diff --git a/keys.nix b/keys.nix index b26f11af..d491d6d5 100644 --- a/keys.nix +++ b/keys.nix @@ -2,21 +2,22 @@ # here all the public keys rec { hosts = { - hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut"; - owl1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1"; - owl2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2"; - eudy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy"; - koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; - bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; - lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; - fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox"; - tent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent"; - apex = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex"; - weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel"; + hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut"; + owl1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1"; + owl2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2"; + eudy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy"; + koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; + bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; + lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; + fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox"; + tent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent"; + apex = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex"; + weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel"; + raccoon = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNQttFvL0dNEyy7klIhLoK4xXOeM2/K9R7lPMTG3qvK raccoon"; }; hostGroup = with hosts; rec { - compute = [ owl1 owl2 fox ]; + compute = [ owl1 owl2 fox raccoon ]; playground = [ eudy koro weasel ]; storage = [ bay lake2 ]; monitor = [ hut ]; diff --git a/m/apex/configuration.nix b/m/apex/configuration.nix index 17828a32..82d38422 100644 --- a/m/apex/configuration.nix +++ b/m/apex/configuration.nix @@ -56,17 +56,6 @@ }; }; - # Use SSH tunnel to reach internal hosts - programs.ssh.extraConfig = '' - Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es - ProxyCommand nc -X connect -x localhost:23080 %h %p - Host raccoon - HostName knights3.bsc.es - ProxyCommand nc -X connect -x localhost:23080 %h %p - Host tent - ProxyJump raccoon - ''; - networking.firewall = { extraCommands = '' # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our diff --git a/m/apex/wireguard.nix b/m/apex/wireguard.nix index 49180a15..4721d2df 100644 --- a/m/apex/wireguard.nix +++ b/m/apex/wireguard.nix @@ -18,18 +18,25 @@ # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA= peers = [ { - name = "Fox"; + name = "fox"; publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y="; - allowedIPs = [ "10.106.0.0/24" ]; + allowedIPs = [ "10.106.0.1/32" ]; endpoint = "fox.ac.upc.edu:666"; # Send keepalives every 25 seconds. Important to keep NAT tables alive. persistentKeepalive = 25; } + { + name = "raccoon"; + publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI="; + allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ]; + } ]; }; }; networking.hosts = { "10.106.0.1" = [ "fox" ]; + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; }; } diff --git a/m/common/base/net.nix b/m/common/base/net.nix index 64e6160a..9fe6c4db 100644 --- a/m/common/base/net.nix +++ b/m/common/base/net.nix @@ -15,8 +15,9 @@ hosts = { "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ]; - "84.88.51.152" = [ "raccoon" ]; "84.88.51.142" = [ "raccoon-ipmi" ]; + "192.168.11.12" = [ "bscpm04.bsc.es" ]; + "192.168.11.15" = [ "gitlab-internal.bsc.es" ]; }; }; } diff --git a/m/common/ssf.nix b/m/common/ssf.nix index 8e8dc6b4..ef74da30 100644 --- a/m/common/ssf.nix +++ b/m/common/ssf.nix @@ -4,7 +4,7 @@ ./xeon.nix ./ssf/fs.nix ./ssf/hosts.nix + ./ssf/hosts-remote.nix ./ssf/net.nix - ./ssf/ssh.nix ]; } diff --git a/m/common/ssf/hosts-remote.nix b/m/common/ssf/hosts-remote.nix new file mode 100644 index 00000000..1660f739 --- /dev/null +++ b/m/common/ssf/hosts-remote.nix @@ -0,0 +1,9 @@ +{ pkgs, ... }: + +{ + networking.hosts = { + # Remote hosts visible from compute nodes + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; + }; +} diff --git a/m/common/ssf/ssh.nix b/m/common/ssf/ssh.nix deleted file mode 100644 index b73abd79..00000000 --- a/m/common/ssf/ssh.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - # Use SSH tunnel to apex to reach internal hosts - programs.ssh.extraConfig = '' - Host tent - ProxyJump raccoon - - # Access raccoon via the HTTP proxy - Host raccoon knights3.bsc.es - HostName knights3.bsc.es - ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p' - - # Make sure we can reach gitlab even if we don't have SSH access to raccoon - Host bscpm04.bsc.es gitlab-internal.bsc.es - ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p' - ''; -} diff --git a/m/fox/configuration.nix b/m/fox/configuration.nix index 3f6742b1..e890ba54 100644 --- a/m/fox/configuration.nix +++ b/m/fox/configuration.nix @@ -45,16 +45,6 @@ services.fail2ban.enable = true; - # Use SSH tunnel to reach internal hosts - programs.ssh.extraConfig = '' - Host bscpm04.bsc.es gitlab-internal.bsc.es tent - ProxyJump raccoon - Host raccoon - ProxyJump apex - HostName 127.0.0.1 - Port 22022 - ''; - networking = { timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ]; hostName = "fox"; diff --git a/m/fox/wireguard.nix b/m/fox/wireguard.nix index 8299d483..7a55c2e9 100644 --- a/m/fox/wireguard.nix +++ b/m/fox/wireguard.nix @@ -24,17 +24,24 @@ peers = [ # List of allowed peers. { - name = "Apex"; + name = "apex"; publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA="; # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing. allowedIPs = [ "10.106.0.30/32" ]; } + { + name = "raccoon"; + publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI="; + allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ]; + } ]; }; }; networking.hosts = { "10.106.0.30" = [ "apex" ]; + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; }; networking.firewall = { diff --git a/m/module/ssh-hut-extern.nix b/m/module/ssh-hut-extern.nix deleted file mode 100644 index cc6b26ac..00000000 --- a/m/module/ssh-hut-extern.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ - programs.ssh.extraConfig = '' - Host apex ssfhead - HostName ssflogin.bsc.es - Host hut - ProxyJump apex - ''; -} diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index cec17e6f..b2c59fe4 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -3,11 +3,12 @@ { imports = [ ../common/base.nix + ../common/ssf/hosts.nix ../module/emulation.nix ../module/debuginfod.nix - ../module/ssh-hut-extern.nix ../module/nvidia.nix ../eudy/kernel/perf.nix + ./wireguard.nix ]; # Don't install Grub on the disk yet @@ -43,6 +44,13 @@ }; }; + # Mount the NFS home + fileSystems."/nfs/home" = { + device = "10.106.0.30:/home"; + fsType = "nfs"; + options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ]; + }; + nix.settings = { extra-substituters = [ "https://jungle.bsc.es/cache" ]; extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ]; diff --git a/m/raccoon/wireguard.nix b/m/raccoon/wireguard.nix new file mode 100644 index 00000000..daf48837 --- /dev/null +++ b/m/raccoon/wireguard.nix @@ -0,0 +1,48 @@ +{ config, pkgs, ... }: + +{ + networking.nat = { + enable = true; + enableIPv6 = false; + externalInterface = "eno0"; + internalInterfaces = [ "wg0" ]; + }; + + networking.firewall = { + allowedUDPPorts = [ 666 ]; + }; + + age.secrets.wgRaccoon.file = ../../secrets/wg-raccoon.age; + + # Enable WireGuard + networking.wireguard.enable = true; + networking.wireguard.interfaces = { + wg0 = { + ips = [ "10.106.0.236/24" ]; + listenPort = 666; + privateKeyFile = config.age.secrets.wgRaccoon.path; + # Public key: QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI= + peers = [ + { + name = "fox"; + publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y="; + allowedIPs = [ "10.106.0.1/32" ]; + endpoint = "fox.ac.upc.edu:666"; + persistentKeepalive = 25; + } + { + name = "apex"; + publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA="; + allowedIPs = [ "10.106.0.30/32" "10.0.40.0/24" ]; + endpoint = "ssfhead.bsc.es:666"; + persistentKeepalive = 25; + } + ]; + }; + }; + + networking.hosts = { + "10.106.0.1" = [ "fox.wg" ]; + "10.106.0.30" = [ "apex.wg" ]; + }; +} diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index 416d8df1..79c9f458 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -3,9 +3,9 @@ { imports = [ ../common/xeon.nix + ../common/ssf/hosts.nix ../module/emulation.nix ../module/debuginfod.nix - ../module/ssh-hut-extern.nix ./monitoring.nix ./nginx.nix ./nix-serve.nix @@ -35,6 +35,7 @@ defaultGateway = "10.0.44.1"; hosts = { "84.88.53.236" = [ "apex" ]; + "10.0.44.1" = [ "raccoon" ]; }; }; diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 96732490..920d52dc 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -4,6 +4,7 @@ let hut = [ keys.hosts.hut ] ++ adminsKeys; fox = [ keys.hosts.fox ] ++ adminsKeys; apex = [ keys.hosts.apex ] ++ adminsKeys; + raccoon = [ keys.hosts.raccoon ] ++ adminsKeys; mon = [ keys.hosts.hut keys.hosts.tent ] ++ adminsKeys; tent = [ keys.hosts.tent ] ++ adminsKeys; # Only expose ceph keys to safe nodes and admins @@ -29,4 +30,5 @@ in "wg-fox.age".publicKeys = fox; "wg-apex.age".publicKeys = apex; + "wg-raccoon.age".publicKeys = raccoon; } diff --git a/secrets/wg-raccoon.age b/secrets/wg-raccoon.age new file mode 100644 index 00000000..f32a2aa3 Binary files /dev/null and b/secrets/wg-raccoon.age differ