From e934a2bc9d6421d37131d88bac6d29bda03a7ddf Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 19 Sep 2025 13:18:12 +0200 Subject: [PATCH 1/6] Use lowercase peer hostnames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Aleix Boné --- m/apex/wireguard.nix | 2 +- m/fox/wireguard.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/m/apex/wireguard.nix b/m/apex/wireguard.nix index 49180a15..607564fe 100644 --- a/m/apex/wireguard.nix +++ b/m/apex/wireguard.nix @@ -18,7 +18,7 @@ # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA= peers = [ { - name = "Fox"; + name = "fox"; publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y="; allowedIPs = [ "10.106.0.0/24" ]; endpoint = "fox.ac.upc.edu:666"; diff --git a/m/fox/wireguard.nix b/m/fox/wireguard.nix index 8299d483..d0b4349e 100644 --- a/m/fox/wireguard.nix +++ b/m/fox/wireguard.nix @@ -24,7 +24,7 @@ peers = [ # List of allowed peers. { - name = "Apex"; + name = "apex"; publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA="; # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing. allowedIPs = [ "10.106.0.30/32" ]; -- 2.51.2 From 3380ec5e05bc87dffb0162c55a75aad8a7d3291c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 19 Sep 2025 13:20:54 +0200 Subject: [PATCH 2/6] Restrict fox peer to a single IP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Aleix Boné --- m/apex/wireguard.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m/apex/wireguard.nix b/m/apex/wireguard.nix index 607564fe..0a6ac5f0 100644 --- a/m/apex/wireguard.nix +++ b/m/apex/wireguard.nix @@ -20,7 +20,7 @@ { name = "fox"; publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y="; - allowedIPs = [ "10.106.0.0/24" ]; + allowedIPs = [ "10.106.0.1/32" ]; endpoint = "fox.ac.upc.edu:666"; # Send keepalives every 25 seconds. Important to keep NAT tables alive. persistentKeepalive = 25; -- 2.51.2 From 26a4a26ce019157307d386fc96f3e25695863db1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 19 Sep 2025 13:26:56 +0200 Subject: [PATCH 3/6] Add raccoon host key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Aleix Boné --- keys.nix | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/keys.nix b/keys.nix index b26f11af..d491d6d5 100644 --- a/keys.nix +++ b/keys.nix @@ -2,21 +2,22 @@ # here all the public keys rec { hosts = { - hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut"; - owl1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1"; - owl2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2"; - eudy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy"; - koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; - bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; - lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; - fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox"; - tent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent"; - apex = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex"; - weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel"; + hut = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut"; + owl1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1"; + owl2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2"; + eudy = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy"; + koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; + bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; + lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; + fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox"; + tent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent"; + apex = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex"; + weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel"; + raccoon = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNQttFvL0dNEyy7klIhLoK4xXOeM2/K9R7lPMTG3qvK raccoon"; }; hostGroup = with hosts; rec { - compute = [ owl1 owl2 fox ]; + compute = [ owl1 owl2 fox raccoon ]; playground = [ eudy koro weasel ]; storage = [ bay lake2 ]; monitor = [ hut ]; -- 2.51.2 From 08e4dda6d2ec954bc653baa592e2663a78149055 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 25 Sep 2025 15:01:33 +0200 Subject: [PATCH 4/6] Add raccoon peer to wireguard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It routes traffic from fox, apex and the compute nodes so that we can reach the git servers and tent. Reviewed-by: Aleix Boné --- m/apex/wireguard.nix | 7 +++++ m/common/base/net.nix | 1 - m/common/ssf.nix | 1 + m/common/ssf/hosts-remote.nix | 9 +++++++ m/fox/wireguard.nix | 7 +++++ m/raccoon/configuration.nix | 1 + m/raccoon/wireguard.nix | 48 ++++++++++++++++++++++++++++++++++ m/tent/configuration.nix | 1 + secrets/secrets.nix | 2 ++ secrets/wg-raccoon.age | Bin 0 -> 697 bytes 10 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 m/common/ssf/hosts-remote.nix create mode 100644 m/raccoon/wireguard.nix create mode 100644 secrets/wg-raccoon.age diff --git a/m/apex/wireguard.nix b/m/apex/wireguard.nix index 0a6ac5f0..4721d2df 100644 --- a/m/apex/wireguard.nix +++ b/m/apex/wireguard.nix @@ -25,11 +25,18 @@ # Send keepalives every 25 seconds. Important to keep NAT tables alive. persistentKeepalive = 25; } + { + name = "raccoon"; + publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI="; + allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ]; + } ]; }; }; networking.hosts = { "10.106.0.1" = [ "fox" ]; + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; }; } diff --git a/m/common/base/net.nix b/m/common/base/net.nix index 64e6160a..88c8c00f 100644 --- a/m/common/base/net.nix +++ b/m/common/base/net.nix @@ -15,7 +15,6 @@ hosts = { "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ]; - "84.88.51.152" = [ "raccoon" ]; "84.88.51.142" = [ "raccoon-ipmi" ]; }; }; diff --git a/m/common/ssf.nix b/m/common/ssf.nix index 8e8dc6b4..98a98e52 100644 --- a/m/common/ssf.nix +++ b/m/common/ssf.nix @@ -4,6 +4,7 @@ ./xeon.nix ./ssf/fs.nix ./ssf/hosts.nix + ./ssf/hosts-remote.nix ./ssf/net.nix ./ssf/ssh.nix ]; diff --git a/m/common/ssf/hosts-remote.nix b/m/common/ssf/hosts-remote.nix new file mode 100644 index 00000000..1660f739 --- /dev/null +++ b/m/common/ssf/hosts-remote.nix @@ -0,0 +1,9 @@ +{ pkgs, ... }: + +{ + networking.hosts = { + # Remote hosts visible from compute nodes + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; + }; +} diff --git a/m/fox/wireguard.nix b/m/fox/wireguard.nix index d0b4349e..7a55c2e9 100644 --- a/m/fox/wireguard.nix +++ b/m/fox/wireguard.nix @@ -29,12 +29,19 @@ # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing. allowedIPs = [ "10.106.0.30/32" ]; } + { + name = "raccoon"; + publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI="; + allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ]; + } ]; }; }; networking.hosts = { "10.106.0.30" = [ "apex" ]; + "10.106.0.236" = [ "raccoon" ]; + "10.0.44.4" = [ "tent" ]; }; networking.firewall = { diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index cec17e6f..7a608e20 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -8,6 +8,7 @@ ../module/ssh-hut-extern.nix ../module/nvidia.nix ../eudy/kernel/perf.nix + ./wireguard.nix ]; # Don't install Grub on the disk yet diff --git a/m/raccoon/wireguard.nix b/m/raccoon/wireguard.nix new file mode 100644 index 00000000..daf48837 --- /dev/null +++ b/m/raccoon/wireguard.nix @@ -0,0 +1,48 @@ +{ config, pkgs, ... }: + +{ + networking.nat = { + enable = true; + enableIPv6 = false; + externalInterface = "eno0"; + internalInterfaces = [ "wg0" ]; + }; + + networking.firewall = { + allowedUDPPorts = [ 666 ]; + }; + + age.secrets.wgRaccoon.file = ../../secrets/wg-raccoon.age; + + # Enable WireGuard + networking.wireguard.enable = true; + networking.wireguard.interfaces = { + wg0 = { + ips = [ "10.106.0.236/24" ]; + listenPort = 666; + privateKeyFile = config.age.secrets.wgRaccoon.path; + # Public key: QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI= + peers = [ + { + name = "fox"; + publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y="; + allowedIPs = [ "10.106.0.1/32" ]; + endpoint = "fox.ac.upc.edu:666"; + persistentKeepalive = 25; + } + { + name = "apex"; + publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA="; + allowedIPs = [ "10.106.0.30/32" "10.0.40.0/24" ]; + endpoint = "ssfhead.bsc.es:666"; + persistentKeepalive = 25; + } + ]; + }; + }; + + networking.hosts = { + "10.106.0.1" = [ "fox.wg" ]; + "10.106.0.30" = [ "apex.wg" ]; + }; +} diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index 416d8df1..9d56ba70 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -35,6 +35,7 @@ defaultGateway = "10.0.44.1"; hosts = { "84.88.53.236" = [ "apex" ]; + "10.0.44.1" = [ "raccoon" ]; }; }; diff --git a/secrets/secrets.nix b/secrets/secrets.nix index 96732490..920d52dc 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -4,6 +4,7 @@ let hut = [ keys.hosts.hut ] ++ adminsKeys; fox = [ keys.hosts.fox ] ++ adminsKeys; apex = [ keys.hosts.apex ] ++ adminsKeys; + raccoon = [ keys.hosts.raccoon ] ++ adminsKeys; mon = [ keys.hosts.hut keys.hosts.tent ] ++ adminsKeys; tent = [ keys.hosts.tent ] ++ adminsKeys; # Only expose ceph keys to safe nodes and admins @@ -29,4 +30,5 @@ in "wg-fox.age".publicKeys = fox; "wg-apex.age".publicKeys = apex; + "wg-raccoon.age".publicKeys = raccoon; } diff --git a/secrets/wg-raccoon.age b/secrets/wg-raccoon.age new file mode 100644 index 0000000000000000000000000000000000000000..f32a2aa395b4fcaf3bdd9cf09e9164d3a85aad62 GIT binary patch literal 697 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCR+k2J4zFIR|6%-1f> zv~&&3NvXq zasr)vT*}dH^9^=0PDxiN@bs!Q)HgHBHIFa~^A9mF&P(%34GD+}Eet9v_Y4ltD+@Mr zb`J}33U%djF0TwU^SAITuME%eu88#T^UY7qa7@qf4Jh|4s>lrqH_fx~4@&e7^~vYb z)zwvqsL1j)cPk7iGxA8z$cPNfH8R)s@yjgD^$066vh)nfa5nVL&oinjuJq)3z`xP` zd5>n%_lLzsEWg+^*k+4KKA2?7U}x?pD Date: Thu, 25 Sep 2025 15:15:43 +0200 Subject: [PATCH 5/6] Remove extra SSH jump configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now have direct visibility among nodes so we don't need any extra SSH configuration to reach them. Reviewed-by: Aleix Boné --- m/apex/configuration.nix | 11 ----------- m/common/base/net.nix | 2 ++ m/common/ssf.nix | 1 - m/common/ssf/ssh.nix | 16 ---------------- m/fox/configuration.nix | 10 ---------- m/module/ssh-hut-extern.nix | 8 -------- m/raccoon/configuration.nix | 2 +- m/tent/configuration.nix | 2 +- 8 files changed, 4 insertions(+), 48 deletions(-) delete mode 100644 m/common/ssf/ssh.nix delete mode 100644 m/module/ssh-hut-extern.nix diff --git a/m/apex/configuration.nix b/m/apex/configuration.nix index 17828a32..82d38422 100644 --- a/m/apex/configuration.nix +++ b/m/apex/configuration.nix @@ -56,17 +56,6 @@ }; }; - # Use SSH tunnel to reach internal hosts - programs.ssh.extraConfig = '' - Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es - ProxyCommand nc -X connect -x localhost:23080 %h %p - Host raccoon - HostName knights3.bsc.es - ProxyCommand nc -X connect -x localhost:23080 %h %p - Host tent - ProxyJump raccoon - ''; - networking.firewall = { extraCommands = '' # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our diff --git a/m/common/base/net.nix b/m/common/base/net.nix index 88c8c00f..9fe6c4db 100644 --- a/m/common/base/net.nix +++ b/m/common/base/net.nix @@ -16,6 +16,8 @@ hosts = { "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ]; "84.88.51.142" = [ "raccoon-ipmi" ]; + "192.168.11.12" = [ "bscpm04.bsc.es" ]; + "192.168.11.15" = [ "gitlab-internal.bsc.es" ]; }; }; } diff --git a/m/common/ssf.nix b/m/common/ssf.nix index 98a98e52..ef74da30 100644 --- a/m/common/ssf.nix +++ b/m/common/ssf.nix @@ -6,6 +6,5 @@ ./ssf/hosts.nix ./ssf/hosts-remote.nix ./ssf/net.nix - ./ssf/ssh.nix ]; } diff --git a/m/common/ssf/ssh.nix b/m/common/ssf/ssh.nix deleted file mode 100644 index b73abd79..00000000 --- a/m/common/ssf/ssh.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - # Use SSH tunnel to apex to reach internal hosts - programs.ssh.extraConfig = '' - Host tent - ProxyJump raccoon - - # Access raccoon via the HTTP proxy - Host raccoon knights3.bsc.es - HostName knights3.bsc.es - ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p' - - # Make sure we can reach gitlab even if we don't have SSH access to raccoon - Host bscpm04.bsc.es gitlab-internal.bsc.es - ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p' - ''; -} diff --git a/m/fox/configuration.nix b/m/fox/configuration.nix index 3f6742b1..e890ba54 100644 --- a/m/fox/configuration.nix +++ b/m/fox/configuration.nix @@ -45,16 +45,6 @@ services.fail2ban.enable = true; - # Use SSH tunnel to reach internal hosts - programs.ssh.extraConfig = '' - Host bscpm04.bsc.es gitlab-internal.bsc.es tent - ProxyJump raccoon - Host raccoon - ProxyJump apex - HostName 127.0.0.1 - Port 22022 - ''; - networking = { timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ]; hostName = "fox"; diff --git a/m/module/ssh-hut-extern.nix b/m/module/ssh-hut-extern.nix deleted file mode 100644 index cc6b26ac..00000000 --- a/m/module/ssh-hut-extern.nix +++ /dev/null @@ -1,8 +0,0 @@ -{ - programs.ssh.extraConfig = '' - Host apex ssfhead - HostName ssflogin.bsc.es - Host hut - ProxyJump apex - ''; -} diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index 7a608e20..b68b5193 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -3,9 +3,9 @@ { imports = [ ../common/base.nix + ../common/ssf/hosts.nix ../module/emulation.nix ../module/debuginfod.nix - ../module/ssh-hut-extern.nix ../module/nvidia.nix ../eudy/kernel/perf.nix ./wireguard.nix diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index 9d56ba70..79c9f458 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -3,9 +3,9 @@ { imports = [ ../common/xeon.nix + ../common/ssf/hosts.nix ../module/emulation.nix ../module/debuginfod.nix - ../module/ssh-hut-extern.nix ./monitoring.nix ./nginx.nix ./nix-serve.nix -- 2.51.2 From 9c3fbc0ec9b1c0acbc0caedd678a4a9f9df35a2a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 19 Sep 2025 13:48:50 +0200 Subject: [PATCH 6/6] Mount apex /home via NFS in raccoon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Aleix Boné --- m/raccoon/configuration.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index b68b5193..b2c59fe4 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -44,6 +44,13 @@ }; }; + # Mount the NFS home + fileSystems."/nfs/home" = { + device = "10.106.0.30:/home"; + fsType = "nfs"; + options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ]; + }; + nix.settings = { extra-substituters = [ "https://jungle.bsc.es/cache" ]; extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ]; -- 2.51.2