From 650be8801550b6fc8afa63766b928d9278c9c9bb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 2 Jun 2025 12:22:41 +0200 Subject: [PATCH 1/7] Create specific SSF rack configuration Allow xeon machines to optionally inherit SSF configuration such as the NFS mount point and the network configuration. --- m/bay/configuration.nix | 2 +- m/common/ssf.nix | 9 +++++++++ m/common/{xeon => ssf}/fs.nix | 0 m/common/{xeon => ssf}/net.nix | 0 m/common/{xeon => ssf}/ssh.nix | 0 m/common/xeon.nix | 5 +---- m/eudy/configuration.nix | 2 +- m/hut/configuration.nix | 2 +- m/koro/configuration.nix | 2 +- m/lake2/configuration.nix | 2 +- m/owl1/configuration.nix | 2 +- m/owl2/configuration.nix | 2 +- 12 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 m/common/ssf.nix rename m/common/{xeon => ssf}/fs.nix (100%) rename m/common/{xeon => ssf}/net.nix (100%) rename m/common/{xeon => ssf}/ssh.nix (100%) diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index 5f215cf1..5c6f93ca 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix ../module/monitoring.nix ]; diff --git a/m/common/ssf.nix b/m/common/ssf.nix new file mode 100644 index 00000000..a01839b3 --- /dev/null +++ b/m/common/ssf.nix @@ -0,0 +1,9 @@ +{ + # Provides the base system for a xeon node in the SSF rack. + imports = [ + ./xeon.nix + ./ssf/fs.nix + ./ssf/net.nix + ./ssf/ssh.nix + ]; +} diff --git a/m/common/xeon/fs.nix b/m/common/ssf/fs.nix similarity index 100% rename from m/common/xeon/fs.nix rename to m/common/ssf/fs.nix diff --git a/m/common/xeon/net.nix b/m/common/ssf/net.nix similarity index 100% rename from m/common/xeon/net.nix rename to m/common/ssf/net.nix diff --git a/m/common/xeon/ssh.nix b/m/common/ssf/ssh.nix similarity index 100% rename from m/common/xeon/ssh.nix rename to m/common/ssf/ssh.nix diff --git a/m/common/xeon.nix b/m/common/xeon.nix index 146e018c..13946609 100644 --- a/m/common/xeon.nix +++ b/m/common/xeon.nix @@ -1,10 +1,7 @@ { - # Provides the base system for a xeon node. + # Provides the base system for a xeon node, not necessarily in the SSF rack. imports = [ ./base.nix ./xeon/console.nix - ./xeon/fs.nix - ./xeon/net.nix - ./xeon/ssh.nix ]; } diff --git a/m/eudy/configuration.nix b/m/eudy/configuration.nix index 29d495a8..d6761354 100644 --- a/m/eudy/configuration.nix +++ b/m/eudy/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix #(modulesPath + "/installer/netboot/netboot-minimal.nix") ./kernel/kernel.nix diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index b00351de..493208c4 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix ../module/ceph.nix ../module/debuginfod.nix diff --git a/m/koro/configuration.nix b/m/koro/configuration.nix index 4214fee1..a106b628 100644 --- a/m/koro/configuration.nix +++ b/m/koro/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix #(modulesPath + "/installer/netboot/netboot-minimal.nix") ../eudy/cpufreq.nix diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 2a29ae7a..e2f350df 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix ../module/monitoring.nix ]; diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index 20e3cf57..e471969d 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix ../module/ceph.nix ../module/emulation.nix ../module/slurm-client.nix diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 54849f6a..e28c5e52 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/xeon.nix + ../common/ssf.nix ../module/ceph.nix ../module/emulation.nix ../module/slurm-client.nix -- 2.49.0 From da20987f99b132066fd4941f8472c3756fc45b16 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 2 Jun 2025 09:07:32 +0200 Subject: [PATCH 2/7] Add tent machine from xeon04 We moved the tent machine to the server room in the BSC building and is now directly connected to the raccoon via NAT. Fixes: https://jungle.bsc.es/git/rarias/jungle/issues/106 --- flake.nix | 1 + m/map.nix | 3 ++- m/raccoon/configuration.nix | 12 ++++++++++++ m/tent/configuration.nix | 38 +++++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 m/tent/configuration.nix diff --git a/flake.nix b/flake.nix index 037bba2c..67ae0875 100644 --- a/flake.nix +++ b/flake.nix @@ -18,6 +18,7 @@ in { nixosConfigurations = { hut = mkConf "hut"; + tent = mkConf "tent"; owl1 = mkConf "owl1"; owl2 = mkConf "owl2"; eudy = mkConf "eudy"; diff --git a/m/map.nix b/m/map.nix index 6c310608..606d4171 100644 --- a/m/map.nix +++ b/m/map.nix @@ -17,7 +17,7 @@ owl1 = { pos=35; size=1; label="SSF-XEON01"; board="S2600WTTR"; sn="BQWL64954172"; contact="rodrigo.arias@bsc.es"; }; owl2 = { pos=34; size=1; label="SSF-XEON02"; board="S2600WTTR"; sn="BQWL64756560"; contact="rodrigo.arias@bsc.es"; }; xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; }; - xeon04 = { pos=32; size=1; label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; }; + # Slot 34 empty koro = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; }; xeon06 = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; }; hut = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; }; @@ -48,6 +48,7 @@ bsc2218 = { raccoon = { board="W2600CR"; sn="QSIP22500829"; contact="rodrigo.arias@bsc.es"; }; + tent = { label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; }; }; upc = { diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index 2482a164..03549035 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -26,6 +26,18 @@ address = "84.88.51.152"; prefixLength = 25; } ]; + interfaces.enp5s0f1.ipv4.addresses = [ { + address = "10.0.44.1"; + prefixLength = 24; + } ]; + nat = { + enable = true; + internalInterfaces = [ "enp5s0f1" ]; + externalInterface = "eno0"; + }; + hosts = { + "10.0.44.4" = [ "tent" ]; + }; }; nix.settings = { diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix new file mode 100644 index 00000000..7cb28454 --- /dev/null +++ b/m/tent/configuration.nix @@ -0,0 +1,38 @@ +{ config, pkgs, ... }: + +{ + imports = [ + ../common/xeon.nix + ../module/emulation.nix + ../module/debuginfod.nix + ]; + + # Select the this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d537675"; + + networking = { + hostName = "tent"; + interfaces.eno1.ipv4.addresses = [ + { + address = "10.0.44.4"; + prefixLength = 24; + } + ]; + + # Only BSC DNSs seem to be reachable from the office VLAN + nameservers = [ "84.88.52.35" "84.88.52.36" ]; + defaultGateway = "10.0.44.1"; + }; + + nix.settings = { + substituters = [ "https://jungle.bsc.es/cache" ]; + trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ]; + }; + + services.prometheus.exporters.node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = 9002; + listenAddress = "127.0.0.1"; + }; +} -- 2.49.0 From e813c811accd7e15c4fee0459cc013ef5ea5e937 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Jun 2025 12:52:10 +0200 Subject: [PATCH 3/7] Don't use proxy in base preset --- m/common/base/ssh.nix | 7 ------- 1 file changed, 7 deletions(-) diff --git a/m/common/base/ssh.nix b/m/common/base/ssh.nix index 92c2c70d..53de4238 100644 --- a/m/common/base/ssh.nix +++ b/m/common/base/ssh.nix @@ -8,13 +8,6 @@ in # Enable the OpenSSH daemon. services.openssh.enable = true; - # Connect to intranet git hosts via proxy - programs.ssh.extraConfig = '' - Host bscpm02.bsc.es bscpm03.bsc.es bscpm04.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es - User git - ProxyCommand nc -X connect -x hut:23080 %h %p - ''; - programs.ssh.knownHosts = hostsKeys // { "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3"; "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS"; -- 2.49.0 From bcb2bb252d6edc53884a3bcfb6020ef91a560f26 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Jun 2025 17:17:29 +0200 Subject: [PATCH 4/7] Add hut SSH configuration from outside SSF LAN --- m/module/ssh-hut-extern.nix | 9 +++++++++ m/raccoon/configuration.nix | 1 + m/tent/configuration.nix | 1 + 3 files changed, 11 insertions(+) create mode 100644 m/module/ssh-hut-extern.nix diff --git a/m/module/ssh-hut-extern.nix b/m/module/ssh-hut-extern.nix new file mode 100644 index 00000000..5eeafb34 --- /dev/null +++ b/m/module/ssh-hut-extern.nix @@ -0,0 +1,9 @@ +{ + programs.ssh.extraConfig = '' + Host ssfhead + HostName ssflogin.bsc.es + Host hut + ProxyJump ssfhead + HostName xeon07 + ''; +} diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix index 03549035..5344751c 100644 --- a/m/raccoon/configuration.nix +++ b/m/raccoon/configuration.nix @@ -5,6 +5,7 @@ ../common/base.nix ../module/emulation.nix ../module/debuginfod.nix + ../module/ssh-hut-extern.nix ../eudy/kernel/perf.nix ]; diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index 7cb28454..47238ac7 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -5,6 +5,7 @@ ../common/xeon.nix ../module/emulation.nix ../module/debuginfod.nix + ../module/ssh-hut-extern.nix ]; # Select the this using the ID to avoid mismatches -- 2.49.0 From c2c4fa0226801838c4e8788860d688f202452cad Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Jun 2025 17:24:40 +0200 Subject: [PATCH 5/7] Add access to tent to all hut users too --- m/common/base/users.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/m/common/base/users.nix b/m/common/base/users.nix index 8e2ca80b..9bce966d 100644 --- a/m/common/base/users.nix +++ b/m/common/base/users.nix @@ -56,7 +56,7 @@ home = "/home/Computational/rpenacob"; description = "Raúl Peñacoba"; group = "Computational"; - hosts = [ "owl1" "owl2" "hut" ]; + hosts = [ "owl1" "owl2" "hut" "tent" ]; hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc" @@ -69,7 +69,7 @@ home = "/home/Computational/anavarro"; description = "Antoni Navarro"; group = "Computational"; - hosts = [ "hut" "raccoon" "fox" ]; + hosts = [ "hut" "tent" "raccoon" "fox" ]; hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead" @@ -82,7 +82,7 @@ home = "/home/Computational/abonerib"; description = "Aleix Boné"; group = "Computational"; - hosts = [ "owl1" "owl2" "hut" "raccoon" "fox" ]; + hosts = [ "owl1" "owl2" "hut" "tent" "raccoon" "fox" ]; hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc" @@ -108,7 +108,7 @@ home = "/home/Computational/dbautist"; description = "Dylan Bautista Cases"; group = "Computational"; - hosts = [ "hut" "raccoon" ]; + hosts = [ "hut" "tent" "raccoon" ]; hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791" @@ -121,7 +121,7 @@ home = "/home/Computational/dalvare1"; description = "David Álvarez"; group = "Computational"; - hosts = [ "hut" "fox" ]; + hosts = [ "hut" "tent" "fox" ]; hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead" @@ -134,7 +134,7 @@ home = "/home/Computational/varcila"; description = "Vincent Arcila"; group = "Computational"; - hosts = [ "hut" "fox" ]; + hosts = [ "hut" "tent" "fox" ]; hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0"; openssh.authorizedKeys.keys = [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch" -- 2.49.0 From ec7c883ae4de72dc80ce8b7f638bbf95457b4a43 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Jun 2025 18:27:56 +0200 Subject: [PATCH 6/7] Add software RAID in tent using 3 disks --- m/tent/configuration.nix | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index 47238ac7..cbe3a6e4 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -36,4 +36,18 @@ port = 9002; listenAddress = "127.0.0.1"; }; + + boot.swraid = { + enable = true; + mdadmConf = '' + DEVICE partitions + ARRAY /dev/md0 metadata=1.2 UUID=496db1e2:056a92aa:a544543f:40db379d + MAILADDR root + ''; + }; + + fileSystems."/vault" = { + device = "/dev/disk/by-label/vault"; + fsType = "ext4"; + }; } -- 2.49.0 From 4f2422eca2bc0d1a849de9d3730efcd2cf3256b0 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Jun 2025 19:07:43 +0200 Subject: [PATCH 7/7] Create directories in /vault for tent users --- m/tent/configuration.nix | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/m/tent/configuration.nix b/m/tent/configuration.nix index cbe3a6e4..74bcb826 100644 --- a/m/tent/configuration.nix +++ b/m/tent/configuration.nix @@ -1,4 +1,4 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: { imports = [ @@ -50,4 +50,21 @@ device = "/dev/disk/by-label/vault"; fsType = "ext4"; }; + + # Make a /vault/$USER directory for each user. + systemd.services.create-vault-dirs = let + # Take only normal users in tent + users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users; + commands = lib.concatLists (lib.mapAttrsToList + (_: user: [ + "install -d -o ${user.name} -g ${user.group} -m 0711 /vault/${user.name}" + ]) users); + script = pkgs.writeShellScript "create-vault-dirs.sh" (lib.concatLines commands); + in { + enable = true; + wants = [ "local-fs.target" ]; + after = [ "local-fs.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig.ExecStart = script; + }; } -- 2.49.0