Add new configuration for apex

Reviewed-by: Aleix Boné <abonerib@bsc.es>
This commit is contained in:
Rodrigo Arias 2025-07-09 11:02:11 +02:00
parent 3d97fada6d
commit 1b731a756a
24 changed files with 172 additions and 109 deletions

View File

@ -27,6 +27,7 @@ in
lake2 = mkConf "lake2";
raccoon = mkConf "raccoon";
fox = mkConf "fox";
apex = mkConf "apex";
};
packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

View File

@ -11,6 +11,7 @@ rec {
lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
tent = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
apex = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
};
hostGroup = with hosts; rec {
@ -19,8 +20,9 @@ rec {
playground = [ eudy koro ];
storage = [ bay lake2 ];
monitor = [ hut ];
login = [ apex ];
system = storage ++ monitor;
system = storage ++ monitor ++ login;
safe = system ++ compute;
all = safe ++ playground;
};

58
m/apex/configuration.nix Normal file
View File

@ -0,0 +1,58 @@
{ lib, config, pkgs, ... }:
{
imports = [
../common/xeon.nix
../common/ssf/hosts.nix
../module/ceph.nix
./nfs.nix
];
# Don't install grub MBR for now
boot.loader.grub.device = "nodev";
boot.initrd.kernelModules = [
"megaraid_sas" # For HW RAID
];
fileSystems."/home" = {
device = "/dev/disk/by-label/home";
fsType = "ext4";
};
# No swap, there is plenty of RAM
swapDevices = lib.mkForce [];
networking = {
hostName = "apex";
defaultGateway = "84.88.53.233";
nameservers = [ "8.8.8.8" ];
# Public facing interface
interfaces.eno1.ipv4.addresses = [ {
address = "84.88.53.236";
prefixLength = 29;
} ];
# Internal LAN to our Ethernet switch
interfaces.eno2.ipv4.addresses = [ {
address = "10.0.40.30";
prefixLength = 24;
} ];
# Infiniband over Omnipath switch (disconnected for now)
# interfaces.ibp5s0 = {};
nat = {
enable = true;
internalInterfaces = [ "eno2" ];
externalInterface = "eno1";
};
};
# Use tent for cache
nix.settings = {
extra-substituters = [ "https://jungle.bsc.es/cache" ];
extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
};
}

37
m/apex/nfs.nix Normal file
View File

@ -0,0 +1,37 @@
{ ... }:
{
services.nfs.server = {
enable = true;
lockdPort = 4001;
mountdPort = 4002;
statdPort = 4000;
exports = ''
/home 10.0.40.0/24(rw,sync,no_subtree_check,root_squash)
'';
};
networking.firewall = {
# Check with `rpcinfo -p`
extraCommands = ''
# Accept NFS traffic from compute nodes but not from the outside
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111 -j nixos-fw-accept
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049 -j nixos-fw-accept
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000 -j nixos-fw-accept
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001 -j nixos-fw-accept
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002 -j nixos-fw-accept
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
# Same but UDP
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111 -j nixos-fw-accept
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049 -j nixos-fw-accept
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000 -j nixos-fw-accept
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001 -j nixos-fw-accept
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002 -j nixos-fw-accept
iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
'';
# Flush all rules and chains on stop so it won't break on start
extraStopCommands = ''
iptables -F
iptables -X
'';
};
}

View File

@ -4,7 +4,7 @@
environment.systemPackages = with pkgs; [
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
ncdu config.boot.kernelPackages.perf ldns
ncdu config.boot.kernelPackages.perf ldns pv
# From bsckgs overlay
osumb
];

View File

@ -11,7 +11,7 @@
};
hosts = {
"84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
"84.88.53.236" = [ "apex" "ssfhead.bsc.es" "ssfhead" ];
"84.88.51.152" = [ "raccoon" ];
"84.88.51.142" = [ "raccoon-ipmi" ];
};

View File

@ -4,6 +4,7 @@
./xeon.nix
./ssf/fs.nix
./ssf/net.nix
./ssf/hosts.nix
./ssf/ssh.nix
];
}

23
m/common/ssf/hosts.nix Normal file
View File

@ -0,0 +1,23 @@
{ pkgs, ... }:
{
networking.hosts = {
# Login
"10.0.40.30" = [ "apex" ];
# Storage
"10.0.40.40" = [ "bay" ]; "10.0.42.40" = [ "bay-ib" ]; "10.0.40.141" = [ "bay-ipmi" ];
"10.0.40.41" = [ "oss01" ]; "10.0.42.41" = [ "oss01-ib0" ]; "10.0.40.142" = [ "oss01-ipmi" ];
"10.0.40.42" = [ "lake2" ]; "10.0.42.42" = [ "lake2-ib" ]; "10.0.40.143" = [ "lake2-ipmi" ];
# Xeon compute
"10.0.40.1" = [ "owl1" ]; "10.0.42.1" = [ "owl1-ib" ]; "10.0.40.101" = [ "owl1-ipmi" ];
"10.0.40.2" = [ "owl2" ]; "10.0.42.2" = [ "owl2-ib" ]; "10.0.40.102" = [ "owl2-ipmi" ];
"10.0.40.3" = [ "xeon03" ]; "10.0.42.3" = [ "xeon03-ib" ]; "10.0.40.103" = [ "xeon03-ipmi" ];
#"10.0.40.4" = [ "tent" ]; "10.0.42.4" = [ "tent-ib" ]; "10.0.40.104" = [ "tent-ipmi" ];
"10.0.40.5" = [ "koro" ]; "10.0.42.5" = [ "koro-ib" ]; "10.0.40.105" = [ "koro-ipmi" ];
"10.0.40.6" = [ "xeon06" ]; "10.0.42.6" = [ "xeon06-ib" ]; "10.0.40.106" = [ "xeon06-ipmi" ];
"10.0.40.7" = [ "hut" ]; "10.0.42.7" = [ "hut-ib" ]; "10.0.40.107" = [ "hut-ipmi" ];
"10.0.40.8" = [ "eudy" ]; "10.0.42.8" = [ "eudy-ib" ]; "10.0.40.108" = [ "eudy-ipmi" ];
};
}

View File

@ -27,64 +27,5 @@
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
'';
};
extraHosts = ''
10.0.40.30 ssfhead
# Node Entry for node: mds01 (ID=72)
10.0.40.40 bay mds01 mds01-eth0
10.0.42.40 bay-ib mds01-ib0
10.0.40.141 bay-ipmi mds01-ipmi0 mds01-ipmi
# Node Entry for node: oss01 (ID=73)
10.0.40.41 oss01 oss01-eth0
10.0.42.41 oss01-ib0
10.0.40.142 oss01-ipmi0 oss01-ipmi
# Node Entry for node: oss02 (ID=74)
10.0.40.42 lake2 oss02 oss02-eth0
10.0.42.42 lake2-ib oss02-ib0
10.0.40.143 lake2-ipmi oss02-ipmi0 oss02-ipmi
# Node Entry for node: xeon01 (ID=15)
10.0.40.1 owl1 xeon01 xeon01-eth0
10.0.42.1 owl1-ib xeon01-ib0
10.0.40.101 owl1-ipmi xeon01-ipmi0 xeon01-ipmi
# Node Entry for node: xeon02 (ID=16)
10.0.40.2 owl2 xeon02 xeon02-eth0
10.0.42.2 owl2-ib xeon02-ib0
10.0.40.102 owl2-ipmi xeon02-ipmi0 xeon02-ipmi
# Node Entry for node: xeon03 (ID=17)
10.0.40.3 xeon03 xeon03-eth0
10.0.42.3 xeon03-ib0
10.0.40.103 xeon03-ipmi0 xeon03-ipmi
# Node Entry for node: xeon04 (ID=18)
10.0.40.4 xeon04 xeon04-eth0
10.0.42.4 xeon04-ib0
10.0.40.104 xeon04-ipmi0 xeon04-ipmi
# Node Entry for node: xeon05 (ID=19)
10.0.40.5 koro xeon05 xeon05-eth0
10.0.42.5 koro-ib xeon05-ib0
10.0.40.105 koro-ipmi xeon05-ipmi0
# Node Entry for node: xeon06 (ID=20)
10.0.40.6 xeon06 xeon06-eth0
10.0.42.6 xeon06-ib0
10.0.40.106 xeon06-ipmi0 xeon06-ipmi
# Node Entry for node: xeon07 (ID=21)
10.0.40.7 hut xeon07 xeon07-eth0
10.0.42.7 hut-ib xeon07-ib0
10.0.40.107 hut-ipmi xeon07-ipmi0 xeon07-ipmi
# Node Entry for node: xeon08 (ID=22)
10.0.40.8 eudy xeon08 xeon08-eth0
10.0.42.8 eudy-ib xeon08-ib0
10.0.40.108 eudy-ipmi xeon08-ipmi0 xeon08-ipmi
'';
};
}

View File

@ -6,7 +6,7 @@
switch-opa = { pos=41; size=1; };
# SSF login
ssfhead = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="operations@bsc.es"; };
apex = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="rodrigo.arias@bsc.es"; };
# Storage
bay = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };

Binary file not shown.

View File

@ -1,11 +1,11 @@
age-encryption.org/v1
-> ssh-ed25519 HY2yRg WUMWvyagPalsy7u1RaEFAwJvFowso1/quNBo+nAkxhQ
OHcebB7koPKhy58A6qngEVNWckkWChyEK3dwgy8EL5o
-> ssh-ed25519 CAWG4Q Yx/HLIryUNE2BaqTl84FrNRy4XLCY2TRkRgbA9k3qU4
LZljfuLS5yMVVK6N57iC6cKEaFP6Hh2OkvWJjuFg8q0
-> ssh-ed25519 xA739A DOXjPRttSWz51Sr7KfjgKfAtaIYMo3foB1Ywqw9HYDY
CA5puXK/1HDOitA2XHBI3OdKmZ7BzHst4DyuWGMC6hE
-> ssh-ed25519 MSF3dg +2LetdIiIZUk7wtHNS1tYsLo4ypwqZ9gpg77RQrnzHU
yIUu8BVbF3dhUx3531RR50/cJQd9gd8VfKUQzEeT/iQ
--- oY/wQ+RjZO2CmKZtbQ0yOVZ5fv2+AlvvkRu1UDfCNAA
_8`G©=C7@x &<26>¢\Ft·)ø¹ç„ÙcPe˜Ã%êÖ½[zX-0Ã[¡í¡ɲ½ótz;%áÊé•Ð~ÂH0ªØƒ*XDØ;ð€
-> ssh-ed25519 HY2yRg d7+nvfAcdC3GjJxipXFrsfGGyP5jAY+gRWRV+4FVYAM
CG7r0bRGgnUWcdfDnpe7HwZ3L/y7b5iuJuqvf15b3/Y
-> ssh-ed25519 CAWG4Q X0vITOErz4wkR3VQYOcVlnrkHtwe+ytdZz1Hcrs4vVs
6IWYOhXLQ+BnML9YfLLHJYEO2CZ/uEc9IBqhoWvjDHI
-> ssh-ed25519 xA739A p5e/0AJtZ0+zbRvkB/usLuxusY8xXRx9Ksi/LQlcIHw
M4S/qlzT9POyJx4gY9lmycstUcdwG2cinN4OlV22zzo
-> ssh-ed25519 MSF3dg Ydl7uBWzBx6sAaxbzC3x8qiaU3ysGqV4rUFLpHCEV30
/1AUHBhCNOs9i7LJbmzwQDHsu+ybzYf6+coztKk5E3U
--- kYt15WxClpT7PXD1oFe9GqJU+OswjH7y9wIc8/GzZ7M
¸«ß“Ñâò`î˜üV4Fÿ¢_k)^ám$uj:ѳð¼ó…á}¨Z]$U]²uœ ¼0¶¿êv8?áXŒPûg%d…#¦d9{rAi³²

Binary file not shown.

Binary file not shown.

View File

@ -1,13 +1,13 @@
age-encryption.org/v1
-> ssh-ed25519 HY2yRg 0tpCZ5yI339pgPKGh3HJ8cnkhKlMoyYiKR1mo1cvkm0
EVVpJ8nyw/W9B65Tw59IjJC5Pb4uQX5LGnzPcf/hUs0
-> ssh-ed25519 G5LX5w YaDAKeAAunommW6q6+hTjrjaadmB17OG89t1Dx/T5z4
tJXdciiBTz9V+0nf1sGAk4vSlOgfeEgrKr+oDJ/4ays
-> ssh-ed25519 CAWG4Q i/cpMcOaZpH7aqwsR/fZiVL9CreL9dkk5F5S9dXrQBY
uU8G51pMH00ywaIVY+AzjpiqzanUYpn9ANRabugSXbE
-> ssh-ed25519 xA739A DTiXqnCz1zNgyLt8VvnOkVLDwfa0qJpUBQw9Ms/qHHA
wKjSYYOUEJkPisxT6MNW1eoYk++ECrs1ib9uEYXsAQY
-> ssh-ed25519 MSF3dg JmvJsExWPW4b6RT62mz4Wscx7EsyDPVf91A9ps9+shM
67jZYnxJpQAhnRWnTOXs+Cu445dRCpDzIGGp1xYuF3s
--- QmdvzR7QqRPxS1fHc8rR/PDZxN8u+BVKAVvE8cMLhqc
ø„»ÿEGš üQ<ùèú¥Kl´U,­äùò[-½¼Âº¨<Ucše<C5A1>³üíŒèžÈó³†)€Ñå†"f¯ËÚè¶ZÂ'C
-> ssh-ed25519 HY2yRg rsbyYULV9S/kz4OzBLQIVfyotgKrzPzvjPNVw69coTo
i9fgGAYTPxJ4Ulft3xzwNPF8v85Ae9ePMNWp593vSfA
-> ssh-ed25519 G5LX5w mhB3iiqV2e+tT31FCREX2Bqq2F2g+vTYvjCuyGSeJxs
Ep9zZykCGFW841S2mnllEi0oPnRiRuYIGtv6ckp+IBg
-> ssh-ed25519 CAWG4Q M0AJEZuiC6FnRy8rAJQ9T9dCXfIfLXGk0uBGhYOxRSg
5jSRNTi0c6we/oLBdUy1am5saH/5Nh1fmVqYajXFbGc
-> ssh-ed25519 xA739A Zf9tUKg4S4UuWMGEtAWVg0pa6vTzKIl2Ty39IjEG2mE
RCSkVFyO2ZuDlAHung9bTeM91aTXxNRJ779kE0C6pK4
-> ssh-ed25519 MSF3dg QLiG9s3mgfO6HnQ8/ReizkGllsjYebIL5ZthSVcD7Ao
YdzcodBarrdg6R96Ys01aEPoeYygbT56yz90BMFfr0U
--- fS/rGOP3IGG8b3bCDy26nBL0P1rtqC70CmKOGDsg8Tw
;YÕÁM¨_<5F>˜Zꙺ:—]Ez89zeú¹á ‘ýDÞX£<19>â‰þÁÒ9{¾x^ïþª L²lˆ<6C>ë Ýã¦9RÿVhWs

Binary file not shown.

View File

@ -1,13 +1,14 @@
age-encryption.org/v1
-> ssh-ed25519 HY2yRg T/Qom1qxE0M+FuvsXD/KZ6Usfp6v3Xwx043kDgxbCz4
6GRg0QjuHd2+d6lJfZqqPMPMjS91HEcJ/W0KRV6Et50
-> ssh-ed25519 G5LX5w pzg0wK+Q6KZP67CkyZNYbNcahlq9SIuFN18H85ARykU
aDSrO49tg/a3GOAJR96lh803bXoZqp/G6VMiSvf91vw
-> ssh-ed25519 CAWG4Q X+F/6LF8VUUoV72iCLzKKpYGRDoUHuBy1E+yr29RKEo
c779vpt/fiN7n0kGAc5jA9fWkzCPrthlNZdN4p6csrk
-> ssh-ed25519 xA739A sbg087VKj/gcycV9JrBNCoCfB4kRMDSVo3EtfpRVDyg
Lv5ges1KmxGwvz4UPZCD0v4YN2ms2Q3wmrJ14XCKYsQ
-> ssh-ed25519 MSF3dg pCLeyeWYbnNWQwwlGcsKz0KZ4BaaYKCGjo0XOPpo+no
IsNxFoB2nTxyThJxtAxSA6gauXHGQJnVefs/K2MZ+DM
--- tgB3F+k1/PQt+r5Cz+FqH31hCZFvr0Y8uZVKkdA80yo
60.ßÄïÔè¶ß(¹sÉ?68§ÆQÍI¿àûdÂñàgbðþž¬Žž`€Ú¸®AÁ”z<L}ó2&w <0B>˜6æ ;F«€rÒBR\ ½ÑÙ‹™h"<22>"¦„~qá×·ÊÆ1ƾòò!({0ó^ÿûQ¼1e†õæ°×à­<C3A0>ˆ+™
-> ssh-ed25519 HY2yRg tdVrzL3EryCEDJSiAjHfr3AC6rhyKLLe9ZaKKa/fyEk
kIbJjp/odUkQ9E2fXpk4zratLieyMNdNLHYGQt8+860
-> ssh-ed25519 G5LX5w A0wBDwowrQyByfinVVrypH5VyvyKk3O3O8+2JnVgcCI
kLiXfQkC+8QycLyyM/6dAKEE6SGxSZJS7PuOTQr10XE
-> ssh-ed25519 CAWG4Q HkbFgDtrbuv+KCwULZppiy88ZHl3kHcdlTVTOfMKTzM
KMGdQl8Gl51gUp1bxEa41a0VBBiHWD81/9C75NX/pzA
-> ssh-ed25519 xA739A XfYFE5jPFvcoTMXtwJgs3+HPLQxRmvz1W7yqE7jSYGE
497iDMqiIx1u+cBu8KZDNF2SPpGCrVqjGdUPD8kEjE4
-> ssh-ed25519 MSF3dg Vbxxsmfoywpi4W9WUMzgay3Nd1UBigliYHD7Wew9AHM
aLt5GN8jJWbbrHfs1321tQz44lBaATe0BipT/EGc80I
--- JHESkz0eGNPo3ZEGALVH4xsQ4p1O/6ShlfOw58fjH1k
³AwN¿gÇñ¦ÅC§¾²Ô¢Ö°7è ÇŸ4#0µ¹¥ssëñÆ-*žñ$Z¬ºáÃÈ[*ý÷iaò{ý?=Ýîÿv-E<E280A2>7ÆÑÉ0ôõ]Žðq0Ú)q"K€Ñ×{BZsÉâæÝ*ál´Î9-E+¦å8<(î•ôa*$dN¬³xd

View File

@ -1,13 +1,12 @@
age-encryption.org/v1
-> ssh-ed25519 G5LX5w V9bHLoGuY4stRwbzVS9Qa0L9yoY+UoCoXc+dJJQW/Ag
2ut9GfdJ3KBCqZRaloZCQsl8MLfaZAZxqj6JtPJzu2k
-> ssh-ed25519 CAWG4Q OAqnIfMECpKglZ7aF9tv/PQinG1Ou2+IEZ+nf4dtQjg
dANdMLe4iI0d6Xd/dIMpZK+mgw2+VmJFQScHaIxD7WI
-> ssh-ed25519 xA739A nVNF4Y6VSa5PP6FFBJpVmoFYYseoFx5F2wJU+Pwk+Xk
A5CiuTSNlX9Y76qhYgblBdJl3zPhtjWho2oL5/sIKu0
-> ssh-ed25519 MSF3dg /WMsGnBGzquIMyw06gHKpSS4OUxheulT59kxi+/pxxU
ppwcv7RLzUbQUM7j0Tb9rRVT9XyPMhqYr2fr4S0nTJY
--- zOe0Ko0oxArbmxePMPDVAT0pDju7IeOAih7sNrDcoVs
iÜkªA
hODVw!Ë ÕØE݈ƒÔ+±§`í¬<C3AD>ÅCî©5<C2A9>L<EFBFBD>At<1A>M^ ˜E<ÏHInnàÃÕoÁ?ój-ö
A³nԔίË>ZÕòzšë…dT½Ìb"(@‹§{_Ú<5F>C
-> ssh-ed25519 G5LX5w 5K0mzfJGvAB2LGmoQ9ZLbWooVEX6F4+fQdo1JUoB3FM
AKGa507bUrYjXFaMQ1MXTDBFYsdS6zbs+flmxYN0UNo
-> ssh-ed25519 CAWG4Q 8KzLc949on8iN1pK8q11OpCIeO71t6b0zxCLHhcQ6ns
uy7z6RdIuoUes+Uap3k5eoFFuu/DcSrEBwq4V4C/ygc
-> ssh-ed25519 xA739A SLx5cKo0fdAHj+cLpJ4FYTWTUTyDsCqKQOufDu3xnGo
VnS/WsiSaf6RpXuhgfij4pYu4p9hlJl1oXrfYY9rKlQ
-> ssh-ed25519 MSF3dg c5ZXvdNxNfZU3HeWsttuhy+UC5JxWN/IFuCuCGbksn4
vcKlIirf+VvERX71YpmwW6zp6ClhlG2PR4R8LIN7cQo
--- pJKICDaYAlxqNnvHIuzB3Yk7tv0ZNYflGTQD+Zk/8+4
Ûh/\J<>J
Ê0?¼ ÒpÎí„@܉7©¡3áÏÈøzޝãåêÉØaÞÁ'Ê,kaìIËXXOZáI\¿–åÇð <09>BPÇÐ/cUÉ¿~BóÓS' Q¥æë·f<06><EFBFBD>er©<72>ÁŠ^<>ãç8lœªV«EòÁ€

Binary file not shown.

Binary file not shown.