diff --git a/keys.nix b/keys.nix index a2b8c2c1..83b8ff4a 100644 --- a/keys.nix +++ b/keys.nix @@ -9,11 +9,12 @@ rec { koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; - fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDa9lId4rB/EKGkkCCVOy0cuId2SYLs+8W8kx0kmpO1y fox"; + fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox"; }; hostGroup = with hosts; rec { - compute = [ owl1 owl2 fox ]; + untrusted = [ fox ]; + compute = [ owl1 owl2 ]; playground = [ eudy koro ]; storage = [ bay lake2 ]; monitor = [ hut ]; diff --git a/m/common/xeon/net.nix b/m/common/xeon/net.nix index 09e83edf..dfd85f88 100644 --- a/m/common/xeon/net.nix +++ b/m/common/xeon/net.nix @@ -85,10 +85,6 @@ 10.0.40.8 eudy xeon08 xeon08-eth0 10.0.42.8 eudy-ib xeon08-ib0 10.0.40.108 eudy-ipmi xeon08-ipmi0 xeon08-ipmi - - # fox - 10.0.40.26 fox - 10.0.40.126 fox-ipmi ''; }; } diff --git a/m/fox/configuration.nix b/m/fox/configuration.nix index 97ac6863..29d8fe95 100644 --- a/m/fox/configuration.nix +++ b/m/fox/configuration.nix @@ -2,11 +2,9 @@ { imports = [ - ../common/xeon.nix - ../module/ceph.nix + ../common/base.nix + ../common/xeon/console.nix ../module/emulation.nix - ../module/slurm-client.nix - ../module/slurm-firewall.nix ]; # Select the this using the ID to avoid mismatches @@ -22,11 +20,30 @@ hardware.cpu.intel.updateMicrocode = lib.mkForce false; networking = { + defaultGateway = "147.83.30.130"; + nameservers = [ "8.8.8.8" ]; hostName = "fox"; - interfaces.enp1s0f0np0.ipv4.addresses = [ { - address = "10.0.40.26"; - prefixLength = 24; - } ]; + interfaces.enp1s0f0np0.ipv4.addresses = [ + { + # UPC network + # Public IP configuration: + # - Hostname: fox.ac.upc.edu + # - IP: 147.83.30.141 + # - Gateway: 147.83.30.130 + # - NetMask: 255.255.255.192 + # Private IP configuration for BMC: + # - Hostname: fox-ipmi.ac.upc.edu + # - IP: 147.83.35.27 + # - Gateway: 147.83.35.2 + # - NetMask: 255.255.255.0 + address = "147.83.30.141"; + prefixLength = 26; # 255.255.255.192 + } + ]; + extraHosts = '' + 147.83.30.141 fox.ac.upc.edu + 147.83.35.27 fox-ipmi.ac.upc.edu + ''; }; # Configure Nvidia driver to use with CUDA @@ -56,20 +73,4 @@ wantedBy = [ "multi-user.target" ]; serviceConfig.ExecStart = script; }; - - # Only allow SSH connections from users who have a SLURM allocation - # See: https://slurm.schedmd.com/pam_slurm_adopt.html - security.pam.services.sshd.rules.account.slurm = { - control = "required"; - enable = true; - modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so"; - args = [ "log_level=debug5" ]; - order = 999999; # Make it last one - }; - - # Disable systemd session (pam_systemd.so) as it will conflict with the - # pam_slurm_adopt.so module. What happens is that the shell is first adopted - # into the slurmstepd task and then into the systemd session, which is not - # what we want, otherwise it will linger even if all jobs are gone. - security.pam.services.sshd.startSession = lib.mkForce false; } diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 7042c913..db5f49fb 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -169,6 +169,9 @@ "8.8.8.8" "ssfhead" "anella-bsc.cesca.cat" + "upc-anella.cesca.cat" + "fox.ac.upc.edu" + "arenys5.ac.upc.edu" ]; }]; relabel_configs = [ @@ -264,17 +267,6 @@ } ]; } - { - job_name = "ipmi-fox"; - metrics_path = "/ipmi"; - static_configs = [ - { targets = [ "127.0.0.1:9290" ]; } - ]; - params = { - target = [ "fox-ipmi" ]; - module = [ "fox" ]; - }; - } ]; }; } diff --git a/m/module/slurm-client.nix b/m/module/slurm-client.nix index 46478a81..21ae9458 100644 --- a/m/module/slurm-client.nix +++ b/m/module/slurm-client.nix @@ -43,13 +43,11 @@ in { clusterName = "jungle"; nodeName = [ "owl[1,2] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl" - "fox Sockets=2 CoresPerSocket=96 ThreadsPerCore=1 Feature=fox" "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" ]; partitionName = [ "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP" - "fox Nodes=fox Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP" ]; # See slurm.conf(5) for more details about these options. @@ -77,7 +75,7 @@ in { SuspendTimeout=60 ResumeProgram=${resumeProgram} ResumeTimeout=300 - SuspendExcNodes=hut,fox + SuspendExcNodes=hut # Turn the nodes off after 1 hour of inactivity SuspendTime=3600 diff --git a/secrets/ceph-user.age b/secrets/ceph-user.age index 880fbbf3..c37e0f47 100644 Binary files a/secrets/ceph-user.age and b/secrets/ceph-user.age differ diff --git a/secrets/gitea-runner-token.age b/secrets/gitea-runner-token.age index 31d52cf5..2b59fef1 100644 Binary files a/secrets/gitea-runner-token.age and b/secrets/gitea-runner-token.age differ diff --git a/secrets/gitlab-bsc-docker-token.age b/secrets/gitlab-bsc-docker-token.age index 74b83e04..f3798053 100644 --- a/secrets/gitlab-bsc-docker-token.age +++ b/secrets/gitlab-bsc-docker-token.age @@ -1,11 +1,10 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg WSdjyQPzBJ4JbzQpGeq1AAYpWKoXmLI1ZtmNmM5QOzs -qGDlDT31DQF1DdHen0+5+52DdsQlabJdA2pOB5O1I6g --> ssh-ed25519 CAWG4Q wioWMDxQjN+d4JdIbCwZg0DLQu1OH2mV6gukRprjuAs -670fE61hidOEh20hHiQAhP0+CjDF0WMBNzgwkGT8Yqg --> ssh-ed25519 MSF3dg DN19uvAEtqq4708P6HpuX9i/o/qAvHX6dj69dCF2H1o -4Lu9GnjiFLMeXJ2C7aVPJsCHCQVlhylNWJi896Av92s ---- 7cKBwOYNOUZ2h3/kAY09aSMASZSxX7hZIT4kvlIiT6w -6fQF5=bX+v e`7/A~PѦ7 -A)h=oZ$ ^V0/܅r -kubĶ:R>^gik_*% a7KG&PIn \ No newline at end of file +-> ssh-ed25519 HY2yRg XPOFoZqY+AnKC77jrgNqAm1ADphurfuhO4NRrfiuUDc +iCfMMpGHyaYHGy6ci8sqjUtcPeteLlyvLGEF79VPOEc +-> ssh-ed25519 CAWG4Q 6OsGrnM+/c5lTN81Rvp166K+ygmSIFeSYzXxYg25KGE +Av1zTw2zK4Gufzti9kQaye7C362GCiDRRHzCqBLR33g +-> ssh-ed25519 MSF3dg 8CHqJ7mEDvjvqbmF+eE6Em1Wi6eHAzEUpiExC1gm7S0 +bdwzYHw3RAbdHq+RsiFUP++sQ586VUlSnAzAOhiQUjI +--- gA5XSUfjUBol938sC5DbUf8PvQUIr2pNkS2nL95OF9c +Ea1G7ݩ[R\{~$GocQwKP&w6] +ѣ^z̄ 1kY2p2Knok/Xpt''$0co= \ No newline at end of file diff --git a/secrets/gitlab-runner-docker-token.age b/secrets/gitlab-runner-docker-token.age index cd1432e3..3efea559 100644 --- a/secrets/gitlab-runner-docker-token.age +++ b/secrets/gitlab-runner-docker-token.age @@ -1,10 +1,9 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg GdmdkW+BqqwBgu30b846jv3J7jtCM+a3rgOERuA050A -FeGqM75jG9egesR+yyVKHm0/M+uBBp5Hclg4+qN0BR8 --> ssh-ed25519 CAWG4Q a0wTWHgulQUYDAMZmXf3dOf6PdYgCqNtSylzWVVRNVM -Bx+WSYaiY4ZwlSZJo2a1XPMQmbKOU7F0tKAqVRLBOPo --> ssh-ed25519 MSF3dg KccUvZZUbxbCrRWUWrX8KcHF6vQ5FV/BqUqI59G7dj4 -CFr7GXpZ9rPgy7HBfOyiYF9FnZUw6KcZwq9f7/0KaU8 ---- E0Rp6RR/8+o0jvB1lRdhnlabxvI6uu/IgL2ZpPXzTc8 -#H$F;%62rfX\Dn шȉx>&;cUI=M?TǸ"pxӭ\sbFWD{ -AW>?UHԳ \ No newline at end of file +-> ssh-ed25519 HY2yRg pXNTB/ailRwSEJG1pXvrzzpz5HqkDZdWVWnOH7JGeQ4 +NzA+2fxfkNRy/u+Zq96A02K1Vxy0ETYZjMkDVTKyCY8 +-> ssh-ed25519 CAWG4Q 7CLJWn+EAxoWDduXaOSrHaBFHQ4GIpYP/62FFTj3ZTI +vSYV1pQg2qI2ngCzM0nCZAnqdz1tbT4hM5m+/TyGU2c +-> ssh-ed25519 MSF3dg Akmp4NcZcDuaYHta/Vej6zulNSrAOCd5lmSV+OiBGC4 +qTxqVzTyywur+GjtUQdbaIUdH1fqCqPe6qPf8iHRa4w +--- uCKNqD1TmZZThOzlpsecBKx/k+noIWhCVMr/pzNwBr8 +r'Ƌs4˺AĥPL7` ) H-0AH5LQeH2bB޲CJG"-S\ H ssh-ed25519 HY2yRg xWRxJGWSzA5aplRYCYLB6aBwrUrQQJ2MtDYaD75V5nI -J07XF3NQiaYKKKNRcNWi9MloJD2wXHd+2K7bo6lF+QU --> ssh-ed25519 CAWG4Q jNWymbyCczcm8RcaIEbFQBlOMALsuxTl4+pLUi0aR20 -z5NixlrRD+Y7Z/aFPs6hiDW4/lp8CBQCeJYpbuG9yYM --> ssh-ed25519 MSF3dg QsUQloEKN3k1G49FQnNR/Do6ILgGpjFcw3zu5kk1Ako -IHwyFWUEWqCStNcFprnpBa8L5J6zKIsn+7HcgGRv3sM ---- oUia0fsL6opeYWACyXtHAu/Ld+bUIt/7S1VszYTvwgU -V*t2-7h&͢_!տ+(n (/}CNͷ|Nu5ù勚Kl"klOXyAe$ \ No newline at end of file +-> ssh-ed25519 HY2yRg s6iI9f25xulF4KXt+XY07kXXPKxXo7f2Ql/OTHN55Hk +WO4Fd2H9c+HL3+XhUF3BmEZVILlcchGxSrSmL2OEdGw +-> ssh-ed25519 CAWG4Q TBkdpx8k8K1NvW3wcvaF7omKFwEJ2DxWJp3tIOTjwCA +LcYgWRix23AQnw0OQ7f8+8S3J84CHUElX1vKZSETiLE +-> ssh-ed25519 MSF3dg WzrF8kjTP7BXXDjmUp7kPCKguthAW12RPo6Vy2RMmh4 +8C3mT9ktudCTANDxhyNszUkbeDG6X4wOJdx825++dYM +--- /w3YQ2UeTi67H1JR0GsdPz2KoLN2Y7BIZfFY+//AWjY +ӣ-`P@ބ)99l ZfV?I>΍w鉐 z40 2{i@ZxAHn% ʤ/WĔl}&얶(KSoz=d \ No newline at end of file diff --git a/secrets/ipmi.yml.age b/secrets/ipmi.yml.age index ec99e58f..02404781 100644 Binary files a/secrets/ipmi.yml.age and b/secrets/ipmi.yml.age differ diff --git a/secrets/jungle-robot-password.age b/secrets/jungle-robot-password.age index 7b8cd842..3e7053e4 100644 Binary files a/secrets/jungle-robot-password.age and b/secrets/jungle-robot-password.age differ diff --git a/secrets/munge-key.age b/secrets/munge-key.age index ce50c123..372d152b 100644 Binary files a/secrets/munge-key.age and b/secrets/munge-key.age differ diff --git a/secrets/nix-serve.age b/secrets/nix-serve.age index ecd0593e..a498056f 100644 Binary files a/secrets/nix-serve.age and b/secrets/nix-serve.age differ