diff --git a/flake.nix b/flake.nix index ca0a530..037bba2 100644 --- a/flake.nix +++ b/flake.nix @@ -25,6 +25,7 @@ in bay = mkConf "bay"; lake2 = mkConf "lake2"; raccoon = mkConf "raccoon"; + fox = mkConf "fox"; }; packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // { diff --git a/keys.nix b/keys.nix index 681fcbc..a2b8c2c 100644 --- a/keys.nix +++ b/keys.nix @@ -9,10 +9,11 @@ rec { koro = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro"; bay = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay"; lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2"; + fox = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDa9lId4rB/EKGkkCCVOy0cuId2SYLs+8W8kx0kmpO1y fox"; }; hostGroup = with hosts; rec { - compute = [ owl1 owl2 ]; + compute = [ owl1 owl2 fox ]; playground = [ eudy koro ]; storage = [ bay lake2 ]; monitor = [ hut ]; diff --git a/m/common/xeon/net.nix b/m/common/xeon/net.nix index a5d87b6..ce1d3bc 100644 --- a/m/common/xeon/net.nix +++ b/m/common/xeon/net.nix @@ -85,6 +85,10 @@ 10.0.40.8 eudy xeon08 xeon08-eth0 10.0.42.8 eudy-ib xeon08-ib0 10.0.40.108 eudy-ipmi xeon08-ipmi0 + + # fox + 10.0.40.26 fox + 10.0.40.126 fox-ipmi ''; }; } diff --git a/m/fox/configuration.nix b/m/fox/configuration.nix new file mode 100644 index 0000000..38d69be --- /dev/null +++ b/m/fox/configuration.nix @@ -0,0 +1,38 @@ +{ lib, config, pkgs, ... }: + +{ + imports = [ + ../common/xeon.nix + ../module/ceph.nix + ../module/emulation.nix + ../module/slurm-client.nix + ../module/slurm-firewall.nix + ]; + + # Select the this using the ID to avoid mismatches + boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103"; + + # No swap, there is plenty of RAM + swapDevices = lib.mkForce []; + + boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; + boot.kernelModules = [ "kvm-amd" ]; + + hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; + hardware.cpu.intel.updateMicrocode = lib.mkForce false; + + networking = { + hostName = "fox"; + interfaces.enp1s0f0np0.ipv4.addresses = [ { + address = "10.0.40.26"; + prefixLength = 24; + } ]; + }; + + # Configure Nvidia driver to use with CUDA + hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production; + hardware.graphics.enable = true; + nixpkgs.config.allowUnfree = true; + nixpkgs.config.nvidia.acceptLicense = true; + services.xserver.videoDrivers = [ "nvidia" ]; +} diff --git a/m/module/slurm-client.nix b/m/module/slurm-client.nix index 7b2acb9..d745310 100644 --- a/m/module/slurm-client.nix +++ b/m/module/slurm-client.nix @@ -43,12 +43,14 @@ in { clusterName = "jungle"; nodeName = [ "owl[1,2] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl" + "fox Sockets=2 CoresPerSocket=96 ThreadsPerCore=2 Feature=fox" "hut Sockets=2 CoresPerSocket=14 ThreadsPerCore=2" ]; partitionName = [ - "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP" - "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP" + "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP" + "fox Nodes=fox Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP" + "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP" ]; # See slurm.conf(5) for more details about these options. diff --git a/secrets/ceph-user.age b/secrets/ceph-user.age index f23e2ff..880fbbf 100644 Binary files a/secrets/ceph-user.age and b/secrets/ceph-user.age differ diff --git a/secrets/gitea-runner-token.age b/secrets/gitea-runner-token.age index 1c799cf..31d52cf 100644 --- a/secrets/gitea-runner-token.age +++ b/secrets/gitea-runner-token.age @@ -1,9 +1,9 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg DQdgCk16Yu524BsrWVf0krnwWzDM6SeaJCgQipOfwCA -Ab9ocqra/UWJZI+QGMlxUhBu5AzqfjPgXl+ENIiHYGs --> ssh-ed25519 CAWG4Q KF9rGCenb3nf+wyz2hyVs/EUEbsmUs5R+1fBxlCibC8 -7++Kxbr3FHVdVfnFdHYdAuR0Tgfd+sRcO6WRss6LhEw --> ssh-ed25519 MSF3dg aUe4DhRsu4X8CFOEAnD/XM/o/0qHYSB522woCaAVh0I -GRcs5cm2YqA/lGhUtbpboBaz7mfgiLaCr+agaB7vACU ---- 9Q7Ou+Pxq+3RZilCb2dKC/pCFjZEt4rp5KnTUUU7WJ8 -1Mw4 :H@/gLtM,ƥ*zNV5mNoj1 $TG_E{%1ǯHAp \ No newline at end of file +-> ssh-ed25519 HY2yRg eRVX5yndWDLg9hw7sY1Iu8pJFy47luHvdL+zZGK2u1s +e1nXXiMW0ywkZYh2s6c7/quGMfBOJOaRhNQDjCD2Iyc +-> ssh-ed25519 CAWG4Q gYG7GRxRpJ0/5Wz0Z0J2wfLfkMFNmcy81dQEewM7gUA +lamdUdx+xOFWF1lmUM4x9TT0cJtKu9Sp7w9JHwm13u0 +-> ssh-ed25519 MSF3dg HEzfpR8alG6WPzhaEjAmmjOFoFcMSQUldx46dBsXri4 +OAD5H/zZGhfevYrFJzJrbNKPomKZDOS9Qx5tmTp78Jo +--- A0sMSiNXWaEIgRXR0x6UAIaluuVH6Zlv4CJ9sI0NXOw +6ph{>F|iv E}{ruƷ}^>c6j gGW:J3||Z \ No newline at end of file diff --git a/secrets/gitlab-bsc-docker-token.age b/secrets/gitlab-bsc-docker-token.age index 74b83e0..8c97f32 100644 --- a/secrets/gitlab-bsc-docker-token.age +++ b/secrets/gitlab-bsc-docker-token.age @@ -1,11 +1,9 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg WSdjyQPzBJ4JbzQpGeq1AAYpWKoXmLI1ZtmNmM5QOzs -qGDlDT31DQF1DdHen0+5+52DdsQlabJdA2pOB5O1I6g --> ssh-ed25519 CAWG4Q wioWMDxQjN+d4JdIbCwZg0DLQu1OH2mV6gukRprjuAs -670fE61hidOEh20hHiQAhP0+CjDF0WMBNzgwkGT8Yqg --> ssh-ed25519 MSF3dg DN19uvAEtqq4708P6HpuX9i/o/qAvHX6dj69dCF2H1o -4Lu9GnjiFLMeXJ2C7aVPJsCHCQVlhylNWJi896Av92s ---- 7cKBwOYNOUZ2h3/kAY09aSMASZSxX7hZIT4kvlIiT6w -6fQF5=bX+v e`7/A~PѦ7 -A)h=oZ$ ^V0/܅r -kubĶ:R>^gik_*% a7KG&PIn \ No newline at end of file +-> ssh-ed25519 HY2yRg 4Xns3jybBuv8flzd+h3DArVBa/AlKjt1J9jAyJsasCE +uyVjJxh5i8aGgAgCpPl6zTYeIkf9mIwURof51IKWvwE +-> ssh-ed25519 CAWG4Q T2r6r1tyNgq1XlYXVtLJFfOfUnm6pSVlPwUqC1pkyRo +9yDoKU0EC34QMUXYnsJvhPCLm6oD9w7NlTi2sheoBqQ +-> ssh-ed25519 MSF3dg Bh9DekFTq+QMUEAonwcaIAJX4Js1O7cHjDniCD0gtm8 +t/Ro0URLeDUWcvb7rlkG2s03PZ+9Rr3N4TIX03tXpVc +--- E5+/D4aK2ihKRR4YC5XOTmUbKgOqBR0Nk0gYvFOzXOI +yKF~djr%'P&_-l&o_rr߁0,U7nCTe[f97C!DEW*LAx6# sqiϟk,ݓ \ No newline at end of file diff --git a/secrets/gitlab-runner-docker-token.age b/secrets/gitlab-runner-docker-token.age index 103d6a6..cd1432e 100644 --- a/secrets/gitlab-runner-docker-token.age +++ b/secrets/gitlab-runner-docker-token.age @@ -1,9 +1,10 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg 0sEIUEJBJQ0k0rBfHaOEbq1pNBqsPin4Xq85v0ds9jY -4wzjLapoOcq53nT2K3hSGED4jTDXci25GLHkl/fL4EI --> ssh-ed25519 CAWG4Q f68ZbJGwXuCZVnqhwbh+8dh0X/MCdjEd+sVtPyBu/hU -u2TQreyWQvP6PGuwuUNKA/AL68560flqSlaItN3k41I --> ssh-ed25519 MSF3dg HdrtRW2j7mfkLH0/4aJK5R0cWdjf56HYtEZgzHi9EAs -A6MF6tXmSUq2RF2bpmav0GFTRERwluSZGh2snP/KqkA ---- drsezqi7J/g8gm6N10SkfeAWnYct99WUraB5djLJqpo -g (J!M63eA?\1yeFN\/MN`K^+"Y^>dH]PJ`x{ڱySyPX{wS ^5XJP;v \ No newline at end of file +-> ssh-ed25519 HY2yRg GdmdkW+BqqwBgu30b846jv3J7jtCM+a3rgOERuA050A +FeGqM75jG9egesR+yyVKHm0/M+uBBp5Hclg4+qN0BR8 +-> ssh-ed25519 CAWG4Q a0wTWHgulQUYDAMZmXf3dOf6PdYgCqNtSylzWVVRNVM +Bx+WSYaiY4ZwlSZJo2a1XPMQmbKOU7F0tKAqVRLBOPo +-> ssh-ed25519 MSF3dg KccUvZZUbxbCrRWUWrX8KcHF6vQ5FV/BqUqI59G7dj4 +CFr7GXpZ9rPgy7HBfOyiYF9FnZUw6KcZwq9f7/0KaU8 +--- E0Rp6RR/8+o0jvB1lRdhnlabxvI6uu/IgL2ZpPXzTc8 +#H$F;%62rfX\Dn шȉx>&;cUI=M?TǸ"pxӭ\sbFWD{ +AW>?UHԳ \ No newline at end of file diff --git a/secrets/gitlab-runner-shell-token.age b/secrets/gitlab-runner-shell-token.age index aaf939c..7c718fd 100644 --- a/secrets/gitlab-runner-shell-token.age +++ b/secrets/gitlab-runner-shell-token.age @@ -1,9 +1,9 @@ age-encryption.org/v1 --> ssh-ed25519 HY2yRg VY8s9s1zuHOv2axmIacwKg2ozsJnskHTQtslRZ3YI1M -fKkJuydLOzF/ciPYSYu4ziSCozdl6sowvDMYZmxqmHY --> ssh-ed25519 CAWG4Q 2ARFd/7RWQ/QOk47FnJFChaVBgoV4LE6EA+JHezkXgg -MV4g4Llv8Qcd/wUgJyoNG5AXb6o3aFTrOYGC+lXlSzw --> ssh-ed25519 MSF3dg SKoxWe8Mi8EkBjkESxStOCI5V4C0KYEXIOx7OdENgTA -p/owKwQ4e4pcGV+hqej2AfPU5QaM2i8VfxhlkjCM4Z4 ---- 0VWKU5CQiGbiOtQ2tsZZg88oZm1qcUDEnU5zDTtV+KU -ucl`M'Vk6Y!=Ly-OAzƘVEKR_qL|1V[)qƓLDy0_q)-T,ƪ_9 ?ib1 \ No newline at end of file +-> ssh-ed25519 HY2yRg xWRxJGWSzA5aplRYCYLB6aBwrUrQQJ2MtDYaD75V5nI +J07XF3NQiaYKKKNRcNWi9MloJD2wXHd+2K7bo6lF+QU +-> ssh-ed25519 CAWG4Q jNWymbyCczcm8RcaIEbFQBlOMALsuxTl4+pLUi0aR20 +z5NixlrRD+Y7Z/aFPs6hiDW4/lp8CBQCeJYpbuG9yYM +-> ssh-ed25519 MSF3dg QsUQloEKN3k1G49FQnNR/Do6ILgGpjFcw3zu5kk1Ako +IHwyFWUEWqCStNcFprnpBa8L5J6zKIsn+7HcgGRv3sM +--- oUia0fsL6opeYWACyXtHAu/Ld+bUIt/7S1VszYTvwgU +V*t2-7h&͢_!տ+(n (/}CNͷ|Nu5ù勚Kl"klOXyAe$ \ No newline at end of file diff --git a/secrets/jungle-robot-password.age b/secrets/jungle-robot-password.age index de9bf22..7b8cd84 100644 Binary files a/secrets/jungle-robot-password.age and b/secrets/jungle-robot-password.age differ diff --git a/secrets/munge-key.age b/secrets/munge-key.age index ead42c8..ce50c12 100644 Binary files a/secrets/munge-key.age and b/secrets/munge-key.age differ diff --git a/secrets/nix-serve.age b/secrets/nix-serve.age index f59c323..ecd0593 100644 Binary files a/secrets/nix-serve.age and b/secrets/nix-serve.age differ