Add AMD uProf section to fox documentation

It tries to dlopen libcrypt.so.1 and libstdc++.so.6, so we make sure
they are available by adding them to the runpath.

.gitignore

@@ -1,2 +1,3 @@
 *.swp
 /result
+/misc

doc/install.md

@@ -150,3 +150,27 @@ And update grub.
 ```
 # nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
 ```
+
+## Chain NixOS in same disk with other systems
+
+To install NixOS on a partition along another system which controls the GRUB,
+first disable the grub device, so the GRUB is not installed in the disk by
+NixOS (only the /boot files will be generated):
+
+```
+boot.loader.grub.device = "nodev";
+```
+
+Then add the following entry to the old GRUB configuration:
+
+```
+menuentry 'NixOS' {
+        insmod chain
+        search --no-floppy --label nixos --set root
+        configfile /boot/grub/grub.cfg
+}
+```
+
+The partition with NixOS must have the label "nixos" for it to be found. New
+system configuration entries will be stored in the GRUB configuration managed
+by NixOS, so there is no need to change the old GRUB settings.

flake.lock

@@ -6,14 +6,15 @@
         "home-manager": "home-manager",
         "nixpkgs": [
           "nixpkgs"
-        ]
+        ],
+        "systems": "systems"
       },
       "locked": {
-        "lastModified": 1690228878,
-        "narHash": "sha256-9Xe7JV0krp4RJC9W9W9WutZVlw6BlHTFMiUP/k48LQY=",
+        "lastModified": 1750173260,
+        "narHash": "sha256-9P1FziAwl5+3edkfFcr5HeGtQUtrSdk/MksX39GieoA=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "d8c973fd228949736dedf61b7f8cc1ece3236792",
+        "rev": "531beac616433bac6f9e2a19feb8e99a22a66baf",
         "type": "github"
       },
       "original": {
@@ -29,17 +30,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1694708510,
-        "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
+        "lastModified": 1749650500,
+        "narHash": "sha256-2MHfVPV6RA7qPSCtXh4+KK0F0UjN+J4z8//+n6NK7Xs=",
         "ref": "refs/heads/master",
-        "rev": "3a4062ac04be6263c64a481420d8e768c2521b80",
-        "revCount": 862,
+        "rev": "9d1944c658929b6f98b3f3803fead4d1b91c4405",
+        "revCount": 961,
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
       },
       "original": {
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
       }
     },
     "darwin": {
@@ -50,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1673295039,
-        "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
+        "lastModified": 1744478979,
+        "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
+        "rev": "43975d782b418ebf4969e9ccba82466728c2851b",
         "type": "github"
       },
       "original": {
@@ -72,11 +73,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1682203081,
-        "narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
+        "lastModified": 1745494811,
+        "narHash": "sha256-YZCh2o9Ua1n9uCvrvi5pRxtuVNml8X2a03qIFfRKpFs=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
+        "rev": "abfad3d2958c9e6300a883bd443512c55dfeb1be",
         "type": "github"
       },
       "original": {
@@ -87,16 +88,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1693663421,
-        "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
+        "lastModified": 1752436162,
+        "narHash": "sha256-Kt1UIPi7kZqkSc5HVj6UY5YLHHEzPBkgpNUByuyxtlw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e56990880811a451abd32515698c712788be5720",
+        "rev": "dfcd5b901dbab46c9c6e80b265648481aafb01f8",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -107,6 +108,21 @@
         "bscpkgs": "bscpkgs",
         "nixpkgs": "nixpkgs"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -1,9 +1,9 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
-    bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
+    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
     bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
   };
 
@@ -17,13 +17,18 @@ let
 in
   {
     nixosConfigurations = {
-      hut   = mkConf "hut";
-      owl1  = mkConf "owl1";
-      owl2  = mkConf "owl2";
-      eudy  = mkConf "eudy";
-      koro  = mkConf "koro";
-      bay   = mkConf "bay";
-      lake2 = mkConf "lake2";
+      hut     = mkConf "hut";
+      tent    = mkConf "tent";
+      owl1    = mkConf "owl1";
+      owl2    = mkConf "owl2";
+      eudy    = mkConf "eudy";
+      koro    = mkConf "koro";
+      bay     = mkConf "bay";
+      lake2   = mkConf "lake2";
+      raccoon = mkConf "raccoon";
+      fox     = mkConf "fox";
+      apex    = mkConf "apex";
+      weasel  = mkConf "weasel";
     };
 
     packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

keys.nix

@@ -2,28 +2,35 @@
 # here all the public keys
 rec {
   hosts = {
-    hut   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
-    owl1  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
-    owl2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
-    eudy  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
-    koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
-    bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
-    lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    hut    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
+    owl1   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
+    owl2   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
+    eudy   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
+    koro   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
+    bay    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
+    lake2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
+    tent   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
+    apex   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
+    weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
   };
 
   hostGroup = with hosts; rec {
-    compute    = [ owl1 owl2 ];
-    playground = [ eudy koro ];
+    compute    = [ owl1 owl2 fox ];
+    playground = [ eudy koro weasel ];
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
+    login      = [ apex ];
 
-    system     = storage ++ monitor;
+    system     = storage ++ monitor ++ login;
     safe       = system ++ compute;
     all        = safe ++ playground;
   };
 
   admins = {
-    rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
-    root   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
+    "rarias@hut"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
+    "rarias@tent" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIwlWSBTZi74WTz5xn6gBvTmCoVltmtIAeM3RMmkh4QZ rarias@tent";
+    "rarias@fox"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDSbw3REAKECV7E2c/e2XJITudJQWq2qDSe2N1JHqHZd rarias@fox";
+    root          = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
   };
 }

m/apex/configuration.nix

@@ -0,0 +1,85 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../common/ssf/hosts.nix
+    ../module/ceph.nix
+    ../module/slurm-server.nix
+    ./nfs.nix
+    ./wireguard.nix
+  ];
+
+  # Don't install grub MBR for now
+  boot.loader.grub.device = "nodev";
+
+  boot.initrd.kernelModules = [
+    "megaraid_sas" # For HW RAID
+  ];
+
+  environment.systemPackages = with pkgs; [
+    storcli # To manage HW RAID
+  ];
+
+  fileSystems."/home" = {
+    device = "/dev/disk/by-label/home";
+    fsType = "ext4";
+  };
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  networking = {
+    hostName = "apex";
+    defaultGateway = "84.88.53.233";
+    nameservers = [ "8.8.8.8" ];
+
+    # Public facing interface
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "84.88.53.236";
+      prefixLength = 29;
+    } ];
+
+    # Internal LAN to our Ethernet switch
+    interfaces.eno2.ipv4.addresses = [ {
+      address = "10.0.40.30";
+      prefixLength = 24;
+    } ];
+
+    # Infiniband over Omnipath switch (disconnected for now)
+    # interfaces.ibp5s0 = {};
+
+    nat = {
+      enable = true;
+      internalInterfaces = [ "eno2" ];
+      externalInterface = "eno1";
+    };
+  };
+
+  # Use SSH tunnel to reach internal hosts
+  programs.ssh.extraConfig = ''
+    Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+    Host raccoon
+      HostName knights3.bsc.es
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+    Host tent
+      ProxyJump raccoon
+  '';
+
+  networking.firewall = {
+    extraCommands = ''
+      # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our
+      # logs. Insert as first position so we also protect SSH.
+      iptables -I nixos-fw 1 -p tcp -s 192.168.8.16 -j nixos-fw-refuse
+      # Same with opsmonweb01.bsc.es which seems to be trying to access via SSH
+      iptables -I nixos-fw 2 -p tcp -s 84.88.52.176 -j nixos-fw-refuse
+    '';
+  };
+
+  # Use tent for cache
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+}

m/apex/nfs.nix

@@ -0,0 +1,48 @@
+{ ... }:
+
+{
+  services.nfs.server = {
+    enable = true;
+    lockdPort = 4001;
+    mountdPort = 4002;
+    statdPort = 4000;
+    exports = ''
+      /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+      /home 10.106.0.0/24(rw,async,no_subtree_check,no_root_squash)
+    '';
+  };
+  networking.firewall = {
+    # Check with `rpcinfo -p`
+    extraCommands = ''
+      # Accept NFS traffic from compute nodes but not from the outside
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+
+      # Accept NFS traffic from wg0
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
+    '';
+  };
+}

m/apex/wireguard.nix

@@ -0,0 +1,35 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgApex.file = ../../secrets/wg-apex.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      ips = [ "10.106.0.30/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgApex.path;
+      # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=
+      peers = [
+        {
+          name = "Fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.0/24" ];
+          endpoint = "fox.ac.upc.edu:666";
+          # Send keepalives every 25 seconds. Important to keep NAT tables alive.
+          persistentKeepalive = 25;
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1" = [ "fox" ];
+  };
+}

m/bay/configuration.nix

@@ -2,21 +2,21 @@
 
 {
   imports = [
-    ../common/main.nix
-    ../common/monitoring.nix
+    ../common/ssf.nix
+    ../module/monitoring.nix
   ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
 
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-
   networking = {
     hostName = "bay";
     interfaces.eno1.ipv4.addresses = [ {
@@ -27,6 +27,16 @@
       address = "10.0.42.40";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from lake2
+        iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
   };
 
   services.ceph = {

m/common/base.nix

@@ -0,0 +1,21 @@
+{
+  # All machines should include this profile.
+  # Includes the basic configuration for an Intel server.
+  imports = [
+    ./base/agenix.nix
+    ./base/always-power-on.nix
+    ./base/august-shutdown.nix
+    ./base/boot.nix
+    ./base/env.nix
+    ./base/fs.nix
+    ./base/hw.nix
+    ./base/net.nix
+    ./base/nix.nix
+    ./base/ntp.nix
+    ./base/rev.nix
+    ./base/ssh.nix
+    ./base/users.nix
+    ./base/watchdog.nix
+    ./base/zsh.nix
+  ];
+}

m/common/base/always-power-on.nix

@@ -0,0 +1,8 @@
+{
+  imports = [
+    ../../module/power-policy.nix
+  ];
+
+  # Turn on as soon as we have power
+  power.policy = "always-on";
+}

m/common/base/august-shutdown.nix

@@ -0,0 +1,14 @@
+{
+  # Shutdown all machines on August 3rd at 22:00, so we can protect the
+  # hardware from spurious electrical peaks on the yearly electrical cut for
+  # manteinance that starts on August 4th.
+  systemd.timers.august-shutdown = {
+    description = "Shutdown on August 3rd for maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-08-03 22:00:00";
+      RandomizedDelaySec = "10min";
+      Unit = "systemd-poweroff.service";
+    };
+  };
+}

m/common/base/boot.nix

@@ -2,7 +2,7 @@
 
 {
   # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = lib.mkForce true;
+  boot.loader.grub.enable = true;
 
   # Enable GRUB2 serial console
   boot.loader.grub.extraConfig = ''
@@ -11,14 +11,12 @@
     terminal_output --append serial
   '';
 
-  # Enable serial console
-  boot.kernelParams = [
-    "console=tty1"
-    "console=ttyS0,115200"
-  ];
-
   boot.kernel.sysctl = {
     "kernel.perf_event_paranoid" = lib.mkDefault "-1";
+
+    # Allow ptracing (i.e. attach with GDB) any process of the same user, see:
+    # https://www.kernel.org/doc/Documentation/security/Yama.txt
+    "kernel.yama.ptrace_scope" = "0";
   };
 
   boot.kernelPackages = pkgs.linuxPackages_latest;

m/common/base/env.nix

@@ -0,0 +1,37 @@
+{ pkgs, config, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns pv
+    # From bsckgs overlay
+    osumb
+  ];
+
+  programs.direnv.enable = true;
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  environment.enableAllTerminfo = true;
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+}

m/common/base/fs.nix

@@ -13,16 +13,12 @@
     [ { device = "/dev/disk/by-label/swap"; }
     ];
 
-  # Mount the home via NFS
-  fileSystems."/home" = {
-    device = "10.0.40.30:/home";
-    fsType = "nfs";
-    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
-  };
-
   # Tracing
   fileSystems."/sys/kernel/tracing" = {
     device = "none";
     fsType = "tracefs";
   };
+
+  # Mount a tmpfs into /tmp
+  boot.tmp.useTmpfs = true;
 }

m/common/base/net.nix

@@ -0,0 +1,22 @@
+{ pkgs, lib, ... }:
+
+{
+  networking = {
+    enableIPv6 = false;
+    useDHCP = false;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
+    };
+
+    # Make sure we use iptables
+    nftables.enable = lib.mkForce false;
+
+    hosts = {
+      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.51.152" = [ "raccoon" ];
+      "84.88.51.142" = [ "raccoon-ipmi" ];
+    };
+  };
+}

m/common/base/nix.nix

@@ -0,0 +1,60 @@
+{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
+
+{
+  nixpkgs.overlays = [
+    bscpkgs.bscOverlay
+    (import ../../../pkgs/overlay.nix)
+  ];
+
+  nixpkgs.config.allowUnfree = true;
+
+  nix = {
+    nixPath = [
+      "nixpkgs=${nixpkgs}"
+      "jungle=${theFlake.outPath}"
+    ];
+
+    registry = {
+      nixpkgs.flake = nixpkgs;
+      jungle.flake = theFlake;
+    };
+
+    settings = {
+      experimental-features = [ "nix-command" "flakes" ];
+      sandbox = "relaxed";
+      trusted-users = [ "@wheel" ];
+      flake-registry = pkgs.writeText "global-registry.json"
+        ''{"flakes":[],"version":2}'';
+      keep-outputs = true;
+    };
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 30d";
+    };
+  };
+
+  # The nix-gc.service can begin its execution *before* /home is mounted,
+  # causing it to remove all gcroots considering them as stale, as it cannot
+  # access the symlink. To prevent this problem, we force the service to wait
+  # until /home is mounted as well as other remote FS like /ceph.
+  systemd.services.nix-gc = {
+    # Start remote-fs.target if not already being started and fail if it fails
+    # to start. It will also be stopped if the remote-fs.target fails after
+    # starting successfully.
+    bindsTo = [ "remote-fs.target" ];
+    # Wait until remote-fs.target fully starts before starting this one.
+    after = [ "remote-fs.target"];
+    # Ensure we can access a remote path inside /home
+    unitConfig.ConditionPathExists = "/home/Computational";
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}

m/common/base/rev.nix

@@ -1,6 +1,7 @@
 { theFlake, ... }:
 
 let
+  # Prevent building a configuration without revision
   rev = if theFlake ? rev then theFlake.rev
     else throw ("Refusing to build from a dirty Git tree!");
 in {
@@ -15,4 +16,6 @@ in {
     DATENOW=$(date --iso-8601=seconds)
     echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
   '';
+
+  system.configurationRevision = rev;
 }

m/common/base/ssh.nix

@@ -1,22 +1,18 @@
 { lib, ... }:
 
 let
-  keys = import ../../keys.nix;
+  keys = import ../../../keys.nix;
   hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
 in
 {
   # Enable the OpenSSH daemon.
   services.openssh.enable = true;
 
-  # Connect to intranet git hosts via proxy
-  programs.ssh.extraConfig = ''
-    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
-      User git
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
-  '';
-
   programs.ssh.knownHosts = hostsKeys // {
     "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
     "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
+    "bscpm04.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT";
+    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
+    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
   };
 }

m/common/base/users.nix

@@ -0,0 +1,178 @@
+{ pkgs, ... }:
+
+{
+  imports = [
+    ../../module/jungle-users.nix
+  ];
+
+  users = {
+    mutableUsers = false;
+    users = {
+      # Generate hashedPassword with `mkpasswd -m sha-512`
+
+      root.openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"
+      ];
+
+      rarias = {
+        uid = 1880;
+        isNormalUser = true;
+        linger = true;
+        home = "/home/Computational/rarias";
+        description = "Rodrigo Arias";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
+        ];
+        shell = pkgs.zsh;
+      };
+
+      arocanon = {
+        uid = 1042;
+        isNormalUser = true;
+        home = "/home/Computational/arocanon";
+        description = "Aleix Roca";
+        group = "Computational";
+        extraGroups = [ "wheel" "tracing" ];
+        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
+        ];
+      };
+    };
+
+    jungleUsers = {
+      rpenacob = {
+        uid = 2761;
+        isNormalUser = true;
+        home = "/home/Computational/rpenacob";
+        description = "Raúl Peñacoba";
+        group = "Computational";
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "fox" ];
+        hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
+        ];
+      };
+
+      anavarro = {
+        uid = 1037;
+        isNormalUser = true;
+        home = "/home/Computational/anavarro";
+        description = "Antoni Navarro";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "raccoon" "fox" "weasel" ];
+        hashedPassword = "$6$EgturvVYXlKgP43g$gTN78LLHIhaF8hsrCXD.O6mKnZSASWSJmCyndTX8QBWT6wTlUhcWVAKz65lFJPXjlJA4u7G1ydYQ0GG6Wk07b1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMsbM21uepnJwPrRe6jYFz8zrZ6AYMtSEvvt4c9spmFP toni@delltoni"
+        ];
+      };
+
+      abonerib = {
+        uid = 4541;
+        isNormalUser = true;
+        home = "/home/Computational/abonerib";
+        description = "Aleix Boné";
+        group = "Computational";
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "raccoon" "fox" "weasel" ];
+        hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
+        ];
+      };
+
+      vlopez = {
+        uid = 4334;
+        isNormalUser = true;
+        home = "/home/Computational/vlopez";
+        description = "Victor López";
+        group = "Computational";
+        hosts = [ "apex" "koro" ];
+        hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
+        ];
+      };
+
+      dbautist = {
+        uid = 5649;
+        isNormalUser = true;
+        home = "/home/Computational/dbautist";
+        description = "Dylan Bautista Cases";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "raccoon" ];
+        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
+        ];
+      };
+
+      dalvare1 = {
+        uid = 2758;
+        isNormalUser = true;
+        home = "/home/Computational/dalvare1";
+        description = "David Álvarez";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "fox" ];
+        hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
+        ];
+      };
+
+      varcila = {
+        uid = 5650;
+        isNormalUser = true;
+        home = "/home/Computational/varcila";
+        description = "Vincent Arcila";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "fox" ];
+        hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
+        ];
+      };
+
+      pmartin1 = {
+        # Arbitrary UID but large so it doesn't collide with other users on ssfhead.
+        uid = 9652;
+        isNormalUser = true;
+        home = "/home/Computational/pmartin1";
+        description = "Pedro J. Martinez-Ferrer";
+        group = "Computational";
+        hosts = [ "fox" ];
+        hashedPassword = "$6$nIgDMGnt4YIZl3G.$.JQ2jXLtDPRKsbsJfJAXdSvjDIzRrg7tNNjPkLPq3KJQhMjfDXRUvzagUHUU2TrE2hHM8/6uq8ex0UdxQ0ysl.";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
+        ];
+      };
+
+      csiringo = {
+        # Arbitrary UID but large so it doesn't collide with other users on ssfhead.
+        uid = 9653;
+        isNormalUser = true;
+        home = "/home/Computational/csiringo";
+        description = "Cesare Siringo";
+        group = "Computational";
+        hosts = [ "apex" "weasel" ];
+        hashedPassword = "$6$0IsZlju8jFukLlAw$VKm0FUXbS.mVmPm3rcJeizTNU4IM5Nmmy21BvzFL.cQwvlGwFI1YWRQm6gsbd4nbg47mPDvYkr/ar0SlgF6GO1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHA65zvvG50iuFEMf+guRwZB65jlGXfGLF4HO+THFaed csiringo@bsc.es"
+        ];
+      };
+    };
+
+    groups = {
+      Computational = { gid = 564; };
+      tracing = { };
+    };
+  };
+}

m/common/base/zsh.nix

@@ -2,7 +2,6 @@
 
 {
   environment.systemPackages = with pkgs; [
-    direnv
     zsh-completions
     nix-zsh-completions
   ];

m/common/main.nix

@@ -1,95 +0,0 @@
-{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
-
-{
-  imports = [
-    ./agenix.nix
-    ./boot.nix
-    ./fs.nix
-    ./hw.nix
-    ./net.nix
-    ./ntp.nix
-    ./slurm.nix
-    ./ssh.nix
-    ./users.nix
-    ./watchdog.nix
-    ./rev.nix
-    ./zsh.nix
-  ];
-
-  nixpkgs.overlays = [
-    bscpkgs.bscOverlay
-    (import ../../pkgs/overlay.nix)
-  ];
-
-  system.configurationRevision =
-    if theFlake ? rev
-    then theFlake.rev
-    else throw ("Refusing to build from a dirty Git tree!");
-
-  nix.nixPath = [
-    "nixpkgs=${nixpkgs}"
-    "jungle=${theFlake.outPath}"
-  ];
-
-  nix.settings.flake-registry =
-    pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
-
-  nix.registry.nixpkgs.flake = nixpkgs;
-  nix.registry.jungle.flake = theFlake;
-
-  environment.systemPackages = with pkgs; [
-    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
-    # From bsckgs overlay
-    bsc.osumb
-  ];
-
-  systemd.services."serial-getty@ttyS0" = {
-    enable = true;
-    wantedBy = [ "getty.target" ];
-    serviceConfig.Restart = "always";
-  };
-
-  # Increase limits
-  security.pam.loginLimits = [
-    {
-      domain = "*";
-      type = "-";
-      item = "memlock";
-      value = "1048576"; # 1 GiB of mem locked
-    }
-  ];
-
-  time.timeZone = "Europe/Madrid";
-  i18n.defaultLocale = "en_DK.UTF-8";
-
-  environment.variables = {
-    EDITOR = "vim";
-    VISUAL = "vim";
-  };
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  nix.settings.sandbox = "relaxed";
-  nix.settings.trusted-users = [ "@wheel" ];
-  nix.gc.automatic = true;
-  nix.gc.dates = "weekly";
-  nix.gc.options = "--delete-older-than 30d";
-
-  programs.bash.promptInit = ''
-    PS1="\h\\$ "
-  '';
-
-  # Copy the NixOS configuration file and link it from the resulting system
-  # (/run/current-system/configuration.nix). This is useful in case you
-  # accidentally delete configuration.nix.
-  #system.copySystemConfiguration = true;
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "22.11"; # Did you read the comment?
-}

m/common/net.nix

@@ -1,94 +0,0 @@
-{ pkgs, ... }:
-
-{
-  # Infiniband (IPoIB)
-  environment.systemPackages = [ pkgs.rdma-core ];
-  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
-
-  networking = {
-    enableIPv6 = false;
-    useDHCP = false;
-    defaultGateway = "10.0.40.30";
-    nameservers = ["8.8.8.8"];
-    proxy = {
-      default = "http://localhost:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
-      # Don't set all_proxy as go complains and breaks the gitlab runner, see:
-      # https://github.com/golang/go/issues/16715
-      allProxy = null;
-    };
-
-    firewall = {
-      enable = true;
-      allowedTCPPorts = [ 22 ];
-      extraCommands = ''
-        # Prevent ssfhead from contacting our slurmd daemon
-        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse
-        # But accept traffic to slurm ports from any other node in the subnet
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
-        # We also need to open the srun port range
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
-      '';
-    };
-
-    extraHosts = ''
-      10.0.40.30      ssfhead
-      84.88.53.236    ssfhead.bsc.es ssfhead
-      
-      # Node Entry for node: mds01 (ID=72)
-      10.0.40.40              bay mds01 mds01-eth0
-      10.0.42.40              bay-ib mds01-ib0
-      10.0.40.141             bay-ipmi mds01-ipmi0
-      
-      # Node Entry for node: oss01 (ID=73)
-      10.0.40.41              oss01 oss01-eth0
-      10.0.42.41              oss01-ib0
-      10.0.40.142             oss01-ipmi0
-      
-      # Node Entry for node: oss02 (ID=74)
-      10.0.40.42              lake2 oss02 oss02-eth0
-      10.0.42.42              lake2-ib oss02-ib0
-      10.0.40.143             lake2-ipmi oss02-ipmi0
-      
-      # Node Entry for node: xeon01 (ID=15)
-      10.0.40.1               owl1 xeon01 xeon01-eth0
-      10.0.42.1               owl1-ib xeon01-ib0
-      10.0.40.101             owl1-ipmi xeon01-ipmi0
-      
-      # Node Entry for node: xeon02 (ID=16)
-      10.0.40.2               owl2 xeon02 xeon02-eth0
-      10.0.42.2               owl2-ib xeon02-ib0
-      10.0.40.102             owl2-ipmi xeon02-ipmi0
-      
-      # Node Entry for node: xeon03 (ID=17)
-      10.0.40.3               xeon03 xeon03-eth0
-      10.0.42.3               xeon03-ib0
-      10.0.40.103             xeon03-ipmi0
-      
-      # Node Entry for node: xeon04 (ID=18)
-      10.0.40.4               xeon04 xeon04-eth0
-      10.0.42.4               xeon04-ib0
-      10.0.40.104             xeon04-ipmi0
-      
-      # Node Entry for node: xeon05 (ID=19)
-      10.0.40.5               koro xeon05 xeon05-eth0
-      10.0.42.5               koro-ib xeon05-ib0
-      10.0.40.105             koro-ipmi xeon05-ipmi0
-      
-      # Node Entry for node: xeon06 (ID=20)
-      10.0.40.6               xeon06 xeon06-eth0
-      10.0.42.6               xeon06-ib0
-      10.0.40.106             xeon06-ipmi0
-      
-      # Node Entry for node: xeon07 (ID=21)
-      10.0.40.7               hut xeon07 xeon07-eth0
-      10.0.42.7               hut-ib xeon07-ib0
-      10.0.40.107             hut-ipmi xeon07-ipmi0
-      
-      # Node Entry for node: xeon08 (ID=22)
-      10.0.40.8               eudy xeon08 xeon08-eth0
-      10.0.42.8               eudy-ib xeon08-ib0
-      10.0.40.108             eudy-ipmi xeon08-ipmi0
-    '';
-  };
-}

m/common/ssf.nix

@@ -0,0 +1,10 @@
+{
+  # Provides the base system for a xeon node in the SSF rack.
+  imports = [
+    ./xeon.nix
+    ./ssf/fs.nix
+    ./ssf/hosts.nix
+    ./ssf/net.nix
+    ./ssf/ssh.nix
+  ];
+}

m/common/ssf/fs.nix

@@ -0,0 +1,8 @@
+{
+  # Mount the home via NFS
+  fileSystems."/home" = {
+    device = "10.0.40.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+}

m/common/ssf/hosts.nix

@@ -0,0 +1,23 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Login
+    "10.0.40.30" = [ "apex" ];
+
+    # Storage
+    "10.0.40.40" = [ "bay" ];   "10.0.42.40" = [ "bay-ib" ];    "10.0.40.141" = [ "bay-ipmi" ];
+    "10.0.40.41" = [ "oss01" ]; "10.0.42.41" = [ "oss01-ib0" ]; "10.0.40.142" = [ "oss01-ipmi" ];
+    "10.0.40.42" = [ "lake2" ]; "10.0.42.42" = [ "lake2-ib" ];  "10.0.40.143" = [ "lake2-ipmi" ];
+
+    # Xeon compute
+    "10.0.40.1" = [ "owl1" ];   "10.0.42.1" = [ "owl1-ib" ];   "10.0.40.101" = [ "owl1-ipmi" ];
+    "10.0.40.2" = [ "owl2" ];   "10.0.42.2" = [ "owl2-ib" ];   "10.0.40.102" = [ "owl2-ipmi" ];
+    "10.0.40.3" = [ "xeon03" ]; "10.0.42.3" = [ "xeon03-ib" ]; "10.0.40.103" = [ "xeon03-ipmi" ];
+    #"10.0.40.4" = [ "tent" ];   "10.0.42.4" = [ "tent-ib" ];   "10.0.40.104" = [ "tent-ipmi" ];
+    "10.0.40.5" = [ "koro" ];   "10.0.42.5" = [ "koro-ib" ];   "10.0.40.105" = [ "koro-ipmi" ];
+    "10.0.40.6" = [ "weasel" ]; "10.0.42.6" = [ "weasel-ib" ]; "10.0.40.106" = [ "weasel-ipmi" ];
+    "10.0.40.7" = [ "hut" ];    "10.0.42.7" = [ "hut-ib" ];    "10.0.40.107" = [ "hut-ipmi" ];
+    "10.0.40.8" = [ "eudy" ];   "10.0.42.8" = [ "eudy-ib" ];   "10.0.40.108" = [ "eudy-ipmi" ];
+  };
+}

m/common/ssf/net.nix

@@ -0,0 +1,23 @@
+{ pkgs, ... }:
+
+{
+  # Infiniband (IPoIB)
+  environment.systemPackages = [ pkgs.rdma-core ];
+  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
+
+  networking = {
+    defaultGateway = "10.0.40.30";
+    nameservers = ["8.8.8.8"];
+
+    firewall = {
+      extraCommands = ''
+        # Prevent ssfhead from contacting our slurmd daemon
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
+        # But accept traffic to slurm ports from any other node in the subnet
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
+        # We also need to open the srun port range
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+      '';
+    };
+  };
+}

m/common/ssf/ssh.nix

@@ -0,0 +1,16 @@
+{
+  # Use SSH tunnel to apex to reach internal hosts
+  programs.ssh.extraConfig = ''
+    Host tent
+      ProxyJump raccoon
+
+    # Access raccoon via the HTTP proxy
+    Host raccoon knights3.bsc.es
+      HostName knights3.bsc.es
+      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
+
+    # Make sure we can reach gitlab even if we don't have SSH access to raccoon
+    Host bscpm04.bsc.es gitlab-internal.bsc.es
+      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
+  '';
+}

m/common/users.nix

@@ -1,75 +0,0 @@
-{ pkgs, ... }:
-
-{
-  users = {
-    mutableUsers = false;
-    users = {
-      # Generate hashedPassword with `mkpasswd -m sha-512`
-
-      root.openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"
-      ];
-
-      rarias = {
-        uid = 1880;
-        isNormalUser = true;
-        home = "/home/Computational/rarias";
-        description = "Rodrigo Arias";
-        group = "Computational";
-        extraGroups = [ "wheel" ];
-        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
-        ];
-        shell = pkgs.zsh;
-      };
-
-      arocanon = {
-        uid = 1042;
-        isNormalUser = true;
-        home = "/home/Computational/arocanon";
-        description = "Aleix Roca";
-        group = "Computational";
-        extraGroups = [ "wheel" ];
-        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
-        ];
-      };
-
-      rpenacob = {
-        uid = 2761;
-        isNormalUser = true;
-        home = "/home/Computational/rpenacob";
-        description = "Raúl Peñacoba";
-        group = "Computational";
-        hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
-        ];
-      };
-
-      anavarro = {
-        uid = 1037;
-        isNormalUser = true;
-        home = "/home/Computational/anavarro";
-        description = "Antoni Navarro";
-        group = "Computational";
-        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
-        ];
-      };
-    };
-
-    groups = {
-      Computational = { gid = 564; };
-    };
-  };
-}

m/common/xeon.nix

@@ -0,0 +1,7 @@
+{
+  # Provides the base system for a xeon node, not necessarily in the SSF rack.
+  imports = [
+    ./base.nix
+    ./xeon/console.nix
+  ];
+}

m/common/xeon/console.nix

@@ -0,0 +1,14 @@
+{
+  # Restart the serial console
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+}

m/eudy/configuration.nix

@@ -2,14 +2,14 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
-    ./slurm.nix
+    ../module/debuginfod.nix
   ];
 
   # Select this using the ID to avoid mismatches

m/eudy/kernel/kernel.nix

@@ -21,9 +21,9 @@ let
   #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
   #};
 
-  kernel = nixos-fcsv3;
+  kernel = nixos-fcs;
 
-  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -40,35 +40,13 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  });
+  }));
 
-  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
-  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
-  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
-
-  # always use fcs_sched_setaffinity
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "7d0bf285fca92badc8df3c9907a9ab30db4418aa";};
-  # remove need_check_cgroup
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "4cc4efaab5e4a0bfa3089e935215b981c1922919";};
-  # merge again fcs_wake and fcs_wait
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "40c6f72f4ae54b0b636b193ac0648fb5730c810d";};
-  # start from scratch, this is the working version with split fcs_wake and fcs_wait
-  nixos-debug = nixos-fcs-kernel {gitCommit = "c9a39d6a4ca83845b4e71fcc268fb0a76aff1bdf"; branch = "fcs-test"; };
-
-  nixos-fcsv1-lockstat = nixos-fcs-kernel {
-    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
     lockStat = true;
   };
-  nixos-fcsv2-lockstat = nixos-fcs-kernel {
-    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
     lockStat = true;
     preempt = true;
   };

m/eudy/slurm.nix

@@ -1,7 +0,0 @@
-{ lib, ... }:
-
-{
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-}

m/fox/configuration.nix

@@ -0,0 +1,127 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+    ../common/xeon/console.nix
+    ../module/amd-uprof.nix
+    ../module/emulation.nix
+    ../module/nvidia.nix
+    ../module/slurm-client.nix
+    ./wireguard.nix
+  ];
+
+  # Don't turn off on August as UPC has different dates.
+  # Fox works fine on power cuts.
+  systemd.timers.august-shutdown.enable = false;
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.kernelModules = [ "kvm-amd" "amd_uncore" ];
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+  hardware.cpu.intel.updateMicrocode = lib.mkForce false;
+
+  # Use performance for benchmarks
+  powerManagement.cpuFreqGovernor = "performance";
+
+  services.amd-uprof.enable = true;
+
+  # Disable NUMA balancing
+  boot.kernel.sysctl."kernel.numa_balancing" = 0;
+
+  # Expose kernel addresses
+  boot.kernel.sysctl."kernel.kptr_restrict" = 0;
+
+  # Disable NMI watchdog to save one hw counter (for AMD uProf)
+  boot.kernel.sysctl."kernel.nmi_watchdog" = 0;
+
+  services.openssh.settings.X11Forwarding = true;
+
+  services.fail2ban.enable = true;
+
+  # Use SSH tunnel to reach internal hosts
+  programs.ssh.extraConfig = ''
+    Host bscpm04.bsc.es gitlab-internal.bsc.es tent
+      ProxyJump raccoon
+    Host raccoon
+      ProxyJump apex
+      HostName 127.0.0.1
+      Port 22022
+  '';
+
+  networking = {
+    timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
+    hostName = "fox";
+    # UPC network (may change over time, use DHCP)
+    # Public IP configuration:
+    # - Hostname: fox.ac.upc.edu
+    # - IP: 147.83.30.141
+    # - Gateway: 147.83.30.130
+    # - NetMask: 255.255.255.192
+    # Private IP configuration for BMC:
+    # - Hostname: fox-ipmi.ac.upc.edu
+    # - IP: 147.83.35.27
+    # - Gateway: 147.83.35.2
+    # - NetMask: 255.255.255.0
+    interfaces.enp1s0f0np0.useDHCP = true;
+  };
+
+  # Use hut for cache
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  # Recommended for new graphics cards
+  hardware.nvidia.open = true;
+
+  # Mount NVME disks
+  fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
+  fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
+
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
+  # Make a /nvme{0,1}/$USER directory for each user.
+  systemd.services.create-nvme-dirs = let
+    # Take only normal users in fox
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /nvme{0,1}/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-nvme-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+
+  # Only allow SSH connections from users who have a SLURM allocation
+  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
+  security.pam.services.sshd.rules.account.slurm = {
+    control = "required";
+    enable = true;
+    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
+    args = [ "log_level=debug5" ];
+    order = 999999; # Make it last one
+  };
+
+  # Disable systemd session (pam_systemd.so) as it will conflict with the
+  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
+  # into the slurmstepd task and then into the systemd session, which is not
+  # what we want, otherwise it will linger even if all jobs are gone.
+  security.pam.services.sshd.startSession = lib.mkForce false;
+}

m/fox/wireguard.nix

@@ -0,0 +1,46 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgFox.file = ../../secrets/wg-fox.age;
+
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      # Determines the IP address and subnet of the server's end of the tunnel interface.
+      ips = [ "10.106.0.1/24" ];
+
+      # The port that WireGuard listens to. Must be accessible by the client.
+      listenPort = 666;
+
+      # Path to the private key file.
+      privateKeyFile = config.age.secrets.wgFox.path;
+      # Public key: VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=
+
+      peers = [
+        # List of allowed peers.
+        { 
+          name = "Apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing.
+          allowedIPs = [ "10.106.0.30/32" ];
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.30" = [ "apex" ];
+  };
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to slurmd from apex (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.30/32 -d 10.106.0.1/32 --dport 6818 -j nixos-fw-accept
+    '';
+  };
+}

m/hut/blackbox.yml

@@ -0,0 +1,14 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+      valid_status_codes: []  # Defaults to 2xx
+      method: GET
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"

m/hut/configuration.nix

@@ -1,22 +1,42 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
 
 {
   imports = [
-    ../common/main.nix
+    ../common/ssf.nix
 
     ../module/ceph.nix
+    ../module/debuginfod.nix
+    ../module/emulation.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm-daemon.nix
     ./nix-serve.nix
+    ./public-inbox.nix
+    ./gitea.nix
+    ./msmtp.nix
+    ./postgresql.nix
+    ./nginx.nix
+    ./p.nix
     #./pxe.nix
   ];
 
-  boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
-
   # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
+
+  fileSystems = {
+    "/" = lib.mkForce {
+      device = "/dev/disk/by-label/nvme";
+      fsType = "ext4";
+      neededForBoot = true;
+      options = [ "noatime" ];
+    };
+
+    "/boot" = lib.mkForce {
+      device = "/dev/disk/by-label/nixos-boot";
+      fsType = "ext4";
+      neededForBoot = true;
+    };
+  };
 
   networking = {
     hostName = "hut";
@@ -28,5 +48,20 @@
       address = "10.0.42.7";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all proxy traffic from compute nodes but not the login
+        iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
+      '';
+      # Flush all rules and chains on stop so it won't break on start
+      extraStopCommands = ''
+        iptables -F
+        iptables -X
+      '';
+    };
   };
+
+  # Allow proxy to bind to the ethernet interface
+  services.openssh.settings.GatewayPorts = "clientspecified";
 }

m/hut/gitea.nix

@@ -0,0 +1,63 @@
+{ config, lib, ... }:
+{
+  age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
+
+  services.gitea = {
+    enable = true;
+    appName = "Gitea in the jungle";
+
+    settings = {
+      server = {
+        ROOT_URL = "https://jungle.bsc.es/git/";
+        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
+        LANDING_PAGE = "explore";
+      };
+      metrics.ENABLED = true;
+      service = {
+        REGISTER_MANUAL_CONFIRM = true;
+        ENABLE_NOTIFY_MAIL = true;
+      };
+      log.LEVEL = "Warn";
+
+      mailer = {
+        ENABLED       = true;
+        FROM          = "jungle-robot@bsc.es";
+        PROTOCOL      = "sendmail";
+        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
+        SENDMAIL_ARGS = "--";
+      };
+    };
+  };
+
+  services.gitea-actions-runner.instances = {
+    runrun = {
+      enable = true;
+      name = "runrun";
+      url = "https://jungle.bsc.es/git/";
+      tokenFile = config.age.secrets.giteaRunnerToken.path;
+      labels = [ "native:host" ];
+      settings.runner.capacity = 8;
+    };
+  };
+
+  systemd.services.gitea-runner-runrun = {
+    path = [ "/run/current-system/sw" ];
+    serviceConfig = {
+      # DynamicUser doesn't work well with SSH
+      DynamicUser = lib.mkForce false;
+      User = "gitea-runner";
+      Group = "gitea-runner";
+    };
+  };
+
+  users.users.gitea-runner = {
+    isSystemUser = true;
+    home = "/var/lib/gitea-runner";
+    description = "Gitea Runner";
+    group = "gitea-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitea-runner = {};
+}
+

m/hut/gitlab-runner.nix

@@ -1,54 +1,111 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
-  age.secrets.nosvToken.file = ../../secrets/nosv-token.age;
+  age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age;
+  age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age;
 
   services.gitlab-runner = {
     enable = true;
     settings.concurrent = 5;
-    services = {
-      ovni-shell = {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+    services = let
+      common-shell = {
         executor = "shell";
-        tagList = [ "nix" "xeon" ];
-        registrationFlags = [
-          # Using space doesn't work, and causes it to misread the next flag
-          "--locked='false'"
-        ];
         environmentVariables = {
           SHELL = "${pkgs.bash}/bin/bash";
         };
       };
-      ovni-docker = {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+      common-docker = {
+        executor = "docker";
         dockerImage = "debian:stable";
-        tagList = [ "docker" "xeon" ];
         registrationFlags = [
-          "--locked='false'"
           "--docker-network-mode host"
         ];
         environmentVariables = {
-          https_proxy = "http://localhost:23080";
-          http_proxy = "http://localhost:23080";
+          https_proxy = "http://hut:23080";
+          http_proxy = "http://hut:23080";
         };
       };
-      nosv-docker = {
-        registrationConfigFile = config.age.secrets.nosvToken.path;
-        dockerImage = "debian:stable";
-        tagList = [ "docker" "xeon" ];
-        registrationFlags = [
-          "--docker-network-mode host"
-          "--docker-cpus 56"
-        ];
+    in {
+      # For pm.bsc.es/gitlab
+      gitlab-pm-shell = common-shell // {
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path;
+      };
+      gitlab-pm-docker = common-docker // {
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path;
+      };
+
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
+        tagList = [ "docker" "hut" ];
         environmentVariables = {
-          https_proxy = "http://localhost:23080";
-          http_proxy = "http://localhost:23080";
+          # We cannot access the hut local interface from docker, so we connect
+          # to hut directly via the ethernet one.
+          https_proxy = "http://hut:23080";
+          http_proxy = "http://hut:23080";
+        };
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerExtraHosts = [
+          # Required to pass the proxy via hut
+          "hut:10.0.40.7"
+        ];
+        dockerDisableCache = true;
+        registrationFlags = [
+          # Increase build log length to 64 MiB
+          "--output-limit 65536"
+        ];
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat > "$HOME/.ssh/config" << EOF
+          Host bscpm04.bsc.es gitlab-internal.bsc.es
+            User git
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          EOF
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          # Required to load SSL certificate paths
+          . ${pkgs.cacert}/nix-support/setup-hook
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
         };
       };
     };
   };
 
+  # DOCKER* chains are useless, override at FORWARD and nixos-fw
+  networking.firewall.extraCommands = ''
+    # Don't forward any traffic from docker
+    iptables -I FORWARD 1 -p all -i docker0 -j nixos-fw-log-refuse
+
+    # Allow incoming traffic from docker to 23080
+    iptables -A nixos-fw -p tcp -i docker0 -d hut --dport 23080 -j ACCEPT
+  '';
+
   #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";

m/hut/gpfs-probe.nix

@@ -0,0 +1,31 @@
+{ pkgs, config, lib, ... }:
+let
+  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
+    ''
+      cp ${./gpfs-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  # Use a new user to handle the SSH keys
+  users.groups.ssh-robot = { };
+  users.users.ssh-robot = {
+    description = "SSH Robot";
+    isNormalUser = true;
+    home = "/var/lib/ssh-robot";
+  };
+
+  systemd.services.gpfs-probe = {
+    description = "Daemon to report GPFS latency via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
+}

m/hut/gpfs-probe.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+N=500
+
+t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
+
+if [ -z "$t" ]; then
+  t="5.00"
+fi
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP gpfs_touch_latency Time to create $N files.
+# TYPE gpfs_touch_latency gauge
+gpfs_touch_latency $t
+EOF

m/hut/ipmi.yml

@@ -1,13 +0,0 @@
-modules:
-        default:
-                collectors:
-                - bmc
-                - ipmi
-                - chassis
-
-        lan:
-                collectors:
-                - ipmi
-                - chassis
-                user: ""
-                pass: ""

m/hut/monitoring.nix

@@ -1,6 +1,22 @@
 { config, lib, ... }:
 
 {
+  imports = [
+    ../module/slurm-exporter.nix
+    ../module/meteocat-exporter.nix
+    ../module/upc-qaire-exporter.nix
+    ./gpfs-probe.nix
+    ../module/nix-daemon-exporter.nix
+  ];
+
+  age.secrets.grafanaJungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    owner = "grafana";
+    mode = "400";
+  };
+
+  age.secrets.ipmiYml.file = ../../secrets/ipmi.yml.age;
+
   services.grafana = {
     enable = true;
     settings = {
@@ -11,14 +27,29 @@
         http_port = 2342;
         http_addr = "127.0.0.1";
       };
+      smtp = {
+        enabled = true;
+        from_address = "jungle-robot@bsc.es";
+        user = "jungle-robot";
+        # Read the password from a file, which is only readable by grafana user
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
+        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
+        host = "mail.bsc.es:465";
+        startTLS_policy = "NoStartTLS";
+      };
       feature_toggles.publicDashboards = true;
+      "auth.anonymous".enabled = true;
+      log.level = "warn";
     };
   };
 
+  # Make grafana alerts also use the proxy
+  systemd.services.grafana.environment = config.networking.proxy.envVars;
+
   services.prometheus = {
     enable = true;
     port = 9001;
-    retentionTime = "1y";
+    retentionTime = "5y";
     listenAddress = "127.0.0.1";
   };
 
@@ -47,13 +78,13 @@
         enable = true;
         group = "root";
         user = "root";
-        configFile = ./ipmi.yml;
-        #extraFlags = [ "--log.level=debug" ];
+        configFile = config.age.secrets.ipmiYml.path;
+        # extraFlags = [ "--log.level=debug" ];
         listenAddress = "127.0.0.1";
       };
       node = {
         enable = true;
-        enabledCollectors = [ "systemd" ];
+        enabledCollectors = [ "systemd" "logind" ];
         port = 9002;
         listenAddress = "127.0.0.1";
       };
@@ -61,6 +92,11 @@
         enable = true;
         listenAddress = "127.0.0.1";
       };
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = ./blackbox.yml;
+      };
     };
 
     scrapeConfigs = [
@@ -73,6 +109,12 @@
             "127.0.0.1:9323"
             "127.0.0.1:9252"
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
+            "127.0.0.1:9341" # Slurm exporter
+            "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9999" # Nix-daemon custom exporter
+            "127.0.0.1:9929" # Meteocat custom exporter
+            "127.0.0.1:9928" # UPC Qaire custom exporter
+            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];
       }
@@ -86,6 +128,74 @@
           ];
         }];
       }
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = { module = [ "http_2xx" ]; };
+        static_configs = [{
+          targets = [
+            "https://www.google.com/robots.txt"
+            "https://pm.bsc.es/"
+            "https://pm.bsc.es/gitlab/"
+            "https://jungle.bsc.es/"
+            "https://gitlab.bsc.es/"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-icmp";
+        metrics_path = "/probe";
+        params = { module = [ "icmp" ]; };
+        static_configs = [{
+          targets = [
+            "1.1.1.1"
+            "8.8.8.8"
+            "ssfhead"
+            "anella-bsc.cesca.cat"
+            "upc-anella.cesca.cat"
+            "fox.ac.upc.edu"
+            "arenys5.ac.upc.edu"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "gitea";
+        static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
+      }
       {
         # Scrape the IPMI info of the hosts remotely via LAN
         job_name = "ipmi-lan";
@@ -107,7 +217,7 @@
             # Sets the "instance" label with the remote host we are querying
             source_labels = [ "__param_target" ];
             separator = ";";
-            regex = "(.*)";
+            regex = "(.*)-ipmi"; # Remove "-ipm̀i" at the end
             target_label = "instance";
             replacement = "\${1}";
             action = "replace";
@@ -138,6 +248,25 @@
           }
         ];
       }
+      {
+        job_name = "ipmi-raccoon";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9291" ]; }
+        ];
+        params = {
+          target = [ "84.88.51.142" ];
+          module = [ "raccoon" ];
+        };
+      }
+      {
+        job_name = "raccoon";
+        static_configs = [
+          {
+            targets = [ "127.0.0.1:19002" ]; # Node exporter
+          }
+        ];
+      }
     ];
   };
 }

m/hut/msmtp.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+{
+  age.secrets.jungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    group = "gitea";
+    mode = "440";
+  };
+
+  programs.msmtp = {
+    enable = true;
+    accounts = {
+      default = {
+        auth = true;
+        tls = true;
+        tls_starttls = false;
+        port = 465;
+        host = "mail.bsc.es";
+        user = "jungle-robot";
+        passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
+        from = "jungle-robot@bsc.es";
+      };
+    };
+  };
+}

m/hut/nginx.nix

@@ -0,0 +1,73 @@
+{ theFlake, pkgs, ... }:
+let
+  website = pkgs.stdenv.mkDerivation {
+    name = "jungle-web";
+    src = theFlake;
+    buildInputs = [ pkgs.hugo ];
+    buildPhase = ''
+      cd web
+      rm -rf public/
+      hugo
+    '';
+    installPhase = ''
+      cp -r public $out
+    '';
+    # Don't mess doc/
+    dontFixup = true;
+  };
+in
+{
+  networking.firewall.allowedTCPPorts = [ 80 ];
+  services.nginx = {
+    enable = true;
+    virtualHosts."jungle.bsc.es" = {
+      root = "${website}";
+      listen = [
+        {
+          addr = "0.0.0.0";
+          port = 80;
+        }
+      ];
+      extraConfig = ''
+        set_real_ip_from 127.0.0.1;
+        set_real_ip_from 84.88.52.107;
+        real_ip_recursive on;
+        real_ip_header X-Forwarded-For;
+
+        location /git {
+          rewrite ^/git$ / break;
+          rewrite ^/git/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:3000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /cache {
+          rewrite ^/cache/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:5000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /lists {
+          proxy_pass http://127.0.0.1:8081;
+          proxy_redirect http:// $scheme://;
+        }
+        location /grafana {
+          proxy_pass http://127.0.0.1:2342;
+          proxy_redirect http:// $scheme://;
+          proxy_set_header Host $host;
+          # Websockets
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+        }
+        location ~ ^/~(.+?)(/.*)?$ {
+          alias /ceph/home/$1/public_html$2;
+          index  index.html index.htm;
+          autoindex on;
+          absolute_redirect off;
+        }
+        location /p/ {
+          alias /ceph/p/;
+        }
+      '';
+    };
+  };
+}

m/hut/p.nix

@@ -0,0 +1,43 @@
+{ pkgs, lib, config, ... }:
+let
+  p = pkgs.writeShellScriptBin "p" ''
+    set -e
+    cd /ceph
+    pastedir="p/$USER"
+    mkdir -p "$pastedir"
+
+    ext="txt"
+
+    if [ -n "$1" ]; then
+      ext="$1"
+    fi
+
+    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
+
+    cat > "$out"
+    chmod go+r "$out"
+    echo "https://jungle.bsc.es/$out"
+  '';
+in
+{
+  environment.systemPackages = with pkgs; [ p ];
+
+  # Make sure we have a directory per user. We cannot use the nice
+  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
+  # may not be mounted when it runs.
+  systemd.services.create-paste-dirs = let
+    # Take only normal users in hut
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "remote-fs.target" ];
+    after = [ "remote-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+}

m/hut/postgresql.nix

@@ -0,0 +1,19 @@
+{ lib, ... }:
+
+{
+  services.postgresql = {
+    enable = true;
+    ensureDatabases = [ "perftestsdb" ];
+    ensureUsers = [
+      { name = "anavarro"; ensureClauses.superuser = true; }
+      { name = "rarias";   ensureClauses.superuser = true; }
+      { name = "grafana"; }
+    ];
+    authentication = ''
+      #type  database     DBuser    auth-method
+      local  perftestsdb  rarias    trust
+      local  perftestsdb  anavarro  trust
+      local  perftestsdb  grafana   trust
+    '';
+  };
+}

m/hut/public-inbox.css

@@ -0,0 +1,79 @@
+/*
+ * CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
+ * Dark color scheme using 216 web-safe colors, inspired
+ * somewhat by the default color scheme in mutt.
+ * It reduces eyestrain for me, and energy usage for all:
+ * https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
+ */
+
+* {
+	font-size: 14px;
+	font-family: monospace;
+}
+
+pre {
+	white-space: pre-wrap;
+	padding: 10px;
+	background: #f5f5f5;
+}
+
+hr {
+	margin: 30px 0;
+}
+
+body {
+	max-width: 120ex; /* 120 columns wide */
+	margin: 50px auto;
+}
+
+/*
+ * Underlined links add visual noise which make them hard-to-read.
+ * Use colors to make them stand out, instead.
+ */
+a:link {
+	color: #007;
+	text-decoration: none;
+}
+a:visited {
+	color:#504;
+}
+a:hover {
+	text-decoration: underline;
+}
+
+/* quoted text in emails gets a different color */
+*.q { color:gray }
+
+/*
+ * these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
+ * (cgit uses <div>, public-inbox uses <span>)
+ */
+*.add { color:darkgreen } /* diff post-image lines */
+*.del { color:darkred } /* diff pre-image lines */
+*.head { color:black } /* diff header (metainformation) */
+*.hunk { color:gray } /* diff hunk-header */
+
+/*
+ * highlight 3.x colors (tested 3.18) for displaying blobs.
+ * This doesn't use most of the colors available, as I find too
+ * many colors overwhelming, so the default is commented out.
+ */
+.hl.num { color:#f30 } /* number */
+.hl.esc { color:#f0f } /* escape character */
+.hl.str { color:#f30 } /* string */
+.hl.ppc { color:#f0f } /* preprocessor */
+.hl.pps { color:#f30 } /* preprocessor string */
+.hl.slc { color:#09f } /* single-line comment */
+.hl.com { color:#09f } /* multi-line comment */
+/* .hl.opt { color:#ccc } */ /* operator */
+/* .hl.ipl { color:#ccc } */ /* interpolation */
+
+/* keyword groups kw[a-z] */
+.hl.kwa { color:#ff0 }
+.hl.kwb { color:#0f0 }
+.hl.kwc { color:#ff0 }
+/* .hl.kwd { color:#ccc } */
+
+/* line-number (unused by public-inbox) */
+/* .hl.lin { color:#ccc } */
+

m/hut/public-inbox.nix

@@ -0,0 +1,47 @@
+{ lib, ... }:
+
+{
+  services.public-inbox = {
+    enable = true;
+    http = {
+      enable = true;
+      port = 8081;
+      mounts = [ "/lists" ];
+    };
+    settings.publicinbox = {
+      css = [ "${./public-inbox.css}" ];
+      wwwlisting = "all";
+    };
+    inboxes = {
+      bscpkgs = {
+        url = "https://jungle.bsc.es/lists/bscpkgs";
+        address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for bscpkgs";
+        listid = "~rodarima/bscpkgs.lists.sr.ht";
+      };
+      jungle = {
+        url = "https://jungle.bsc.es/lists/jungle";
+        address = [ "~rodarima/jungle@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for jungle";
+        listid = "~rodarima/jungle.lists.sr.ht";
+      };
+    };
+  };
+
+  # We need access to the network for the watch service, as we will fetch the
+  # emails directly from the IMAP server.
+  systemd.services.public-inbox-watch.serviceConfig = {
+    PrivateNetwork = lib.mkForce false;
+    RestrictAddressFamilies = lib.mkForce [ "AF_UNIX"  "AF_INET" "AF_INET6" ];
+    KillSignal = "SIGKILL"; # Avoid slow shutdown
+
+    # Required for chmod(..., 02750) on directories by git, from
+    # systemd.exec(8):
+    # > Note that this restricts marking of any type of file system object with
+    # > these bits, including both regular files and directories (where the SGID
+    # > is a different meaning than for files, see documentation).
+    RestrictSUIDSGID = lib.mkForce false;
+  };
+}

m/hut/slurm-daemon.nix

@@ -1,7 +0,0 @@
-{ ... }:
-
-{
-  services.slurm = {
-    server.enable = true;
-  };
-}

m/hut/targets.yml

@@ -1,15 +1,15 @@
 - targets:
-  - 10.0.40.101
-  - 10.0.40.102
-  - 10.0.40.103
-  - 10.0.40.104
-  - 10.0.40.105
-  - 10.0.40.106
-  - 10.0.40.107
-  - 10.0.40.108
+  - owl1-ipmi
+  - owl2-ipmi
+  - xeon03-ipmi
+  - xeon04-ipmi
+  - koro-ipmi
+  - weasel-ipmi
+  - hut-ipmi
+  - eudy-ipmi
   # Storage
-  - 10.0.40.141
-  - 10.0.40.142
-  - 10.0.40.143
+  - bay-ipmi
+  - oss01-ipmi
+  - lake2-ipmi
   labels:
     job: ipmi-lan

m/koro/configuration.nix

@@ -2,13 +2,11 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix
     ../eudy/users.nix
-    ../eudy/slurm.nix
-    ./users.nix
     ./kernel.nix
   ];
 

m/koro/kernel.nix

@@ -1,9 +1,29 @@
 { pkgs, lib, ... }:
 
 let
-  kernel = nixos-fcsv4;
+  #fcs-devel = pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = /mnt/data/kernel/fcs/kernel/src;
+  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
+  #};
 
-  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
+  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
+  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
+  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
+  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = builtins.fetchGit {
+  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+  #     rev = gitCommit;
+  #     ref = "fcs";
+  #   };
+  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
+  #};
+
+  kernel = nixos-fcs;
+
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -20,27 +40,13 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  });
+  }));
 
-  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
-  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
-  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
-  nixos-fcsv4 = nixos-fcs-kernel {gitCommit = "c94c3d946f33ac3e5782a02ee002cc1164c0cb4f";};
-
-  nixos-fcsv1-lockstat = nixos-fcs-kernel {
-    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
     lockStat = true;
   };
-  nixos-fcsv2-lockstat = nixos-fcs-kernel {
-    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
     lockStat = true;
     preempt = true;
   };
@@ -60,5 +66,5 @@ in {
   
   # enable memory overcommit, needed to build a taglibc system using nix after
   # increasing the openblas memory footprint
-  boot.kernel.sysctl."vm.overcommit_memory" = lib.mkForce 1;
+  boot.kernel.sysctl."vm.overcommit_memory" = 1;
 }

m/koro/users.nix

@@ -1,17 +0,0 @@
-{ ... }:
-
-{
-  users.users = {
-    vlopez = {
-      uid = 4334;
-      isNormalUser = true;
-      home = "/home/Computational/vlopez";
-      description = "Victor López";
-      group = "Computational";
-      hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
-      ];
-    };
-  };
-}

m/lake2/configuration.nix

@@ -2,20 +2,20 @@
 
 {
   imports = [
-    ../common/main.nix
-    ../common/monitoring.nix
+    ../common/ssf.nix
+    ../module/monitoring.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
 
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-
   services.ceph = {
     enable = true;
     global = {
@@ -49,6 +49,16 @@
       address = "10.0.42.42";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from bay
+        iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
   };
 
   # Missing service for volumes, see:

m/map.nix

@@ -0,0 +1,70 @@
+{
+  # In physical order from top to bottom (see note below)
+  ssf = {
+    # Switches for Ethernet and OmniPath
+    switch-C6-S1A-05 = { pos=42; size=1; model="Dell S3048-ON"; };
+    switch-opa = { pos=41; size=1; };
+
+    # SSF login
+    apex = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="rodrigo.arias@bsc.es"; };
+
+    # Storage
+    bay   = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };
+    lake1 = { pos=37; size=1; label="OSS01"; board="S2600WT2R"; sn="BQWL64850234"; contact="rodrigo.arias@bsc.es"; };
+    lake2 = { pos=36; size=1; label="OSS02"; board="S2600WT2R"; sn="BQWL64850266"; contact="rodrigo.arias@bsc.es"; };
+
+    # Compute xeon
+    owl1   = { pos=35; size=1; label="SSF-XEON01"; board="S2600WTTR"; sn="BQWL64954172"; contact="rodrigo.arias@bsc.es"; };
+    owl2   = { pos=34; size=1; label="SSF-XEON02"; board="S2600WTTR"; sn="BQWL64756560"; contact="rodrigo.arias@bsc.es"; };
+    xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; };
+    # Slot 34 empty
+    koro   = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; };
+    weasel = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
+    hut    = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; };
+    eudy   = { pos=28; size=1; label="SSF-XEON08"; board="S2600WTTR"; sn="BQWL64756586"; contact="aleix.rocanonell@bsc.es"; };
+
+    # 16 KNL nodes, 4 per chassis
+    knl01_04 = { pos=26; size=2; label="KNL01..KNL04"; board="HNS7200APX"; };
+    knl05_08 = { pos=24; size=2; label="KNL05..KNL18"; board="HNS7200APX"; };
+    knl09_12 = { pos=22; size=2; label="KNL09..KNL12"; board="HNS7200APX"; };
+    knl13_16 = { pos=20; size=2; label="KNL13..KNL16"; board="HNS7200APX"; };
+
+    # Slot 19 empty
+
+    # EPI (hw team, guessed order)
+    epi01 = { pos=18; size=1; contact="joan.cabre@bsc.es"; };
+    epi02 = { pos=17; size=1; contact="joan.cabre@bsc.es"; };
+    epi03 = { pos=16; size=1; contact="joan.cabre@bsc.es"; };
+    anon  = { pos=14; size=2; }; # Unlabeled machine. Operative
+
+    # These are old and decommissioned (off)
+    power8    = { pos=12; size=2; label="BSCPOWER8N3";   decommissioned=true; };
+    powern1   = { pos=8;  size=4; label="BSCPOWERN1";    decommissioned=true; };
+    gustafson = { pos=7;  size=1; label="gustafson";     decommissioned=true; };
+    odap01    = { pos=3;  size=4; label="ODAP01";        decommissioned=true; };
+    amhdal    = { pos=2;  size=1; label="AMHDAL";        decommissioned=true; }; # sic
+    moore     = { pos=1;  size=1; label="moore (earth)"; decommissioned=true; };
+  };
+
+  bsc2218 = {
+    raccoon = { board="W2600CR"; sn="QSIP22500829"; contact="rodrigo.arias@bsc.es"; };
+    tent    = { label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  upc = {
+    fox = { board="H13DSG-O-CPU"; sn="UM24CS600392"; prod="AS-4125GS-TNRT"; prod_sn="E508839X5103339"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  # NOTE: Position is specified in "U" units (44.45 mm) and starts at 1 from the
+  # bottom. Example:
+  #
+  #  |   ...  | - [pos+size] <--- Label in chassis
+  #  +--------+
+  #  |  node  | - [pos+1]
+  #  |   2U   | - [pos]
+  #  +------- +
+  #  |   ...  | - [pos-1]
+  #
+  # NOTE: The board and sn refers to the FRU information (Board Product and
+  # Board Serial) via `ipmitool fru print 0`.
+}

m/module/amd-uprof.nix

@@ -0,0 +1,49 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options = {
+    services.amd-uprof = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable AMD uProf.";
+      };
+    };
+  };
+
+  # Only setup amd-uprof if enabled
+  config = lib.mkIf config.services.amd-uprof.enable {
+
+    # First make sure that we add the module to the list of available modules
+    # in the kernel matching the same kernel version of this configuration.
+    boot.extraModulePackages = with config.boot.kernelPackages; [ amd-uprof-driver ];
+    boot.kernelModules = [ "AMDPowerProfiler" ];
+
+    # Make the userspace tools available in $PATH.
+    environment.systemPackages = with pkgs; [ amd-uprof ];
+
+    # The AMDPowerProfiler module doesn't create the /dev device nor it emits
+    # any uevents, so we cannot use udev rules to automatically create the
+    # device. Instead, we run a systemd unit that does it after loading the
+    # modules.
+    systemd.services.amd-uprof-device = {
+      description = "Create /dev/AMDPowerProfiler device";
+      after = [ "systemd-modules-load.service" ];
+      wantedBy = [ "multi-user.target" ];
+      unitConfig.ConditionPathExists = [
+          "/proc/AMDPowerProfiler/device"
+          "!/dev/AMDPowerProfiler"
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = pkgs.writeShellScript "add-amd-uprof-dev.sh" ''
+          mknod /dev/AMDPowerProfiler -m 666 c $(< /proc/AMDPowerProfiler/device) 0
+        '';
+        ExecStop = pkgs.writeShellScript "remove-amd-uprof-dev.sh" ''
+          rm -f /dev/AMDPowerProfiler
+        '';
+      };
+    };
+  };
+}

m/module/ceph.nix

@@ -3,7 +3,6 @@
 # Mounts the /ceph filesystem at boot
 {
   environment.systemPackages = with pkgs; [
-    ceph
     ceph-client
     fio # For benchmarks
   ];

m/module/debuginfod.nix

@@ -0,0 +1,3 @@
+{
+  services.nixseparatedebuginfod.enable = true;
+}

m/module/emulation.nix

@@ -0,0 +1,3 @@
+{
+  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
+}

m/module/hut-substituter.nix

@@ -0,0 +1,10 @@
+{ config, ... }:
+{
+  nix.settings =
+    # Don't add hut as a cache to itself
+    assert config.networking.hostName != "hut";
+    {
+      extra-substituters = [ "http://hut/cache" ];
+      extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+    };
+}

m/module/jungle-users.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+
+with lib;
+
+{
+  options = {
+    users.jungleUsers = mkOption {
+      type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
+      description = ''
+        Same as users.users but with the extra `hosts` attribute, which controls
+        access to the nodes by `networking.hostName`.
+      '';
+    };
+  };
+
+  config = let
+    allowedUser = host: userConf: builtins.elem host userConf.hosts;
+    filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
+    removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
+    currentHost = config.networking.hostName;
+  in {
+    users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
+  };
+}

m/module/meteocat-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-meteocat-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.meteocat-exporter}/bin/meteocat-exporter";
+    };
+  };
+}

m/module/nix-daemon-builds.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Locate nix daemon pid
+nd=$(pgrep -o nix-daemon)
+
+# Locate children of nix-daemon
+pids1=$(tr ' ' '\n' < "/proc/$nd/task/$nd/children")
+
+# For each children, locate 2nd level children
+pids2=$(echo "$pids1" | xargs -I @ /bin/sh -c 'cat /proc/@/task/*/children' | tr ' ' '\n')
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP nix_daemon_build Nix daemon derivation build state.
+# TYPE nix_daemon_build gauge
+EOF
+
+for pid in $pids2; do
+  name=$(cat /proc/$pid/environ 2>/dev/null | tr '\0' '\n' | rg "^name=(.+)" - --replace '$1' | tr -dc ' [:alnum:]_\-\.')
+  user=$(ps -o uname= -p "$pid")
+  if [ -n "$name" -a -n "$user" ]; then
+    printf 'nix_daemon_build{user="%s",name="%s"} 1\n' "$user" "$name"
+  fi
+done

m/module/nix-daemon-exporter.nix

@@ -0,0 +1,23 @@
+{ pkgs, config, lib, ... }:
+let
+  script = pkgs.runCommand "nix-daemon-exporter.sh" { }
+    ''
+      cp ${./nix-daemon-builds.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  systemd.services.nix-daemon-exporter = {
+    description = "Daemon to export nix-daemon metrics";
+    path = [ pkgs.procps pkgs.ripgrep ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9999,fork EXEC:${script}";
+      # Needed root to read the environment, potentially unsafe
+      User = "root";
+      Group = "root";
+    };
+  };
+}

m/module/nvidia.nix

@@ -0,0 +1,20 @@
+{ lib, config, pkgs, ... }:
+{
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.nvidia.open = lib.mkDefault (builtins.abort "hardware.nvidia.open not set");
+  hardware.graphics.enable = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  # enable support for derivations which require nvidia-gpu to be available
+  # > requiredSystemFeatures = [ "cuda" ];
+  programs.nix-required-mounts.enable = true;
+  programs.nix-required-mounts.presets.nvidia-gpu.enable = true;
+  # They forgot to add the symlink
+  programs.nix-required-mounts.allowedPatterns.nvidia-gpu.paths = [
+    config.systemd.tmpfiles.settings.graphics-driver."/run/opengl-driver"."L+".argument
+  ];
+
+  environment.systemPackages = [ pkgs.cudainfo ];
+}

m/module/p.nix

@@ -0,0 +1,68 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.p;
+in
+{
+  options = {
+    services.p = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the p service.";
+      };
+      path = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/p";
+        description = "Where to save the pasted files on disk.";
+      };
+      url = lib.mkOption {
+        type = lib.types.str;
+        default = "https://jungle.bsc.es/p";
+        description = "URL prefix for the printed file.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = let 
+      p = pkgs.writeShellScriptBin "p" ''
+        set -e
+        pastedir="${cfg.path}/$USER"
+        cd "$pastedir"
+
+        ext="txt"
+        if [ -n "$1" ]; then
+          ext="$1"
+        fi
+
+        out=$(mktemp "XXXXXXXX.$ext")
+        cat > "$out"
+        chmod go+r "$out"
+        echo "${cfg.url}/$USER/$out"
+      '';
+    in [ p ];
+
+    systemd.services.p = let
+      # Take only normal users
+      users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+      # Create a directory for each user
+      commands = lib.concatLists (lib.mapAttrsToList (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 ${cfg.path}/${user.name}"
+      ]) users);
+    in {
+      description = "P service setup";
+      requires = [ "network-online.target" ];
+      #wants = [ "remote-fs.target" ];
+      #after = [ "remote-fs.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = pkgs.writeShellScript "p-init.sh" (''
+
+          install -d -o root -g root -m 0755 ${cfg.path}
+
+        '' + (lib.concatLines commands));
+      };
+    };
+  };
+}

m/module/power-policy.nix

@@ -0,0 +1,33 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.power.policy;
+in
+{
+  options = {
+    power.policy = mkOption {
+      type = types.nullOr (types.enum [ "always-on" "previous" "always-off" ]);
+      default = null;
+      description = "Set power policy to use via IPMI.";
+    };
+  };
+
+  config = mkIf (cfg != null) {
+    systemd.services."power-policy" = {
+      description = "Set power policy to use via IPMI";
+      wantedBy = [ "multi-user.target" ];
+      unitConfig = {
+        StartLimitBurst = "10";
+        StartLimitIntervalSec = "10m";
+      };
+      serviceConfig = {
+        ExecStart = "${pkgs.ipmitool}/bin/ipmitool chassis policy ${cfg}";
+        Type = "oneshot";
+        Restart = "on-failure";
+        RestartSec = "5s";
+      };
+    };
+  };
+}

m/module/slurm-client.nix

@@ -0,0 +1,18 @@
+{ lib, ... }:
+
+{
+  imports = [
+    ./slurm-common.nix
+  ];
+
+  systemd.services.slurmd.serviceConfig = {
+    # Kill all processes in the control group on stop/restart. This will kill
+    # all the jobs running, so ensure that we only upgrade when the nodes are
+    # not in use. See:
+    # https://github.com/NixOS/nixpkgs/commit/ae93ed0f0d4e7be0a286d1fca86446318c0c6ffb
+    # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24
+    KillMode = lib.mkForce "control-group";
+  };
+
+  services.slurm.client.enable = true;
+}

m/module/slurm-common.nix

@@ -1,8 +1,7 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, ... }:
 
 let
-  suspendProgram = pkgs.writeScript "suspend.sh" ''
-    #!/usr/bin/env bash
+  suspendProgram = pkgs.writeShellScript "suspend.sh" ''
     exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
     set -x
     export "PATH=/run/current-system/sw/bin:$PATH"
@@ -14,8 +13,7 @@ let
     done
   '';
 
-  resumeProgram = pkgs.writeScript "resume.sh" ''
-    #!/usr/bin/env bash
+  resumeProgram = pkgs.writeShellScript "resume.sh" ''
     exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
     set -x
     export "PATH=/run/current-system/sw/bin:$PATH"
@@ -28,27 +26,17 @@ let
   '';
 
 in {
-  systemd.services.slurmd.serviceConfig = {
-    # Kill all processes in the control group on stop/restart. This will kill
-    # all the jobs running, so ensure that we only upgrade when the nodes are
-    # not in use. See:
-    # https://github.com/NixOS/nixpkgs/commit/ae93ed0f0d4e7be0a286d1fca86446318c0c6ffb
-    # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24
-    KillMode = lib.mkForce "control-group";
-  };
-
   services.slurm = {
-    client.enable = true;
-    controlMachine = "hut";
+    controlMachine = "apex";
     clusterName = "jungle";
     nodeName = [
       "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
-      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
+      "fox       Sockets=8 CoresPerSocket=24 ThreadsPerCore=1"
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP"
-      "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "fox Nodes=fox          Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -76,16 +64,44 @@ in {
       SuspendTimeout=60
       ResumeProgram=${resumeProgram}
       ResumeTimeout=300
-      SuspendExcNodes=hut
+      SuspendExcNodes=fox
 
       # Turn the nodes off after 1 hour of inactivity
       SuspendTime=3600
 
       # Reduce port range so we can allow only this range in the firewall
       SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
     '';
   };
 
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
   age.secrets.mungeKey = {
     file = ../../secrets/munge-key.age;
     owner = "munge";

m/module/slurm-exporter.nix

@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+# See also: https://github.com/NixOS/nixpkgs/pull/112010
+# And: https://github.com/NixOS/nixpkgs/pull/115839
+
+with lib;
+
+{
+  systemd.services."prometheus-slurm-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = ''
+        ${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
+      '';
+      Environment = [
+        "PATH=${pkgs.slurm}/bin"
+        # We need to specify the slurm config to be able to talk to the slurmd
+        # daemon.
+        "SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
+      ];
+    };
+  };
+}

m/module/slurm-hut-nix-store.nix

@@ -0,0 +1,19 @@
+{ ... }:
+
+{
+  # Mount the hut nix store via NFS
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  systemd.services.slurmd.serviceConfig = {
+    # When running a job, bind the hut store in /nix/store so the paths are
+    # available too.
+    # FIXME: This doesn't keep the programs in /run/current-system/sw/bin
+    # available in the store. Ideally they should be merged but the overlay FS
+    # doesn't work when the underlying directories change.
+    BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
+  };
+}

m/module/slurm-server.nix

@@ -0,0 +1,23 @@
+{ ... }:
+
+{
+  imports = [
+    ./slurm-common.nix
+  ];
+
+  services.slurm.server.enable = true;
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to controller from compute nodes
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from compute nodes for srun
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+
+      # Accept slurm connections to controller from fox (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from fox for srun (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 60000:61000 -j nixos-fw-accept
+    '';
+  };
+}

m/module/ssh-hut-extern.nix

@@ -0,0 +1,8 @@
+{
+  programs.ssh.extraConfig = ''
+    Host apex ssfhead
+      HostName ssflogin.bsc.es
+    Host hut
+      ProxyJump apex
+  '';
+}

m/module/upc-qaire-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-upc-qaire-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.upc-qaire-exporter}/bin/upc-qaire-exporter";
+    };
+  };
+}

m/module/vpn-dac.nix

@@ -0,0 +1,35 @@
+{config, ...}:
+{
+  age.secrets.vpn-dac-login.file = ../../secrets/vpn-dac-login.age;
+  age.secrets.vpn-dac-client-key.file = ../../secrets/vpn-dac-client-key.age;
+
+  services.openvpn.servers = {
+    # systemctl status openvpn-dac.service
+    dac = {
+      config = ''
+        client
+        dev tun
+        proto tcp
+        remote vpn.ac.upc.edu 1194
+        remote vpn.ac.upc.edu 80
+        resolv-retry infinite
+        nobind
+        persist-key
+        persist-tun
+        ca ${./vpn-dac/ca.crt}
+        cert ${./vpn-dac/client.crt}
+        # Only key needs to be secret
+        key ${config.age.secrets.vpn-dac-client-key.path}
+        remote-cert-tls server
+        comp-lzo
+        verb 3
+        auth-user-pass ${config.age.secrets.vpn-dac-login.path}
+        reneg-sec 0
+
+        # Only route fox-ipmi
+        pull-filter ignore "route "
+        route 147.83.35.27 255.255.255.255
+      '';
+    };
+  };
+}

m/module/vpn-dac/ca.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFUjCCBDqgAwIBAgIJAJH118PApk5hMA0GCSqGSIb3DQEBCwUAMIHLMQswCQYD
+VQQGEwJFUzESMBAGA1UECBMJQmFyY2Vsb25hMRIwEAYDVQQHEwlCYXJjZWxvbmEx
+LTArBgNVBAoTJFVuaXZlcnNpdGF0IFBvbGl0ZWNuaWNhIGRlIENhdGFsdW55YTEk
+MCIGA1UECxMbQXJxdWl0ZWN0dXJhIGRlIENvbXB1dGFkb3JzMRAwDgYDVQQDEwdM
+Q0FDIENBMQ0wCwYDVQQpEwRMQ0FDMR4wHAYJKoZIhvcNAQkBFg9sY2FjQGFjLnVw
+Yy5lZHUwHhcNMTYwMTEyMTI0NDIxWhcNNDYwMTEyMTI0NDIxWjCByzELMAkGA1UE
+BhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0w
+KwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAi
+BgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENB
+QyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMu
+ZWR1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0CteSeof7Xwi51kC
+F0nQ4E9iR5Lq7wtfRuVPn6JJcIxJJ6+F9gr4R/HIHTztW4XAzReE36DYfexupx3D
+6UgQIkMLlVyGqRbulNF+RnCx20GosF7Dm4RGBVvOxBP1PGjYq/A+XhaaDAFd0cOF
+LMNkzuYP7PF0bnBEaHnxmN8bPmuyDyas7fK9AAc3scyWT2jSBPbOVFvCJwPg8MH9
+V/h+hKwL/7hRt1MVfVv2qyIuKwTki8mUt0RcVbP7oJoRY5K1+R52phIz/GL/b4Fx
+L6MKXlQxLi8vzP4QZXgCMyV7oFNdU3VqCEXBA11YIRvsOZ4QS19otIk/ZWU5x+HH
+LAIJ7wIDAQABo4IBNTCCATEwHQYDVR0OBBYEFNyezX1cH1N4QR14ebBpljqmtE7q
+MIIBAAYDVR0jBIH4MIH1gBTcns19XB9TeEEdeHmwaZY6prRO6qGB0aSBzjCByzEL
+MAkGA1UEBhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vs
+b25hMS0wKwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVu
+eWExJDAiBgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UE
+AxMHTENBQyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0Bh
+Yy51cGMuZWR1ggkAkfXXw8CmTmEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsF
+AAOCAQEAUAmOvVXIQrR+aZVO0bOTeugKBHB75eTIZSIHIn2oDUvDbAP5GXIJ56A1
+6mZXxemSMY8/9k+pRcwJhfat3IgvAN159XSqf9kRv0NHgc3FWUI1Qv/BsAn0vJO/
+oK0dbmbbRWqt86qNrCN+cUfz5aovvxN73jFfnvfDQFBk/8enj9wXxYfokjjLPR1Q
++oTkH8dY68qf71oaUB9MndppPEPSz0K1S6h1XxvJoSu9MVSXOQHiq1cdZdxRazI3
+4f7q9sTCL+khwDAuZxAYzlEYxFFa/NN8PWU6xPw6V+t/aDhOiXUPJQB/O/K7mw3Z
+TQQx5NqM7B5jjak5fauR3/oRD8XXsA==
+-----END CERTIFICATE-----

m/module/vpn-dac/client.crt

@@ -0,0 +1,100 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 2 (0x2)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
+        Validity
+            Not Before: Jan 12 12:45:41 2016 GMT
+            Not After : Jan 12 12:45:41 2046 GMT
+        Subject: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=client/name=LCAC/emailAddress=lcac@ac.upc.edu
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:97:99:fa:7a:0e:4d:e2:1d:a5:b1:a8:14:18:64:
+                    c7:66:bf:de:99:1d:92:3b:86:82:4d:95:39:f7:a6:
+                    56:49:97:14:4f:e3:37:00:6c:f4:d0:1d:56:79:e7:
+                    19:b5:dd:36:15:8e:1d:57:7b:59:29:d2:11:bf:58:
+                    48:e0:f7:41:3d:16:64:8d:a2:0b:4a:ac:fa:c6:83:
+                    dc:10:2a:2c:d9:97:48:ee:11:2a:bc:4b:60:dd:b9:
+                    2e:8f:45:ca:87:0b:38:65:1c:f8:a2:1d:f9:50:aa:
+                    6e:60:f9:48:df:57:12:23:e1:e7:0c:81:5c:9f:c5:
+                    b2:e6:99:99:95:30:6d:57:36:06:8c:fd:fb:f9:4f:
+                    60:d2:3c:ba:ae:28:56:2f:da:58:5c:e8:c5:7b:ec:
+                    76:d9:28:6e:fb:8c:07:f9:d7:23:c3:72:76:3c:fa:
+                    dc:20:67:8f:cc:16:e0:91:07:d5:68:f9:20:4d:7d:
+                    5c:2d:02:04:16:76:52:f3:53:be:a3:dc:0d:d5:fb:
+                    6b:55:29:f3:52:35:c8:7d:99:d1:4a:94:be:b1:8e:
+                    fd:85:18:25:eb:41:e9:56:da:af:62:84:20:0a:00:
+                    17:94:92:94:91:6a:f8:54:37:17:ee:1e:bb:fb:93:
+                    71:91:d9:e4:e9:b8:3b:18:7d:6d:7d:4c:ce:58:55:
+                    f9:41
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Comment: 
+                Easy-RSA Generated Certificate
+            X509v3 Subject Key Identifier: 
+                1B:88:06:D5:33:1D:5C:48:46:B5:DE:78:89:36:96:91:3A:74:43:18
+            X509v3 Authority Key Identifier: 
+                keyid:DC:9E:CD:7D:5C:1F:53:78:41:1D:78:79:B0:69:96:3A:A6:B4:4E:EA
+                DirName:/C=ES/ST=Barcelona/L=Barcelona/O=Universitat Politecnica de Catalunya/OU=Arquitectura de Computadors/CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
+                serial:91:F5:D7:C3:C0:A6:4E:61
+
+            X509v3 Extended Key Usage: 
+                TLS Web Client Authentication
+            X509v3 Key Usage: 
+                Digital Signature
+            X509v3 Subject Alternative Name: 
+                DNS:client
+    Signature Algorithm: sha256WithRSAEncryption
+         42:e8:50:b2:e7:88:75:86:0b:bb:29:e3:aa:c6:0e:4c:e8:ea:
+         3d:0c:02:31:7f:3b:80:0c:3f:80:af:45:d6:62:27:a0:0e:e7:
+         26:09:12:97:95:f8:d9:9b:89:b5:ef:56:64:f1:de:82:74:e0:
+         31:0a:cc:90:0a:bd:50:b8:54:95:0a:ae:3b:40:df:76:b6:d1:
+         01:2e:f3:96:9f:52:d4:e9:14:6d:b7:14:9d:45:99:33:36:2a:
+         01:0b:15:1a:ed:55:dc:64:83:65:1a:06:42:d9:c7:dc:97:d4:
+         02:81:c2:58:2b:ea:e4:b7:ae:84:3a:e4:3f:f1:2e:fa:ec:f3:
+         40:5d:b8:6a:d5:5e:e1:e8:2f:e2:2f:48:a4:38:a1:4f:22:e3:
+         4f:66:94:aa:02:78:9a:2b:7a:5d:aa:aa:51:a5:e3:d0:91:e9:
+         1d:f9:08:ed:8b:51:c9:a6:af:46:85:b5:1c:ed:12:a1:28:33:
+         75:36:00:d8:5c:14:65:96:c0:28:7d:47:50:a4:89:5f:b0:72:
+         1a:4b:13:17:26:0f:f0:b8:65:3c:e9:96:36:f9:bf:90:59:33:
+         87:1f:01:03:25:f8:f0:3a:9b:33:02:d0:0a:43:b5:0a:cf:62:
+         a1:45:38:37:07:9d:9c:94:0b:31:c6:3c:34:b7:fc:5a:0c:e4:
+         bf:23:f6:7d
+-----BEGIN CERTIFICATE-----
+MIIFqjCCBJKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCByzELMAkGA1UEBhMCRVMx
+EjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0wKwYDVQQK
+EyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAiBgNVBAsT
+G0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENBQyBDQTEN
+MAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MB4X
+DTE2MDExMjEyNDU0MVoXDTQ2MDExMjEyNDU0MVowgcoxCzAJBgNVBAYTAkVTMRIw
+EAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNlbG9uYTEtMCsGA1UEChMk
+VW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1bnlhMSQwIgYDVQQLExtB
+cnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxDzANBgNVBAMTBmNsaWVudDENMAsG
+A1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAl5n6eg5N4h2lsagUGGTHZr/emR2S
+O4aCTZU596ZWSZcUT+M3AGz00B1WeecZtd02FY4dV3tZKdIRv1hI4PdBPRZkjaIL
+Sqz6xoPcECos2ZdI7hEqvEtg3bkuj0XKhws4ZRz4oh35UKpuYPlI31cSI+HnDIFc
+n8Wy5pmZlTBtVzYGjP37+U9g0jy6rihWL9pYXOjFe+x22Shu+4wH+dcjw3J2PPrc
+IGePzBbgkQfVaPkgTX1cLQIEFnZS81O+o9wN1ftrVSnzUjXIfZnRSpS+sY79hRgl
+60HpVtqvYoQgCgAXlJKUkWr4VDcX7h67+5Nxkdnk6bg7GH1tfUzOWFX5QQIDAQAB
+o4IBljCCAZIwCQYDVR0TBAIwADAtBglghkgBhvhCAQ0EIBYeRWFzeS1SU0EgR2Vu
+ZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQbiAbVMx1cSEa13niJNpaROnRD
+GDCCAQAGA1UdIwSB+DCB9YAU3J7NfVwfU3hBHXh5sGmWOqa0TuqhgdGkgc4wgcsx
+CzAJBgNVBAYTAkVTMRIwEAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNl
+bG9uYTEtMCsGA1UEChMkVW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1
+bnlhMSQwIgYDVQQLExtBcnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxEDAOBgNV
+BAMTB0xDQUMgQ0ExDTALBgNVBCkTBExDQUMxHjAcBgkqhkiG9w0BCQEWD2xjYWNA
+YWMudXBjLmVkdYIJAJH118PApk5hMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAsGA1Ud
+DwQEAwIHgDARBgNVHREECjAIggZjbGllbnQwDQYJKoZIhvcNAQELBQADggEBAELo
+ULLniHWGC7sp46rGDkzo6j0MAjF/O4AMP4CvRdZiJ6AO5yYJEpeV+NmbibXvVmTx
+3oJ04DEKzJAKvVC4VJUKrjtA33a20QEu85afUtTpFG23FJ1FmTM2KgELFRrtVdxk
+g2UaBkLZx9yX1AKBwlgr6uS3roQ65D/xLvrs80BduGrVXuHoL+IvSKQ4oU8i409m
+lKoCeJorel2qqlGl49CR6R35CO2LUcmmr0aFtRztEqEoM3U2ANhcFGWWwCh9R1Ck
+iV+wchpLExcmD/C4ZTzpljb5v5BZM4cfAQMl+PA6mzMC0ApDtQrPYqFFODcHnZyU
+CzHGPDS3/FoM5L8j9n0=
+-----END CERTIFICATE-----

m/owl1/configuration.nix

@@ -2,9 +2,13 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/ssf.nix
     ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -2,9 +2,13 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/ssf.nix
     ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/raccoon/configuration.nix

@@ -0,0 +1,94 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+    ../module/nvidia.nix
+    ../eudy/kernel/perf.nix
+  ];
+
+  # Don't install Grub on the disk yet
+  boot.loader.grub.device = "nodev";
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS1,115200"
+  ];
+
+  networking = {
+    hostName = "raccoon";
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    defaultGateway = "84.88.51.129";
+    interfaces.eno0.ipv4.addresses = [ {
+      address = "84.88.51.152";
+      prefixLength = 25;
+    } ];
+    interfaces.enp5s0f1.ipv4.addresses = [ {
+      address = "10.0.44.1";
+      prefixLength = 24;
+    } ];
+    nat = {
+      enable = true;
+      internalInterfaces = [ "enp5s0f1" ];
+      externalInterface = "eno0";
+    };
+    hosts = {
+      "10.0.44.4" = [ "tent" ];
+      "84.88.53.236" = [ "apex" ];
+    };
+  };
+
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  # Enable performance governor
+  powerManagement.cpuFreqGovernor = "performance";
+
+  hardware.nvidia.open = false; # Maxwell is older than Turing architecture
+
+  services.openssh.settings.X11Forwarding = true;
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
+  users.motd = ''
+    ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⡎⢰⣧⠀⠀⠀⠁⠈⠛⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⣴⡦⠶⠟⠓⠚⠻⡄⠀
+    ⠀⠀⠀⠀⠀⠀⣧⠀⣱⣀⣰⣧⠀⢀⠀⣘⣿⣿⣦⣶⣄⣠⡀⠀⠀⣀⣀⣤⣴⣄⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠿⠏⠁⠀⣀⣠⣶⣿⡶⣿⠀
+    ⠀⠀⠀⠀⠀⠀⣹⣆⠘⣿⣿⣿⣇⢸⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣶⣶⣦⡀⣀⣤⣠⣤⡾⠋⠀⢀⣤⣶⣿⣿⣿⣿⣿⣿⣿⡀
+    ⠀⠀⠀⠀⠀⠀⠘⢿⡄⢼⣿⣿⣿⣿⣿⡟⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣵⣾⡾⠙⣋⣩⣽⣿⣿⣿⣿⢋⡼⠁
+    ⠀⠀⠀⠀⠀⠀⠀⠈⢻⣄⠸⢿⣿⣿⠿⠷⠀⠈⠀⣭⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣾⣿⣿⣿⣿⣿⣿⠇⡼⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⣯⡀⠀⢼⡿⠀⠀⠀⢼⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣿⡿⣿⣿⣿⠿⣿⣯⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢋⡼⠁⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢻⡏⠠⣦⠁⠀⠀⠀⠀⠀⠟⠛⠛⣿⣿⣿⣿⣿⠿⠁⠀⠁⢿⠙⠁⠀⠛⠹⣿⣏⣾⣿⣿⣿⣿⣿⣿⣿⣿⠿⠃⣹⠁⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣘⣧⠀⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⡿⡿⠀⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀⢹⣿⠿⢿⣿⣿⣿⣿⣿⠋⢀⡤⠛⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⡯⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⢸⣿⣿⣿⠛⠉⠀⣰⠷⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠇⠀⠀⠀⠀⠀⢀⣿⡇⠀⠀⢻⣿⣿⠁⠀⠀⢠⣾⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠟⢿⣿⣄⡀⢸⣿⡀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⠀⠀⠀⢰⣿⣿⡛⣿⣿⡄⢠⡺⠿⡍⠁⢀⣤⣿⣿⣿⠿⣷⣮⣉⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⠀⠀⠈⣧⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⠉⠃⠀⣴⣿⣟⠻⣿⣿⣿⡇⢸⣿⣶⠀⢀⣾⣿⣿⣟⠿⣷⣾⣿⣿⣿⣿⣦⣤⣤⡤⠀⠀⠀⠀⠀⠁⠀⠀⠀⣼⠗⠀⠀⠀⠀
+    ⠀⠀⠐⢄⡀⠀⠀⠀⢘⡀⠀⢶⣾⣿⣿⣿⣿⡿⠋⠁⠈⠻⠉⠀⠚⠻⣿⣿⣿⣶⣾⣿⣿⣿⣿⣿⣿⣷⣬⣤⣶⣦⡀⣾⣶⣇⠀⠀⠈⢉⣷⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠈⠓⠶⢦⡽⠄⣈⣿⣿⣿⣿⣿⠏⠀⠀⠀⠀⠀⠀⠀⠀⠀⠹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡓⠙⣿⡟⠀⠀⠀⠈⠛⣷⣶⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⢀⣬⠆⢠⣍⣛⠻⣿⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣉⣀⡀⠀⠀⠈⠛⢿⣦⡀
+    ⠐⠒⠒⠶⠶⠶⢦⣬⣟⣥⣀⡉⠛⠻⠶⢁⣤⣾⣿⣿⣿⣷⡄⠀⠀⠀⠀⠀⢸⣿⣿⣿⣿⣿⣟⡛⠿⠭⠭⠭⠭⠭⠿⠿⠿⢿⣿⣟⠃⠀⠀⠀⠹⣟⠓
+    ⠀⣀⣠⠤⠤⢤⣤⣾⣤⡄⣉⣉⣙⣓⡂⣿⣿⣭⣹⣿⣿⣿⣿⡰⣂⣀⢀⠀⠻⣿⠛⠻⠟⠡⣶⣾⣿⣿⣿⣿⣿⣿⣿⡖⠒⠒⠒⠛⠷⢤⡀⢰⣴⣿⡆
+    ⠀⠀⠀⢀⣠⡴⠾⠟⠻⣟⡉⠉⠉⠉⢁⢿⣿⣿⣿⣿⣿⣿⡿⣱⣿⣭⡌⠤⠀⠀⠐⣶⣌⡻⣶⣭⡻⢿⣿⣿⣿⣿⣿⣯⣥⣤⣦⠀⠠⣴⣶⣶⣿⡟⢿
+    ⢀⠔⠊⠉⠀⠀⠀⠀⢸⣯⣤⠀⠀⠠⣼⣮⣟⣿⣿⣿⣻⣭⣾⣿⣿⣷⣶⣦⠶⣚⣾⣿⣿⣷⣜⣿⣿⣶⣝⢿⣿⣿⣿⣿⣷⣦⣄⣰⡄⠈⢿⣿⡿⣇⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠈⢡⢇⠀⠀⣠⣿⣿⣿⣯⣟⣛⣛⣛⣛⣛⣩⣭⣴⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣻⣿⣧⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠏⠀⢹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣍⣿⣿⣿⣿⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣾⡁⢈⣾⣿⡿⠛⣛⣿⣿⣿⣿ DO YOU BRING FEEDS? ⣿⣿⣿⣿⣿⣿⡏⠈⠙⠈⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⡿⠛⠉⣽⣿⣷⣾⡿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠷⠌⠛⠉⠀⠁⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠹⠋⠀⢻⣿⣿⣿⣿⠿⢿⣿⣿⣿⣿⣿⣿⠿⣿⣿⣿⣿⠿⠛⠋⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠁⠀⠀⠀⠀⠀⠈⠉⠉⠀⠀⠈⠋⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 
+  '';
+}

m/tent/blackbox.yml

@@ -0,0 +1,14 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip4"
+      follow_redirects: true
+      valid_status_codes: []  # Defaults to 2xx
+      method: GET
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"

m/tent/configuration.nix

@@ -0,0 +1,83 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+    ./monitoring.nix
+    ./nginx.nix
+    ./nix-serve.nix
+    ./gitlab-runner.nix
+    ./gitea.nix
+    ../hut/public-inbox.nix
+    ../hut/msmtp.nix
+    ../module/p.nix
+    ../module/vpn-dac.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d537675";
+
+  networking = {
+    hostName = "tent";
+    interfaces.eno1.ipv4.addresses = [
+      {
+        address = "10.0.44.4";
+        prefixLength = 24;
+      }
+    ];
+
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    search = [ "bsc.es" "ac.upc.edu" ];
+    defaultGateway = "10.0.44.1";
+    hosts = {
+      "84.88.53.236" = [ "apex" ];
+    };
+  };
+
+  services.p.enable = true;
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
+  boot.swraid = {
+    enable = true;
+    mdadmConf = ''
+      DEVICE partitions
+      ARRAY /dev/md0 metadata=1.2 UUID=496db1e2:056a92aa:a544543f:40db379d
+      MAILADDR root
+    '';
+  };
+
+  fileSystems."/vault" = {
+    device = "/dev/disk/by-label/vault";
+    fsType = "ext4";
+  };
+
+  # Make a /vault/$USER directory for each user.
+  systemd.services.create-vault-dirs = let
+    # Take only normal users in tent
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0711 /vault/home/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-vault-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+
+  # disable automatic garbage collector
+  nix.gc.automatic = lib.mkForce false;
+}

m/tent/gitea.nix

@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+{
+  services.gitea = {
+    enable = true;
+    appName = "Gitea in the jungle";
+
+    settings = {
+      server = {
+        ROOT_URL = "https://jungle.bsc.es/git/";
+        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
+        LANDING_PAGE = "explore";
+      };
+      metrics.ENABLED = true;
+      service = {
+        DISABLE_REGISTRATION = true;
+        REGISTER_MANUAL_CONFIRM = true;
+        ENABLE_NOTIFY_MAIL = true;
+      };
+      log.LEVEL = "Warn";
+
+      mailer = {
+        ENABLED       = true;
+        FROM          = "jungle-robot@bsc.es";
+        PROTOCOL      = "sendmail";
+        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
+        SENDMAIL_ARGS = "--";
+      };
+    };
+  };
+}

m/tent/gitlab-runner.nix

@@ -0,0 +1,93 @@
+{ pkgs, lib, config, ... }:
+
+{
+  age.secrets.tent-gitlab-runner-pm-shell.file = ../../secrets/tent-gitlab-runner-pm-shell-token.age;
+  age.secrets.tent-gitlab-runner-pm-docker.file = ../../secrets/tent-gitlab-runner-pm-docker-token.age;
+  age.secrets.tent-gitlab-runner-bsc-docker.file = ../../secrets/tent-gitlab-runner-bsc-docker-token.age;
+
+  services.gitlab-runner = let sec = config.age.secrets; in {
+    enable = true;
+    settings.concurrent = 5;
+    services = {
+      # For gitlab.pm.bsc.es
+      gitlab-pm-shell = {
+        executor = "shell";
+        environmentVariables = {
+          SHELL = "${pkgs.bash}/bin/bash";
+        };
+        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-shell.path;
+        preGetSourcesScript = pkgs.writeScript "setup" ''
+          echo "This is the preGetSources script running, brace for impact"
+          env
+        '';
+      };
+      gitlab-pm-docker = {
+        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-docker.path;
+        executor = "docker";
+        dockerImage = "debian:stable";
+      };
+
+      # For gitlab.bsc.es
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = sec.tent-gitlab-runner-bsc-docker.path;
+        tagList = [ "docker" "tent" "nix" ];
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerDisableCache = true;
+        registrationFlags = [
+          # Increase build log length to 64 MiB
+          "--output-limit 65536"
+        ];
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          # Required to load SSL certificate paths
+          . ${pkgs.cacert}/nix-support/setup-hook
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+        };
+      };
+    };
+  };
+
+  systemd.services.gitlab-runner.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "gitlab-runner";
+    Group = "gitlab-runner";
+    ExecStart = lib.mkForce
+      ''${pkgs.gitlab-runner}/bin/gitlab-runner run --config ''${HOME}/.gitlab-runner/config.toml --listen-address "127.0.0.1:9252" --working-directory ''${HOME}'';
+  };
+
+  users.users.gitlab-runner = {
+    uid = config.ids.uids.gitlab-runner;
+    home = "/var/lib/gitlab-runner";
+    description = "Gitlab Runner";
+    group = "gitlab-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;
+}

m/tent/monitoring.nix

@@ -0,0 +1,217 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ../module/meteocat-exporter.nix
+    ../module/upc-qaire-exporter.nix
+    ../module/nix-daemon-exporter.nix
+  ];
+
+  age.secrets.grafanaJungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    owner = "grafana";
+    mode = "400";
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        domain = "jungle.bsc.es";
+        root_url = "%(protocol)s://%(domain)s/grafana";
+        serve_from_sub_path = true;
+        http_port = 2342;
+        http_addr = "127.0.0.1";
+      };
+      smtp = {
+        enabled = true;
+        from_address = "jungle-robot@bsc.es";
+        user = "jungle-robot";
+        # Read the password from a file, which is only readable by grafana user
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
+        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
+        host = "mail.bsc.es:465";
+        startTLS_policy = "NoStartTLS";
+      };
+      feature_toggles.publicDashboards = true;
+      "auth.anonymous".enabled = true;
+      log.level = "warn";
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    retentionTime = "5y";
+    listenAddress = "127.0.0.1";
+  };
+
+  # We need access to the devices to monitor the disk space
+  systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+  systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+  # Credentials for IPMI exporter
+  age.secrets.ipmiYml = {
+    file = ../../secrets/ipmi.yml.age;
+    owner = "ipmi-exporter";
+  };
+
+  # Create an IPMI group and assign the ipmi0 device
+  users.groups.ipmi = {};
+  services.udev.extraRules = ''
+    SUBSYSTEM=="ipmi", KERNEL=="ipmi0", GROUP="ipmi", MODE="0660"
+  '';
+
+  # Add a new ipmi-exporter user that can read the ipmi0 device
+  users.users.ipmi-exporter = {
+    isSystemUser = true;
+    group = "ipmi";
+  };
+
+  # Disable dynamic user so we have the ipmi-exporter user available for the credentials
+  systemd.services.prometheus-ipmi-exporter.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    PrivateDevices = lib.mkForce false;
+    User = lib.mkForce "ipmi-exporter";
+    Group = lib.mkForce "ipmi";
+    RestrictNamespaces = lib.mkForce false;
+    # Fake uid to 0 so it shuts up
+    ExecStart = let
+      cfg = config.services.prometheus.exporters.ipmi;
+    in lib.mkForce (lib.concatStringsSep " " ([
+      "${pkgs.util-linux}/bin/unshare --map-user 0"
+      "${pkgs.prometheus-ipmi-exporter}/bin/ipmi_exporter"
+      "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
+      "--config.file ${lib.escapeShellArg cfg.configFile}"
+    ] ++ cfg.extraFlags));
+  };
+
+  services.prometheus = {
+    exporters = {
+      ipmi = {
+        enable = true;
+        configFile = config.age.secrets.ipmiYml.path;
+        #extraFlags = [ "--log.level=debug" ];
+        listenAddress = "127.0.0.1";
+      };
+      node = {
+        enable = true;
+        enabledCollectors = [ "logind" ];
+        port = 9002;
+        listenAddress = "127.0.0.1";
+      };
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = ./blackbox.yml;
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "local";
+        static_configs = [{
+          targets = [
+            "127.0.0.1:9002" # Node exporter
+            #"127.0.0.1:9115" # Blackbox exporter
+            "127.0.0.1:9290" # IPMI exporter for local node
+            "127.0.0.1:9928" # UPC Qaire custom exporter
+            "127.0.0.1:9929" # Meteocat custom exporter
+            "127.0.0.1:9999" # Nix-daemon custom exporter
+          ];
+        }];
+      }
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = { module = [ "http_2xx" ]; };
+        static_configs = [{
+          targets = [
+            "https://www.google.com/robots.txt"
+            "https://pm.bsc.es/"
+            "https://pm.bsc.es/gitlab/"
+            "https://jungle.bsc.es/"
+            "https://gitlab.bsc.es/"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-icmp";
+        metrics_path = "/probe";
+        params = { module = [ "icmp" ]; };
+        static_configs = [{
+          targets = [
+            "1.1.1.1"
+            "8.8.8.8"
+            "ssfhead"
+            "raccoon"
+            "anella-bsc.cesca.cat"
+            "upc-anella.cesca.cat"
+            "fox.ac.upc.edu"
+            "fox-ipmi.ac.upc.edu"
+            "arenys5.ac.upc.edu"
+            "arenys0-2.ac.upc.edu"
+            "epi01.bsc.es"
+            "axle.bsc.es"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "ipmi-raccoon";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9290" ]; }
+        ];
+        params = {
+          target = [ "raccoon-ipmi" ];
+          module = [ "raccoon" ];
+        };
+      }
+      {
+        job_name = "ipmi-fox";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9290" ]; }
+        ];
+        params = {
+          target = [ "fox-ipmi.ac.upc.edu" ];
+          module = [ "fox" ];
+        };
+      }
+    ];
+  };
+}

m/tent/nginx.nix

@@ -0,0 +1,73 @@
+{ theFlake, pkgs, ... }:
+let
+  website = pkgs.stdenv.mkDerivation {
+    name = "jungle-web";
+    src = theFlake;
+    buildInputs = [ pkgs.hugo ];
+    buildPhase = ''
+      cd web
+      rm -rf public/
+      hugo
+    '';
+    installPhase = ''
+      cp -r public $out
+    '';
+    # Don't mess doc/
+    dontFixup = true;
+  };
+in
+{
+  networking.firewall.allowedTCPPorts = [ 80 ];
+  services.nginx = {
+    enable = true;
+    virtualHosts."jungle.bsc.es" = {
+      root = "${website}";
+      listen = [
+        {
+          addr = "0.0.0.0";
+          port = 80;
+        }
+      ];
+      extraConfig = ''
+        set_real_ip_from 127.0.0.1;
+        set_real_ip_from 84.88.52.107;
+        real_ip_recursive on;
+        real_ip_header X-Forwarded-For;
+
+        location /git {
+          rewrite ^/git$ / break;
+          rewrite ^/git/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:3000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /cache {
+          rewrite ^/cache/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:5000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /lists {
+          proxy_pass http://127.0.0.1:8081;
+          proxy_redirect http:// $scheme://;
+        }
+        location /grafana {
+          proxy_pass http://127.0.0.1:2342;
+          proxy_redirect http:// $scheme://;
+          proxy_set_header Host $host;
+          # Websockets
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+        }
+        location ~ ^/~(.+?)(/.*)?$ {
+          alias /vault/home/$1/public_html$2;
+          index  index.html index.htm;
+          autoindex on;
+          absolute_redirect off;
+        }
+        location /p/ {
+          alias /var/lib/p/;
+        }
+      '';
+    };
+  };
+}

m/tent/nix-serve.nix

@@ -0,0 +1,16 @@
+{ config, ... }:
+
+{
+  age.secrets.nixServe.file = ../../secrets/nix-serve.age;
+
+  services.nix-serve = {
+    enable = true;
+    # Only listen locally, as we serve it via ssh
+    bindAddress = "127.0.0.1";
+    port = 5000;
+
+    secretKeyFile = config.age.secrets.nixServe.path;
+    # Public key:
+    # jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=
+  };
+}

m/weasel/configuration.nix

@@ -0,0 +1,32 @@
+{ lib, ... }:
+
+{
+  imports = [
+    ../common/ssf.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d5356ca";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  # Users with sudo access
+  users.groups.wheel.members = [ "abonerib" "anavarro" ];
+
+  # Run julia installed with juliaup using julia's own libraries:
+  # NIX_LD_LIBRARY_PATH=~/.julia/juliaup/${VERS}/lib/julia ~/.juliaup/bin/julia
+  programs.nix-ld.enable = true;
+
+  networking = {
+    hostName = "weasel";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.6";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.6";
+      prefixLength = 24;
+    } ];
+  };
+}

pkgs/amd-uprof/default.nix

@@ -0,0 +1,82 @@
+{ stdenv
+, lib
+, curl
+, cacert
+, runCommandLocal
+, autoPatchelfHook
+, elfutils
+, glib
+, libGL
+, ncurses5
+, xorg
+, zlib
+, libxkbcommon
+, freetype
+, fontconfig
+, libGLU
+, dbus
+, rocmPackages
+, libxcrypt-legacy
+}:
+
+let
+  version = "5.1.701";
+  tarball = "AMDuProf_Linux_x64_${version}.tar.bz2";
+
+  uprofSrc = runCommandLocal tarball {
+    nativeBuildInputs = [ curl ];
+    outputHash = "sha256-j9gxcBcIg6Zhc5FglUXf/VV9bKSo+PAKeootbN7ggYk=";
+    SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt";
+  } ''
+    curl \
+    -o $out \
+    'https://download.amd.com/developer/eula/uprof/uprof-5-1/${tarball}' \
+    -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0' \
+    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \
+    -H 'Accept-Language: en-US,en;q=0.5' \
+    -H 'Accept-Encoding: gzip, deflate, br, zstd' \
+    -H 'Referer: https://www.amd.com/' 2>&1 | tr '\r' '\n'
+  '';
+
+in
+  stdenv.mkDerivation {
+    pname = "AMD-uProf";
+    inherit version;
+    src = uprofSrc;
+    dontStrip = true;
+    phases = [ "installPhase" "fixupPhase" ];
+    nativeBuildInputs = [ autoPatchelfHook ];
+    buildInputs = [
+      stdenv.cc.cc.lib
+      ncurses5
+      elfutils
+      glib
+      libGL
+      libGLU
+      libxcrypt-legacy
+      xorg.libX11
+      xorg.libXext
+      xorg.libXi
+      xorg.libXmu
+      xorg.libxcb
+      xorg.xcbutilwm
+      xorg.xcbutilrenderutil
+      xorg.xcbutilkeysyms
+      xorg.xcbutilimage
+      fontconfig.lib
+      libxkbcommon
+      zlib
+      freetype
+      dbus
+      rocmPackages.rocprofiler
+    ];
+    installPhase = ''
+      set -x
+      mkdir -p $out
+      tar -x -v -C $out --strip-components=1 -f $src
+      rm $out/bin/AMDPowerProfilerDriverSource.tar.gz
+      patchelf --replace-needed libroctracer64.so.1 libroctracer64.so $out/bin/ProfileAgents/x64/libAMDGpuAgent.so
+      patchelf --add-needed libcrypt.so.1 --add-needed libstdc++.so.6 $out/bin/AMDuProfSys
+      set +x
+    '';
+  }

pkgs/amd-uprof/driver.nix

@@ -0,0 +1,35 @@
+{ stdenv
+, lib
+, amd-uprof
+, curl
+, cacert
+, kernel
+, runCommandLocal
+}:
+
+let
+  version = amd-uprof.version;
+  tarball = amd-uprof.src;
+in stdenv.mkDerivation {
+  pname = "AMDPowerProfilerDriver";
+  inherit version;
+  src = runCommandLocal "AMDPowerProfilerDriverSource.tar.gz" { } ''
+    set -x
+    tar -x -f ${tarball} AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz
+    mv AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz $out
+    set +x
+  '';
+  hardeningDisable = [ "pic" "format" ];
+  nativeBuildInputs = kernel.moduleBuildDependencies;
+  patches = [ ./makefile.patch ./hrtimer.patch ];
+  makeFlags = [
+    "KERNEL_VERSION=${kernel.modDirVersion}"
+    "KERNEL_DIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
+    "INSTALL_MOD_PATH=$(out)"
+  ];
+  meta = {
+    description = "AMD Power Profiler Driver";
+    homepage = "https://www.amd.com/es/developer/uprof.html";
+    platforms = lib.platforms.linux;
+  };
+}

pkgs/amd-uprof/hrtimer.patch

@@ -0,0 +1,31 @@
+--- a/src/PmcTimerConfig.c	2025-09-04 12:17:16.771707049 +0200
++++ b/src/PmcTimerConfig.c	2025-09-04 12:17:04.878515468 +0200
+@@ -99,7 +99,7 @@ static void PmcInitTimer(void* pInfo)
+ 
+     DRVPRINT("pTimerConfig(%p)", pTimerConfig);
+ 
+-    hrtimer_init(&pTimerConfig->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
++    hrtimer_setup(&pTimerConfig->m_hrTimer, PmcTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ }
+ 
+ int PmcSetupTimer(ClientContext* pClientCtx)
+@@ -157,7 +157,6 @@ int PmcSetupTimer(ClientContext* pClient
+                 {
+                     /* Interval in ms */
+                     pTimerConfig->m_time = ktime_set(interval / 1000, interval * 1000000);
+-                    pTimerConfig->m_hrTimer.function = PmcTimerCallback;
+ 
+                     DRVPRINT("retVal(%d) m_time(%lld)", retVal, (long long int) pTimerConfig->m_time);
+                 }
+--- a/src/PwrProfTimer.c	2025-09-04 12:18:08.750544327 +0200
++++ b/src/PwrProfTimer.c	2025-09-04 12:18:28.557863382 +0200
+@@ -573,8 +573,7 @@ void InitHrTimer(uint32 cpu)
+     pCoreClientData = &per_cpu(g_coreClientData, cpu);
+ 
+     // initialize HR timer
+-    hrtimer_init(&pCoreClientData->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+-    pCoreClientData->m_hrTimer.function = &HrTimerCallback;
++    hrtimer_setup(&pCoreClientData->m_hrTimer, &HrTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ 
+     return;
+ } // InitHrTimer

pkgs/amd-uprof/makefile.patch

@@ -0,0 +1,66 @@
+--- a/Makefile	2025-06-19 20:36:49.346693267 +0200
++++ b/Makefile	2025-06-19 20:42:29.778088660 +0200
+@@ -27,7 +27,7 @@ MODULE_VERSION=$(shell cat AMDPowerProfi
+ MODULE_NAME_KO=$(MODULE_NAME).ko
+ 
+ # check is module inserted
+-MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
++#MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
+ 
+ # check pcore dkms status
+ PCORE_DKMS_STATUS=$(shell dkms status | grep $(MODULE_NAME) | grep $(MODULE_VERSION))
+@@ -50,7 +50,7 @@ endif
+ # “-Wno-missing-attributes” is added for GCC version >= 9.0 and kernel version <= 5.00
+ G_VERSION=9
+ K_VERSION=5
+-KERNEL_MAJOR_VERSION=$(shell uname -r | cut -f1 -d.)
++KERNEL_MAJOR_VERSION=$(shell echo "$(KERNEL_VERSION)" | cut -f1 -d.)
+ GCCVERSION = $(shell gcc -dumpversion | cut -f1 -d.)
+ ifeq ($(G_VERSION),$(firstword $(sort $(GCCVERSION) $(G_VERSION))))
+ 	ifeq ($(K_VERSION),$(lastword $(sort $(KERNEL_MAJOR_VERSION) $(K_VERSION))))
+@@ -66,17 +66,7 @@ ${MODULE_NAME}-objs :=  src/PmcDataBuffe
+ 
+ # make
+ all:
+-	@chmod a+x ./AMDPPcert.sh
+-	@./AMDPPcert.sh 0 1; echo $$? > $(PWD)/sign_status;
+-	@SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 1 ]; then \
+-			exit 1; \
+-		fi
+-	@make -C /lib/modules/$(KERNEL_VERSION)/build M=$(PWD) $(MAKE_OPTS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" modules
+-	@SIGSTATUS3=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS3 -eq 0 ]; then \
+-			./AMDPPcert.sh 1 $(MODULE_NAME_KO); \
+-		fi
++	make -C $(KERNEL_DIR) M=$(PWD) $(MAKE_OPTS) CFLAGS_MODULE="$(EXTRA_CFLAGS)" modules
+ 
+ # make clean
+ clean:
+@@ -84,23 +74,9 @@ clean:
+ 
+ # make install
+ install:
+-	@mkdir -p /lib/modules/`uname -r`/kernel/drivers/extra
+-	@rm  -f /lib/modules/`uname -r`/kernel/drivers/extra/$(MODULE_NAME_KO)
+-	@cp $(MODULE_NAME_KO) /lib/modules/`uname -r`/kernel/drivers/extra/
+-	@depmod -a
+-	@if [ ! -z "$(MODPROBE_OUTPUT)" ]; then \
+-		echo "Uninstalling AMDPowerProfiler Linux kernel module.";\
+-		rmmod $(MODULE_NAME);\
+-	fi
+-	@modprobe $(MODULE_NAME) 2> $(PWD)/sign_status1; \
+-		cat $(PWD)/sign_status1 | grep "Key was rejected by service"; \
+-		echo $$? > $(PWD)/sign_status; SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 0 ]; then \
+-			echo "ERROR: Secure Boot enabled, correct key is not yet enrolled in BIOS key table"; \
+-			exit 1; \
+-		else \
+-			cat $(PWD)/sign_status1; \
+-		fi
++	mkdir -p $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++	cp -a $(MODULE_NAME_KO) $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++
+ # make dkms
+ dkms:
+ 	@chmod a+x ./AMDPPcert.sh