Fetch website from its own git repository

Reviewed-by: Aleix Boné <abonerib@bsc.es>

doc/trim.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Trims the jungle repository by moving the website to its own repository and
+# removing it from jungle. It also removes big pdf files and kernel
+# configurations so the jungle repository is small.
+
+set -e
+
+if [ -e oldjungle -o -e newjungle -o -e website ]; then
+  echo "remove oldjungle/, newjungle/ and website/ first"
+  exit 1
+fi
+
+# Clone the old jungle repo
+git clone gitea@tent:rarias/jungle.git oldjungle
+
+# First split the website into a new repository
+mkdir website && git -C website init -b master
+git-filter-repo \
+  --path web \
+  --subdirectory-filter web \
+  --source oldjungle \
+  --target website
+
+# Then remove the website, pdf files and big kernel configs
+mkdir newjungle && git -C newjungle init -b master
+git-filter-repo \
+  --invert-paths \
+  --path web \
+  --path-glob 'doc*.pdf' \
+  --path-glob '**/kernel/configs/lockdep' \
+  --path-glob '**/kernel/configs/defconfig' \
+  --source oldjungle \
+  --target newjungle
+
+set -x
+
+du -sh oldjungle newjungle website
+#  57M  oldjungle
+# 2,3M  newjungle
+# 6,4M  website
+
+du -sh --exclude=.git oldjungle newjungle website
+#  30M  oldjungle
+# 700K  newjungle
+# 3,5M  website

keys.nix

@@ -13,11 +13,11 @@ rec {
     tent    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
     apex    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
     weasel  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
+    raccoon = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNQttFvL0dNEyy7klIhLoK4xXOeM2/K9R7lPMTG3qvK raccoon";
   };
 
   hostGroup = with hosts; rec {
-    untrusted  = [ fox ];
+    compute    = [ owl1 owl2 fox raccoon ];
-    compute    = [ owl1 owl2 ];
     playground = [ eudy koro weasel ];
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
@@ -31,6 +31,7 @@ rec {
   admins = {
     "rarias@hut"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
     "rarias@tent" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIwlWSBTZi74WTz5xn6gBvTmCoVltmtIAeM3RMmkh4QZ rarias@tent";
+    "rarias@fox"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDSbw3REAKECV7E2c/e2XJITudJQWq2qDSe2N1JHqHZd rarias@fox";
     root          = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
   };
 }

m/apex/configuration.nix

@@ -5,7 +5,10 @@
     ../common/xeon.nix
     ../common/ssf/hosts.nix
     ../module/ceph.nix
+    ../module/hut-substituter.nix
+    ../module/slurm-server.nix
     ./nfs.nix
+    ./wireguard.nix
   ];
 
   # Don't install grub MBR for now
@@ -54,17 +57,6 @@
     };
   };
 
-  # Use SSH tunnel to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
-    Host raccoon
-      HostName knights3.bsc.es
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
-    Host tent
-      ProxyJump raccoon
-  '';
-
   networking.firewall = {
     extraCommands = ''
       # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our
@@ -74,10 +66,4 @@
       iptables -I nixos-fw 2 -p tcp -s 84.88.52.176 -j nixos-fw-refuse
     '';
   };
-
-  # Use tent for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
 }

m/apex/nfs.nix

@@ -8,6 +8,7 @@
     statdPort = 4000;
     exports = ''
       /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+      /home 10.106.0.0/24(rw,async,no_subtree_check,no_root_squash)
     '';
   };
   networking.firewall = {
@@ -27,6 +28,21 @@
       iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
       iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
       iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+
+      # Accept NFS traffic from wg0
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
     '';
   };
 }

m/apex/wireguard.nix

@@ -0,0 +1,42 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgApex.file = ../../secrets/wg-apex.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      ips = [ "10.106.0.30/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgApex.path;
+      # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=
+      peers = [
+        {
+          name = "fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.1/32" ];
+          endpoint = "fox.ac.upc.edu:666";
+          # Send keepalives every 25 seconds. Important to keep NAT tables alive.
+          persistentKeepalive = 25;
+        }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1" = [ "fox" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+}

m/bay/configuration.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ../common/ssf.nix
+    ../module/hut-substituter.nix
     ../module/monitoring.nix
   ];
 

m/common/base/net.nix

@@ -14,9 +14,10 @@
     nftables.enable = lib.mkForce false;
 
     hosts = {
-      "84.88.53.236" = [ "apex" "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
-      "84.88.51.152" = [ "raccoon" ];
       "84.88.51.142" = [ "raccoon-ipmi" ];
+      "192.168.11.12" = [ "bscpm04.bsc.es" ];
+      "192.168.11.15" = [ "gitlab-internal.bsc.es" ];
     };
   };
 }

m/common/base/users.nix

@@ -154,6 +154,32 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
         ];
       };
+
+      csiringo = {
+        uid = 9653;
+        isNormalUser = true;
+        home = "/home/Computational/csiringo";
+        description = "Cesare Siringo";
+        group = "Computational";
+        hosts = [ ];
+        hashedPassword = "$6$0IsZlju8jFukLlAw$VKm0FUXbS.mVmPm3rcJeizTNU4IM5Nmmy21BvzFL.cQwvlGwFI1YWRQm6gsbd4nbg47mPDvYkr/ar0SlgF6GO1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHA65zvvG50iuFEMf+guRwZB65jlGXfGLF4HO+THFaed csiringo@bsc.es"
+        ];
+      };
+
+      acinca = {
+        uid = 9654;
+        isNormalUser = true;
+        home = "/home/Computational/acinca";
+        description = "Arnau Cinca";
+        group = "Computational";
+        hosts = [ "apex" "hut" "fox" "owl1" "owl2" ];
+        hashedPassword = "$6$S6PUeRpdzYlidxzI$szyvWejQ4hEN76yBYhp1diVO5ew1FFg.cz4lKiXt2Idy4XdpifwrFTCIzLTs5dvYlR62m7ekA5MrhcVxR5F/q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFmMqKqPg4uocNOr3O41kLbZMOMJn3m2ZdN1JvTR96z3 bsccns@arnau-bsc"
+        ];
+      };
     };
 
     groups = {

m/common/ssf.nix

@@ -4,7 +4,7 @@
     ./xeon.nix
     ./ssf/fs.nix
     ./ssf/hosts.nix
+    ./ssf/hosts-remote.nix
     ./ssf/net.nix
-    ./ssf/ssh.nix
   ];
 }

m/common/ssf/hosts-remote.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Remote hosts visible from compute nodes
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+}

m/common/ssf/ssh.nix

@@ -1,16 +0,0 @@
-{
-  # Use SSH tunnel to apex to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host tent
-      ProxyJump raccoon
-
-    # Access raccoon via the HTTP proxy
-    Host raccoon knights3.bsc.es
-      HostName knights3.bsc.es
-      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
-
-    # Make sure we can reach gitlab even if we don't have SSH access to raccoon
-    Host bscpm04.bsc.es gitlab-internal.bsc.es
-      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
-  '';
-}

m/eudy/configuration.nix

@@ -9,6 +9,7 @@
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
+    ../module/hut-substituter.nix
     ../module/debuginfod.nix
   ];
 

m/fox/configuration.nix

@@ -4,8 +4,12 @@
   imports = [
     ../common/base.nix
     ../common/xeon/console.nix
+    ../module/amd-uprof.nix
     ../module/emulation.nix
     ../module/nvidia.nix
+    ../module/slurm-client.nix
+    ../module/hut-substituter.nix
+    ./wireguard.nix
   ];
 
   # Don't turn off on August as UPC has different dates.
@@ -19,7 +23,7 @@
   swapDevices = lib.mkForce [];
 
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
-  boot.kernelModules = [ "kvm-amd" "amd_uncore" ];
+  boot.kernelModules = [ "kvm-amd" "amd_uncore" "amd_hsmp" ];
 
   hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
   hardware.cpu.intel.updateMicrocode = lib.mkForce false;
@@ -27,14 +31,21 @@
   # Use performance for benchmarks
   powerManagement.cpuFreqGovernor = "performance";
 
+  services.amd-uprof.enable = true;
+
   # Disable NUMA balancing
   boot.kernel.sysctl."kernel.numa_balancing" = 0;
 
   # Expose kernel addresses
   boot.kernel.sysctl."kernel.kptr_restrict" = 0;
 
+  # Disable NMI watchdog to save one hw counter (for AMD uProf)
+  boot.kernel.sysctl."kernel.nmi_watchdog" = 0;
+
   services.openssh.settings.X11Forwarding = true;
 
+  services.fail2ban.enable = true;
+
   networking = {
     timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
     hostName = "fox";
@@ -52,12 +63,6 @@
     interfaces.enp1s0f0np0.useDHCP = true;
   };
 
-  # Use hut for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
-
   # Recommended for new graphics cards
   hardware.nvidia.open = true;
 
@@ -65,6 +70,13 @@
   fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
   fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
 
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
   # Make a /nvme{0,1}/$USER directory for each user.
   systemd.services.create-nvme-dirs = let
     # Take only normal users in fox
@@ -81,4 +93,20 @@
     wantedBy = [ "multi-user.target" ];
     serviceConfig.ExecStart = script;
   };
+
+  # Only allow SSH connections from users who have a SLURM allocation
+  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
+  security.pam.services.sshd.rules.account.slurm = {
+    control = "required";
+    enable = true;
+    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
+    args = [ "log_level=debug5" ];
+    order = 999999; # Make it last one
+  };
+
+  # Disable systemd session (pam_systemd.so) as it will conflict with the
+  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
+  # into the slurmstepd task and then into the systemd session, which is not
+  # what we want, otherwise it will linger even if all jobs are gone.
+  security.pam.services.sshd.startSession = lib.mkForce false;
 }

m/fox/wireguard.nix

@@ -0,0 +1,53 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgFox.file = ../../secrets/wg-fox.age;
+
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      # Determines the IP address and subnet of the server's end of the tunnel interface.
+      ips = [ "10.106.0.1/24" ];
+
+      # The port that WireGuard listens to. Must be accessible by the client.
+      listenPort = 666;
+
+      # Path to the private key file.
+      privateKeyFile = config.age.secrets.wgFox.path;
+      # Public key: VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=
+
+      peers = [
+        # List of allowed peers.
+        { 
+          name = "apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing.
+          allowedIPs = [ "10.106.0.30/32" ];
+        }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.30" = [ "apex" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to slurmd from apex (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.30/32 -d 10.106.0.1/32 --dport 6818 -j nixos-fw-accept
+    '';
+  };
+}

m/hut/configuration.nix

@@ -7,11 +7,9 @@
     ../module/ceph.nix
     ../module/debuginfod.nix
     ../module/emulation.nix
-    ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm-server.nix
     ./nix-serve.nix
     ./public-inbox.nix
     ./gitea.nix

m/hut/nginx.nix

@@ -2,10 +2,13 @@
 let
   website = pkgs.stdenv.mkDerivation {
     name = "jungle-web";
-    src = theFlake;
+    src = pkgs.fetchgit {
+      url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
+      rev = "739bf0175a7f05380fe7ad7023ff1d60db1710e1";
+      hash = "sha256-ea5DzhYTzZ9TmqD+x95rdNdLbxPnBluqlYH2NmBYmc4=";
+    };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''
-      cd web
       rm -rf public/
       hugo
     '';

m/hut/slurm-server.nix

@@ -1,7 +0,0 @@
-{ ... }:
-
-{
-  services.slurm = {
-    server.enable = true;
-  };
-}

m/lake2/configuration.nix

@@ -4,6 +4,7 @@
   imports = [
     ../common/ssf.nix
     ../module/monitoring.nix
+    ../module/hut-substituter.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";

m/module/amd-uprof.nix

@@ -0,0 +1,49 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options = {
+    services.amd-uprof = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable AMD uProf.";
+      };
+    };
+  };
+
+  # Only setup amd-uprof if enabled
+  config = lib.mkIf config.services.amd-uprof.enable {
+
+    # First make sure that we add the module to the list of available modules
+    # in the kernel matching the same kernel version of this configuration.
+    boot.extraModulePackages = with config.boot.kernelPackages; [ amd-uprof-driver ];
+    boot.kernelModules = [ "AMDPowerProfiler" ];
+
+    # Make the userspace tools available in $PATH.
+    environment.systemPackages = with pkgs; [ amd-uprof ];
+
+    # The AMDPowerProfiler module doesn't create the /dev device nor it emits
+    # any uevents, so we cannot use udev rules to automatically create the
+    # device. Instead, we run a systemd unit that does it after loading the
+    # modules.
+    systemd.services.amd-uprof-device = {
+      description = "Create /dev/AMDPowerProfiler device";
+      after = [ "systemd-modules-load.service" ];
+      wantedBy = [ "multi-user.target" ];
+      unitConfig.ConditionPathExists = [
+          "/proc/AMDPowerProfiler/device"
+          "!/dev/AMDPowerProfiler"
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = pkgs.writeShellScript "add-amd-uprof-dev.sh" ''
+          mknod /dev/AMDPowerProfiler -m 666 c $(< /proc/AMDPowerProfiler/device) 0
+        '';
+        ExecStop = pkgs.writeShellScript "remove-amd-uprof-dev.sh" ''
+          rm -f /dev/AMDPowerProfiler
+        '';
+      };
+    };
+  };
+}

m/module/hut-substituter.nix

@@ -6,5 +6,8 @@
     {
       extra-substituters = [ "http://hut/cache" ];
       extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+
+      # Set a low timeout in case hut is down
+      connect-timeout = 3; # seconds
     };
 }

m/module/slurm-client.nix

@@ -1,33 +1,10 @@
-{ config, pkgs, lib, ... }:
+{ lib, ... }:
 
-let
+{
-  suspendProgram = pkgs.writeScript "suspend.sh" ''
+  imports = [
-    #!/usr/bin/env bash
+    ./slurm-common.nix
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+  ];
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Shutting down host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
-    done
-  '';
 
-  resumeProgram = pkgs.writeScript "resume.sh" ''
-    #!/usr/bin/env bash
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Starting host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
-    done
-  '';
-
-in {
   systemd.services.slurmd.serviceConfig = {
     # Kill all processes in the control group on stop/restart. This will kill
     # all the jobs running, so ensure that we only upgrade when the nodes are
@@ -35,92 +12,13 @@ in {
     # https://github.com/NixOS/nixpkgs/commit/ae93ed0f0d4e7be0a286d1fca86446318c0c6ffb
     # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24
     KillMode = lib.mkForce "control-group";
+
+    # If slurmd fails to contact the control server it will fail, causing the
+    # node to remain out of service until manually restarted. Always try to
+    # restart it.
+    Restart = "always";
+    RestartSec = "30s";
   };
 
-  services.slurm = {
+  services.slurm.client.enable = true;
-    client.enable = true;
-    controlMachine = "hut";
-    clusterName = "jungle";
-    nodeName = [
-      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
-      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
-    ];
-
-    partitionName = [
-      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-    ];
-
-    # See slurm.conf(5) for more details about these options.
-    extraConfig = ''
-      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
-      # not with Intel MPI. For that use the compatibility shim libpmi.so
-      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
-      # library in SLURM (--mpi=pmix). See more details here:
-      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
-      MpiDefault=pmix
-
-      # When a node reboots return that node to the slurm queue as soon as it
-      # becomes operative again.
-      ReturnToService=2
-
-      # Track all processes by using a cgroup
-      ProctrackType=proctrack/cgroup
-
-      # Enable task/affinity to allow the jobs to run in a specified subset of
-      # the resources. Use the task/cgroup plugin to enable process containment.
-      TaskPlugin=task/affinity,task/cgroup
-
-      # Power off unused nodes until they are requested
-      SuspendProgram=${suspendProgram}
-      SuspendTimeout=60
-      ResumeProgram=${resumeProgram}
-      ResumeTimeout=300
-      SuspendExcNodes=hut
-
-      # Turn the nodes off after 1 hour of inactivity
-      SuspendTime=3600
-
-      # Reduce port range so we can allow only this range in the firewall
-      SrunPortRange=60000-61000
-
-      # Use cores as consumable resources. In SLURM terms, a core may have
-      # multiple hardware threads (or CPUs).
-      SelectType=select/cons_tres
-
-      # Ignore memory constraints and only use unused cores to share a node with
-      # other jobs.
-      SelectTypeParameters=CR_Core
-
-      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
-      # This sets up the "extern" step into which ssh-launched processes will be
-      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
-      # when a task runs (srun) so we can ssh early.
-      PrologFlags=Alloc,Contain,X11
-
-      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
-      # adopted by the external step, similar to tasks running in regular steps
-      # LaunchParameters=ulimit_pam_adopt
-      SlurmdDebug=debug5
-      #DebugFlags=Protocol,Cgroup
-    '';
-
-    extraCgroupConfig = ''
-      CgroupPlugin=cgroup/v2
-      #ConstrainCores=yes
-    '';
-  };
-
-  # Place the slurm config in /etc as this will be required by PAM
-  environment.etc.slurm.source = config.services.slurm.etcSlurm;
-
-  age.secrets.mungeKey = {
-    file = ../../secrets/munge-key.age;
-    owner = "munge";
-    group = "munge";
-  };
-
-  services.munge = {
-    enable = true;
-    password = config.age.secrets.mungeKey.path;
-  };
 }

m/module/slurm-common.nix

@@ -0,0 +1,115 @@
+{ config, pkgs, ... }:
+
+let
+  suspendProgram = pkgs.writeShellScript "suspend.sh" ''
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Shutting down host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
+    done
+  '';
+
+  resumeProgram = pkgs.writeShellScript "resume.sh" ''
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Starting host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
+    done
+  '';
+
+in {
+  services.slurm = {
+    controlMachine = "apex";
+    clusterName = "jungle";
+    nodeName = [
+      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
+      "fox       Sockets=8 CoresPerSocket=24 ThreadsPerCore=1"
+    ];
+
+    partitionName = [
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "fox Nodes=fox          Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+    ];
+
+    # See slurm.conf(5) for more details about these options.
+    extraConfig = ''
+      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
+      # not with Intel MPI. For that use the compatibility shim libpmi.so
+      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
+      # library in SLURM (--mpi=pmix). See more details here:
+      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
+      MpiDefault=pmix
+
+      # When a node reboots return that node to the slurm queue as soon as it
+      # becomes operative again.
+      ReturnToService=2
+
+      # Track all processes by using a cgroup
+      ProctrackType=proctrack/cgroup
+
+      # Enable task/affinity to allow the jobs to run in a specified subset of
+      # the resources. Use the task/cgroup plugin to enable process containment.
+      TaskPlugin=task/affinity,task/cgroup
+
+      # Power off unused nodes until they are requested
+      SuspendProgram=${suspendProgram}
+      SuspendTimeout=60
+      ResumeProgram=${resumeProgram}
+      ResumeTimeout=300
+      SuspendExcNodes=fox
+
+      # Turn the nodes off after 1 hour of inactivity
+      SuspendTime=3600
+
+      # Reduce port range so we can allow only this range in the firewall
+      SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
+    '';
+  };
+
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
+  age.secrets.mungeKey = {
+    file = ../../secrets/munge-key.age;
+    owner = "munge";
+    group = "munge";
+  };
+
+  services.munge = {
+    enable = true;
+    password = config.age.secrets.mungeKey.path;
+  };
+}

m/module/slurm-server.nix

@@ -0,0 +1,23 @@
+{ ... }:
+
+{
+  imports = [
+    ./slurm-common.nix
+  ];
+
+  services.slurm.server.enable = true;
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to controller from compute nodes
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from compute nodes for srun
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+
+      # Accept slurm connections to controller from fox (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from fox for srun (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 60000:61000 -j nixos-fw-accept
+    '';
+  };
+}

m/module/ssh-hut-extern.nix

@@ -1,8 +0,0 @@
-{
-  programs.ssh.extraConfig = ''
-    Host apex ssfhead
-      HostName ssflogin.bsc.es
-    Host hut
-      ProxyJump apex
-  '';
-}

m/raccoon/configuration.nix

@@ -3,11 +3,13 @@
 {
   imports = [
     ../common/base.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
     ../module/nvidia.nix
     ../eudy/kernel/perf.nix
+    ./wireguard.nix
+    ../module/hut-substituter.nix
   ];
 
   # Don't install Grub on the disk yet
@@ -39,12 +41,15 @@
     };
     hosts = {
       "10.0.44.4" = [ "tent" ];
+      "84.88.53.236" = [ "apex" ];
     };
   };
 
-  nix.settings = {
+  # Mount the NFS home
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+  fileSystems."/nfs/home" = {
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
   };
 
   # Enable performance governor

m/raccoon/wireguard.nix

@@ -0,0 +1,48 @@
+{ config, pkgs, ... }:
+
+{
+  networking.nat = {
+    enable = true;
+    enableIPv6 = false;
+    externalInterface = "eno0";
+    internalInterfaces = [ "wg0" ];
+  };
+
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgRaccoon.file = ../../secrets/wg-raccoon.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    wg0 = {
+      ips = [ "10.106.0.236/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgRaccoon.path;
+      # Public key: QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=
+      peers = [
+        {
+          name = "fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.1/32" ];
+          endpoint = "fox.ac.upc.edu:666";
+          persistentKeepalive = 25;
+        }
+        {
+          name = "apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          allowedIPs = [ "10.106.0.30/32" "10.0.40.0/24" ];
+          endpoint = "ssfhead.bsc.es:666";
+          persistentKeepalive = 25;
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1"  = [ "fox.wg" ];
+    "10.106.0.30" = [ "apex.wg" ];
+  };
+}

m/tent/configuration.nix

@@ -3,9 +3,9 @@
 {
   imports = [
     ../common/xeon.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
     ./monitoring.nix
     ./nginx.nix
     ./nix-serve.nix
@@ -15,6 +15,7 @@
     ../hut/msmtp.nix
     ../module/p.nix
     ../module/vpn-dac.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -33,6 +34,10 @@
     nameservers = [ "84.88.52.35" "84.88.52.36" ];
     search = [ "bsc.es" "ac.upc.edu" ];
     defaultGateway = "10.0.44.1";
+    hosts = {
+      "84.88.53.236" = [ "apex" ];
+      "10.0.44.1" = [ "raccoon" ];
+    };
   };
 
   services.p.enable = true;

m/tent/nginx.nix

@@ -2,10 +2,13 @@
 let
   website = pkgs.stdenv.mkDerivation {
     name = "jungle-web";
-    src = theFlake;
+    src = pkgs.fetchgit {
+      url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
+      rev = "739bf0175a7f05380fe7ad7023ff1d60db1710e1";
+      hash = "sha256-ea5DzhYTzZ9TmqD+x95rdNdLbxPnBluqlYH2NmBYmc4=";
+    };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''
-      cd web
       rm -rf public/
       hugo
     '';
@@ -67,6 +70,9 @@ in
         location /p/ {
           alias /var/lib/p/;
         }
+        location /pub/ {
+          alias /vault/pub/;
+        }
       '';
     };
   };

m/weasel/configuration.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ../common/ssf.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select this using the ID to avoid mismatches
@@ -14,6 +15,10 @@
   # Users with sudo access
   users.groups.wheel.members = [ "abonerib" "anavarro" ];
 
+  # Run julia installed with juliaup using julia's own libraries:
+  # NIX_LD_LIBRARY_PATH=~/.julia/juliaup/${VERS}/lib/julia ~/.juliaup/bin/julia
+  programs.nix-ld.enable = true;
+
   networking = {
     hostName = "weasel";
     interfaces.eno1.ipv4.addresses = [ {

pkgs/amd-uprof/default.nix

@@ -0,0 +1,89 @@
+{ stdenv
+, lib
+, curl
+, cacert
+, runCommandLocal
+, autoPatchelfHook
+, elfutils
+, glib
+, libGL
+, ncurses5
+, xorg
+, zlib
+, libxkbcommon
+, freetype
+, fontconfig
+, libGLU
+, dbus
+, rocmPackages
+, libxcrypt-legacy
+, numactl
+, radare2
+}:
+
+let
+  version = "5.1.701";
+  tarball = "AMDuProf_Linux_x64_${version}.tar.bz2";
+
+  # NOTE: Remember to update the radare2 patch below if AMDuProfPcm changes.
+  uprofSrc = runCommandLocal tarball {
+    nativeBuildInputs = [ curl ];
+    outputHash = "sha256-j9gxcBcIg6Zhc5FglUXf/VV9bKSo+PAKeootbN7ggYk=";
+    SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt";
+  } ''
+    curl \
+    -o $out \
+    'https://download.amd.com/developer/eula/uprof/uprof-5-1/${tarball}' \
+    -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0' \
+    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \
+    -H 'Accept-Language: en-US,en;q=0.5' \
+    -H 'Accept-Encoding: gzip, deflate, br, zstd' \
+    -H 'Referer: https://www.amd.com/' 2>&1 | tr '\r' '\n'
+  '';
+
+in
+  stdenv.mkDerivation {
+    pname = "AMD-uProf";
+    inherit version;
+    src = uprofSrc;
+    dontStrip = true;
+    phases = [ "installPhase" "fixupPhase" ];
+    nativeBuildInputs = [ autoPatchelfHook radare2 ];
+    buildInputs = [
+      stdenv.cc.cc.lib
+      ncurses5
+      elfutils
+      glib
+      libGL
+      libGLU
+      libxcrypt-legacy
+      xorg.libX11
+      xorg.libXext
+      xorg.libXi
+      xorg.libXmu
+      xorg.libxcb
+      xorg.xcbutilwm
+      xorg.xcbutilrenderutil
+      xorg.xcbutilkeysyms
+      xorg.xcbutilimage
+      fontconfig.lib
+      libxkbcommon
+      zlib
+      freetype
+      dbus
+      rocmPackages.rocprofiler
+      numactl
+    ];
+    installPhase = ''
+      set -x
+      mkdir -p $out
+      tar -x -v -C $out --strip-components=1 -f $src
+      rm $out/bin/AMDPowerProfilerDriverSource.tar.gz
+      patchelf --replace-needed libroctracer64.so.1 libroctracer64.so $out/bin/ProfileAgents/x64/libAMDGpuAgent.so
+      patchelf --add-needed libcrypt.so.1 --add-needed libstdc++.so.6 $out/bin/AMDuProfSys
+      echo "16334a51fcc48668307ad94e20482ca4  $out/bin/AMDuProfPcm" | md5sum -c -
+      radare2 -w -q -i ${./libnuma.r2} $out/bin/AMDuProfPcm
+      patchelf --add-needed libnuma.so $out/bin/AMDuProfPcm
+      set +x
+    '';
+  }

pkgs/amd-uprof/driver.nix

@@ -0,0 +1,33 @@
+{ stdenv
+, lib
+, amd-uprof
+, kernel
+, runCommandLocal
+}:
+
+let
+  version = amd-uprof.version;
+  tarball = amd-uprof.src;
+in stdenv.mkDerivation {
+  pname = "AMDPowerProfilerDriver";
+  inherit version;
+  src = runCommandLocal "AMDPowerProfilerDriverSource.tar.gz" { } ''
+    set -x
+    tar -x -f ${tarball} AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz
+    mv AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz $out
+    set +x
+  '';
+  hardeningDisable = [ "pic" "format" ];
+  nativeBuildInputs = kernel.moduleBuildDependencies;
+  patches = [ ./makefile.patch ./hrtimer.patch ];
+  makeFlags = [
+    "KERNEL_VERSION=${kernel.modDirVersion}"
+    "KERNEL_DIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
+    "INSTALL_MOD_PATH=$(out)"
+  ];
+  meta = {
+    description = "AMD Power Profiler Driver";
+    homepage = "https://www.amd.com/es/developer/uprof.html";
+    platforms = lib.platforms.linux;
+  };
+}

pkgs/amd-uprof/hrtimer.patch

@@ -0,0 +1,31 @@
+--- a/src/PmcTimerConfig.c	2025-09-04 12:17:16.771707049 +0200
++++ b/src/PmcTimerConfig.c	2025-09-04 12:17:04.878515468 +0200
+@@ -99,7 +99,7 @@ static void PmcInitTimer(void* pInfo)
+ 
+     DRVPRINT("pTimerConfig(%p)", pTimerConfig);
+ 
+-    hrtimer_init(&pTimerConfig->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
++    hrtimer_setup(&pTimerConfig->m_hrTimer, PmcTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ }
+ 
+ int PmcSetupTimer(ClientContext* pClientCtx)
+@@ -157,7 +157,6 @@ int PmcSetupTimer(ClientContext* pClient
+                 {
+                     /* Interval in ms */
+                     pTimerConfig->m_time = ktime_set(interval / 1000, interval * 1000000);
+-                    pTimerConfig->m_hrTimer.function = PmcTimerCallback;
+ 
+                     DRVPRINT("retVal(%d) m_time(%lld)", retVal, (long long int) pTimerConfig->m_time);
+                 }
+--- a/src/PwrProfTimer.c	2025-09-04 12:18:08.750544327 +0200
++++ b/src/PwrProfTimer.c	2025-09-04 12:18:28.557863382 +0200
+@@ -573,8 +573,7 @@ void InitHrTimer(uint32 cpu)
+     pCoreClientData = &per_cpu(g_coreClientData, cpu);
+ 
+     // initialize HR timer
+-    hrtimer_init(&pCoreClientData->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+-    pCoreClientData->m_hrTimer.function = &HrTimerCallback;
++    hrtimer_setup(&pCoreClientData->m_hrTimer, &HrTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ 
+     return;
+ } // InitHrTimer

pkgs/amd-uprof/libnuma.r2

@@ -0,0 +1,10 @@
+# Patch arguments to call sym std::string::find(char const*, unsigned long, unsigned long)
+# so it matches NixOS:
+#
+# Change OS name to NixOS
+wz NixOS @ 0x00550a43
+# And set the length to 5 characters
+wa mov ecx, 5 @0x00517930
+#
+# Then change the argument to dlopen() so it only uses libnuma.so
+wz libnuma.so @ 0x00562940

pkgs/amd-uprof/makefile.patch

@@ -0,0 +1,66 @@
+--- a/Makefile	2025-06-19 20:36:49.346693267 +0200
++++ b/Makefile	2025-06-19 20:42:29.778088660 +0200
+@@ -27,7 +27,7 @@ MODULE_VERSION=$(shell cat AMDPowerProfi
+ MODULE_NAME_KO=$(MODULE_NAME).ko
+ 
+ # check is module inserted
+-MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
++#MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
+ 
+ # check pcore dkms status
+ PCORE_DKMS_STATUS=$(shell dkms status | grep $(MODULE_NAME) | grep $(MODULE_VERSION))
+@@ -50,7 +50,7 @@ endif
+ # “-Wno-missing-attributes” is added for GCC version >= 9.0 and kernel version <= 5.00
+ G_VERSION=9
+ K_VERSION=5
+-KERNEL_MAJOR_VERSION=$(shell uname -r | cut -f1 -d.)
++KERNEL_MAJOR_VERSION=$(shell echo "$(KERNEL_VERSION)" | cut -f1 -d.)
+ GCCVERSION = $(shell gcc -dumpversion | cut -f1 -d.)
+ ifeq ($(G_VERSION),$(firstword $(sort $(GCCVERSION) $(G_VERSION))))
+ 	ifeq ($(K_VERSION),$(lastword $(sort $(KERNEL_MAJOR_VERSION) $(K_VERSION))))
+@@ -66,17 +66,7 @@ ${MODULE_NAME}-objs :=  src/PmcDataBuffe
+ 
+ # make
+ all:
+-	@chmod a+x ./AMDPPcert.sh
+-	@./AMDPPcert.sh 0 1; echo $$? > $(PWD)/sign_status;
+-	@SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 1 ]; then \
+-			exit 1; \
+-		fi
+-	@make -C /lib/modules/$(KERNEL_VERSION)/build M=$(PWD) $(MAKE_OPTS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" modules
+-	@SIGSTATUS3=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS3 -eq 0 ]; then \
+-			./AMDPPcert.sh 1 $(MODULE_NAME_KO); \
+-		fi
++	make -C $(KERNEL_DIR) M=$(PWD) $(MAKE_OPTS) CFLAGS_MODULE="$(EXTRA_CFLAGS)" modules
+ 
+ # make clean
+ clean:
+@@ -84,23 +74,9 @@ clean:
+ 
+ # make install
+ install:
+-	@mkdir -p /lib/modules/`uname -r`/kernel/drivers/extra
+-	@rm  -f /lib/modules/`uname -r`/kernel/drivers/extra/$(MODULE_NAME_KO)
+-	@cp $(MODULE_NAME_KO) /lib/modules/`uname -r`/kernel/drivers/extra/
+-	@depmod -a
+-	@if [ ! -z "$(MODPROBE_OUTPUT)" ]; then \
+-		echo "Uninstalling AMDPowerProfiler Linux kernel module.";\
+-		rmmod $(MODULE_NAME);\
+-	fi
+-	@modprobe $(MODULE_NAME) 2> $(PWD)/sign_status1; \
+-		cat $(PWD)/sign_status1 | grep "Key was rejected by service"; \
+-		echo $$? > $(PWD)/sign_status; SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 0 ]; then \
+-			echo "ERROR: Secure Boot enabled, correct key is not yet enrolled in BIOS key table"; \
+-			exit 1; \
+-		else \
+-			cat $(PWD)/sign_status1; \
+-		fi
++	mkdir -p $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++	cp -a $(MODULE_NAME_KO) $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++
+ # make dkms
+ dkms:
+ 	@chmod a+x ./AMDPPcert.sh

pkgs/overlay.nix

@@ -53,4 +53,15 @@ final: prev:
   meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { };
   upc-qaire-exporter = prev.callPackage ./upc-qaire-exporter/default.nix { };
   cudainfo = prev.callPackage ./cudainfo/default.nix { };
+
+  amd-uprof = prev.callPackage ./amd-uprof/default.nix { };
+
+  # FIXME: Extend this to all linuxPackages variants. Open problem, see:
+  # https://discourse.nixos.org/t/whats-the-right-way-to-make-a-custom-kernel-module-available/4636
+  linuxPackages = prev.linuxPackages.extend (_final: _prev: {
+    amd-uprof-driver = _prev.callPackage ./amd-uprof/driver.nix { };
+  });
+  linuxPackages_latest = prev.linuxPackages_latest.extend(_final: _prev: {
+    amd-uprof-driver = _prev.callPackage ./amd-uprof/driver.nix { };
+  });
 }

secrets/gitea-runner-token.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg d7+nvfAcdC3GjJxipXFrsfGGyP5jAY+gRWRV+4FVYAM
+-> ssh-ed25519 HY2yRg gKGxsjHfpiRDQ6Tuvcx7pjKgrVUGweotuplLYwCGvik
-CG7r0bRGgnUWcdfDnpe7HwZ3L/y7b5iuJuqvf15b3/Y
+DSz9j/stVyB1lXpVP+kg+H+RDgSftREGFFLQZClC3kI
--> ssh-ed25519 CAWG4Q X0vITOErz4wkR3VQYOcVlnrkHtwe+ytdZz1Hcrs4vVs
+-> ssh-ed25519 cK5kHw 17DpKekfNVy4V742QSd61r2w6iawtOJR7Ct3UflDXio
-6IWYOhXLQ+BnML9YfLLHJYEO2CZ/uEc9IBqhoWvjDHI
+hsqTEPCYjHKvndMWPl4GpG23CzjGgVrS+cLIymISJHU
--> ssh-ed25519 xA739A p5e/0AJtZ0+zbRvkB/usLuxusY8xXRx9Ksi/LQlcIHw
+-> ssh-ed25519 CAWG4Q oK01d4pbBqEZVsymSiKijPvJo714xsMSRMbzkssJKiw
-M4S/qlzT9POyJx4gY9lmycstUcdwG2cinN4OlV22zzo
+hs0tVFkqtIHXg9jtC2iDgCtefFcWvGJkXB+HJUcqXQs
--> ssh-ed25519 MSF3dg Ydl7uBWzBx6sAaxbzC3x8qiaU3ysGqV4rUFLpHCEV30
+-> ssh-ed25519 xA739A KxO+AawfLMERHwzt3YnZRwPFlCfGETma7fo8M+ZtsAY
-/1AUHBhCNOs9i7LJbmzwQDHsu+ybzYf6+coztKk5E3U
+eSn0+/rhLQxNKt5xKubKck8Nxun2Sh3eJqBU/hwgzZM
---- kYt15WxClpT7PXD1oFe9GqJU+OswjH7y9wIc8/GzZ7M
+-> ssh-ed25519 MSF3dg OyaZBLB2kO8fU139lXbbC404gT7IzIWk+BMhYzabBDg
-<EFBFBD><EFBFBD>h<>ߓ<><DF93><EFBFBD>`<60><><EFBFBD>V4F<34><46>_k)^<5E>m$uj:ѳ<><D1B3><17><><EFBFBD>}<7D>Z]$U]<12>u<EFBFBD> <20>0<EFBFBD><30><EFBFBD>v8<76>?<3F>X<EFBFBD>P<EFBFBD>g%d<>#<23>d9{rAi<41><69>
+/fiPFfBJcb+e40+fZbwCw7niF2hh+JxUPiKSiwUSOWg
+--- ycZyGX+Li+LsOuweF9OVPl8aoMaRgp/RdFbDrPszkUs
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD>YM<EFBFBD><EFBFBD>:E O<><4F>2<EFBFBD>r=<15>&4<><04>CQΣ<51><CEA3>hC<68><43><EFBFBD>cb<63>^Sy<53><79>%	<09><>x-vC`g<><15><><EFBFBD><EFBFBD>W^<5E><>wVG<0B><><EFBFBD>

secrets/secrets.nix

@@ -2,6 +2,9 @@ let
   keys = import ../keys.nix;
   adminsKeys = builtins.attrValues keys.admins;
   hut = [ keys.hosts.hut ] ++ adminsKeys;
+  fox = [ keys.hosts.fox ] ++ adminsKeys;
+  apex = [ keys.hosts.apex ] ++ adminsKeys;
+  raccoon = [ keys.hosts.raccoon ] ++ adminsKeys;
   mon = [ keys.hosts.hut keys.hosts.tent ] ++ adminsKeys;
   tent = [ keys.hosts.tent ] ++ adminsKeys;
   # Only expose ceph keys to safe nodes and admins
@@ -24,4 +27,8 @@ in
 
   "ceph-user.age".publicKeys = safe;
   "munge-key.age".publicKeys = safe;
+
+  "wg-fox.age".publicKeys = fox;
+  "wg-apex.age".publicKeys = apex;
+  "wg-raccoon.age".publicKeys = raccoon;
 }

secrets/tent-gitlab-runner-bsc-docker-token.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w HlQ4V8lBd3im5j8KHEuQZBTuztvPj1QoWdv6FL6qzGI
+-> ssh-ed25519 G5LX5w Zhbs+NM/SI49qQ0X8bBpWUWxYM0vUKCXNAnPpIE2NR0
-Jpt91X1UIIVFQt1X6Q//kALn+Cetp/LqBZZvTuhFthw
+CkBUmJ26EkwHztT8Pz0UGq2KZwN0Xz8iYQ9cEHL9OWQ
--> ssh-ed25519 CAWG4Q StnngJAcuAwUnTrXDR3nJ2KFN0jNdTqSz+/1TfmWkzA
+-> ssh-ed25519 cK5kHw 5KjUXJywRDp2A7l5ukTCS+WIAalxwP1f71ejGxwNrX4
-CR4AQ6fqaJVY1mdUIX1gzaZwRs1sU8F8hHztnkN8vN0
+JW8OLmfkULXo9AwYMGNyOgZ+nQ0MVc0PCM4kKPIo6V4
--> ssh-ed25519 xA739A xya5A5t63Owx+VrGgUfV/lIP8b/xV1cerMpuZBLaDVM
+-> ssh-ed25519 CAWG4Q cVjY3R0ZHAfokA4kWlu5vOl2Gs7mdqRgRk4WSUOXAjg
-w+pA583yUnFq2AvGBGzWbQIGQEY9WqW0CSLQ9v+SG0c
+IxEDvuximW99EqxmpW+Btpm0Zydmwg/u87bqnl26NYc
--> ssh-ed25519 MSF3dg aXkLxCyYdOwVopHHmpXEI6WlAIizKdJi4IO0KEdhS3s
+-> ssh-ed25519 xA739A hmuwZuxmJnuAjmU4X8yhPQ+hPWvN1G+ZS0pvD7fHamg
-WKXkTszZN66+QZdSDJ4D9q7xgYWMfliOLCubIF2Dqkc
+fnAPW6ZCrv5pSO4RQhhr8xz7ij7jAZJk0ApWluOXDng
---- uVWoU2lMkqQ/9Z0BqKRCeUpsKi8lwmHukT/FV8wYMbg
+-> ssh-ed25519 MSF3dg SSGLcWnum0Qo/0OnKDZVg9xAZMwGwVNYYmRJXxb4GU0
-<EFBFBD><EFBFBD>1G+<2B>6<EFBFBD><36>g[|x]2T<32>й<EFBFBD><D0B9><EFBFBD> <20>CKu)<29><><EFBFBD>]<5D><>8֓<38><D693><EFBFBD><EFBFBD>l<EFBFBD><6C>S<EFBFBD><53><EFBFBD>Q<EFBFBD><07><>x<EFBFBD><78><EFBFBD><EFBFBD>#7r<37>k{*<2A><>3ս~C<>b<EFBFBD><62><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڵ<EFBFBD>Np<1E><05>]J]h<>je+d%Е<>#<23>m<EFBFBD>?=6}<7D> 
+pdl6kATG7n2oMsoUboBfu+vDKurJcH1UvUa70rfMQkE
+--- a2ZQAeAQlO9DWnegIAq6NpI1Po6f38l+hitZvq+zIW8
+<EFBFBD>\ֺ"^<5E>DT<44>H<EFBFBD><48>3<EFBFBD><33><EFBFBD>_|.h<0E><><EFBFBD><EFBFBD><03>^<5E>n<14><0E><><EFBFBD><EFBFBD><1A>g<EFBFBD>S<EFBFBD>]_<><5F>?n<>z~2<>!<21>p7<70><37><<3C><14>ʨD?<3F>~<02>F<EFBFBD>$<24>`<60>q+<2B><><EFBFBD>SW<53>(+<2B><>P<EFBFBD>c<1E>u[<5B>m<EFBFBD>`O<>ܛ<EFBFBD>ϖT

secrets/tent-gitlab-runner-pm-docker-token.age

@@ -1,11 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w sg9SmahxBg35MDIxhrp4oHkaTaxsKoVQju2eNhCt0BM
+-> ssh-ed25519 G5LX5w VKM/Y6Wy0gmb2gc4Q00VzHQ4IAxfSyshuDoaAzlEkFM
-CZ64dEGqz2tbkG8KtimZvLUEMrQpVVBJP7Fu46WTMgc
+vf18uoEN5ZLJ4HcJg85epaseh1CRL9/ncXtU2HpH+QE
--> ssh-ed25519 CAWG4Q jzS1R14W1CWxdziMLG/yCGPLWSkiyE+9lqyCVe491ng
+-> ssh-ed25519 cK5kHw sMuG07kjlI6VjPjELOUPzkn+KT9Yq7BPf0zSATM2aGI
-acJo/nhKq3pSPoFEPaFLN1fzHHbEzstNoLtohWAHKiM
+/eODwL8KwyVgFjBK2MJlbqjN7mEvXCSsjq9D96szrng
--> ssh-ed25519 xA739A qeGJoLeSIQwLU2Yg+Gi2bikHJ3HscLfyo1msqL3JwHw
+-> ssh-ed25519 CAWG4Q t3/Ty7yCqC5x8KQY4VaHSQ9Q3epqMpXoBDKyKx9+VzE
-tTwaxRBKTl/SoyY/LnxR/j/5WvCNX5VeZLKi018YMrY
+JwgUsqMd+1jFZvFp9/SIoowbhSMVEkKp03T69+OHjho
--> ssh-ed25519 MSF3dg Wym7Uyf1XvH1H6mNDERkO8opkMiN0zzXm2PjXftEOWs
+-> ssh-ed25519 xA739A 0ohmKK427+4vupivrtjXp0dDK8wT4XUA9rWgcsCGKgA
-Uw8ZwwKIB5UqgVuoSLE2QajNDJZkH7/Y3Nsy+WFl7Xs
+msbeQyz3pL8RLtAeXX5tsfyHyOXxhfYpqaLEKnRxpPQ
---- 94hGVbYiCGZdMEJesCMLh7IZi+w5l/Kr1lZJHQgrc0o
+-> ssh-ed25519 MSF3dg H+6jAoP7/Dxp8C/7Bk1C4CT1hpkUhtbnTWWIxkO24Ec
-j5j磛<6A><04><>J<EFBFBD><4A><EFBFBD>a<EFBFBD>]<5D>a%dr<64><72>FDT<44><54>^<5E><>Q<EFBFBD>s/<2F>kwB<77>$<24><>$<24><>H<EFBFBD>'<27><><EFBFBD><EFBFBD><EFBFBD>w<14><?^|<7C><07>h$<24>ؗ<EFBFBD>GI<47>ĕsT2RU<52><55>*/O<>7<EFBFBD><37><EFBFBD>G<EFBFBD>pͪ<70>4<EFBFBD><34><EFBFBD>M9<4D>j<><06>
+SrMuUG93T5lUw3xINEen5EEKLXJizIGFhBO1fVroFHE
+--- tIPnH9cxTV3m3qzvZB97Egz+raWwZJ182BXXKDu8f+o
+<EFBFBD><EFBFBD>f#<23>,|<7C>Ey.v<>DL<44>Ӻ<05>J
PX<50><07><>`<60><><EFBFBD><EFBFBD>-#<23>F<EFBFBD>Ubs<62>(Q!?<3F><1A>#xJG?5<><35><EFBFBD><EFBFBD><EFBFBD>~<7E><>6MA<15>U<><55><EFBFBD>C
<01><>M<>$+}W<>NϨG!<21><><EFBFBD><EFBFBD>a<EFBFBD><61><EFBFBD><EFBFBD>%<25>ǽ<EFBFBD>G

secrets/tent-gitlab-runner-pm-shell-token.age

@@ -1,12 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w 5K0mzfJGvAB2LGmoQ9ZLbWooVEX6F4+fQdo1JUoB3FM
+-> ssh-ed25519 G5LX5w 1KfTmTRP3iSdcclf/FuIpFWpy1tgKs5ED+qSYWo7inY
-AKGa507bUrYjXFaMQ1MXTDBFYsdS6zbs+flmxYN0UNo
+RX6Q1nLFF/yiVLpkWrl0BI0PpLoBi753+y8l/AXjNE4
--> ssh-ed25519 CAWG4Q 8KzLc949on8iN1pK8q11OpCIeO71t6b0zxCLHhcQ6ns
+-> ssh-ed25519 cK5kHw TP7+OQpQSNuyArnUo1C97J3P3oB0YtzCEPeVvlzsYHE
-uy7z6RdIuoUes+Uap3k5eoFFuu/DcSrEBwq4V4C/ygc
+Bsy5KPNHTVNHnF1sxOvlfJq3CNMVFaXdYkRG2vSj7qM
--> ssh-ed25519 xA739A SLx5cKo0fdAHj+cLpJ4FYTWTUTyDsCqKQOufDu3xnGo
+-> ssh-ed25519 CAWG4Q eQyzwNaH6CfaYIjs8abEuQxt6vxRXsGz69UletMUVDE
-VnS/WsiSaf6RpXuhgfij4pYu4p9hlJl1oXrfYY9rKlQ
+FDcynPO7xg4PWez5Z8gTg5LyE0Wgb3zT9i3Kon67QsU
--> ssh-ed25519 MSF3dg c5ZXvdNxNfZU3HeWsttuhy+UC5JxWN/IFuCuCGbksn4
+-> ssh-ed25519 xA739A 2JuLai2fUu3dZBydS8cMrLrEUIUkz4NNaiupoBOtTwU
-vcKlIirf+VvERX71YpmwW6zp6ClhlG2PR4R8LIN7cQo
+sdM3X+XRzysop7yqa76Z7FAwTHOj91STCtZvfIgCdB0
---- pJKICDaYAlxqNnvHIuzB3Yk7tv0ZNYflGTQD+Zk/8+4
+-> ssh-ed25519 MSF3dg fSPkiWnpInX1V5p3afPCoPotcGFoWFiOMPThtY927lc
-<EFBFBD>h/\J<>J
+8v7E/3l0xA2VWZPXzkN4NmnaA0KJutLMurn/ZXZmhxA
-<EFBFBD>0?<3F> <20>p<EFBFBD><70><EFBFBD>@܉7<DC89><37>3<EFBFBD><33><EFBFBD><EFBFBD>z<EFBFBD><7A><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>a<EFBFBD><61>'<27>,ka<6B>I<EFBFBD>XXOZ<4F>I\<5C><><EFBFBD><EFBFBD><EFBFBD>	<09>BP<42><50>/cUɿ~B<><42>S'Q<><51><EFBFBD><EFBFBD>f<06><><EFBFBD>er<65><72><EFBFBD><EFBFBD>^<5E><><EFBFBD><EFBFBD>8l<38><6C>V<EFBFBD>E<EFBFBD><45><EFBFBD>
+--- MQkyBx9hT4ILYXKoZT18PWny1QbDFymcZr63zjMN/qQ
+-b<>#<23><>M.<16>@<40>t<EFBFBD><74><EFBFBD>ŵ}+ό#@<40><><EFBFBD><EFBFBD><EFBFBD>k<EFBFBD>y<EFBFBD><79><EFBFBD>?v<><76>n<1F><>T<EFBFBD>+<2B><><EFBFBD>[<5B>Q<EFBFBD> gA<67><41><EFBFBD>

secrets/vpn-dac-login.age

@@ -1,12 +1,14 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w /RF8uZ/KahUqjEFILbF3+Jin+U0SQdoQChcc9RJ9axc
+-> ssh-ed25519 G5LX5w SRJhNenoQXbT1FgX3TMPnVH5P6oe2eHot+M1YsEjsEk
-aEmPk++86nBR6d2BIa/oaUdyiLS6cH8TUoYJE3bxba4
+hfTSLgKi98Eh7JK5o7x2POpTEtQlQCpEa3keUFYCuME
--> ssh-ed25519 CAWG4Q qHyh9nQi8c3z/KHby9y5vhzN0Dwz0zca98ebjJmXrzs
+-> ssh-ed25519 cK5kHw z5TwWJTkvx7HztjXHJW/aCOtOfPrQaLP0gyIT7rXcyU
-ZbmwNzrSSQ3RvskE8SqcBa0vMy8pzm/HPGHLm5zuPGQ
+b4NCpHfasgvkLLr+6LcWUl60p59aSNnfp3bl2OFYXo0
--> ssh-ed25519 xA739A FlGbfS4bUxA3gVDzb3yPjp4hV8a7aiNBLUctnN3bGEY
+-> ssh-ed25519 CAWG4Q 4VpS1/OnFe8nxcQbRTKNhjsh/ZQ5cbhSMXwK/jjQ+3o
-3fI6SyVjVhh2M8uc/XV3blpdQMPMYi2qzaHNXvx0bvM
+WF9wvOkqVml4UcEzyzeumKuUwCwwr2zvKLMg+PCB8nk
--> ssh-ed25519 MSF3dg 0Bs/aW0nNISS+93It75o6hKZWa7S+LF5bF5ApsJ2fQ8
+-> ssh-ed25519 xA739A 67FhuJ070jBVMt/xbKHWhfri6iIm0FyaFvzQabsvFBM
-y7o0KYDHEen13ndIxg/mYil3eMxxzvYF2pWqhMb+rBU
+1G5/913dDv/r/6p1x/c5YiUnZzrX/LvIj33KW+PN0KU
---- Iqo75G4+02Y9nc1OOkcEx+iQlKnGYCekAx76tRH53wA
+-> ssh-ed25519 MSF3dg Bj/yB4N2wkyHCHC22tcjjJAA4ebSamN0Z4UVX3ZnryI
-<10>
+6D/ZgTs+j+MGDAbPU5zyK0i9zN6tQy68IcOnQZ27mYg
-<EFBFBD>X<EFBFBD><EFBFBD>%f<0C><><12>hX<0B><>R<>c<EFBFBD>+z<><7A>eg<65>
& <20>d<EFBFBD><64><EFBFBD>ק<06><>A<EFBFBD><41><EFBFBD>чXM<58>1<EFBFBD>
+--- 169erk3ICSYLs4FPEuXCn7QlekWhsmSn0Lr+/R14I5Q
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><05>ҽ3<D2BD>s<EFBFBD>
+w<EFBFBD><EFBFBD>4D<EFBFBD><EFBFBD>b.<2E><><EFBFBD>"|<7C><><EFBFBD>)"<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>;<3B>.<2E>ɫ7)<29>LeC<05>=S؟

secrets/wg-fox.age

@@ -0,0 +1,14 @@
+age-encryption.org/v1
+-> ssh-ed25519 cDBabA heyW9/cxgwFX9IexQIXjAQDWGQPNcMXcArQp2Rxsqx4
+o9MQ7EH8PDDjsJdpH9F3Xq2zUoaDAJQlfFmYucSFs6Y
+-> ssh-ed25519 cK5kHw Sza4pos7K3qW3omEeyidI/jszJNf9smemSZnUJfCIww
+D6vazXki7hIYraIuSiGPS+FPbkFUwHhHWDf52OhEIMg
+-> ssh-ed25519 CAWG4Q YexIHueOIMmIN8JIDyNUOKBkyz/k18HqV3hTXh48KlM
+xh8UJzzWT6ByN+Dpn4JrMNsjGC/uc/v6LynwjBDz9NQ
+-> ssh-ed25519 xA739A KySG3TXdqfCMUkVEDGa74B0op745s3XGYxFLyAXSQAc
+5EI/yb5ctW9Qu18bHm3/sK97kwGcKzzmWvPSCWm89XA
+-> ssh-ed25519 MSF3dg MNxnNj0fHmri8ophexXPNjRUBUWrzcuk5S1mucxUMTE
+GVFWXtISEU8ZmlwL4nh4weAgfGrt2GHX0DTzbpS6zg8
+--- UdrqkYG2ZApAuwdZeNhC50NP2rkD/Ol6y8nJa4RHx7Y
+<EFBFBD>ܻ<EFBFBD>m(<28><><EFBFBD>><3E>H<48>Y87<><37>G<0F>+*<12><><EFBFBD><EFBFBD>9V<>.<2E><><EFBFBD><EFBFBD><03><><EFBFBD>p<EFBFBD>Oo<4F>=+哇<>P0<50><30>{<7B>)<29><17><><EFBFBD><EFBFBD>><3E>z3P^
+u

web/content/fox/_index.md

@@ -21,17 +21,28 @@ the detailed specifications:
 
 ## Access
 
-To access the machine, request a SLURM session from [hut](/hut) using the `fox`
+To access the machine, request a SLURM session from [apex](/apex) using the `fox`
-partition:
+partition. If you need the machine for performance measurements, use an
+exclusive reservation:
 
-    hut% salloc -p fox
+    apex% salloc -p fox --exclusive
 
-Then connect via ssh:
+Otherwise, specify the CPUs that you need so other users can also use the node
+at the same time:
 
-    hut% ssh fox
+    apex% salloc -p fox -c 8
+
+Then use srun to execute an interactive shell:
+
+    apex% srun --pty $SHELL
     fox%
 
-Follow [these steps](/access) if you don't have access to hut or fox.
+Make sure you get all CPUs you expect:
+
+    fox% grep Cpus_allowed_list /proc/self/status
+    Cpus_allowed_list:	0-191
+
+Follow [these steps](/access) if you don't have access to apex or fox.
 
 ## CUDA
 
@@ -85,13 +96,22 @@ Then just run `nix develop` from the same directory:
     Cuda compilation tools, release 12.4, V12.4.99
     Build cuda_12.4.r12.4/compiler.33961263_0
 
+## AMD uProf
+
+The [AMD uProf](https://www.amd.com/en/developer/uprof.html) performance
+analysis tool-suite is installed and ready to use.
+
+See the [AMD uProf user guide](https://docs.amd.com/r/en-US/57368-uProf-user-guide)
+([PDF backup for v5.1](https://jungle.bsc.es/pub/57368-uprof-user-guide.pdf))
+for more details on how to use the tools. To use the GUI make sure that you
+connect to fox using X11 forwarding.
+
 ## Filesystems
 
 The machine has several file systems available.
 
-- `$HOME`: Mounted via NFS across all nodes. It is slow and has low capacity.
+- `/nfs/home`: The `/home` from apex via NFS, which is also shared with other
-  Don't abuse.
+  xeon machines. It has about 2 ms of latency, so not suitable for quick random
-- `/ceph/home/$USER`: Shared Ceph file system across jungle nodes. Slow but high
+  access.
-  capacity. Stores three redundant copies of every file.
 - `/nvme{0,1}/$USER`: The two local NVME disks, very fast and large capacity.
 - `/tmp`: tmpfs, fast but not backed by a disk. Will be erased on reboot.

web/content/posts/2025-09-26/_index.md

@@ -0,0 +1,49 @@
+---
+title: "Update 2025-09-26"
+author: "Rodrigo Arias Mallo"
+date: 2025-09-26
+---
+
+This is a summary of notable changes introduced in the last two years. We
+continue to maintain all machines updated to the last NixOS release (currently
+NixOS 25.05).
+
+### New compute node: fox
+
+We have a new [fox machine](/fox), with two AMD Genoa 9684X CPUs and two NVIDIA
+RTX4000 GPUs. During the last months we have been doing some tests and it seems
+that most of the components work well. We have configured CUDA to use the NVIDIA
+GPUs, as well as AMD uProf to trace performance and energy counters from the
+CPUs.
+
+### Upgraded login node: apex
+
+We have upgraded the operating system on the login node to NixOS, which now runs
+Linux 6.15.6. During the upgrade, we have detected a problem with the storage
+disks. The `/` and `/home` partitions sit on a
+[RAID 5](https://en.wikipedia.org/wiki/Standard_RAID_levels#RAID_5),
+transparently handled by a RAID hardware controller which starts its own
+firmware before passing the control to the BIOS to continue the boot sequence. A
+problem during the startup of the firmware prevented the node to even reach the
+BIOS screen.
+
+After a long debugging session, we detected that the flash memory that stores
+the firmware of the hardware controller was likely to be the issue, since
+[memory cells](https://en.wikipedia.org/wiki/Flash_memory#Principles_of_operation)
+may lose charge over time and can end up corrupting the content. We flashed
+the latest firmware so the memory cells are charged again with the new bits and
+that fixed the problem. Hopefully we will be able to use it for some more years.
+
+The SLURM server has been moved to apex which allows users to also submit jobs
+to fox.
+
+### Migrated machines to BSC building
+
+The server room had a temperature issue that had been affecting our machines
+since the end of February of 2025. As the summer approached, the temperature
+exceeded the safe limits for our hardware, so we had to shutdown the cluster.
+
+![Room temperature](temp.png)
+
+Since then, we have moved the cluster to BSC premises, where it now rests at a
+stable temperature, so hopefully we won't have more unscheduled downtime.