Add remote sblame probe to prometheus

Reviewed-by: Aleix Boné <abonerib@bsc.es>

flake.lock

@@ -2,16 +2,16 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1752436162,
-        "narHash": "sha256-Kt1UIPi7kZqkSc5HVj6UY5YLHHEzPBkgpNUByuyxtlw=",
+        "lastModified": 1767634882,
+        "narHash": "sha256-2GffSfQxe3sedHzK+sTKlYo/NTIAGzbFCIsNMUPAAnk=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "dfcd5b901dbab46c9c6e80b265648481aafb01f8",
+        "rev": "3c9db02515ef1d9b6b709fc60ba9a540957f661c",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-25.05",
+        "ref": "nixos-25.11",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,6 +1,6 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
   };
 
   outputs = { self, nixpkgs, ... }:

keys.nix

@@ -22,8 +22,9 @@ rec {
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
     login      = [ apex ];
+    services   = [ tent ];
 
-    system     = storage ++ monitor ++ login;
+    system     = storage ++ monitor ++ login ++ services;
     safe       = system ++ compute;
     all        = safe ++ playground;
   };

m/apex/configuration.nix

@@ -57,6 +57,18 @@
     };
   };
 
+  services.fail2ban = {
+    enable = true;
+    maxretry = 5;
+    bantime-increment = {
+      enable = true; # Double ban time on each attack
+      maxtime = "7d"; # Ban up to a week
+    };
+  };
+
+  # Disable SSH login with password, allow only keypair
+  services.openssh.settings.PasswordAuthentication = false;
+
   networking.firewall = {
     extraCommands = ''
       # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our

m/apex/nfs.nix

@@ -7,7 +7,7 @@
     mountdPort = 4002;
     statdPort = 4000;
     exports = ''
-      /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+      /home 10.0.40.0/21(rw,async,no_subtree_check,no_root_squash)
       /home 10.106.0.0/24(rw,async,no_subtree_check,no_root_squash)
     '';
   };
@@ -15,19 +15,19 @@
     # Check with `rpcinfo -p`
     extraCommands = ''
       # Accept NFS traffic from compute nodes but not from the outside
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 20048 -j nixos-fw-accept
       # Same but UDP
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 20048 -j nixos-fw-accept
 
       # Accept NFS traffic from wg0
       iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept

m/bay/configuration.nix

@@ -24,7 +24,7 @@
       address = "10.0.40.40";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.40";
       prefixLength = 24;
     } ];
@@ -35,7 +35,7 @@
         # Accept monitoring requests from hut
         iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
         # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/21 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
       '';
     };
   };

m/common/base/env.nix

@@ -1,11 +1,36 @@
-{ pkgs, config, ... }:
+{ pkgs, ... }:
 
 {
   environment.systemPackages = with pkgs; [
-    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns pv
-    # From bsckgs overlay
+    cmake
+    ethtool
+    file
+    freeipmi
+    git
+    gnumake
+    home-manager
+    htop
+    ipmitool
+    ldns
+    lm_sensors
+    ncdu
+    nix-diff
+    nix-index
+    nix-output-monitor
+    nixfmt-tree
+    nixos-option
+    pciutils
+    perf
+    pv
+    ripgrep
+    tcpdump
+    tmux
+    tree
+    vim
+    wget
+
+    # From jungle overlay
+    nixgen
     osumb
   ];
 

m/common/base/users.nix

@@ -139,6 +139,7 @@
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
         ];
+        shell = pkgs.zsh;
       };
 
       pmartin1 = {
@@ -193,6 +194,32 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOlRX7ZCnqtUJYCxKgWmgSrFCYuA2LHY96rVwqxXPl86 aaguirre@BSC-8488184117"
         ];
       };
+
+      emonteir = {
+        uid = 9656;
+        isNormalUser = true;
+        home = "/home/Computational/emonteir";
+        description = "Erwin Royson Monteiro";
+        group = "Computational";
+        hosts = [ "apex" "fox" ];
+        hashedPassword = "$6$0mU88zd3ZuK5NiJQ$DFWL5RMLH6esQM5UyhBCiiNryw4lDDmvJp7Usz3tmevnsiSJr6u0RsUKAnR/K8GRBFrV1.GocrgNjKjik5GY//";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKZKot/Y3F5Wq9pQIXlCbyvQuVVeWMCsAC96Nd+LTcG erwin@Oreo"
+        ];
+      };
+
+      ssanzmar = {
+        uid = 9657;
+        isNormalUser = true;
+        home = "/home/Computational/ssanzmar";
+        description = "Sergio Sanz Martínez";
+        group = "Computational";
+        hosts = [ "apex" "fox" ];
+        hashedPassword = "$6$HUjNDJeJMmNQ6M64$laXSOZcXg6o4v2r8Jm8Xj9kmqw7veCY32po3TVDPRR4WlyxvOeqwoKr4NjlUlPPpKN55Oot3ZYHi.9iNXsH5E1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIELrsRRHXryrdA2ZBx5XmdGxL4DC5bmJydhBeTWQ0SQ sergio.sanz.martinez@estudiantat.upc.edu"
+        ];
+      };
     };
 
     groups = {

m/common/base/watchdog.nix

@@ -5,5 +5,5 @@
   boot.kernelModules = [ "ipmi_watchdog" ];
 
   # Enable systemd watchdog with 30 s interval
-  systemd.watchdog.runtimeTime = "30s";
+  systemd.settings.Manager.RuntimeWatchdogSec = 30;
 }

m/eudy/kernel/perf.nix

@@ -1,11 +1,6 @@
-{ config, pkgs, lib, ... }:
+{ pkgs, lib, ... }:
 
 {
-  # add the perf tool
-  environment.systemPackages = with pkgs; [
-    config.boot.kernelPackages.perf
-  ];
-
   # allow non-root users to read tracing data from the kernel
   boot.kernel.sysctl."kernel.perf_event_paranoid" = -2;
   boot.kernel.sysctl."kernel.kptr_restrict" = 0;

m/fox/configuration.nix

@@ -93,20 +93,4 @@
     wantedBy = [ "multi-user.target" ];
     serviceConfig.ExecStart = script;
   };
-
-  # Only allow SSH connections from users who have a SLURM allocation
-  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
-  security.pam.services.sshd.rules.account.slurm = {
-    control = "required";
-    enable = true;
-    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
-    args = [ "log_level=debug5" ];
-    order = 999999; # Make it last one
-  };
-
-  # Disable systemd session (pam_systemd.so) as it will conflict with the
-  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
-  # into the slurmstepd task and then into the systemd session, which is not
-  # what we want, otherwise it will linger even if all jobs are gone.
-  security.pam.services.sshd.startSession = lib.mkForce false;
 }

m/hut/configuration.nix

@@ -45,7 +45,7 @@
       address = "10.0.40.7";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.7";
       prefixLength = 24;
     } ];

m/hut/gitlab-runner.nix

@@ -51,6 +51,7 @@
           "/nix/store:/nix/store:ro"
           "/nix/var/nix/db:/nix/var/nix/db:ro"
           "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+          "/var/run/postgresql/:/var/run/postgresql/"
         ];
         dockerExtraHosts = [
           # Required to pass the proxy via hut

m/hut/monitoring.nix

@@ -5,7 +5,7 @@
     ../module/slurm-exporter.nix
     ../module/meteocat-exporter.nix
     ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
+    ./ssh-robot-probes.nix
     ../module/nix-daemon-exporter.nix
   ];
 
@@ -111,6 +111,7 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9967" # SLURM custom exporter
             "127.0.0.1:9999" # Nix-daemon custom exporter
             "127.0.0.1:9929" # Meteocat custom exporter
             "127.0.0.1:9928" # UPC Qaire custom exporter

m/hut/nginx.nix

@@ -4,8 +4,8 @@ let
     name = "jungle-web";
     src = pkgs.fetchgit {
       url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
-      rev = "52abaf4d71652a9ef77a0b098db14ca33bffff4c";
-      hash = "sha256-/ul9GazbOrOkmlvSgDz/+2W+V+ir5725Y7mVLc3rb0M=";
+      rev = "5f18335d14126d2fef134c0cd441771436f7dfa1";
+      hash = "sha256-s9VBF91sQ7hg9+lrwNFPYgoXTTyXaQcAulCiGJgWERo=";
     };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''

m/hut/postgresql.nix

@@ -8,12 +8,14 @@
       { name = "anavarro"; ensureClauses.superuser = true; }
       { name = "rarias";   ensureClauses.superuser = true; }
       { name = "grafana"; }
+      { name = "gitlab-runner"; }
     ];
     authentication = ''
-      #type  database     DBuser    auth-method
-      local  perftestsdb  rarias    trust
-      local  perftestsdb  anavarro  trust
-      local  perftestsdb  grafana   trust
+      #type  database     DBuser         auth-method
+      local  perftestsdb  rarias         trust
+      local  perftestsdb  anavarro       trust
+      local  perftestsdb  grafana        trust
+      local  perftestsdb  gitlab-runner  trust
     '';
   };
 }

m/hut/sblame-probe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+EOF
+ssh bsc015557@glogin2.bsc.es "timeout 3 command sblame -E"

m/hut/ssh-robot-probes.nix

@@ -6,6 +6,12 @@ let
       chmod +x $out
     ''
   ;
+  sblame-probe-script = pkgs.runCommand "sblame-probe.sh" { }
+    ''
+      cp ${./sblame-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
 in
 {
   # Use a new user to handle the SSH keys
@@ -28,4 +34,17 @@ in
       Group = "ssh-robot";
     };
   };
+
+  systemd.services.sblame-probe = {
+    description = "Daemon to report SLURM statistics via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9967,fork EXEC:${sblame-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
 }

m/lake2/configuration.nix

@@ -46,7 +46,7 @@
       address = "10.0.40.42";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.42";
       prefixLength = 24;
     } ];
@@ -57,7 +57,7 @@
         # Accept monitoring requests from hut
         iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
         # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/21 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
       '';
     };
   };

m/module/debuginfod.nix

@@ -1,3 +1,10 @@
 {
-  services.nixseparatedebuginfod.enable = true;
+  services.nixseparatedebuginfod2 = {
+    enable = true;
+    substituters = [
+      "local:"
+      "https://cache.nixos.org"
+      "http://hut/cache"
+    ];
+  };
 }

m/module/slurm-client.nix

@@ -1,4 +1,4 @@
-{ lib, ... }:
+{ lib, pkgs, ... }:
 
 {
   imports = [
@@ -21,4 +21,20 @@
   };
 
   services.slurm.client.enable = true;
+
+  # Only allow SSH connections from users who have a SLURM allocation
+  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
+  security.pam.services.sshd.rules.account.slurm = {
+    control = "required";
+    enable = true;
+    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
+    args = [ "log_level=debug5" ];
+    order = 999999; # Make it last one
+  };
+
+  # Disable systemd session (pam_systemd.so) as it will conflict with the
+  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
+  # into the slurmstepd task and then into the systemd session, which is not
+  # what we want, otherwise it will linger even if all jobs are gone.
+  security.pam.services.sshd.startSession = lib.mkForce false;
 }

m/module/slurm-common.nix

@@ -1,31 +1,6 @@
 { config, pkgs, ... }:
 
-let
-  suspendProgram = pkgs.writeShellScript "suspend.sh" ''
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Shutting down host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
-    done
-  '';
-
-  resumeProgram = pkgs.writeShellScript "resume.sh" ''
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Starting host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
-    done
-  '';
-
-in {
+{
   services.slurm = {
     controlMachine = "apex";
     clusterName = "jungle";
@@ -59,16 +34,6 @@ in {
       # the resources. Use the task/cgroup plugin to enable process containment.
       TaskPlugin=task/affinity,task/cgroup
 
-      # Power off unused nodes until they are requested
-      SuspendProgram=${suspendProgram}
-      SuspendTimeout=60
-      ResumeProgram=${resumeProgram}
-      ResumeTimeout=300
-      SuspendExcNodes=fox
-
-      # Turn the nodes off after 1 hour of inactivity
-      SuspendTime=3600
-
       # Reduce port range so we can allow only this range in the firewall
       SrunPortRange=60000-61000
 

m/module/tc1-board.nix

@@ -0,0 +1,27 @@
+{ lib, pkgs, ... }:
+
+{
+  # Allow user access to FTDI USB device
+  services.udev.packages = lib.singleton (pkgs.writeTextFile {
+    # Needs to be < 73
+    name = "60-ftdi-tc1.rules";
+    text = ''
+      # Bus 003 Device 003: ID 0403:6011 Future Technology Devices International, Ltd FT4232H Quad HS USB-UART/FIFO IC
+      # Use := to make sure it doesn't get changed later
+      SUBSYSTEMS=="usb", ATTRS{idVendor}=="0403", ATTRS{idProduct}=="6011", MODE:="0666"
+    '';
+    destination = "/etc/udev/rules.d/60-ftdi-tc1.rules";
+  });
+
+  # Allow access to USB for docker in GitLab runner
+  services.gitlab-runner = {
+    services.gitlab-bsc-docker = {
+      registrationFlags = [
+        # We need raw access to the USB port to reboot the board
+        "--docker-devices /dev/bus/usb/003/003"
+        # And TTY access for the serial port
+        "--docker-devices /dev/ttyUSB2"
+      ];
+    };
+  };
+}

m/owl1/configuration.nix

@@ -20,7 +20,7 @@
       address = "10.0.40.1";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.1";
       prefixLength = 24;
     } ];

m/owl2/configuration.nix

@@ -21,7 +21,7 @@
       prefixLength = 24;
     } ];
     # Watch out! The OmniPath device is not in the same place here:
-    interfaces.ibp129s0.ipv4.addresses = [ {
+    interfaces.ibs801.ipv4.addresses = [ {
       address = "10.0.42.2";
       prefixLength = 24;
     } ];

m/tent/configuration.nix

@@ -16,6 +16,8 @@
     ../module/p.nix
     ../module/vpn-dac.nix
     ../module/hut-substituter.nix
+    ../module/tc1-board.nix
+    ../module/ceph.nix
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -63,6 +65,13 @@
     fsType = "ext4";
   };
 
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
   # Make a /vault/$USER directory for each user.
   systemd.services.create-vault-dirs = let
     # Take only normal users in tent

m/tent/gitea.nix

@@ -1,4 +1,7 @@
 { config, lib, ... }:
+let
+  cfg = config.services.gitea;
+in
 {
   services.gitea = {
     enable = true;
@@ -26,5 +29,54 @@
         SENDMAIL_ARGS = "--";
       };
     };
+
+    dump = {
+      enable = false; # Do not enable NixOS module, use our custom systemd script below
+      backupDir = "/vault/backup/gitea";
+    };
   };
+
+  systemd.services.gitea-backup = let
+    exe = lib.getExe cfg.package;
+  in {
+    description = "Gitea daily backup";
+    after = [ "gitea.service" ];
+    path = [ cfg.package ];
+
+    environment = {
+      USER = cfg.user;
+      HOME = cfg.stateDir;
+      GITEA_WORK_DIR = cfg.stateDir;
+      GITEA_CUSTOM = cfg.customDir;
+    };
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = cfg.user;
+      WorkingDirectory = cfg.dump.backupDir;
+    };
+
+    script = ''
+      name="gitea-dump-$(date +%a).${cfg.dump.type}"
+      ${exe} dump --type ${cfg.dump.type} --file - >"$name.tmp"
+      mv "$name.tmp" "$name"
+      cp "$name" "/ceph/backup/gitea/$name"
+    '';
+  };
+
+  # Create also the /ceph directories if needed
+  systemd.tmpfiles.rules = [
+    "d /ceph/backup/gitea/ 0750 ${cfg.user} ${cfg.group} - -"
+    "z /ceph/backup/gitea/ 0750 ${cfg.user} ${cfg.group} - -"
+  ];
+
+  systemd.timers.gitea-backup = {
+    description = "Update timer for gitea-backup";
+    partOf = [ "gitea-backup.service" ];
+    wantedBy = [ "timers.target" ];
+    timerConfig.OnCalendar = cfg.dump.interval;
+  };
+
+  # Allow gitea user to send mail
+  users.users.gitea.extraGroups = [ "mail-robot" ];
 }

m/tent/gitlab-runner.nix

@@ -43,6 +43,7 @@
         registrationFlags = [
           # Increase build log length to 64 MiB
           "--output-limit 65536"
+          "--docker-network-mode host"
         ];
         preBuildScript = pkgs.writeScript "setup-container" ''
           mkdir -p -m 0755 /nix/var/log/nix/drvs

m/tent/nginx.nix

@@ -4,8 +4,8 @@ let
     name = "jungle-web";
     src = pkgs.fetchgit {
       url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
-      rev = "52abaf4d71652a9ef77a0b098db14ca33bffff4c";
-      hash = "sha256-/ul9GazbOrOkmlvSgDz/+2W+V+ir5725Y7mVLc3rb0M=";
+      rev = "5f18335d14126d2fef134c0cd441771436f7dfa1";
+      hash = "sha256-s9VBF91sQ7hg9+lrwNFPYgoXTTyXaQcAulCiGJgWERo=";
     };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''

m/weasel/configuration.nix

@@ -25,7 +25,7 @@
       address = "10.0.40.6";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.6";
       prefixLength = 24;
     } ];

overlay.nix

@@ -30,13 +30,15 @@ let
       amd-uprof-driver = _prev.callPackage ./pkgs/amd-uprof/driver.nix { };
     });
     lmbench = callPackage ./pkgs/lmbench/default.nix { };
-    mcxx = callPackage ./pkgs/mcxx/default.nix { };
+    # Broken and unmantained
+    # mcxx = callPackage ./pkgs/mcxx/default.nix { };
     meteocat-exporter = prev.callPackage ./pkgs/meteocat-exporter/default.nix { };
     mpi = final.mpich; # Set MPICH as default
     mpich = callPackage ./pkgs/mpich/default.nix { mpich = prev.mpich; };
     nanos6 = callPackage ./pkgs/nanos6/default.nix { };
     nanos6Debug = final.nanos6.override { enableDebug = true; };
     nixtools = callPackage ./pkgs/nixtools/default.nix { };
+    nixgen = callPackage ./pkgs/nixgen/default.nix { };
     # Broken because of pkgsStatic.libcap
     # See: https://github.com/NixOS/nixpkgs/pull/268791
     #nix-wrap = callPackage ./pkgs/nix-wrap/default.nix { };
@@ -100,14 +102,16 @@ let
   pkgsTopLevel = filterAttrs (_: isDerivation) bscPkgs;
 
   # Native build in that platform doesn't imply cross build works
-  canCrossCompile = platform: pkg:
+  canCrossCompile = platform: default: pkg:
     (isDerivation pkg) &&
-    # Must be defined explicitly
-    (pkg.meta.cross or false) &&
-    (meta.availableOn platform pkg);
+    # If meta.cross is undefined, use default
+    (pkg.meta.cross or default) &&
+    (meta.availableOn final.pkgsCross.${platform}.stdenv.hostPlatform pkg);
 
   # For now only RISC-V
-  crossSet = { riscv64 = final.pkgsCross.riscv64.bsc.pkgsTopLevel; };
+  crossSet = genAttrs [ "riscv64" ] (platform:
+    filterAttrs (_: canCrossCompile platform true)
+      final.pkgsCross.${platform}.bsc.pkgsTopLevel);
 
   buildList = name: paths:
     final.runCommandLocal name { } ''
@@ -127,7 +131,7 @@ let
   # For now only RISC-V
   crossList = buildList "ci-cross"
     (filter
-      (canCrossCompile final.pkgsCross.riscv64.stdenv.hostPlatform)
+      (canCrossCompile "riscv64" false) # opt-in (pkgs with: meta.cross = true)
         (builtins.attrValues crossSet.riscv64));
 
 in bscPkgs // {

pkgs/amd-uprof/default.nix

@@ -90,7 +90,7 @@ in
     meta = {
       description = "Performance analysis tool-suite for x86 based applications";
       homepage = "https://www.amd.com/es/developer/uprof.html";
-      platforms = lib.platforms.linux;
+      platforms = [ "x86_64-linux" ];
       license = lib.licenses.unfree;
       maintainers = with lib.maintainers.bsc; [ rarias varcila ];
     };

pkgs/amd-uprof/driver.nix

@@ -19,7 +19,7 @@ in stdenv.mkDerivation {
   '';
   hardeningDisable = [ "pic" "format" ];
   nativeBuildInputs = kernel.moduleBuildDependencies;
-  patches = [ ./makefile.patch ./hrtimer.patch ];
+  patches = [ ./makefile.patch ./hrtimer.patch ./remove-wr-rdmsrq.patch ];
   makeFlags = [
     "KERNEL_VERSION=${kernel.modDirVersion}"
     "KERNEL_DIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"

pkgs/amd-uprof/remove-wr-rdmsrq.patch

@@ -0,0 +1,20 @@
+diff --git a/inc/PwrProfAsm.h b/inc/PwrProfAsm.h
+index d77770a..c93a0e9 100644
+--- a/inc/PwrProfAsm.h
++++ b/inc/PwrProfAsm.h
+@@ -347,6 +347,7 @@
+ 
+ #endif
+ 
++/*
+ #define rdmsrq(msr,val1,val2,val3,val4) ({ \
+         __asm__ __volatile__( \
+                               "rdmsr\n" \
+@@ -362,6 +363,7 @@
+                               :"c"(msr), "a"(val1), "d"(val2), "S"(val3), "D"(val4) \
+                             ); \
+     })
++*/
+ 
+ #define rdmsrpw(msr,val1,val2,val3,val4) ({ \
+         __asm__ __volatile__( \

pkgs/cudainfo/default.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, lib
 , cudatoolkit
 , cudaPackages
 , autoAddDriverRunpath
@@ -11,7 +12,7 @@ stdenv.mkDerivation (finalAttrs: {
   src = ./.;
   buildInputs = [
     cudatoolkit # Required for nvcc
-    cudaPackages.cuda_cudart.static # Required for -lcudart_static
+    (lib.getOutput "static" cudaPackages.cuda_cudart) # Required for -lcudart_static
     autoAddDriverRunpath
   ];
   installPhase = ''
@@ -40,4 +41,9 @@ stdenv.mkDerivation (finalAttrs: {
     '';
     installPhase = "touch $out";
   };
+
+  meta = {
+    platforms = [ "x86_64-linux" ];
+    maintainers = with lib.maintainers.bsc; [ rarias ];
+  };
 })

pkgs/gpi-2/default.nix

@@ -9,7 +9,6 @@
 , automake
 , libtool
 , mpi
-, rsync
 , gfortran
 }:
 
@@ -44,13 +43,24 @@ stdenv.mkDerivation rec {
 
   configureFlags = [
     "--with-infiniband=${rdma-core-all}"
-    "--with-mpi=${mpiAll}"
+    "--with-mpi=yes" # fixes mpi detection when cross-compiling
     "--with-slurm"
     "CFLAGS=-fPIC"
     "CXXFLAGS=-fPIC"
   ];
 
-  buildInputs = [ slurm mpiAll rdma-core-all autoconf automake libtool rsync gfortran ];
+  nativeBuildInputs = [
+    autoconf
+    automake
+    gfortran
+    libtool
+  ];
+
+  buildInputs = [
+    slurm
+    mpiAll
+    rdma-core-all
+  ];
 
   hardeningDisable = [ "all" ];
 
@@ -60,5 +70,6 @@ stdenv.mkDerivation rec {
     maintainers = with lib.maintainers.bsc; [ rarias ];
     platforms = lib.platforms.linux;
     license = lib.licenses.gpl3Plus;
+    cross = false; # infiniband detection does not work
   };
 }

pkgs/intel-oneapi/2023.nix

@@ -10,7 +10,7 @@
 , zlib
 , autoPatchelfHook
 , libfabric
-, gcc13
+, gcc
 , wrapCCWith
 }:
 
@@ -33,8 +33,6 @@ let
     maintainers = with lib.maintainers.bsc; [ abonerib ];
   };
 
-  gcc = gcc13;
-
   v = {
     hpckit   = "2023.1.0";
     compiler = "2023.1.0";

pkgs/llvm-ompss2/clang.nix

@@ -16,19 +16,19 @@
 , useGit ? false
 , gitUrl ? "ssh://git@bscpm04.bsc.es/llvm-ompss/llvm-mono.git"
 , gitBranch ? "master"
-, gitCommit ? "880e2341c56bad1dc14e8c369fb3356bec19018e"
+, gitCommit ? "872ba63f86edaefc9787984ef3fae9f2f94e0124" # github-release-2025.11
 }:
 
 let
   stdenv = llvmPackages_latest.stdenv;
 
   release = rec {
-    version = "2025.06";
+    version = "2025.11";
     src = fetchFromGitHub {
       owner = "bsc-pm";
       repo = "llvm";
       rev = "refs/tags/github-release-${version}";
-      hash = "sha256-ww9PpRmtz/M9IyLiZ8rAehx2UW4VpQt+svf4XfKBzKo=";
+      hash = "sha256-UgwMTUkM9Z87dDH205swZFBeFhrcbLAxginViG40pBM=";
     };
   };
 

pkgs/llvm-ompss2/default.nix

@@ -27,10 +27,10 @@ let
   # We need to replace the lld linker from bintools with our linker just built,
   # otherwise we run into incompatibility issues when mixing compiler and linker
   # versions.
-  bintools-unwrapped = llvmPackages_latest.tools.bintools-unwrapped.override {
+  bintools-unwrapped = llvmPackages_latest.bintools-unwrapped.override {
     lld = clangOmpss2Unwrapped;
   };
-  bintools = llvmPackages_latest.tools.bintools.override {
+  bintools = llvmPackages_latest.bintools.override {
     bintools = bintools-unwrapped;
   };
   targetConfig = stdenv.targetPlatform.config;

pkgs/mcxx/default.nix

@@ -65,6 +65,7 @@ stdenv.mkDerivation rec {
   ];
 
   meta = {
+    broken = true;
     homepage = "https://github.com/bsc-pm/mcxx";
     description = "C/C++/Fortran source-to-source compilation infrastructure aimed at fast prototyping";
     maintainers = with lib.maintainers.bsc; [ rpenacob ];

pkgs/meteocat-exporter/default.nix

@@ -1,9 +1,11 @@
 { python3Packages, lib }:
 
-python3Packages.buildPythonApplication rec {
+python3Packages.buildPythonApplication {
   pname = "meteocat-exporter";
   version = "1.0";
 
+  pyproject = true;
+
   src = ./.;
 
   doCheck = false;

pkgs/mpich/default.nix

@@ -6,6 +6,13 @@
 , pmix
 , gfortran
 , symlinkJoin
+# Disabled when cross-compiling
+# To fix cross compilation, we should fill the values in:
+# https://github.com/pmodels/mpich/blob/main/maint/fcrosscompile/cross_values.txt.in
+# For each arch
+, enableFortran ? stdenv.hostPlatform == stdenv.buildPlatform
+, perl
+, targetPackages
 }:
 
 let
@@ -15,10 +22,13 @@ let
     paths = [ pmix.dev pmix.out ];
   };
 in mpich.overrideAttrs (old: {
-  buildInput = old.buildInputs ++ [
+  buildInputs = old.buildInputs ++ [
     libfabric
     pmixAll
   ];
+  nativeBuildInputs = old.nativeBuildInputs ++ [
+    perl
+  ];
   configureFlags = [
     "--enable-shared"
     "--enable-sharedlib"
@@ -31,10 +41,21 @@ in mpich.overrideAttrs (old: {
   ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
     "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
     "FCFLAGS=-fallow-argument-mismatch"
+  ] ++ lib.optionals (!enableFortran) [
+    "--disable-fortran"
   ];
+
+  preFixup = ''
+    sed -i 's:^CC=.*:CC=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' $out/bin/mpicc
+    sed -i 's:^CXX=.*:CXX=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' $out/bin/mpicxx
+  '' + lib.optionalString enableFortran ''
+    sed -i 's:^FC=.*:FC=${targetPackages.gfortran or gfortran}/bin/${targetPackages.gfortran.targetPrefix or gfortran.targetPrefix}gfortran:' $out/bin/mpifort
+  '';
+
   hardeningDisable = [ "all" ];
 
   meta = old.meta // {
     maintainers = old.meta.maintainers ++ (with lib.maintainers.bsc; [ rarias ]);
+    cross = true;
   };
 })

pkgs/nixgen/default.nix

@@ -0,0 +1,22 @@
+{
+  stdenv
+, lib
+}:
+
+stdenv.mkDerivation {
+  pname = "nixgen";
+  version = "0.0.1";
+  src = ./nixgen;
+  dontUnpack = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -a $src $out/bin/nixgen
+  '';
+  meta = {
+    description = "Quickly generate flake.nix from command line";
+    maintainers = with lib.maintainers.bsc; [ rarias ];
+    platforms = lib.platforms.linux;
+    license = lib.licenses.gpl3Plus;
+  };
+}

pkgs/nixgen/nixgen

@@ -0,0 +1,97 @@
+#!/bin/sh
+#
+# Copyright (c) 2025, Barcelona Supercomputing Center (BSC)
+# SPDX-License-Identifier: GPL-3.0+
+# Author: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
+
+function usage() {
+  echo "USAGE: nixgen [-f] [package [...]] [-b package [...]]" >&2
+  echo "  Generates a flake.nix file with the given packages." >&2
+  echo "  After flake.nix is created, use 'nix develop' to enter the shell." >&2
+  echo "OPTIONS" >&2
+  echo "  -f               Overwrite existing flake.nix (default: no)." >&2
+  echo "  packages...      Add these packages to the shell." >&2
+  echo "  -b packages...   Add the dependencies needed to build these packages." >&2
+  echo "EXAMPLE" >&2
+  echo "  $ nixgen ovni bigotes -b nosv tampi" >&2
+  echo "  Adds the packages ovni and bigotes as well as all required dependencies" >&2
+  echo "  to build nosv and tampi." >&2
+  echo "AUTHOR" >&2
+  echo "  Rodrigo Arias Mallo <rodrigo.arias@bsc.es>" >&2
+  exit 1
+}
+
+mode=package
+packages=
+inputsFrom=
+force=
+
+if [[ $# -eq 0 ]]; then
+  usage
+fi
+
+while [[ $# -gt 0 ]]; do
+    case $1 in -b)
+      mode=build
+      shift
+      ;;
+    -f)
+      force=1
+      shift
+      ;;
+    -h)
+      usage
+      ;;
+    -*|--*)
+      echo "error: unknown option $1" >&2
+      exit 1
+      ;;
+    *)
+      if [ "$mode" == "package" ]; then
+        packages+="${packages:+ }$1"
+      else
+        inputsFrom+="${inputsFrom:+ }$1"
+      fi
+      shift
+      ;;
+  esac
+done
+
+if [ ! "$force" -a -e flake.nix ]; then
+  echo "error: flake.nix exists, force overwrite with -f" >&2
+  exit 1
+fi
+
+cat > flake.nix <<EOF
+{
+  inputs.jungle.url = "git+https://jungle.bsc.es/git/rarias/jungle";
+  outputs = { self, jungle }:
+  let
+    nixpkgs = jungle.inputs.nixpkgs;
+    customOverlay = (final: prev: {
+      # Example overlay, for now empty
+    });
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [
+        # Apply jungle overlay to get our BSC custom packages
+        jungle.outputs.bscOverlay
+        # And on top apply our local changes to customize for cluster
+        customOverlay
+      ];
+    };
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell {
+      pname = "devshell";
+      # Include these packages in the shell
+      packages = with pkgs; [
+        $packages
+      ];
+      # The dependencies needed to build these packages will be also included
+      inputsFrom = with pkgs; [
+        $inputsFrom
+      ];
+    };
+  };
+}
+EOF

pkgs/nodes/default.nix

@@ -14,19 +14,19 @@
 , useGit ? false
 , gitUrl ? "ssh://git@gitlab-internal.bsc.es/nos-v/nodes.git"
 , gitBranch ? "master"
-, gitCommit ? "6002ec9ae6eb876d962cc34366952a3b26599ba6"
+, gitCommit ? "511489e71504a44381e0930562e7ac80ac69a848" # version-1.4
 }:
 
 with lib;
 
 let
   release = rec {
-    version = "1.3";
+    version = "1.4";
     src = fetchFromGitHub {
       owner = "bsc-pm";
       repo = "nodes";
       rev = "version-${version}";
-      hash = "sha256-cFb9pxcjtkMmH0CsGgUO9LTdXDNh7MCqicgGWawLrsU=";
+      hash = "sha256-+lR/R0l3fGZO3XG7whMorFW2y2YZ0ZFnLeOHyQYrAsQ=";
     };
   };
 

pkgs/nosv/default.nix

@@ -13,19 +13,19 @@
 , useGit ? false
 , gitUrl ? "git@gitlab-internal.bsc.es:nos-v/nos-v.git"
 , gitBranch ? "master"
-, gitCommit ? "9f47063873c3aa9d6a47482a82c5000a8c813dd8"
+, gitCommit ? "1108e4786b58e0feb9a16fa093010b763eb2f8e8" # version 4.0.0
 }:
 
 with lib;
 
 let
   release = rec {
-    version = "3.2.0";
+    version = "4.0.0";
     src = fetchFromGitHub {
       owner = "bsc-pm";
       repo = "nos-v";
       rev = "${version}";
-      hash = "sha256-yaz92426EM8trdkBJlISmAoG9KJCDTvoAW/HKrasvOw=";
+      hash = "sha256-llaq73bd/YxLVKNlMebnUHKa4z3sdcsuDUoVwUxNuw8=";
     };
   };
 

pkgs/osu/default.nix

@@ -32,6 +32,11 @@ stdenv.mkDerivation rec {
       "CXX=mpicxx"
   ];
 
+  env = {
+    MPICH_CC="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
+    MPICH_CXX="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++";
+  };
+
   postInstall = ''
     mkdir -p $out/bin
     for f in $(find $out -executable -type f); do
@@ -44,5 +49,6 @@ stdenv.mkDerivation rec {
     homepage = "http://mvapich.cse.ohio-state.edu/benchmarks/";
     maintainers = [ ];
     platforms = lib.platforms.all;
+    cross = true;
   };
 }

pkgs/ovni/default.nix

@@ -7,7 +7,7 @@
 , useGit ? false
 , gitBranch ? "master"
 , gitUrl ? "ssh://git@bscpm04.bsc.es/rarias/ovni.git"
-, gitCommit ? "e4f62382076f0cf0b1d08175cf57cc0bc51abc61"
+, gitCommit ? "06432668f346c8bdc1006fabc23e94ccb81b0d8b" # version 1.13.0
 , enableDebug ? false
 # Only enable MPI if the build is native (fails on cross-compilation)
 , useMpi ? (stdenv.buildPlatform.canExecute stdenv.hostPlatform)
@@ -15,13 +15,13 @@
 
 let
   release = rec {
-    version = "1.12.0";
+    version = "1.13.0";
     src = fetchFromGitHub {
       owner = "bsc-pm";
       repo = "ovni";
       rev = "${version}";
-      hash = "sha256-H04JvsVKrdqr3ON7JhU0g17jjlg/jzQ7eTfx9vUNd3E=";
-    } // { shortRev = "a73afcf"; };
+      hash = "sha256-0l2ryIyWNiZqeYdVlnj/WnQGS3xFCY4ICG8JedX424w=";
+    } // { shortRev = "0643266"; };
   };
 
   git = rec {

pkgs/paraver/default.nix

@@ -12,7 +12,7 @@
 , paraverKernel
 , openssl
 , glibcLocales
-, wrapGAppsHook
+, wrapGAppsHook3
 }:
 
 let
@@ -64,7 +64,7 @@ stdenv.mkDerivation rec {
     autoconf
     automake
     autoreconfHook
-    wrapGAppsHook
+    wrapGAppsHook3
   ];
 
   buildInputs = [

pkgs/sonar/default.nix

@@ -35,5 +35,6 @@ stdenv.mkDerivation rec {
     maintainers = with lib.maintainers.bsc; [ rarias ];
     platforms = lib.platforms.linux;
     license = lib.licenses.mit;
+    cross = true;
   };
 }

pkgs/tagaspi/default.nix

@@ -5,23 +5,14 @@
 , automake
 , autoconf
 , libtool
-, mpi
 , autoreconfHook
 , gpi-2
 , boost
 , numactl
 , rdma-core
 , gfortran
-, symlinkJoin
 }:
 
-let
-  mpiAll = symlinkJoin {
-    name = "mpi-all";
-    paths = [ mpi.all ];
-  };
-in
-
 stdenv.mkDerivation rec {
   pname = "tagaspi";
   enableParallelBuilding = true;
@@ -35,16 +26,18 @@ stdenv.mkDerivation rec {
     hash = "sha256-RGG/Re2uM293HduZfGzKUWioDtwnSYYdfeG9pVrX9EM=";
   };
 
-  buildInputs = [
+  nativeBuildInputs = [
     autoreconfHook
     automake
     autoconf
     libtool
+    gfortran
+  ];
+
+  buildInputs = [
     boost
     numactl
     rdma-core
-    gfortran
-    mpiAll
   ];
 
   dontDisableStatic = true;
@@ -63,5 +56,6 @@ stdenv.mkDerivation rec {
     maintainers = with lib.maintainers.bsc; [ rarias ];
     platforms = lib.platforms.linux;
     license = lib.licenses.gpl3Plus;
+    cross = false; # gpi-2 cannot cross
   };
 }

pkgs/tampi/default.nix

@@ -68,5 +68,6 @@ in stdenv.mkDerivation {
     maintainers = with lib.maintainers.bsc; [ rarias ];
     platforms = lib.platforms.linux;
     license = lib.licenses.gpl3Plus;
+    cross = true;
   };
 }

pkgs/upc-qaire-exporter/default.nix

@@ -1,9 +1,11 @@
 { python3Packages, lib }:
 
-python3Packages.buildPythonApplication rec {
+python3Packages.buildPythonApplication {
   pname = "upc-qaire-exporter";
   version = "1.0";
 
+  pyproject = true;
+
   src = ./.;
 
   doCheck = false;

secrets/ceph-user.age

@@ -1,25 +1,29 @@
 age-encryption.org/v1
--> ssh-ed25519 AY8zKw /gmhFOFqOs8IobAImvQVKeM5Y6k0FpuR61/Cu5drVVI
-g9FXJg2oIoien0zJ70FWHwSTM8SBwbpS188S3Swj7EM
--> ssh-ed25519 sgAamA opPjlWPhSiI0Rd5l7kd204S5FXFLcQcQftyKb7MDmnU
-3XrRDVnglCP+vBwvfd1rP5gHttsGDHyXwbf10a8/kKY
--> ssh-ed25519 HY2yRg QKZbubM76C3tobPoyCFDRclA9Pzb2fC7s4WOoIgdORc
-K5kckU0KhQFTE6SikJXFJgM41Tco5+VqOsaG0qLrY1Q
--> ssh-ed25519 fw2Xhg +ohqts8dLFjvdHxrGHcOGxU0dm+V3N//giljHkobpDM
-jR/UzGrfS9lrJ/VeolKLxfzeJAf2fIB2pdIn/6ukqNk
--> ssh-ed25519 tcumPQ 3DPkDPIQQSVtXSLzIRETsIyXQ0k1o18Evn6vf+l/6R8
-bLXF62OmJjnOT1vvgq3+AcOKKSG5NonrK5EqCVc0Mwo
--> ssh-ed25519 JJ1LWg 2Wefc7eLolMU5InEmCNTq21Mf71mI0a2N1HgDrlHvy4
-qXFW9CQBnrzubZ0mzS0Io2WGRrwGBkmeYndBTcZn/fM
--> ssh-ed25519 cDBabA oiH36AoIt/fFFYgnoxtH7OoetP+2/wjtn8qo3RJDSHc
-qKmkxy1aZGP4ZwC0iH7n7hiJ0+rFQYvjQb5O1a1Z0r4
--> ssh-ed25519 cK5kHw bX3RtO5StMejUYWAaA37fjHA5nO7Xs1vWDQk3yOjs2o
-Egxmcf8FKAd+E5hMLmhV1yQsCo5rJyUazf1szOvpTAM
--> ssh-ed25519 CAWG4Q oKqqRDJH0w8lsoQBQk0w8PO+z5gFNmSaGBUSumvDp1I
-m1zWp9MfViAmtpbJhqOHraIokDaPKb0DvvO4vAGCTWI
--> ssh-ed25519 xA739A G26kPOz6sbFATs+KAr7gbDvji13eA1smFusQAOJXMwA
-Sppvz7A103kZoNxoGsd6eXeCvVh7mBE2MRwLFj9O1dY
--> ssh-ed25519 MSF3dg 55ekNcp+inbUd+GQ/VZ7BoBASaJ8YDqF74CVXy1PUxQ
-aTHLLAbzQPWWld/OT3BKebc6FcmsqMTaWCPBGm1UHic
---- mVkAMnI9XQhS3fMiFuuXP/yLR9wEG9+Rr8pA4Uc0avY
-<04>DU <20><>s<EFBFBD><73><EFBFBD><EFBFBD>j<EFBFBD><6A>M<EFBFBD><4D>$<24>[<5B>M<EFBFBD><4D><EFBFBD><03>[_<>K7s<37>ju<>v<EFBFBD>D<EFBFBD>4<EFBFBD>g<EFBFBD><67>܄3<>Gn<47><6E><EFBFBD> ɽ<>P<EFBFBD>7~rZs<><73>
+-> ssh-ed25519 AY8zKw Crgof1PMHzv3jBw8VeJAst6FKSoyqPFdANFpf79CAgo
+7fagE5BmlWdTsdY/i3RbExu1KBcjW1LQXbYwu6chxlk
+-> ssh-ed25519 sgAamA tGRCaK8mjvz65YziXjRcjMOHIRoyGNJFzBEEbivXPDo
+YLzE5a3J81r+gzkfZIeh9gS+mXzMooC82tBbZ+C3C8o
+-> ssh-ed25519 HY2yRg +vhO1/vdGPM1JnZRsvVnViFWaFWUZ7MIqvWdePivkxA
+2K+JdN82DTeGh9QwZBTaghg8C5BCLoEsOgTCM64PU28
+-> ssh-ed25519 fw2Xhg NHDn0dq32I/AVdUZlpzBX6retlEYEUipde7A9R90qW4
+SJO78ooqEwfHlBRW+YCzgSQJb1JHNo8jz37t3qvLClE
+-> ssh-ed25519 G5LX5w d4HfLzI2623artkR2FIfRJgr5yb2BKZJUWqPnwOWDCk
+Kh50QESJZSjaJPyp3xroHGn0fD5pPNEYgKkDdqxGpjs
+-> ssh-ed25519 tcumPQ wQyOKtT15Qezs3cyv5/xxIPVD7Jyk6N6ZLkfxxBHLTo
+rKlRBjJdfDVT6U8211+ssFF8yY9yRs1u3GhCSvsw2oE
+-> ssh-ed25519 JJ1LWg 98tF1MdA244xNny4w3RnMFuubf4WcuQaZf2bN2Uq8Qc
+MA1Xh1H9vHisVYdqkxNeBkngtn8cYuT2eSimvooIXYo
+-> ssh-ed25519 cDBabA imJ0rXLQETELP7yo3sArhqA9nJwY+S6gkC7tA7CJsQA
+pKMHW/KDAoEj5ZD64VKekg6et9hlS2PKSgDw3eB3eu8
+-> ssh-ed25519 WY7yGw +2g5021/02HvLxLqq42ynr6qKgOKJ3J5GgB1a1bmFXg
+fYvj52R6bM6ngPOZ2lwVezTJnx+8LJBbdnaapKKbyd0
+-> ssh-ed25519 cK5kHw fLZ6yF3NggJ724rjYqhs5ZZh1xUExuK+ITAyqONluzk
+NS9OMX70XEHrbPQnmC4KB/eoiHChIb8DwDLYJiwOLUU
+-> ssh-ed25519 CAWG4Q tVduE/wMzdfS+DjNbU3Q4blNhL/A63IehNSZGJkJjD0
+jEBB5zG+gLA/88YF+KqWQsNH7lfCsWNvAkrgfbescFs
+-> ssh-ed25519 xA739A ZhFvev77I+YOl1YSHKn2ZcEvGoLjWOILufjd4q/k8HM
+YXEtHHtjPQlgZW60zHgHm7CLI6vYiRo+AM8QERL9tCg
+-> ssh-ed25519 MSF3dg 9DvLNheBU1vlfW2zNNxBrGnJ6k4P5ox7s+OGKlgRdyQ
+wseHfLGHz0huNi5sZsNOfeNkm6Kjjx0SZ8lK4/oXtUQ
+--- bnJE+14onuSla0XmckD4z/wChWGZh6exbkcbyhcmNYU
+<EFBFBD><EFBFBD>t<>N猈<><10>U<EFBFBD>w▮i2<69><32>-<2D>iV'(<1E>IF<49><46>	S<><53>xs/s<><73>	<09><>NDm<44>Q<EFBFBD><51><EFBFBD>o<EFBFBD><6F><EFBFBD><EFBFBD>wZv<7F><76>.\

secrets/gitlab-runner-docker-token.age

@@ -1,13 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg U2KQWviZIVNemm9e8h7H+eOzoYNxXgLLS3hsZLMAuGk
-6n5dH1McNzk3rscP4v2pqZYDWtUFMd15rZsEd/mqIFM
--> ssh-ed25519 cK5kHw Ebrj/cpz1cFWAYAV9OxgyyH85OEMUnfUIV66p7jaoFY
-6J7hWqODtS/fIF4BpxhxbrxZq5vbolvbLqRKqazT02M
--> ssh-ed25519 CAWG4Q mXqoQH9ycHF7u0y8mazCgynHxNLxTnrmQHke+2a5QCc
-mq6PdSF+KOqthuXwzTCsOQsi5KG0z1wHUck+bSTyOBY
--> ssh-ed25519 xA739A TADeswueqDEroZWLjMw3RDNwVQ2xRD+JUMVZENovn0M
-KFlnSjVFbjc+ZsbY8Ed7edC5B01TJGzd/dSryiLArPc
--> ssh-ed25519 MSF3dg Pq+ZD8AqJGDHDbd4PO1ngNFST8+6C2ghZkO/knKzzEc
-wyiL/u38hdQMokmfTsBrY7CtYwc+31FG4EDaqVEn31U
---- 1z4cOipayh0zYkvasEVEvGreajegE/dqBV7b6E7aFh0
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD>R<EFBFBD>@<40>/i<>I'<27><><EFBFBD>Nx<4E>r"<1D>`<1E>O<EFBFBD><4F><EFBFBD>y<><79>8<EFBFBD><38> \/<2F><>I<19><17>D<EFBFBD>`<60>ߓ<EFBFBD><DF93><EFBFBD><1E><04>uy<75><79><EFBFBD>:9Lt<4C><1D><><EFBFBD>؋<EFBFBD><D88B><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>AU<41><55><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>`<60>;<3B>q8<71>GLU#<23>i<EFBFBD>y<EFBFBD><79>i<03>ڜ
+-> ssh-ed25519 HY2yRg eHM55QsHK1ca9b5nP3EoVUZYu0w2d4B5tkilNK0j/lw
+6Na6lkMe0fOd7+vNP1fLIaVEQDUw5m65Wh8jUH1I6C0
+-> ssh-ed25519 cK5kHw 0ekhoBYwF7OSWwn4P5f/J4gXb9UHJAWGKV0yI7HCzzE
+2Q+Tt5jXAB9ip9jf1z+jeM4FSiqd1w5DNtbqtacuOcM
+-> ssh-ed25519 CAWG4Q Jmw4v9efOFXHjjNky96q/d6vGBP5dNM4wK9zoGrwOh8
+u5I17wcIq8/2ARWckDXsYckhfX0jWE4AEm5mip/KHws
+-> ssh-ed25519 xA739A 10pPeC2YG9DJzaQlt7p+fGo27VDiL2dN6JmvY2npcUw
+4aRV8DekYeL9HagGWgOSjlYnPKmYdKZH8Aw4lRdm+r8
+-> ssh-ed25519 MSF3dg hDwIE3Su6cN3sq2E5v/oy6vTNfxTT1ZPts85//gIhwY
+aoiaGjQYJB1ededhIuVBCKDRLIOVThWz1pSTvg65J3Y
+--- OYPAGb5U/nwLOIV5VchSvxhChjNnwzbEgU9glSkWCl4
+<EFBFBD>=<3D><><EFBFBD>c<EFBFBD>WȟJSaІ&<26><1F>ቧ)E<><0B> C<><43>J~u<>c<63><7F>2<EFBFBD><32>v<EFBFBD><76><EFBFBD><03><>s<EFBFBD><73><EFBFBD>vf<76><10><>X7(<28>~<7E><1A>=XCi;<3B>״<EFBFBD>\ߢ<><DFA2><EFBFBD>ܣ<EFBFBD><10><><07>ɳCe<43>D;;X*<2A>3<EFBFBD>i<EFBFBD><69>r<EFBFBD>Em<45><6D><

secrets/vpn-dac-login.age

@@ -1,14 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w SRJhNenoQXbT1FgX3TMPnVH5P6oe2eHot+M1YsEjsEk
-hfTSLgKi98Eh7JK5o7x2POpTEtQlQCpEa3keUFYCuME
--> ssh-ed25519 cK5kHw z5TwWJTkvx7HztjXHJW/aCOtOfPrQaLP0gyIT7rXcyU
-b4NCpHfasgvkLLr+6LcWUl60p59aSNnfp3bl2OFYXo0
--> ssh-ed25519 CAWG4Q 4VpS1/OnFe8nxcQbRTKNhjsh/ZQ5cbhSMXwK/jjQ+3o
-WF9wvOkqVml4UcEzyzeumKuUwCwwr2zvKLMg+PCB8nk
--> ssh-ed25519 xA739A 67FhuJ070jBVMt/xbKHWhfri6iIm0FyaFvzQabsvFBM
-1G5/913dDv/r/6p1x/c5YiUnZzrX/LvIj33KW+PN0KU
--> ssh-ed25519 MSF3dg Bj/yB4N2wkyHCHC22tcjjJAA4ebSamN0Z4UVX3ZnryI
-6D/ZgTs+j+MGDAbPU5zyK0i9zN6tQy68IcOnQZ27mYg
---- 169erk3ICSYLs4FPEuXCn7QlekWhsmSn0Lr+/R14I5Q
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD><05>ҽ3<D2BD>s<EFBFBD>
-w<EFBFBD><EFBFBD>4D<EFBFBD><EFBFBD>b.<2E><><EFBFBD>"|<7C><><EFBFBD>)"<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>;<3B>.<2E>ɫ7)<29>LeC<05>=S؟
+-> ssh-ed25519 G5LX5w /9lcJOXC9CN02+XLswUaJ0H7jU6Xhjd8Xg4+KY0l1Vc
+fCLzsLc9zrocM8SHOKyZwt6eUEr8r1WLug9RLi63KU0
+-> ssh-ed25519 cK5kHw 1qza6h2NRSs4g8LYdFU7E+Dn1CgdtCU7DPdYInP1GwM
+/6uk7pTFkNTRTI7nA+x4y4CyOBVQVXX2lnpOg3ktPe4
+-> ssh-ed25519 CAWG4Q o+vyzcejSaNVYPSGzzOdzaqPByZ6zA1uaJf4KOg+wQA
+wfZmWrDSfRV8C+Hu+SeZDcomf/qigBqxuQK77SfnuEo
+-> ssh-ed25519 xA739A +rBsOC+IBE3lmc/pfrziftLIqMSyaGMsggRjC5Pqwl0
+xa7ulLz2+YC3g2hu7e9XhRYDIUb2sriaaigJRYF2oB8
+-> ssh-ed25519 MSF3dg TK6PmKjjQt8ni0mJLCt7P41lUsgimlj3o5Q6n3N+DE4
+ne+s3ctcg8cBjY06LY2lrW7wcxomvKHxu6MlirEA8Kg
+--- eorg2ckkUZ1Ogi4iTTg2MoiVBwl1F0RCmH2D8N1d1So
+<EFBFBD><EFBFBD><EFBFBD>8<1C><><EFBFBD><EFBFBD><EFBFBD><12>i<17>$]K<>J=2Z<1D><>ӼF<D3BC>][<14><><EFBFBD>8<EFBFBD><38>ޤ<12>	<09>=<3D><>
LD/<2F>gz

secrets/wg-fox.age

@@ -1,14 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 cDBabA heyW9/cxgwFX9IexQIXjAQDWGQPNcMXcArQp2Rxsqx4
-o9MQ7EH8PDDjsJdpH9F3Xq2zUoaDAJQlfFmYucSFs6Y
--> ssh-ed25519 cK5kHw Sza4pos7K3qW3omEeyidI/jszJNf9smemSZnUJfCIww
-D6vazXki7hIYraIuSiGPS+FPbkFUwHhHWDf52OhEIMg
--> ssh-ed25519 CAWG4Q YexIHueOIMmIN8JIDyNUOKBkyz/k18HqV3hTXh48KlM
-xh8UJzzWT6ByN+Dpn4JrMNsjGC/uc/v6LynwjBDz9NQ
--> ssh-ed25519 xA739A KySG3TXdqfCMUkVEDGa74B0op745s3XGYxFLyAXSQAc
-5EI/yb5ctW9Qu18bHm3/sK97kwGcKzzmWvPSCWm89XA
--> ssh-ed25519 MSF3dg MNxnNj0fHmri8ophexXPNjRUBUWrzcuk5S1mucxUMTE
-GVFWXtISEU8ZmlwL4nh4weAgfGrt2GHX0DTzbpS6zg8
---- UdrqkYG2ZApAuwdZeNhC50NP2rkD/Ol6y8nJa4RHx7Y
-<EFBFBD>ܻ<EFBFBD>m(<28><><EFBFBD>><3E>H<48>Y87<><37>G<0F>+*<12><><EFBFBD><EFBFBD>9V<>.<2E><><EFBFBD><EFBFBD><03><><EFBFBD>p<EFBFBD>Oo<4F>=+哇<>P0<50><30>{<7B>)<29><17><><EFBFBD><EFBFBD>><3E>z3P^
-u
+-> ssh-ed25519 cDBabA So/Tqwdwd7G0PbE4RwH2qDrNcdqTkhFjF4IJrLKKpkM
+MEA5dzlUeFXm3pa+ndxrcE0ZWdO00Xf98+Q8U9LZ+cQ
+-> ssh-ed25519 cK5kHw sCHD/hHBOfMBUQXkLG3MBPNC4ebLOXW37OlF/C8FEjU
+4TFbKoy23Ic2vteXZ02fMrFxyb4NxyWaSo5I8dn48mI
+-> ssh-ed25519 CAWG4Q KYGPAXTx8H5cBC3YIBxi5B7OeF15C9rEIPFCcG0vEDw
+9LC2Zvp1Oiau1/hfPf+nJknl6BUSr+lzTn6TozZNxJg
+-> ssh-ed25519 xA739A hpvNBHPgYRtUx0HyUAdCW8s7QTmGyPXwzRHb8qYoeG0
+QkUZINY7Fr7HpyY6lbIMcP+hGO3oCmLL6N+yDN4weyk
+-> ssh-ed25519 MSF3dg P9TmEfXS+hyxsbVKja58UWAFpad0ZS3LhwrMkLnSNAY
+hiHuh7HhoYwHi2KFbCczXJoF3On9eqjD1Wsp9Q1NW/w
+--- SN3peoDvjXuD/Q4DdebQFam1CE22NyGZlMmnKyCTuX8
+s<0F><><14><>&׳֦<D7B3><D6A6><EFBFBD><EFBFBD>
}<7D>#In0&<26><1F>{<7B>1<EFBFBD><31>.