Update website

Add apex page and replace bscpkgs references for jungle after the merge.

See: rarias/jungle-website#1
Reviewed-by: Aleix Boné <abonerib@bsc.es>

.gitea/workflows/ci.yaml

@@ -0,0 +1,20 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:all:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link --print-out-paths .#bsc.ci.all
+  build:cross:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link --print-out-paths .#bsc.ci.cross

.gitignore

@@ -1,3 +1,3 @@
-*.swp
+**.swp
 /result
 /misc

.gitlab-ci.yml

@@ -0,0 +1,6 @@
+build:bsc-ci.all:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix build -L --no-link --print-out-paths .#bsc-ci.all

COPYING

@@ -0,0 +1,21 @@
+Copyright (c) 2020-2025 Barcelona Supercomputing Center
+Copyright (c) 2003-2020 Eelco Dolstra and the Nixpkgs/NixOS contributors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,9 @@
+# Jungle
+
+This repository provides two components that can be used independently:
+
+- A Nix overlay with packages used at BSC (formerly known as bscpkgs). Access
+  them directly with `nix shell .#<pkgname>`.
+
+- NixOS configurations for jungle machines. Use `nixos-rebuild switch --flake .`
+  to upgrade the current machine.

default.nix

@@ -0,0 +1,19 @@
+let
+  bscOverlay = import ./overlay.nix;
+
+  # read flake.lock and determine revision from there
+  lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+  inherit (lock.nodes.nixpkgs.locked) rev narHash;
+  fetchedNixpkgs = builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
+    sha256 = narHash;
+  };
+in
+{ overlays ? [ ]
+, nixpkgs ? fetchedNixpkgs
+, ...
+}@attrs:
+import nixpkgs (
+  (builtins.removeAttrs attrs [ "overlays" "nixpkgs" ]) //
+  { overlays = [ bscOverlay ] ++ overlays; }
+)

doc/maintainers.md

@@ -0,0 +1,30 @@
+# Maintainers
+
+## Role of a maintainer
+The responsibilities of maintainers are quite lax, and similar in spirit to
+[nixpkgs' maintainers][1]:
+
+    The main responsibility of a maintainer is to keep the packages they
+    maintain in a functioning state, and keep up with updates. In order to do
+    that, they are empowered to make decisions over the packages they maintain.
+
+    That being said, the maintainer is not alone in proposing changes to the
+    packages. Anybody (both bots and humans) can send PRs to bump or tweak the
+    package.
+
+In practice, this means that when updating or proposing changes to a package,
+we will notify maintainers by mentioning them in Gitea so they can test changes
+and give feedback.
+
+Since we do bi-yearly release cycles, there is no expectation from maintainers
+to update packages at each upstream release. Nevertheless, on each release cycle
+we may request help from maintainers when updating or testing their packages.
+
+## Becoming a maintainer
+
+
+You'll have to add yourself in the `maintainers.nix` list; your username should
+match your `bsc.es` email. Then you can add yourself to the `meta.maintainers`
+of any package you are interested in maintaining.
+
+[1]: [https://github.com/NixOS/nixpkgs/tree/nixos-25.05/maintainers]

doc/trim.sh

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Trims the jungle repository by moving the website to its own repository and
+# removing it from jungle. It also removes big pdf files and kernel
+# configurations so the jungle repository is small.
+
+set -e
+
+if [ -e oldjungle -o -e newjungle -o -e website ]; then
+  echo "remove oldjungle/, newjungle/ and website/ first"
+  exit 1
+fi
+
+# Clone the old jungle repo
+git clone gitea@tent:rarias/jungle.git oldjungle
+
+# First split the website into a new repository
+mkdir website && git -C website init -b master
+git-filter-repo \
+  --path web \
+  --subdirectory-filter web \
+  --source oldjungle \
+  --target website
+
+# Then remove the website, pdf files and big kernel configs
+mkdir newjungle && git -C newjungle init -b master
+git-filter-repo \
+  --invert-paths \
+  --path web \
+  --path-glob 'doc*.pdf' \
+  --path-glob '**/kernel/configs/lockdep' \
+  --path-glob '**/kernel/configs/defconfig' \
+  --source oldjungle \
+  --target newjungle
+
+set -x
+
+du -sh oldjungle newjungle website
+#  57M  oldjungle
+# 2,3M  newjungle
+# 6,4M  website
+
+du -sh --exclude=.git oldjungle newjungle website
+#  30M  oldjungle
+# 700K  newjungle
+# 3,5M  website

flake.lock

@@ -1,128 +1,25 @@
 {
   "nodes": {
-    "agenix": {
-      "inputs": {
-        "darwin": "darwin",
-        "home-manager": "home-manager",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1723293904,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
-        "owner": "ryantm",
-        "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
-        "type": "github"
-      },
-      "original": {
-        "owner": "ryantm",
-        "repo": "agenix",
-        "type": "github"
-      }
-    },
-    "bscpkgs": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1732868163,
-        "narHash": "sha256-qck4h298AgcNI6BnGhEwl26MTLXjumuJVr+9kak7uPo=",
-        "ref": "refs/heads/master",
-        "rev": "6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f",
-        "revCount": 952,
-        "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
-      },
-      "original": {
-        "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
-      }
-    },
-    "darwin": {
-      "inputs": {
-        "nixpkgs": [
-          "agenix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1700795494,
-        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
-        "owner": "lnl7",
-        "repo": "nix-darwin",
-        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
-        "type": "github"
-      },
-      "original": {
-        "owner": "lnl7",
-        "ref": "master",
-        "repo": "nix-darwin",
-        "type": "github"
-      }
-    },
-    "home-manager": {
-      "inputs": {
-        "nixpkgs": [
-          "agenix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1703113217,
-        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
-        "owner": "nix-community",
-        "repo": "home-manager",
-        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "home-manager",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736867362,
-        "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=",
+        "lastModified": 1752436162,
+        "narHash": "sha256-Kt1UIPi7kZqkSc5HVj6UY5YLHHEzPBkgpNUByuyxtlw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc",
+        "rev": "dfcd5b901dbab46c9c6e80b265648481aafb01f8",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "agenix": "agenix",
-        "bscpkgs": "bscpkgs",
         "nixpkgs": "nixpkgs"
       }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -1,19 +1,22 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
-    agenix.url = "github:ryantm/agenix";
-    agenix.inputs.nixpkgs.follows = "nixpkgs";
-    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
-    bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
   };
 
-  outputs = { self, nixpkgs, agenix, bscpkgs, ... }:
+  outputs = { self, nixpkgs, ... }:
 let
   mkConf = name: nixpkgs.lib.nixosSystem {
     system = "x86_64-linux";
-    specialArgs = { inherit nixpkgs bscpkgs agenix; theFlake = self; };
+    specialArgs = { inherit nixpkgs; theFlake = self; };
     modules = [ "${self.outPath}/m/${name}/configuration.nix" ];
   };
+  # For now we only support x86
+  system = "x86_64-linux";
+  pkgs = import nixpkgs {
+    inherit system;
+    overlays = [ self.overlays.default ];
+    config.allowUnfree = true;
+  };
 in
   {
     nixosConfigurations = {
@@ -27,11 +30,23 @@ in
       lake2   = mkConf "lake2";
       raccoon = mkConf "raccoon";
       fox     = mkConf "fox";
+      apex    = mkConf "apex";
+      weasel  = mkConf "weasel";
     };
 
-    packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
-      bscpkgs = bscpkgs.packages.x86_64-linux;
-      nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
+    bscOverlay = import ./overlay.nix;
+    overlays.default = self.bscOverlay;
+
+    # full nixpkgs with our overlay applied
+    legacyPackages.${system} = pkgs;
+
+    hydraJobs = self.legacyPackages.${system}.bsc.hydraJobs;
+
+    # propagate nixpkgs lib, so we can do bscpkgs.lib
+    lib = nixpkgs.lib // {
+      maintainers = nixpkgs.lib.maintainers // {
+        bsc = import ./pkgs/maintainers.nix;
+      };
     };
   };
 }

keys.nix

@@ -2,25 +2,28 @@
 # here all the public keys
 rec {
   hosts = {
-    hut   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
-    owl1  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
-    owl2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
-    eudy  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
-    koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
-    bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
-    lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
-    fox   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
-    tent  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
+    hut     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
+    owl1    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
+    owl2    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
+    eudy    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
+    koro    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
+    bay     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
+    lake2   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
+    tent    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
+    apex    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
+    weasel  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
+    raccoon = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNQttFvL0dNEyy7klIhLoK4xXOeM2/K9R7lPMTG3qvK raccoon";
   };
 
   hostGroup = with hosts; rec {
-    untrusted  = [ fox ];
-    compute    = [ owl1 owl2 ];
-    playground = [ eudy koro ];
+    compute    = [ owl1 owl2 fox raccoon ];
+    playground = [ eudy koro weasel ];
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
+    login      = [ apex ];
 
-    system     = storage ++ monitor;
+    system     = storage ++ monitor ++ login;
     safe       = system ++ compute;
     all        = safe ++ playground;
   };
@@ -28,6 +31,7 @@ rec {
   admins = {
     "rarias@hut"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
     "rarias@tent" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIwlWSBTZi74WTz5xn6gBvTmCoVltmtIAeM3RMmkh4QZ rarias@tent";
+    "rarias@fox"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDSbw3REAKECV7E2c/e2XJITudJQWq2qDSe2N1JHqHZd rarias@fox";
     root          = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
   };
 }

m/apex/configuration.nix

@@ -0,0 +1,69 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../common/ssf/hosts.nix
+    ../module/ceph.nix
+    ../module/hut-substituter.nix
+    ../module/slurm-server.nix
+    ./nfs.nix
+    ./wireguard.nix
+  ];
+
+  # Don't install grub MBR for now
+  boot.loader.grub.device = "nodev";
+
+  boot.initrd.kernelModules = [
+    "megaraid_sas" # For HW RAID
+  ];
+
+  environment.systemPackages = with pkgs; [
+    storcli # To manage HW RAID
+  ];
+
+  fileSystems."/home" = {
+    device = "/dev/disk/by-label/home";
+    fsType = "ext4";
+  };
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  networking = {
+    hostName = "apex";
+    defaultGateway = "84.88.53.233";
+    nameservers = [ "8.8.8.8" ];
+
+    # Public facing interface
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "84.88.53.236";
+      prefixLength = 29;
+    } ];
+
+    # Internal LAN to our Ethernet switch
+    interfaces.eno2.ipv4.addresses = [ {
+      address = "10.0.40.30";
+      prefixLength = 24;
+    } ];
+
+    # Infiniband over Omnipath switch (disconnected for now)
+    # interfaces.ibp5s0 = {};
+
+    nat = {
+      enable = true;
+      internalInterfaces = [ "eno2" ];
+      externalInterface = "eno1";
+    };
+  };
+
+  networking.firewall = {
+    extraCommands = ''
+      # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our
+      # logs. Insert as first position so we also protect SSH.
+      iptables -I nixos-fw 1 -p tcp -s 192.168.8.16 -j nixos-fw-refuse
+      # Same with opsmonweb01.bsc.es which seems to be trying to access via SSH
+      iptables -I nixos-fw 2 -p tcp -s 84.88.52.176 -j nixos-fw-refuse
+    '';
+  };
+}

m/apex/nfs.nix

@@ -0,0 +1,48 @@
+{ ... }:
+
+{
+  services.nfs.server = {
+    enable = true;
+    lockdPort = 4001;
+    mountdPort = 4002;
+    statdPort = 4000;
+    exports = ''
+      /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+      /home 10.106.0.0/24(rw,async,no_subtree_check,no_root_squash)
+    '';
+  };
+  networking.firewall = {
+    # Check with `rpcinfo -p`
+    extraCommands = ''
+      # Accept NFS traffic from compute nodes but not from the outside
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+
+      # Accept NFS traffic from wg0
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -i wg0 -s 10.106.0.0/24 --dport 20048 -j nixos-fw-accept
+    '';
+  };
+}

m/apex/wireguard.nix

@@ -0,0 +1,42 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgApex.file = ../../secrets/wg-apex.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      ips = [ "10.106.0.30/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgApex.path;
+      # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=
+      peers = [
+        {
+          name = "fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.1/32" ];
+          endpoint = "fox.ac.upc.edu:666";
+          # Send keepalives every 25 seconds. Important to keep NAT tables alive.
+          persistentKeepalive = 25;
+        }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1" = [ "fox" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+}

m/bay/configuration.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ../common/ssf.nix
+    ../module/hut-substituter.nix
     ../module/monitoring.nix
   ];
 

m/common/base.nix

@@ -3,6 +3,7 @@
   # Includes the basic configuration for an Intel server.
   imports = [
     ./base/agenix.nix
+    ./base/always-power-on.nix
     ./base/august-shutdown.nix
     ./base/boot.nix
     ./base/env.nix
@@ -10,6 +11,7 @@
     ./base/hw.nix
     ./base/net.nix
     ./base/nix.nix
+    ./base/sys-devices.nix
     ./base/ntp.nix
     ./base/rev.nix
     ./base/ssh.nix

m/common/base/agenix.nix

@@ -1,9 +1,8 @@
-{ agenix, ... }:
+{ pkgs, ... }:
 
 {
-  imports = [ agenix.nixosModules.default ];
+  imports = [ ../../module/agenix.nix ];
 
-  environment.systemPackages = [
-    agenix.packages.x86_64-linux.default
-  ];
+  # Add agenix to system packages
+  environment.systemPackages = [ pkgs.agenix ];
 }

m/common/base/always-power-on.nix

@@ -0,0 +1,8 @@
+{
+  imports = [
+    ../../module/power-policy.nix
+  ];
+
+  # Turn on as soon as we have power
+  power.policy = "always-on";
+}

m/common/base/august-shutdown.nix

@@ -1,12 +1,12 @@
 {
-  # Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
+  # Shutdown all machines on August 3rd at 22:00, so we can protect the
   # hardware from spurious electrical peaks on the yearly electrical cut for
   # manteinance that starts on August 4th.
   systemd.timers.august-shutdown = {
-    description = "Shutdown on August 2nd for maintenance";
+    description = "Shutdown on August 3rd for maintenance";
     wantedBy = [ "timers.target" ];
     timerConfig = {
-      OnCalendar = "*-08-02 11:00:00";
+      OnCalendar = "*-08-03 22:00:00";
       RandomizedDelaySec = "10min";
       Unit = "systemd-poweroff.service";
     };

m/common/base/env.nix

@@ -3,8 +3,8 @@
 {
   environment.systemPackages = with pkgs; [
     vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
+    nix-diff ipmitool freeipmi ethtool lm_sensors cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns pv
     # From bsckgs overlay
     osumb
   ];

m/common/base/net.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, lib, ... }:
 
 {
   networking = {
@@ -10,10 +10,14 @@
       allowedTCPPorts = [ 22 ];
     };
 
+    # Make sure we use iptables
+    nftables.enable = lib.mkForce false;
+
     hosts = {
       "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
-      "84.88.51.152" = [ "raccoon" ];
       "84.88.51.142" = [ "raccoon-ipmi" ];
+      "192.168.11.12" = [ "bscpm04.bsc.es" ];
+      "192.168.11.15" = [ "gitlab-internal.bsc.es" ];
     };
   };
 }

m/common/base/nix.nix

@@ -1,11 +1,12 @@
-{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
+{ pkgs, nixpkgs, theFlake,  ... }:
 
 {
   nixpkgs.overlays = [
-    bscpkgs.bscOverlay
-    (import ../../../pkgs/overlay.nix)
+    (import ../../../overlay.nix)
   ];
 
+  nixpkgs.config.allowUnfree = true;
+
   nix = {
     nixPath = [
       "nixpkgs=${nixpkgs}"

m/common/base/sys-devices.nix

@@ -0,0 +1,9 @@
+{
+  nix.settings.system-features = [ "sys-devices" ];
+
+  programs.nix-required-mounts.enable = true;
+  programs.nix-required-mounts.allowedPatterns.sys-devices.paths = [
+    "/sys/devices/system/cpu"
+    "/sys/devices/system/node"
+  ];
+}

m/common/base/users.nix

@@ -56,7 +56,7 @@
         home = "/home/Computational/rpenacob";
         description = "Raúl Peñacoba";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "tent" "fox" ];
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "fox" ];
         hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
@@ -69,10 +69,10 @@
         home = "/home/Computational/anavarro";
         description = "Antoni Navarro";
         group = "Computational";
-        hosts = [ "hut" "tent" "raccoon" "fox" ];
-        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
+        hosts = [ "apex" "hut" "tent" "raccoon" "fox" "weasel" ];
+        hashedPassword = "$6$EgturvVYXlKgP43g$gTN78LLHIhaF8hsrCXD.O6mKnZSASWSJmCyndTX8QBWT6wTlUhcWVAKz65lFJPXjlJA4u7G1ydYQ0GG6Wk07b1";
         openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMsbM21uepnJwPrRe6jYFz8zrZ6AYMtSEvvt4c9spmFP toni@delltoni"
         ];
       };
 
@@ -82,7 +82,7 @@
         home = "/home/Computational/abonerib";
         description = "Aleix Boné";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "tent" "raccoon" "fox" ];
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "raccoon" "fox" "weasel" ];
         hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
@@ -95,7 +95,7 @@
         home = "/home/Computational/vlopez";
         description = "Victor López";
         group = "Computational";
-        hosts = [ "koro" ];
+        hosts = [ "apex" "koro" ];
         hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
@@ -108,7 +108,7 @@
         home = "/home/Computational/dbautist";
         description = "Dylan Bautista Cases";
         group = "Computational";
-        hosts = [ "hut" "tent" "raccoon" ];
+        hosts = [ "apex" "hut" "tent" "raccoon" ];
         hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
@@ -121,7 +121,7 @@
         home = "/home/Computational/dalvare1";
         description = "David Álvarez";
         group = "Computational";
-        hosts = [ "hut" "tent" "fox" ];
+        hosts = [ "apex" "hut" "tent" "fox" ];
         hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
@@ -134,7 +134,7 @@
         home = "/home/Computational/varcila";
         description = "Vincent Arcila";
         group = "Computational";
-        hosts = [ "hut" "tent" "fox" ];
+        hosts = [ "apex" "hut" "tent" "fox" ];
         hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
@@ -154,6 +154,45 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
         ];
       };
+
+      csiringo = {
+        uid = 9653;
+        isNormalUser = true;
+        home = "/home/Computational/csiringo";
+        description = "Cesare Siringo";
+        group = "Computational";
+        hosts = [ ];
+        hashedPassword = "$6$0IsZlju8jFukLlAw$VKm0FUXbS.mVmPm3rcJeizTNU4IM5Nmmy21BvzFL.cQwvlGwFI1YWRQm6gsbd4nbg47mPDvYkr/ar0SlgF6GO1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHA65zvvG50iuFEMf+guRwZB65jlGXfGLF4HO+THFaed csiringo@bsc.es"
+        ];
+      };
+
+      acinca = {
+        uid = 9654;
+        isNormalUser = true;
+        home = "/home/Computational/acinca";
+        description = "Arnau Cinca";
+        group = "Computational";
+        hosts = [ "apex" "hut" "fox" "owl1" "owl2" ];
+        hashedPassword = "$6$S6PUeRpdzYlidxzI$szyvWejQ4hEN76yBYhp1diVO5ew1FFg.cz4lKiXt2Idy4XdpifwrFTCIzLTs5dvYlR62m7ekA5MrhcVxR5F/q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFmMqKqPg4uocNOr3O41kLbZMOMJn3m2ZdN1JvTR96z3 bsccns@arnau-bsc"
+        ];
+      };
+
+      aaguirre = {
+        uid = 9655;
+        isNormalUser = true;
+        home = "/home/Computational/aaguirre";
+        description = "Alejandro Aguirre";
+        group = "Computational";
+        hosts = [ "apex" "hut" ];
+        hashedPassword = "$6$TXRXQT6jjBvxkxU6$E.sh5KspAm1qeG5Ct7OPHpo8REmbGDwjFGvqeGgTVz3GASGOAnPL7UMZsMAsAKBoahOw.v8LNno6XGrTEPzZH1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOlRX7ZCnqtUJYCxKgWmgSrFCYuA2LHY96rVwqxXPl86 aaguirre@BSC-8488184117"
+        ];
+      };
     };
 
     groups = {

m/common/ssf.nix

@@ -3,7 +3,8 @@
   imports = [
     ./xeon.nix
     ./ssf/fs.nix
+    ./ssf/hosts.nix
+    ./ssf/hosts-remote.nix
     ./ssf/net.nix
-    ./ssf/ssh.nix
   ];
 }

m/common/ssf/hosts-remote.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Remote hosts visible from compute nodes
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+}

m/common/ssf/hosts.nix

@@ -0,0 +1,23 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Login
+    "10.0.40.30" = [ "apex" ];
+
+    # Storage
+    "10.0.40.40" = [ "bay" ];   "10.0.42.40" = [ "bay-ib" ];    "10.0.40.141" = [ "bay-ipmi" ];
+    "10.0.40.41" = [ "oss01" ]; "10.0.42.41" = [ "oss01-ib0" ]; "10.0.40.142" = [ "oss01-ipmi" ];
+    "10.0.40.42" = [ "lake2" ]; "10.0.42.42" = [ "lake2-ib" ];  "10.0.40.143" = [ "lake2-ipmi" ];
+
+    # Xeon compute
+    "10.0.40.1" = [ "owl1" ];   "10.0.42.1" = [ "owl1-ib" ];   "10.0.40.101" = [ "owl1-ipmi" ];
+    "10.0.40.2" = [ "owl2" ];   "10.0.42.2" = [ "owl2-ib" ];   "10.0.40.102" = [ "owl2-ipmi" ];
+    "10.0.40.3" = [ "xeon03" ]; "10.0.42.3" = [ "xeon03-ib" ]; "10.0.40.103" = [ "xeon03-ipmi" ];
+    #"10.0.40.4" = [ "tent" ];   "10.0.42.4" = [ "tent-ib" ];   "10.0.40.104" = [ "tent-ipmi" ];
+    "10.0.40.5" = [ "koro" ];   "10.0.42.5" = [ "koro-ib" ];   "10.0.40.105" = [ "koro-ipmi" ];
+    "10.0.40.6" = [ "weasel" ]; "10.0.42.6" = [ "weasel-ib" ]; "10.0.40.106" = [ "weasel-ipmi" ];
+    "10.0.40.7" = [ "hut" ];    "10.0.42.7" = [ "hut-ib" ];    "10.0.40.107" = [ "hut-ipmi" ];
+    "10.0.40.8" = [ "eudy" ];   "10.0.42.8" = [ "eudy-ib" ];   "10.0.40.108" = [ "eudy-ipmi" ];
+  };
+}

m/common/ssf/net.nix

@@ -9,14 +9,6 @@
     defaultGateway = "10.0.40.30";
     nameservers = ["8.8.8.8"];
 
-    proxy = {
-      default = "http://hut:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40,hut";
-      # Don't set all_proxy as go complains and breaks the gitlab runner, see:
-      # https://github.com/golang/go/issues/16715
-      allProxy = null;
-    };
-
     firewall = {
       extraCommands = ''
         # Prevent ssfhead from contacting our slurmd daemon
@@ -27,64 +19,5 @@
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
       '';
     };
-
-    extraHosts = ''
-      10.0.40.30              ssfhead
-      
-      # Node Entry for node: mds01 (ID=72)
-      10.0.40.40              bay mds01 mds01-eth0
-      10.0.42.40              bay-ib mds01-ib0
-      10.0.40.141             bay-ipmi mds01-ipmi0 mds01-ipmi
-      
-      # Node Entry for node: oss01 (ID=73)
-      10.0.40.41              oss01 oss01-eth0
-      10.0.42.41              oss01-ib0
-      10.0.40.142             oss01-ipmi0 oss01-ipmi
-      
-      # Node Entry for node: oss02 (ID=74)
-      10.0.40.42              lake2 oss02 oss02-eth0
-      10.0.42.42              lake2-ib oss02-ib0
-      10.0.40.143             lake2-ipmi oss02-ipmi0 oss02-ipmi
-      
-      # Node Entry for node: xeon01 (ID=15)
-      10.0.40.1               owl1 xeon01 xeon01-eth0
-      10.0.42.1               owl1-ib xeon01-ib0
-      10.0.40.101             owl1-ipmi xeon01-ipmi0 xeon01-ipmi
-      
-      # Node Entry for node: xeon02 (ID=16)
-      10.0.40.2               owl2 xeon02 xeon02-eth0
-      10.0.42.2               owl2-ib xeon02-ib0
-      10.0.40.102             owl2-ipmi xeon02-ipmi0 xeon02-ipmi
-      
-      # Node Entry for node: xeon03 (ID=17)
-      10.0.40.3               xeon03 xeon03-eth0
-      10.0.42.3               xeon03-ib0
-      10.0.40.103             xeon03-ipmi0 xeon03-ipmi
-      
-      # Node Entry for node: xeon04 (ID=18)
-      10.0.40.4               xeon04 xeon04-eth0
-      10.0.42.4               xeon04-ib0
-      10.0.40.104             xeon04-ipmi0 xeon04-ipmi
-      
-      # Node Entry for node: xeon05 (ID=19)
-      10.0.40.5               koro xeon05 xeon05-eth0
-      10.0.42.5               koro-ib xeon05-ib0
-      10.0.40.105             koro-ipmi xeon05-ipmi0
-      
-      # Node Entry for node: xeon06 (ID=20)
-      10.0.40.6               xeon06 xeon06-eth0
-      10.0.42.6               xeon06-ib0
-      10.0.40.106             xeon06-ipmi0 xeon06-ipmi
-      
-      # Node Entry for node: xeon07 (ID=21)
-      10.0.40.7               hut xeon07 xeon07-eth0
-      10.0.42.7               hut-ib xeon07-ib0
-      10.0.40.107             hut-ipmi xeon07-ipmi0 xeon07-ipmi
-      
-      # Node Entry for node: xeon08 (ID=22)
-      10.0.40.8               eudy xeon08 xeon08-eth0
-      10.0.42.8               eudy-ib xeon08-ib0
-      10.0.40.108             eudy-ipmi xeon08-ipmi0 xeon08-ipmi
-    '';
   };
 }

m/common/ssf/ssh.nix

@@ -1,8 +0,0 @@
-{
-  # Connect to intranet git hosts via proxy
-  programs.ssh.extraConfig = ''
-    # Connect to BSC machines via hut proxy too
-    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-      ProxyCommand nc -X connect -x hut:23080 %h %p
-  '';
-}

m/eudy/configuration.nix

@@ -9,6 +9,7 @@
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
+    ../module/hut-substituter.nix
     ../module/debuginfod.nix
   ];
 

m/fox/configuration.nix

@@ -4,9 +4,18 @@
   imports = [
     ../common/base.nix
     ../common/xeon/console.nix
+    ../module/amd-uprof.nix
     ../module/emulation.nix
+    ../module/nvidia.nix
+    ../module/slurm-client.nix
+    ../module/hut-substituter.nix
+    ./wireguard.nix
   ];
 
+  # Don't turn off on August as UPC has different dates.
+  # Fox works fine on power cuts.
+  systemd.timers.august-shutdown.enable = false;
+
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
 
@@ -14,7 +23,7 @@
   swapDevices = lib.mkForce [];
 
   boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
-  boot.kernelModules = [ "kvm-amd" "amd_uncore" ];
+  boot.kernelModules = [ "kvm-amd" "amd_uncore" "amd_hsmp" ];
 
   hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
   hardware.cpu.intel.updateMicrocode = lib.mkForce false;
@@ -22,14 +31,21 @@
   # Use performance for benchmarks
   powerManagement.cpuFreqGovernor = "performance";
 
+  services.amd-uprof.enable = true;
+
   # Disable NUMA balancing
   boot.kernel.sysctl."kernel.numa_balancing" = 0;
 
   # Expose kernel addresses
   boot.kernel.sysctl."kernel.kptr_restrict" = 0;
 
+  # Disable NMI watchdog to save one hw counter (for AMD uProf)
+  boot.kernel.sysctl."kernel.nmi_watchdog" = 0;
+
   services.openssh.settings.X11Forwarding = true;
 
+  services.fail2ban.enable = true;
+
   networking = {
     timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
     hostName = "fox";
@@ -47,23 +63,20 @@
     interfaces.enp1s0f0np0.useDHCP = true;
   };
 
-  # Use hut for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
-
-  # Configure Nvidia driver to use with CUDA
-  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
-  hardware.graphics.enable = true;
-  nixpkgs.config.allowUnfree = true;
-  nixpkgs.config.nvidia.acceptLicense = true;
-  services.xserver.videoDrivers = [ "nvidia" ];
+  # Recommended for new graphics cards
+  hardware.nvidia.open = true;
 
   # Mount NVME disks
   fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
   fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
 
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
   # Make a /nvme{0,1}/$USER directory for each user.
   systemd.services.create-nvme-dirs = let
     # Take only normal users in fox
@@ -80,4 +93,20 @@
     wantedBy = [ "multi-user.target" ];
     serviceConfig.ExecStart = script;
   };
+
+  # Only allow SSH connections from users who have a SLURM allocation
+  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
+  security.pam.services.sshd.rules.account.slurm = {
+    control = "required";
+    enable = true;
+    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
+    args = [ "log_level=debug5" ];
+    order = 999999; # Make it last one
+  };
+
+  # Disable systemd session (pam_systemd.so) as it will conflict with the
+  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
+  # into the slurmstepd task and then into the systemd session, which is not
+  # what we want, otherwise it will linger even if all jobs are gone.
+  security.pam.services.sshd.startSession = lib.mkForce false;
 }

m/fox/wireguard.nix

@@ -0,0 +1,54 @@
+{ config, ... }:
+
+{
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgFox.file = ../../secrets/wg-fox.age;
+
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    # "wg0" is the network interface name. You can name the interface arbitrarily.
+    wg0 = {
+      # Determines the IP address and subnet of the server's end of the tunnel interface.
+      ips = [ "10.106.0.1/24" ];
+
+      # The port that WireGuard listens to. Must be accessible by the client.
+      listenPort = 666;
+
+      # Path to the private key file.
+      privateKeyFile = config.age.secrets.wgFox.path;
+      # Public key: VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=
+
+      peers = [
+        # List of allowed peers.
+        {
+          name = "apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing.
+          allowedIPs = [ "10.106.0.30/32" "10.0.40.7/32" ];
+        }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.30" = [ "apex" ];
+    "10.0.40.7" = [ "hut" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to slurmd from apex (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.30/32 -d 10.106.0.1/32 --dport 6818 -j nixos-fw-accept
+    '';
+  };
+}

m/hut/blackbox.yml

@@ -3,160 +3,12 @@ modules:
     prober: http
     timeout: 5s
     http:
-      proxy_url: "http://127.0.0.1:23080"
-      skip_resolve_phase_with_proxy: true
-      follow_redirects: true
-      valid_status_codes: []  # Defaults to 2xx
-      method: GET
-  http_with_proxy:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      skip_resolve_phase_with_proxy: true
-  http_with_proxy_and_headers:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      proxy_connect_header:
-        Proxy-Authorization:
-          - Bearer token
-  http_post_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Content-Type: application/json
-      body: '{}'
-  http_post_body_file:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      body_file: "/files/body.txt"
-  http_basic_auth_example:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Host: "login.example.com"
-      basic_auth:
-        username: "username"
-        password: "mysecret"
-  http_2xx_oauth_client_credentials:
-    prober: http
-    timeout: 5s
-    http:
-      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
       follow_redirects: true
       preferred_ip_protocol: "ip4"
-      valid_status_codes:
-        - 200
-        - 201
-      oauth2:
-        client_id: "client_id"
-        client_secret: "client_secret"
-        token_url: "https://api.example.com/token"
-        endpoint_params:
-          grant_type: "client_credentials"
-  http_custom_ca_example:
-    prober: http
-    http:
+      valid_status_codes: []  # Defaults to 2xx
       method: GET
-      tls_config:
-        ca_file: "/certs/my_cert.crt"
-  http_gzip:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-  http_gzip_with_accept_encoding:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-      headers:
-        Accept-Encoding: gzip
-  tls_connect:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      tls: true
-  tcp_connect_example:
-    prober: tcp
-    timeout: 5s
-  imap_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "OK.*STARTTLS"
-        - send: ". STARTTLS"
-        - expect: "OK"
-        - starttls: true
-        - send: ". capability"
-        - expect: "CAPABILITY IMAP4rev1"
-  smtp_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "^220 ([^ ]+) ESMTP (.+)$"
-        - send: "EHLO prober\r"
-        - expect: "^250-STARTTLS"
-        - send: "STARTTLS\r"
-        - expect: "^220"
-        - starttls: true
-        - send: "EHLO prober\r"
-        - expect: "^250-AUTH"
-        - send: "QUIT\r"
-  irc_banner_example:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - send: "NICK prober"
-        - send: "USER prober prober prober :prober"
-        - expect: "PING :([^ ]+)"
-          send: "PONG ${1}"
-        - expect: "^:[^ ]+ 001"
   icmp:
     prober: icmp
     timeout: 5s
     icmp:
       preferred_ip_protocol: "ip4"
-  dns_udp_example:
-    prober: dns
-    timeout: 5s
-    dns:
-      query_name: "www.prometheus.io"
-      query_type: "A"
-      valid_rcodes:
-        - NOERROR
-      validate_answer_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-        fail_if_all_match_regexp:
-          - ".*127.0.0.1"
-        fail_if_not_matches_regexp:
-          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
-        fail_if_none_matches_regexp:
-          - "127.0.0.1"
-      validate_authority_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-      validate_additional_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-  dns_soa:
-    prober: dns
-    dns:
-      query_name: "prometheus.io"
-      query_type: "SOA"
-  dns_tcp_example:
-    prober: dns
-    dns:
-      transport_protocol: "tcp" # defaults to "udp"
-      preferred_ip_protocol: "ip4" # defaults to "ip6"
-      query_name: "www.prometheus.io"

m/hut/configuration.nix

@@ -7,11 +7,9 @@
     ../module/ceph.nix
     ../module/debuginfod.nix
     ../module/emulation.nix
-    ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm-server.nix
     ./nix-serve.nix
     ./public-inbox.nix
     ./gitea.nix

m/hut/gpfs-probe.sh

@@ -2,20 +2,10 @@
 
 N=500
 
-t_proj=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
-t_scratch=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/scratch/bsc15/rodrigo/probe/gpfs.{1..$N} 2>&1; rm -f /gpfs/scratch/bsc15/rodrigo/probe/gpfs.{1..$N}")
-t_home=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /home/bsc/bsc015557/.gpfs/{1..$N} 2>&1; rm -f /home/bsc/bsc015557/.gpfs/{1..$N}")
+t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
 
-if [ -z "$t_proj" ]; then
-  t_proj="5.00"
-fi
-
-if [ -z "$t_scratch" ]; then
-  t_scratch="5.00"
-fi
-
-if [ -z "$t_home" ]; then
-  t_home="5.00"
+if [ -z "$t" ]; then
+  t="5.00"
 fi
 
 cat <<EOF
@@ -24,7 +14,5 @@ Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
 
 # HELP gpfs_touch_latency Time to create $N files.
 # TYPE gpfs_touch_latency gauge
-gpfs_touch_latency{partition="projects"} $t_proj
-gpfs_touch_latency{partition="home"} $t_home
-gpfs_touch_latency{partition="scratch"} $t_scratch
+gpfs_touch_latency $t
 EOF

m/hut/monitoring.nix

@@ -267,14 +267,6 @@
           }
         ];
       }
-      {
-        job_name = "tent";
-        static_configs = [
-          {
-            targets = [ "127.0.0.1:29002" ]; # Node exporter
-          }
-        ];
-      }
     ];
   };
 }

m/hut/nginx.nix

@@ -2,10 +2,13 @@
 let
   website = pkgs.stdenv.mkDerivation {
     name = "jungle-web";
-    src = theFlake;
+    src = pkgs.fetchgit {
+      url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
+      rev = "52abaf4d71652a9ef77a0b098db14ca33bffff4c";
+      hash = "sha256-/ul9GazbOrOkmlvSgDz/+2W+V+ir5725Y7mVLc3rb0M=";
+    };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''
-      cd web
       rm -rf public/
       hugo
     '';

m/hut/slurm-server.nix

@@ -1,7 +0,0 @@
-{ ... }:
-
-{
-  services.slurm = {
-    server.enable = true;
-  };
-}

m/hut/targets.yml

@@ -4,7 +4,7 @@
   - xeon03-ipmi
   - xeon04-ipmi
   - koro-ipmi
-  - xeon06-ipmi
+  - weasel-ipmi
   - hut-ipmi
   - eudy-ipmi
   # Storage

m/lake2/configuration.nix

@@ -4,6 +4,7 @@
   imports = [
     ../common/ssf.nix
     ../module/monitoring.nix
+    ../module/hut-substituter.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";

m/map.nix

@@ -6,7 +6,7 @@
     switch-opa = { pos=41; size=1; };
 
     # SSF login
-    ssfhead = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="operations@bsc.es"; };
+    apex = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="rodrigo.arias@bsc.es"; };
 
     # Storage
     bay   = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };
@@ -19,7 +19,7 @@
     xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; };
     # Slot 34 empty
     koro   = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; };
-    xeon06 = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
+    weasel = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
     hut    = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; };
     eudy   = { pos=28; size=1; label="SSF-XEON08"; board="S2600WTTR"; sn="BQWL64756586"; contact="aleix.rocanonell@bsc.es"; };
 

m/module/agenix.nix

@@ -0,0 +1,357 @@
+{
+  config,
+  options,
+  lib,
+  pkgs,
+  ...
+}:
+with lib;
+let
+  cfg = config.age;
+
+  isDarwin = lib.attrsets.hasAttrByPath [ "environment" "darwinConfig" ] options;
+
+  ageBin = config.age.ageBin;
+
+  users = config.users.users;
+
+  sysusersEnabled =
+    if isDarwin then
+      false
+    else
+      options.systemd ? sysusers && (config.systemd.sysusers.enable || config.services.userborn.enable);
+
+  mountCommand =
+    if isDarwin then
+      ''
+        if ! diskutil info "${cfg.secretsMountPoint}" &> /dev/null; then
+            num_sectors=1048576
+            dev=$(hdiutil attach -nomount ram://"$num_sectors" | sed 's/[[:space:]]*$//')
+            newfs_hfs -v agenix "$dev"
+            mount -t hfs -o nobrowse,nodev,nosuid,-m=0751 "$dev" "${cfg.secretsMountPoint}"
+        fi
+      ''
+    else
+      ''
+        grep -q "${cfg.secretsMountPoint} ramfs" /proc/mounts ||
+          mount -t ramfs none "${cfg.secretsMountPoint}" -o nodev,nosuid,mode=0751
+      '';
+  newGeneration = ''
+    _agenix_generation="$(basename "$(readlink ${cfg.secretsDir})" || echo 0)"
+    (( ++_agenix_generation ))
+    echo "[agenix] creating new generation in ${cfg.secretsMountPoint}/$_agenix_generation"
+    mkdir -p "${cfg.secretsMountPoint}"
+    chmod 0751 "${cfg.secretsMountPoint}"
+    ${mountCommand}
+    mkdir -p "${cfg.secretsMountPoint}/$_agenix_generation"
+    chmod 0751 "${cfg.secretsMountPoint}/$_agenix_generation"
+  '';
+
+  chownGroup = if isDarwin then "admin" else "keys";
+  # chown the secrets mountpoint and the current generation to the keys group
+  # instead of leaving it root:root.
+  chownMountPoint = ''
+    chown :${chownGroup} "${cfg.secretsMountPoint}" "${cfg.secretsMountPoint}/$_agenix_generation"
+  '';
+
+  setTruePath = secretType: ''
+    ${
+      if secretType.symlink then
+        ''
+          _truePath="${cfg.secretsMountPoint}/$_agenix_generation/${secretType.name}"
+        ''
+      else
+        ''
+          _truePath="${secretType.path}"
+        ''
+    }
+  '';
+
+  installSecret = secretType: ''
+    ${setTruePath secretType}
+    echo "decrypting '${secretType.file}' to '$_truePath'..."
+    TMP_FILE="$_truePath.tmp"
+
+    IDENTITIES=()
+    for identity in ${toString cfg.identityPaths}; do
+      test -r "$identity" || continue
+      test -s "$identity" || continue
+      IDENTITIES+=(-i)
+      IDENTITIES+=("$identity")
+    done
+
+    test "''${#IDENTITIES[@]}" -eq 0 && echo "[agenix] WARNING: no readable identities found!"
+
+    mkdir -p "$(dirname "$_truePath")"
+    [ "${secretType.path}" != "${cfg.secretsDir}/${secretType.name}" ] && mkdir -p "$(dirname "${secretType.path}")"
+    (
+      umask u=r,g=,o=
+      test -f "${secretType.file}" || echo '[agenix] WARNING: encrypted file ${secretType.file} does not exist!'
+      test -d "$(dirname "$TMP_FILE")" || echo "[agenix] WARNING: $(dirname "$TMP_FILE") does not exist!"
+      LANG=${
+        config.i18n.defaultLocale or "C"
+      } ${ageBin} --decrypt "''${IDENTITIES[@]}" -o "$TMP_FILE" "${secretType.file}"
+    )
+    chmod ${secretType.mode} "$TMP_FILE"
+    mv -f "$TMP_FILE" "$_truePath"
+
+    ${optionalString secretType.symlink ''
+      [ "${secretType.path}" != "${cfg.secretsDir}/${secretType.name}" ] && ln -sfT "${cfg.secretsDir}/${secretType.name}" "${secretType.path}"
+    ''}
+  '';
+
+  testIdentities = map (path: ''
+    test -f ${path} || echo '[agenix] WARNING: config.age.identityPaths entry ${path} not present!'
+  '') cfg.identityPaths;
+
+  cleanupAndLink = ''
+    _agenix_generation="$(basename "$(readlink ${cfg.secretsDir})" || echo 0)"
+    (( ++_agenix_generation ))
+    echo "[agenix] symlinking new secrets to ${cfg.secretsDir} (generation $_agenix_generation)..."
+    ln -sfT "${cfg.secretsMountPoint}/$_agenix_generation" ${cfg.secretsDir}
+
+    (( _agenix_generation > 1 )) && {
+    echo "[agenix] removing old secrets (generation $(( _agenix_generation - 1 )))..."
+    rm -rf "${cfg.secretsMountPoint}/$(( _agenix_generation - 1 ))"
+    }
+  '';
+
+  installSecrets = builtins.concatStringsSep "\n" (
+    [ "echo '[agenix] decrypting secrets...'" ]
+    ++ testIdentities
+    ++ (map installSecret (builtins.attrValues cfg.secrets))
+    ++ [ cleanupAndLink ]
+  );
+
+  chownSecret = secretType: ''
+    ${setTruePath secretType}
+    chown ${secretType.owner}:${secretType.group} "$_truePath"
+  '';
+
+  chownSecrets = builtins.concatStringsSep "\n" (
+    [ "echo '[agenix] chowning...'" ]
+    ++ [ chownMountPoint ]
+    ++ (map chownSecret (builtins.attrValues cfg.secrets))
+  );
+
+  secretType = types.submodule (
+    { config, ... }:
+    {
+      options = {
+        name = mkOption {
+          type = types.str;
+          default = config._module.args.name;
+          defaultText = literalExpression "config._module.args.name";
+          description = ''
+            Name of the file used in {option}`age.secretsDir`
+          '';
+        };
+        file = mkOption {
+          type = types.path;
+          description = ''
+            Age file the secret is loaded from.
+          '';
+        };
+        path = mkOption {
+          type = types.str;
+          default = "${cfg.secretsDir}/${config.name}";
+          defaultText = literalExpression ''
+            "''${cfg.secretsDir}/''${config.name}"
+          '';
+          description = ''
+            Path where the decrypted secret is installed.
+          '';
+        };
+        mode = mkOption {
+          type = types.str;
+          default = "0400";
+          description = ''
+            Permissions mode of the decrypted secret in a format understood by chmod.
+          '';
+        };
+        owner = mkOption {
+          type = types.str;
+          default = "0";
+          description = ''
+            User of the decrypted secret.
+          '';
+        };
+        group = mkOption {
+          type = types.str;
+          default = users.${config.owner}.group or "0";
+          defaultText = literalExpression ''
+            users.''${config.owner}.group or "0"
+          '';
+          description = ''
+            Group of the decrypted secret.
+          '';
+        };
+        symlink = mkEnableOption "symlinking secrets to their destination" // {
+          default = true;
+        };
+      };
+    }
+  );
+in
+{
+  imports = [
+    (mkRenamedOptionModule [ "age" "sshKeyPaths" ] [ "age" "identityPaths" ])
+  ];
+
+  options.age = {
+    ageBin = mkOption {
+      type = types.str;
+      default = "${pkgs.age}/bin/age";
+      defaultText = literalExpression ''
+        "''${pkgs.age}/bin/age"
+      '';
+      description = ''
+        The age executable to use.
+      '';
+    };
+    secrets = mkOption {
+      type = types.attrsOf secretType;
+      default = { };
+      description = ''
+        Attrset of secrets.
+      '';
+    };
+    secretsDir = mkOption {
+      type = types.path;
+      default = "/run/agenix";
+      description = ''
+        Folder where secrets are symlinked to
+      '';
+    };
+    secretsMountPoint = mkOption {
+      type =
+        types.addCheck types.str (
+          s:
+          (builtins.match "[ \t\n]*" s) == null # non-empty
+          && (builtins.match ".+/" s) == null
+        ) # without trailing slash
+        // {
+          description = "${types.str.description} (with check: non-empty without trailing slash)";
+        };
+      default = "/run/agenix.d";
+      description = ''
+        Where secrets are created before they are symlinked to {option}`age.secretsDir`
+      '';
+    };
+    identityPaths = mkOption {
+      type = types.listOf types.path;
+      default =
+        if isDarwin then
+          [
+            "/etc/ssh/ssh_host_ed25519_key"
+            "/etc/ssh/ssh_host_rsa_key"
+          ]
+        else if (config.services.openssh.enable or false) then
+          map (e: e.path) (
+            lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys
+          )
+        else
+          [ ];
+      defaultText = literalExpression ''
+        if isDarwin
+        then [
+          "/etc/ssh/ssh_host_ed25519_key"
+          "/etc/ssh/ssh_host_rsa_key"
+        ]
+        else if (config.services.openssh.enable or false)
+        then map (e: e.path) (lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys)
+        else [];
+      '';
+      description = ''
+        Path to SSH keys to be used as identities in age decryption.
+      '';
+    };
+  };
+
+  config = mkIf (cfg.secrets != { }) (mkMerge [
+    {
+      assertions = [
+        {
+          assertion = cfg.identityPaths != [ ];
+          message = "age.identityPaths must be set, for example by enabling openssh.";
+        }
+      ];
+    }
+    (optionalAttrs (!isDarwin) {
+      # When using sysusers we no longer be started as an activation script
+      # because those are started in initrd while sysusers is started later.
+      systemd.services.agenix-install-secrets = mkIf sysusersEnabled {
+        wantedBy = [ "sysinit.target" ];
+        after = [ "systemd-sysusers.service" ];
+        unitConfig.DefaultDependencies = "no";
+
+        path = [ pkgs.mount ];
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = pkgs.writeShellScript "agenix-install" (concatLines [
+            newGeneration
+            installSecrets
+            chownSecrets
+          ]);
+          RemainAfterExit = true;
+        };
+      };
+
+      # Create a new directory full of secrets for symlinking (this helps
+      # ensure removed secrets are actually removed, or at least become
+      # invalid symlinks).
+      system.activationScripts = mkIf (!sysusersEnabled) {
+        agenixNewGeneration = {
+          text = newGeneration;
+          deps = [
+            "specialfs"
+          ];
+        };
+
+        agenixInstall = {
+          text = installSecrets;
+          deps = [
+            "agenixNewGeneration"
+            "specialfs"
+          ];
+        };
+
+        # So user passwords can be encrypted.
+        users.deps = [ "agenixInstall" ];
+
+        # Change ownership and group after users and groups are made.
+        agenixChown = {
+          text = chownSecrets;
+          deps = [
+            "users"
+            "groups"
+          ];
+        };
+
+        # So other activation scripts can depend on agenix being done.
+        agenix = {
+          text = "";
+          deps = [ "agenixChown" ];
+        };
+      };
+    })
+
+    (optionalAttrs isDarwin {
+      launchd.daemons.activate-agenix = {
+        script = ''
+          set -e
+          set -o pipefail
+          export PATH="${pkgs.gnugrep}/bin:${pkgs.coreutils}/bin:@out@/sw/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+          ${newGeneration}
+          ${installSecrets}
+          ${chownSecrets}
+          exit 0
+        '';
+        serviceConfig = {
+          RunAtLoad = true;
+          KeepAlive.SuccessfulExit = false;
+        };
+      };
+    })
+  ]);
+}

m/module/amd-uprof.nix

@@ -0,0 +1,49 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options = {
+    services.amd-uprof = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable AMD uProf.";
+      };
+    };
+  };
+
+  # Only setup amd-uprof if enabled
+  config = lib.mkIf config.services.amd-uprof.enable {
+
+    # First make sure that we add the module to the list of available modules
+    # in the kernel matching the same kernel version of this configuration.
+    boot.extraModulePackages = with config.boot.kernelPackages; [ amd-uprof-driver ];
+    boot.kernelModules = [ "AMDPowerProfiler" ];
+
+    # Make the userspace tools available in $PATH.
+    environment.systemPackages = with pkgs; [ amd-uprof ];
+
+    # The AMDPowerProfiler module doesn't create the /dev device nor it emits
+    # any uevents, so we cannot use udev rules to automatically create the
+    # device. Instead, we run a systemd unit that does it after loading the
+    # modules.
+    systemd.services.amd-uprof-device = {
+      description = "Create /dev/AMDPowerProfiler device";
+      after = [ "systemd-modules-load.service" ];
+      wantedBy = [ "multi-user.target" ];
+      unitConfig.ConditionPathExists = [
+          "/proc/AMDPowerProfiler/device"
+          "!/dev/AMDPowerProfiler"
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        ExecStart = pkgs.writeShellScript "add-amd-uprof-dev.sh" ''
+          mknod /dev/AMDPowerProfiler -m 666 c $(< /proc/AMDPowerProfiler/device) 0
+        '';
+        ExecStop = pkgs.writeShellScript "remove-amd-uprof-dev.sh" ''
+          rm -f /dev/AMDPowerProfiler
+        '';
+      };
+    };
+  };
+}

m/module/hut-substituter.nix

@@ -6,5 +6,8 @@
     {
       extra-substituters = [ "http://hut/cache" ];
       extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+
+      # Set a low timeout in case hut is down
+      connect-timeout = 3; # seconds
     };
 }

m/module/nvidia.nix

@@ -0,0 +1,20 @@
+{ lib, config, pkgs, ... }:
+{
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.nvidia.open = lib.mkDefault (builtins.abort "hardware.nvidia.open not set");
+  hardware.graphics.enable = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  # enable support for derivations which require nvidia-gpu to be available
+  # > requiredSystemFeatures = [ "cuda" ];
+  programs.nix-required-mounts.enable = true;
+  programs.nix-required-mounts.presets.nvidia-gpu.enable = true;
+  # They forgot to add the symlink
+  programs.nix-required-mounts.allowedPatterns.nvidia-gpu.paths = [
+    config.systemd.tmpfiles.settings.graphics-driver."/run/opengl-driver"."L+".argument
+  ];
+
+  environment.systemPackages = [ pkgs.cudainfo ];
+}

m/module/power-policy.nix

@@ -0,0 +1,33 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.power.policy;
+in
+{
+  options = {
+    power.policy = mkOption {
+      type = types.nullOr (types.enum [ "always-on" "previous" "always-off" ]);
+      default = null;
+      description = "Set power policy to use via IPMI.";
+    };
+  };
+
+  config = mkIf (cfg != null) {
+    systemd.services."power-policy" = {
+      description = "Set power policy to use via IPMI";
+      wantedBy = [ "multi-user.target" ];
+      unitConfig = {
+        StartLimitBurst = "10";
+        StartLimitIntervalSec = "10m";
+      };
+      serviceConfig = {
+        ExecStart = "${pkgs.ipmitool}/bin/ipmitool chassis policy ${cfg}";
+        Type = "oneshot";
+        Restart = "on-failure";
+        RestartSec = "5s";
+      };
+    };
+  };
+}

m/module/slurm-client.nix

@@ -1,33 +1,10 @@
-{ config, pkgs, lib, ... }:
+{ lib, ... }:
 
-let
-  suspendProgram = pkgs.writeScript "suspend.sh" ''
-    #!/usr/bin/env bash
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Shutting down host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
-    done
-  '';
+{
+  imports = [
+    ./slurm-common.nix
+  ];
 
-  resumeProgram = pkgs.writeScript "resume.sh" ''
-    #!/usr/bin/env bash
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Starting host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
-    done
-  '';
-
-in {
   systemd.services.slurmd.serviceConfig = {
     # Kill all processes in the control group on stop/restart. This will kill
     # all the jobs running, so ensure that we only upgrade when the nodes are
@@ -35,92 +12,13 @@ in {
     # https://github.com/NixOS/nixpkgs/commit/ae93ed0f0d4e7be0a286d1fca86446318c0c6ffb
     # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24
     KillMode = lib.mkForce "control-group";
+
+    # If slurmd fails to contact the control server it will fail, causing the
+    # node to remain out of service until manually restarted. Always try to
+    # restart it.
+    Restart = "always";
+    RestartSec = "30s";
   };
 
-  services.slurm = {
-    client.enable = true;
-    controlMachine = "hut";
-    clusterName = "jungle";
-    nodeName = [
-      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
-      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
-    ];
-
-    partitionName = [
-      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-    ];
-
-    # See slurm.conf(5) for more details about these options.
-    extraConfig = ''
-      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
-      # not with Intel MPI. For that use the compatibility shim libpmi.so
-      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
-      # library in SLURM (--mpi=pmix). See more details here:
-      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
-      MpiDefault=pmix
-
-      # When a node reboots return that node to the slurm queue as soon as it
-      # becomes operative again.
-      ReturnToService=2
-
-      # Track all processes by using a cgroup
-      ProctrackType=proctrack/cgroup
-
-      # Enable task/affinity to allow the jobs to run in a specified subset of
-      # the resources. Use the task/cgroup plugin to enable process containment.
-      TaskPlugin=task/affinity,task/cgroup
-
-      # Power off unused nodes until they are requested
-      SuspendProgram=${suspendProgram}
-      SuspendTimeout=60
-      ResumeProgram=${resumeProgram}
-      ResumeTimeout=300
-      SuspendExcNodes=hut
-
-      # Turn the nodes off after 1 hour of inactivity
-      SuspendTime=3600
-
-      # Reduce port range so we can allow only this range in the firewall
-      SrunPortRange=60000-61000
-
-      # Use cores as consumable resources. In SLURM terms, a core may have
-      # multiple hardware threads (or CPUs).
-      SelectType=select/cons_tres
-
-      # Ignore memory constraints and only use unused cores to share a node with
-      # other jobs.
-      SelectTypeParameters=CR_Core
-
-      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
-      # This sets up the "extern" step into which ssh-launched processes will be
-      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
-      # when a task runs (srun) so we can ssh early.
-      PrologFlags=Alloc,Contain,X11
-
-      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
-      # adopted by the external step, similar to tasks running in regular steps
-      # LaunchParameters=ulimit_pam_adopt
-      SlurmdDebug=debug5
-      #DebugFlags=Protocol,Cgroup
-    '';
-
-    extraCgroupConfig = ''
-      CgroupPlugin=cgroup/v2
-      #ConstrainCores=yes
-    '';
-  };
-
-  # Place the slurm config in /etc as this will be required by PAM
-  environment.etc.slurm.source = config.services.slurm.etcSlurm;
-
-  age.secrets.mungeKey = {
-    file = ../../secrets/munge-key.age;
-    owner = "munge";
-    group = "munge";
-  };
-
-  services.munge = {
-    enable = true;
-    password = config.age.secrets.mungeKey.path;
-  };
+  services.slurm.client.enable = true;
 }

m/module/slurm-common.nix

@@ -0,0 +1,115 @@
+{ config, pkgs, ... }:
+
+let
+  suspendProgram = pkgs.writeShellScript "suspend.sh" ''
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Shutting down host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
+    done
+  '';
+
+  resumeProgram = pkgs.writeShellScript "resume.sh" ''
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Starting host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
+    done
+  '';
+
+in {
+  services.slurm = {
+    controlMachine = "apex";
+    clusterName = "jungle";
+    nodeName = [
+      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
+      "fox       Sockets=8 CoresPerSocket=24 ThreadsPerCore=1"
+    ];
+
+    partitionName = [
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "fox Nodes=fox          Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+    ];
+
+    # See slurm.conf(5) for more details about these options.
+    extraConfig = ''
+      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
+      # not with Intel MPI. For that use the compatibility shim libpmi.so
+      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
+      # library in SLURM (--mpi=pmix). See more details here:
+      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
+      MpiDefault=pmix
+
+      # When a node reboots return that node to the slurm queue as soon as it
+      # becomes operative again.
+      ReturnToService=2
+
+      # Track all processes by using a cgroup
+      ProctrackType=proctrack/cgroup
+
+      # Enable task/affinity to allow the jobs to run in a specified subset of
+      # the resources. Use the task/cgroup plugin to enable process containment.
+      TaskPlugin=task/affinity,task/cgroup
+
+      # Power off unused nodes until they are requested
+      SuspendProgram=${suspendProgram}
+      SuspendTimeout=60
+      ResumeProgram=${resumeProgram}
+      ResumeTimeout=300
+      SuspendExcNodes=fox
+
+      # Turn the nodes off after 1 hour of inactivity
+      SuspendTime=3600
+
+      # Reduce port range so we can allow only this range in the firewall
+      SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
+    '';
+  };
+
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
+  age.secrets.mungeKey = {
+    file = ../../secrets/munge-key.age;
+    owner = "munge";
+    group = "munge";
+  };
+
+  services.munge = {
+    enable = true;
+    password = config.age.secrets.mungeKey.path;
+  };
+}

m/module/slurm-server.nix

@@ -0,0 +1,23 @@
+{ ... }:
+
+{
+  imports = [
+    ./slurm-common.nix
+  ];
+
+  services.slurm.server.enable = true;
+
+  networking.firewall = {
+    extraCommands = ''
+      # Accept slurm connections to controller from compute nodes
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from compute nodes for srun
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+
+      # Accept slurm connections to controller from fox (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 6817 -j nixos-fw-accept
+      # Accept slurm connections from fox for srun (via wireguard)
+      iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.1/32 --dport 60000:61000 -j nixos-fw-accept
+    '';
+  };
+}

m/module/ssh-hut-extern.nix

@@ -1,9 +0,0 @@
-{
-  programs.ssh.extraConfig = ''
-    Host ssfhead
-      HostName ssflogin.bsc.es
-    Host hut
-      ProxyJump ssfhead
-      HostName xeon07
-  '';
-}

m/raccoon/configuration.nix

@@ -3,10 +3,13 @@
 {
   imports = [
     ../common/base.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
+    ../module/nvidia.nix
     ../eudy/kernel/perf.nix
+    ./wireguard.nix
+    ../module/hut-substituter.nix
   ];
 
   # Don't install Grub on the disk yet
@@ -38,26 +41,21 @@
     };
     hosts = {
       "10.0.44.4" = [ "tent" ];
+      "84.88.53.236" = [ "apex" ];
     };
   };
 
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
   };
 
   # Enable performance governor
   powerManagement.cpuFreqGovernor = "performance";
 
-  # Configure Nvidia driver to use with CUDA
-  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
-  hardware.graphics.enable = true;
-  nixpkgs.config.allowUnfree = true;
-  nixpkgs.config.nvidia.acceptLicense = true;
-  services.xserver.videoDrivers = [ "nvidia" ];
-
-  # Disable garbage collection for now
-  nix.gc.automatic = lib.mkForce false;
+  hardware.nvidia.open = false; # Maxwell is older than Turing architecture
 
   services.openssh.settings.X11Forwarding = true;
 

m/raccoon/wireguard.nix

@@ -0,0 +1,48 @@
+{ config, pkgs, ... }:
+
+{
+  networking.nat = {
+    enable = true;
+    enableIPv6 = false;
+    externalInterface = "eno0";
+    internalInterfaces = [ "wg0" ];
+  };
+
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgRaccoon.file = ../../secrets/wg-raccoon.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    wg0 = {
+      ips = [ "10.106.0.236/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgRaccoon.path;
+      # Public key: QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=
+      peers = [
+        {
+          name = "fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.1/32" ];
+          endpoint = "fox.ac.upc.edu:666";
+          persistentKeepalive = 25;
+        }
+        {
+          name = "apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          allowedIPs = [ "10.106.0.30/32" "10.0.40.0/24" ];
+          endpoint = "ssfhead.bsc.es:666";
+          persistentKeepalive = 25;
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1"  = [ "fox.wg" ];
+    "10.106.0.30" = [ "apex.wg" ];
+  };
+}

m/tent/configuration.nix

@@ -3,9 +3,9 @@
 {
   imports = [
     ../common/xeon.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
     ./monitoring.nix
     ./nginx.nix
     ./nix-serve.nix
@@ -15,6 +15,7 @@
     ../hut/msmtp.nix
     ../module/p.nix
     ../module/vpn-dac.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -33,6 +34,10 @@
     nameservers = [ "84.88.52.35" "84.88.52.36" ];
     search = [ "bsc.es" "ac.upc.edu" ];
     defaultGateway = "10.0.44.1";
+    hosts = {
+      "84.88.53.236" = [ "apex" ];
+      "10.0.44.1" = [ "raccoon" ];
+    };
   };
 
   services.p.enable = true;

m/tent/nginx.nix

@@ -2,10 +2,13 @@
 let
   website = pkgs.stdenv.mkDerivation {
     name = "jungle-web";
-    src = theFlake;
+    src = pkgs.fetchgit {
+      url = "https://jungle.bsc.es/git/rarias/jungle-website.git";
+      rev = "52abaf4d71652a9ef77a0b098db14ca33bffff4c";
+      hash = "sha256-/ul9GazbOrOkmlvSgDz/+2W+V+ir5725Y7mVLc3rb0M=";
+    };
     buildInputs = [ pkgs.hugo ];
     buildPhase = ''
-      cd web
       rm -rf public/
       hugo
     '';
@@ -67,6 +70,9 @@ in
         location /p/ {
           alias /var/lib/p/;
         }
+        location /pub/ {
+          alias /vault/pub/;
+        }
       '';
     };
   };

m/weasel/configuration.nix

@@ -0,0 +1,33 @@
+{ lib, ... }:
+
+{
+  imports = [
+    ../common/ssf.nix
+    ../module/hut-substituter.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d5356ca";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  # Users with sudo access
+  users.groups.wheel.members = [ "abonerib" "anavarro" ];
+
+  # Run julia installed with juliaup using julia's own libraries:
+  # NIX_LD_LIBRARY_PATH=~/.julia/juliaup/${VERS}/lib/julia ~/.juliaup/bin/julia
+  programs.nix-ld.enable = true;
+
+  networking = {
+    hostName = "weasel";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.6";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.6";
+      prefixLength = 24;
+    } ];
+  };
+}

overlay.nix

@@ -0,0 +1,157 @@
+final: /* Future last stage */
+prev:  /* Previous stage */
+
+with final.lib;
+
+let
+  callPackage = final.callPackage;
+
+  bscPkgs = {
+    agenix = prev.callPackage ./pkgs/agenix/default.nix { };
+    amd-uprof = prev.callPackage ./pkgs/amd-uprof/default.nix { };
+    bench6 = callPackage ./pkgs/bench6/default.nix { };
+    bigotes = callPackage ./pkgs/bigotes/default.nix { };
+    clangOmpss2 = callPackage ./pkgs/llvm-ompss2/default.nix { };
+    clangOmpss2Nanos6 = callPackage ./pkgs/llvm-ompss2/default.nix { ompss2rt = final.nanos6; };
+    clangOmpss2Nodes = callPackage ./pkgs/llvm-ompss2/default.nix { ompss2rt = final.nodes; openmp = final.openmp; };
+    clangOmpss2NodesOmpv = callPackage ./pkgs/llvm-ompss2/default.nix { ompss2rt = final.nodes; openmp = final.openmpv; };
+    clangOmpss2Unwrapped = callPackage ./pkgs/llvm-ompss2/clang.nix { };
+    cudainfo = prev.callPackage ./pkgs/cudainfo/default.nix { };
+    #extrae = callPackage ./pkgs/extrae/default.nix { }; # Broken and outdated
+    gpi-2 = callPackage ./pkgs/gpi-2/default.nix { };
+    intelPackages_2023 = callPackage ./pkgs/intel-oneapi/2023.nix { };
+    jemallocNanos6 = callPackage ./pkgs/nanos6/jemalloc.nix { };
+    # FIXME: Extend this to all linuxPackages variants. Open problem, see:
+    # https://discourse.nixos.org/t/whats-the-right-way-to-make-a-custom-kernel-module-available/4636
+    linuxPackages = prev.linuxPackages.extend (_final: _prev: {
+      amd-uprof-driver = _prev.callPackage ./pkgs/amd-uprof/driver.nix { };
+    });
+    linuxPackages_latest = prev.linuxPackages_latest.extend(_final: _prev: {
+      amd-uprof-driver = _prev.callPackage ./pkgs/amd-uprof/driver.nix { };
+    });
+    lmbench = callPackage ./pkgs/lmbench/default.nix { };
+    mcxx = callPackage ./pkgs/mcxx/default.nix { };
+    meteocat-exporter = prev.callPackage ./pkgs/meteocat-exporter/default.nix { };
+    mpi = final.mpich; # Set MPICH as default
+    mpich = callPackage ./pkgs/mpich/default.nix { mpich = prev.mpich; };
+    nanos6 = callPackage ./pkgs/nanos6/default.nix { };
+    nanos6Debug = final.nanos6.override { enableDebug = true; };
+    nixtools = callPackage ./pkgs/nixtools/default.nix { };
+    # Broken because of pkgsStatic.libcap
+    # See: https://github.com/NixOS/nixpkgs/pull/268791
+    #nix-wrap = callPackage ./pkgs/nix-wrap/default.nix { };
+    nodes = callPackage ./pkgs/nodes/default.nix { };
+    nosv = callPackage ./pkgs/nosv/default.nix { };
+    openmp = callPackage ./pkgs/llvm-ompss2/openmp.nix { monorepoSrc = final.clangOmpss2Unwrapped.src; version = final.clangOmpss2Unwrapped.version; };
+    openmpv = final.openmp.override { enableNosv = true; enableOvni = true; };
+    osumb = callPackage ./pkgs/osu/default.nix { };
+    ovni = callPackage ./pkgs/ovni/default.nix { };
+    ovniGit = final.ovni.override { useGit = true; };
+    paraverKernel = callPackage ./pkgs/paraver/kernel.nix { };
+    prometheus-slurm-exporter = prev.callPackage ./pkgs/slurm-exporter/default.nix { };
+    #pscom = callPackage ./pkgs/parastation/pscom.nix { }; # Unmaintaned
+    #psmpi = callPackage ./pkgs/parastation/psmpi.nix { }; # Unmaintaned
+    sonar = callPackage ./pkgs/sonar/default.nix { };
+    stdenvClangOmpss2 = final.stdenv.override { cc = final.clangOmpss2; allowedRequisites = null; };
+    stdenvClangOmpss2Nanos6 = final.stdenv.override { cc = final.clangOmpss2Nanos6; allowedRequisites = null; };
+    stdenvClangOmpss2Nodes = final.stdenv.override { cc = final.clangOmpss2Nodes; allowedRequisites = null; };
+    stdenvClangOmpss2NodesOmpv = final.stdenv.override { cc = final.clangOmpss2NodesOmpv; allowedRequisites = null; };
+    tagaspi = callPackage ./pkgs/tagaspi/default.nix { };
+    tampi = callPackage ./pkgs/tampi/default.nix { };
+    upc-qaire-exporter = prev.callPackage ./pkgs/upc-qaire-exporter/default.nix { };
+    wxparaver = callPackage ./pkgs/paraver/default.nix { };
+  };
+
+  tests = rec {
+    hwloc = callPackage ./test/bugs/hwloc.nix { };
+    #sigsegv = callPackage ./test/reproducers/sigsegv.nix { };
+    hello-c = callPackage ./test/compilers/hello-c.nix { };
+    hello-cpp = callPackage ./test/compilers/hello-cpp.nix { };
+    lto = callPackage ./test/compilers/lto.nix { };
+    asan = callPackage ./test/compilers/asan.nix { };
+    intel2023-icx-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv; };
+    intel2023-icc-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv-icc; };
+    intel2023-icx-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv; };
+    intel2023-icc-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv-icc; };
+    intel2023-ifort   = callPackage ./test/compilers/hello-f.nix {
+      stdenv = final.intelPackages_2023.stdenv-ifort;
+    };
+    clangOmpss2-lto   = lto.override       { stdenv = final.stdenvClangOmpss2Nanos6; };
+    clangOmpss2-asan  = asan.override      { stdenv = final.stdenvClangOmpss2Nanos6; };
+    clangOmpss2-task  = callPackage ./test/compilers/ompss2.nix {
+      stdenv = final.stdenvClangOmpss2Nanos6;
+    };
+    clangNodes-task = callPackage ./test/compilers/ompss2.nix {
+      stdenv = final.stdenvClangOmpss2Nodes;
+    };
+    clangNosvOpenmp-task = callPackage ./test/compilers/clang-openmp.nix {
+      stdenv = final.stdenvClangOmpss2Nodes;
+    };
+    clangNosvOmpv-nosv = callPackage ./test/compilers/clang-openmp-nosv.nix {
+      stdenv = final.stdenvClangOmpss2NodesOmpv;
+    };
+    clangNosvOmpv-ld = callPackage ./test/compilers/clang-openmp-ld.nix {
+      stdenv = final.stdenvClangOmpss2NodesOmpv;
+    };
+  };
+
+  # For now, only build toplevel packages in CI/Hydra
+  pkgsTopLevel = filterAttrs (_: isDerivation) bscPkgs;
+
+  # Native build in that platform doesn't imply cross build works
+  canCrossCompile = platform: pkg:
+    (isDerivation pkg) &&
+    # Must be defined explicitly
+    (pkg.meta.cross or false) &&
+    (meta.availableOn platform pkg);
+
+  # For now only RISC-V
+  crossSet = { riscv64 = final.pkgsCross.riscv64.bsc.pkgsTopLevel; };
+
+  buildList = name: paths:
+    final.runCommandLocal name { } ''
+      printf '%s\n' ${toString paths} | tee $out
+    '';
+
+  buildList' = name: paths:
+    final.runCommandLocal name { } ''
+      deps="${toString paths}"
+      cat $deps
+      printf '%s\n' $deps >$out
+    '';
+
+  pkgsList = buildList "ci-pkgs" (builtins.attrValues pkgsTopLevel);
+  testsList = buildList "ci-tests" (collect isDerivation tests);
+  allList = buildList' "ci-all" [ pkgsList testsList ];
+  # For now only RISC-V
+  crossList = buildList "ci-cross"
+    (filter
+      (canCrossCompile final.pkgsCross.riscv64.stdenv.hostPlatform)
+        (builtins.attrValues crossSet.riscv64));
+
+in bscPkgs // {
+
+  lib = prev.lib // {
+    maintainers = prev.lib.maintainers // {
+      bsc = import ./pkgs/maintainers.nix;
+    };
+  };
+
+  # Prevent accidental usage of bsc-ci attribute
+  bsc-ci = throw "the bsc-ci attribute is deprecated, use bsc.ci";
+
+  # Internal for our CI tests
+  bsc = {
+    # CI targets for nix build
+    ci = { pkgs = pkgsList; tests = testsList; all = allList; cross = crossList; };
+
+    # Direct access to package sets
+    tests = tests;
+    pkgs = bscPkgs;
+    pkgsTopLevel = pkgsTopLevel;
+    cross = crossSet;
+
+    # Hydra uses attribute sets of pkgs
+    hydraJobs = { tests = tests; pkgs = pkgsTopLevel; cross = crossSet; };
+  };
+}

pkgs/agenix/agenix.sh

@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+PACKAGE="agenix"
+
+function show_help () {
+  echo "$PACKAGE - edit and rekey age secret files"
+  echo " "
+  echo "$PACKAGE -e FILE [-i PRIVATE_KEY]"
+  echo "$PACKAGE -r [-i PRIVATE_KEY]"
+  echo ' '
+  echo 'options:'
+  echo '-h, --help                show help'
+  # shellcheck disable=SC2016
+  echo '-e, --edit FILE           edits FILE using $EDITOR'
+  echo '-r, --rekey               re-encrypts all secrets with specified recipients'
+  echo '-d, --decrypt FILE        decrypts FILE to STDOUT'
+  echo '-i, --identity            identity to use when decrypting'
+  echo '-v, --verbose             verbose output'
+  echo ' '
+  echo 'FILE an age-encrypted file'
+  echo ' '
+  echo 'PRIVATE_KEY a path to a private SSH key used to decrypt file'
+  echo ' '
+  echo 'EDITOR environment variable of editor to use when editing FILE'
+  echo ' '
+  echo 'If STDIN is not interactive, EDITOR will be set to "cp /dev/stdin"'
+  echo ' '
+  echo 'RULES environment variable with path to Nix file specifying recipient public keys.'
+  echo "Defaults to './secrets.nix'"
+  echo ' '
+  echo "agenix version: @version@"
+  echo "age binary path: @ageBin@"
+  echo "age version: $(@ageBin@ --version)"
+}
+
+function warn() {
+  printf '%s\n' "$*" >&2
+}
+
+function err() {
+  warn "$*"
+  exit 1
+}
+
+test $# -eq 0 && (show_help && exit 1)
+
+REKEY=0
+DECRYPT_ONLY=0
+DEFAULT_DECRYPT=(--decrypt)
+
+while test $# -gt 0; do
+  case "$1" in
+    -h|--help)
+      show_help
+      exit 0
+      ;;
+    -e|--edit)
+      shift
+      if test $# -gt 0; then
+        export FILE=$1
+      else
+        echo "no FILE specified"
+        exit 1
+      fi
+      shift
+      ;;
+    -i|--identity)
+      shift
+      if test $# -gt 0; then
+        DEFAULT_DECRYPT+=(--identity "$1")
+      else
+        echo "no PRIVATE_KEY specified"
+        exit 1
+      fi
+      shift
+      ;;
+    -r|--rekey)
+      shift
+      REKEY=1
+      ;;
+    -d|--decrypt)
+      shift
+      DECRYPT_ONLY=1
+      if test $# -gt 0; then
+        export FILE=$1
+      else
+        echo "no FILE specified"
+        exit 1
+      fi
+      shift
+      ;;
+    -v|--verbose)
+      shift
+      set -x
+      ;;
+    *)
+      show_help
+      exit 1
+      ;;
+  esac
+done
+
+RULES=${RULES:-./secrets.nix}
+function cleanup {
+    if [ -n "${CLEARTEXT_DIR+x}" ]
+    then
+        rm -rf -- "$CLEARTEXT_DIR"
+    fi
+    if [ -n "${REENCRYPTED_DIR+x}" ]
+    then
+        rm -rf -- "$REENCRYPTED_DIR"
+    fi
+}
+trap "cleanup" 0 2 3 15
+
+function keys {
+    (@nixInstantiate@ --json --eval --strict -E "(let rules = import $RULES; in rules.\"$1\".publicKeys)" | @jqBin@ -r .[]) || exit 1
+}
+
+function armor {
+    (@nixInstantiate@ --json --eval --strict -E "(let rules = import $RULES; in (builtins.hasAttr \"armor\" rules.\"$1\" && rules.\"$1\".armor))") || exit 1
+}
+
+function decrypt {
+    FILE=$1
+    KEYS=$2
+    if [ -z "$KEYS" ]
+    then
+        err "There is no rule for $FILE in $RULES."
+    fi
+
+    if [ -f "$FILE" ]
+    then
+        DECRYPT=("${DEFAULT_DECRYPT[@]}")
+        if [[ "${DECRYPT[*]}" != *"--identity"* ]]; then
+            if [ -f "$HOME/.ssh/id_rsa" ]; then
+                DECRYPT+=(--identity "$HOME/.ssh/id_rsa")
+            fi
+            if [ -f "$HOME/.ssh/id_ed25519" ]; then
+                DECRYPT+=(--identity "$HOME/.ssh/id_ed25519")
+            fi
+        fi
+        if [[ "${DECRYPT[*]}" != *"--identity"* ]]; then
+          err "No identity found to decrypt $FILE. Try adding an SSH key at $HOME/.ssh/id_rsa or $HOME/.ssh/id_ed25519 or using the --identity flag to specify a file."
+        fi
+
+        @ageBin@ "${DECRYPT[@]}" -- "$FILE" || exit 1
+    fi
+}
+
+function edit {
+    FILE=$1
+    KEYS=$(keys "$FILE") || exit 1
+    ARMOR=$(armor "$FILE") || exit 1
+
+    CLEARTEXT_DIR=$(@mktempBin@ -d)
+    CLEARTEXT_FILE="$CLEARTEXT_DIR/$(basename -- "$FILE")"
+    DEFAULT_DECRYPT+=(-o "$CLEARTEXT_FILE")
+
+    decrypt "$FILE" "$KEYS" || exit 1
+
+    [ ! -f "$CLEARTEXT_FILE" ] || cp -- "$CLEARTEXT_FILE" "$CLEARTEXT_FILE.before"
+
+    [ -t 0 ] || EDITOR='cp -- /dev/stdin'
+
+    $EDITOR "$CLEARTEXT_FILE"
+
+    if [ ! -f "$CLEARTEXT_FILE" ]
+    then
+      warn "$FILE wasn't created."
+      return
+    fi
+    [ -f "$FILE" ] && [ "$EDITOR" != ":" ] && @diffBin@ -q -- "$CLEARTEXT_FILE.before" "$CLEARTEXT_FILE" && warn "$FILE wasn't changed, skipping re-encryption." && return
+
+    ENCRYPT=()
+    if [[ "$ARMOR" == "true" ]]; then
+        ENCRYPT+=(--armor)
+    fi
+    while IFS= read -r key
+    do
+        if [ -n "$key" ]; then
+            ENCRYPT+=(--recipient "$key")
+        fi
+    done <<< "$KEYS"
+
+    REENCRYPTED_DIR=$(@mktempBin@ -d)
+    REENCRYPTED_FILE="$REENCRYPTED_DIR/$(basename -- "$FILE")"
+
+    ENCRYPT+=(-o "$REENCRYPTED_FILE")
+
+    @ageBin@ "${ENCRYPT[@]}" <"$CLEARTEXT_FILE" || exit 1
+
+    mkdir -p -- "$(dirname -- "$FILE")"
+
+    mv -f -- "$REENCRYPTED_FILE" "$FILE"
+}
+
+function rekey {
+    FILES=$( (@nixInstantiate@ --json --eval -E "(let rules = import $RULES; in builtins.attrNames rules)"  | @jqBin@ -r .[]) || exit 1)
+
+    for FILE in $FILES
+    do
+        warn "rekeying $FILE..."
+        EDITOR=: edit "$FILE"
+        cleanup
+    done
+}
+
+[ $REKEY -eq 1 ] && rekey && exit 0
+[ $DECRYPT_ONLY -eq 1 ] && DEFAULT_DECRYPT+=("-o" "-") && decrypt "${FILE}" "$(keys "$FILE")" && exit 0
+edit "$FILE" && cleanup && exit 0

pkgs/agenix/default.nix

@@ -0,0 +1,66 @@
+{
+  lib,
+  stdenv,
+  age,
+  jq,
+  nix,
+  mktemp,
+  diffutils,
+  replaceVars,
+  ageBin ? "${age}/bin/age",
+  shellcheck,
+}:
+let
+  bin = "${placeholder "out"}/bin/agenix";
+in
+stdenv.mkDerivation rec {
+  pname = "agenix";
+  version = "0.15.0";
+  src = replaceVars ./agenix.sh {
+    inherit ageBin version;
+    jqBin = "${jq}/bin/jq";
+    nixInstantiate = "${nix}/bin/nix-instantiate";
+    mktempBin = "${mktemp}/bin/mktemp";
+    diffBin = "${diffutils}/bin/diff";
+  };
+  dontUnpack = true;
+  doInstallCheck = true;
+  installCheckInputs = [ shellcheck ];
+  postInstallCheck = ''
+    shellcheck ${bin}
+    ${bin} -h | grep ${version}
+
+    test_tmp=$(mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir')
+    export HOME="$test_tmp/home"
+    export NIX_STORE_DIR="$test_tmp/nix/store"
+    export NIX_STATE_DIR="$test_tmp/nix/var"
+    mkdir -p "$HOME" "$NIX_STORE_DIR" "$NIX_STATE_DIR"
+    function cleanup {
+      rm -rf "$test_tmp"
+    }
+    trap "cleanup" 0 2 3 15
+
+    mkdir -p $HOME/.ssh
+    cp -r "${./example}" $HOME/secrets
+    chmod -R u+rw $HOME/secrets
+    (
+    umask u=rw,g=r,o=r
+    cp ${./example_keys/user1.pub} $HOME/.ssh/id_ed25519.pub
+    chown $UID $HOME/.ssh/id_ed25519.pub
+    )
+    (
+    umask u=rw,g=,o=
+    cp ${./example_keys/user1} $HOME/.ssh/id_ed25519
+    chown $UID $HOME/.ssh/id_ed25519
+    )
+
+    cd $HOME/secrets
+    test $(${bin} -d secret1.age) = "hello"
+  '';
+
+  installPhase = ''
+    install -D $src ${bin}
+  '';
+
+  meta.description = "age-encrypted secrets for NixOS";
+}

pkgs/agenix/example/-leading-hyphen-filename.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 V3XmEA zirqdzZZ1E+sedBn7fbEHq4ntLEkokZ4GctarBBOHXY
+Rvs5YHaAUeCZyNwPedubPcHClWYIuXXWA5zadXPWY6w
+-> ssh-ed25519 KLPP8w BVp4rDkOYSQyn8oVeHFeinSqW+pdVtxBF9+5VM1yORY
+bMwppAi8Nhz0328taU4AzUkTVyWtSLvFZG6c5W/Fs78
+--- xCbqLhXAcOziO2wmbjTiSQfZvt5Rlsc4SCvF+iEzpQA
+ôKB£î/²ZÅÈrÙ%
¾à4¡´—Mq5×Ô_ÌÂÝ’‹†ã„Ò11ܨqM;& ¢‡LríÂÒføû”]>N

pkgs/agenix/example/armored-secret.age

@@ -0,0 +1,7 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFYzWG1FQSBpZkZW
+aFpLNnJxc0VUMHRmZ2dZS0pjMGVENnR3OHd5K0RiT1RjRUhibFZBCnN5UG5vUjA3
+SXpsNGtiVUw4T0tIVFo5Wkk5QS9NQlBndzVvektiQ0ozc0kKLS0tIGxyY1Q4dEZ1
+VGZEanJyTFNta2JNRmpZb2FnK2JyS1hSVml1UGdMNWZKQXMKYla+wTXcRedyZoEb
+LVWaSx49WoUTU0KBPJg9RArxaeC23GoCDzR/aM/1DvYU
+-----END AGE ENCRYPTED FILE-----

pkgs/agenix/example/passwordfile-user1.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 KLPP8w s1DYZRlZuSsyhmZCF1lFB+E9vB8bZ/+ZhBRlx8nprwE
+nmYVCsVBrX2CFXXPU+D+bbkkIe/foofp+xoUrg9DHZw
+-> ssh-ed25519 V3XmEA Pwv3oCwcY0DX8rY48UNfsj9RumWsn4dbgorYHCwObgI
+FKxRYkL3JHtJxUwymWDF0rAtJ33BivDI6IfPsfumM90
+-> V'v(/u$-grease em/Vgf 2qDuk
+7I3iiQLPGi1COML9u/JeYkr7EqbSLoU
+--- 57WJRigUGtmcObrssS3s4PvmR8wgh1AOC/ijJn1s3xI
+<EFBFBD>'K©Æ·Y&‘7GÆOÝòFj
±kÆXç«BnuJöê:9Ê(’ÙÏX¬#¼AíÄÞÃÚ§j’,ê_ÈþÝ?ÝZ“¥vœ¹V’96]oks~%£c	Îe^CÅ%JQ5€<H¢z}îCý,°pŒ¿*!W§§ÈA±º­Ò…dC¼K)¿¢-žy

pkgs/agenix/example/secret2.age

@@ -0,0 +1,5 @@
+age-encryption.org/v1
+-> ssh-ed25519 V3XmEA OB4+1FbPhQ3r6iGksM7peWX5it8NClpXIq/o5nnP7GA
+FmHVUj+A5i5+bDFgySQskmlvynnosJiWUTJmBRiNA9I
+--- tP+3mFVtd7ogVu1Lkboh55zoi5a77Ht08Uc/QuIviv4
+¤¬Xæ{”ïOŠ£èätMXxÔvÓª(¬IÁmyPÇï¸è+3²S3i

pkgs/agenix/example/secrets.nix

@@ -0,0 +1,23 @@
+let
+  user1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL0idNvgGiucWgup/mP78zyC23uFjYq0evcWdjGQUaBH";
+  system1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPJDyIr/FSz1cJdcoW69R+NrWzwGK/+3gJpqD1t8L2zE";
+in
+{
+  "secret1.age".publicKeys = [
+    user1
+    system1
+  ];
+  "secret2.age".publicKeys = [ user1 ];
+  "passwordfile-user1.age".publicKeys = [
+    user1
+    system1
+  ];
+  "-leading-hyphen-filename.age".publicKeys = [
+    user1
+    system1
+  ];
+  "armored-secret.age" = {
+    publicKeys = [ user1 ];
+    armor = true;
+  };
+}

pkgs/agenix/example_keys/system1

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACDyQ8iK/xUs9XCXXKFuvUfja1s8Biv/t4Caag9bfC9sxAAAAJA3yvCWN8rw
+lgAAAAtzc2gtZWQyNTUxOQAAACDyQ8iK/xUs9XCXXKFuvUfja1s8Biv/t4Caag9bfC9sxA
+AAAEA+J2V6AG1NriAIvnNKRauIEh1JE9HSdhvKJ68a5Fm0w/JDyIr/FSz1cJdcoW69R+Nr
+WzwGK/+3gJpqD1t8L2zEAAAADHJ5YW50bUBob21lMQE=
+-----END OPENSSH PRIVATE KEY-----

pkgs/agenix/example_keys/system1.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPJDyIr/FSz1cJdcoW69R+NrWzwGK/+3gJpqD1t8L2zE

pkgs/agenix/example_keys/user1

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACC9InTb4BornFoLqf5j+/M8gtt7hY2KtHr3FnYxkFGgRwAAAJC2JJ8htiSf
+IQAAAAtzc2gtZWQyNTUxOQAAACC9InTb4BornFoLqf5j+/M8gtt7hY2KtHr3FnYxkFGgRw
+AAAEDxt5gC/s53IxiKAjfZJVCCcFIsdeERdIgbYhLO719+Kb0idNvgGiucWgup/mP78zyC
+23uFjYq0evcWdjGQUaBHAAAADHJ5YW50bUBob21lMQE=
+-----END OPENSSH PRIVATE KEY-----

pkgs/agenix/example_keys/user1.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL0idNvgGiucWgup/mP78zyC23uFjYq0evcWdjGQUaBH

pkgs/agenix/update.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+
+# All operations are done relative to root
+GITROOT=$(git rev-parse --show-toplevel)
+cd "$GITROOT"
+
+REVISION=${1:-main}
+
+TMPCLONE=$(mktemp -d)
+trap "rm -rf ${TMPCLONE}" EXIT
+
+git clone https://github.com/ryantm/agenix.git --revision="$REVISION" "$TMPCLONE" --depth=1
+
+cp "${TMPCLONE}/pkgs/agenix.sh" pkgs/agenix/agenix.sh
+cp "${TMPCLONE}/pkgs/agenix.nix" pkgs/agenix/default.nix
+sed -i 's#../example#./example#' pkgs/agenix/default.nix
+
+cp "${TMPCLONE}/example/"* pkgs/agenix/example/
+cp "${TMPCLONE}/example_keys/"* pkgs/agenix/example_keys/
+
+cp "${TMPCLONE}/modules/age.nix" m/module/agenix.nix

pkgs/amd-uprof/default.nix

@@ -0,0 +1,98 @@
+{ stdenv
+, lib
+, curl
+, cacert
+, runCommandLocal
+, autoPatchelfHook
+, elfutils
+, glib
+, libGL
+, ncurses5
+, xorg
+, zlib
+, libxkbcommon
+, freetype
+, fontconfig
+, libGLU
+, dbus
+, rocmPackages
+, libxcrypt-legacy
+, numactl
+, radare2
+}:
+
+let
+  version = "5.1.701";
+  tarball = "AMDuProf_Linux_x64_${version}.tar.bz2";
+
+  # NOTE: Remember to update the radare2 patch below if AMDuProfPcm changes.
+  uprofSrc = runCommandLocal tarball {
+    nativeBuildInputs = [ curl ];
+    outputHash = "sha256-j9gxcBcIg6Zhc5FglUXf/VV9bKSo+PAKeootbN7ggYk=";
+    SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt";
+  } ''
+    curl \
+    -o $out \
+    'https://download.amd.com/developer/eula/uprof/uprof-5-1/${tarball}' \
+    -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0' \
+    -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \
+    -H 'Accept-Language: en-US,en;q=0.5' \
+    -H 'Accept-Encoding: gzip, deflate, br, zstd' \
+    -H 'Referer: https://www.amd.com/' 2>&1 | tr '\r' '\n'
+  '';
+
+in
+  stdenv.mkDerivation {
+    pname = "AMD-uProf";
+    inherit version;
+    src = uprofSrc;
+    dontStrip = true;
+    phases = [ "installPhase" "fixupPhase" ];
+    nativeBuildInputs = [ autoPatchelfHook radare2 ];
+    buildInputs = [
+      stdenv.cc.cc.lib
+      ncurses5
+      elfutils
+      glib
+      libGL
+      libGLU
+      libxcrypt-legacy
+      xorg.libX11
+      xorg.libXext
+      xorg.libXi
+      xorg.libXmu
+      xorg.libxcb
+      xorg.xcbutilwm
+      xorg.xcbutilrenderutil
+      xorg.xcbutilkeysyms
+      xorg.xcbutilimage
+      fontconfig.lib
+      libxkbcommon
+      zlib
+      freetype
+      dbus
+      rocmPackages.rocprofiler
+      numactl
+    ];
+    installPhase = ''
+      set -x
+      mkdir -p $out
+      tar -x -v -C $out --strip-components=1 -f $src
+      rm $out/bin/AMDPowerProfilerDriverSource.tar.gz
+      patchelf --replace-needed libroctracer64.so.1 libroctracer64.so $out/bin/ProfileAgents/x64/libAMDGpuAgent.so
+      patchelf --add-needed libcrypt.so.1 --add-needed libstdc++.so.6 $out/bin/AMDuProfSys
+      echo "16334a51fcc48668307ad94e20482ca4  $out/bin/AMDuProfPcm" | md5sum -c -
+      radare2 -w -q -i ${./libnuma.r2} $out/bin/AMDuProfPcm
+      patchelf --add-needed libnuma.so $out/bin/AMDuProfPcm
+      set +x
+    '';
+
+    meta = {
+      description = "Performance analysis tool-suite for x86 based applications";
+      homepage = "https://www.amd.com/es/developer/uprof.html";
+      platforms = lib.platforms.linux;
+      license = lib.licenses.unfree;
+      maintainers = with lib.maintainers.bsc; [ rarias varcila ];
+    };
+
+  }

pkgs/amd-uprof/driver.nix

@@ -0,0 +1,35 @@
+{ stdenv
+, lib
+, amd-uprof
+, kernel
+, runCommandLocal
+}:
+
+let
+  version = amd-uprof.version;
+  tarball = amd-uprof.src;
+in stdenv.mkDerivation {
+  pname = "AMDPowerProfilerDriver";
+  inherit version;
+  src = runCommandLocal "AMDPowerProfilerDriverSource.tar.gz" { } ''
+    set -x
+    tar -x -f ${tarball} AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz
+    mv AMDuProf_Linux_x64_${version}/bin/AMDPowerProfilerDriverSource.tar.gz $out
+    set +x
+  '';
+  hardeningDisable = [ "pic" "format" ];
+  nativeBuildInputs = kernel.moduleBuildDependencies;
+  patches = [ ./makefile.patch ./hrtimer.patch ];
+  makeFlags = [
+    "KERNEL_VERSION=${kernel.modDirVersion}"
+    "KERNEL_DIR=${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
+    "INSTALL_MOD_PATH=$(out)"
+  ];
+  meta = {
+    description = "AMD Power Profiler Driver";
+    homepage = "https://www.amd.com/es/developer/uprof.html";
+    platforms = lib.platforms.linux;
+    license = lib.licenses.unfree;
+    maintainers = with lib.maintainers.bsc; [ rarias varcila ];
+  };
+}

pkgs/amd-uprof/hrtimer.patch

@@ -0,0 +1,31 @@
+--- a/src/PmcTimerConfig.c	2025-09-04 12:17:16.771707049 +0200
++++ b/src/PmcTimerConfig.c	2025-09-04 12:17:04.878515468 +0200
+@@ -99,7 +99,7 @@ static void PmcInitTimer(void* pInfo)
+ 
+     DRVPRINT("pTimerConfig(%p)", pTimerConfig);
+ 
+-    hrtimer_init(&pTimerConfig->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
++    hrtimer_setup(&pTimerConfig->m_hrTimer, PmcTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ }
+ 
+ int PmcSetupTimer(ClientContext* pClientCtx)
+@@ -157,7 +157,6 @@ int PmcSetupTimer(ClientContext* pClient
+                 {
+                     /* Interval in ms */
+                     pTimerConfig->m_time = ktime_set(interval / 1000, interval * 1000000);
+-                    pTimerConfig->m_hrTimer.function = PmcTimerCallback;
+ 
+                     DRVPRINT("retVal(%d) m_time(%lld)", retVal, (long long int) pTimerConfig->m_time);
+                 }
+--- a/src/PwrProfTimer.c	2025-09-04 12:18:08.750544327 +0200
++++ b/src/PwrProfTimer.c	2025-09-04 12:18:28.557863382 +0200
+@@ -573,8 +573,7 @@ void InitHrTimer(uint32 cpu)
+     pCoreClientData = &per_cpu(g_coreClientData, cpu);
+ 
+     // initialize HR timer
+-    hrtimer_init(&pCoreClientData->m_hrTimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+-    pCoreClientData->m_hrTimer.function = &HrTimerCallback;
++    hrtimer_setup(&pCoreClientData->m_hrTimer, &HrTimerCallback, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
+ 
+     return;
+ } // InitHrTimer

pkgs/amd-uprof/libnuma.r2

@@ -0,0 +1,10 @@
+# Patch arguments to call sym std::string::find(char const*, unsigned long, unsigned long)
+# so it matches NixOS:
+#
+# Change OS name to NixOS
+wz NixOS @ 0x00550a43
+# And set the length to 5 characters
+wa mov ecx, 5 @0x00517930
+#
+# Then change the argument to dlopen() so it only uses libnuma.so
+wz libnuma.so @ 0x00562940

pkgs/amd-uprof/makefile.patch

@@ -0,0 +1,66 @@
+--- a/Makefile	2025-06-19 20:36:49.346693267 +0200
++++ b/Makefile	2025-06-19 20:42:29.778088660 +0200
+@@ -27,7 +27,7 @@ MODULE_VERSION=$(shell cat AMDPowerProfi
+ MODULE_NAME_KO=$(MODULE_NAME).ko
+ 
+ # check is module inserted
+-MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
++#MODPROBE_OUTPUT=$(shell lsmod | grep $(MODULE_NAME))
+ 
+ # check pcore dkms status
+ PCORE_DKMS_STATUS=$(shell dkms status | grep $(MODULE_NAME) | grep $(MODULE_VERSION))
+@@ -50,7 +50,7 @@ endif
+ # “-Wno-missing-attributes” is added for GCC version >= 9.0 and kernel version <= 5.00
+ G_VERSION=9
+ K_VERSION=5
+-KERNEL_MAJOR_VERSION=$(shell uname -r | cut -f1 -d.)
++KERNEL_MAJOR_VERSION=$(shell echo "$(KERNEL_VERSION)" | cut -f1 -d.)
+ GCCVERSION = $(shell gcc -dumpversion | cut -f1 -d.)
+ ifeq ($(G_VERSION),$(firstword $(sort $(GCCVERSION) $(G_VERSION))))
+ 	ifeq ($(K_VERSION),$(lastword $(sort $(KERNEL_MAJOR_VERSION) $(K_VERSION))))
+@@ -66,17 +66,7 @@ ${MODULE_NAME}-objs :=  src/PmcDataBuffe
+ 
+ # make
+ all:
+-	@chmod a+x ./AMDPPcert.sh
+-	@./AMDPPcert.sh 0 1; echo $$? > $(PWD)/sign_status;
+-	@SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 1 ]; then \
+-			exit 1; \
+-		fi
+-	@make -C /lib/modules/$(KERNEL_VERSION)/build M=$(PWD) $(MAKE_OPTS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" modules
+-	@SIGSTATUS3=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS3 -eq 0 ]; then \
+-			./AMDPPcert.sh 1 $(MODULE_NAME_KO); \
+-		fi
++	make -C $(KERNEL_DIR) M=$(PWD) $(MAKE_OPTS) CFLAGS_MODULE="$(EXTRA_CFLAGS)" modules
+ 
+ # make clean
+ clean:
+@@ -84,23 +74,9 @@ clean:
+ 
+ # make install
+ install:
+-	@mkdir -p /lib/modules/`uname -r`/kernel/drivers/extra
+-	@rm  -f /lib/modules/`uname -r`/kernel/drivers/extra/$(MODULE_NAME_KO)
+-	@cp $(MODULE_NAME_KO) /lib/modules/`uname -r`/kernel/drivers/extra/
+-	@depmod -a
+-	@if [ ! -z "$(MODPROBE_OUTPUT)" ]; then \
+-		echo "Uninstalling AMDPowerProfiler Linux kernel module.";\
+-		rmmod $(MODULE_NAME);\
+-	fi
+-	@modprobe $(MODULE_NAME) 2> $(PWD)/sign_status1; \
+-		cat $(PWD)/sign_status1 | grep "Key was rejected by service"; \
+-		echo $$? > $(PWD)/sign_status; SIGSTATUS1=`cat $(PWD)/sign_status | tr -d '\n'`; \
+-                if [ $$SIGSTATUS1 -eq 0 ]; then \
+-			echo "ERROR: Secure Boot enabled, correct key is not yet enrolled in BIOS key table"; \
+-			exit 1; \
+-		else \
+-			cat $(PWD)/sign_status1; \
+-		fi
++	mkdir -p $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++	cp -a $(MODULE_NAME_KO) $(INSTALL_MOD_PATH)/lib/modules/$(KERNEL_VERSION)/kernel/drivers/extra/
++
+ # make dkms
+ dkms:
+ 	@chmod a+x ./AMDPPcert.sh

pkgs/babeltrace/default.nix

@@ -0,0 +1,25 @@
+{ stdenv, lib, fetchurl, pkg-config, glib, libuuid, popt, elfutils, swig4, python3 }:
+
+stdenv.mkDerivation rec {
+  name = "babeltrace-1.5.8";
+
+  src = fetchurl {
+    url = "https://www.efficios.com/files/babeltrace/${name}.tar.bz2";
+    sha256 = "1hkg3phnamxfrhwzmiiirbhdgckzfkqwhajl0lmr1wfps7j47wcz";
+  };
+
+  nativeBuildInputs = [ pkg-config ];
+  buildInputs = [ glib libuuid popt elfutils swig4 python3 ];
+
+  meta = with lib; {
+    description = "Command-line tool and library to read and convert LTTng tracefiles";
+    homepage = "https://www.efficios.com/babeltrace";
+    license = licenses.mit;
+    platforms = platforms.linux;
+    maintainers = [ maintainers.bjornfor ];
+  };
+
+  configureFlags = [
+    "--enable-python-bindings"
+  ];
+}

pkgs/babeltrace2/default.nix

@@ -0,0 +1,34 @@
+{
+  stdenv
+, fetchurl
+, pkg-config
+, glib
+, libuuid
+, popt
+, elfutils
+, python3
+, swig4
+, ncurses
+, breakpointHook
+}:
+
+stdenv.mkDerivation rec {
+  pname = "babeltrace2";
+  version = "2.0.3";
+
+  src = fetchurl {
+    url = "https://www.efficios.com/files/babeltrace/${pname}-${version}.tar.bz2";
+    sha256 = "1804pyq7fz6rkcz4r1abkkn0pfnss13m6fd8if32s42l4lajadm5";
+  };
+
+  enableParallelBuilding = true;
+  nativeBuildInputs = [ pkg-config ];
+  buildInputs = [ glib libuuid popt elfutils python3 swig4 ncurses breakpointHook ];
+  hardeningDisable = [ "all" ];
+
+  configureFlags = [
+    "--enable-python-plugins"
+    "--enable-python-bindings"
+  ];
+
+}

pkgs/bench6/default.nix

@@ -0,0 +1,70 @@
+{
+  stdenv
+, lib
+, bigotes
+, cmake
+, clangOmpss2
+, openmp
+, openmpv
+, nanos6
+, nodes
+, nosv
+, mpi
+, tampi
+, openblas
+, ovni
+, gitBranch ? "master"
+, gitURL ? "ssh://git@bscpm04.bsc.es/rarias/bench6.git"
+, gitCommit ? "bf29a53113737c3aa74d2fe3d55f59868faea7b4"
+}:
+
+stdenv.mkDerivation rec {
+  pname = "bench6";
+  version = "${src.shortRev}";
+
+  src = builtins.fetchGit {
+    url = gitURL;
+    ref = gitBranch;
+    rev = gitCommit;
+  };
+
+  nativeBuildInputs = [
+    cmake
+    clangOmpss2
+  ];
+
+  buildInputs = [
+    bigotes
+    openmp
+    openmpv
+    nanos6
+    nodes
+    nosv
+    mpi
+    tampi
+    openblas
+    openblas.dev
+    ovni
+  ];
+
+  env = {
+    NANOS6_HOME = nanos6;
+    NODES_HOME = nodes;
+    NOSV_HOME = nosv;
+  };
+
+  cmakeFlags = [
+    "-DCMAKE_C_COMPILER=clang"
+    "-DCMAKE_CXX_COMPILER=clang++"
+  ];
+  hardeningDisable = [ "all" ];
+  dontStrip = true;
+
+  meta = {
+    homepage = "https://gitlab.pm.bsc.es/rarias/bench6";
+    description = "Set of micro-benchmarks for OmpSs-2 and several mini-apps";
+    maintainers = with lib.maintainers.bsc; [ rarias ];
+    platforms = lib.platforms.linux;
+    license = lib.licenses.gpl3Plus;
+  };
+}

pkgs/bigotes/default.nix

@@ -0,0 +1,26 @@
+{
+  stdenv
+, lib
+, fetchFromGitHub
+, cmake
+}:
+
+stdenv.mkDerivation {
+  pname = "bigotes";
+  version = "9dce13";
+  src = fetchFromGitHub {
+    owner = "rodarima";
+    repo = "bigotes";
+    rev = "9dce13446a8da30bea552d569d260d54e0188518";
+    sha256 = "sha256-ktxM3pXiL8YXSK+/IKWYadijhYXqGoLY6adLk36iigE=";
+  };
+  nativeBuildInputs = [ cmake ];
+
+  meta = {
+    homepage = "https://github.com/rodarima/bigotes";
+    description = "Versatile benchmark tool";
+    maintainers = with lib.maintainers.bsc; [ rarias ];
+    platforms = lib.platforms.linux;
+    license = lib.licenses.gpl3Plus;
+  };
+}

pkgs/clsync/default.nix

@@ -0,0 +1,54 @@
+{ stdenv
+, fetchFromGitHub
+, libcap
+, libcgroup
+, libmhash
+, doxygen
+, graphviz
+, autoreconfHook
+, pkg-config
+, glib
+}:
+
+let
+  version = "0.4.4";
+
+in stdenv.mkDerivation {
+  pname = "clsync";
+  inherit version;
+
+  src = fetchFromGitHub {
+    repo = "clsync";
+    owner = "clsync";
+    rev = "v${version}";
+    sha256 = "0sdiyfwp0iqr6l1sirm51pirzmhi4jzgky5pzfj24nn71q3fwqgz";
+  };
+
+  outputs = [ "out" "dev" ];
+
+  buildInputs = [
+    autoreconfHook
+    libcap
+    libcgroup
+    libmhash
+    doxygen
+    graphviz
+    pkg-config
+    glib
+  ];
+
+  preConfigure = ''
+    ./configure --help
+  '';
+
+  enableParallelBuilding = true;
+
+  meta = with lib; {
+    description = "File live sync daemon based on inotify/kqueue/bsm (Linux, FreeBSD), written in GNU C";
+    homepage = "https://github.com/clsync/clsync";
+    license = licenses.gpl3Plus;
+    maintainers = [ ];
+    platforms = platforms.linux;
+  };
+}
+

pkgs/cn6/default.nix

@@ -0,0 +1,51 @@
+{
+  stdenv
+, lib
+, babeltrace2
+, pkg-config
+, uthash
+, enableTest ? false
+, mpi ? null
+, clangOmpss2 ? null
+, tampi ? null
+}:
+
+with lib;
+
+assert (enableTest -> (mpi != null));
+assert (enableTest -> (clangOmpss2 != null));
+assert (enableTest -> (tampi != null));
+
+stdenv.mkDerivation rec {
+  pname = "cn6";
+  version = "${src.shortRev}";
+
+  buildInputs = [
+    babeltrace2
+    pkg-config
+    uthash
+    mpi
+  ] ++ optionals (enableTest) [ mpi clangOmpss2 tampi ];
+
+  src = builtins.fetchGit {
+    url = "ssh://git@bscpm04.bsc.es/rarias/cn6.git";
+    ref = "master";
+    rev = "c72c3b66b720c2a33950f536fc819051c8f20a69";
+  };
+
+  makeFlags = [ "PREFIX=$(out)" ];
+
+  postBuild = optionalString (enableTest) ''
+    (
+      cd test
+      make timediff timediff_mpi
+    )
+  '';
+
+  postInstall = optionalString (enableTest) ''
+    (
+      cd test
+      cp timediff timediff_mpi sync-err.sh $out/bin/
+    )
+  '';
+}

pkgs/cpuid/default.nix

@@ -0,0 +1,21 @@
+{
+  stdenv
+, perl # For the pod2man command
+}:
+
+stdenv.mkDerivation rec {
+  version = "20201006";
+  pname = "cpuid";
+
+  buildInputs = [ perl ];
+
+  # Replace /usr install directory for $out
+  postPatch = ''
+    sed -i "s@/usr@$out@g" Makefile
+  '';
+
+  src = builtins.fetchTarball {
+    url = "http://www.etallen.com/cpuid/${pname}-${version}.src.tar.gz";
+    sha256 = "04qhs938gs1kjxpsrnfy6lbsircsprfyh4db62s5cf83a1nrwn9w";
+  };
+}

pkgs/cudainfo/Makefile

@@ -0,0 +1,12 @@
+HOSTCXX  ?= g++
+NVCC     := nvcc -ccbin $(HOSTCXX)
+CXXFLAGS := -m64
+
+# Target rules
+all: cudainfo
+
+cudainfo: cudainfo.cpp
+	$(NVCC) $(CXXFLAGS) -o $@ $<
+
+clean:
+	rm -f cudainfo cudainfo.o

pkgs/cudainfo/cudainfo.cpp

@@ -0,0 +1,600 @@
+/*
+ * Copyright 1993-2015 NVIDIA Corporation.  All rights reserved.
+ *
+ * Please refer to the NVIDIA end user license agreement (EULA) associated
+ * with this source code for terms and conditions that govern your use of
+ * this software. Any use, reproduction, disclosure, or distribution of
+ * this software and related documentation outside the terms of the EULA
+ * is strictly prohibited.
+ *
+ */
+/* This sample queries the properties of the CUDA devices present in the system via CUDA Runtime API. */
+
+// Shared Utilities (QA Testing)
+
+// std::system includes
+#include <memory>
+#include <iostream>
+
+#include <cuda_runtime.h>
+
+// This will output the proper CUDA error strings in the event that a CUDA host call returns an error
+#define checkCudaErrors(val)           check ( (val), #val, __FILE__, __LINE__ )
+
+// CUDA Runtime error messages
+#ifdef __DRIVER_TYPES_H__
+static const char *_cudaGetErrorEnum(cudaError_t error)
+{
+    switch (error)
+    {
+        case cudaSuccess:
+            return "cudaSuccess";
+
+        case cudaErrorMissingConfiguration:
+            return "cudaErrorMissingConfiguration";
+
+        case cudaErrorMemoryAllocation:
+            return "cudaErrorMemoryAllocation";
+
+        case cudaErrorInitializationError:
+            return "cudaErrorInitializationError";
+
+        case cudaErrorLaunchFailure:
+            return "cudaErrorLaunchFailure";
+
+        case cudaErrorPriorLaunchFailure:
+            return "cudaErrorPriorLaunchFailure";
+
+        case cudaErrorLaunchTimeout:
+            return "cudaErrorLaunchTimeout";
+
+        case cudaErrorLaunchOutOfResources:
+            return "cudaErrorLaunchOutOfResources";
+
+        case cudaErrorInvalidDeviceFunction:
+            return "cudaErrorInvalidDeviceFunction";
+
+        case cudaErrorInvalidConfiguration:
+            return "cudaErrorInvalidConfiguration";
+
+        case cudaErrorInvalidDevice:
+            return "cudaErrorInvalidDevice";
+
+        case cudaErrorInvalidValue:
+            return "cudaErrorInvalidValue";
+
+        case cudaErrorInvalidPitchValue:
+            return "cudaErrorInvalidPitchValue";
+
+        case cudaErrorInvalidSymbol:
+            return "cudaErrorInvalidSymbol";
+
+        case cudaErrorMapBufferObjectFailed:
+            return "cudaErrorMapBufferObjectFailed";
+
+        case cudaErrorUnmapBufferObjectFailed:
+            return "cudaErrorUnmapBufferObjectFailed";
+
+        case cudaErrorInvalidHostPointer:
+            return "cudaErrorInvalidHostPointer";
+
+        case cudaErrorInvalidDevicePointer:
+            return "cudaErrorInvalidDevicePointer";
+
+        case cudaErrorInvalidTexture:
+            return "cudaErrorInvalidTexture";
+
+        case cudaErrorInvalidTextureBinding:
+            return "cudaErrorInvalidTextureBinding";
+
+        case cudaErrorInvalidChannelDescriptor:
+            return "cudaErrorInvalidChannelDescriptor";
+
+        case cudaErrorInvalidMemcpyDirection:
+            return "cudaErrorInvalidMemcpyDirection";
+
+        case cudaErrorAddressOfConstant:
+            return "cudaErrorAddressOfConstant";
+
+        case cudaErrorTextureFetchFailed:
+            return "cudaErrorTextureFetchFailed";
+
+        case cudaErrorTextureNotBound:
+            return "cudaErrorTextureNotBound";
+
+        case cudaErrorSynchronizationError:
+            return "cudaErrorSynchronizationError";
+
+        case cudaErrorInvalidFilterSetting:
+            return "cudaErrorInvalidFilterSetting";
+
+        case cudaErrorInvalidNormSetting:
+            return "cudaErrorInvalidNormSetting";
+
+        case cudaErrorMixedDeviceExecution:
+            return "cudaErrorMixedDeviceExecution";
+
+        case cudaErrorCudartUnloading:
+            return "cudaErrorCudartUnloading";
+
+        case cudaErrorUnknown:
+            return "cudaErrorUnknown";
+
+        case cudaErrorNotYetImplemented:
+            return "cudaErrorNotYetImplemented";
+
+        case cudaErrorMemoryValueTooLarge:
+            return "cudaErrorMemoryValueTooLarge";
+
+        case cudaErrorInvalidResourceHandle:
+            return "cudaErrorInvalidResourceHandle";
+
+        case cudaErrorNotReady:
+            return "cudaErrorNotReady";
+
+        case cudaErrorInsufficientDriver:
+            return "cudaErrorInsufficientDriver";
+
+        case cudaErrorSetOnActiveProcess:
+            return "cudaErrorSetOnActiveProcess";
+
+        case cudaErrorInvalidSurface:
+            return "cudaErrorInvalidSurface";
+
+        case cudaErrorNoDevice:
+            return "cudaErrorNoDevice";
+
+        case cudaErrorECCUncorrectable:
+            return "cudaErrorECCUncorrectable";
+
+        case cudaErrorSharedObjectSymbolNotFound:
+            return "cudaErrorSharedObjectSymbolNotFound";
+
+        case cudaErrorSharedObjectInitFailed:
+            return "cudaErrorSharedObjectInitFailed";
+
+        case cudaErrorUnsupportedLimit:
+            return "cudaErrorUnsupportedLimit";
+
+        case cudaErrorDuplicateVariableName:
+            return "cudaErrorDuplicateVariableName";
+
+        case cudaErrorDuplicateTextureName:
+            return "cudaErrorDuplicateTextureName";
+
+        case cudaErrorDuplicateSurfaceName:
+            return "cudaErrorDuplicateSurfaceName";
+
+        case cudaErrorDevicesUnavailable:
+            return "cudaErrorDevicesUnavailable";
+
+        case cudaErrorInvalidKernelImage:
+            return "cudaErrorInvalidKernelImage";
+
+        case cudaErrorNoKernelImageForDevice:
+            return "cudaErrorNoKernelImageForDevice";
+
+        case cudaErrorIncompatibleDriverContext:
+            return "cudaErrorIncompatibleDriverContext";
+
+        case cudaErrorPeerAccessAlreadyEnabled:
+            return "cudaErrorPeerAccessAlreadyEnabled";
+
+        case cudaErrorPeerAccessNotEnabled:
+            return "cudaErrorPeerAccessNotEnabled";
+
+        case cudaErrorDeviceAlreadyInUse:
+            return "cudaErrorDeviceAlreadyInUse";
+
+        case cudaErrorProfilerDisabled:
+            return "cudaErrorProfilerDisabled";
+
+        case cudaErrorProfilerNotInitialized:
+            return "cudaErrorProfilerNotInitialized";
+
+        case cudaErrorProfilerAlreadyStarted:
+            return "cudaErrorProfilerAlreadyStarted";
+
+        case cudaErrorProfilerAlreadyStopped:
+            return "cudaErrorProfilerAlreadyStopped";
+
+        /* Since CUDA 4.0*/
+        case cudaErrorAssert:
+            return "cudaErrorAssert";
+
+        case cudaErrorTooManyPeers:
+            return "cudaErrorTooManyPeers";
+
+        case cudaErrorHostMemoryAlreadyRegistered:
+            return "cudaErrorHostMemoryAlreadyRegistered";
+
+        case cudaErrorHostMemoryNotRegistered:
+            return "cudaErrorHostMemoryNotRegistered";
+
+        /* Since CUDA 5.0 */
+        case cudaErrorOperatingSystem:
+            return "cudaErrorOperatingSystem";
+
+        case cudaErrorPeerAccessUnsupported:
+            return "cudaErrorPeerAccessUnsupported";
+
+        case cudaErrorLaunchMaxDepthExceeded:
+            return "cudaErrorLaunchMaxDepthExceeded";
+
+        case cudaErrorLaunchFileScopedTex:
+            return "cudaErrorLaunchFileScopedTex";
+
+        case cudaErrorLaunchFileScopedSurf:
+            return "cudaErrorLaunchFileScopedSurf";
+
+        case cudaErrorSyncDepthExceeded:
+            return "cudaErrorSyncDepthExceeded";
+
+        case cudaErrorLaunchPendingCountExceeded:
+            return "cudaErrorLaunchPendingCountExceeded";
+
+        case cudaErrorNotPermitted:
+            return "cudaErrorNotPermitted";
+
+        case cudaErrorNotSupported:
+            return "cudaErrorNotSupported";
+
+        /* Since CUDA 6.0 */
+        case cudaErrorHardwareStackError:
+            return "cudaErrorHardwareStackError";
+
+        case cudaErrorIllegalInstruction:
+            return "cudaErrorIllegalInstruction";
+
+        case cudaErrorMisalignedAddress:
+            return "cudaErrorMisalignedAddress";
+
+        case cudaErrorInvalidAddressSpace:
+            return "cudaErrorInvalidAddressSpace";
+
+        case cudaErrorInvalidPc:
+            return "cudaErrorInvalidPc";
+
+        case cudaErrorIllegalAddress:
+            return "cudaErrorIllegalAddress";
+
+        /* Since CUDA 6.5*/
+        case cudaErrorInvalidPtx:
+            return "cudaErrorInvalidPtx";
+
+        case cudaErrorInvalidGraphicsContext:
+            return "cudaErrorInvalidGraphicsContext";
+
+        case cudaErrorStartupFailure:
+            return "cudaErrorStartupFailure";
+
+        case cudaErrorApiFailureBase:
+            return "cudaErrorApiFailureBase";
+    }
+
+    return "<unknown>";
+}
+#endif
+
+template< typename T >
+void check(T result, char const *const func, const char *const file, int const line)
+{
+    if (result)
+    {
+        fprintf(stderr, "CUDA error at %s:%d code=%d(%s) \"%s\" \n",
+                file, line, static_cast<unsigned int>(result), _cudaGetErrorEnum(result), func);
+        cudaDeviceReset();
+        // Make sure we call CUDA Device Reset before exiting
+        exit(EXIT_FAILURE);
+    }
+}
+
+int *pArgc = NULL;
+char **pArgv = NULL;
+
+#if CUDART_VERSION < 5000
+
+// CUDA-C includes
+#include <cuda.h>
+
+// This function wraps the CUDA Driver API into a template function
+template <class T>
+inline void getCudaAttribute(T *attribute, CUdevice_attribute device_attribute, int device)
+{
+    CUresult error =    cuDeviceGetAttribute(attribute, device_attribute, device);
+
+    if (CUDA_SUCCESS != error) {
+        fprintf(stderr, "cuSafeCallNoSync() Driver API error = %04d from file <%s>, line %i.\n",
+                error, __FILE__, __LINE__);
+
+        // cudaDeviceReset causes the driver to clean up all state. While
+        // not mandatory in normal operation, it is good practice.  It is also
+        // needed to ensure correct operation when the application is being
+        // profiled. Calling cudaDeviceReset causes all profile data to be
+        // flushed before the application exits
+        cudaDeviceReset();
+        exit(EXIT_FAILURE);
+    }
+}
+
+#endif /* CUDART_VERSION < 5000 */
+
+// Beginning of GPU Architecture definitions
+inline int ConvertSMVer2Cores(int major, int minor)
+{
+    // Defines for GPU Architecture types (using the SM version to determine the # of cores per SM
+    typedef struct {
+        int SM; // 0xMm (hexidecimal notation), M = SM Major version, and m = SM minor version
+        int Cores;
+    } sSMtoCores;
+
+    sSMtoCores nGpuArchCoresPerSM[] = {
+        { 0x20, 32 }, // Fermi Generation (SM 2.0) GF100 class
+        { 0x21, 48 }, // Fermi Generation (SM 2.1) GF10x class
+        { 0x30, 192}, // Kepler Generation (SM 3.0) GK10x class
+        { 0x32, 192}, // Kepler Generation (SM 3.2) GK10x class
+        { 0x35, 192}, // Kepler Generation (SM 3.5) GK11x class
+        { 0x37, 192}, // Kepler Generation (SM 3.7) GK21x class
+        { 0x50, 128}, // Maxwell Generation (SM 5.0) GM10x class
+        { 0x52, 128}, // Maxwell Generation (SM 5.2) GM20x class
+        {   -1, -1 }
+    };
+
+    int index = 0;
+
+    while (nGpuArchCoresPerSM[index].SM != -1) {
+        if (nGpuArchCoresPerSM[index].SM == ((major << 4) + minor)) {
+            return nGpuArchCoresPerSM[index].Cores;
+        }
+
+        index++;
+    }
+
+    // If we don't find the values, we default use the previous one to run properly
+    printf("MapSMtoCores for SM %d.%d is undefined.  Default to use %d Cores/SM\n", major, minor, nGpuArchCoresPerSM[index-1].Cores);
+    return nGpuArchCoresPerSM[index-1].Cores;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Program main
+////////////////////////////////////////////////////////////////////////////////
+int
+main(int argc, char **argv)
+{
+    pArgc = &argc;
+    pArgv = argv;
+
+    printf("%s Starting...\n\n", argv[0]);
+    printf(" CUDA Device Query (Runtime API) version (CUDART static linking)\n\n");
+
+    int deviceCount = 0;
+    cudaError_t error_id = cudaGetDeviceCount(&deviceCount);
+
+    if (error_id != cudaSuccess) {
+        printf("cudaGetDeviceCount failed: %s (%d)\n",
+			cudaGetErrorString(error_id), (int) error_id);
+        printf("Result = FAIL\n");
+        exit(EXIT_FAILURE);
+    }
+
+    // This function call returns 0 if there are no CUDA capable devices.
+    if (deviceCount == 0)
+        printf("There are no available device(s) that support CUDA\n");
+    else
+        printf("Detected %d CUDA Capable device(s)\n", deviceCount);
+
+    int dev, driverVersion = 0, runtimeVersion = 0;
+
+    for (dev = 0; dev < deviceCount; ++dev) {
+        cudaSetDevice(dev);
+        cudaDeviceProp deviceProp;
+        cudaGetDeviceProperties(&deviceProp, dev);
+
+        printf("\nDevice %d: \"%s\"\n", dev, deviceProp.name);
+
+        // Console log
+        cudaDriverGetVersion(&driverVersion);
+        cudaRuntimeGetVersion(&runtimeVersion);
+        printf("  CUDA Driver Version / Runtime Version          %d.%d / %d.%d\n", driverVersion/1000, (driverVersion%100)/10, runtimeVersion/1000, (runtimeVersion%100)/10);
+        printf("  CUDA Capability Major/Minor version number:    %d.%d\n", deviceProp.major, deviceProp.minor);
+
+        printf("  Total amount of global memory:                 %.0f MBytes (%llu bytes)\n",
+                (float)deviceProp.totalGlobalMem/1048576.0f, (unsigned long long) deviceProp.totalGlobalMem);
+
+        printf("  (%2d) Multiprocessors, (%3d) CUDA Cores/MP:     %d CUDA Cores\n",
+               deviceProp.multiProcessorCount,
+               ConvertSMVer2Cores(deviceProp.major, deviceProp.minor),
+               ConvertSMVer2Cores(deviceProp.major, deviceProp.minor) * deviceProp.multiProcessorCount);
+        printf("  GPU Max Clock rate:                            %.0f MHz (%0.2f GHz)\n", deviceProp.clockRate * 1e-3f, deviceProp.clockRate * 1e-6f);
+
+
+#if CUDART_VERSION >= 5000
+        // This is supported in CUDA 5.0 (runtime API device properties)
+        printf("  Memory Clock rate:                             %.0f Mhz\n", deviceProp.memoryClockRate * 1e-3f);
+        printf("  Memory Bus Width:                              %d-bit\n",   deviceProp.memoryBusWidth);
+
+        if (deviceProp.l2CacheSize) {
+            printf("  L2 Cache Size:                                 %d bytes\n", deviceProp.l2CacheSize);
+        }
+
+#else
+        // This only available in CUDA 4.0-4.2 (but these were only exposed in the CUDA Driver API)
+        int memoryClock;
+        getCudaAttribute<int>(&memoryClock, CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, dev);
+        printf("  Memory Clock rate:                             %.0f Mhz\n", memoryClock * 1e-3f);
+        int memBusWidth;
+        getCudaAttribute<int>(&memBusWidth, CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, dev);
+        printf("  Memory Bus Width:                              %d-bit\n", memBusWidth);
+        int L2CacheSize;
+        getCudaAttribute<int>(&L2CacheSize, CU_DEVICE_ATTRIBUTE_L2_CACHE_SIZE, dev);
+
+        if (L2CacheSize) {
+            printf("  L2 Cache Size:                                 %d bytes\n", L2CacheSize);
+        }
+
+#endif
+
+        printf("  Maximum Texture Dimension Size (x,y,z)         1D=(%d), 2D=(%d, %d), 3D=(%d, %d, %d)\n",
+               deviceProp.maxTexture1D   , deviceProp.maxTexture2D[0], deviceProp.maxTexture2D[1],
+               deviceProp.maxTexture3D[0], deviceProp.maxTexture3D[1], deviceProp.maxTexture3D[2]);
+        printf("  Maximum Layered 1D Texture Size, (num) layers  1D=(%d), %d layers\n",
+               deviceProp.maxTexture1DLayered[0], deviceProp.maxTexture1DLayered[1]);
+        printf("  Maximum Layered 2D Texture Size, (num) layers  2D=(%d, %d), %d layers\n",
+               deviceProp.maxTexture2DLayered[0], deviceProp.maxTexture2DLayered[1], deviceProp.maxTexture2DLayered[2]);
+
+
+        printf("  Total amount of constant memory:               %lu bytes\n", deviceProp.totalConstMem);
+        printf("  Total amount of shared memory per block:       %lu bytes\n", deviceProp.sharedMemPerBlock);
+        printf("  Total number of registers available per block: %d\n", deviceProp.regsPerBlock);
+        printf("  Warp size:                                     %d\n", deviceProp.warpSize);
+        printf("  Maximum number of threads per multiprocessor:  %d\n", deviceProp.maxThreadsPerMultiProcessor);
+        printf("  Maximum number of threads per block:           %d\n", deviceProp.maxThreadsPerBlock);
+        printf("  Max dimension size of a thread block (x,y,z): (%d, %d, %d)\n",
+               deviceProp.maxThreadsDim[0],
+               deviceProp.maxThreadsDim[1],
+               deviceProp.maxThreadsDim[2]);
+        printf("  Max dimension size of a grid size    (x,y,z): (%d, %d, %d)\n",
+               deviceProp.maxGridSize[0],
+               deviceProp.maxGridSize[1],
+               deviceProp.maxGridSize[2]);
+        printf("  Maximum memory pitch:                          %lu bytes\n", deviceProp.memPitch);
+        printf("  Texture alignment:                             %lu bytes\n", deviceProp.textureAlignment);
+        printf("  Concurrent copy and kernel execution:          %s with %d copy engine(s)\n", (deviceProp.deviceOverlap ? "Yes" : "No"), deviceProp.asyncEngineCount);
+        printf("  Run time limit on kernels:                     %s\n", deviceProp.kernelExecTimeoutEnabled ? "Yes" : "No");
+        printf("  Integrated GPU sharing Host Memory:            %s\n", deviceProp.integrated ? "Yes" : "No");
+        printf("  Support host page-locked memory mapping:       %s\n", deviceProp.canMapHostMemory ? "Yes" : "No");
+        printf("  Alignment requirement for Surfaces:            %s\n", deviceProp.surfaceAlignment ? "Yes" : "No");
+        printf("  Device has ECC support:                        %s\n", deviceProp.ECCEnabled ? "Enabled" : "Disabled");
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+        printf("  CUDA Device Driver Mode (TCC or WDDM):         %s\n", deviceProp.tccDriver ? "TCC (Tesla Compute Cluster Driver)" : "WDDM (Windows Display Driver Model)");
+#endif
+        printf("  Device supports Unified Addressing (UVA):      %s\n", deviceProp.unifiedAddressing ? "Yes" : "No");
+        printf("  Device PCI Domain ID / Bus ID / location ID:   %d / %d / %d\n", deviceProp.pciDomainID, deviceProp.pciBusID, deviceProp.pciDeviceID);
+
+        const char *sComputeMode[] = {
+            "Default (multiple host threads can use ::cudaSetDevice() with device simultaneously)",
+            "Exclusive (only one host thread in one process is able to use ::cudaSetDevice() with this device)",
+            "Prohibited (no host thread can use ::cudaSetDevice() with this device)",
+            "Exclusive Process (many threads in one process is able to use ::cudaSetDevice() with this device)",
+            "Unknown",
+            NULL
+        };
+        printf("  Compute Mode:\n");
+        printf("     < %s >\n", sComputeMode[deviceProp.computeMode]);
+    }
+
+    // If there are 2 or more GPUs, query to determine whether RDMA is supported
+    if (deviceCount >= 2)
+    {
+        cudaDeviceProp prop[64];
+        int gpuid[64]; // we want to find the first two GPU's that can support P2P
+        int gpu_p2p_count = 0;
+
+        for (int i=0; i < deviceCount; i++)
+        {
+            checkCudaErrors(cudaGetDeviceProperties(&prop[i], i));
+
+            // Only boards based on Fermi or later can support P2P
+            if ((prop[i].major >= 2)
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+                // on Windows (64-bit), the Tesla Compute Cluster driver for windows must be enabled to supprot this
+                && prop[i].tccDriver
+#endif
+               )
+            {
+                // This is an array of P2P capable GPUs
+                gpuid[gpu_p2p_count++] = i;
+            }
+        }
+
+        // Show all the combinations of support P2P GPUs
+        int can_access_peer_0_1, can_access_peer_1_0;
+
+        if (gpu_p2p_count >= 2)
+        {
+            for (int i = 0; i < gpu_p2p_count-1; i++)
+            {
+                for (int j = 1; j < gpu_p2p_count; j++)
+                {
+                    checkCudaErrors(cudaDeviceCanAccessPeer(&can_access_peer_0_1, gpuid[i], gpuid[j]));
+                    printf("> Peer access from %s (GPU%d) -> %s (GPU%d) : %s\n", prop[gpuid[i]].name, gpuid[i],
+                           prop[gpuid[j]].name, gpuid[j] ,
+                           can_access_peer_0_1 ? "Yes" : "No");
+                }
+            }
+
+            for (int j = 1; j < gpu_p2p_count; j++)
+            {
+                for (int i = 0; i < gpu_p2p_count-1; i++)
+                {
+                    checkCudaErrors(cudaDeviceCanAccessPeer(&can_access_peer_1_0, gpuid[j], gpuid[i]));
+                    printf("> Peer access from %s (GPU%d) -> %s (GPU%d) : %s\n", prop[gpuid[j]].name, gpuid[j],
+                           prop[gpuid[i]].name, gpuid[i] ,
+                           can_access_peer_1_0 ? "Yes" : "No");
+                }
+            }
+        }
+    }
+
+    // csv masterlog info
+    // *****************************
+    // exe and CUDA driver name
+    printf("\n");
+    std::string sProfileString = "deviceQuery, CUDA Driver = CUDART";
+    char cTemp[128];
+
+    // driver version
+    sProfileString += ", CUDA Driver Version = ";
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+    sprintf_s(cTemp, 10, "%d.%d", driverVersion/1000, (driverVersion%100)/10);
+#else
+    sprintf(cTemp, "%d.%d", driverVersion/1000, (driverVersion%100)/10);
+#endif
+    sProfileString +=  cTemp;
+
+    // Runtime version
+    sProfileString += ", CUDA Runtime Version = ";
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+    sprintf_s(cTemp, 10, "%d.%d", runtimeVersion/1000, (runtimeVersion%100)/10);
+#else
+    sprintf(cTemp, "%d.%d", runtimeVersion/1000, (runtimeVersion%100)/10);
+#endif
+    sProfileString +=  cTemp;
+
+    // Device count
+    sProfileString += ", NumDevs = ";
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+    sprintf_s(cTemp, 10, "%d", deviceCount);
+#else
+    sprintf(cTemp, "%d", deviceCount);
+#endif
+    sProfileString += cTemp;
+
+    // Print Out all device Names
+    for (dev = 0; dev < deviceCount; ++dev)
+    {
+#if defined(WIN32) || defined(_WIN32) || defined(WIN64) || defined(_WIN64)
+        sprintf_s(cTemp, 13, ", Device%d = ", dev);
+#else
+        sprintf(cTemp, ", Device%d = ", dev);
+#endif
+        cudaDeviceProp deviceProp;
+        cudaGetDeviceProperties(&deviceProp, dev);
+        sProfileString += cTemp;
+        sProfileString += deviceProp.name;
+    }
+
+    sProfileString += "\n";
+    printf("%s", sProfileString.c_str());
+
+    printf("Result = PASS\n");
+
+    // finish
+    // cudaDeviceReset causes the driver to clean up all state. While
+    // not mandatory in normal operation, it is good practice.  It is also
+    // needed to ensure correct operation when the application is being
+    // profiled. Calling cudaDeviceReset causes all profile data to be
+    // flushed before the application exits
+    cudaDeviceReset();
+    return 0;
+}

pkgs/cudainfo/default.nix

@@ -0,0 +1,43 @@
+{
+  stdenv
+, cudatoolkit
+, cudaPackages
+, autoAddDriverRunpath
+, strace
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  name = "cudainfo";
+  src = ./.;
+  buildInputs = [
+    cudatoolkit # Required for nvcc
+    cudaPackages.cuda_cudart.static # Required for -lcudart_static
+    autoAddDriverRunpath
+  ];
+  installPhase = ''
+    mkdir -p $out/bin
+    cp -a cudainfo $out/bin
+  '';
+  passthru.gpuCheck = stdenv.mkDerivation {
+    name = "cudainfo-test";
+    requiredSystemFeatures = [ "cuda" ];
+    dontBuild = true;
+    nativeCheckInputs = [
+      finalAttrs.finalPackage # The cudainfo package from above
+      strace # When it fails, it will show the trace
+    ];
+    dontUnpack = true;
+    doCheck = true;
+    checkPhase = ''
+      if ! cudainfo; then
+        set -x
+        cudainfo=$(command -v cudainfo)
+        ldd $cudainfo
+        readelf -d $cudainfo
+        strace -f $cudainfo
+        set +x
+      fi
+    '';
+    installPhase = "touch $out";
+  };
+})

pkgs/dummy/default.nix

@@ -0,0 +1,25 @@
+{
+  stdenv
+}:
+
+stdenv.mkDerivation rec {
+  name = "dummy";
+
+  src = null;
+  dontUnpack = true;
+  dontBuild = true;
+
+  programPath = "/bin/dummy";
+
+  installPhase = ''
+    mkdir -p $out/bin
+
+    cat > $out/bin/dummy <<EOF
+    #!/bin/sh
+    echo Hello worlda!
+
+    EOF
+
+    chmod +x $out/bin/dummy
+  '';
+}

pkgs/extrae/PTR.patch

@@ -0,0 +1,13 @@
+diff --git a/src/merger/common/bfd_manager.c b/src/merger/common/bfd_manager.c
+index 5f9dacf9..5231e3eb 100644
+--- a/src/merger/common/bfd_manager.c
++++ b/src/merger/common/bfd_manager.c
+@@ -225,7 +225,7 @@ asymbol **BFDmanager_getDefaultSymbols (void)
+  *
+  * @return No return value.
+  */
+-static void BFDmanager_findAddressInSection (bfd * abfd, asection * section, PTR data)
++static void BFDmanager_findAddressInSection (bfd * abfd, asection * section, void * data)
+ {
+ #if HAVE_BFD_GET_SECTION_SIZE || HAVE_BFD_SECTION_SIZE || HAVE_BFD_GET_SECTION_SIZE_BEFORE_RELOC
+ 	bfd_size_type size;

pkgs/extrae/default.nix

@@ -0,0 +1,123 @@
+{ stdenv
+, lib
+, fetchFromGitHub
+, boost
+, libdwarf
+, libelf
+, libxml2
+, libunwind
+, papi
+, binutils-unwrapped
+, libiberty
+, gfortran
+, xml2
+, which
+, libbfd
+, mpi ? null
+, cuda ? null
+, llvmPackages
+, autoreconfHook
+#, python3Packages
+, installShellFiles
+, symlinkJoin
+, enablePapi ? stdenv.hostPlatform == stdenv.buildPlatform # Disabled when cross-compiling
+}:
+
+let
+  libdwarfBundle = symlinkJoin {
+    name = "libdwarfBundle";
+    paths = [ libdwarf.dev libdwarf.lib libdwarf.out ];
+  };
+in
+
+stdenv.mkDerivation rec {
+  pname = "extrae";
+  version = "4.0.1";
+  src = fetchFromGitHub {
+    owner = "bsc-performance-tools";
+    repo = "extrae";
+    rev = "${version}";
+    sha256 = "SlMYxNQXJ0Xg90HmpnotUR3tEPVVBXhk1NtEBJwGBR4=";
+  };
+
+  patches = [
+    # FIXME: Waiting for German to merge this patch. Still not in master, merged
+    # on 2023-03-01 in devel branch (after 3 years), see:
+    # https://github.com/bsc-performance-tools/extrae/pull/45
+    ./use-command.patch
+    # https://github.com/bsc-performance-tools/extrae/issues/71
+    ./PTR.patch
+  ];
+
+  enableParallelBuilding = true;
+  hardeningDisable = [ "all" ];
+
+  nativeBuildInputs = [ installShellFiles ];
+
+  buildInputs = [
+    autoreconfHook
+    gfortran
+    libunwind
+    binutils-unwrapped
+    boost
+    boost.dev
+    libiberty
+    mpi
+    xml2
+    which
+    libxml2.dev
+    libbfd
+    #python3Packages.sphinx
+  ]
+  ++ lib.optional stdenv.cc.isClang llvmPackages.openmp;
+    
+  preConfigure = ''
+    configureFlagsArray=(
+      --enable-posix-clock
+      --with-binutils="${binutils-unwrapped} ${libiberty}"
+      --with-dwarf=${libdwarfBundle}
+      --with-elf=${libelf}
+      --with-boost=${boost.dev}
+      --enable-instrument-io
+      --enable-instrument-dynamic-memory
+      --without-memkind
+      --enable-merge-in-trace
+      --disable-online
+      --without-opencl
+      --enable-pebs-sampling
+      --enable-sampling
+      --with-unwind=${libunwind.dev}
+      --with-xml-prefix=${libxml2.dev}
+      ${lib.optionalString enablePapi "--with-papi=${papi}"}
+      ${if (mpi != null) then ''--with-mpi=${mpi}''
+        else ''--without-mpi''}
+      --without-dyninst)
+  '';
+
+  # Install the manuals only by hand, as we don't want to pull the complete
+  # LaTeX world
+
+  # FIXME: sphinx is broken
+  #postBuild = ''
+  #  make -C docs man
+  #'';
+  #
+  #postInstall = ''
+  #  installManPage docs/builds/man/*/*
+  #'';
+
+#  ++ (
+#    if (openmp)
+#    then [ "--enable-openmp" ]
+#    else []
+#  );
+
+  meta = {
+    homepage = "https://github.com/bsc-performance-tools/extrae";
+    description = "Instrumentation framework to generate execution traces of the most used parallel runtimes";
+    maintainers = [ ];
+    broken = true;
+    platforms = lib.platforms.linux;
+    license = lib.licenses.lgpl21Plus;
+  };
+}

pkgs/extrae/use-command.patch

@@ -0,0 +1,24 @@
+diff --git a/substitute b/substitute
+index d5615606..82ca91a5 100755
+--- a/substitute
++++ b/substitute
+@@ -16,7 +16,7 @@ UNAME=`uname`
+ if [ "${UNAME}" = "Darwin" -o "${UNAME}" = "AIX" ] ; then
+ 	TMPFILE=substitute-$$
+ 	${SED} "s|${KEY}|${VALUE}|g" < ${FILE} >${TMPFILE}
+-	/bin/mv -f ${TMPFILE} ${FILE}
++	command mv -f ${TMPFILE} ${FILE}
+ else
+ 	${SED} "s|${KEY}|${VALUE}|g" -i ${FILE}
+ fi
+diff --git a/substitute-all b/substitute-all
+index 48c6b76a..eda7a0f2 100755
+--- a/substitute-all
++++ b/substitute-all
+@@ -23,5 +23,5 @@ fi
+ 
+ echo "Applying modification in ${PATHTOCHANGE} - Key = ${KEY} for value = ${VALUE}"
+ 
+-/usr/bin/find ${PATHTOCHANGE} -type f -exec ${SCRIPT_LOCATION} "${SED}" "${KEY}" "${VALUE}" {} \;
++command find ${PATHTOCHANGE} -type f -exec ${SCRIPT_LOCATION} "${SED}" "${KEY}" "${VALUE}" {} \;
+ 

pkgs/fftw/default.nix

@@ -0,0 +1,58 @@
+{ fetchurl, stdenv, lib, llvmPackages ? null, precision ? "double", perl, mpi }:
+
+with lib;
+
+assert stdenv.cc.isClang -> llvmPackages != null;
+assert elem precision [ "single" "double" "long-double" "quad-precision" ];
+
+let
+  version = "3.3.8";
+  withDoc = stdenv.cc.isGNU;
+in
+
+stdenv.mkDerivation {
+  name = "fftw-${precision}-${version}";
+
+  src = fetchurl {
+    urls = [
+      "http://fftw.org/fftw-${version}.tar.gz"
+      "ftp://ftp.fftw.org/pub/fftw/fftw-${version}.tar.gz"
+    ];
+    sha256 = "00z3k8fq561wq2khssqg0kallk0504dzlx989x3vvicjdqpjc4v1";
+  };
+
+  outputs = [ "out" "dev" "man" ]
+    ++ optional withDoc "info"; # it's dev-doc only
+  outputBin = "dev"; # fftw-wisdom
+
+  buildInputs = [ mpi ]
+  ++ lib.optionals stdenv.cc.isClang [
+    # TODO: This may mismatch the LLVM version sin the stdenv, see #79818.
+    llvmPackages.openmp
+  ];
+
+  configureFlags =
+    [ "--enable-shared"
+      "--enable-threads"
+      "--enable-mpi"
+      "--disable-openmp"
+    ]
+    ++ optional (precision != "double") "--enable-${precision}"
+    # all x86_64 have sse2
+    # however, not all float sizes fit
+    ++ optional (stdenv.isx86_64 && (precision == "single" || precision == "double") )  "--enable-sse2"
+    # doc generation causes Fortran wrapper generation which hard-codes gcc
+    ++ optional (!withDoc) "--disable-doc";
+
+  enableParallelBuilding = true;
+
+  checkInputs = [ perl ];
+
+  meta = with lib; {
+    description = "Fastest Fourier Transform in the West library";
+    homepage = "http://www.fftw.org/";
+    license = licenses.gpl2Plus;
+    maintainers = [ maintainers.spwhitt ];
+    platforms = platforms.unix;
+  };
+}

pkgs/gpi-2/default.nix

@@ -0,0 +1,64 @@
+{
+  stdenv
+, lib
+, fetchurl
+, symlinkJoin
+, slurm
+, rdma-core
+, autoconf
+, automake
+, libtool
+, mpi
+, rsync
+, gfortran
+}:
+
+let
+  rdma-core-all = symlinkJoin {
+    name ="rdma-core-all";
+    paths = [ rdma-core.dev rdma-core.out ];
+  };
+  mpiAll = symlinkJoin {
+    name = "mpi-all";
+    paths = [ mpi.all ];
+  };
+in
+
+stdenv.mkDerivation rec {
+  pname = "GPI-2";
+  version = "tagaspi-2021.11";
+
+  src = fetchurl {
+    url = "https://pm.bsc.es/gitlab/interoperability/extern/GPI-2/-/archive/${version}/GPI-2-${version}.tar.gz";
+    hash = "sha256-eY2wpyTpnOXRoAcYoAP82Jq9Q7p5WwDpMj+f1vEX5zw=";
+  };
+
+  enableParallelBuilding = true;
+
+  patches = [ ./rdma-core.patch ./max-mem.patch ];
+
+  preConfigure = ''
+    patchShebangs autogen.sh
+    ./autogen.sh
+  '';
+
+  configureFlags = [
+    "--with-infiniband=${rdma-core-all}"
+    "--with-mpi=${mpiAll}"
+    "--with-slurm"
+    "CFLAGS=-fPIC"
+    "CXXFLAGS=-fPIC"
+  ];
+
+  buildInputs = [ slurm mpiAll rdma-core-all autoconf automake libtool rsync gfortran ];
+
+  hardeningDisable = [ "all" ];
+
+  meta = {
+    homepage = "https://pm.bsc.es/gitlab/interoperability/extern/GPI-2";
+    description = "GPI-2 extended for supporting Task-Aware GASPI (TAGASPI) library";
+    maintainers = with lib.maintainers.bsc; [ rarias ];
+    platforms = lib.platforms.linux;
+    license = lib.licenses.gpl3Plus;
+  };
+}

pkgs/gpi-2/max-mem.patch

@@ -0,0 +1,10 @@
+--- a/tests/tests/segments/max_mem.c	2025-09-12 13:30:53.449353591 +0200
++++ b/tests/tests/segments/max_mem.c	2025-09-12 13:33:49.750352401 +0200
+@@ -1,5 +1,7 @@
+ #include <test_utils.h>
+ 
++gaspi_size_t gaspi_get_system_mem (void);
++
+ /* Test allocates 45% of system memory and creates a segment that
+    large or if several ranks per node exist, divided among that
+    number */

pkgs/gpi-2/rdma-core.patch

@@ -0,0 +1,12 @@
+--- a/src/devices/ib/GPI2_IB.h	2025-09-12 13:25:31.564181121 +0200
++++ b/src/devices/ib/GPI2_IB.h	2025-09-12 13:24:49.105422150 +0200
+@@ -26,6 +26,9 @@ along with GPI-2. If not, see <http://ww
+ 
+ #include "GPI2_Dev.h"
+ 
++/* Missing prototype as driver.h is now private */
++int ibv_read_sysfs_file(const char *dir, const char *file, char *buf, size_t size);
++
+ #define GASPI_GID_INDEX   (0)
+ #define PORT_LINK_UP      (5)
+ #define MAX_INLINE_BYTES  (128)

pkgs/groff/0001-Fix-cross-compilation-by-looking-for-ar.patch

@@ -0,0 +1,46 @@
+From 1454525f70b43a6957b7c9e1870e997368787da3 Mon Sep 17 00:00:00 2001
+From: Samuel Dionne-Riel <samuel@dionne-riel.com>
+Date: Fri, 8 Nov 2019 21:59:21 -0500
+Subject: [PATCH] Fix cross-compilation by looking for `ar`.
+
+---
+ Makefile.am  | 2 +-
+ configure.ac | 2 ++
+ 2 files changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile.am b/Makefile.am
+index d18c49b8..b1b53338 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -494,7 +494,7 @@ CCC=@CXX@
+ # INSTALL_INFO
+ # LN_S
+ 
+-AR=ar
++AR=@AR@
+ ETAGS=etags
+ ETAGSFLAGS=
+ # Flag that tells etags to assume C++.
+diff --git a/configure.ac b/configure.ac
+index 28e75f17..2449b9f5 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -37,6 +37,7 @@ AC_CONFIG_AUX_DIR([build-aux])
+ 
+ AC_CONFIG_HEADERS([src/include/config.h:src/include/config.hin])
+ AC_CONFIG_SRCDIR([src/roff/groff/groff.cpp])
++AC_CONFIG_MACRO_DIR([m4])
+ 
+ AC_USE_SYSTEM_EXTENSIONS
+ 
+@@ -72,6 +73,7 @@ GROFF_DOC_CHECK
+ GROFF_MAKEINFO
+ GROFF_TEXI2DVI
+ AC_PROG_RANLIB
++AC_CHECK_TOOL([AR], [ar], [ar])
+ GROFF_INSTALL_SH
+ GROFF_INSTALL_INFO
+ AC_PROG_INSTALL
+-- 
+2.23.0
+

pkgs/groff/default.nix

@@ -0,0 +1,127 @@
+{ stdenv, lib, fetchurl, perl
+, ghostscript #for postscript and html output
+, psutils, netpbm #for html output
+, buildPackages
+, autoreconfHook
+, pkg-config
+, texinfo
+}:
+
+stdenv.mkDerivation rec {
+  pname = "groff";
+  version = "1.22.4";
+
+  src = fetchurl {
+    url = "mirror://gnu/groff/${pname}-${version}.tar.gz";
+    sha256 = "14q2mldnr1vx0l9lqp9v2f6iww24gj28iyh4j2211hyynx67p3p7";
+  };
+
+  enableParallelBuilding = false;
+
+  patches = [
+    ./0001-Fix-cross-compilation-by-looking-for-ar.patch
+  ];
+
+  postPatch = lib.optionalString (psutils != null) ''
+    substituteInPlace src/preproc/html/pre-html.cpp \
+      --replace "psselect" "${psutils}/bin/psselect"
+  '' + lib.optionalString (netpbm != null) ''
+    substituteInPlace src/preproc/html/pre-html.cpp \
+      --replace "pnmcut" "${lib.getBin netpbm}/bin/pnmcut" \
+      --replace "pnmcrop" "${lib.getBin netpbm}/bin/pnmcrop" \
+      --replace "pnmtopng" "${lib.getBin netpbm}/bin/pnmtopng"
+    substituteInPlace tmac/www.tmac.in \
+      --replace "pnmcrop" "${lib.getBin netpbm}/bin/pnmcrop" \
+      --replace "pngtopnm" "${lib.getBin netpbm}/bin/pngtopnm" \
+      --replace "@PNMTOPS_NOSETPAGE@" "${lib.getBin netpbm}/bin/pnmtops -nosetpage"
+  '';
+
+  buildInputs = [ ghostscript psutils netpbm perl ];
+  nativeBuildInputs = [ autoreconfHook pkg-config texinfo ];
+
+  # Builds running without a chroot environment may detect the presence
+  # of /usr/X11 in the host system, leading to an impure build of the
+  # package. To avoid this issue, X11 support is explicitly disabled.
+  # Note: If we ever want to *enable* X11 support, then we'll probably
+  # have to pass "--with-appresdir", too.
+  configureFlags = [
+    "--without-x"
+  ] ++ lib.optionals (ghostscript != null) [
+    "--with-gs=${ghostscript}/bin/gs"
+  ] ++ lib.optionals (stdenv.buildPlatform != stdenv.hostPlatform) [
+    "ac_cv_path_PERL=${buildPackages.perl}/bin/perl"
+  ];
+
+  makeFlags = lib.optionals (stdenv.buildPlatform != stdenv.hostPlatform) [
+    # Trick to get the build system find the proper 'native' groff
+    # http://www.mail-archive.com/bug-groff@gnu.org/msg01335.html
+    "GROFF_BIN_PATH=${buildPackages.groff}/bin"
+    "GROFFBIN=${buildPackages.groff}/bin/groff"
+  ];
+
+  doCheck = true;
+
+  postInstall = ''
+    for f in 'man.local' 'mdoc.local'; do
+        cat '${./site.tmac}' >>"$out/share/groff/site-tmac/$f"
+    done
+
+    moveToOutput bin/gropdf $out
+    moveToOutput bin/pdfmom $out
+    moveToOutput bin/roff2text $out
+    moveToOutput bin/roff2pdf $out
+    moveToOutput bin/roff2ps $out
+    moveToOutput bin/roff2dvi $out
+    moveToOutput bin/roff2ps $out
+    moveToOutput bin/roff2html $out
+    moveToOutput bin/glilypond $out
+    moveToOutput bin/mmroff $out
+    moveToOutput bin/roff2x $out
+    moveToOutput bin/afmtodit $out
+    moveToOutput bin/gperl $out
+    moveToOutput bin/chem $out
+    moveToOutput share/groff/${version}/font/devpdf $out
+
+    # idk if this is needed, but Fedora does it
+    moveToOutput share/groff/${version}/tmac/pdf.tmac $out
+
+    moveToOutput bin/gpinyin $out
+    moveToOutput lib/groff/gpinyin $out
+    substituteInPlace $out/bin/gpinyin \
+      --replace $out/lib/groff/gpinyin $out/lib/groff/gpinyin
+
+    moveToOutput bin/groffer $out
+    moveToOutput lib/groff/groffer $out
+    substituteInPlace $out/bin/groffer \
+      --replace $out/lib/groff/groffer $out/lib/groff/groffer
+
+    moveToOutput bin/grog $out
+    moveToOutput lib/groff/grog $out
+    substituteInPlace $out/bin/grog \
+      --replace $out/lib/groff/grog $out/lib/groff/grog
+
+  '' + lib.optionalString (stdenv.buildPlatform != stdenv.hostPlatform) ''
+    find $out/ -type f -print0 | xargs --null sed -i 's|${buildPackages.perl}|${perl}|'
+  '';
+
+  meta = with lib; {
+    homepage = "https://www.gnu.org/software/groff/";
+    description = "GNU Troff, a typesetting package that reads plain text and produces formatted output";
+    license = licenses.gpl3Plus;
+    platforms = platforms.all;
+    maintainers = with maintainers; [ pSub ];
+
+    longDescription = ''
+      groff is the GNU implementation of troff, a document formatting
+      system.  Included in this release are implementations of troff,
+      pic, eqn, tbl, grn, refer, -man, -mdoc, -mom, and -ms macros,
+      and drivers for PostScript, TeX dvi format, HP LaserJet 4
+      printers, Canon CAPSL printers, HTML and XHTML format (beta
+      status), and typewriter-like devices.  Also included is a
+      modified version of the Berkeley -me macros, the enhanced
+      version gxditview of the X11 xditview previewer, and an
+      implementation of the -mm macros.
+    '';
+
+  };
+}