Use NixOS attributes for the install section

Allows the LazyUnmount option and avoids the stage1 hack with
/nix//store.

.gitignore

@@ -1,3 +1,2 @@
 *.swp
 /result
-/misc

doc/install.md

@@ -150,27 +150,3 @@ And update grub.
 ```
 # nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
 ```
-
-## Chain NixOS in same disk with other systems
-
-To install NixOS on a partition along another system which controls the GRUB,
-first disable the grub device, so the GRUB is not installed in the disk by
-NixOS (only the /boot files will be generated):
-
-```
-boot.loader.grub.device = "nodev";
-```
-
-Then add the following entry to the old GRUB configuration:
-
-```
-menuentry 'NixOS' {
-        insmod chain
-        search --no-floppy --label nixos --set root
-        configfile /boot/grub/grub.cfg
-}
-```
-
-The partition with NixOS must have the label "nixos" for it to be found. New
-system configuration entries will be stored in the GRUB configuration managed
-by NixOS, so there is no need to change the old GRUB settings.

flake.lock

@@ -6,15 +6,14 @@
         "home-manager": "home-manager",
         "nixpkgs": [
           "nixpkgs"
-        ],
-        "systems": "systems"
+        ]
       },
       "locked": {
-        "lastModified": 1723293904,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "lastModified": 1690228878,
+        "narHash": "sha256-9Xe7JV0krp4RJC9W9W9WutZVlw6BlHTFMiUP/k48LQY=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "rev": "d8c973fd228949736dedf61b7f8cc1ece3236792",
         "type": "github"
       },
       "original": {
@@ -30,17 +29,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1732868163,
-        "narHash": "sha256-qck4h298AgcNI6BnGhEwl26MTLXjumuJVr+9kak7uPo=",
+        "lastModified": 1694708510,
+        "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
         "ref": "refs/heads/master",
-        "rev": "6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f",
-        "revCount": 952,
+        "rev": "3a4062ac04be6263c64a481420d8e768c2521b80",
+        "revCount": 862,
         "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
       },
       "original": {
         "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
       }
     },
     "darwin": {
@@ -51,11 +50,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1700795494,
-        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
+        "lastModified": 1673295039,
+        "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
+        "rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
         "type": "github"
       },
       "original": {
@@ -73,11 +72,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1703113217,
-        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
+        "lastModified": 1682203081,
+        "narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
+        "rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
         "type": "github"
       },
       "original": {
@@ -88,16 +87,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736867362,
-        "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=",
+        "lastModified": 1693663421,
+        "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc",
+        "rev": "e56990880811a451abd32515698c712788be5720",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -108,21 +107,6 @@
         "bscpkgs": "bscpkgs",
         "nixpkgs": "nixpkgs"
       }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -1,9 +1,9 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
-    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
+    bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
     bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
   };
 
@@ -18,15 +18,12 @@ in
   {
     nixosConfigurations = {
       hut   = mkConf "hut";
-      tent    = mkConf "tent";
       owl1  = mkConf "owl1";
       owl2  = mkConf "owl2";
       eudy  = mkConf "eudy";
       koro  = mkConf "koro";
       bay   = mkConf "bay";
       lake2 = mkConf "lake2";
-      raccoon = mkConf "raccoon";
-      fox     = mkConf "fox";
     };
 
     packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

keys.nix

@@ -9,12 +9,9 @@ rec {
     koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
     bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
     lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
-    fox   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
-    tent  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
   };
 
   hostGroup = with hosts; rec {
-    untrusted  = [ fox ];
     compute    = [ owl1 owl2 ];
     playground = [ eudy koro ];
     storage    = [ bay lake2 ];
@@ -26,8 +23,7 @@ rec {
   };
 
   admins = {
-    "rarias@hut"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
-    "rarias@tent" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIwlWSBTZi74WTz5xn6gBvTmCoVltmtIAeM3RMmkh4QZ rarias@tent";
+    rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
     root   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
   };
 }

m/bay/configuration.nix

@@ -2,21 +2,21 @@
 
 {
   imports = [
-    ../common/ssf.nix
-    ../module/monitoring.nix
+    ../common/main.nix
+    ../common/monitoring.nix
   ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
 
-  boot.kernel.sysctl = {
-    "kernel.yama.ptrace_scope" = lib.mkForce "1";
-  };
-
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+
   networking = {
     hostName = "bay";
     interfaces.eno1.ipv4.addresses = [ {
@@ -27,16 +27,6 @@
       address = "10.0.42.40";
       prefixLength = 24;
     } ];
-    firewall = {
-      extraCommands = ''
-        # Accept all incoming TCP traffic from lake2
-        iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
-        # Accept monitoring requests from hut
-        iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
-        # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
-      '';
-    };
   };
 
   services.ceph = {

m/common/base.nix

@@ -1,20 +0,0 @@
-{
-  # All machines should include this profile.
-  # Includes the basic configuration for an Intel server.
-  imports = [
-    ./base/agenix.nix
-    ./base/august-shutdown.nix
-    ./base/boot.nix
-    ./base/env.nix
-    ./base/fs.nix
-    ./base/hw.nix
-    ./base/net.nix
-    ./base/nix.nix
-    ./base/ntp.nix
-    ./base/rev.nix
-    ./base/ssh.nix
-    ./base/users.nix
-    ./base/watchdog.nix
-    ./base/zsh.nix
-  ];
-}

m/common/base/august-shutdown.nix

@@ -1,14 +0,0 @@
-{
-  # Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
-  # hardware from spurious electrical peaks on the yearly electrical cut for
-  # manteinance that starts on August 4th.
-  systemd.timers.august-shutdown = {
-    description = "Shutdown on August 2nd for maintenance";
-    wantedBy = [ "timers.target" ];
-    timerConfig = {
-      OnCalendar = "*-08-02 11:00:00";
-      RandomizedDelaySec = "10min";
-      Unit = "systemd-poweroff.service";
-    };
-  };
-}

m/common/base/env.nix

@@ -1,37 +0,0 @@
-{ pkgs, config, ... }:
-
-{
-  environment.systemPackages = with pkgs; [
-    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
-    # From bsckgs overlay
-    osumb
-  ];
-
-  programs.direnv.enable = true;
-
-  # Increase limits
-  security.pam.loginLimits = [
-    {
-      domain = "*";
-      type = "-";
-      item = "memlock";
-      value = "1048576"; # 1 GiB of mem locked
-    }
-  ];
-
-  environment.enableAllTerminfo = true;
-
-  environment.variables = {
-    EDITOR = "vim";
-    VISUAL = "vim";
-  };
-
-  programs.bash.promptInit = ''
-    PS1="\h\\$ "
-  '';
-
-  time.timeZone = "Europe/Madrid";
-  i18n.defaultLocale = "en_DK.UTF-8";
-}

m/common/base/net.nix

@@ -1,19 +0,0 @@
-{ pkgs, ... }:
-
-{
-  networking = {
-    enableIPv6 = false;
-    useDHCP = false;
-
-    firewall = {
-      enable = true;
-      allowedTCPPorts = [ 22 ];
-    };
-
-    hosts = {
-      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
-      "84.88.51.152" = [ "raccoon" ];
-      "84.88.51.142" = [ "raccoon-ipmi" ];
-    };
-  };
-}

m/common/base/nix.nix

@@ -1,58 +0,0 @@
-{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
-
-{
-  nixpkgs.overlays = [
-    bscpkgs.bscOverlay
-    (import ../../../pkgs/overlay.nix)
-  ];
-
-  nix = {
-    nixPath = [
-      "nixpkgs=${nixpkgs}"
-      "jungle=${theFlake.outPath}"
-    ];
-
-    registry = {
-      nixpkgs.flake = nixpkgs;
-      jungle.flake = theFlake;
-    };
-
-    settings = {
-      experimental-features = [ "nix-command" "flakes" ];
-      sandbox = "relaxed";
-      trusted-users = [ "@wheel" ];
-      flake-registry = pkgs.writeText "global-registry.json"
-        ''{"flakes":[],"version":2}'';
-      keep-outputs = true;
-    };
-
-    gc = {
-      automatic = true;
-      dates = "weekly";
-      options = "--delete-older-than 30d";
-    };
-  };
-
-  # The nix-gc.service can begin its execution *before* /home is mounted,
-  # causing it to remove all gcroots considering them as stale, as it cannot
-  # access the symlink. To prevent this problem, we force the service to wait
-  # until /home is mounted as well as other remote FS like /ceph.
-  systemd.services.nix-gc = {
-    # Start remote-fs.target if not already being started and fail if it fails
-    # to start. It will also be stopped if the remote-fs.target fails after
-    # starting successfully.
-    bindsTo = [ "remote-fs.target" ];
-    # Wait until remote-fs.target fully starts before starting this one.
-    after = [ "remote-fs.target"];
-    # Ensure we can access a remote path inside /home
-    unitConfig.ConditionPathExists = "/home/Computational";
-  };
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "22.11"; # Did you read the comment?
-}

m/common/base/users.nix

@@ -1,164 +0,0 @@
-{ pkgs, ... }:
-
-{
-  imports = [
-    ../../module/jungle-users.nix
-  ];
-
-  users = {
-    mutableUsers = false;
-    users = {
-      # Generate hashedPassword with `mkpasswd -m sha-512`
-
-      root.openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"
-      ];
-
-      rarias = {
-        uid = 1880;
-        isNormalUser = true;
-        linger = true;
-        home = "/home/Computational/rarias";
-        description = "Rodrigo Arias";
-        group = "Computational";
-        extraGroups = [ "wheel" ];
-        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
-        ];
-        shell = pkgs.zsh;
-      };
-
-      arocanon = {
-        uid = 1042;
-        isNormalUser = true;
-        home = "/home/Computational/arocanon";
-        description = "Aleix Roca";
-        group = "Computational";
-        extraGroups = [ "wheel" "tracing" ];
-        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
-        ];
-      };
-    };
-
-    jungleUsers = {
-      rpenacob = {
-        uid = 2761;
-        isNormalUser = true;
-        home = "/home/Computational/rpenacob";
-        description = "Raúl Peñacoba";
-        group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "tent" "fox" ];
-        hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
-        ];
-      };
-
-      anavarro = {
-        uid = 1037;
-        isNormalUser = true;
-        home = "/home/Computational/anavarro";
-        description = "Antoni Navarro";
-        group = "Computational";
-        hosts = [ "hut" "tent" "raccoon" "fox" ];
-        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
-        ];
-      };
-
-      abonerib = {
-        uid = 4541;
-        isNormalUser = true;
-        home = "/home/Computational/abonerib";
-        description = "Aleix Boné";
-        group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "tent" "raccoon" "fox" ];
-        hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
-        ];
-      };
-
-      vlopez = {
-        uid = 4334;
-        isNormalUser = true;
-        home = "/home/Computational/vlopez";
-        description = "Victor López";
-        group = "Computational";
-        hosts = [ "koro" ];
-        hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
-        ];
-      };
-
-      dbautist = {
-        uid = 5649;
-        isNormalUser = true;
-        home = "/home/Computational/dbautist";
-        description = "Dylan Bautista Cases";
-        group = "Computational";
-        hosts = [ "hut" "tent" "raccoon" ];
-        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
-        ];
-      };
-
-      dalvare1 = {
-        uid = 2758;
-        isNormalUser = true;
-        home = "/home/Computational/dalvare1";
-        description = "David Álvarez";
-        group = "Computational";
-        hosts = [ "hut" "tent" "fox" ];
-        hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
-        ];
-      };
-
-      varcila = {
-        uid = 5650;
-        isNormalUser = true;
-        home = "/home/Computational/varcila";
-        description = "Vincent Arcila";
-        group = "Computational";
-        hosts = [ "hut" "tent" "fox" ];
-        hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
-        ];
-      };
-
-      pmartin1 = {
-        # Arbitrary UID but large so it doesn't collide with other users on ssfhead.
-        uid = 9652;
-        isNormalUser = true;
-        home = "/home/Computational/pmartin1";
-        description = "Pedro J. Martinez-Ferrer";
-        group = "Computational";
-        hosts = [ "fox" ];
-        hashedPassword = "$6$nIgDMGnt4YIZl3G.$.JQ2jXLtDPRKsbsJfJAXdSvjDIzRrg7tNNjPkLPq3KJQhMjfDXRUvzagUHUU2TrE2hHM8/6uq8ex0UdxQ0ysl.";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
-        ];
-      };
-    };
-
-    groups = {
-      Computational = { gid = 564; };
-      tracing = { };
-    };
-  };
-}

m/common/boot.nix

@@ -2,7 +2,7 @@
 
 {
   # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = true;
+  boot.loader.grub.enable = lib.mkForce true;
 
   # Enable GRUB2 serial console
   boot.loader.grub.extraConfig = ''
@@ -11,12 +11,14 @@
     terminal_output --append serial
   '';
 
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
   boot.kernel.sysctl = {
     "kernel.perf_event_paranoid" = lib.mkDefault "-1";
-
-    # Allow ptracing (i.e. attach with GDB) any process of the same user, see:
-    # https://www.kernel.org/doc/Documentation/security/Yama.txt
-    "kernel.yama.ptrace_scope" = "0";
   };
 
   boot.kernelPackages = pkgs.linuxPackages_latest;

m/common/fs.nix

@@ -13,12 +13,16 @@
     [ { device = "/dev/disk/by-label/swap"; }
     ];
 
+  # Mount the home via NFS
+  fileSystems."/home" = {
+    device = "10.0.40.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
   # Tracing
   fileSystems."/sys/kernel/tracing" = {
     device = "none";
     fsType = "tracefs";
   };
-
-  # Mount a tmpfs into /tmp
-  boot.tmp.useTmpfs = true;
 }

m/common/main.nix

@@ -0,0 +1,97 @@
+{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
+
+{
+  imports = [
+    ./agenix.nix
+    ./boot.nix
+    ./fs.nix
+    ./hw.nix
+    ./net.nix
+    ./ntp.nix
+    ./slurm.nix
+    ./ssh.nix
+    ./users.nix
+    ./watchdog.nix
+    ./rev.nix
+    ./zsh.nix
+  ];
+
+  nixpkgs.overlays = [
+    bscpkgs.bscOverlay
+    (import ../../pkgs/overlay.nix)
+  ];
+
+  system.configurationRevision =
+    if theFlake ? rev
+    then theFlake.rev
+    else throw ("Refusing to build from a dirty Git tree!");
+
+  nix.nixPath = [
+    "nixpkgs=${nixpkgs}"
+    "jungle=${theFlake.outPath}"
+  ];
+
+  nix.settings.flake-registry =
+    pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
+
+  nix.registry.nixpkgs.flake = nixpkgs;
+  nix.registry.jungle.flake = theFlake;
+
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns
+    # From bsckgs overlay
+    bsc.osumb
+  ];
+
+  programs.direnv.enable = true;
+
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.sandbox = "relaxed";
+  nix.settings.trusted-users = [ "@wheel" ];
+  nix.gc.automatic = true;
+  nix.gc.dates = "weekly";
+  nix.gc.options = "--delete-older-than 30d";
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  # Copy the NixOS configuration file and link it from the resulting system
+  # (/run/current-system/configuration.nix). This is useful in case you
+  # accidentally delete configuration.nix.
+  #system.copySystemConfiguration = true;
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}

m/common/net.nix

@@ -6,21 +6,24 @@
   boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
 
   networking = {
+    enableIPv6 = false;
+    useDHCP = false;
     defaultGateway = "10.0.40.30";
     nameservers = ["8.8.8.8"];
-
     proxy = {
-      default = "http://hut:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40,hut";
+      default = "http://localhost:23080/";
+      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
       # Don't set all_proxy as go complains and breaks the gitlab runner, see:
       # https://github.com/golang/go/issues/16715
       allProxy = null;
     };
 
     firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
       extraCommands = ''
         # Prevent ssfhead from contacting our slurmd daemon
-        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse
         # But accept traffic to slurm ports from any other node in the subnet
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
         # We also need to open the srun port range
@@ -30,41 +33,42 @@
 
     extraHosts = ''
       10.0.40.30      ssfhead
+      84.88.53.236    ssfhead.bsc.es ssfhead
       
       # Node Entry for node: mds01 (ID=72)
       10.0.40.40              bay mds01 mds01-eth0
       10.0.42.40              bay-ib mds01-ib0
-      10.0.40.141             bay-ipmi mds01-ipmi0 mds01-ipmi
+      10.0.40.141             bay-ipmi mds01-ipmi0
       
       # Node Entry for node: oss01 (ID=73)
       10.0.40.41              oss01 oss01-eth0
       10.0.42.41              oss01-ib0
-      10.0.40.142             oss01-ipmi0 oss01-ipmi
+      10.0.40.142             oss01-ipmi0
       
       # Node Entry for node: oss02 (ID=74)
       10.0.40.42              lake2 oss02 oss02-eth0
       10.0.42.42              lake2-ib oss02-ib0
-      10.0.40.143             lake2-ipmi oss02-ipmi0 oss02-ipmi
+      10.0.40.143             lake2-ipmi oss02-ipmi0
       
       # Node Entry for node: xeon01 (ID=15)
       10.0.40.1               owl1 xeon01 xeon01-eth0
       10.0.42.1               owl1-ib xeon01-ib0
-      10.0.40.101             owl1-ipmi xeon01-ipmi0 xeon01-ipmi
+      10.0.40.101             owl1-ipmi xeon01-ipmi0
       
       # Node Entry for node: xeon02 (ID=16)
       10.0.40.2               owl2 xeon02 xeon02-eth0
       10.0.42.2               owl2-ib xeon02-ib0
-      10.0.40.102             owl2-ipmi xeon02-ipmi0 xeon02-ipmi
+      10.0.40.102             owl2-ipmi xeon02-ipmi0
       
       # Node Entry for node: xeon03 (ID=17)
       10.0.40.3               xeon03 xeon03-eth0
       10.0.42.3               xeon03-ib0
-      10.0.40.103             xeon03-ipmi0 xeon03-ipmi
+      10.0.40.103             xeon03-ipmi0
       
       # Node Entry for node: xeon04 (ID=18)
       10.0.40.4               xeon04 xeon04-eth0
       10.0.42.4               xeon04-ib0
-      10.0.40.104             xeon04-ipmi0 xeon04-ipmi
+      10.0.40.104             xeon04-ipmi0
       
       # Node Entry for node: xeon05 (ID=19)
       10.0.40.5               koro xeon05 xeon05-eth0
@@ -74,17 +78,17 @@
       # Node Entry for node: xeon06 (ID=20)
       10.0.40.6               xeon06 xeon06-eth0
       10.0.42.6               xeon06-ib0
-      10.0.40.106             xeon06-ipmi0 xeon06-ipmi
+      10.0.40.106             xeon06-ipmi0
       
       # Node Entry for node: xeon07 (ID=21)
       10.0.40.7               hut xeon07 xeon07-eth0
       10.0.42.7               hut-ib xeon07-ib0
-      10.0.40.107             hut-ipmi xeon07-ipmi0 xeon07-ipmi
+      10.0.40.107             hut-ipmi xeon07-ipmi0
       
       # Node Entry for node: xeon08 (ID=22)
       10.0.40.8               eudy xeon08 xeon08-eth0
       10.0.42.8               eudy-ib xeon08-ib0
-      10.0.40.108             eudy-ipmi xeon08-ipmi0 xeon08-ipmi
+      10.0.40.108             eudy-ipmi xeon08-ipmi0
     '';
   };
 }

m/common/rev.nix

@@ -1,7 +1,6 @@
 { theFlake, ... }:
 
 let
-  # Prevent building a configuration without revision
   rev = if theFlake ? rev then theFlake.rev
     else throw ("Refusing to build from a dirty Git tree!");
 in {
@@ -16,6 +15,4 @@ in {
     DATENOW=$(date --iso-8601=seconds)
     echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
   '';
-
-  system.configurationRevision = rev;
 }

m/common/slurm.nix

@@ -47,7 +47,8 @@ in {
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP"
+      "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -82,37 +83,9 @@ in {
 
       # Reduce port range so we can allow only this range in the firewall
       SrunPortRange=60000-61000
-
-      # Use cores as consumable resources. In SLURM terms, a core may have
-      # multiple hardware threads (or CPUs).
-      SelectType=select/cons_tres
-
-      # Ignore memory constraints and only use unused cores to share a node with
-      # other jobs.
-      SelectTypeParameters=CR_Core
-
-      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
-      # This sets up the "extern" step into which ssh-launched processes will be
-      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
-      # when a task runs (srun) so we can ssh early.
-      PrologFlags=Alloc,Contain,X11
-
-      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
-      # adopted by the external step, similar to tasks running in regular steps
-      # LaunchParameters=ulimit_pam_adopt
-      SlurmdDebug=debug5
-      #DebugFlags=Protocol,Cgroup
-    '';
-
-    extraCgroupConfig = ''
-      CgroupPlugin=cgroup/v2
-      #ConstrainCores=yes
     '';
   };
 
-  # Place the slurm config in /etc as this will be required by PAM
-  environment.etc.slurm.source = config.services.slurm.etcSlurm;
-
   age.secrets.mungeKey = {
     file = ../../secrets/munge-key.age;
     owner = "munge";

m/common/ssf.nix

@@ -1,9 +0,0 @@
-{
-  # Provides the base system for a xeon node in the SSF rack.
-  imports = [
-    ./xeon.nix
-    ./ssf/fs.nix
-    ./ssf/net.nix
-    ./ssf/ssh.nix
-  ];
-}

m/common/ssf/fs.nix

@@ -1,8 +0,0 @@
-{
-  # Mount the home via NFS
-  fileSystems."/home" = {
-    device = "10.0.40.30:/home";
-    fsType = "nfs";
-    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
-  };
-}

m/common/ssf/ssh.nix

@@ -1,8 +0,0 @@
-{
-  # Connect to intranet git hosts via proxy
-  programs.ssh.extraConfig = ''
-    # Connect to BSC machines via hut proxy too
-    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-      ProxyCommand nc -X connect -x hut:23080 %h %p
-  '';
-}

m/common/ssh.nix

@@ -1,18 +1,22 @@
 { lib, ... }:
 
 let
-  keys = import ../../../keys.nix;
+  keys = import ../../keys.nix;
   hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
 in
 {
   # Enable the OpenSSH daemon.
   services.openssh.enable = true;
 
+  # Connect to intranet git hosts via proxy
+  programs.ssh.extraConfig = ''
+    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
+      User git
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+  '';
+
   programs.ssh.knownHosts = hostsKeys // {
     "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
     "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
-    "bscpm04.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT";
-    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
-    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
   };
 }

m/common/users.nix

@@ -0,0 +1,75 @@
+{ pkgs, ... }:
+
+{
+  users = {
+    mutableUsers = false;
+    users = {
+      # Generate hashedPassword with `mkpasswd -m sha-512`
+
+      root.openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"
+      ];
+
+      rarias = {
+        uid = 1880;
+        isNormalUser = true;
+        home = "/home/Computational/rarias";
+        description = "Rodrigo Arias";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
+        ];
+        shell = pkgs.zsh;
+      };
+
+      arocanon = {
+        uid = 1042;
+        isNormalUser = true;
+        home = "/home/Computational/arocanon";
+        description = "Aleix Roca";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
+        ];
+      };
+
+      rpenacob = {
+        uid = 2761;
+        isNormalUser = true;
+        home = "/home/Computational/rpenacob";
+        description = "Raúl Peñacoba";
+        group = "Computational";
+        hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
+        ];
+      };
+
+      anavarro = {
+        uid = 1037;
+        isNormalUser = true;
+        home = "/home/Computational/anavarro";
+        description = "Antoni Navarro";
+        group = "Computational";
+        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
+        ];
+      };
+    };
+
+    groups = {
+      Computational = { gid = 564; };
+    };
+  };
+}

m/common/xeon.nix

@@ -1,7 +0,0 @@
-{
-  # Provides the base system for a xeon node, not necessarily in the SSF rack.
-  imports = [
-    ./base.nix
-    ./xeon/console.nix
-  ];
-}

m/common/xeon/console.nix

@@ -1,14 +0,0 @@
-{
-  # Restart the serial console
-  systemd.services."serial-getty@ttyS0" = {
-    enable = true;
-    wantedBy = [ "getty.target" ];
-    serviceConfig.Restart = "always";
-  };
-
-  # Enable serial console
-  boot.kernelParams = [
-    "console=tty1"
-    "console=ttyS0,115200"
-  ];
-}

m/eudy/configuration.nix

@@ -2,14 +2,14 @@
 
 {
   imports = [
-    ../common/ssf.nix
+    ../common/main.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
-    ../module/debuginfod.nix
+    ./slurm.nix
   ];
 
   # Select this using the ID to avoid mismatches

m/eudy/kernel/kernel.nix

@@ -21,9 +21,9 @@ let
   #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
   #};
 
-  kernel = nixos-fcs;
+  kernel = nixos-fcsv3;
 
-  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -40,13 +40,35 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  }));
+  });
 
-  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
-  nixos-fcs-lockstat = nixos-fcs.override {
+  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
+  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
+  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
+
+  # always use fcs_sched_setaffinity
+  #nixos-debug = nixos-fcs-kernel {gitCommit = "7d0bf285fca92badc8df3c9907a9ab30db4418aa";};
+  # remove need_check_cgroup
+  #nixos-debug = nixos-fcs-kernel {gitCommit = "4cc4efaab5e4a0bfa3089e935215b981c1922919";};
+  # merge again fcs_wake and fcs_wait
+  #nixos-debug = nixos-fcs-kernel {gitCommit = "40c6f72f4ae54b0b636b193ac0648fb5730c810d";};
+  # start from scratch, this is the working version with split fcs_wake and fcs_wait
+  nixos-debug = nixos-fcs-kernel {gitCommit = "c9a39d6a4ca83845b4e71fcc268fb0a76aff1bdf"; branch = "fcs-test"; };
+
+  nixos-fcsv1-lockstat = nixos-fcs-kernel {
+    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
     lockStat = true;
   };
-  nixos-fcs-lockstat-preempt = nixos-fcs.override {
+  nixos-fcsv2-lockstat = nixos-fcs-kernel {
+    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
+    lockStat = true;
+  };
+  nixos-fcsv3-lockstat = nixos-fcs-kernel {
+    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+    lockStat = true;
+  };
+  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
+    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
     lockStat = true;
     preempt = true;
   };

m/eudy/slurm.nix

@@ -0,0 +1,7 @@
+{ lib, ... }:
+
+{
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+}

m/fox/configuration.nix

@@ -1,83 +0,0 @@
-{ lib, config, pkgs, ... }:
-
-{
-  imports = [
-    ../common/base.nix
-    ../common/xeon/console.nix
-    ../module/emulation.nix
-  ];
-
-  # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
-
-  # No swap, there is plenty of RAM
-  swapDevices = lib.mkForce [];
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
-  boot.kernelModules = [ "kvm-amd" "amd_uncore" ];
-
-  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-  hardware.cpu.intel.updateMicrocode = lib.mkForce false;
-
-  # Use performance for benchmarks
-  powerManagement.cpuFreqGovernor = "performance";
-
-  # Disable NUMA balancing
-  boot.kernel.sysctl."kernel.numa_balancing" = 0;
-
-  # Expose kernel addresses
-  boot.kernel.sysctl."kernel.kptr_restrict" = 0;
-
-  services.openssh.settings.X11Forwarding = true;
-
-  networking = {
-    timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
-    hostName = "fox";
-    # UPC network (may change over time, use DHCP)
-    # Public IP configuration:
-    # - Hostname: fox.ac.upc.edu
-    # - IP: 147.83.30.141
-    # - Gateway: 147.83.30.130
-    # - NetMask: 255.255.255.192
-    # Private IP configuration for BMC:
-    # - Hostname: fox-ipmi.ac.upc.edu
-    # - IP: 147.83.35.27
-    # - Gateway: 147.83.35.2
-    # - NetMask: 255.255.255.0
-    interfaces.enp1s0f0np0.useDHCP = true;
-  };
-
-  # Use hut for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
-
-  # Configure Nvidia driver to use with CUDA
-  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
-  hardware.graphics.enable = true;
-  nixpkgs.config.allowUnfree = true;
-  nixpkgs.config.nvidia.acceptLicense = true;
-  services.xserver.videoDrivers = [ "nvidia" ];
-
-  # Mount NVME disks
-  fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
-  fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
-
-  # Make a /nvme{0,1}/$USER directory for each user.
-  systemd.services.create-nvme-dirs = let
-    # Take only normal users in fox
-    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-    commands = lib.concatLists (lib.mapAttrsToList
-      (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0755 /nvme{0,1}/${user.name}"
-      ]) users);
-    script = pkgs.writeShellScript "create-nvme-dirs.sh" (lib.concatLines commands);
-  in {
-    enable = true;
-    wants = [ "local-fs.target" ];
-    after = [ "local-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig.ExecStart = script;
-  };
-}

m/hut/blackbox.yml

@@ -1,162 +0,0 @@
-modules:
-  http_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      proxy_url: "http://127.0.0.1:23080"
-      skip_resolve_phase_with_proxy: true
-      follow_redirects: true
-      valid_status_codes: []  # Defaults to 2xx
-      method: GET
-  http_with_proxy:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      skip_resolve_phase_with_proxy: true
-  http_with_proxy_and_headers:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      proxy_connect_header:
-        Proxy-Authorization:
-          - Bearer token
-  http_post_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Content-Type: application/json
-      body: '{}'
-  http_post_body_file:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      body_file: "/files/body.txt"
-  http_basic_auth_example:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Host: "login.example.com"
-      basic_auth:
-        username: "username"
-        password: "mysecret"
-  http_2xx_oauth_client_credentials:
-    prober: http
-    timeout: 5s
-    http:
-      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
-      follow_redirects: true
-      preferred_ip_protocol: "ip4"
-      valid_status_codes:
-        - 200
-        - 201
-      oauth2:
-        client_id: "client_id"
-        client_secret: "client_secret"
-        token_url: "https://api.example.com/token"
-        endpoint_params:
-          grant_type: "client_credentials"
-  http_custom_ca_example:
-    prober: http
-    http:
-      method: GET
-      tls_config:
-        ca_file: "/certs/my_cert.crt"
-  http_gzip:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-  http_gzip_with_accept_encoding:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-      headers:
-        Accept-Encoding: gzip
-  tls_connect:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      tls: true
-  tcp_connect_example:
-    prober: tcp
-    timeout: 5s
-  imap_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "OK.*STARTTLS"
-        - send: ". STARTTLS"
-        - expect: "OK"
-        - starttls: true
-        - send: ". capability"
-        - expect: "CAPABILITY IMAP4rev1"
-  smtp_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "^220 ([^ ]+) ESMTP (.+)$"
-        - send: "EHLO prober\r"
-        - expect: "^250-STARTTLS"
-        - send: "STARTTLS\r"
-        - expect: "^220"
-        - starttls: true
-        - send: "EHLO prober\r"
-        - expect: "^250-AUTH"
-        - send: "QUIT\r"
-  irc_banner_example:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - send: "NICK prober"
-        - send: "USER prober prober prober :prober"
-        - expect: "PING :([^ ]+)"
-          send: "PONG ${1}"
-        - expect: "^:[^ ]+ 001"
-  icmp:
-    prober: icmp
-    timeout: 5s
-    icmp:
-      preferred_ip_protocol: "ip4"
-  dns_udp_example:
-    prober: dns
-    timeout: 5s
-    dns:
-      query_name: "www.prometheus.io"
-      query_type: "A"
-      valid_rcodes:
-        - NOERROR
-      validate_answer_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-        fail_if_all_match_regexp:
-          - ".*127.0.0.1"
-        fail_if_not_matches_regexp:
-          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
-        fail_if_none_matches_regexp:
-          - "127.0.0.1"
-      validate_authority_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-      validate_additional_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-  dns_soa:
-    prober: dns
-    dns:
-      query_name: "prometheus.io"
-      query_type: "SOA"
-  dns_tcp_example:
-    prober: dns
-    dns:
-      transport_protocol: "tcp" # defaults to "udp"
-      preferred_ip_protocol: "ip4" # defaults to "ip6"
-      query_name: "www.prometheus.io"

m/hut/configuration.nix

@@ -1,44 +1,22 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, ... }:
 
 {
   imports = [
-    ../common/ssf.nix
+    ../common/main.nix
 
     ../module/ceph.nix
-    ../module/debuginfod.nix
-    ../module/emulation.nix
-    ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm-server.nix
+    ./slurm-daemon.nix
     ./nix-serve.nix
-    ./public-inbox.nix
-    ./gitea.nix
-    ./msmtp.nix
-    ./postgresql.nix
-    ./nginx.nix
-    ./p.nix
     #./pxe.nix
   ];
 
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
+
   # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
-
-  fileSystems = {
-    "/" = lib.mkForce {
-      device = "/dev/disk/by-label/nvme";
-      fsType = "ext4";
-      neededForBoot = true;
-      options = [ "noatime" ];
-    };
-
-    "/boot" = lib.mkForce {
-      device = "/dev/disk/by-label/nixos-boot";
-      fsType = "ext4";
-      neededForBoot = true;
-    };
-  };
+  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
 
   networking = {
     hostName = "hut";
@@ -50,20 +28,5 @@
       address = "10.0.42.7";
       prefixLength = 24;
     } ];
-    firewall = {
-      extraCommands = ''
-        # Accept all proxy traffic from compute nodes but not the login
-        iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
-      '';
-      # Flush all rules and chains on stop so it won't break on start
-      extraStopCommands = ''
-        iptables -F
-        iptables -X
-      '';
   };
-  };
-
-  # Allow proxy to bind to the ethernet interface
-  services.openssh.settings.GatewayPorts = "clientspecified";
 }

m/hut/gitea.nix

@@ -1,63 +0,0 @@
-{ config, lib, ... }:
-{
-  age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
-
-  services.gitea = {
-    enable = true;
-    appName = "Gitea in the jungle";
-
-    settings = {
-      server = {
-        ROOT_URL = "https://jungle.bsc.es/git/";
-        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
-        LANDING_PAGE = "explore";
-      };
-      metrics.ENABLED = true;
-      service = {
-        REGISTER_MANUAL_CONFIRM = true;
-        ENABLE_NOTIFY_MAIL = true;
-      };
-      log.LEVEL = "Warn";
-
-      mailer = {
-        ENABLED       = true;
-        FROM          = "jungle-robot@bsc.es";
-        PROTOCOL      = "sendmail";
-        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
-        SENDMAIL_ARGS = "--";
-      };
-    };
-  };
-
-  services.gitea-actions-runner.instances = {
-    runrun = {
-      enable = true;
-      name = "runrun";
-      url = "https://jungle.bsc.es/git/";
-      tokenFile = config.age.secrets.giteaRunnerToken.path;
-      labels = [ "native:host" ];
-      settings.runner.capacity = 8;
-    };
-  };
-
-  systemd.services.gitea-runner-runrun = {
-    path = [ "/run/current-system/sw" ];
-    serviceConfig = {
-      # DynamicUser doesn't work well with SSH
-      DynamicUser = lib.mkForce false;
-      User = "gitea-runner";
-      Group = "gitea-runner";
-    };
-  };
-
-  users.users.gitea-runner = {
-    isSystemUser = true;
-    home = "/var/lib/gitea-runner";
-    description = "Gitea Runner";
-    group = "gitea-runner";
-    extraGroups = [ "docker" ];
-    createHome = true;
-  };
-  users.groups.gitea-runner = {};
-}
-

m/hut/gitlab-runner.nix

@@ -1,111 +1,54 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age;
-  age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age;
-  age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age;
+  age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
+  age.secrets.nosvToken.file = ../../secrets/nosv-token.age;
 
   services.gitlab-runner = {
     enable = true;
     settings.concurrent = 5;
-    services = let
-      common-shell = {
+    services = {
+      ovni-shell = {
+        registrationConfigFile = config.age.secrets.ovniToken.path;
         executor = "shell";
+        tagList = [ "nix" "xeon" ];
+        registrationFlags = [
+          # Using space doesn't work, and causes it to misread the next flag
+          "--locked='false'"
+        ];
         environmentVariables = {
           SHELL = "${pkgs.bash}/bin/bash";
         };
       };
-      common-docker = {
-        executor = "docker";
+      ovni-docker = {
+        registrationConfigFile = config.age.secrets.ovniToken.path;
         dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
         registrationFlags = [
+          "--locked='false'"
           "--docker-network-mode host"
         ];
         environmentVariables = {
-          https_proxy = "http://hut:23080";
-          http_proxy = "http://hut:23080";
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
         };
       };
-    in {
-      # For pm.bsc.es/gitlab
-      gitlab-pm-shell = common-shell // {
-        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path;
-      };
-      gitlab-pm-docker = common-docker // {
-        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path;
-      };
-
-      gitlab-bsc-docker = {
-        # gitlab.bsc.es still uses the old token mechanism
-        registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
-        tagList = [ "docker" "hut" ];
-        environmentVariables = {
-          # We cannot access the hut local interface from docker, so we connect
-          # to hut directly via the ethernet one.
-          https_proxy = "http://hut:23080";
-          http_proxy = "http://hut:23080";
-        };
-        executor = "docker";
-        dockerImage = "alpine";
-        dockerVolumes = [
-          "/nix/store:/nix/store:ro"
-          "/nix/var/nix/db:/nix/var/nix/db:ro"
-          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
-        ];
-        dockerExtraHosts = [
-          # Required to pass the proxy via hut
-          "hut:10.0.40.7"
-        ];
-        dockerDisableCache = true;
+      nosv-docker = {
+        registrationConfigFile = config.age.secrets.nosvToken.path;
+        dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
         registrationFlags = [
-          # Increase build log length to 64 MiB
-          "--output-limit 65536"
+          "--docker-network-mode host"
+          "--docker-cpus 56"
         ];
-        preBuildScript = pkgs.writeScript "setup-container" ''
-          mkdir -p -m 0755 /nix/var/log/nix/drvs
-          mkdir -p -m 0755 /nix/var/nix/gcroots
-          mkdir -p -m 0755 /nix/var/nix/profiles
-          mkdir -p -m 0755 /nix/var/nix/temproots
-          mkdir -p -m 0755 /nix/var/nix/userpool
-          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
-          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
-          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
-          mkdir -p -m 0700 "$HOME/.nix-defexpr"
-          mkdir -p -m 0700 "$HOME/.ssh"
-          cat > "$HOME/.ssh/config" << EOF
-          Host bscpm04.bsc.es gitlab-internal.bsc.es
-            User git
-            ProxyCommand nc -X connect -x hut:23080 %h %p
-          Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-            ProxyCommand nc -X connect -x hut:23080 %h %p
-          EOF
-          cat >> "$HOME/.ssh/known_hosts" << EOF
-          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
-          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
-          EOF
-          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
-          # Required to load SSL certificate paths
-          . ${pkgs.cacert}/nix-support/setup-hook
-        '';
         environmentVariables = {
-          ENV = "/etc/profile";
-          USER = "root";
-          NIX_REMOTE = "daemon";
-          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
         };
       };
     };
   };
 
-  # DOCKER* chains are useless, override at FORWARD and nixos-fw
-  networking.firewall.extraCommands = ''
-    # Don't forward any traffic from docker
-    iptables -I FORWARD 1 -p all -i docker0 -j nixos-fw-log-refuse
-
-    # Allow incoming traffic from docker to 23080
-    iptables -A nixos-fw -p tcp -i docker0 -d hut --dport 23080 -j ACCEPT
-  '';
-
   #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";

m/hut/gpfs-probe.nix

@@ -1,31 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
-    ''
-      cp ${./gpfs-probe.sh} $out;
-      chmod +x $out
-    ''
-  ;
-in
-{
-  # Use a new user to handle the SSH keys
-  users.groups.ssh-robot = { };
-  users.users.ssh-robot = {
-    description = "SSH Robot";
-    isNormalUser = true;
-    home = "/var/lib/ssh-robot";
-  };
-
-  systemd.services.gpfs-probe = {
-    description = "Daemon to report GPFS latency via SSH";
-    path = [ pkgs.openssh pkgs.netcat ];
-    after = [ "network.target" ];
-    wantedBy = [ "default.target" ];
-    serviceConfig = {
-      Type = "simple";
-      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
-      User = "ssh-robot";
-      Group = "ssh-robot";
-    };
-  };
-}

m/hut/gpfs-probe.sh

@@ -1,30 +0,0 @@
-#!/bin/sh
-
-N=500
-
-t_proj=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
-t_scratch=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/scratch/bsc15/rodrigo/probe/gpfs.{1..$N} 2>&1; rm -f /gpfs/scratch/bsc15/rodrigo/probe/gpfs.{1..$N}")
-t_home=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /home/bsc/bsc015557/.gpfs/{1..$N} 2>&1; rm -f /home/bsc/bsc015557/.gpfs/{1..$N}")
-
-if [ -z "$t_proj" ]; then
-  t_proj="5.00"
-fi
-
-if [ -z "$t_scratch" ]; then
-  t_scratch="5.00"
-fi
-
-if [ -z "$t_home" ]; then
-  t_home="5.00"
-fi
-
-cat <<EOF
-HTTP/1.1 200 OK
-Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
-
-# HELP gpfs_touch_latency Time to create $N files.
-# TYPE gpfs_touch_latency gauge
-gpfs_touch_latency{partition="projects"} $t_proj
-gpfs_touch_latency{partition="home"} $t_home
-gpfs_touch_latency{partition="scratch"} $t_scratch
-EOF

m/hut/ipmi.yml

@@ -0,0 +1,13 @@
+modules:
+        default:
+                collectors:
+                - bmc
+                - ipmi
+                - chassis
+
+        lan:
+                collectors:
+                - ipmi
+                - chassis
+                user: ""
+                pass: ""

m/hut/monitoring.nix

@@ -1,22 +1,6 @@
 { config, lib, ... }:
 
 {
-  imports = [
-    ../module/slurm-exporter.nix
-    ../module/meteocat-exporter.nix
-    ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
-    ../module/nix-daemon-exporter.nix
-  ];
-
-  age.secrets.grafanaJungleRobotPassword = {
-    file = ../../secrets/jungle-robot-password.age;
-    owner = "grafana";
-    mode = "400";
-  };
-
-  age.secrets.ipmiYml.file = ../../secrets/ipmi.yml.age;
-
   services.grafana = {
     enable = true;
     settings = {
@@ -27,29 +11,14 @@
         http_port = 2342;
         http_addr = "127.0.0.1";
       };
-      smtp = {
-        enabled = true;
-        from_address = "jungle-robot@bsc.es";
-        user = "jungle-robot";
-        # Read the password from a file, which is only readable by grafana user
-        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
-        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
-        host = "mail.bsc.es:465";
-        startTLS_policy = "NoStartTLS";
-      };
       feature_toggles.publicDashboards = true;
-      "auth.anonymous".enabled = true;
-      log.level = "warn";
     };
   };
 
-  # Make grafana alerts also use the proxy
-  systemd.services.grafana.environment = config.networking.proxy.envVars;
-
   services.prometheus = {
     enable = true;
     port = 9001;
-    retentionTime = "5y";
+    retentionTime = "1y";
     listenAddress = "127.0.0.1";
   };
 
@@ -78,13 +47,13 @@
         enable = true;
         group = "root";
         user = "root";
-        configFile = config.age.secrets.ipmiYml.path;
-        # extraFlags = [ "--log.level=debug" ];
+        configFile = ./ipmi.yml;
+        #extraFlags = [ "--log.level=debug" ];
         listenAddress = "127.0.0.1";
       };
       node = {
         enable = true;
-        enabledCollectors = [ "systemd" "logind" ];
+        enabledCollectors = [ "systemd" ];
         port = 9002;
         listenAddress = "127.0.0.1";
       };
@@ -92,11 +61,6 @@
         enable = true;
         listenAddress = "127.0.0.1";
       };
-      blackbox = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        configFile = ./blackbox.yml;
-      };
     };
 
     scrapeConfigs = [
@@ -109,12 +73,6 @@
             "127.0.0.1:9323"
             "127.0.0.1:9252"
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
-            "127.0.0.1:9341" # Slurm exporter
-            "127.0.0.1:9966" # GPFS custom exporter
-            "127.0.0.1:9999" # Nix-daemon custom exporter
-            "127.0.0.1:9929" # Meteocat custom exporter
-            "127.0.0.1:9928" # UPC Qaire custom exporter
-            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];
       }
@@ -128,74 +86,6 @@
           ];
         }];
       }
-      {
-        job_name = "blackbox-http";
-        metrics_path = "/probe";
-        params = { module = [ "http_2xx" ]; };
-        static_configs = [{
-          targets = [
-            "https://www.google.com/robots.txt"
-            "https://pm.bsc.es/"
-            "https://pm.bsc.es/gitlab/"
-            "https://jungle.bsc.es/"
-            "https://gitlab.bsc.es/"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox-icmp";
-        metrics_path = "/probe";
-        params = { module = [ "icmp" ]; };
-        static_configs = [{
-          targets = [
-            "1.1.1.1"
-            "8.8.8.8"
-            "ssfhead"
-            "anella-bsc.cesca.cat"
-            "upc-anella.cesca.cat"
-            "fox.ac.upc.edu"
-            "arenys5.ac.upc.edu"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "gitea";
-        static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
-      }
       {
         # Scrape the IPMI info of the hosts remotely via LAN
         job_name = "ipmi-lan";
@@ -217,7 +107,7 @@
             # Sets the "instance" label with the remote host we are querying
             source_labels = [ "__param_target" ];
             separator = ";";
-            regex = "(.*)-ipmi"; # Remove "-ipm̀i" at the end
+            regex = "(.*)";
             target_label = "instance";
             replacement = "\${1}";
             action = "replace";
@@ -248,33 +138,6 @@
           }
         ];
       }
-      {
-        job_name = "ipmi-raccoon";
-        metrics_path = "/ipmi";
-        static_configs = [
-          { targets = [ "127.0.0.1:9291" ]; }
-        ];
-        params = {
-          target = [ "84.88.51.142" ];
-          module = [ "raccoon" ];
-        };
-      }
-      {
-        job_name = "raccoon";
-        static_configs = [
-          {
-            targets = [ "127.0.0.1:19002" ]; # Node exporter
-          }
-        ];
-      }
-      {
-        job_name = "tent";
-        static_configs = [
-          {
-            targets = [ "127.0.0.1:29002" ]; # Node exporter
-          }
-        ];
-      }
     ];
   };
 }

m/hut/msmtp.nix

@@ -1,24 +0,0 @@
-{ config, lib, ... }:
-{
-  age.secrets.jungleRobotPassword = {
-    file = ../../secrets/jungle-robot-password.age;
-    group = "gitea";
-    mode = "440";
-  };
-
-  programs.msmtp = {
-    enable = true;
-    accounts = {
-      default = {
-        auth = true;
-        tls = true;
-        tls_starttls = false;
-        port = 465;
-        host = "mail.bsc.es";
-        user = "jungle-robot";
-        passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
-        from = "jungle-robot@bsc.es";
-      };
-    };
-  };
-}

m/hut/nginx.nix

@@ -1,73 +0,0 @@
-{ theFlake, pkgs, ... }:
-let
-  website = pkgs.stdenv.mkDerivation {
-    name = "jungle-web";
-    src = theFlake;
-    buildInputs = [ pkgs.hugo ];
-    buildPhase = ''
-      cd web
-      rm -rf public/
-      hugo
-    '';
-    installPhase = ''
-      cp -r public $out
-    '';
-    # Don't mess doc/
-    dontFixup = true;
-  };
-in
-{
-  networking.firewall.allowedTCPPorts = [ 80 ];
-  services.nginx = {
-    enable = true;
-    virtualHosts."jungle.bsc.es" = {
-      root = "${website}";
-      listen = [
-        {
-          addr = "0.0.0.0";
-          port = 80;
-        }
-      ];
-      extraConfig = ''
-        set_real_ip_from 127.0.0.1;
-        set_real_ip_from 84.88.52.107;
-        real_ip_recursive on;
-        real_ip_header X-Forwarded-For;
-
-        location /git {
-          rewrite ^/git$ / break;
-          rewrite ^/git/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:3000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /cache {
-          rewrite ^/cache/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:5000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /lists {
-          proxy_pass http://127.0.0.1:8081;
-          proxy_redirect http:// $scheme://;
-        }
-        location /grafana {
-          proxy_pass http://127.0.0.1:2342;
-          proxy_redirect http:// $scheme://;
-          proxy_set_header Host $host;
-          # Websockets
-          proxy_http_version 1.1;
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection "upgrade";
-        }
-        location ~ ^/~(.+?)(/.*)?$ {
-          alias /ceph/home/$1/public_html$2;
-          index  index.html index.htm;
-          autoindex on;
-          absolute_redirect off;
-        }
-        location /p/ {
-          alias /ceph/p/;
-        }
-      '';
-    };
-  };
-}

m/hut/p.nix

@@ -1,43 +0,0 @@
-{ pkgs, lib, config, ... }:
-let
-  p = pkgs.writeShellScriptBin "p" ''
-    set -e
-    cd /ceph
-    pastedir="p/$USER"
-    mkdir -p "$pastedir"
-
-    ext="txt"
-
-    if [ -n "$1" ]; then
-      ext="$1"
-    fi
-
-    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
-
-    cat > "$out"
-    chmod go+r "$out"
-    echo "https://jungle.bsc.es/$out"
-  '';
-in
-{
-  environment.systemPackages = with pkgs; [ p ];
-
-  # Make sure we have a directory per user. We cannot use the nice
-  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
-  # may not be mounted when it runs.
-  systemd.services.create-paste-dirs = let
-    # Take only normal users in hut
-    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-    commands = lib.concatLists (lib.mapAttrsToList
-      (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
-      ]) users);
-    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
-  in {
-    enable = true;
-    wants = [ "remote-fs.target" ];
-    after = [ "remote-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig.ExecStart = script;
-  };
-}

m/hut/postgresql.nix

@@ -1,19 +0,0 @@
-{ lib, ... }:
-
-{
-  services.postgresql = {
-    enable = true;
-    ensureDatabases = [ "perftestsdb" ];
-    ensureUsers = [
-      { name = "anavarro"; ensureClauses.superuser = true; }
-      { name = "rarias";   ensureClauses.superuser = true; }
-      { name = "grafana"; }
-    ];
-    authentication = ''
-      #type  database     DBuser    auth-method
-      local  perftestsdb  rarias    trust
-      local  perftestsdb  anavarro  trust
-      local  perftestsdb  grafana   trust
-    '';
-  };
-}

m/hut/public-inbox.css

@@ -1,79 +0,0 @@
-/*
- * CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
- * Dark color scheme using 216 web-safe colors, inspired
- * somewhat by the default color scheme in mutt.
- * It reduces eyestrain for me, and energy usage for all:
- * https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
- */
-
-* {
-	font-size: 14px;
-	font-family: monospace;
-}
-
-pre {
-	white-space: pre-wrap;
-	padding: 10px;
-	background: #f5f5f5;
-}
-
-hr {
-	margin: 30px 0;
-}
-
-body {
-	max-width: 120ex; /* 120 columns wide */
-	margin: 50px auto;
-}
-
-/*
- * Underlined links add visual noise which make them hard-to-read.
- * Use colors to make them stand out, instead.
- */
-a:link {
-	color: #007;
-	text-decoration: none;
-}
-a:visited {
-	color:#504;
-}
-a:hover {
-	text-decoration: underline;
-}
-
-/* quoted text in emails gets a different color */
-*.q { color:gray }
-
-/*
- * these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
- * (cgit uses <div>, public-inbox uses <span>)
- */
-*.add { color:darkgreen } /* diff post-image lines */
-*.del { color:darkred } /* diff pre-image lines */
-*.head { color:black } /* diff header (metainformation) */
-*.hunk { color:gray } /* diff hunk-header */
-
-/*
- * highlight 3.x colors (tested 3.18) for displaying blobs.
- * This doesn't use most of the colors available, as I find too
- * many colors overwhelming, so the default is commented out.
- */
-.hl.num { color:#f30 } /* number */
-.hl.esc { color:#f0f } /* escape character */
-.hl.str { color:#f30 } /* string */
-.hl.ppc { color:#f0f } /* preprocessor */
-.hl.pps { color:#f30 } /* preprocessor string */
-.hl.slc { color:#09f } /* single-line comment */
-.hl.com { color:#09f } /* multi-line comment */
-/* .hl.opt { color:#ccc } */ /* operator */
-/* .hl.ipl { color:#ccc } */ /* interpolation */
-
-/* keyword groups kw[a-z] */
-.hl.kwa { color:#ff0 }
-.hl.kwb { color:#0f0 }
-.hl.kwc { color:#ff0 }
-/* .hl.kwd { color:#ccc } */
-
-/* line-number (unused by public-inbox) */
-/* .hl.lin { color:#ccc } */
-

m/hut/public-inbox.nix

@@ -1,47 +0,0 @@
-{ lib, ... }:
-
-{
-  services.public-inbox = {
-    enable = true;
-    http = {
-      enable = true;
-      port = 8081;
-      mounts = [ "/lists" ];
-    };
-    settings.publicinbox = {
-      css = [ "${./public-inbox.css}" ];
-      wwwlisting = "all";
-    };
-    inboxes = {
-      bscpkgs = {
-        url = "https://jungle.bsc.es/lists/bscpkgs";
-        address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
-        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
-        description = "Patches for bscpkgs";
-        listid = "~rodarima/bscpkgs.lists.sr.ht";
-      };
-      jungle = {
-        url = "https://jungle.bsc.es/lists/jungle";
-        address = [ "~rodarima/jungle@lists.sr.ht" ];
-        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
-        description = "Patches for jungle";
-        listid = "~rodarima/jungle.lists.sr.ht";
-      };
-    };
-  };
-
-  # We need access to the network for the watch service, as we will fetch the
-  # emails directly from the IMAP server.
-  systemd.services.public-inbox-watch.serviceConfig = {
-    PrivateNetwork = lib.mkForce false;
-    RestrictAddressFamilies = lib.mkForce [ "AF_UNIX"  "AF_INET" "AF_INET6" ];
-    KillSignal = "SIGKILL"; # Avoid slow shutdown
-
-    # Required for chmod(..., 02750) on directories by git, from
-    # systemd.exec(8):
-    # > Note that this restricts marking of any type of file system object with
-    # > these bits, including both regular files and directories (where the SGID
-    # > is a different meaning than for files, see documentation).
-    RestrictSUIDSGID = lib.mkForce false;
-  };
-}

m/hut/targets.yml

@@ -1,15 +1,15 @@
 - targets:
-  - owl1-ipmi
-  - owl2-ipmi
-  - xeon03-ipmi
-  - xeon04-ipmi
-  - koro-ipmi
-  - xeon06-ipmi
-  - hut-ipmi
-  - eudy-ipmi
+  - 10.0.40.101
+  - 10.0.40.102
+  - 10.0.40.103
+  - 10.0.40.104
+  - 10.0.40.105
+  - 10.0.40.106
+  - 10.0.40.107
+  - 10.0.40.108
   # Storage
-  - bay-ipmi
-  - oss01-ipmi
-  - lake2-ipmi
+  - 10.0.40.141
+  - 10.0.40.142
+  - 10.0.40.143
   labels:
     job: ipmi-lan

m/koro/configuration.nix

@@ -2,11 +2,13 @@
 
 {
   imports = [
-    ../common/ssf.nix
+    ../common/main.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix
     ../eudy/users.nix
+    ../eudy/slurm.nix
+    ./users.nix
     ./kernel.nix
   ];
 

m/koro/kernel.nix

@@ -1,29 +1,9 @@
 { pkgs, lib, ... }:
 
 let
-  #fcs-devel = pkgs.linuxPackages_custom {
-  #   version = "6.2.8";
-  #   src = /mnt/data/kernel/fcs/kernel/src;
-  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
-  #};
+  kernel = nixos-fcsv4;
 
-  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
-  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
-  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
-  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
-  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
-  #   version = "6.2.8";
-  #   src = builtins.fetchGit {
-  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
-  #     rev = gitCommit;
-  #     ref = "fcs";
-  #   };
-  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
-  #};
-
-  kernel = nixos-fcs;
-
-  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -40,13 +20,27 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  }));
+  });
 
-  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
-  nixos-fcs-lockstat = nixos-fcs.override {
+  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
+  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
+  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
+  nixos-fcsv4 = nixos-fcs-kernel {gitCommit = "c94c3d946f33ac3e5782a02ee002cc1164c0cb4f";};
+
+  nixos-fcsv1-lockstat = nixos-fcs-kernel {
+    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
     lockStat = true;
   };
-  nixos-fcs-lockstat-preempt = nixos-fcs.override {
+  nixos-fcsv2-lockstat = nixos-fcs-kernel {
+    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
+    lockStat = true;
+  };
+  nixos-fcsv3-lockstat = nixos-fcs-kernel {
+    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+    lockStat = true;
+  };
+  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
+    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
     lockStat = true;
     preempt = true;
   };
@@ -66,5 +60,5 @@ in {
   
   # enable memory overcommit, needed to build a taglibc system using nix after
   # increasing the openblas memory footprint
-  boot.kernel.sysctl."vm.overcommit_memory" = 1;
+  boot.kernel.sysctl."vm.overcommit_memory" = lib.mkForce 1;
 }

m/koro/users.nix

@@ -0,0 +1,17 @@
+{ ... }:
+
+{
+  users.users = {
+    vlopez = {
+      uid = 4334;
+      isNormalUser = true;
+      home = "/home/Computational/vlopez";
+      description = "Victor López";
+      group = "Computational";
+      hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
+      openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
+      ];
+    };
+  };
+}

m/lake2/configuration.nix

@@ -2,20 +2,20 @@
 
 {
   imports = [
-    ../common/ssf.nix
-    ../module/monitoring.nix
+    ../common/main.nix
+    ../common/monitoring.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
 
-  boot.kernel.sysctl = {
-    "kernel.yama.ptrace_scope" = lib.mkForce "1";
-  };
-
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+
   services.ceph = {
     enable = true;
     global = {
@@ -49,16 +49,6 @@
       address = "10.0.42.42";
       prefixLength = 24;
     } ];
-    firewall = {
-      extraCommands = ''
-        # Accept all incoming TCP traffic from bay
-        iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
-        # Accept monitoring requests from hut
-        iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
-        # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
-      '';
-    };
   };
 
   # Missing service for volumes, see:

m/map.nix

@@ -1,70 +0,0 @@
-{
-  # In physical order from top to bottom (see note below)
-  ssf = {
-    # Switches for Ethernet and OmniPath
-    switch-C6-S1A-05 = { pos=42; size=1; model="Dell S3048-ON"; };
-    switch-opa = { pos=41; size=1; };
-
-    # SSF login
-    ssfhead = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="operations@bsc.es"; };
-
-    # Storage
-    bay   = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };
-    lake1 = { pos=37; size=1; label="OSS01"; board="S2600WT2R"; sn="BQWL64850234"; contact="rodrigo.arias@bsc.es"; };
-    lake2 = { pos=36; size=1; label="OSS02"; board="S2600WT2R"; sn="BQWL64850266"; contact="rodrigo.arias@bsc.es"; };
-
-    # Compute xeon
-    owl1   = { pos=35; size=1; label="SSF-XEON01"; board="S2600WTTR"; sn="BQWL64954172"; contact="rodrigo.arias@bsc.es"; };
-    owl2   = { pos=34; size=1; label="SSF-XEON02"; board="S2600WTTR"; sn="BQWL64756560"; contact="rodrigo.arias@bsc.es"; };
-    xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; };
-    # Slot 34 empty
-    koro   = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; };
-    xeon06 = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
-    hut    = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; };
-    eudy   = { pos=28; size=1; label="SSF-XEON08"; board="S2600WTTR"; sn="BQWL64756586"; contact="aleix.rocanonell@bsc.es"; };
-
-    # 16 KNL nodes, 4 per chassis
-    knl01_04 = { pos=26; size=2; label="KNL01..KNL04"; board="HNS7200APX"; };
-    knl05_08 = { pos=24; size=2; label="KNL05..KNL18"; board="HNS7200APX"; };
-    knl09_12 = { pos=22; size=2; label="KNL09..KNL12"; board="HNS7200APX"; };
-    knl13_16 = { pos=20; size=2; label="KNL13..KNL16"; board="HNS7200APX"; };
-
-    # Slot 19 empty
-
-    # EPI (hw team, guessed order)
-    epi01 = { pos=18; size=1; contact="joan.cabre@bsc.es"; };
-    epi02 = { pos=17; size=1; contact="joan.cabre@bsc.es"; };
-    epi03 = { pos=16; size=1; contact="joan.cabre@bsc.es"; };
-    anon  = { pos=14; size=2; }; # Unlabeled machine. Operative
-
-    # These are old and decommissioned (off)
-    power8    = { pos=12; size=2; label="BSCPOWER8N3";   decommissioned=true; };
-    powern1   = { pos=8;  size=4; label="BSCPOWERN1";    decommissioned=true; };
-    gustafson = { pos=7;  size=1; label="gustafson";     decommissioned=true; };
-    odap01    = { pos=3;  size=4; label="ODAP01";        decommissioned=true; };
-    amhdal    = { pos=2;  size=1; label="AMHDAL";        decommissioned=true; }; # sic
-    moore     = { pos=1;  size=1; label="moore (earth)"; decommissioned=true; };
-  };
-
-  bsc2218 = {
-    raccoon = { board="W2600CR"; sn="QSIP22500829"; contact="rodrigo.arias@bsc.es"; };
-    tent    = { label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; };
-  };
-
-  upc = {
-    fox = { board="H13DSG-O-CPU"; sn="UM24CS600392"; prod="AS-4125GS-TNRT"; prod_sn="E508839X5103339"; contact="rodrigo.arias@bsc.es"; };
-  };
-
-  # NOTE: Position is specified in "U" units (44.45 mm) and starts at 1 from the
-  # bottom. Example:
-  #
-  #  |   ...  | - [pos+size] <--- Label in chassis
-  #  +--------+
-  #  |  node  | - [pos+1]
-  #  |   2U   | - [pos]
-  #  +------- +
-  #  |   ...  | - [pos-1]
-  #
-  # NOTE: The board and sn refers to the FRU information (Board Product and
-  # Board Serial) via `ipmitool fru print 0`.
-}

m/module/ceph.nix

@@ -3,6 +3,7 @@
 # Mounts the /ceph filesystem at boot
 {
   environment.systemPackages = with pkgs; [
+    ceph
     ceph-client
     fio # For benchmarks
   ];

m/module/debuginfod.nix

@@ -1,3 +0,0 @@
-{
-  services.nixseparatedebuginfod.enable = true;
-}

m/module/emulation.nix

@@ -1,3 +0,0 @@
-{
-  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
-}

m/module/hut-substituter.nix

@@ -1,10 +0,0 @@
-{ config, ... }:
-{
-  nix.settings =
-    # Don't add hut as a cache to itself
-    assert config.networking.hostName != "hut";
-    {
-      extra-substituters = [ "http://hut/cache" ];
-      extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-    };
-}

m/module/jungle-users.nix

@@ -1,24 +0,0 @@
-{ config, lib, ... }:
-
-with lib;
-
-{
-  options = {
-    users.jungleUsers = mkOption {
-      type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
-      description = ''
-        Same as users.users but with the extra `hosts` attribute, which controls
-        access to the nodes by `networking.hostName`.
-      '';
-    };
-  };
-
-  config = let
-    allowedUser = host: userConf: builtins.elem host userConf.hosts;
-    filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
-    removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
-    currentHost = config.networking.hostName;
-  in {
-    users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
-  };
-}

m/module/meteocat-exporter.nix

@@ -1,17 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-{
-  systemd.services."prometheus-meteocat-exporter" = {
-    wantedBy = [ "multi-user.target" ];
-    after = [ "network.target" ];
-    serviceConfig = {
-      Restart = mkDefault "always";
-      PrivateTmp = mkDefault true;
-      WorkingDirectory = mkDefault "/tmp";
-      DynamicUser = mkDefault true;
-      ExecStart = "${pkgs.meteocat-exporter}/bin/meteocat-exporter";
-    };
-  };
-}

m/module/nix-daemon-builds.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-
-# Locate nix daemon pid
-nd=$(pgrep -o nix-daemon)
-
-# Locate children of nix-daemon
-pids1=$(tr ' ' '\n' < "/proc/$nd/task/$nd/children")
-
-# For each children, locate 2nd level children
-pids2=$(echo "$pids1" | xargs -I @ /bin/sh -c 'cat /proc/@/task/*/children' | tr ' ' '\n')
-
-cat <<EOF
-HTTP/1.1 200 OK
-Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
-
-# HELP nix_daemon_build Nix daemon derivation build state.
-# TYPE nix_daemon_build gauge
-EOF
-
-for pid in $pids2; do
-  name=$(cat /proc/$pid/environ 2>/dev/null | tr '\0' '\n' | rg "^name=(.+)" - --replace '$1' | tr -dc ' [:alnum:]_\-\.')
-  user=$(ps -o uname= -p "$pid")
-  if [ -n "$name" -a -n "$user" ]; then
-    printf 'nix_daemon_build{user="%s",name="%s"} 1\n' "$user" "$name"
-  fi
-done

m/module/nix-daemon-exporter.nix

@@ -1,23 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  script = pkgs.runCommand "nix-daemon-exporter.sh" { }
-    ''
-      cp ${./nix-daemon-builds.sh} $out;
-      chmod +x $out
-    ''
-  ;
-in
-{
-  systemd.services.nix-daemon-exporter = {
-    description = "Daemon to export nix-daemon metrics";
-    path = [ pkgs.procps pkgs.ripgrep ];
-    wantedBy = [ "default.target" ];
-    serviceConfig = {
-      Type = "simple";
-      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9999,fork EXEC:${script}";
-      # Needed root to read the environment, potentially unsafe
-      User = "root";
-      Group = "root";
-    };
-  };
-}

m/module/p.nix

@@ -1,68 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cfg = config.services.p;
-in
-{
-  options = {
-    services.p = {
-      enable = lib.mkOption {
-        type = lib.types.bool;
-        default = false;
-        description = "Whether to enable the p service.";
-      };
-      path = lib.mkOption {
-        type = lib.types.str;
-        default = "/var/lib/p";
-        description = "Where to save the pasted files on disk.";
-      };
-      url = lib.mkOption {
-        type = lib.types.str;
-        default = "https://jungle.bsc.es/p";
-        description = "URL prefix for the printed file.";
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    environment.systemPackages = let 
-      p = pkgs.writeShellScriptBin "p" ''
-        set -e
-        pastedir="${cfg.path}/$USER"
-        cd "$pastedir"
-
-        ext="txt"
-        if [ -n "$1" ]; then
-          ext="$1"
-        fi
-
-        out=$(mktemp "XXXXXXXX.$ext")
-        cat > "$out"
-        chmod go+r "$out"
-        echo "${cfg.url}/$USER/$out"
-      '';
-    in [ p ];
-
-    systemd.services.p = let
-      # Take only normal users
-      users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-      # Create a directory for each user
-      commands = lib.concatLists (lib.mapAttrsToList (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0755 ${cfg.path}/${user.name}"
-      ]) users);
-    in {
-      description = "P service setup";
-      requires = [ "network-online.target" ];
-      #wants = [ "remote-fs.target" ];
-      #after = [ "remote-fs.target" ];
-      wantedBy = [ "multi-user.target" ];
-      serviceConfig = {
-        ExecStart = pkgs.writeShellScript "p-init.sh" (''
-
-          install -d -o root -g root -m 0755 ${cfg.path}
-
-        '' + (lib.concatLines commands));
-      };
-    };
-  };
-}

m/module/shared-nix-store.nix

@@ -0,0 +1,69 @@
+{ ... }:
+{
+  # Don't make the nix store read-only, as this would prevent the overlay FS
+  # from being able to mount it.
+  boot.readOnlyNixStore = false;
+
+  # The nix-daemon.socket has an unnecessary dependency over the /nix/store
+  # mount point. But that mount point won't be provided until the network is
+  # ready. However, the network-address-eno1.service, has a dependency over
+  # sockets.target, causing a cycle.
+  # One solution is to make the nix-daemon.socket depend only on the socket
+  # patch (which is already covered by ConditionPathIsReadWrite =
+  # /nix/var/nix/daemon-socket), instead on the /nix/store.
+  #
+  # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor =
+  # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get
+  # added by systemd when the override config is merged with the one that Nix
+  # provides:
+  #
+  # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor
+  # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket
+  # 
+  # To fix this, the Nix package is patched to only depend on /nix/var instead.
+  # See ../../pkgs/overlay.nix for details.
+
+  # Mount the hut nix store via NFS in read-only mode.
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  # A workdir is also needed, so setup a permanent dir using tmpfiles.
+  systemd.tmpfiles.rules = [
+    "d /mnt/nix-work 0700 root root -"
+  ];
+
+  # Mount an overlay in /nix/store using as lower layer the NFS store and upper
+  # layer the disk nix store. The destination is still the nix store in
+  # /nix/store (confusing). We need rw access, as the daemon need to write the
+  # lock files to build derivations locally. Use a systemd mount unit directly
+  # so we can specify the LazyUmount option and we avoid having it mounted
+  # in the stage1 before systemd.
+  systemd.mounts = [
+    {
+      what = "overlay";
+      type = "overlay";
+      where = "/nix/store";
+      # We need the local-fs.target to be ready, so the network interfaces can
+      # be configured to the network.target is reached. So make this a netdev
+      # mount.
+      options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
+      description = "Overlay /nix/store mount";
+      mountConfig = {
+        LazyUnmount = true;
+      };
+
+      # Run the unit after remote-fs-pre.target but before the remote-fs.target
+      after = [ "remote-fs-pre.target"];
+      before = [ "umount.target" "remote-fs.target" ];
+      # Install by using wantedBy over remote-fs.target
+      wantedBy = [ "remote-fs.target" ];
+      unitConfig = {
+        # We need to wait for the NFS mount
+        RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
+      };
+    }
+  ];
+}

m/module/slurm-exporter.nix

@@ -1,28 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-# See also: https://github.com/NixOS/nixpkgs/pull/112010
-# And: https://github.com/NixOS/nixpkgs/pull/115839
-
-with lib;
-
-{
-  systemd.services."prometheus-slurm-exporter" = {
-    wantedBy = [ "multi-user.target" ];
-    after = [ "network.target" ];
-    serviceConfig = {
-      Restart = mkDefault "always";
-      PrivateTmp = mkDefault true;
-      WorkingDirectory = mkDefault "/tmp";
-      DynamicUser = mkDefault true;
-      ExecStart = ''
-        ${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
-      '';
-      Environment = [
-        "PATH=${pkgs.slurm}/bin"
-        # We need to specify the slurm config to be able to talk to the slurmd
-        # daemon.
-        "SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
-      ];
-    };
-  };
-}

m/module/slurm-hut-nix-store.nix

@@ -1,19 +0,0 @@
-{ ... }:
-
-{
-  # Mount the hut nix store via NFS
-  fileSystems."/mnt/hut-nix-store" = {
-    device = "hut:/nix/store";
-    fsType = "nfs";
-    options = [ "ro" ];
-  };
-
-  systemd.services.slurmd.serviceConfig = {
-    # When running a job, bind the hut store in /nix/store so the paths are
-    # available too.
-    # FIXME: This doesn't keep the programs in /run/current-system/sw/bin
-    # available in the store. Ideally they should be merged but the overlay FS
-    # doesn't work when the underlying directories change.
-    BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
-  };
-}

m/module/ssh-hut-extern.nix

@@ -1,9 +0,0 @@
-{
-  programs.ssh.extraConfig = ''
-    Host ssfhead
-      HostName ssflogin.bsc.es
-    Host hut
-      ProxyJump ssfhead
-      HostName xeon07
-  '';
-}

m/module/upc-qaire-exporter.nix

@@ -1,17 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-with lib;
-
-{
-  systemd.services."prometheus-upc-qaire-exporter" = {
-    wantedBy = [ "multi-user.target" ];
-    after = [ "network.target" ];
-    serviceConfig = {
-      Restart = mkDefault "always";
-      PrivateTmp = mkDefault true;
-      WorkingDirectory = mkDefault "/tmp";
-      DynamicUser = mkDefault true;
-      ExecStart = "${pkgs.upc-qaire-exporter}/bin/upc-qaire-exporter";
-    };
-  };
-}

m/module/vpn-dac.nix

@@ -1,35 +0,0 @@
-{config, ...}:
-{
-  age.secrets.vpn-dac-login.file = ../../secrets/vpn-dac-login.age;
-  age.secrets.vpn-dac-client-key.file = ../../secrets/vpn-dac-client-key.age;
-
-  services.openvpn.servers = {
-    # systemctl status openvpn-dac.service
-    dac = {
-      config = ''
-        client
-        dev tun
-        proto tcp
-        remote vpn.ac.upc.edu 1194
-        remote vpn.ac.upc.edu 80
-        resolv-retry infinite
-        nobind
-        persist-key
-        persist-tun
-        ca ${./vpn-dac/ca.crt}
-        cert ${./vpn-dac/client.crt}
-        # Only key needs to be secret
-        key ${config.age.secrets.vpn-dac-client-key.path}
-        remote-cert-tls server
-        comp-lzo
-        verb 3
-        auth-user-pass ${config.age.secrets.vpn-dac-login.path}
-        reneg-sec 0
-
-        # Only route fox-ipmi
-        pull-filter ignore "route "
-        route 147.83.35.27 255.255.255.255
-      '';
-    };
-  };
-}

m/module/vpn-dac/ca.crt

@@ -1,31 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFUjCCBDqgAwIBAgIJAJH118PApk5hMA0GCSqGSIb3DQEBCwUAMIHLMQswCQYD
-VQQGEwJFUzESMBAGA1UECBMJQmFyY2Vsb25hMRIwEAYDVQQHEwlCYXJjZWxvbmEx
-LTArBgNVBAoTJFVuaXZlcnNpdGF0IFBvbGl0ZWNuaWNhIGRlIENhdGFsdW55YTEk
-MCIGA1UECxMbQXJxdWl0ZWN0dXJhIGRlIENvbXB1dGFkb3JzMRAwDgYDVQQDEwdM
-Q0FDIENBMQ0wCwYDVQQpEwRMQ0FDMR4wHAYJKoZIhvcNAQkBFg9sY2FjQGFjLnVw
-Yy5lZHUwHhcNMTYwMTEyMTI0NDIxWhcNNDYwMTEyMTI0NDIxWjCByzELMAkGA1UE
-BhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0w
-KwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAi
-BgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENB
-QyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMu
-ZWR1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0CteSeof7Xwi51kC
-F0nQ4E9iR5Lq7wtfRuVPn6JJcIxJJ6+F9gr4R/HIHTztW4XAzReE36DYfexupx3D
-6UgQIkMLlVyGqRbulNF+RnCx20GosF7Dm4RGBVvOxBP1PGjYq/A+XhaaDAFd0cOF
-LMNkzuYP7PF0bnBEaHnxmN8bPmuyDyas7fK9AAc3scyWT2jSBPbOVFvCJwPg8MH9
-V/h+hKwL/7hRt1MVfVv2qyIuKwTki8mUt0RcVbP7oJoRY5K1+R52phIz/GL/b4Fx
-L6MKXlQxLi8vzP4QZXgCMyV7oFNdU3VqCEXBA11YIRvsOZ4QS19otIk/ZWU5x+HH
-LAIJ7wIDAQABo4IBNTCCATEwHQYDVR0OBBYEFNyezX1cH1N4QR14ebBpljqmtE7q
-MIIBAAYDVR0jBIH4MIH1gBTcns19XB9TeEEdeHmwaZY6prRO6qGB0aSBzjCByzEL
-MAkGA1UEBhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vs
-b25hMS0wKwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVu
-eWExJDAiBgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UE
-AxMHTENBQyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0Bh
-Yy51cGMuZWR1ggkAkfXXw8CmTmEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsF
-AAOCAQEAUAmOvVXIQrR+aZVO0bOTeugKBHB75eTIZSIHIn2oDUvDbAP5GXIJ56A1
-6mZXxemSMY8/9k+pRcwJhfat3IgvAN159XSqf9kRv0NHgc3FWUI1Qv/BsAn0vJO/
-oK0dbmbbRWqt86qNrCN+cUfz5aovvxN73jFfnvfDQFBk/8enj9wXxYfokjjLPR1Q
-+oTkH8dY68qf71oaUB9MndppPEPSz0K1S6h1XxvJoSu9MVSXOQHiq1cdZdxRazI3
-4f7q9sTCL+khwDAuZxAYzlEYxFFa/NN8PWU6xPw6V+t/aDhOiXUPJQB/O/K7mw3Z
-TQQx5NqM7B5jjak5fauR3/oRD8XXsA==
------END CERTIFICATE-----

m/module/vpn-dac/client.crt

@@ -1,100 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 2 (0x2)
-    Signature Algorithm: sha256WithRSAEncryption
-        Issuer: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
-        Validity
-            Not Before: Jan 12 12:45:41 2016 GMT
-            Not After : Jan 12 12:45:41 2046 GMT
-        Subject: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=client/name=LCAC/emailAddress=lcac@ac.upc.edu
-        Subject Public Key Info:
-            Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
-                Modulus:
-                    00:97:99:fa:7a:0e:4d:e2:1d:a5:b1:a8:14:18:64:
-                    c7:66:bf:de:99:1d:92:3b:86:82:4d:95:39:f7:a6:
-                    56:49:97:14:4f:e3:37:00:6c:f4:d0:1d:56:79:e7:
-                    19:b5:dd:36:15:8e:1d:57:7b:59:29:d2:11:bf:58:
-                    48:e0:f7:41:3d:16:64:8d:a2:0b:4a:ac:fa:c6:83:
-                    dc:10:2a:2c:d9:97:48:ee:11:2a:bc:4b:60:dd:b9:
-                    2e:8f:45:ca:87:0b:38:65:1c:f8:a2:1d:f9:50:aa:
-                    6e:60:f9:48:df:57:12:23:e1:e7:0c:81:5c:9f:c5:
-                    b2:e6:99:99:95:30:6d:57:36:06:8c:fd:fb:f9:4f:
-                    60:d2:3c:ba:ae:28:56:2f:da:58:5c:e8:c5:7b:ec:
-                    76:d9:28:6e:fb:8c:07:f9:d7:23:c3:72:76:3c:fa:
-                    dc:20:67:8f:cc:16:e0:91:07:d5:68:f9:20:4d:7d:
-                    5c:2d:02:04:16:76:52:f3:53:be:a3:dc:0d:d5:fb:
-                    6b:55:29:f3:52:35:c8:7d:99:d1:4a:94:be:b1:8e:
-                    fd:85:18:25:eb:41:e9:56:da:af:62:84:20:0a:00:
-                    17:94:92:94:91:6a:f8:54:37:17:ee:1e:bb:fb:93:
-                    71:91:d9:e4:e9:b8:3b:18:7d:6d:7d:4c:ce:58:55:
-                    f9:41
-                Exponent: 65537 (0x10001)
-        X509v3 extensions:
-            X509v3 Basic Constraints: 
-                CA:FALSE
-            Netscape Comment: 
-                Easy-RSA Generated Certificate
-            X509v3 Subject Key Identifier: 
-                1B:88:06:D5:33:1D:5C:48:46:B5:DE:78:89:36:96:91:3A:74:43:18
-            X509v3 Authority Key Identifier: 
-                keyid:DC:9E:CD:7D:5C:1F:53:78:41:1D:78:79:B0:69:96:3A:A6:B4:4E:EA
-                DirName:/C=ES/ST=Barcelona/L=Barcelona/O=Universitat Politecnica de Catalunya/OU=Arquitectura de Computadors/CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
-                serial:91:F5:D7:C3:C0:A6:4E:61
-
-            X509v3 Extended Key Usage: 
-                TLS Web Client Authentication
-            X509v3 Key Usage: 
-                Digital Signature
-            X509v3 Subject Alternative Name: 
-                DNS:client
-    Signature Algorithm: sha256WithRSAEncryption
-         42:e8:50:b2:e7:88:75:86:0b:bb:29:e3:aa:c6:0e:4c:e8:ea:
-         3d:0c:02:31:7f:3b:80:0c:3f:80:af:45:d6:62:27:a0:0e:e7:
-         26:09:12:97:95:f8:d9:9b:89:b5:ef:56:64:f1:de:82:74:e0:
-         31:0a:cc:90:0a:bd:50:b8:54:95:0a:ae:3b:40:df:76:b6:d1:
-         01:2e:f3:96:9f:52:d4:e9:14:6d:b7:14:9d:45:99:33:36:2a:
-         01:0b:15:1a:ed:55:dc:64:83:65:1a:06:42:d9:c7:dc:97:d4:
-         02:81:c2:58:2b:ea:e4:b7:ae:84:3a:e4:3f:f1:2e:fa:ec:f3:
-         40:5d:b8:6a:d5:5e:e1:e8:2f:e2:2f:48:a4:38:a1:4f:22:e3:
-         4f:66:94:aa:02:78:9a:2b:7a:5d:aa:aa:51:a5:e3:d0:91:e9:
-         1d:f9:08:ed:8b:51:c9:a6:af:46:85:b5:1c:ed:12:a1:28:33:
-         75:36:00:d8:5c:14:65:96:c0:28:7d:47:50:a4:89:5f:b0:72:
-         1a:4b:13:17:26:0f:f0:b8:65:3c:e9:96:36:f9:bf:90:59:33:
-         87:1f:01:03:25:f8:f0:3a:9b:33:02:d0:0a:43:b5:0a:cf:62:
-         a1:45:38:37:07:9d:9c:94:0b:31:c6:3c:34:b7:fc:5a:0c:e4:
-         bf:23:f6:7d
------BEGIN CERTIFICATE-----
-MIIFqjCCBJKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCByzELMAkGA1UEBhMCRVMx
-EjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0wKwYDVQQK
-EyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAiBgNVBAsT
-G0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENBQyBDQTEN
-MAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MB4X
-DTE2MDExMjEyNDU0MVoXDTQ2MDExMjEyNDU0MVowgcoxCzAJBgNVBAYTAkVTMRIw
-EAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNlbG9uYTEtMCsGA1UEChMk
-VW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1bnlhMSQwIgYDVQQLExtB
-cnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxDzANBgNVBAMTBmNsaWVudDENMAsG
-A1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MIIBIjAN
-BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAl5n6eg5N4h2lsagUGGTHZr/emR2S
-O4aCTZU596ZWSZcUT+M3AGz00B1WeecZtd02FY4dV3tZKdIRv1hI4PdBPRZkjaIL
-Sqz6xoPcECos2ZdI7hEqvEtg3bkuj0XKhws4ZRz4oh35UKpuYPlI31cSI+HnDIFc
-n8Wy5pmZlTBtVzYGjP37+U9g0jy6rihWL9pYXOjFe+x22Shu+4wH+dcjw3J2PPrc
-IGePzBbgkQfVaPkgTX1cLQIEFnZS81O+o9wN1ftrVSnzUjXIfZnRSpS+sY79hRgl
-60HpVtqvYoQgCgAXlJKUkWr4VDcX7h67+5Nxkdnk6bg7GH1tfUzOWFX5QQIDAQAB
-o4IBljCCAZIwCQYDVR0TBAIwADAtBglghkgBhvhCAQ0EIBYeRWFzeS1SU0EgR2Vu
-ZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQbiAbVMx1cSEa13niJNpaROnRD
-GDCCAQAGA1UdIwSB+DCB9YAU3J7NfVwfU3hBHXh5sGmWOqa0TuqhgdGkgc4wgcsx
-CzAJBgNVBAYTAkVTMRIwEAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNl
-bG9uYTEtMCsGA1UEChMkVW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1
-bnlhMSQwIgYDVQQLExtBcnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxEDAOBgNV
-BAMTB0xDQUMgQ0ExDTALBgNVBCkTBExDQUMxHjAcBgkqhkiG9w0BCQEWD2xjYWNA
-YWMudXBjLmVkdYIJAJH118PApk5hMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAsGA1Ud
-DwQEAwIHgDARBgNVHREECjAIggZjbGllbnQwDQYJKoZIhvcNAQELBQADggEBAELo
-ULLniHWGC7sp46rGDkzo6j0MAjF/O4AMP4CvRdZiJ6AO5yYJEpeV+NmbibXvVmTx
-3oJ04DEKzJAKvVC4VJUKrjtA33a20QEu85afUtTpFG23FJ1FmTM2KgELFRrtVdxk
-g2UaBkLZx9yX1AKBwlgr6uS3roQ65D/xLvrs80BduGrVXuHoL+IvSKQ4oU8i409m
-lKoCeJorel2qqlGl49CR6R35CO2LUcmmr0aFtRztEqEoM3U2ANhcFGWWwCh9R1Ck
-iV+wchpLExcmD/C4ZTzpljb5v5BZM4cfAQMl+PA6mzMC0ApDtQrPYqFFODcHnZyU
-CzHGPDS3/FoM5L8j9n0=
------END CERTIFICATE-----

m/owl1/configuration.nix

@@ -2,13 +2,10 @@
 
 {
   imports = [
-    ../common/ssf.nix
+    ../common/main.nix
     ../module/ceph.nix
-    ../module/emulation.nix
-    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
-    ../module/debuginfod.nix
-    ../module/hut-substituter.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -2,13 +2,10 @@
 
 {
   imports = [
-    ../common/ssf.nix
+    ../common/main.nix
     ../module/ceph.nix
-    ../module/emulation.nix
-    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
-    ../module/debuginfod.nix
-    ../module/hut-substituter.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/raccoon/configuration.nix

@@ -1,100 +0,0 @@
-{ config, pkgs, lib, modulesPath, ... }:
-
-{
-  imports = [
-    ../common/base.nix
-    ../module/emulation.nix
-    ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
-    ../eudy/kernel/perf.nix
-  ];
-
-  # Don't install Grub on the disk yet
-  boot.loader.grub.device = "nodev";
-
-  # Enable serial console
-  boot.kernelParams = [
-    "console=tty1"
-    "console=ttyS1,115200"
-  ];
-
-  networking = {
-    hostName = "raccoon";
-    # Only BSC DNSs seem to be reachable from the office VLAN
-    nameservers = [ "84.88.52.35" "84.88.52.36" ];
-    defaultGateway = "84.88.51.129";
-    interfaces.eno0.ipv4.addresses = [ {
-      address = "84.88.51.152";
-      prefixLength = 25;
-    } ];
-    interfaces.enp5s0f1.ipv4.addresses = [ {
-      address = "10.0.44.1";
-      prefixLength = 24;
-    } ];
-    nat = {
-      enable = true;
-      internalInterfaces = [ "enp5s0f1" ];
-      externalInterface = "eno0";
-    };
-    hosts = {
-      "10.0.44.4" = [ "tent" ];
-    };
-  };
-
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
-
-  # Enable performance governor
-  powerManagement.cpuFreqGovernor = "performance";
-
-  # Configure Nvidia driver to use with CUDA
-  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
-  hardware.graphics.enable = true;
-  nixpkgs.config.allowUnfree = true;
-  nixpkgs.config.nvidia.acceptLicense = true;
-  services.xserver.videoDrivers = [ "nvidia" ];
-
-  # Disable garbage collection for now
-  nix.gc.automatic = lib.mkForce false;
-
-  services.openssh.settings.X11Forwarding = true;
-
-  services.prometheus.exporters.node = {
-    enable = true;
-    enabledCollectors = [ "systemd" ];
-    port = 9002;
-    listenAddress = "127.0.0.1";
-  };
-
-  users.motd = ''
-    ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⡎⢰⣧⠀⠀⠀⠁⠈⠛⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⣴⡦⠶⠟⠓⠚⠻⡄⠀
-    ⠀⠀⠀⠀⠀⠀⣧⠀⣱⣀⣰⣧⠀⢀⠀⣘⣿⣿⣦⣶⣄⣠⡀⠀⠀⣀⣀⣤⣴⣄⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠿⠏⠁⠀⣀⣠⣶⣿⡶⣿⠀
-    ⠀⠀⠀⠀⠀⠀⣹⣆⠘⣿⣿⣿⣇⢸⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣶⣶⣦⡀⣀⣤⣠⣤⡾⠋⠀⢀⣤⣶⣿⣿⣿⣿⣿⣿⣿⡀
-    ⠀⠀⠀⠀⠀⠀⠘⢿⡄⢼⣿⣿⣿⣿⣿⡟⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣵⣾⡾⠙⣋⣩⣽⣿⣿⣿⣿⢋⡼⠁
-    ⠀⠀⠀⠀⠀⠀⠀⠈⢻⣄⠸⢿⣿⣿⠿⠷⠀⠈⠀⣭⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣾⣿⣿⣿⣿⣿⣿⠇⡼⠁⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⣯⡀⠀⢼⡿⠀⠀⠀⢼⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣿⡿⣿⣿⣿⠿⣿⣯⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢋⡼⠁⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢻⡏⠠⣦⠁⠀⠀⠀⠀⠀⠟⠛⠛⣿⣿⣿⣿⣿⠿⠁⠀⠁⢿⠙⠁⠀⠛⠹⣿⣏⣾⣿⣿⣿⣿⣿⣿⣿⣿⠿⠃⣹⠁⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣘⣧⠀⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⡿⡿⠀⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀⢹⣿⠿⢿⣿⣿⣿⣿⣿⠋⢀⡤⠛⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⡯⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⢸⣿⣿⣿⠛⠉⠀⣰⠷⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠇⠀⠀⠀⠀⠀⢀⣿⡇⠀⠀⢻⣿⣿⠁⠀⠀⢠⣾⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠟⢿⣿⣄⡀⢸⣿⡀⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⠀⠀⠀⢰⣿⣿⡛⣿⣿⡄⢠⡺⠿⡍⠁⢀⣤⣿⣿⣿⠿⣷⣮⣉⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⠀⠀⠈⣧⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⠉⠃⠀⣴⣿⣟⠻⣿⣿⣿⡇⢸⣿⣶⠀⢀⣾⣿⣿⣟⠿⣷⣾⣿⣿⣿⣿⣦⣤⣤⡤⠀⠀⠀⠀⠀⠁⠀⠀⠀⣼⠗⠀⠀⠀⠀
-    ⠀⠀⠐⢄⡀⠀⠀⠀⢘⡀⠀⢶⣾⣿⣿⣿⣿⡿⠋⠁⠈⠻⠉⠀⠚⠻⣿⣿⣿⣶⣾⣿⣿⣿⣿⣿⣿⣷⣬⣤⣶⣦⡀⣾⣶⣇⠀⠀⠈⢉⣷⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠈⠓⠶⢦⡽⠄⣈⣿⣿⣿⣿⣿⠏⠀⠀⠀⠀⠀⠀⠀⠀⠀⠹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡓⠙⣿⡟⠀⠀⠀⠈⠛⣷⣶⡄⠀
-    ⠀⠀⠀⠀⠀⠀⠀⢀⣬⠆⢠⣍⣛⠻⣿⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣉⣀⡀⠀⠀⠈⠛⢿⣦⡀
-    ⠐⠒⠒⠶⠶⠶⢦⣬⣟⣥⣀⡉⠛⠻⠶⢁⣤⣾⣿⣿⣿⣷⡄⠀⠀⠀⠀⠀⢸⣿⣿⣿⣿⣿⣟⡛⠿⠭⠭⠭⠭⠭⠿⠿⠿⢿⣿⣟⠃⠀⠀⠀⠹⣟⠓
-    ⠀⣀⣠⠤⠤⢤⣤⣾⣤⡄⣉⣉⣙⣓⡂⣿⣿⣭⣹⣿⣿⣿⣿⡰⣂⣀⢀⠀⠻⣿⠛⠻⠟⠡⣶⣾⣿⣿⣿⣿⣿⣿⣿⡖⠒⠒⠒⠛⠷⢤⡀⢰⣴⣿⡆
-    ⠀⠀⠀⢀⣠⡴⠾⠟⠻⣟⡉⠉⠉⠉⢁⢿⣿⣿⣿⣿⣿⣿⡿⣱⣿⣭⡌⠤⠀⠀⠐⣶⣌⡻⣶⣭⡻⢿⣿⣿⣿⣿⣿⣯⣥⣤⣦⠀⠠⣴⣶⣶⣿⡟⢿
-    ⢀⠔⠊⠉⠀⠀⠀⠀⢸⣯⣤⠀⠀⠠⣼⣮⣟⣿⣿⣿⣻⣭⣾⣿⣿⣷⣶⣦⠶⣚⣾⣿⣿⣷⣜⣿⣿⣶⣝⢿⣿⣿⣿⣿⣷⣦⣄⣰⡄⠈⢿⣿⡿⣇⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠈⢡⢇⠀⠀⣠⣿⣿⣿⣯⣟⣛⣛⣛⣛⣛⣩⣭⣴⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣻⣿⣧⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠏⠀⢹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣍⣿⣿⣿⣿⡄⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣾⡁⢈⣾⣿⡿⠛⣛⣿⣿⣿⣿ DO YOU BRING FEEDS? ⣿⣿⣿⣿⣿⣿⡏⠈⠙⠈⠁⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⡿⠛⠉⣽⣿⣷⣾⡿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠷⠌⠛⠉⠀⠁⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠹⠋⠀⢻⣿⣿⣿⣿⠿⢿⣿⣿⣿⣿⣿⣿⠿⣿⣿⣿⣿⠿⠛⠋⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
-    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠁⠀⠀⠀⠀⠀⠈⠉⠉⠀⠀⠈⠋⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 
-  '';
-}

m/tent/blackbox.yml

@@ -1,14 +0,0 @@
-modules:
-  http_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      preferred_ip_protocol: "ip4"
-      follow_redirects: true
-      valid_status_codes: []  # Defaults to 2xx
-      method: GET
-  icmp:
-    prober: icmp
-    timeout: 5s
-    icmp:
-      preferred_ip_protocol: "ip4"

m/tent/configuration.nix

@@ -1,80 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-{
-  imports = [
-    ../common/xeon.nix
-    ../module/emulation.nix
-    ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
-    ./monitoring.nix
-    ./nginx.nix
-    ./nix-serve.nix
-    ./gitlab-runner.nix
-    ./gitea.nix
-    ../hut/public-inbox.nix
-    ../hut/msmtp.nix
-    ../module/p.nix
-    ../module/vpn-dac.nix
-  ];
-
-  # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d537675";
-
-  networking = {
-    hostName = "tent";
-    interfaces.eno1.ipv4.addresses = [
-      {
-        address = "10.0.44.4";
-        prefixLength = 24;
-      }
-    ];
-
-    # Only BSC DNSs seem to be reachable from the office VLAN
-    nameservers = [ "84.88.52.35" "84.88.52.36" ];
-    search = [ "bsc.es" "ac.upc.edu" ];
-    defaultGateway = "10.0.44.1";
-  };
-
-  services.p.enable = true;
-
-  services.prometheus.exporters.node = {
-    enable = true;
-    enabledCollectors = [ "systemd" ];
-    port = 9002;
-    listenAddress = "127.0.0.1";
-  };
-
-  boot.swraid = {
-    enable = true;
-    mdadmConf = ''
-      DEVICE partitions
-      ARRAY /dev/md0 metadata=1.2 UUID=496db1e2:056a92aa:a544543f:40db379d
-      MAILADDR root
-    '';
-  };
-
-  fileSystems."/vault" = {
-    device = "/dev/disk/by-label/vault";
-    fsType = "ext4";
-  };
-
-  # Make a /vault/$USER directory for each user.
-  systemd.services.create-vault-dirs = let
-    # Take only normal users in tent
-    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-    commands = lib.concatLists (lib.mapAttrsToList
-      (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0711 /vault/home/${user.name}"
-      ]) users);
-    script = pkgs.writeShellScript "create-vault-dirs.sh" (lib.concatLines commands);
-  in {
-    enable = true;
-    wants = [ "local-fs.target" ];
-    after = [ "local-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig.ExecStart = script;
-  };
-
-  # disable automatic garbage collector
-  nix.gc.automatic = lib.mkForce false;
-}

m/tent/gitea.nix

@@ -1,30 +0,0 @@
-{ config, lib, ... }:
-{
-  services.gitea = {
-    enable = true;
-    appName = "Gitea in the jungle";
-
-    settings = {
-      server = {
-        ROOT_URL = "https://jungle.bsc.es/git/";
-        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
-        LANDING_PAGE = "explore";
-      };
-      metrics.ENABLED = true;
-      service = {
-        DISABLE_REGISTRATION = true;
-        REGISTER_MANUAL_CONFIRM = true;
-        ENABLE_NOTIFY_MAIL = true;
-      };
-      log.LEVEL = "Warn";
-
-      mailer = {
-        ENABLED       = true;
-        FROM          = "jungle-robot@bsc.es";
-        PROTOCOL      = "sendmail";
-        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
-        SENDMAIL_ARGS = "--";
-      };
-    };
-  };
-}

m/tent/gitlab-runner.nix

@@ -1,93 +0,0 @@
-{ pkgs, lib, config, ... }:
-
-{
-  age.secrets.tent-gitlab-runner-pm-shell.file = ../../secrets/tent-gitlab-runner-pm-shell-token.age;
-  age.secrets.tent-gitlab-runner-pm-docker.file = ../../secrets/tent-gitlab-runner-pm-docker-token.age;
-  age.secrets.tent-gitlab-runner-bsc-docker.file = ../../secrets/tent-gitlab-runner-bsc-docker-token.age;
-
-  services.gitlab-runner = let sec = config.age.secrets; in {
-    enable = true;
-    settings.concurrent = 5;
-    services = {
-      # For gitlab.pm.bsc.es
-      gitlab-pm-shell = {
-        executor = "shell";
-        environmentVariables = {
-          SHELL = "${pkgs.bash}/bin/bash";
-        };
-        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-shell.path;
-        preGetSourcesScript = pkgs.writeScript "setup" ''
-          echo "This is the preGetSources script running, brace for impact"
-          env
-        '';
-      };
-      gitlab-pm-docker = {
-        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-docker.path;
-        executor = "docker";
-        dockerImage = "debian:stable";
-      };
-
-      # For gitlab.bsc.es
-      gitlab-bsc-docker = {
-        # gitlab.bsc.es still uses the old token mechanism
-        registrationConfigFile = sec.tent-gitlab-runner-bsc-docker.path;
-        tagList = [ "docker" "tent" "nix" ];
-        executor = "docker";
-        dockerImage = "alpine";
-        dockerVolumes = [
-          "/nix/store:/nix/store:ro"
-          "/nix/var/nix/db:/nix/var/nix/db:ro"
-          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
-        ];
-        dockerDisableCache = true;
-        registrationFlags = [
-          # Increase build log length to 64 MiB
-          "--output-limit 65536"
-        ];
-        preBuildScript = pkgs.writeScript "setup-container" ''
-          mkdir -p -m 0755 /nix/var/log/nix/drvs
-          mkdir -p -m 0755 /nix/var/nix/gcroots
-          mkdir -p -m 0755 /nix/var/nix/profiles
-          mkdir -p -m 0755 /nix/var/nix/temproots
-          mkdir -p -m 0755 /nix/var/nix/userpool
-          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
-          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
-          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
-          mkdir -p -m 0700 "$HOME/.nix-defexpr"
-          mkdir -p -m 0700 "$HOME/.ssh"
-          cat >> "$HOME/.ssh/known_hosts" << EOF
-          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
-          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
-          EOF
-          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
-          # Required to load SSL certificate paths
-          . ${pkgs.cacert}/nix-support/setup-hook
-        '';
-        environmentVariables = {
-          ENV = "/etc/profile";
-          USER = "root";
-          NIX_REMOTE = "daemon";
-          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
-        };
-      };
-    };
-  };
-
-  systemd.services.gitlab-runner.serviceConfig = {
-    DynamicUser = lib.mkForce false;
-    User = "gitlab-runner";
-    Group = "gitlab-runner";
-    ExecStart = lib.mkForce
-      ''${pkgs.gitlab-runner}/bin/gitlab-runner run --config ''${HOME}/.gitlab-runner/config.toml --listen-address "127.0.0.1:9252" --working-directory ''${HOME}'';
-  };
-
-  users.users.gitlab-runner = {
-    uid = config.ids.uids.gitlab-runner;
-    home = "/var/lib/gitlab-runner";
-    description = "Gitlab Runner";
-    group = "gitlab-runner";
-    extraGroups = [ "docker" ];
-    createHome = true;
-  };
-  users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;
-}

m/tent/monitoring.nix

@@ -1,217 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-{
-  imports = [
-    ../module/meteocat-exporter.nix
-    ../module/upc-qaire-exporter.nix
-    ../module/nix-daemon-exporter.nix
-  ];
-
-  age.secrets.grafanaJungleRobotPassword = {
-    file = ../../secrets/jungle-robot-password.age;
-    owner = "grafana";
-    mode = "400";
-  };
-
-  services.grafana = {
-    enable = true;
-    settings = {
-      server = {
-        domain = "jungle.bsc.es";
-        root_url = "%(protocol)s://%(domain)s/grafana";
-        serve_from_sub_path = true;
-        http_port = 2342;
-        http_addr = "127.0.0.1";
-      };
-      smtp = {
-        enabled = true;
-        from_address = "jungle-robot@bsc.es";
-        user = "jungle-robot";
-        # Read the password from a file, which is only readable by grafana user
-        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
-        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
-        host = "mail.bsc.es:465";
-        startTLS_policy = "NoStartTLS";
-      };
-      feature_toggles.publicDashboards = true;
-      "auth.anonymous".enabled = true;
-      log.level = "warn";
-    };
-  };
-
-  services.prometheus = {
-    enable = true;
-    port = 9001;
-    retentionTime = "5y";
-    listenAddress = "127.0.0.1";
-  };
-
-  # We need access to the devices to monitor the disk space
-  systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
-  systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
-
-  # Credentials for IPMI exporter
-  age.secrets.ipmiYml = {
-    file = ../../secrets/ipmi.yml.age;
-    owner = "ipmi-exporter";
-  };
-
-  # Create an IPMI group and assign the ipmi0 device
-  users.groups.ipmi = {};
-  services.udev.extraRules = ''
-    SUBSYSTEM=="ipmi", KERNEL=="ipmi0", GROUP="ipmi", MODE="0660"
-  '';
-
-  # Add a new ipmi-exporter user that can read the ipmi0 device
-  users.users.ipmi-exporter = {
-    isSystemUser = true;
-    group = "ipmi";
-  };
-
-  # Disable dynamic user so we have the ipmi-exporter user available for the credentials
-  systemd.services.prometheus-ipmi-exporter.serviceConfig = {
-    DynamicUser = lib.mkForce false;
-    PrivateDevices = lib.mkForce false;
-    User = lib.mkForce "ipmi-exporter";
-    Group = lib.mkForce "ipmi";
-    RestrictNamespaces = lib.mkForce false;
-    # Fake uid to 0 so it shuts up
-    ExecStart = let
-      cfg = config.services.prometheus.exporters.ipmi;
-    in lib.mkForce (lib.concatStringsSep " " ([
-      "${pkgs.util-linux}/bin/unshare --map-user 0"
-      "${pkgs.prometheus-ipmi-exporter}/bin/ipmi_exporter"
-      "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
-      "--config.file ${lib.escapeShellArg cfg.configFile}"
-    ] ++ cfg.extraFlags));
-  };
-
-  services.prometheus = {
-    exporters = {
-      ipmi = {
-        enable = true;
-        configFile = config.age.secrets.ipmiYml.path;
-        #extraFlags = [ "--log.level=debug" ];
-        listenAddress = "127.0.0.1";
-      };
-      node = {
-        enable = true;
-        enabledCollectors = [ "logind" ];
-        port = 9002;
-        listenAddress = "127.0.0.1";
-      };
-      blackbox = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        configFile = ./blackbox.yml;
-      };
-    };
-
-    scrapeConfigs = [
-      {
-        job_name = "local";
-        static_configs = [{
-          targets = [
-            "127.0.0.1:9002" # Node exporter
-            #"127.0.0.1:9115" # Blackbox exporter
-            "127.0.0.1:9290" # IPMI exporter for local node
-            "127.0.0.1:9928" # UPC Qaire custom exporter
-            "127.0.0.1:9929" # Meteocat custom exporter
-            "127.0.0.1:9999" # Nix-daemon custom exporter
-          ];
-        }];
-      }
-      {
-        job_name = "blackbox-http";
-        metrics_path = "/probe";
-        params = { module = [ "http_2xx" ]; };
-        static_configs = [{
-          targets = [
-            "https://www.google.com/robots.txt"
-            "https://pm.bsc.es/"
-            "https://pm.bsc.es/gitlab/"
-            "https://jungle.bsc.es/"
-            "https://gitlab.bsc.es/"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:9115";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox-icmp";
-        metrics_path = "/probe";
-        params = { module = [ "icmp" ]; };
-        static_configs = [{
-          targets = [
-            "1.1.1.1"
-            "8.8.8.8"
-            "ssfhead"
-            "raccoon"
-            "anella-bsc.cesca.cat"
-            "upc-anella.cesca.cat"
-            "fox.ac.upc.edu"
-            "fox-ipmi.ac.upc.edu"
-            "arenys5.ac.upc.edu"
-            "arenys0-2.ac.upc.edu"
-            "epi01.bsc.es"
-            "axle.bsc.es"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:9115";
-          }
-        ];
-      }
-      {
-        job_name = "ipmi-raccoon";
-        metrics_path = "/ipmi";
-        static_configs = [
-          { targets = [ "127.0.0.1:9290" ]; }
-        ];
-        params = {
-          target = [ "raccoon-ipmi" ];
-          module = [ "raccoon" ];
-        };
-      }
-      {
-        job_name = "ipmi-fox";
-        metrics_path = "/ipmi";
-        static_configs = [
-          { targets = [ "127.0.0.1:9290" ]; }
-        ];
-        params = {
-          target = [ "fox-ipmi.ac.upc.edu" ];
-          module = [ "fox" ];
-        };
-      }
-    ];
-  };
-}

m/tent/nginx.nix

@@ -1,73 +0,0 @@
-{ theFlake, pkgs, ... }:
-let
-  website = pkgs.stdenv.mkDerivation {
-    name = "jungle-web";
-    src = theFlake;
-    buildInputs = [ pkgs.hugo ];
-    buildPhase = ''
-      cd web
-      rm -rf public/
-      hugo
-    '';
-    installPhase = ''
-      cp -r public $out
-    '';
-    # Don't mess doc/
-    dontFixup = true;
-  };
-in
-{
-  networking.firewall.allowedTCPPorts = [ 80 ];
-  services.nginx = {
-    enable = true;
-    virtualHosts."jungle.bsc.es" = {
-      root = "${website}";
-      listen = [
-        {
-          addr = "0.0.0.0";
-          port = 80;
-        }
-      ];
-      extraConfig = ''
-        set_real_ip_from 127.0.0.1;
-        set_real_ip_from 84.88.52.107;
-        real_ip_recursive on;
-        real_ip_header X-Forwarded-For;
-
-        location /git {
-          rewrite ^/git$ / break;
-          rewrite ^/git/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:3000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /cache {
-          rewrite ^/cache/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:5000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /lists {
-          proxy_pass http://127.0.0.1:8081;
-          proxy_redirect http:// $scheme://;
-        }
-        location /grafana {
-          proxy_pass http://127.0.0.1:2342;
-          proxy_redirect http:// $scheme://;
-          proxy_set_header Host $host;
-          # Websockets
-          proxy_http_version 1.1;
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection "upgrade";
-        }
-        location ~ ^/~(.+?)(/.*)?$ {
-          alias /vault/home/$1/public_html$2;
-          index  index.html index.htm;
-          autoindex on;
-          absolute_redirect off;
-        }
-        location /p/ {
-          alias /var/lib/p/;
-        }
-      '';
-    };
-  };
-}

m/tent/nix-serve.nix

@@ -1,16 +0,0 @@
-{ config, ... }:
-
-{
-  age.secrets.nixServe.file = ../../secrets/nix-serve.age;
-
-  services.nix-serve = {
-    enable = true;
-    # Only listen locally, as we serve it via ssh
-    bindAddress = "127.0.0.1";
-    port = 5000;
-
-    secretKeyFile = config.age.secrets.nixServe.path;
-    # Public key:
-    # jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=
-  };
-}

pkgs/ceph.nix

@@ -0,0 +1,405 @@
+{ lib
+, stdenv
+, runCommand
+, fetchurl
+, fetchFromGitHub
+, fetchPypi
+
+# Build time
+, cmake
+, ensureNewerSourcesHook
+, fmt
+, git
+, makeWrapper
+, nasm
+, pkg-config
+, which
+
+# Tests
+, nixosTests
+
+# Runtime dependencies
+, arrow-cpp
+, babeltrace
+, boost179
+, bzip2
+, cryptsetup
+, cunit
+, doxygen
+, gperf
+, graphviz
+, gtest
+, icu
+, libcap
+, libcap_ng
+, libnl
+, libxml2
+, lttng-ust
+, lua
+, lz4
+, oath-toolkit
+, openldap
+, python310
+, rdkafka
+, rocksdb
+, snappy
+, sqlite
+, utf8proc
+, zlib
+, zstd
+
+# Optional Dependencies
+, curl ? null
+, expat ? null
+, fuse ? null
+, libatomic_ops ? null
+, libedit ? null
+, libs3 ? null
+, yasm ? null
+
+# Mallocs
+, gperftools ? null
+, jemalloc ? null
+
+# Crypto Dependencies
+, cryptopp ? null
+, nspr ? null
+, nss ? null
+
+# Linux Only Dependencies
+, linuxHeaders
+, util-linux
+, libuuid
+, udev
+, keyutils
+, rdma-core
+, rabbitmq-c
+, libaio ? null
+, libxfs ? null
+, liburing ? null
+, zfs ? null
+, ...
+}:
+
+# We must have one crypto library
+assert cryptopp != null || (nss != null && nspr != null);
+
+let
+  shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
+
+  optYasm = shouldUsePkg yasm;
+  optExpat = shouldUsePkg expat;
+  optCurl = shouldUsePkg curl;
+  optFuse = shouldUsePkg fuse;
+  optLibedit = shouldUsePkg libedit;
+  optLibatomic_ops = shouldUsePkg libatomic_ops;
+  optLibs3 = shouldUsePkg libs3;
+
+  optJemalloc = shouldUsePkg jemalloc;
+  optGperftools = shouldUsePkg gperftools;
+
+  optCryptopp = shouldUsePkg cryptopp;
+  optNss = shouldUsePkg nss;
+  optNspr = shouldUsePkg nspr;
+
+  optLibaio = shouldUsePkg libaio;
+  optLibxfs = shouldUsePkg libxfs;
+  optZfs = shouldUsePkg zfs;
+
+  # Downgrade rocksdb, 7.10 breaks ceph
+  rocksdb' = rocksdb.overrideAttrs {
+    version = "7.9.2";
+    src = fetchFromGitHub {
+      owner = "facebook";
+      repo = "rocksdb";
+      rev = "refs/tags/v7.9.2";
+      hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
+    };
+  };
+
+  hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
+
+  # Malloc implementation (can be jemalloc, tcmalloc or null)
+  malloc = if optJemalloc != null then optJemalloc else optGperftools;
+
+  # We prefer nss over cryptopp
+  cryptoStr = if optNss != null && optNspr != null then "nss" else
+    if optCryptopp != null then "cryptopp" else "none";
+
+  cryptoLibsMap = {
+    nss = [ optNss optNspr ];
+    cryptopp = [ optCryptopp ];
+    none = [ ];
+  };
+
+  getMeta = description: with lib; {
+     homepage = "https://ceph.io/en/";
+     inherit description;
+     license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+     maintainers = with maintainers; [ adev ak johanot krav ];
+     platforms = [ "x86_64-linux" "aarch64-linux" ];
+   };
+
+  ceph-common = with python.pkgs; buildPythonPackage {
+    pname = "ceph-common";
+    inherit src version;
+
+    sourceRoot = "ceph-${version}/src/python-common";
+
+    propagatedBuildInputs = [
+      pyyaml
+    ];
+
+    nativeCheckInputs = [
+      pytestCheckHook
+    ];
+
+    disabledTests = [
+      # requires network access
+      "test_valid_addr"
+    ];
+
+    meta = getMeta "Ceph common module for code shared by manager modules";
+  };
+
+  # Watch out for python <> boost compatibility
+  python = python310.override {
+    packageOverrides = self: super: {
+      sqlalchemy = super.sqlalchemy.overridePythonAttrs rec {
+        version = "1.4.46";
+        src = fetchPypi {
+          pname = "SQLAlchemy";
+          inherit version;
+          hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA=";
+        };
+        disabledTestPaths = [
+          "test/aaa_profiling"
+          "test/ext/mypy"
+        ];
+      };
+    };
+  };
+
+  boost = boost179.override {
+    enablePython = true;
+    inherit python;
+  };
+
+  # TODO: split this off in build and runtime environment
+  ceph-python-env = python.withPackages (ps: with ps; [
+    ceph-common
+
+    # build time
+    cython
+
+    # debian/control
+    bcrypt
+    cherrypy
+    influxdb
+    jinja2
+    kubernetes
+    natsort
+    numpy
+    pecan
+    prettytable
+    pyjwt
+    pyopenssl
+    python-dateutil
+    pyyaml
+    requests
+    routes
+    scikit-learn
+    scipy
+    setuptools
+    sphinx
+    virtualenv
+    werkzeug
+
+    # src/pybind/mgr/requirements-required.txt
+    cryptography
+    jsonpatch
+
+    # src/tools/cephfs/shell/setup.py
+    cmd2
+    colorama
+  ]);
+  inherit (ceph-python-env.python) sitePackages;
+
+  version = "18.2.0";
+  src = fetchurl {
+    url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
+    hash = "sha256:0k9nl6xi5brva51rr14m7ig27mmmd7vrpchcmqc40q3c2khn6ns9";
+  };
+in rec {
+  ceph = stdenv.mkDerivation {
+    pname = "ceph";
+    inherit src version;
+
+    nativeBuildInputs = [
+      cmake
+      fmt
+      git
+      makeWrapper
+      nasm
+      pkg-config
+      python
+      python.pkgs.python # for the toPythonPath function
+      python.pkgs.wrapPython
+      which
+      (ensureNewerSourcesHook { year = "1980"; })
+      # for building docs/man-pages presumably
+      doxygen
+      graphviz
+    ];
+
+    enableParallelBuilding = true;
+
+    buildInputs = cryptoLibsMap.${cryptoStr} ++ [
+      arrow-cpp
+      babeltrace
+      boost
+      bzip2
+      ceph-python-env
+      cryptsetup
+      cunit
+      gperf
+      gtest
+      icu
+      libcap
+      libnl
+      libxml2
+      lttng-ust
+      lua
+      lz4
+      malloc
+      oath-toolkit
+      openldap
+      optLibatomic_ops
+      optLibs3
+      optYasm
+      rdkafka
+      rocksdb'
+      snappy
+      sqlite
+      utf8proc
+      zlib
+      zstd
+    ] ++ lib.optionals stdenv.isLinux [
+      keyutils
+      libcap_ng
+      liburing
+      libuuid
+      linuxHeaders
+      optLibaio
+      optLibxfs
+      optZfs
+      rabbitmq-c
+      rdma-core
+      udev
+      util-linux
+    ] ++ lib.optionals hasRadosgw [
+      optCurl
+      optExpat
+      optFuse
+      optLibedit
+    ];
+
+    pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
+
+    preConfigure =''
+      substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
+      substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
+      substituteInPlace src/common/module.c --replace "/bin/grep" "grep"
+
+      # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
+      # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others
+      export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages}
+      patchShebangs src/
+    '';
+
+    cmakeFlags = [
+      "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
+
+      "-DWITH_CEPHFS_SHELL:BOOL=ON"
+      "-DWITH_SYSTEMD:BOOL=OFF"
+      # `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
+      #     CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
+      #     Command failed: 2
+      #
+      #        'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
+      #
+      #     See also
+      #
+      #        /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
+      # and that file contains:
+      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
+      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
+      #       71 |     ::THRIFT_CLOSESOCKET(socket_);
+      #          |       ^~~~~~~~~~~~~~~~~~
+      # Looks like `close()` is somehow not included.
+      # But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
+      # So it's proably not worth trying to fix that for this Ceph version,
+      # and instead just disable Ceph's Jaeger support.
+      "-DWITH_JAEGER:BOOL=OFF"
+      "-DWITH_TESTS:BOOL=OFF"
+
+      # Use our own libraries, where possible
+      "-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
+      "-DWITH_SYSTEM_BOOST:BOOL=ON"
+      "-DWITH_SYSTEM_GTEST:BOOL=ON"
+      "-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
+      "-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
+      "-DWITH_SYSTEM_ZSTD:BOOL=ON"
+
+      # TODO breaks with sandbox, tries to download stuff with npm
+      "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
+      # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
+      ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
+    ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON";
+
+    postFixup = ''
+      wrapPythonPrograms
+      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
+
+      # Test that ceph-volume exists since the build system has a tendency to
+      # silently drop it with misconfigurations.
+      test -f $out/bin/ceph-volume
+    '';
+
+    outputs = [ "out" "lib" "dev" "doc" "man" ];
+
+    doCheck = false; # uses pip to install things from the internet
+
+    # Takes 7+h to build with 2 cores.
+    requiredSystemFeatures = [ "big-parallel" ];
+
+    meta = getMeta "Distributed storage system";
+
+    passthru = {
+      inherit version;
+      tests = {
+        inherit (nixosTests)
+          ceph-multi-node
+          ceph-single-node
+          ceph-single-node-bluestore;
+      };
+    };
+  };
+
+  ceph-client = runCommand "ceph-client-${version}" {
+      meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
+    } ''
+      mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
+      cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
+      cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
+      cp -r ${ceph}/bin/rbd-replay* $out/bin
+      cp -r ${ceph}/sbin/mount.ceph $out/bin
+      cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
+      ln -s bin $out/sbin
+      cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
+      cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
+      # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
+      substituteInPlace $out/bin/ceph          --replace ${ceph} $out
+      substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
+   '';
+}

pkgs/meteocat-exporter/default.nix

@@ -1,25 +0,0 @@
-{ python3Packages, lib }:
-
-python3Packages.buildPythonApplication rec {
-  pname = "meteocat-exporter";
-  version = "1.0";
-
-  src = ./.;
-
-  doCheck = false;
-
-  build-system = with python3Packages; [
-    setuptools
-  ];
-
-  dependencies = with python3Packages; [
-    beautifulsoup4
-    lxml
-    prometheus-client
-  ];
-
-  meta = with lib; {
-    description = "MeteoCat Prometheus Exporter";
-    platforms = platforms.linux;
-  };
-}

pkgs/meteocat-exporter/meteocat-exporter

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-
-import time
-from prometheus_client import start_http_server, Gauge
-from bs4 import BeautifulSoup
-from urllib import request
-
-# Configuration -------------------------------------------
-meteo_station = "X8" # Barcelona - Zona Universitària
-listening_port = 9929
-update_period = 60 * 5 # Each 5 min
-# ---------------------------------------------------------
-
-metric_tmin = Gauge('meteocat_temp_min', 'Min temperature')
-metric_tmax = Gauge('meteocat_temp_max', 'Max temperature')
-metric_tavg = Gauge('meteocat_temp_avg', 'Average temperature')
-metric_srad = Gauge('meteocat_solar_radiation', 'Solar radiation')
-
-def update(st):
-    url = 'https://www.meteo.cat/observacions/xema/dades?codi=' + st
-    response = request.urlopen(url)
-    data = response.read()
-    soup = BeautifulSoup(data, 'lxml')
-    table = soup.find("table", {"class" : "tblperiode"})
-    rows = table.find_all('tr')
-    row = rows[-1] # Take the last row
-    row_data = []
-    header = row.find('th')
-    header_text = header.text.strip()
-    row_data.append(header_text)
-    for col in row.find_all('td'):
-        row_data.append(col.text)
-    try:
-        # Sometimes it will return '(s/d)' and fail to parse
-        metric_tavg.set(float(row_data[1]))
-        metric_tmax.set(float(row_data[2]))
-        metric_tmin.set(float(row_data[3]))
-        metric_srad.set(float(row_data[10]))
-        #print("ok: temp_avg={}".format(float(row_data[1])))
-    except:
-        print("cannot parse row: {}".format(row))
-        metric_tavg.set(float("nan"))
-        metric_tmax.set(float("nan"))
-        metric_tmin.set(float("nan"))
-        metric_srad.set(float("nan"))
-
-if __name__ == '__main__':
-    start_http_server(port=listening_port, addr="localhost")
-    while True:
-        try:
-            update(meteo_station)
-        except:
-            print("update failed")
-        time.sleep(update_period)

pkgs/meteocat-exporter/setup.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-
-from setuptools import setup, find_packages
-
-setup(name='meteocat-exporter',
-      version='1.0',
-      # Modules to import from other scripts:
-      packages=find_packages(),
-      # Executables
-      scripts=["meteocat-exporter"],
-     )

pkgs/mpich-fix-hwtopo.patch

@@ -1,36 +0,0 @@
-diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
-index 33e88bc..ee3641c 100644
---- a/src/util/mpir_hwtopo.c
-+++ b/src/util/mpir_hwtopo.c
-@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
- #ifdef HAVE_HWLOC
-     bindset = hwloc_bitmap_alloc();
-     hwloc_topology_init(&hwloc_topology);
--    char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
--    if (xmlfile != NULL) {
--        int rc;
--        rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
--        if (rc == 0) {
--            /* To have hwloc still actually call OS-specific hooks, the
--             * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
--             * file is really the underlying system. */
--            hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
--        }
--        MPL_free(xmlfile);
--    }
-
-     hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
-     if (!hwloc_topology_load(hwloc_topology)) 
-
---- a/src/mpi/init/local_proc_attrs.c
-+++ b/src/mpi/init/local_proc_attrs.c
-@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
-     /* Set the number of tag bits. The device may override this value. */
-     MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
-
--    char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
--    MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
--    MPL_free(requested_kinds);
--
-     return mpi_errno;
- }

pkgs/nix-socket.patch

@@ -0,0 +1,11 @@
+--- a/misc/systemd/nix-daemon.socket.in	1970-01-01 01:00:01.000000000 +0100
++++ b/misc/systemd/nix-daemon.socket.in	2023-09-18 17:53:32.351760208 +0200
+@@ -1,7 +1,7 @@
+ [Unit]
+ Description=Nix Daemon Socket
+ Before=multi-user.target
+-RequiresMountsFor=@storedir@
++RequiresMountsFor=@localstatedir@
+ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
+ 
+ [Socket]

pkgs/overlay.nix

@@ -1,23 +1,14 @@
 final: prev:
 {
+  bsc = prev.bsc.extend (bscFinal: bscPrev: {
     # Set MPICH as default
-  mpi = final.mpich;
+    mpi = bscFinal.mpich;
 
     # Configure the network for MPICH
-  mpich = with final; let
-    # pmix comes with the libraries in .out and headers in .dev
-    pmixAll = symlinkJoin {
-      name = "pmix-all";
-      paths = [ pmix.dev pmix.out ];
-    };
-  in prev.mpich.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [
-      # See https://github.com/pmodels/mpich/issues/6946
-      ./mpich-fix-hwtopo.patch
-    ];
+    mpich = with final; prev.mpich.overrideAttrs (old: {
       buildInput = old.buildInputs ++ [
         libfabric
-      pmixAll
+        pmix
       ];
       configureFlags = [
         "--enable-shared"
@@ -25,35 +16,24 @@ final: prev:
         "--with-pm=no"
         "--with-device=ch4:ofi"
         "--with-pmi=pmix"
-      "--with-pmix=${pmixAll}"
-      "--with-libfabric=${libfabric}"
+        "--with-pmix=${final.pmix}"
+        "--with-libfabric=${final.libfabric}"
         "--enable-g=log"
       ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
         "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
         "FCFLAGS=-fallow-argument-mismatch"
       ];
     });
-
-  slurm = prev.slurm.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [
-      # See https://bugs.schedmd.com/show_bug.cgi?id=19324
-      ./slurm-rank-expansion.patch
-    ];
-    # Install also the pam_slurm_adopt library to restrict users from accessing
-    # nodes with no job allocated.
-    postBuild = (old.postBuild or "") + ''
-      pushd contribs/pam_slurm_adopt
-        make "PAM_DIR=$out/lib/security"
-      popd
-    '';
-    postInstall = (old.postInstall or "") + ''
-      pushd contribs/pam_slurm_adopt
-        make "PAM_DIR=$out/lib/security" install
-      popd
-    '';
   });
 
-  prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
-  meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { };
-  upc-qaire-exporter = prev.callPackage ./upc-qaire-exporter/default.nix { };
+  # Update ceph to 18.2.0 until it lands in nixpkgs, see:
+  # https://github.com/NixOS/nixpkgs/pull/247849
+  inherit (prev.callPackage ./ceph.nix {
+    lua = prev.lua5_4;
+    fmt = prev.fmt_8;
+  }) ceph ceph-client;
+
+  nix = prev.nix.overrideAttrs (old: {
+    patches = old.patches ++ [ ./nix-socket.patch ];
+  });
 }

pkgs/slurm-exporter.nix

@@ -1,22 +0,0 @@
-{ buildGoModule, fetchFromGitHub, lib }:
-
-buildGoModule rec {
-  pname = "prometheus-slurm-exporter";
-  version = "0.20";
-
-  src = fetchFromGitHub {
-    rev = version;
-    owner = "vpenso";
-    repo = pname;
-    sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
-  };
-
-  vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
-  doCheck = false;
-
-  meta = with lib; {
-    description = "Prometheus SLURM Exporter";
-    homepage = "https://github.com/vpenso/prometheus-slurm-exporter";
-    platforms = platforms.linux;
-  };
-}

pkgs/slurm-rank-expansion.patch

@@ -1,11 +0,0 @@
---- a/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:05:24.815313882 +0100
-+++ b/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:09:53.936900823 +0100
-@@ -314,7 +314,7 @@ static void _dmdx_req(buf_t *buf, int no
- 	}
- 
- 	nsptr = pmixp_nspaces_local();
--	if (nsptr->ntasks <= rank) {
-+	if ((long) nsptr->ntasks <= (long) rank) {
- 		char *nodename = pmixp_info_job_host(nodeid);
- 		PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
- 			    nodename, ns, nsptr->ntasks, rank);

pkgs/upc-qaire-exporter/default.nix

@@ -1,24 +0,0 @@
-{ python3Packages, lib }:
-
-python3Packages.buildPythonApplication rec {
-  pname = "upc-qaire-exporter";
-  version = "1.0";
-
-  src = ./.;
-
-  doCheck = false;
-
-  build-system = with python3Packages; [
-    setuptools
-  ];
-
-  dependencies = with python3Packages; [
-    prometheus-client
-    requests
-  ];
-
-  meta = with lib; {
-    description = "UPC Qaire Prometheus Exporter";
-    platforms = platforms.linux;
-  };
-}

pkgs/upc-qaire-exporter/setup.py

@@ -1,11 +0,0 @@
-#!/usr/bin/env python
-
-from setuptools import setup, find_packages
-
-setup(name='upc-qaire-exporter',
-      version='1.0',
-      # Modules to import from other scripts:
-      packages=find_packages(),
-      # Executables
-      scripts=["upc-qaire-exporter"],
-     )

pkgs/upc-qaire-exporter/upc-qaire-exporter

@@ -1,74 +0,0 @@
-#!/usr/bin/env python3
-
-import time
-from prometheus_client import start_http_server, Gauge
-import requests, json
-from datetime import datetime, timedelta
-
-# Configuration -------------------------------------------
-listening_port = 9928
-update_period = 60 * 5 # Each 5 min
-# ---------------------------------------------------------
-
-metric_temp = Gauge('upc_c6_s302_temp', 'UPC C6 S302 temperature sensor')
-
-def genparams():
-    d = {}
-    d['topic'] = 'TEMPERATURE'
-    d['shift_dates_to'] = ''
-    d['datapoints'] = 301
-    d['devicesAndColors'] = '1148418@@@#40ACB6'
-
-    now = datetime.now()
-
-    d['fromDate'] = now.strftime('%d/%m/%Y')
-    d['toDate'] = now.strftime('%d/%m/%Y')
-    d['serviceFrequency'] = 'NONE'
-
-    # WTF!
-    for i in range(7):
-        for j in range(48):
-            key = 'week.days[{}].hours[{}].value'.format(i, j)
-            d[key] = 'OPEN'
-
-    return d
-
-def measure():
-    # First we need to load session
-    s = requests.Session()
-    r = s.get("https://upc.edu/sirena")
-    if r.status_code != 200:
-        print("bad HTTP status code on new session: {}".format(r.status_code))
-        return
-
-    if s.cookies.get("JSESSIONID") is None:
-        print("cannot get JSESSIONID")
-        return
-
-    # Now we can pull the data
-    url = "https://upcsirena.app.dexma.com/l_12535/analysis/by_datapoints/data.json"
-    r = s.post(url, data=genparams())
-
-    if r.status_code != 200:
-        print("bad HTTP status code on data: {}".format(r.status_code))
-        return
-
-    #print(r.text)
-    j = json.loads(r.content)
-
-    # Just take the last one
-    last = j['data']['chartElementList'][-1]
-    temp = last['values']['1148418-Temperatura']
-
-    return temp
-
-if __name__ == '__main__':
-    start_http_server(port=listening_port, addr="localhost")
-    while True:
-        try:
-            metric_temp.set(measure())
-        except:
-            print("measure failed")
-            metric_temp.set(float("nan"))
-
-        time.sleep(update_period)

secrets/ceph-user.age

@@ -1,19 +1,21 @@
 age-encryption.org/v1
--> ssh-ed25519 AY8zKw xeyzSqfio6SMS9SqywR+7II80D12Oha9T5zOgAIABSQ
-ST26VaF2G1xv9l7d3jWKG32ssOivfwx+p9jLLV7ZFnU
--> ssh-ed25519 sgAamA HrRx+x7NjXKVDaealWFo+Q8zMAdzoj6nTBxw0KMi3jE
-nlcEVTDTe1mPeS16/t9GYRnSSkm5EjpeiBZPIC/2f8U
--> ssh-ed25519 HY2yRg NDp5vUeX35rDV78DFQi9fsc71pQNVE8YQ1StCp+YjTg
-MdUAWHd1k6Jed2pp7Wct/DgF6ShqXFwNxPaXeBOLAcs
--> ssh-ed25519 tcumPQ d0zVVB8t7W9KUapOsnsrvpAj7LgM9zS0yCv8SQnF0g8
-aAPaWRTEBEQgmCkRG69NuWZ/lEva7vH+L8ifQSE0Z1I
--> ssh-ed25519 JJ1LWg 4l8GZNdGOSbqKvmKq1q1aPvjeQIwpgbJj4DBYBse7x4
-rNhTiZlwzyOiCLzYRSzJ5AHebbv94dOgl1UyNmDJD8A
--> ssh-ed25519 CAWG4Q vGhwJDLJIAU8BpV6GP8Dnz2pvTAMufY4v4nvrr2O9yw
-hNZZFDYUMPQNM5+qcc5arIgqQw0PXuqq1WWDTpE+EHo
--> ssh-ed25519 xA739A 8eEi9S5dMWPVR4fKVZdV5eHBOJVf2Ap+3qHSYtYHYgc
-GcgzvJiqsNyZTVk12Z0FEnqB4LgfQ1xjKQwXdto1Hjs
--> ssh-ed25519 MSF3dg oUY9IjDR6hi1qbrCV5z5IcYj85cMppxO94iqkD60Eww
-cBzFGrhh+kWjIi0llw2RqACU1pa7XT9kqWkSeAY8VGI
---- q7AaMOj7ZaS+Mf6trWK56o/1q/c2urrQBPAqk4PtATA
-~<7E><>k-/<2F>Xw<58><77><1D>V<EFBFBD>(<18><>Z<EFBFBD>d\<5C><>t<EFBFBD>'q<><71><EFBFBD>3<EFBFBD><33>R<EFBFBD>a\yFW<46>
+-> ssh-ed25519 AY8zKw J00a6ZOhkupkhLU5WQ0kD05HEF4KKsSs2hwjHKbnnHU
+J14VoNOCqLpScVO7OLXbqTcLI4tcVUHt5cqY/XQmbGs
+-> ssh-ed25519 sgAamA k8R/bSUdvVmlBI6yHPi5NBQPBGM36lPJwsir8DFGgxE
+4ZKC3gYvic6AVrNGgNjwztbUzhxP8ViX5O3wFo9wlrk
+-> ssh-ed25519 HY2yRg 966xf2fTnA6Wq0uYXbXZQOManqITJcCbQS9LZCGEOh4
+Qg5echQSrzqeDqvaMx+5fqi8XyTjAeCsY/UFJX6YnDs
+-> ssh-ed25519 tcumPQ e0U2okrGIoUpLfPYjIRx1V92rE3hZW13nJef+l3kBQg
+LejAUKBl+tPhwocCF00ZHTzFISnwX8og8GvemiMIcyo
+-> ssh-ed25519 JJ1LWg QkzTsPq9Gdh+FNz/a4bDb9LQOreFyxeTC51UNd1fsj0
+ayrlKenETfQzH1Z9drVEWqszQebicGVJve0/pCnxAE8
+-> ssh-ed25519 CAWG4Q lJLW9+dxvyoD4hYzeXeE/4rzJ6HIeEQOB1+fbhV3xw0
+T2RrVCtTuQvya9HiJB7txk3QGrntpsMX9Tt1cyXoW5E
+-> ssh-ed25519 MSF3dg JOZkFb2CfqWKvZIz7lYxXWgv8iEVDkQF8hInDMZvknc
+MHDWxjUw4dNiC1h4MrU9uKKcI3rwkxABm0+5FYMZkok
+-> ~8m;7f-grease
+lDIullfC98RhpTZ4Mk87Td+VtPmwPdgz+iIilpKugUkmV5r4Uqd7yE+5ArA6ekr/
+G/X4EA
+--- Cz4sv9ZunBcVdZCozdTh1zlg1zIASjk2MjYeYfcN9eA
+<EFBFBD>N	<09>$[H<><48>Q<EFBFBD><51><EFBFBD>
+d<EFBFBD><EFBFBD><EFBFBD>'<27><><EFBFBD>7<EFBFBD><1F>Ͳ)<29><><EFBFBD><17>x9y<39><79><EFBFBD>E<04><><EFBFBD>M7^<5E>[<5B>M<EFBFBD>+<2B>&<26><><EFBFBD><0E>$8tM<74>в

secrets/gitea-runner-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 HY2yRg WUMWvyagPalsy7u1RaEFAwJvFowso1/quNBo+nAkxhQ
-OHcebB7koPKhy58A6qngEVNWckkWChyEK3dwgy8EL5o
--> ssh-ed25519 CAWG4Q Yx/HLIryUNE2BaqTl84FrNRy4XLCY2TRkRgbA9k3qU4
-LZljfuLS5yMVVK6N57iC6cKEaFP6Hh2OkvWJjuFg8q0
--> ssh-ed25519 xA739A DOXjPRttSWz51Sr7KfjgKfAtaIYMo3foB1Ywqw9HYDY
-CA5puXK/1HDOitA2XHBI3OdKmZ7BzHst4DyuWGMC6hE
--> ssh-ed25519 MSF3dg +2LetdIiIZUk7wtHNS1tYsLo4ypwqZ9gpg77RQrnzHU
-yIUu8BVbF3dhUx3531RR50/cJQd9gd8VfKUQzEeT/iQ
---- oY/wQ+RjZO2CmKZtbQ0yOVZ5fv2+AlvvkRu1UDfCNAA
-_8`G<>=C7@x&<26><>\Ft<46>)<29><><EFBFBD><EFBFBD><EFBFBD>cPe<50><65>%<25>ֽ[zX-0<>[<11><><EFBFBD>ɲ<><C9B2>tz<74><7A>;%<25><><EFBFBD><EFBFBD><EFBFBD>~<7E>H0<48>؃*XD<58>;<3B><>

secrets/gitlab-runner-docker-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 HY2yRg 6C5Cv7ILdBrpMkCTT/insUY0kyQWbfgU500Ai8ePOXY
-tMw6ehFrsq2dvDEXkLOJwrNZfI28trlr9uy3xW/fzpA
--> ssh-ed25519 CAWG4Q x/j+364IYURgt7fhIPBzabbWMEg08nX8MRrJM/1Q6RU
-AL5Ut2rDr3UXcQXMZJ53ZMf5wMHmT83whx0ntJfW/WU
--> ssh-ed25519 xA739A QjXftBsoGV1rVeHSKcsjp+HMpRVsaHOeeGdDcF6ZWg4
-ovVoYPaPn3liGPAxHWY37CBIUFjAXurv6jMWs2He3HQ
--> ssh-ed25519 MSF3dg FG0CQOj9fRlneW5QrWiy5ksRpicUwHqX9QMpZWhDImw
-L20n1vZRepsRPT4xM6TO6PcI/MJxw4mBLUF0EPv9Uhs
---- DEi7iuzkniq0JPatJ5f2KhrhxWid7ojHpvNfUCGxFtk
-<EFBFBD><EFBFBD>%	n<><6E>!;^Q<>
rqG<71>:<3A>jC.8l<38>|<7C><>o<EFBFBD><1E><>$LYy<59>N<EFBFBD>b<EFBFBD><1E><>:<14>{<7B><><EFBFBD>fާxTS\<5C>t<04>U<EFBFBD><55>\F<>)%<25><><EFBFBD>KL<4B>㙇p<E39987>:><3E><><EFBFBD><EFBFBD>&<1B>)<0B>Q<EFBFBD>1<>H܃V<DC83>Sޑ<53>n<>