Delay nix-gc until /home is mounted

Prevents starting the garbage collector before the remote FS are
mounted, in particular /home. Otherwise, all the gcroots which have
symlinks in /home will be considered stale and they will be removed.

See: #79
Reviewed-by: Aleix Roca Nonell <aleix.rocanonell@bsc.es>

doc/install.md

@@ -150,3 +150,27 @@ And update grub.
 ```
 # nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
 ```
+
+## Chain NixOS in same disk with other systems
+
+To install NixOS on a partition along another system which controls the GRUB,
+first disable the grub device, so the GRUB is not installed in the disk by
+NixOS (only the /boot files will be generated):
+
+```
+boot.loader.grub.device = "nodev";
+```
+
+Then add the following entry to the old GRUB configuration:
+
+```
+menuentry 'NixOS' {
+        insmod chain
+        search --no-floppy --label nixos --set root
+        configfile /boot/grub/grub.cfg
+}
+```
+
+The partition with NixOS must have the label "nixos" for it to be found. New
+system configuration entries will be stored in the GRUB configuration managed
+by NixOS, so there is no need to change the old GRUB settings.

flake.lock

@@ -6,14 +6,15 @@
         "home-manager": "home-manager",
         "nixpkgs": [
           "nixpkgs"
-        ]
+        ],
+        "systems": "systems"
       },
       "locked": {
-        "lastModified": 1690228878,
-        "narHash": "sha256-9Xe7JV0krp4RJC9W9W9WutZVlw6BlHTFMiUP/k48LQY=",
+        "lastModified": 1720546205,
+        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "d8c973fd228949736dedf61b7f8cc1ece3236792",
+        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
         "type": "github"
       },
       "original": {
@@ -29,17 +30,17 @@
         ]
       },
       "locked": {
-        "lastModified": 1694077645,
-        "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
+        "lastModified": 1713974364,
+        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
         "ref": "refs/heads/master",
-        "rev": "6122fef92701701e1a0622550ac0fc5c2beb5906",
-        "revCount": 860,
+        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
+        "revCount": 937,
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
       },
       "original": {
         "type": "git",
-        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
       }
     },
     "darwin": {
@@ -50,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1673295039,
-        "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
+        "lastModified": 1700795494,
+        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
+        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
         "type": "github"
       },
       "original": {
@@ -72,11 +73,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1682203081,
-        "narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
+        "lastModified": 1703113217,
+        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
+        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
         "type": "github"
       },
       "original": {
@@ -87,11 +88,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1693663421,
-        "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
+        "lastModified": 1720957393,
+        "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "e56990880811a451abd32515698c712788be5720",
+        "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
         "type": "github"
       },
       "original": {
@@ -107,6 +108,21 @@
         "bscpkgs": "bscpkgs",
         "nixpkgs": "nixpkgs"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -3,7 +3,7 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
-    bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
+    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
     bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
   };
 
@@ -17,15 +17,19 @@ let
 in
   {
     nixosConfigurations = {
-      hut   = mkConf "hut";
-      owl1  = mkConf "owl1";
-      owl2  = mkConf "owl2";
-      eudy  = mkConf "eudy";
-      koro  = mkConf "koro";
-      bay   = mkConf "bay";
-      lake2 = mkConf "lake2";
+      hut     = mkConf "hut";
+      owl1    = mkConf "owl1";
+      owl2    = mkConf "owl2";
+      eudy    = mkConf "eudy";
+      koro    = mkConf "koro";
+      bay     = mkConf "bay";
+      lake2   = mkConf "lake2";
+      raccoon = mkConf "raccoon";
     };
 
-    packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs;
+    packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
+      bscpkgs = bscpkgs.packages.x86_64-linux;
+      nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
+    };
   };
 }

m/bay/configuration.nix

@@ -2,21 +2,21 @@
 
 {
   imports = [
-    ../common/main.nix
-    ../common/monitoring.nix
+    ../common/xeon.nix
+    ../module/monitoring.nix
   ];
 
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
 
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-
   networking = {
     hostName = "bay";
     interfaces.eno1.ipv4.addresses = [ {
@@ -27,6 +27,16 @@
       address = "10.0.42.40";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from lake2
+        iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
   };
 
   services.ceph = {

m/common/base.nix

@@ -0,0 +1,20 @@
+{
+  # All machines should include this profile.
+  # Includes the basic configuration for an Intel server.
+  imports = [
+    ./base/agenix.nix
+    ./base/august-shutdown.nix
+    ./base/boot.nix
+    ./base/env.nix
+    ./base/fs.nix
+    ./base/hw.nix
+    ./base/net.nix
+    ./base/nix.nix
+    ./base/ntp.nix
+    ./base/rev.nix
+    ./base/ssh.nix
+    ./base/users.nix
+    ./base/watchdog.nix
+    ./base/zsh.nix
+  ];
+}

m/common/base/august-shutdown.nix

@@ -0,0 +1,14 @@
+{
+  # Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
+  # hardware from spurious electrical peaks on the yearly electrical cut for
+  # manteinance that starts on August 4th.
+  systemd.timers.august-shutdown = {
+    description = "Shutdown on August 2nd for maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-08-02 11:00:00";
+      RandomizedDelaySec = "10min";
+      Unit = "systemd-poweroff.service";
+    };
+  };
+}

m/common/base/boot.nix

@@ -2,7 +2,7 @@
 
 {
   # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = lib.mkForce true;
+  boot.loader.grub.enable = true;
 
   # Enable GRUB2 serial console
   boot.loader.grub.extraConfig = ''
@@ -11,14 +11,12 @@
     terminal_output --append serial
   '';
 
-  # Enable serial console
-  boot.kernelParams = [
-    "console=tty1"
-    "console=ttyS0,115200"
-  ];
-
   boot.kernel.sysctl = {
     "kernel.perf_event_paranoid" = lib.mkDefault "-1";
+
+    # Allow ptracing (i.e. attach with GDB) any process of the same user, see:
+    # https://www.kernel.org/doc/Documentation/security/Yama.txt
+    "kernel.yama.ptrace_scope" = "0";
   };
 
   boot.kernelPackages = pkgs.linuxPackages_latest;

m/common/base/env.nix

@@ -0,0 +1,35 @@
+{ pkgs, config, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns
+    # From bsckgs overlay
+    osumb
+  ];
+
+  programs.direnv.enable = true;
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+}

m/common/base/fs.nix

@@ -13,16 +13,12 @@
     [ { device = "/dev/disk/by-label/swap"; }
     ];
 
-  # Mount the home via NFS
-  fileSystems."/home" = {
-    device = "10.0.40.30:/home";
-    fsType = "nfs";
-    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
-  };
-
   # Tracing
   fileSystems."/sys/kernel/tracing" = {
     device = "none";
     fsType = "tracefs";
   };
+
+  # Mount a tmpfs into /tmp
+  boot.tmp.useTmpfs = true;
 }

m/common/base/net.nix

@@ -0,0 +1,19 @@
+{ pkgs, ... }:
+
+{
+  networking = {
+    enableIPv6 = false;
+    useDHCP = false;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
+    };
+
+    hosts = {
+      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.51.152" = [ "raccoon" ];
+      "84.88.51.142" = [ "raccoon-ipmi" ];
+    };
+  };
+}

m/common/base/nix.nix

@@ -0,0 +1,57 @@
+{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
+
+{
+  nixpkgs.overlays = [
+    bscpkgs.bscOverlay
+    (import ../../../pkgs/overlay.nix)
+  ];
+
+  nix = {
+    nixPath = [
+      "nixpkgs=${nixpkgs}"
+      "jungle=${theFlake.outPath}"
+    ];
+
+    registry = {
+      nixpkgs.flake = nixpkgs;
+      jungle.flake = theFlake;
+    };
+
+    settings = {
+      experimental-features = [ "nix-command" "flakes" ];
+      sandbox = "relaxed";
+      trusted-users = [ "@wheel" ];
+      flake-registry = pkgs.writeText "global-registry.json"
+        ''{"flakes":[],"version":2}'';
+    };
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 30d";
+    };
+  };
+
+  # The nix-gc.service can begin its execution *before* /home is mounted,
+  # causing it to remove all gcroots considering them as stale, as it cannot
+  # access the symlink. To prevent this problem, we force the service to wait
+  # until /home is mounted as well as other remote FS like /ceph.
+  systemd.services.nix-gc = {
+    # Start remote-fs.target if not already being started and fail if it fails
+    # to start. It will also be stopped if the remote-fs.target fails after
+    # starting successfully.
+    bindsTo = [ "remote-fs.target" ];
+    # Wait until remote-fs.target fully starts before starting this one.
+    after = [ "remote-fs.target"];
+    # Ensure we can access a remote path inside /home
+    unitConfig.ConditionPathExists = "/home/Computational";
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}

m/common/base/rev.nix

@@ -1,6 +1,7 @@
 { theFlake, ... }:
 
 let
+  # Prevent building a configuration without revision
   rev = if theFlake ? rev then theFlake.rev
     else throw ("Refusing to build from a dirty Git tree!");
 in {
@@ -15,4 +16,6 @@ in {
     DATENOW=$(date --iso-8601=seconds)
     echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
   '';
+
+  system.configurationRevision = rev;
 }

m/common/base/ssh.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 
 let
-  keys = import ../../keys.nix;
+  keys = import ../../../keys.nix;
   hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
 in
 {
@@ -12,7 +12,7 @@ in
   programs.ssh.extraConfig = ''
     Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
       User git
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
+      ProxyCommand nc -X connect -x hut:23080 %h %p
   '';
 
   programs.ssh.knownHosts = hostsKeys // {

m/common/base/users.nix

@@ -1,6 +1,10 @@
 { pkgs, ... }:
 
 {
+  imports = [
+    ../../module/jungle-users.nix
+  ];
+
   users = {
     mutableUsers = false;
     users = {
@@ -42,13 +46,16 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
         ];
       };
+    };
 
+    jungleUsers = {
       rpenacob = {
         uid = 2761;
         isNormalUser = true;
         home = "/home/Computational/rpenacob";
         description = "Raúl Peñacoba";
         group = "Computational";
+        hosts = [ "owl1" "owl2" "hut" ];
         hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
@@ -61,11 +68,51 @@
         home = "/home/Computational/anavarro";
         description = "Antoni Navarro";
         group = "Computational";
+        hosts = [ "hut" "raccoon" ];
         hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
         ];
       };
+
+      abonerib = {
+        uid = 4541;
+        isNormalUser = true;
+        home = "/home/Computational/abonerib";
+        description = "Aleix Boné";
+        group = "Computational";
+        hosts = [ "owl1" "owl2" "hut" "raccoon" ];
+        hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
+        ];
+      };
+
+      vlopez = {
+        uid = 4334;
+        isNormalUser = true;
+        home = "/home/Computational/vlopez";
+        description = "Victor López";
+        group = "Computational";
+        hosts = [ "koro" ];
+        hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
+        ];
+      };
+
+      dbautist = {
+        uid = 5649;
+        isNormalUser = true;
+        home = "/home/Computational/dbautist";
+        description = "Dylan Bautista Cases";
+        group = "Computational";
+        hosts = [ "hut" ];
+        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
+        ];
+      };
     };
 
     groups = {

m/common/base/zsh.nix

@@ -2,7 +2,6 @@
 
 {
   environment.systemPackages = with pkgs; [
-    direnv
     zsh-completions
     nix-zsh-completions
   ];

m/common/main.nix

@@ -1,94 +0,0 @@
-{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
-
-{
-  imports = [
-    ./agenix.nix
-    ./boot.nix
-    ./fs.nix
-    ./hw.nix
-    ./net.nix
-    ./ntp.nix
-    ./slurm.nix
-    ./ssh.nix
-    ./users.nix
-    ./watchdog.nix
-    ./rev.nix
-    ./zsh.nix
-  ];
-
-  nixpkgs.overlays = [
-    bscpkgs.bscOverlay
-    (import ../../pkgs/overlay.nix)
-  ];
-
-  system.configurationRevision =
-    if theFlake ? rev
-    then theFlake.rev
-    else throw ("Refusing to build from a dirty Git tree!");
-
-  nix.nixPath = [
-    "nixpkgs=${nixpkgs}"
-    "bscpkgs=${bscpkgs}"
-    "jungle=${theFlake.outPath}"
-  ];
-
-  nix.registry.nixpkgs.flake = nixpkgs;
-  nix.registry.bscpkgs.flake = bscpkgs;
-  nix.registry.jungle.flake = theFlake;
-
-  environment.systemPackages = with pkgs; [
-    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
-    # From bsckgs overlay
-    bsc.osumb
-  ];
-
-  systemd.services."serial-getty@ttyS0" = {
-    enable = true;
-    wantedBy = [ "getty.target" ];
-    serviceConfig.Restart = "always";
-  };
-
-  # Increase limits
-  security.pam.loginLimits = [
-    {
-      domain = "*";
-      type = "-";
-      item = "memlock";
-      value = "1048576"; # 1 GiB of mem locked
-    }
-  ];
-
-  time.timeZone = "Europe/Madrid";
-  i18n.defaultLocale = "en_DK.UTF-8";
-
-  environment.variables = {
-    EDITOR = "vim";
-    VISUAL = "vim";
-  };
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  nix.settings.sandbox = "relaxed";
-  nix.settings.trusted-users = [ "@wheel" ];
-  nix.gc.automatic = true;
-  nix.gc.dates = "weekly";
-  nix.gc.options = "--delete-older-than 30d";
-
-  programs.bash.promptInit = ''
-    PS1="\h\\$ "
-  '';
-
-  # Copy the NixOS configuration file and link it from the resulting system
-  # (/run/current-system/configuration.nix). This is useful in case you
-  # accidentally delete configuration.nix.
-  #system.copySystemConfiguration = true;
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "22.11"; # Did you read the comment?
-}

m/common/xeon.nix

@@ -0,0 +1,9 @@
+{
+  # Provides the base system for a xeon node.
+  imports = [
+    ./base.nix
+    ./xeon/fs.nix
+    ./xeon/console.nix
+    ./xeon/net.nix
+  ];
+}

m/common/xeon/console.nix

@@ -0,0 +1,14 @@
+{
+  # Restart the serial console
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+}

m/common/xeon/fs.nix

@@ -0,0 +1,8 @@
+{
+  # Mount the home via NFS
+  fileSystems."/home" = {
+    device = "10.0.40.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+}

m/common/xeon/net.nix

@@ -6,12 +6,11 @@
   boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
 
   networking = {
-    enableIPv6 = false;
-    useDHCP = false;
     defaultGateway = "10.0.40.30";
     nameservers = ["8.8.8.8"];
+
     proxy = {
-      default = "http://localhost:23080/";
+      default = "http://hut:23080/";
       noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
       # Don't set all_proxy as go complains and breaks the gitlab runner, see:
       # https://github.com/golang/go/issues/16715
@@ -19,11 +18,9 @@
     };
 
     firewall = {
-      enable = true;
-      allowedTCPPorts = [ 22 ];
       extraCommands = ''
         # Prevent ssfhead from contacting our slurmd daemon
-        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
         # But accept traffic to slurm ports from any other node in the subnet
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
         # We also need to open the srun port range
@@ -32,8 +29,7 @@
     };
 
     extraHosts = ''
-      10.0.40.30      ssfhead
-      84.88.53.236    ssfhead.bsc.es ssfhead
+      10.0.40.30              ssfhead
       
       # Node Entry for node: mds01 (ID=72)
       10.0.40.40              bay mds01 mds01-eth0

m/eudy/configuration.nix

@@ -2,14 +2,14 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
-    ./slurm.nix
+    ../module/debuginfod.nix
   ];
 
   # Select this using the ID to avoid mismatches

m/eudy/kernel/kernel.nix

@@ -21,9 +21,9 @@ let
   #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
   #};
 
-  kernel = nixos-fcsv3;
+  kernel = nixos-fcs;
 
-  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -40,35 +40,13 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  });
+  }));
 
-  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
-  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
-  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
-
-  # always use fcs_sched_setaffinity
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "7d0bf285fca92badc8df3c9907a9ab30db4418aa";};
-  # remove need_check_cgroup
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "4cc4efaab5e4a0bfa3089e935215b981c1922919";};
-  # merge again fcs_wake and fcs_wait
-  #nixos-debug = nixos-fcs-kernel {gitCommit = "40c6f72f4ae54b0b636b193ac0648fb5730c810d";};
-  # start from scratch, this is the working version with split fcs_wake and fcs_wait
-  nixos-debug = nixos-fcs-kernel {gitCommit = "c9a39d6a4ca83845b4e71fcc268fb0a76aff1bdf"; branch = "fcs-test"; };
-
-  nixos-fcsv1-lockstat = nixos-fcs-kernel {
-    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
     lockStat = true;
   };
-  nixos-fcsv2-lockstat = nixos-fcs-kernel {
-    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
     lockStat = true;
     preempt = true;
   };

m/eudy/slurm.nix

@@ -1,7 +0,0 @@
-{ lib, ... }:
-
-{
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-}

m/hut/blackbox.yml

@@ -0,0 +1,162 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      proxy_url: "http://127.0.0.1:23080"
+      skip_resolve_phase_with_proxy: true
+      follow_redirects: true
+      valid_status_codes: []  # Defaults to 2xx
+      method: GET
+  http_with_proxy:
+    prober: http
+    http:
+      proxy_url: "http://127.0.0.1:3128"
+      skip_resolve_phase_with_proxy: true
+  http_with_proxy_and_headers:
+    prober: http
+    http:
+      proxy_url: "http://127.0.0.1:3128"
+      proxy_connect_header:
+        Proxy-Authorization:
+          - Bearer token
+  http_post_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      headers:
+        Content-Type: application/json
+      body: '{}'
+  http_post_body_file:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      body_file: "/files/body.txt"
+  http_basic_auth_example:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      headers:
+        Host: "login.example.com"
+      basic_auth:
+        username: "username"
+        password: "mysecret"
+  http_2xx_oauth_client_credentials:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+      valid_status_codes:
+        - 200
+        - 201
+      oauth2:
+        client_id: "client_id"
+        client_secret: "client_secret"
+        token_url: "https://api.example.com/token"
+        endpoint_params:
+          grant_type: "client_credentials"
+  http_custom_ca_example:
+    prober: http
+    http:
+      method: GET
+      tls_config:
+        ca_file: "/certs/my_cert.crt"
+  http_gzip:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+  http_gzip_with_accept_encoding:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+      headers:
+        Accept-Encoding: gzip
+  tls_connect:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      tls: true
+  tcp_connect_example:
+    prober: tcp
+    timeout: 5s
+  imap_starttls:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - expect: "OK.*STARTTLS"
+        - send: ". STARTTLS"
+        - expect: "OK"
+        - starttls: true
+        - send: ". capability"
+        - expect: "CAPABILITY IMAP4rev1"
+  smtp_starttls:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - expect: "^220 ([^ ]+) ESMTP (.+)$"
+        - send: "EHLO prober\r"
+        - expect: "^250-STARTTLS"
+        - send: "STARTTLS\r"
+        - expect: "^220"
+        - starttls: true
+        - send: "EHLO prober\r"
+        - expect: "^250-AUTH"
+        - send: "QUIT\r"
+  irc_banner_example:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - send: "NICK prober"
+        - send: "USER prober prober prober :prober"
+        - expect: "PING :([^ ]+)"
+          send: "PONG ${1}"
+        - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"
+  dns_udp_example:
+    prober: dns
+    timeout: 5s
+    dns:
+      query_name: "www.prometheus.io"
+      query_type: "A"
+      valid_rcodes:
+        - NOERROR
+      validate_answer_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+        fail_if_all_match_regexp:
+          - ".*127.0.0.1"
+        fail_if_not_matches_regexp:
+          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
+        fail_if_none_matches_regexp:
+          - "127.0.0.1"
+      validate_authority_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+      validate_additional_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+  dns_soa:
+    prober: dns
+    dns:
+      query_name: "prometheus.io"
+      query_type: "SOA"
+  dns_tcp_example:
+    prober: dns
+    dns:
+      transport_protocol: "tcp" # defaults to "udp"
+      preferred_ip_protocol: "ip4" # defaults to "ip6"
+      query_name: "www.prometheus.io"

m/hut/configuration.nix

@@ -2,19 +2,24 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
 
     ../module/ceph.nix
+    ../module/debuginfod.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
-    ./slurm-daemon.nix
+    ./slurm-server.nix
     ./nix-serve.nix
+    ./public-inbox.nix
+    ./gitea.nix
+    ./msmtp.nix
+    ./postgresql.nix
     #./pxe.nix
   ];
 
-  boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
-
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
 
@@ -28,5 +33,15 @@
       address = "10.0.42.7";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all proxy traffic from compute nodes but not the login
+        iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
+      '';
+    };
   };
+
+  # Allow proxy to bind to the ethernet interface
+  services.openssh.settings.GatewayPorts = "clientspecified";
 }

m/hut/gitea.nix

@@ -0,0 +1,63 @@
+{ config, lib, ... }:
+{
+  age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
+
+  services.gitea = {
+    enable = true;
+    appName = "Gitea in the jungle";
+
+    settings = {
+      server = {
+        ROOT_URL = "https://jungle.bsc.es/git/";
+        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
+        LANDING_PAGE = "explore";
+      };
+      metrics.ENABLED = true;
+      service = {
+        REGISTER_MANUAL_CONFIRM = true;
+        ENABLE_NOTIFY_MAIL = true;
+      };
+      log.LEVEL = "Warn";
+
+      mailer = {
+        ENABLED       = true;
+        FROM          = "jungle-robot@bsc.es";
+        PROTOCOL      = "sendmail";
+        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
+        SENDMAIL_ARGS = "--";
+      };
+    };
+  };
+
+  services.gitea-actions-runner.instances = {
+    runrun = {
+      enable = true;
+      name = "runrun";
+      url = "https://jungle.bsc.es/git/";
+      tokenFile = config.age.secrets.giteaRunnerToken.path;
+      labels = [ "native:host" ];
+      settings.runner.capacity = 8;
+    };
+  };
+
+  systemd.services.gitea-runner-runrun = {
+    path = [ "/run/current-system/sw" ];
+    serviceConfig = {
+      # DynamicUser doesn't work well with SSH
+      DynamicUser = lib.mkForce false;
+      User = "gitea-runner";
+      Group = "gitea-runner";
+    };
+  };
+
+  users.users.gitea-runner = {
+    isSystemUser = true;
+    home = "/var/lib/gitea-runner";
+    description = "Gitea Runner";
+    group = "gitea-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitea-runner = {};
+}
+

m/hut/gitlab-runner.nix

@@ -1,31 +1,23 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
-  age.secrets.nosvToken.file = ../../secrets/nosv-token.age;
+  age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age;
+  age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age;
 
   services.gitlab-runner = {
     enable = true;
     settings.concurrent = 5;
-    services = {
-      ovni-shell = {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+    services = let
+      common-shell = {
         executor = "shell";
-        tagList = [ "nix" "xeon" ];
-        registrationFlags = [
-          # Using space doesn't work, and causes it to misread the next flag
-          "--locked='false'"
-        ];
         environmentVariables = {
           SHELL = "${pkgs.bash}/bin/bash";
         };
       };
-      ovni-docker = {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+      common-docker = {
+        executor = "docker";
         dockerImage = "debian:stable";
-        tagList = [ "docker" "xeon" ];
         registrationFlags = [
-          "--locked='false'"
           "--docker-network-mode host"
         ];
         environmentVariables = {
@@ -33,18 +25,13 @@
           http_proxy = "http://localhost:23080";
         };
       };
-      nosv-docker = {
-        registrationConfigFile = config.age.secrets.nosvToken.path;
-        dockerImage = "debian:stable";
-        tagList = [ "docker" "xeon" ];
-        registrationFlags = [
-          "--docker-network-mode host"
-          "--docker-cpus 56"
-        ];
-        environmentVariables = {
-          https_proxy = "http://localhost:23080";
-          http_proxy = "http://localhost:23080";
-        };
+    in {
+      # For pm.bsc.es/gitlab
+      gitlab-pm-shell = common-shell // {
+        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path;
+      };
+      gitlab-pm-docker = common-docker // {
+        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path;
       };
     };
   };

m/hut/monitoring.nix

@@ -1,6 +1,14 @@
 { config, lib, ... }:
 
 {
+  imports = [ ../module/slurm-exporter.nix ];
+
+  age.secrets.grafanaJungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    owner = "grafana";
+    mode = "400";
+  };
+
   services.grafana = {
     enable = true;
     settings = {
@@ -11,10 +19,25 @@
         http_port = 2342;
         http_addr = "127.0.0.1";
       };
+      smtp = {
+        enabled = true;
+        from_address = "jungle-robot@bsc.es";
+        user = "jungle-robot";
+        # Read the password from a file, which is only readable by grafana user
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
+        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
+        host = "mail.bsc.es:465";
+        startTLS_policy = "NoStartTLS";
+      };
       feature_toggles.publicDashboards = true;
+      "auth.anonymous".enabled = true;
+      log.level = "warn";
     };
   };
 
+  # Make grafana alerts also use the proxy
+  systemd.services.grafana.environment = config.networking.proxy.envVars;
+
   services.prometheus = {
     enable = true;
     port = 9001;
@@ -61,6 +84,11 @@
         enable = true;
         listenAddress = "127.0.0.1";
       };
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = ./blackbox.yml;
+      };
     };
 
     scrapeConfigs = [
@@ -73,6 +101,8 @@
             "127.0.0.1:9323"
             "127.0.0.1:9252"
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
+            "127.0.0.1:9341" # Slurm exporter
+            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];
       }
@@ -86,6 +116,71 @@
           ];
         }];
       }
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = { module = [ "http_2xx" ]; };
+        static_configs = [{
+          targets = [
+            "https://www.google.com/robots.txt"
+            "https://pm.bsc.es/"
+            "https://pm.bsc.es/gitlab/"
+            "https://jungle.bsc.es/"
+            "https://gitlab.bsc.es/"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-icmp";
+        metrics_path = "/probe";
+        params = { module = [ "icmp" ]; };
+        static_configs = [{
+          targets = [
+            "1.1.1.1"
+            "8.8.8.8"
+            "ssfhead"
+            "anella-bsc.cesca.cat"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "gitea";
+        static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
+      }
       {
         # Scrape the IPMI info of the hosts remotely via LAN
         job_name = "ipmi-lan";
@@ -138,6 +233,17 @@
           }
         ];
       }
+      {
+        job_name = "ipmi-raccoon";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9291" ]; }
+        ];
+        params = {
+          target = [ "84.88.51.142" ];
+          module = [ "raccoon" ];
+        };
+      }
     ];
   };
 }

m/hut/msmtp.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+{
+  age.secrets.jungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    group = "gitea";
+    mode = "440";
+  };
+
+  programs.msmtp = {
+    enable = true;
+    accounts = {
+      default = {
+        auth = true;
+        tls = true;
+        tls_starttls = false;
+        port = 465;
+        host = "mail.bsc.es";
+        user = "jungle-robot";
+        passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
+        from = "jungle-robot@bsc.es";
+      };
+    };
+  };
+}

m/hut/postgresql.nix

@@ -0,0 +1,19 @@
+{ lib, ... }:
+
+{
+  services.postgresql = {
+    enable = true;
+    ensureDatabases = [ "perftestsdb" ];
+    ensureUsers = [
+      { name = "anavarro"; ensureClauses.superuser = true; }
+      { name = "rarias";   ensureClauses.superuser = true; }
+      { name = "grafana"; }
+    ];
+    authentication = ''
+      #type  database     DBuser    auth-method
+      local  perftestsdb  rarias    trust
+      local  perftestsdb  anavarro  trust
+      local  perftestsdb  grafana   trust
+    '';
+  };
+}

m/hut/public-inbox.css

@@ -0,0 +1,79 @@
+/*
+ * CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
+ * Dark color scheme using 216 web-safe colors, inspired
+ * somewhat by the default color scheme in mutt.
+ * It reduces eyestrain for me, and energy usage for all:
+ * https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
+ */
+
+* {
+	font-size: 14px;
+	font-family: monospace;
+}
+
+pre {
+	white-space: pre-wrap;
+	padding: 10px;
+	background: #f5f5f5;
+}
+
+hr {
+	margin: 30px 0;
+}
+
+body {
+	max-width: 120ex; /* 120 columns wide */
+	margin: 50px auto;
+}
+
+/*
+ * Underlined links add visual noise which make them hard-to-read.
+ * Use colors to make them stand out, instead.
+ */
+a:link {
+	color: #007;
+	text-decoration: none;
+}
+a:visited {
+	color:#504;
+}
+a:hover {
+	text-decoration: underline;
+}
+
+/* quoted text in emails gets a different color */
+*.q { color:gray }
+
+/*
+ * these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
+ * (cgit uses <div>, public-inbox uses <span>)
+ */
+*.add { color:darkgreen } /* diff post-image lines */
+*.del { color:darkred } /* diff pre-image lines */
+*.head { color:black } /* diff header (metainformation) */
+*.hunk { color:gray } /* diff hunk-header */
+
+/*
+ * highlight 3.x colors (tested 3.18) for displaying blobs.
+ * This doesn't use most of the colors available, as I find too
+ * many colors overwhelming, so the default is commented out.
+ */
+.hl.num { color:#f30 } /* number */
+.hl.esc { color:#f0f } /* escape character */
+.hl.str { color:#f30 } /* string */
+.hl.ppc { color:#f0f } /* preprocessor */
+.hl.pps { color:#f30 } /* preprocessor string */
+.hl.slc { color:#09f } /* single-line comment */
+.hl.com { color:#09f } /* multi-line comment */
+/* .hl.opt { color:#ccc } */ /* operator */
+/* .hl.ipl { color:#ccc } */ /* interpolation */
+
+/* keyword groups kw[a-z] */
+.hl.kwa { color:#ff0 }
+.hl.kwb { color:#0f0 }
+.hl.kwc { color:#ff0 }
+/* .hl.kwd { color:#ccc } */
+
+/* line-number (unused by public-inbox) */
+/* .hl.lin { color:#ccc } */
+

m/hut/public-inbox.nix

@@ -0,0 +1,47 @@
+{ lib, ... }:
+
+{
+  services.public-inbox = {
+    enable = true;
+    http = {
+      enable = true;
+      port = 8081;
+      mounts = [ "/lists" ];
+    };
+    settings.publicinbox = {
+      css = [ "${./public-inbox.css}" ];
+      wwwlisting = "all";
+    };
+    inboxes = {
+      bscpkgs = {
+        url = "https://jungle.bsc.es/lists/bscpkgs";
+        address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for bscpkgs";
+        listid = "~rodarima/bscpkgs.lists.sr.ht";
+      };
+      jungle = {
+        url = "https://jungle.bsc.es/lists/jungle";
+        address = [ "~rodarima/jungle@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for jungle";
+        listid = "~rodarima/jungle.lists.sr.ht";
+      };
+    };
+  };
+
+  # We need access to the network for the watch service, as we will fetch the
+  # emails directly from the IMAP server.
+  systemd.services.public-inbox-watch.serviceConfig = {
+    PrivateNetwork = lib.mkForce false;
+    RestrictAddressFamilies = lib.mkForce [ "AF_UNIX"  "AF_INET" "AF_INET6" ];
+    KillSignal = "SIGKILL"; # Avoid slow shutdown
+
+    # Required for chmod(..., 02750) on directories by git, from
+    # systemd.exec(8):
+    # > Note that this restricts marking of any type of file system object with
+    # > these bits, including both regular files and directories (where the SGID
+    # > is a different meaning than for files, see documentation).
+    RestrictSUIDSGID = lib.mkForce false;
+  };
+}

m/koro/configuration.nix

@@ -2,13 +2,11 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix
     ../eudy/users.nix
-    ../eudy/slurm.nix
-    ./users.nix
     ./kernel.nix
   ];
 

m/koro/kernel.nix

@@ -1,9 +1,29 @@
 { pkgs, lib, ... }:
 
 let
-  kernel = nixos-fcsv4;
+  #fcs-devel = pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = /mnt/data/kernel/fcs/kernel/src;
+  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
+  #};
 
-  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
+  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
+  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
+  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
+  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = builtins.fetchGit {
+  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+  #     rev = gitCommit;
+  #     ref = "fcs";
+  #   };
+  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
+  #};
+
+  kernel = nixos-fcs;
+
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
     version = "6.2.8";
     src = builtins.fetchGit {
       url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
@@ -20,27 +40,13 @@ let
     };
     kernelPatches = [];
     extraMeta.branch = lib.versions.majorMinor version;
-  });
+  }));
 
-  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
-  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
-  nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
-  nixos-fcsv4 = nixos-fcs-kernel {gitCommit = "c94c3d946f33ac3e5782a02ee002cc1164c0cb4f";};
-
-  nixos-fcsv1-lockstat = nixos-fcs-kernel {
-    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
     lockStat = true;
   };
-  nixos-fcsv2-lockstat = nixos-fcs-kernel {
-    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
-    lockStat = true;
-  };
-  nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
-    gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
     lockStat = true;
     preempt = true;
   };
@@ -60,5 +66,5 @@ in {
   
   # enable memory overcommit, needed to build a taglibc system using nix after
   # increasing the openblas memory footprint
-  boot.kernel.sysctl."vm.overcommit_memory" = lib.mkForce 1;
+  boot.kernel.sysctl."vm.overcommit_memory" = 1;
 }

m/koro/users.nix

@@ -1,17 +0,0 @@
-{ ... }:
-
-{
-  users.users = {
-    vlopez = {
-      uid = 4334;
-      isNormalUser = true;
-      home = "/home/Computational/vlopez";
-      description = "Victor López";
-      group = "Computational";
-      hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
-      openssh.authorizedKeys.keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
-      ];
-    };
-  };
-}

m/lake2/configuration.nix

@@ -2,20 +2,20 @@
 
 {
   imports = [
-    ../common/main.nix
-    ../common/monitoring.nix
+    ../common/xeon.nix
+    ../module/monitoring.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
 
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
   environment.systemPackages = with pkgs; [
     ceph
   ];
 
-  services.slurm = {
-    client.enable = lib.mkForce false;
-  };
-
   services.ceph = {
     enable = true;
     global = {
@@ -49,6 +49,16 @@
       address = "10.0.42.42";
       prefixLength = 24;
     } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from bay
+        iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
   };
 
   # Missing service for volumes, see:

m/module/ceph.nix

@@ -3,7 +3,6 @@
 # Mounts the /ceph filesystem at boot
 {
   environment.systemPackages = with pkgs; [
-    ceph
     ceph-client
     fio # For benchmarks
   ];

m/module/debuginfod.nix

@@ -0,0 +1,3 @@
+{
+  services.nixseparatedebuginfod.enable = true;
+}

m/module/emulation.nix

@@ -0,0 +1,3 @@
+{
+  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
+}

m/module/jungle-users.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+
+with lib;
+
+{
+  options = {
+    users.jungleUsers = mkOption {
+      type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
+      description = ''
+        Same as users.users but with the extra `hosts` attribute, which controls
+        access to the nodes by `networking.hostName`.
+      '';
+    };
+  };
+
+  config = let
+    allowedUser = host: userConf: builtins.elem host userConf.hosts;
+    filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
+    removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
+    currentHost = config.networking.hostName;
+  in {
+    users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
+  };
+}

m/module/slurm-client.nix

@@ -47,8 +47,8 @@ in {
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP"
-      "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -83,6 +83,14 @@ in {
 
       # Reduce port range so we can allow only this range in the firewall
       SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
     '';
   };
 

m/module/slurm-exporter.nix

@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+# See also: https://github.com/NixOS/nixpkgs/pull/112010
+# And: https://github.com/NixOS/nixpkgs/pull/115839
+
+with lib;
+
+{
+  systemd.services."prometheus-slurm-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = ''
+        ${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
+      '';
+      Environment = [
+        "PATH=${pkgs.slurm}/bin"
+        # We need to specify the slurm config to be able to talk to the slurmd
+        # daemon.
+        "SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
+      ];
+    };
+  };
+}

m/module/slurm-hut-nix-store.nix

@@ -0,0 +1,19 @@
+{ ... }:
+
+{
+  # Mount the hut nix store via NFS
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  systemd.services.slurmd.serviceConfig = {
+    # When running a job, bind the hut store in /nix/store so the paths are
+    # available too.
+    # FIXME: This doesn't keep the programs in /run/current-system/sw/bin
+    # available in the store. Ideally they should be merged but the overlay FS
+    # doesn't work when the underlying directories change.
+    BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
+  };
+}

m/owl1/configuration.nix

@@ -2,9 +2,12 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -2,9 +2,12 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
     ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/raccoon/configuration.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+  ];
+
+  # Don't install Grub on the disk yet
+  boot.loader.grub.device = "nodev";
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS1,115200"
+  ];
+
+  networking = {
+    hostName = "raccoon";
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    defaultGateway = "84.88.51.129";
+    interfaces.eno0.ipv4.addresses = [ {
+      address = "84.88.51.152";
+      prefixLength = 25;
+    } ];
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.graphics.enable = true;
+  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  users.motd = ''
+    ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⡎⢰⣧⠀⠀⠀⠁⠈⠛⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⣴⡦⠶⠟⠓⠚⠻⡄⠀
+    ⠀⠀⠀⠀⠀⠀⣧⠀⣱⣀⣰⣧⠀⢀⠀⣘⣿⣿⣦⣶⣄⣠⡀⠀⠀⣀⣀⣤⣴⣄⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠿⠏⠁⠀⣀⣠⣶⣿⡶⣿⠀
+    ⠀⠀⠀⠀⠀⠀⣹⣆⠘⣿⣿⣿⣇⢸⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣶⣶⣦⡀⣀⣤⣠⣤⡾⠋⠀⢀⣤⣶⣿⣿⣿⣿⣿⣿⣿⡀
+    ⠀⠀⠀⠀⠀⠀⠘⢿⡄⢼⣿⣿⣿⣿⣿⡟⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣵⣾⡾⠙⣋⣩⣽⣿⣿⣿⣿⢋⡼⠁
+    ⠀⠀⠀⠀⠀⠀⠀⠈⢻⣄⠸⢿⣿⣿⠿⠷⠀⠈⠀⣭⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣾⣿⣿⣿⣿⣿⣿⠇⡼⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⣯⡀⠀⢼⡿⠀⠀⠀⢼⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣿⡿⣿⣿⣿⠿⣿⣯⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢋⡼⠁⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢻⡏⠠⣦⠁⠀⠀⠀⠀⠀⠟⠛⠛⣿⣿⣿⣿⣿⠿⠁⠀⠁⢿⠙⠁⠀⠛⠹⣿⣏⣾⣿⣿⣿⣿⣿⣿⣿⣿⠿⠃⣹⠁⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣘⣧⠀⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⡿⡿⠀⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀⢹⣿⠿⢿⣿⣿⣿⣿⣿⠋⢀⡤⠛⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⡯⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⢸⣿⣿⣿⠛⠉⠀⣰⠷⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠇⠀⠀⠀⠀⠀⢀⣿⡇⠀⠀⢻⣿⣿⠁⠀⠀⢠⣾⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠟⢿⣿⣄⡀⢸⣿⡀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⠀⠀⠀⢰⣿⣿⡛⣿⣿⡄⢠⡺⠿⡍⠁⢀⣤⣿⣿⣿⠿⣷⣮⣉⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⠀⠀⠈⣧⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⠉⠃⠀⣴⣿⣟⠻⣿⣿⣿⡇⢸⣿⣶⠀⢀⣾⣿⣿⣟⠿⣷⣾⣿⣿⣿⣿⣦⣤⣤⡤⠀⠀⠀⠀⠀⠁⠀⠀⠀⣼⠗⠀⠀⠀⠀
+    ⠀⠀⠐⢄⡀⠀⠀⠀⢘⡀⠀⢶⣾⣿⣿⣿⣿⡿⠋⠁⠈⠻⠉⠀⠚⠻⣿⣿⣿⣶⣾⣿⣿⣿⣿⣿⣿⣷⣬⣤⣶⣦⡀⣾⣶⣇⠀⠀⠈⢉⣷⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠈⠓⠶⢦⡽⠄⣈⣿⣿⣿⣿⣿⠏⠀⠀⠀⠀⠀⠀⠀⠀⠀⠹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡓⠙⣿⡟⠀⠀⠀⠈⠛⣷⣶⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⢀⣬⠆⢠⣍⣛⠻⣿⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣉⣀⡀⠀⠀⠈⠛⢿⣦⡀
+    ⠐⠒⠒⠶⠶⠶⢦⣬⣟⣥⣀⡉⠛⠻⠶⢁⣤⣾⣿⣿⣿⣷⡄⠀⠀⠀⠀⠀⢸⣿⣿⣿⣿⣿⣟⡛⠿⠭⠭⠭⠭⠭⠿⠿⠿⢿⣿⣟⠃⠀⠀⠀⠹⣟⠓
+    ⠀⣀⣠⠤⠤⢤⣤⣾⣤⡄⣉⣉⣙⣓⡂⣿⣿⣭⣹⣿⣿⣿⣿⡰⣂⣀⢀⠀⠻⣿⠛⠻⠟⠡⣶⣾⣿⣿⣿⣿⣿⣿⣿⡖⠒⠒⠒⠛⠷⢤⡀⢰⣴⣿⡆
+    ⠀⠀⠀⢀⣠⡴⠾⠟⠻⣟⡉⠉⠉⠉⢁⢿⣿⣿⣿⣿⣿⣿⡿⣱⣿⣭⡌⠤⠀⠀⠐⣶⣌⡻⣶⣭⡻⢿⣿⣿⣿⣿⣿⣯⣥⣤⣦⠀⠠⣴⣶⣶⣿⡟⢿
+    ⢀⠔⠊⠉⠀⠀⠀⠀⢸⣯⣤⠀⠀⠠⣼⣮⣟⣿⣿⣿⣻⣭⣾⣿⣿⣷⣶⣦⠶⣚⣾⣿⣿⣷⣜⣿⣿⣶⣝⢿⣿⣿⣿⣿⣷⣦⣄⣰⡄⠈⢿⣿⡿⣇⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠈⢡⢇⠀⠀⣠⣿⣿⣿⣯⣟⣛⣛⣛⣛⣛⣩⣭⣴⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣻⣿⣧⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠏⠀⢹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣍⣿⣿⣿⣿⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣾⡁⢈⣾⣿⡿⠛⣛⣿⣿⣿⣿ DO YOU BRING FEEDS? ⣿⣿⣿⣿⣿⣿⡏⠈⠙⠈⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⡿⠛⠉⣽⣿⣷⣾⡿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠷⠌⠛⠉⠀⠁⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠹⠋⠀⢻⣿⣿⣿⣿⠿⢿⣿⣿⣿⣿⣿⣿⠿⣿⣿⣿⣿⠿⠛⠋⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠁⠀⠀⠀⠀⠀⠈⠉⠉⠀⠀⠈⠋⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 
+  '';
+}

pkgs/ceph.nix

@@ -1,405 +0,0 @@
-{ lib
-, stdenv
-, runCommand
-, fetchurl
-, fetchFromGitHub
-, fetchPypi
-
-# Build time
-, cmake
-, ensureNewerSourcesHook
-, fmt
-, git
-, makeWrapper
-, nasm
-, pkg-config
-, which
-
-# Tests
-, nixosTests
-
-# Runtime dependencies
-, arrow-cpp
-, babeltrace
-, boost179
-, bzip2
-, cryptsetup
-, cunit
-, doxygen
-, gperf
-, graphviz
-, gtest
-, icu
-, libcap
-, libcap_ng
-, libnl
-, libxml2
-, lttng-ust
-, lua
-, lz4
-, oath-toolkit
-, openldap
-, python310
-, rdkafka
-, rocksdb
-, snappy
-, sqlite
-, utf8proc
-, zlib
-, zstd
-
-# Optional Dependencies
-, curl ? null
-, expat ? null
-, fuse ? null
-, libatomic_ops ? null
-, libedit ? null
-, libs3 ? null
-, yasm ? null
-
-# Mallocs
-, gperftools ? null
-, jemalloc ? null
-
-# Crypto Dependencies
-, cryptopp ? null
-, nspr ? null
-, nss ? null
-
-# Linux Only Dependencies
-, linuxHeaders
-, util-linux
-, libuuid
-, udev
-, keyutils
-, rdma-core
-, rabbitmq-c
-, libaio ? null
-, libxfs ? null
-, liburing ? null
-, zfs ? null
-, ...
-}:
-
-# We must have one crypto library
-assert cryptopp != null || (nss != null && nspr != null);
-
-let
-  shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
-
-  optYasm = shouldUsePkg yasm;
-  optExpat = shouldUsePkg expat;
-  optCurl = shouldUsePkg curl;
-  optFuse = shouldUsePkg fuse;
-  optLibedit = shouldUsePkg libedit;
-  optLibatomic_ops = shouldUsePkg libatomic_ops;
-  optLibs3 = shouldUsePkg libs3;
-
-  optJemalloc = shouldUsePkg jemalloc;
-  optGperftools = shouldUsePkg gperftools;
-
-  optCryptopp = shouldUsePkg cryptopp;
-  optNss = shouldUsePkg nss;
-  optNspr = shouldUsePkg nspr;
-
-  optLibaio = shouldUsePkg libaio;
-  optLibxfs = shouldUsePkg libxfs;
-  optZfs = shouldUsePkg zfs;
-
-  # Downgrade rocksdb, 7.10 breaks ceph
-  rocksdb' = rocksdb.overrideAttrs {
-    version = "7.9.2";
-    src = fetchFromGitHub {
-      owner = "facebook";
-      repo = "rocksdb";
-      rev = "refs/tags/v7.9.2";
-      hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
-    };
-  };
-
-  hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
-
-  # Malloc implementation (can be jemalloc, tcmalloc or null)
-  malloc = if optJemalloc != null then optJemalloc else optGperftools;
-
-  # We prefer nss over cryptopp
-  cryptoStr = if optNss != null && optNspr != null then "nss" else
-    if optCryptopp != null then "cryptopp" else "none";
-
-  cryptoLibsMap = {
-    nss = [ optNss optNspr ];
-    cryptopp = [ optCryptopp ];
-    none = [ ];
-  };
-
-  getMeta = description: with lib; {
-     homepage = "https://ceph.io/en/";
-     inherit description;
-     license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
-     maintainers = with maintainers; [ adev ak johanot krav ];
-     platforms = [ "x86_64-linux" "aarch64-linux" ];
-   };
-
-  ceph-common = with python.pkgs; buildPythonPackage {
-    pname = "ceph-common";
-    inherit src version;
-
-    sourceRoot = "ceph-${version}/src/python-common";
-
-    propagatedBuildInputs = [
-      pyyaml
-    ];
-
-    nativeCheckInputs = [
-      pytestCheckHook
-    ];
-
-    disabledTests = [
-      # requires network access
-      "test_valid_addr"
-    ];
-
-    meta = getMeta "Ceph common module for code shared by manager modules";
-  };
-
-  # Watch out for python <> boost compatibility
-  python = python310.override {
-    packageOverrides = self: super: {
-      sqlalchemy = super.sqlalchemy.overridePythonAttrs rec {
-        version = "1.4.46";
-        src = fetchPypi {
-          pname = "SQLAlchemy";
-          inherit version;
-          hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA=";
-        };
-        disabledTestPaths = [
-          "test/aaa_profiling"
-          "test/ext/mypy"
-        ];
-      };
-    };
-  };
-
-  boost = boost179.override {
-    enablePython = true;
-    inherit python;
-  };
-
-  # TODO: split this off in build and runtime environment
-  ceph-python-env = python.withPackages (ps: with ps; [
-    ceph-common
-
-    # build time
-    cython
-
-    # debian/control
-    bcrypt
-    cherrypy
-    influxdb
-    jinja2
-    kubernetes
-    natsort
-    numpy
-    pecan
-    prettytable
-    pyjwt
-    pyopenssl
-    python-dateutil
-    pyyaml
-    requests
-    routes
-    scikit-learn
-    scipy
-    setuptools
-    sphinx
-    virtualenv
-    werkzeug
-
-    # src/pybind/mgr/requirements-required.txt
-    cryptography
-    jsonpatch
-
-    # src/tools/cephfs/shell/setup.py
-    cmd2
-    colorama
-  ]);
-  inherit (ceph-python-env.python) sitePackages;
-
-  version = "18.2.0";
-  src = fetchurl {
-    url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
-    hash = "sha256:0k9nl6xi5brva51rr14m7ig27mmmd7vrpchcmqc40q3c2khn6ns9";
-  };
-in rec {
-  ceph = stdenv.mkDerivation {
-    pname = "ceph";
-    inherit src version;
-
-    nativeBuildInputs = [
-      cmake
-      fmt
-      git
-      makeWrapper
-      nasm
-      pkg-config
-      python
-      python.pkgs.python # for the toPythonPath function
-      python.pkgs.wrapPython
-      which
-      (ensureNewerSourcesHook { year = "1980"; })
-      # for building docs/man-pages presumably
-      doxygen
-      graphviz
-    ];
-
-    enableParallelBuilding = true;
-
-    buildInputs = cryptoLibsMap.${cryptoStr} ++ [
-      arrow-cpp
-      babeltrace
-      boost
-      bzip2
-      ceph-python-env
-      cryptsetup
-      cunit
-      gperf
-      gtest
-      icu
-      libcap
-      libnl
-      libxml2
-      lttng-ust
-      lua
-      lz4
-      malloc
-      oath-toolkit
-      openldap
-      optLibatomic_ops
-      optLibs3
-      optYasm
-      rdkafka
-      rocksdb'
-      snappy
-      sqlite
-      utf8proc
-      zlib
-      zstd
-    ] ++ lib.optionals stdenv.isLinux [
-      keyutils
-      libcap_ng
-      liburing
-      libuuid
-      linuxHeaders
-      optLibaio
-      optLibxfs
-      optZfs
-      rabbitmq-c
-      rdma-core
-      udev
-      util-linux
-    ] ++ lib.optionals hasRadosgw [
-      optCurl
-      optExpat
-      optFuse
-      optLibedit
-    ];
-
-    pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
-
-    preConfigure =''
-      substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
-      substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
-      substituteInPlace src/common/module.c --replace "/bin/grep" "grep"
-
-      # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
-      # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others
-      export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages}
-      patchShebangs src/
-    '';
-
-    cmakeFlags = [
-      "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
-
-      "-DWITH_CEPHFS_SHELL:BOOL=ON"
-      "-DWITH_SYSTEMD:BOOL=OFF"
-      # `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
-      #     CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
-      #     Command failed: 2
-      #
-      #        'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
-      #
-      #     See also
-      #
-      #        /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
-      # and that file contains:
-      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
-      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
-      #       71 |     ::THRIFT_CLOSESOCKET(socket_);
-      #          |       ^~~~~~~~~~~~~~~~~~
-      # Looks like `close()` is somehow not included.
-      # But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
-      # So it's proably not worth trying to fix that for this Ceph version,
-      # and instead just disable Ceph's Jaeger support.
-      "-DWITH_JAEGER:BOOL=OFF"
-      "-DWITH_TESTS:BOOL=OFF"
-
-      # Use our own libraries, where possible
-      "-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
-      "-DWITH_SYSTEM_BOOST:BOOL=ON"
-      "-DWITH_SYSTEM_GTEST:BOOL=ON"
-      "-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
-      "-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
-      "-DWITH_SYSTEM_ZSTD:BOOL=ON"
-
-      # TODO breaks with sandbox, tries to download stuff with npm
-      "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
-      # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
-      ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
-    ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON";
-
-    postFixup = ''
-      wrapPythonPrograms
-      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
-
-      # Test that ceph-volume exists since the build system has a tendency to
-      # silently drop it with misconfigurations.
-      test -f $out/bin/ceph-volume
-    '';
-
-    outputs = [ "out" "lib" "dev" "doc" "man" ];
-
-    doCheck = false; # uses pip to install things from the internet
-
-    # Takes 7+h to build with 2 cores.
-    requiredSystemFeatures = [ "big-parallel" ];
-
-    meta = getMeta "Distributed storage system";
-
-    passthru = {
-      inherit version;
-      tests = {
-        inherit (nixosTests)
-          ceph-multi-node
-          ceph-single-node
-          ceph-single-node-bluestore;
-      };
-    };
-  };
-
-  ceph-client = runCommand "ceph-client-${version}" {
-      meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
-    } ''
-      mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
-      cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
-      cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
-      cp -r ${ceph}/bin/rbd-replay* $out/bin
-      cp -r ${ceph}/sbin/mount.ceph $out/bin
-      cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
-      ln -s bin $out/sbin
-      cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
-      cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
-      # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
-      substituteInPlace $out/bin/ceph          --replace ${ceph} $out
-      substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
-   '';
-}

pkgs/mpich-fix-hwtopo.patch

@@ -0,0 +1,36 @@
+diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
+index 33e88bc..ee3641c 100644
+--- a/src/util/mpir_hwtopo.c
++++ b/src/util/mpir_hwtopo.c
+@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
+ #ifdef HAVE_HWLOC
+     bindset = hwloc_bitmap_alloc();
+     hwloc_topology_init(&hwloc_topology);
+-    char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
+-    if (xmlfile != NULL) {
+-        int rc;
+-        rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
+-        if (rc == 0) {
+-            /* To have hwloc still actually call OS-specific hooks, the
+-             * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
+-             * file is really the underlying system. */
+-            hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
+-        }
+-        MPL_free(xmlfile);
+-    }
+
+     hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
+     if (!hwloc_topology_load(hwloc_topology)) 
+
+--- a/src/mpi/init/local_proc_attrs.c
++++ b/src/mpi/init/local_proc_attrs.c
+@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
+     /* Set the number of tag bits. The device may override this value. */
+     MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
+
+-    char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
+-    MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
+-    MPL_free(requested_kinds);
+-
+     return mpi_errno;
+ }

pkgs/overlay.nix

@@ -1,35 +1,45 @@
 final: prev:
 {
-  bsc = prev.bsc.extend (bscFinal: bscPrev: {
-    # Set MPICH as default
-    mpi = bscFinal.mpich;
+  # Set MPICH as default
+  mpi = final.mpich;
 
-    # Configure the network for MPICH
-    mpich = with final; prev.mpich.overrideAttrs (old: {
-      buildInput = old.buildInputs ++ [
-        libfabric
-        pmix
-      ];
-      configureFlags = [
-        "--enable-shared"
-        "--enable-sharedlib"
-        "--with-pm=no"
-        "--with-device=ch4:ofi"
-        "--with-pmi=pmix"
-        "--with-pmix=${final.pmix}"
-        "--with-libfabric=${final.libfabric}"
-        "--enable-g=log"
-      ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
-        "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
-        "FCFLAGS=-fallow-argument-mismatch"
-      ];
-    });
+  # Configure the network for MPICH
+  mpich = with final; let
+    # pmix comes with the libraries in .out and headers in .dev
+    pmixAll = symlinkJoin {
+      name = "pmix-all";
+      paths = [ pmix.dev pmix.out ];
+    };
+  in prev.mpich.overrideAttrs (old: {
+    patches = [
+      # See https://github.com/pmodels/mpich/issues/6946
+      ./mpich-fix-hwtopo.patch
+    ];
+    buildInput = old.buildInputs ++ [
+      libfabric
+      pmixAll
+    ];
+    configureFlags = [
+      "--enable-shared"
+      "--enable-sharedlib"
+      "--with-pm=no"
+      "--with-device=ch4:ofi"
+      "--with-pmi=pmix"
+      "--with-pmix=${pmixAll}"
+      "--with-libfabric=${libfabric}"
+      "--enable-g=log"
+    ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
+      "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
+      "FCFLAGS=-fallow-argument-mismatch"
+    ];
   });
 
-  # Update ceph to 18.2.0 until it lands in nixpkgs, see:
-  # https://github.com/NixOS/nixpkgs/pull/247849
-  inherit (prev.callPackage ./ceph.nix {
-    lua = prev.lua5_4;
-    fmt = prev.fmt_8;
-  }) ceph ceph-client;
+  slurm = prev.slurm.overrideAttrs (old: {
+    patches = (old.patches or []) ++ [
+      # See https://bugs.schedmd.com/show_bug.cgi?id=19324
+      ./slurm-rank-expansion.patch
+    ];
+  });
+
+  prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
 }

pkgs/slurm-exporter.nix

@@ -0,0 +1,22 @@
+{ buildGoModule, fetchFromGitHub, lib }:
+
+buildGoModule rec {
+  pname = "prometheus-slurm-exporter";
+  version = "0.20";
+
+  src = fetchFromGitHub {
+    rev = version;
+    owner = "vpenso";
+    repo = pname;
+    sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
+  };
+
+  vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Prometheus SLURM Exporter";
+    homepage = "https://github.com/vpenso/prometheus-slurm-exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/slurm-rank-expansion.patch

@@ -0,0 +1,11 @@
+--- a/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:05:24.815313882 +0100
++++ b/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:09:53.936900823 +0100
+@@ -314,7 +314,7 @@ static void _dmdx_req(buf_t *buf, int no
+ 	}
+ 
+ 	nsptr = pmixp_nspaces_local();
+-	if (nsptr->ntasks <= rank) {
++	if ((long) nsptr->ntasks <= (long) rank) {
+ 		char *nodename = pmixp_info_job_host(nodeid);
+ 		PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
+ 			    nodename, ns, nsptr->ntasks, rank);

secrets/gitea-runner-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg DQdgCk16Yu524BsrWVf0krnwWzDM6SeaJCgQipOfwCA
+Ab9ocqra/UWJZI+QGMlxUhBu5AzqfjPgXl+ENIiHYGs
+-> ssh-ed25519 CAWG4Q KF9rGCenb3nf+wyz2hyVs/EUEbsmUs5R+1fBxlCibC8
+7++Kxbr3FHVdVfnFdHYdAuR0Tgfd+sRcO6WRss6LhEw
+-> ssh-ed25519 MSF3dg aUe4DhRsu4X8CFOEAnD/XM/o/0qHYSB522woCaAVh0I
+GRcs5cm2YqA/lGhUtbpboBaz7mfgiLaCr+agaB7vACU
+--- 9Q7Ou+Pxq+3RZilCb2dKC/pCFjZEt4rp5KnTUUU7WJ8
+1¬
Mw4‘Í	ì:Hµ@Á/ägLtMÇ,ßÆ¥ô*¡žzñNV5ˆm‚ÍNŽoÞáj1$÷TøG_³E{Œ%“‰1ǯ‘<>H£îAÛp™

secrets/gitlab-runner-docker-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg WvKK6U1wQtx2pbUDfuaUIXTQiCulDkz7hgUCSwMfMzQ
+jLktUMqKuVxukqzz++pHOKvmucUQqeKYy5IwBma7KxY
+-> ssh-ed25519 CAWG4Q XKGuNNoYFl9bdZzsqYYTY7GsEt5sypLW4R+1uk78NmU
+8dIA2GzRAwTGM5CDHSM2BUBsbXzEAUssWUz2PY2PaTg
+-> ssh-ed25519 MSF3dg T630RsKuZIF/bp+KITnIIWWHsg6M/VQGqbWQZxqT+AA
+SraZcgZJVtmUzHF/XR9J7aK5t5EDNpkC/av/WJUT/G8
+--- /12G8pj9sbs591OM/ryhoLnSWWmzYcoqprk9uN/3g18
+ä·ù
¼Â‡%å]yi"ô<>»LÓâùH`ªa$Æþ)¦9ve<76>.0úmÉK<EFBFBD>vƒÀïu"|1cÞ-%ÔÕ"åWFï¡ÞA«<41>hº$•ºj<eñ¶xÅLx«ç.?œÈâ:L…¬–ƒ,ëu»|³‹F|Õi²äÔ

secrets/jungle-robot-password.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg 3L1Y5upc5qN6fgiFAox5rD/W8n0eQUv5mT39QAdO5Ac
+XkWsmPmzRgHjsvJgsDKJRgHZ7/sBZFmd1Doppj/y390
+-> ssh-ed25519 CAWG4Q v03Qr+fckdIpsxvQG/viKxlF8WNpO4XUe//QcPzH4k0
+afUwi3ccDCRfUxPDdF7ZkoL+0UX1XwqVtiyabDWjVQk
+-> ssh-ed25519 MSF3dg c2hEUk4LslJpiL7v/4UpT8fK7ZiBJ8+uRhZ/vBoRUDE
+YX9EpnJpHo1eDsZtapTVY6jD+81kb588Oik4NoY9jro
+--- LhUkopNtCsyHCLzEYzBFs+vekOkAR4B3VBaiMF/ZF8w
+oË<EFBFBD>…×à»ÂC‚ßHãáàùýy—LØ”ItMèÕåµI×±sMÆ\Í1-±K”ˆ¤‰G:õ™<02>¦
+ÝgáºÙbpF¼Ó¶Í%Y·

secrets/nosv-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 HY2yRg hrdS7Dl/j+u3XVfM79ZJpZSlre9TcD7DTQ+EEAT6kEE
-avUO96P1h7w2BYWgrQ7GpUgdaCV9AZL7eOTTcF9gfro
--> ssh-ed25519 CAWG4Q A5raRY1CAgFYZgoQ92GMyNejYNdHx/7Y6uTS+EjLPWA
-FRFqT2Jz7qRcybaxkQTKHGl797LVXoHpYG4RZSrX/70
--> ssh-ed25519 MSF3dg D+R80Bg7W9AuiOMAqtGFZQl994dRBIegYRLmmTaeZ3o
-BHvZsugRiuZ91b4jk91h30o3eF3hadSnVCwxXge95T8
--> BT/El`a-grease W{nq|Vm )bld 2Nl}4 N$#JGB4t
-oLG+0S1aGfO/ohCfgGmhDhwwLi4H
---- 2I5C+FvBG/K1ZHh7C5QD39feTSLoFGwcTeZAmeILNsI
-¹õW©o÷ ÙÄd;ËÐC¾.¹¡_(“u
G¡€‰#ìvâœgÉ<67>†õõy¹Y‰žl9ŒÈ
¡Ïµ.Œé0x<30>Þ½úN. /ü<>tB×b‡ü¼K¼ì:Q×—È\¹ÀÍT_´»Átxïm’——_JñÞž-š

secrets/secrets.nix

@@ -6,9 +6,11 @@ let
   safe = keys.hostGroup.safe ++ adminsKeys;
 in
 {
-  "ovni-token.age".publicKeys = hut;
-  "nosv-token.age".publicKeys = hut;
+  "gitea-runner-token.age".publicKeys = hut;
+  "gitlab-runner-docker-token.age".publicKeys = hut;
+  "gitlab-runner-shell-token.age".publicKeys = hut;
   "nix-serve.age".publicKeys = hut;
+  "jungle-robot-password.age".publicKeys = hut;
 
   "ceph-user.age".publicKeys = safe;
   "munge-key.age".publicKeys = safe;

web/content/git/_index.md

@@ -0,0 +1,6 @@
+---
+title: "Git"
+description: "Gitea instance"
+---
+
+If you are reading this page, the proxy to the Gitea service is not working.

web/content/intro-nix/_index.md

@@ -1,7 +0,0 @@
----
-title: "Intro to nix"
-date: 2023-09-15
----
-
-Basic introduction to Nix for users of the jungle machines. You should be able
-to access the jungle machines, otherwise [request access](/access).

web/content/intro-nix/ch1.md

@@ -1,100 +0,0 @@
----
-title: "Chapter 1: Packages"
-description: "Here we show where packages come from"
-date: 2023-06-13T19:36:57+02:00
-weight: 1
----
-
-In this chapter we describe where the packages available in the cluster come
-from and how to load them.
-
-## Where packages come from
-
-The packages in the jungle cluster are constructed by *layers*. Each layer
-applies some changes over the previous one:
-
-1. The first layer is [nixpkgs][1], a large repository of packages maintained by
-   the NixOS community. It provides packages like gcc, bash, gcc or the linux
-   kernel.
-
-[1]: https://github.com/NixOS/nixpkgs/
-
-2. The second layer is [bscpkgs][2], it takes the nixpkgs set of packages and
-   expands it by adding custom packages from the BSC such as Nanos6, nOS-V,
-   NODES, ovni or wxparaver.
-
-[2]: https://pm.bsc.es/gitlab/rarias/bscpkgs
-
-3. The third layer is [jungle][3], it takes the extended packages from bscpkgs
-   and configures them for the jungle cluster. For example, we configure MPICH
-   to use the OmniPath network and set it as the default implementation.
-
-[3]: https://pm.bsc.es/gitlab/rarias/jungle
-
-These layers are called *overlays* in Nix and they are the default mechanism
-used to modify the packages. Generally you will use the packages defined in the
-last layer (jungle) but you can define your own additional layer to specify
-custom changes. For example, instead of choosing MPICH, you may want to use
-Intel MPI instead by default.
-
-## Loading packages in an ephemeral shell
-
-You can manually load packages in a *new* shell with `nix shell jungle#<pkg>`,
-for example:
-
-```
-hut% which ovniemu
-ovniemu not found
-hut% nix shell jungle#bsc.ovni
-hut% which ovniemu
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
-hut% exit
-hut%
-```
-
-You can also specify multiple packages by listing them as parameters of `nix
-shell`:
-
-```
-hut% nix shell jungle#bsc.ovni jungle#bsc.osumb
-hut% which osu_bw
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
-hut% exit
-```
-
-Or make the bash (zsh in this case) shell expand them:
-
-```
-hut% echo nix shell jungle#bsc.{ovni,osumb}
-nix shell jungle#bsc.ovni jungle#bsc.osumb
-hut% nix shell jungle#bsc.{ovni,osumb}
-hut% which osu_bw
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
-hut% exit
-```
-
-You can use TAB to see which packages are available:
-
-```
-hut% nix shell jungle#bsc.n<TAB>
-jungle\#bsc.nanos6              jungle\#bsc.nixtools
-jungle\#bsc.nanos6Debug         jungle\#bsc.nix-wrap
-jungle\#bsc.nanos6Git           jungle\#bsc.nodes
-jungle\#bsc.nanos6GlibcxxDebug  jungle\#bsc.nodesGit
-jungle\#bsc.nanos6-icc          jungle\#bsc.nodesRelease
-jungle\#bsc.nanos6-icx          jungle\#bsc.nodesWithOvni
-jungle\#bsc.nanos6Release       jungle\#bsc.nosv
-jungle\#bsc.nix-mn4
-```
-
-Notice that these packages are evaluated at the moment the command is invoked.
-So if you come back a month later and run the same command, you may find that
-the packages have been updated and that could be problematic.
-
-In the next section we will create a new flake that defines the packages of the
-shell and also records the exact version of the packages that we used at the
-evaluation time for future use.
-
-In the [next chapter](../ch2) we will see how to create a permanent shell that
-will retain the same packages even if they are upgraded in the cluster, until we
-decide to upgrade them.

web/content/intro-nix/ch2.md

@@ -1,155 +0,0 @@
----
-title: "Chapter 2: Your first shell"
-date: 2023-09-15
-weight: 2
----
-
-## Creating a shell with flake.nix
-
-First, create an empty git repository where your shells will live:
-
-```txt
-hut% mkdir jungle-examples
-hut% cd jungle-examples
-hut% git init
-Initialized empty Git repository in /home/Computational/rarias/jungle-examples/.git/
-```
-
-And then, place a file named `flake.nix` on the repo with this content:
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovni osumb # other packages here...
-      ];
-    };
-  };
-}
-```
-
-This file defines a how to create a shell in the Nix language with the
-`pkgs.mkShell` function using the packages listed in `buildInputs`. It also
-requests the packages to be taken from the *jungle* input, which corresponds to
-the set of packages that [we defined earlier](../ch1#where-packages-come-from),
-tuned for the cluster. We will describe it in more detail later.
-
-The tool `nix develop` tries to find a flake.nix in the current directory and
-enter the shell described by `devShells.x86_64-linux.default` (or the
-corresponding architecture).
-
-Now, **it is important that all the files of the repository are committed in
-git**, as nix will only read what is in the index of git. If we try to enter the
-shell with the `nix develop` command, it will complain and fail:
-
-```txt
-hut% nix develop
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-error: getting status of '/nix/store/0ccnxa25whszw7mgbgyzdm4nqc0zwnm8-source/flake.nix': No such file or directory
-```
-
-The first warning states that the git directory has modified files not added to
-the index. Then the error occurs because the flake.nix is not in the index of
-git, so `nix develop` doesn't see it. So let's add it to a commit and try again:
-
-```txt
-hut% git add flake.nix
-hut% git commit flake.nix -m 'First shell'
-[master (root-commit) eb8a4ac] First shell
- 1 file changed, 13 insertions(+)
- create mode 100644 flake.nix
-hut% nix develop
-warning: creating lock file '/home/Computational/rarias/jungle-examples/flake.lock'
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-[nix-develop]$
-```
-
-In the `flake.nix` we have set the shell prompt to `[nix-develop]` so we can
-easily spot that we are inside a `nix develop` shell. To exit:
-
-```txt
-[nix-develop]$ exit
-hut%
-```
-
-## Using the flake.lock file
-
-Now we see the `creating lock file` message and the git tree becomes dirty
-again (however, we enter the shell successfully).
-
-This `flake.lock` file that has been created collects the current state of the
-jungle packages in a file, so future invocations will use the same versions. We
-can see more details with `nix flake metadata`:
-
-```txt
-hut% nix flake metadata
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-Resolved URL:  git+file:///home/Computational/rarias/jungle-examples
-Locked URL:    git+file:///home/Computational/rarias/jungle-examples
-Path:          /nix/store/bckxqjkkv52hy4pzgb96r7fchhmvmql8-source
-Revision:      eb8a4ac544a74e3995d859c751e9ff4339de6509-dirty
-Last modified: 2023-09-15 13:06:12
-Inputs:
-└───jungle: path:/nix/store/3wv6q0f3pkgw840nnkn4jsp9xi650dyj-source?lastModified=1694772033&narHash=sha256-7a09O0Jb8WncxeB32ywmQEMqJdEFLrOG/XVT9bdII6I%3D&rev=653d411b9e46076a7878be9574ed6b3bd627cff1&revCount=195
-    ├───agenix: github:ryantm/agenix/d8c973fd228949736dedf61b7f8cc1ece3236792
-    │   ├───darwin: github:lnl7/nix-darwin/87b9d090ad39b25b2400029c64825fc2a8868943
-    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
-    │   ├───home-manager: github:nix-community/home-manager/32d3e39c491e2f91152c84f8ad8b003420eab0a1
-    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
-    │   └───nixpkgs follows input 'jungle/nixpkgs'
-    ├───bscpkgs: git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=3a4062ac04be6263c64a481420d8e768c2521b80
-    │   └───nixpkgs follows input 'jungle/nixpkgs'
-    └───nixpkgs: github:NixOS/nixpkgs/e56990880811a451abd32515698c712788be5720
-```
-
-Now, as long as we keep these two files `flake.nix` and `flake.lock`, we can
-reproduce the same shell in the future, so let's add the lock file into git too.
-
-```txt
-hut% git commit -m 'Add flake.lock file'
-[master d3725ec] Add flake.lock file
- 1 file changed, 135 insertions(+)
- create mode 100644 flake.lock
-hut% git status
-On branch master
-nothing to commit, working tree clean
-```
-
-## Using the shell with nix develop
-
-Now, the invocations of `nix develop` won't complain that the git tree is dirty
-anymore and will enter the shell:
-
-```txt
-hut% nix develop
-[nix-develop]$
-```
-
-And the requested packages are now available:
-
-```txt
-[nix-develop]$ which ovniemu
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
-```
-
-The packages of the shell are listed in the `$buildInputs` variable, in case you
-need to examine them:
-
-```txt
-[nix-develop]$ printf '%s\n' $buildInputs
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
-[nix-develop]$ exit
-hut%
-```
-
-In the [next chapter](../ch3) we will see how to add more packages and also how to modify
-them.

web/content/intro-nix/ch3.md

@@ -1,160 +0,0 @@
----
-title: "Chapter 3: Custom packages"
-date: 2023-09-15
-weight: 3
----
-
-## Adding more packages
-
-So far we have define all the packages using:
-
-```nix
-pkgs.mkShell rec {
-  pname = "my-shell";
-  buildInputs = with pkgs.bsc; [
-    ovni osumb # other packages here...
-  ];
-};
-```
-
-This line specifies that all packages come from the `pkgs.bsc` set. We can add
-additional packages adding them to the list:
-
-```nix
-pkgs.mkShell rec {
-  pname = "my-shell";
-  buildInputs = with pkgs.bsc; [
-    ovni osumb sonar
-  ];
-};
-```
-
-And running `nix develop` again:
-
-```txt
-hut% nix develop
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-[nix-develop]$ printf '%s\n' $buildInputs
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
-/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0
-```
-
-In the jungle cluster, the default MPI implementation is currently set to MPICH,
-as it can be shown with ldd:
-
-```txt
-[nix-develop]$ ldd $(which ovnisync) | grep mpi
-        libmpi.so.12 => /nix/store/nnnaly6hgylravdrmqkhpx1ndg5p79nc-mpich-4.1.2/lib/libmpi.so.12 (0x00007ffff5200000)
-```
-
-Now, what if we want to replace the MPI implementation by another one?
-
-## Modifying a package
-
-You notice that the packages we are using are coming directly from the ones
-specified in jungle. However, what if we need to modify some option at build
-time or change a dependency?
-
-The Nix language is used to describe how to build each package, and can be
-extended to create derived versions very easily.
-
-Let's focus on the `ovni` package. First, to load the definition we can use the
-`nix edit` command, which opens the definition file using the editor defined in
-`$EDITOR`:
-
-```txt
-hut% nix edit jungle#bsc.ovni
-...
-```
-
-This particular package has several inputs that can be modified directly:
-
-```txt
-{
-  stdenv
-, lib
-, cmake
-, mpi
-, fetchFromGitHub
-, useGit ? false
-, gitBranch ? "master"
-, gitUrl ? "ssh://git@bscpm03.bsc.es/rarias/ovni.git"
-, gitCommit ? "d0a47783f20f8b177a48418966dae45454193a6a"
-, enableDebug ? false
-}:
-...
-```
-
-For example, the `enableDebug` flag, currently set to false, affects how the
-build is configured:
-
-```txt
-cmakeBuildType = if (enableDebug) then "Debug" else "Release";
-```
-
-Now, to change this option we could replace `ovni` for our version:
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovniDebug osumb sonar
-      ];
-    };
-  };
-}
-```
-
-And then, when we now enter the develop shell we can see that ovni gets build
-with the Debug option:
-
-```txt
-hut% nix develop -L
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-ovni> unpacking sources
-ovni> unpacking source archive /nix/store/cz4si0vsw85r9s6dyiqr5ybngh9aympi-source
-ovni> source root is source
-ovni> patching sources
-ovni> updateAutotoolsGnuConfigScriptsPhase
-ovni> configuring
-ovni> fixing cmake files...
-ovni> cmake flags: ... -DCMAKE_BUILD_TYPE=Debug ...
-...
-[nix-develop]$ which ovniver
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver
-
-[nix-develop]$ file $(which ovniver) | fold
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver: ELF 64-bit L
-SB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /nix/st
-ore/9la894yvmmksqlapd4v16wvxpaw3rg70-glibc-2.37-8/lib/ld-linux-x86-64.so.2, for
-GNU/Linux 3.10.0, with debug_info, not stripped
-```
-
-And we see that the ovniver program is now compiled with debug symbols.
-
-However, this *only* replaces the ovni package that we specify in the shell. The
-sonar library also depends on ovni, but that package is still using the old one:
-
-```txt
-[nix-develop]$ find $buildInputs -name 'libovni.so.1'
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/lib/libovni.so.1
-
-[nix-develop]$ find $buildInputs -name 'libsonar-mpi.so'
-/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so
-
-[nix-develop]$ ldd /nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so | grep ovni
-        libovni.so.1 => /nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/lib/libovni.so.1 (0x00007ffff7f8d000)
-```
-
-In the [next chapter](../ch4) we will see how to replace packages in such a way
-that all the dependences are automatically updated too.

web/content/intro-nix/ch4.md

@@ -1,29 +0,0 @@
----
-title: "Chapter 4: Adding an overlay"
-date: 2023-09-15
-weight: 4
----
-
-NOTE: We shouldn't be instructing users to use an overlay to replace packages in
-`bsc.` until we have determined if we move them to the root attribute set
-first!
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovniDebug osumb sonar
-      ];
-    };
-  };
-}
-```

web/content/lists/_index.md

@@ -0,0 +1,6 @@
+---
+title: "Lists"
+description: "Mailing lists"
+---
+
+If you are reading this page, the proxy to the public-inbox service is not working.

web/content/owl/_index.md

@@ -8,3 +8,11 @@ description: "Low system noise"
 Much like the silent flight of an owl at night, these nodes are configured to
 minimize the system noise and let programs run undisturbed. The list of nodes is
 `owl[1-2]` and are available for jobs with SLURM.
+
+The contents of the nix store of the hut node is made available in the owl nodes
+when a job is running. This allows jobs to access the same paths that are on hut
+to load dependencies.
+
+For now, only the hut node can be used to build new derivations so that they
+appear in the compute nodes. This applies to the `nix build`, `nix develop` and
+`nix shell` commands.

web/hugo.toml

@@ -2,7 +2,27 @@ baseURL = 'https://jungle.bsc.es/'
 languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
-sectionPagesMenu = "main"
 
-[params]
-ShowBreadCrumbs = true
+[[menu.main]]
+identifier = "grafana"
+name = "Grafana"
+url = "/grafana/"
+weight = 10
+
+[[menu.main]]
+identifier = "Git"
+name = "Git"
+url = "/git/"
+weight = 20
+
+[[menu.main]]
+identifier = "Lists"
+name = "Lists"
+url = "/lists/"
+weight = 30
+
+[[menu.main]]
+identifier = "Posts"
+name = "Posts"
+url = "/posts/"
+weight = 40

web/themes/PaperMod/layouts/partials/header.html

@@ -100,9 +100,9 @@
                 {{- $separator := or $label_text (not site.Params.disableThemeToggle)}}
                 {{- with site.Home.AllTranslations }}
                 <ul class="lang-switch">
-                    {{- if $separator }}<li>|</li>{{ end }}
                     {{- range . -}}
                     {{- if ne $lang .Lang }}
+                    {{- if $separator }}<li>|</li>{{ end }}
                     <li>
                         <a href="{{- .Permalink -}}" title="{{ .Language.Params.languageAltTitle | default (.Language.LanguageName | emojify) | default (.Lang | title) }}"
                             aria-label="{{ .Language.LanguageName | default (.Lang | title) }}">

web/themes/PaperMod/layouts/partials/post_meta.html

@@ -17,5 +17,5 @@
 {{- end }}
 
 {{- with ($scratch.Get "meta") }}
-{{- delimit . " · " -}}
+{{- delimit . " · " | safeHTML  -}}
 {{- end -}}