Add quickstart guide

Use agenix to store the credentials safely.

.gitignore

@@ -0,0 +1,3 @@
+*.swp
+/result
+/misc

doc/install.md

@@ -0,0 +1,176 @@
+# Installing NixOS in a new node
+
+This article shows the steps to install NixOS in a node following the
+configuration of the repo.
+
+## Enable the serial console
+
+By default, the nodes have the serial console disabled in the GRUB and also boot
+without the serial enabled.
+
+To enable the serial console in the GRUB, set in /etc/default/grub the following
+lines:
+
+```
+GRUB_TERMINAL="console serial"
+GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
+```
+
+To boot Linux with the serial enabled, so you can see the boot log and login via
+serial set:
+
+```
+GRUB_CMDLINE_LINUX="console=ttyS0,115200n8 console=tty0"
+```
+
+Then update the grub config:
+
+```
+# grub2-mkconfig -o /boot/grub2/grub.cfg
+```
+
+And reboot.
+
+## Prepare the disk
+
+Create a main partition and label it `nixos` following [the manual][1].
+
+[1]: https://nixos.org/manual/nixos/stable/index.html#sec-installation-manual-partitioning.
+
+```
+# disk=/dev/sdX
+# parted $disk -- mklabel msdos
+# parted $disk -- mkpart primary 1MB -8GB
+# parted $disk -- mkpart primary linux-swap -8GB 100%
+# parted $disk -- set 1 boot on
+```
+
+Then create an etx4 filesystem, labeled `nixos` where the system will be
+installed. **Ensure that no other partition has the same label.**
+
+```
+# mkfs.ext4 -L nixos "${disk}1"
+# mkswap -L swap "${disk}2"
+# mount ${disk}1 /mnt
+# lsblk -f $disk
+NAME   FSTYPE LABEL UUID                                 MOUNTPOINT
+sdX
+`-sdX1 ext4   nixos 10d73b75-809c-4fa3-b99d-4fab2f0d0d8e /mnt
+```
+
+## Prepare nix and nixos-install
+
+Mount the nix store from the hut node in read-only /nix.
+
+```
+# mkdir /nix
+# mount -o ro hut:/nix /nix
+```
+
+Get the nix binary and nixos-install tool from hut:
+
+```
+# ssh hut 'readlink -f $(which nix)'
+/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin/nix
+# ssh hut 'readlink -f $(which nixos-install)'
+/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/nixos-install
+```
+
+And add them to the PATH:
+
+```
+# export PATH=$PATH:/nix/store/0sxbaj71c4c4n43qhdxm31f56gjalksw-nix-2.13.3/bin
+# export PATH=$PATH:/nix/store/9yq8ps06ysr2pfiwiij39ny56yk3pdcs-nixos-install/bin/
+# nix --version
+nix (Nix) 2.13.3
+```
+
+## Adapt owl configuration
+
+Clone owl repo:
+
+```
+$ git clone git@bscpm03.bsc.es:rarias/owl.git
+$ cd owl
+```
+
+Edit the configuration to your needs.
+
+## Install from another Linux OS
+
+Install nixOS into the storage drive.
+
+```
+# nixos-install --flake --root /mnt .#xeon0X
+```
+
+At this point, the nixOS grub has been installed into the nixos device, which
+is not the default boot device. To keep both the old Linux and NixOS grubs, add
+an entry into the old Linux grub to jump into the new grub.
+
+```
+# echo "
+
+menuentry 'NixOS' {
+    insmod chain
+    search --no-floppy --label nixos --set root
+    configfile /boot/grub/grub.cfg
+} " >> /etc/grub.d/40_custom
+```
+
+Rebuild grub config.
+
+```
+# grub2-mkconfig -o /boot/grub/grub.cfg
+```
+
+To boot into NixOS manually, reboot and select NixOS in the grub menu to boot
+into NixOS.
+
+To temporarily boot into NixOS only on the next reboot run:
+
+```
+# grub2-reboot 'NixOS'
+```
+
+To permanently boot into NixOS as the default boot OS, edit `/etc/default/grub/`:
+
+```
+GRUB_DEFAULT='NixOS'
+```
+
+And update grub.
+
+```
+# grub2-mkconfig -o /boot/grub/grub.cfg
+```
+
+## Build the nixos kexec image
+
+```
+# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
+```
+
+## Chain NixOS in same disk with other systems
+
+To install NixOS on a partition along another system which controls the GRUB,
+first disable the grub device, so the GRUB is not installed in the disk by
+NixOS (only the /boot files will be generated):
+
+```
+boot.loader.grub.device = "nodev";
+```
+
+Then add the following entry to the old GRUB configuration:
+
+```
+menuentry 'NixOS' {
+        insmod chain
+        search --no-floppy --label nixos --set root
+        configfile /boot/grub/grub.cfg
+}
+```
+
+The partition with NixOS must have the label "nixos" for it to be found. New
+system configuration entries will be stored in the GRUB configuration managed
+by NixOS, so there is no need to change the old GRUB settings.

flake.lock

@@ -0,0 +1,130 @@
+{
+  "nodes": {
+    "agenix": {
+      "inputs": {
+        "darwin": "darwin",
+        "home-manager": "home-manager",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1723293904,
+        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "owner": "ryantm",
+        "repo": "agenix",
+        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ryantm",
+        "repo": "agenix",
+        "type": "github"
+      }
+    },
+    "bscpkgs": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1732868163,
+        "narHash": "sha256-qck4h298AgcNI6BnGhEwl26MTLXjumuJVr+9kak7uPo=",
+        "ref": "refs/heads/master",
+        "rev": "6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f",
+        "revCount": 952,
+        "type": "git",
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+      }
+    },
+    "darwin": {
+      "inputs": {
+        "nixpkgs": [
+          "agenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1700795494,
+        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
+        "owner": "lnl7",
+        "repo": "nix-darwin",
+        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "lnl7",
+        "ref": "master",
+        "repo": "nix-darwin",
+        "type": "github"
+      }
+    },
+    "home-manager": {
+      "inputs": {
+        "nixpkgs": [
+          "agenix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1703113217,
+        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "home-manager",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1736867362,
+        "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "agenix": "agenix",
+        "bscpkgs": "bscpkgs",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,36 @@
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    agenix.url = "github:ryantm/agenix";
+    agenix.inputs.nixpkgs.follows = "nixpkgs";
+    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
+    bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, agenix, bscpkgs, ... }:
+let
+  mkConf = name: nixpkgs.lib.nixosSystem {
+    system = "x86_64-linux";
+    specialArgs = { inherit nixpkgs bscpkgs agenix; theFlake = self; };
+    modules = [ "${self.outPath}/m/${name}/configuration.nix" ];
+  };
+in
+  {
+    nixosConfigurations = {
+      hut     = mkConf "hut";
+      owl1    = mkConf "owl1";
+      owl2    = mkConf "owl2";
+      eudy    = mkConf "eudy";
+      koro    = mkConf "koro";
+      bay     = mkConf "bay";
+      lake2   = mkConf "lake2";
+      raccoon = mkConf "raccoon";
+      fox     = mkConf "fox";
+    };
+
+    packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
+      bscpkgs = bscpkgs.packages.x86_64-linux;
+      nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
+    };
+  };
+}

keys.nix

@@ -0,0 +1,30 @@
+# As agenix needs to parse the secrets from a standalone .nix file, we describe
+# here all the public keys
+rec {
+  hosts = {
+    hut   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
+    owl1  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
+    owl2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
+    eudy  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
+    koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
+    bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
+    lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDa9lId4rB/EKGkkCCVOy0cuId2SYLs+8W8kx0kmpO1y fox";
+  };
+
+  hostGroup = with hosts; rec {
+    compute    = [ owl1 owl2 fox ];
+    playground = [ eudy koro ];
+    storage    = [ bay lake2 ];
+    monitor    = [ hut ];
+
+    system     = storage ++ monitor;
+    safe       = system ++ compute;
+    all        = safe ++ playground;
+  };
+
+  admins = {
+    rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
+    root   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
+  };
+}

m/bay/configuration.nix

@@ -0,0 +1,107 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/monitoring.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
+
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
+  environment.systemPackages = with pkgs; [
+    ceph
+  ];
+
+  networking = {
+    hostName = "bay";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.40";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.40";
+      prefixLength = 24;
+    } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from lake2
+        iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
+  };
+
+  services.ceph = {
+    enable = true;
+    global = {
+      fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b";
+      monHost = "10.0.40.40";
+      monInitialMembers = "bay";
+      clusterNetwork = "10.0.40.40/24"; # Use Ethernet only
+    };
+    extraConfig = {
+      # Only log to stderr so it appears in the journal
+      "log_file" = "/dev/null";
+      "mon_cluster_log_file" = "/dev/null";
+      "log_to_stderr" = "true";
+      "err_to_stderr" = "true";
+      "log_to_file" = "false";
+    };
+    mds = {
+      enable = true;
+      daemons = [ "mds0" "mds1" ];
+      extraConfig = {
+        "host" = "bay";
+      };
+    };
+    mgr = {
+      enable = true;
+      daemons = [ "bay" ];
+    };
+    mon = {
+      enable = true;
+      daemons = [ "bay" ];
+    };
+    osd = {
+      enable = true;
+      # One daemon per NVME disk
+      daemons = [ "0" "1" "2" "3" ];
+      extraConfig = {
+        "osd crush chooseleaf type" = "0";
+        "osd journal size" = "10000";
+        "osd pool default min size" = "2";
+        "osd pool default pg num" = "200";
+        "osd pool default pgp num" = "200";
+        "osd pool default size" = "3";
+      };
+    };
+  };
+
+  # Missing service for volumes, see:
+  # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/
+  systemd.services.ceph-volume = {
+    enable = true;
+    description = "Ceph Volume activation";
+    unitConfig = {
+      Type = "oneshot";
+      After = "local-fs.target";
+      Wants = "local-fs.target";
+    };
+    path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ];
+    serviceConfig = {
+      KillMode = "none";
+      Environment = "CEPH_VOLUME_TIMEOUT=10000";
+      ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'";
+      TimeoutSec = "0";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
+}

m/common/base.nix

@@ -0,0 +1,20 @@
+{
+  # All machines should include this profile.
+  # Includes the basic configuration for an Intel server.
+  imports = [
+    ./base/agenix.nix
+    ./base/august-shutdown.nix
+    ./base/boot.nix
+    ./base/env.nix
+    ./base/fs.nix
+    ./base/hw.nix
+    ./base/net.nix
+    ./base/nix.nix
+    ./base/ntp.nix
+    ./base/rev.nix
+    ./base/ssh.nix
+    ./base/users.nix
+    ./base/watchdog.nix
+    ./base/zsh.nix
+  ];
+}

m/common/base/agenix.nix

@@ -0,0 +1,9 @@
+{ agenix, ... }:
+
+{
+  imports = [ agenix.nixosModules.default ];
+
+  environment.systemPackages = [
+    agenix.packages.x86_64-linux.default
+  ];
+}

m/common/base/august-shutdown.nix

@@ -0,0 +1,14 @@
+{
+  # Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
+  # hardware from spurious electrical peaks on the yearly electrical cut for
+  # manteinance that starts on August 4th.
+  systemd.timers.august-shutdown = {
+    description = "Shutdown on August 2nd for maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-08-02 11:00:00";
+      RandomizedDelaySec = "10min";
+      Unit = "systemd-poweroff.service";
+    };
+  };
+}

m/common/base/boot.nix

@@ -0,0 +1,37 @@
+{ lib, pkgs, ... }:
+
+{
+  # Use the GRUB 2 boot loader.
+  boot.loader.grub.enable = true;
+
+  # Enable GRUB2 serial console
+  boot.loader.grub.extraConfig = ''
+    serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+    terminal_input --append serial
+    terminal_output --append serial
+  '';
+
+  boot.kernel.sysctl = {
+    "kernel.perf_event_paranoid" = lib.mkDefault "-1";
+
+    # Allow ptracing (i.e. attach with GDB) any process of the same user, see:
+    # https://www.kernel.org/doc/Documentation/security/Yama.txt
+    "kernel.yama.ptrace_scope" = "0";
+  };
+
+  boot.kernelPackages = pkgs.linuxPackages_latest;
+
+  #boot.kernelPatches = lib.singleton {
+  #  name = "osnoise-tracer";
+  #  patch = null;
+  #  extraStructuredConfig = with lib.kernel; {
+  #    OSNOISE_TRACER = yes;
+  #    HWLAT_TRACER = yes;
+  #  };
+  #};
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "nvme" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+}

m/common/base/env.nix

@@ -0,0 +1,35 @@
+{ pkgs, config, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns
+    # From bsckgs overlay
+    osumb
+  ];
+
+  programs.direnv.enable = true;
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+}

m/common/base/fs.nix

@@ -0,0 +1,24 @@
+{ ... }:
+
+{
+  fileSystems."/" =
+    { device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+
+  # Trim unused blocks weekly
+  services.fstrim.enable = true;
+
+  swapDevices =
+    [ { device = "/dev/disk/by-label/swap"; }
+    ];
+
+  # Tracing
+  fileSystems."/sys/kernel/tracing" = {
+    device = "none";
+    fsType = "tracefs";
+  };
+
+  # Mount a tmpfs into /tmp
+  boot.tmp.useTmpfs = true;
+}

m/common/base/hw.nix

@@ -0,0 +1,14 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

m/common/base/net.nix

@@ -0,0 +1,19 @@
+{ pkgs, ... }:
+
+{
+  networking = {
+    enableIPv6 = false;
+    useDHCP = false;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
+    };
+
+    hosts = {
+      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.51.152" = [ "raccoon" ];
+      "84.88.51.142" = [ "raccoon-ipmi" ];
+    };
+  };
+}

m/common/base/nix.nix

@@ -0,0 +1,57 @@
+{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
+
+{
+  nixpkgs.overlays = [
+    bscpkgs.bscOverlay
+    (import ../../../pkgs/overlay.nix)
+  ];
+
+  nix = {
+    nixPath = [
+      "nixpkgs=${nixpkgs}"
+      "jungle=${theFlake.outPath}"
+    ];
+
+    registry = {
+      nixpkgs.flake = nixpkgs;
+      jungle.flake = theFlake;
+    };
+
+    settings = {
+      experimental-features = [ "nix-command" "flakes" ];
+      sandbox = "relaxed";
+      trusted-users = [ "@wheel" ];
+      flake-registry = pkgs.writeText "global-registry.json"
+        ''{"flakes":[],"version":2}'';
+    };
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 30d";
+    };
+  };
+
+  # The nix-gc.service can begin its execution *before* /home is mounted,
+  # causing it to remove all gcroots considering them as stale, as it cannot
+  # access the symlink. To prevent this problem, we force the service to wait
+  # until /home is mounted as well as other remote FS like /ceph.
+  systemd.services.nix-gc = {
+    # Start remote-fs.target if not already being started and fail if it fails
+    # to start. It will also be stopped if the remote-fs.target fails after
+    # starting successfully.
+    bindsTo = [ "remote-fs.target" ];
+    # Wait until remote-fs.target fully starts before starting this one.
+    after = [ "remote-fs.target"];
+    # Ensure we can access a remote path inside /home
+    unitConfig.ConditionPathExists = "/home/Computational";
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}

m/common/base/ntp.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+
+{
+  services.ntp.enable = true;
+
+  # Use the NTP server at BSC, as we don't have direct access
+  # to the outside world
+  networking.timeServers = [ "84.88.52.36" ];
+}

m/common/base/rev.nix

@@ -0,0 +1,21 @@
+{ theFlake, ... }:
+
+let
+  # Prevent building a configuration without revision
+  rev = if theFlake ? rev then theFlake.rev
+    else throw ("Refusing to build from a dirty Git tree!");
+in {
+  # Save the commit of the config in /etc/configrev
+  environment.etc.configrev.text = rev + "\n";
+
+  # Keep a log with the config over time
+  system.activationScripts.configRevLog.text = ''
+    BOOTED=$(cat /run/booted-system/etc/configrev 2>/dev/null || echo unknown)
+    CURRENT=$(cat /run/current-system/etc/configrev 2>/dev/null || echo unknown)
+    NEXT=${rev}
+    DATENOW=$(date --iso-8601=seconds)
+    echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
+  '';
+
+  system.configurationRevision = rev;
+}

m/common/base/ssh.nix

@@ -0,0 +1,28 @@
+{ lib, ... }:
+
+let
+  keys = import ../../../keys.nix;
+  hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
+in
+{
+  # Enable the OpenSSH daemon.
+  services.openssh.enable = true;
+
+  # Connect to intranet git hosts via proxy
+  programs.ssh.extraConfig = ''
+    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
+      User git
+      ProxyCommand nc -X connect -x hut:23080 %h %p
+
+    # Connect to BSC machines via hut proxy too
+    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+      ProxyCommand nc -X connect -x hut:23080 %h %p
+  '';
+
+  programs.ssh.knownHosts = hostsKeys // {
+    "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
+    "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
+    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
+    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
+  };
+}

m/common/base/users.nix

@@ -0,0 +1,135 @@
+{ pkgs, ... }:
+
+{
+  imports = [
+    ../../module/jungle-users.nix
+  ];
+
+  users = {
+    mutableUsers = false;
+    users = {
+      # Generate hashedPassword with `mkpasswd -m sha-512`
+
+      root.openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut"
+      ];
+
+      rarias = {
+        uid = 1880;
+        isNormalUser = true;
+        home = "/home/Computational/rarias";
+        description = "Rodrigo Arias";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
+        ];
+        shell = pkgs.zsh;
+      };
+
+      arocanon = {
+        uid = 1042;
+        isNormalUser = true;
+        home = "/home/Computational/arocanon";
+        description = "Aleix Roca";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
+        ];
+      };
+    };
+
+    jungleUsers = {
+      rpenacob = {
+        uid = 2761;
+        isNormalUser = true;
+        home = "/home/Computational/rpenacob";
+        description = "Raúl Peñacoba";
+        group = "Computational";
+        hosts = [ "owl1" "owl2" "hut" ];
+        hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
+        ];
+      };
+
+      anavarro = {
+        uid = 1037;
+        isNormalUser = true;
+        home = "/home/Computational/anavarro";
+        description = "Antoni Navarro";
+        group = "Computational";
+        hosts = [ "hut" "raccoon" "fox" ];
+        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
+        ];
+      };
+
+      abonerib = {
+        uid = 4541;
+        isNormalUser = true;
+        home = "/home/Computational/abonerib";
+        description = "Aleix Boné";
+        group = "Computational";
+        hosts = [ "owl1" "owl2" "hut" "raccoon" ];
+        hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
+        ];
+      };
+
+      vlopez = {
+        uid = 4334;
+        isNormalUser = true;
+        home = "/home/Computational/vlopez";
+        description = "Victor López";
+        group = "Computational";
+        hosts = [ "koro" ];
+        hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
+        ];
+      };
+
+      dbautist = {
+        uid = 5649;
+        isNormalUser = true;
+        home = "/home/Computational/dbautist";
+        description = "Dylan Bautista Cases";
+        group = "Computational";
+        hosts = [ "hut" ];
+        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
+        ];
+      };
+
+      dalvare1 = {
+        uid = 2758;
+        isNormalUser = true;
+        home = "/home/Computational/dalvare1";
+        description = "David Álvarez";
+        group = "Computational";
+        hosts = [ "hut" "fox" ];
+        hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
+        ];
+      };
+    };
+
+    groups = {
+      Computational = { gid = 564; };
+    };
+  };
+}

m/common/base/watchdog.nix

@@ -0,0 +1,9 @@
+{ ... }:
+
+{
+  # The boards have a BMC watchdog controlled by IPMI
+  boot.kernelModules = [ "ipmi_watchdog" ];
+
+  # Enable systemd watchdog with 30 s interval
+  systemd.watchdog.runtimeTime = "30s";
+}

m/common/base/zsh.nix

@@ -0,0 +1,91 @@
+{ pkgs, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    zsh-completions
+    nix-zsh-completions
+  ];
+
+  programs.zsh = {
+    enable = true;
+    histSize = 1000000;
+
+    shellInit = ''
+      # Disable new user prompt
+      if [ ! -e ~/.zshrc ]; then
+        touch ~/.zshrc
+      fi
+    '';
+
+    promptInit = ''
+      # Note that to manually override this in ~/.zshrc you should run `prompt off`
+      # before setting your PS1 and etc. Otherwise this will likely to interact with
+      # your ~/.zshrc configuration in unexpected ways as the default prompt sets
+      # a lot of different prompt variables.
+      autoload -U promptinit && promptinit && prompt default && setopt prompt_sp
+    '';
+
+    # Taken from Ulli Kehrle config:
+    # https://git.hrnz.li/Ulli/nixos/src/commit/2e203b8d8d671f4e3ced0f1744a51d5c6ee19846/profiles/shell.nix#L199-L205
+    interactiveShellInit = ''
+      source "${pkgs.zsh-history-substring-search}/share/zsh-history-substring-search/zsh-history-substring-search.zsh"
+
+      # Save history immediately, but only load it when the shell starts
+      setopt inc_append_history
+
+      # dircolors doesn't support alacritty:
+      # https://lists.gnu.org/archive/html/bug-coreutils/2019-05/msg00029.html
+      export LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=00:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.avif=01;35:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:*~=00;90:*#=00;90:*.bak=00;90:*.old=00;90:*.orig=00;90:*.part=00;90:*.rej=00;90:*.swp=00;90:*.tmp=00;90:*.dpkg-dist=00;90:*.dpkg-old=00;90:*.ucf-dist=00;90:*.ucf-new=00;90:*.ucf-old=00;90:*.rpmnew=00;90:*.rpmorig=00;90:*.rpmsave=00;90:';
+
+      # From Arch Linux and GRML
+      bindkey "^R" history-incremental-pattern-search-backward
+      bindkey "^S" history-incremental-pattern-search-forward
+
+      # Auto rehash for new binaries
+      zstyle ':completion:*' rehash true
+      # show a nice menu with the matches
+      zstyle ':completion:*' menu yes select
+
+      bindkey '^[OA' history-substring-search-up   # Up
+      bindkey '^[[A' history-substring-search-up   # Up
+
+      bindkey '^[OB' history-substring-search-down # Down
+      bindkey '^[[B' history-substring-search-down # Down
+
+      bindkey '\e[1~' beginning-of-line            # Home
+      bindkey '\e[7~' beginning-of-line            # Home
+      bindkey '\e[H'  beginning-of-line            # Home
+      bindkey '\eOH'  beginning-of-line            # Home
+
+      bindkey '\e[4~' end-of-line                  # End
+      bindkey '\e[8~' end-of-line                  # End
+      bindkey '\e[F ' end-of-line                  # End
+      bindkey '\eOF'  end-of-line                  # End
+
+      bindkey '^?'    backward-delete-char         # Backspace
+      bindkey '\e[3~' delete-char                  # Del
+      # bindkey '\e[3;5~' delete-char                # sometimes Del, sometimes C-Del
+      bindkey '\e[2~' overwrite-mode               # Ins
+
+      bindkey '^H'      backward-kill-word         # C-Backspace
+
+      bindkey '5~'      kill-word                  # C-Del
+      bindkey '^[[3;5~' kill-word                  # C-Del
+      bindkey '^[[3^'   kill-word                  # C-Del
+
+      bindkey "^[[1;5H" backward-kill-line         # C-Home
+      bindkey "^[[7^"   backward-kill-line         # C-Home
+
+      bindkey "^[[1;5F" kill-line                  # C-End
+      bindkey "^[[8^"   kill-line                  # C-End
+
+      bindkey '^[[1;5C' forward-word               # C-Right
+      bindkey '^[0c'    forward-word               # C-Right
+      bindkey '^[[5C'   forward-word               # C-Right
+
+      bindkey '^[[1;5D' backward-word              # C-Left
+      bindkey '^[0d'    backward-word              # C-Left
+      bindkey '^[[5D'   backward-word              # C-Left
+    '';
+  };
+}

m/common/xeon.nix

@@ -0,0 +1,9 @@
+{
+  # Provides the base system for a xeon node.
+  imports = [
+    ./base.nix
+    ./xeon/fs.nix
+    ./xeon/console.nix
+    ./xeon/net.nix
+  ];
+}

m/common/xeon/console.nix

@@ -0,0 +1,14 @@
+{
+  # Restart the serial console
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+}

m/common/xeon/fs.nix

@@ -1,5 +1,3 @@
-{ ... }:
-
 {
   # Mount the home via NFS
   fileSystems."/home" = {
@@ -7,10 +5,4 @@
     fsType = "nfs";
     options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
   };
-
-  # Tracing
-  fileSystems."/sys/kernel/tracing" = {
-    device = "none";
-    fsType = "tracefs";
-  };
 }

m/common/xeon/net.nix

@@ -0,0 +1,94 @@
+{ pkgs, ... }:
+
+{
+  # Infiniband (IPoIB)
+  environment.systemPackages = [ pkgs.rdma-core ];
+  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
+
+  networking = {
+    defaultGateway = "10.0.40.30";
+    nameservers = ["8.8.8.8"];
+
+    proxy = {
+      default = "http://hut:23080/";
+      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
+      # Don't set all_proxy as go complains and breaks the gitlab runner, see:
+      # https://github.com/golang/go/issues/16715
+      allProxy = null;
+    };
+
+    firewall = {
+      extraCommands = ''
+        # Prevent ssfhead from contacting our slurmd daemon
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
+        # But accept traffic to slurm ports from any other node in the subnet
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
+        # We also need to open the srun port range
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+      '';
+    };
+
+    extraHosts = ''
+      10.0.40.30              ssfhead
+      
+      # Node Entry for node: mds01 (ID=72)
+      10.0.40.40              bay mds01 mds01-eth0
+      10.0.42.40              bay-ib mds01-ib0
+      10.0.40.141             bay-ipmi mds01-ipmi0 mds01-ipmi
+      
+      # Node Entry for node: oss01 (ID=73)
+      10.0.40.41              oss01 oss01-eth0
+      10.0.42.41              oss01-ib0
+      10.0.40.142             oss01-ipmi0 oss01-ipmi
+      
+      # Node Entry for node: oss02 (ID=74)
+      10.0.40.42              lake2 oss02 oss02-eth0
+      10.0.42.42              lake2-ib oss02-ib0
+      10.0.40.143             lake2-ipmi oss02-ipmi0 oss02-ipmi
+      
+      # Node Entry for node: xeon01 (ID=15)
+      10.0.40.1               owl1 xeon01 xeon01-eth0
+      10.0.42.1               owl1-ib xeon01-ib0
+      10.0.40.101             owl1-ipmi xeon01-ipmi0 xeon01-ipmi
+      
+      # Node Entry for node: xeon02 (ID=16)
+      10.0.40.2               owl2 xeon02 xeon02-eth0
+      10.0.42.2               owl2-ib xeon02-ib0
+      10.0.40.102             owl2-ipmi xeon02-ipmi0 xeon02-ipmi
+      
+      # Node Entry for node: xeon03 (ID=17)
+      10.0.40.3               xeon03 xeon03-eth0
+      10.0.42.3               xeon03-ib0
+      10.0.40.103             xeon03-ipmi0 xeon03-ipmi
+      
+      # Node Entry for node: xeon04 (ID=18)
+      10.0.40.4               xeon04 xeon04-eth0
+      10.0.42.4               xeon04-ib0
+      10.0.40.104             xeon04-ipmi0 xeon04-ipmi
+      
+      # Node Entry for node: xeon05 (ID=19)
+      10.0.40.5               koro xeon05 xeon05-eth0
+      10.0.42.5               koro-ib xeon05-ib0
+      10.0.40.105             koro-ipmi xeon05-ipmi0
+      
+      # Node Entry for node: xeon06 (ID=20)
+      10.0.40.6               xeon06 xeon06-eth0
+      10.0.42.6               xeon06-ib0
+      10.0.40.106             xeon06-ipmi0 xeon06-ipmi
+      
+      # Node Entry for node: xeon07 (ID=21)
+      10.0.40.7               hut xeon07 xeon07-eth0
+      10.0.42.7               hut-ib xeon07-ib0
+      10.0.40.107             hut-ipmi xeon07-ipmi0 xeon07-ipmi
+      
+      # Node Entry for node: xeon08 (ID=22)
+      10.0.40.8               eudy xeon08 xeon08-eth0
+      10.0.42.8               eudy-ib xeon08-ib0
+      10.0.40.108             eudy-ipmi xeon08-ipmi0 xeon08-ipmi
+
+      # fox
+      10.0.40.26              fox
+      10.0.40.126             fox-ipmi
+    '';
+  };
+}

m/eudy/configuration.nix

@@ -0,0 +1,37 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    #(modulesPath + "/installer/netboot/netboot-minimal.nix")
+
+    ./kernel/kernel.nix
+    ./cpufreq.nix
+    ./fs.nix
+    ./users.nix
+    ../module/debuginfod.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53564b";
+
+  # disable automatic garbage collector
+  nix.gc.automatic = lib.mkForce false;
+
+  # members of the tracing group can use the lttng-provided kernel events
+  # without root permissions
+  users.groups.tracing.members = [ "arocanon" ];
+
+  # set up both ethernet and infiniband ips
+  networking = {
+    hostName = "eudy";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.8";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.8";
+      prefixLength = 24;
+    } ];
+  };
+}

m/eudy/cpufreq.nix

@@ -0,0 +1,40 @@
+{ lib, ... }:
+
+{
+  # Disable frequency boost by default. Use the intel_pstate driver instead of
+  # acpi_cpufreq driver because the acpi_cpufreq driver does not read the
+  # complete range of P-States [1]. Use the intel_pstate passive mode [2] to
+  # disable HWP, which allows a core to "select P-states by itself". Also, this
+  # disables intel governors, which confusingly, have the same names as the
+  # generic ones but behave differently [3].
+
+  # Essentially, we use the generic governors, but use the intel driver to read
+  # the P-state list.
+
+  # [1] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#intel-pstate-vs-acpi-cpufreq
+  # [2] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#passive-mode
+  # [3] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#active-mode
+  # https://www.kernel.org/doc/html/latest/admin-guide/pm/cpufreq.html
+
+  # set intel_pstate to passive mode
+  boot.kernelParams = [
+    "intel_pstate=passive"
+  ];
+  # Disable frequency boost
+  system.activationScripts = {
+    disableFrequencyBoost.text = ''
+      echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo
+    '';
+  };
+
+  ## disable intel_pstate
+  #boot.kernelParams = [
+  #  "intel_pstate=disable"
+  #];
+  ## Disable frequency boost
+  #system.activationScripts = {
+  #  disableFrequencyBoost.text = ''
+  #    echo 0 > /sys/devices/system/cpu/cpufreq/boost
+  #  '';
+  #};
+}

m/eudy/fs.nix

@@ -0,0 +1,13 @@
+{ ... }:
+
+{
+  fileSystems."/nix" = {
+    device = "/dev/disk/by-label/optane";
+    fsType = "ext4";
+    neededForBoot = true;
+  };
+  fileSystems."/mnt/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "ext4";
+  };
+}

m/eudy/kernel/kernel.nix

@@ -0,0 +1,70 @@
+{ pkgs, lib, ... }:
+
+let
+  #fcs-devel = pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = /mnt/data/kernel/fcs/kernel/src;
+  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
+  #};
+
+  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
+  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
+  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
+  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
+  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = builtins.fetchGit {
+  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+  #     rev = gitCommit;
+  #     ref = "fcs";
+  #   };
+  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
+  #};
+
+  kernel = nixos-fcs;
+
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+    version = "6.2.8";
+    src = builtins.fetchGit {
+      url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+      rev = gitCommit;
+      ref = branch;
+    };
+    structuredExtraConfig = with lib.kernel; {
+      # add general custom kernel options here
+    } // lib.optionalAttrs lockStat {
+      LOCK_STAT = yes;
+    } // lib.optionalAttrs preempt {
+      PREEMPT = lib.mkForce yes;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+    };
+    kernelPatches = [];
+    extraMeta.branch = lib.versions.majorMinor version;
+  }));
+
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
+    lockStat = true;
+  };
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
+    lockStat = true;
+    preempt = true;
+  };
+  latest = pkgs.linuxPackages_latest;
+
+in {
+  imports = [
+    ./lttng.nix
+    ./perf.nix
+  ];
+  boot.kernelPackages = lib.mkForce kernel;
+
+  # disable all cpu mitigations
+  boot.kernelParams = [
+    "mitigations=off"
+  ];
+  
+  # enable memory overcommit, needed to build a taglibc system using nix after
+  # increasing the openblas memory footprint
+  boot.kernel.sysctl."vm.overcommit_memory" = 1;
+}

m/eudy/kernel/lttng.nix

@@ -0,0 +1,43 @@
+{ config, pkgs, lib, ... }:
+
+let
+
+  # The lttng btrfs probe crashes at compile time because of an undefined
+  # function. This disables the btrfs tracepoints to avoid the issue.
+
+  # Also enable lockdep tracepoints, this is disabled by default because it
+  # does not work well on architectures other than x86_64 (i think that arm) as
+  # I was told on the mailing list.
+  lttng-modules-fixed = config.boot.kernelPackages.lttng-modules.overrideAttrs (finalAttrs: previousAttrs: {
+    patchPhase = (lib.optionalString (previousAttrs ? patchPhase) previousAttrs.patchPhase) + ''
+      # disable btrfs
+      substituteInPlace src/probes/Kbuild \
+        --replace "  obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" "  #obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o"
+
+      # enable lockdep tracepoints
+      substituteInPlace src/probes/Kbuild \
+        --replace "#ifneq (\$(CONFIG_LOCKDEP),)"                  "ifneq (\$(CONFIG_LOCKDEP),)" \
+        --replace "#  obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" "  obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" \
+        --replace "#endif # CONFIG_LOCKDEP"                       "endif # CONFIG_LOCKDEP"
+    '';
+  });
+in {
+
+  # add the lttng tools and modules to the system environment
+  boot.extraModulePackages = [ lttng-modules-fixed ];
+  environment.systemPackages = with pkgs; [
+    lttng-tools lttng-ust babeltrace
+  ];
+
+  # start the lttng root daemon to manage kernel events
+  systemd.services.lttng-sessiond = {
+    wantedBy = [ "multi-user.target" ];
+    description = "LTTng session daemon for the root user";
+    serviceConfig = {
+      User = "root";
+      ExecStart = ''
+        ${pkgs.lttng-tools}/bin/lttng-sessiond
+      '';
+    };
+  };
+}

m/eudy/kernel/perf.nix

@@ -0,0 +1,22 @@
+{ config, pkgs, lib, ... }:
+
+{
+  # add the perf tool
+  environment.systemPackages = with pkgs; [
+    config.boot.kernelPackages.perf
+  ];
+
+  # allow non-root users to read tracing data from the kernel
+  boot.kernel.sysctl."kernel.perf_event_paranoid" = -2;
+  boot.kernel.sysctl."kernel.kptr_restrict" = 0;
+
+  # specify additionl options to the tracefs directory to allow members of the
+  # tracing group to access tracefs.
+  fileSystems."/sys/kernel/tracing" = {
+    options = [
+      "mode=755"
+      "gid=tracing"
+    ];
+  };
+}
+

m/eudy/users.nix

@@ -0,0 +1,11 @@
+{ ... }:
+
+{
+  security.sudo.extraRules= [{
+    users = [ "arocanon" ];
+    commands = [{
+      command = "ALL" ;
+      options= [ "NOPASSWD" ]; # "SETENV" # Adding the following could be a good idea
+    }];
+  }];
+}

m/fox/configuration.nix

@@ -0,0 +1,75 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
+    ../module/slurm-firewall.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.kernelModules = [ "kvm-amd" ];
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+  hardware.cpu.intel.updateMicrocode = lib.mkForce false;
+
+  networking = {
+    hostName = "fox";
+    interfaces.enp1s0f0np0.ipv4.addresses = [ {
+      address = "10.0.40.26";
+      prefixLength = 24;
+    } ];
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.graphics.enable = true;
+  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  # Mount NVME disks
+  fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
+  fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
+
+  # Make a /nvme{0,1}/$USER directory for each user.
+  systemd.services.create-nvme-dirs = let
+    # Take only normal users in fox
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /nvme{0,1}/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-nvme-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+
+  # Only allow SSH connections from users who have a SLURM allocation
+  # See: https://slurm.schedmd.com/pam_slurm_adopt.html
+  security.pam.services.sshd.rules.account.slurm = {
+    control = "required";
+    enable = true;
+    modulePath = "${pkgs.slurm}/lib/security/pam_slurm_adopt.so";
+    args = [ "log_level=debug5" ];
+    order = 999999; # Make it last one
+  };
+
+  # Disable systemd session (pam_systemd.so) as it will conflict with the
+  # pam_slurm_adopt.so module. What happens is that the shell is first adopted
+  # into the slurmstepd task and then into the systemd session, which is not
+  # what we want, otherwise it will linger even if all jobs are gone.
+  security.pam.services.sshd.startSession = lib.mkForce false;
+}

m/hut/blackbox.yml

@@ -0,0 +1,162 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      proxy_url: "http://127.0.0.1:23080"
+      skip_resolve_phase_with_proxy: true
+      follow_redirects: true
+      valid_status_codes: []  # Defaults to 2xx
+      method: GET
+  http_with_proxy:
+    prober: http
+    http:
+      proxy_url: "http://127.0.0.1:3128"
+      skip_resolve_phase_with_proxy: true
+  http_with_proxy_and_headers:
+    prober: http
+    http:
+      proxy_url: "http://127.0.0.1:3128"
+      proxy_connect_header:
+        Proxy-Authorization:
+          - Bearer token
+  http_post_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      headers:
+        Content-Type: application/json
+      body: '{}'
+  http_post_body_file:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      body_file: "/files/body.txt"
+  http_basic_auth_example:
+    prober: http
+    timeout: 5s
+    http:
+      method: POST
+      headers:
+        Host: "login.example.com"
+      basic_auth:
+        username: "username"
+        password: "mysecret"
+  http_2xx_oauth_client_credentials:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+      valid_status_codes:
+        - 200
+        - 201
+      oauth2:
+        client_id: "client_id"
+        client_secret: "client_secret"
+        token_url: "https://api.example.com/token"
+        endpoint_params:
+          grant_type: "client_credentials"
+  http_custom_ca_example:
+    prober: http
+    http:
+      method: GET
+      tls_config:
+        ca_file: "/certs/my_cert.crt"
+  http_gzip:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+  http_gzip_with_accept_encoding:
+    prober: http
+    http:
+      method: GET
+      compression: gzip
+      headers:
+        Accept-Encoding: gzip
+  tls_connect:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      tls: true
+  tcp_connect_example:
+    prober: tcp
+    timeout: 5s
+  imap_starttls:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - expect: "OK.*STARTTLS"
+        - send: ". STARTTLS"
+        - expect: "OK"
+        - starttls: true
+        - send: ". capability"
+        - expect: "CAPABILITY IMAP4rev1"
+  smtp_starttls:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - expect: "^220 ([^ ]+) ESMTP (.+)$"
+        - send: "EHLO prober\r"
+        - expect: "^250-STARTTLS"
+        - send: "STARTTLS\r"
+        - expect: "^220"
+        - starttls: true
+        - send: "EHLO prober\r"
+        - expect: "^250-AUTH"
+        - send: "QUIT\r"
+  irc_banner_example:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+        - send: "NICK prober"
+        - send: "USER prober prober prober :prober"
+        - expect: "PING :([^ ]+)"
+          send: "PONG ${1}"
+        - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"
+  dns_udp_example:
+    prober: dns
+    timeout: 5s
+    dns:
+      query_name: "www.prometheus.io"
+      query_type: "A"
+      valid_rcodes:
+        - NOERROR
+      validate_answer_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+        fail_if_all_match_regexp:
+          - ".*127.0.0.1"
+        fail_if_not_matches_regexp:
+          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
+        fail_if_none_matches_regexp:
+          - "127.0.0.1"
+      validate_authority_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+      validate_additional_rrs:
+        fail_if_matches_regexp:
+          - ".*127.0.0.1"
+  dns_soa:
+    prober: dns
+    dns:
+      query_name: "prometheus.io"
+      query_type: "SOA"
+  dns_tcp_example:
+    prober: dns
+    dns:
+      transport_protocol: "tcp" # defaults to "udp"
+      preferred_ip_protocol: "ip4" # defaults to "ip6"
+      query_name: "www.prometheus.io"

m/hut/configuration.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+
+    ../module/ceph.nix
+    ../module/debuginfod.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
+    ./gitlab-runner.nix
+    ./monitoring.nix
+    ./nfs.nix
+    ./slurm-server.nix
+    ./nix-serve.nix
+    ./public-inbox.nix
+    ./gitea.nix
+    ./msmtp.nix
+    ./postgresql.nix
+    ./nginx.nix
+    ./p.nix
+    #./pxe.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
+
+  fileSystems = {
+    "/" = lib.mkForce {
+      device = "/dev/disk/by-label/nvme";
+      fsType = "ext4";
+      neededForBoot = true;
+      options = [ "noatime" ];
+    };
+
+    "/boot" = lib.mkForce {
+      device = "/dev/disk/by-label/nixos-boot";
+      fsType = "ext4";
+      neededForBoot = true;
+    };
+  };
+
+  networking = {
+    hostName = "hut";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.7";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.7";
+      prefixLength = 24;
+    } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all proxy traffic from compute nodes but not the login
+        iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
+      '';
+    };
+  };
+
+  # Allow proxy to bind to the ethernet interface
+  services.openssh.settings.GatewayPorts = "clientspecified";
+}

m/hut/gitea.nix

@@ -0,0 +1,63 @@
+{ config, lib, ... }:
+{
+  age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
+
+  services.gitea = {
+    enable = true;
+    appName = "Gitea in the jungle";
+
+    settings = {
+      server = {
+        ROOT_URL = "https://jungle.bsc.es/git/";
+        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
+        LANDING_PAGE = "explore";
+      };
+      metrics.ENABLED = true;
+      service = {
+        REGISTER_MANUAL_CONFIRM = true;
+        ENABLE_NOTIFY_MAIL = true;
+      };
+      log.LEVEL = "Warn";
+
+      mailer = {
+        ENABLED       = true;
+        FROM          = "jungle-robot@bsc.es";
+        PROTOCOL      = "sendmail";
+        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
+        SENDMAIL_ARGS = "--";
+      };
+    };
+  };
+
+  services.gitea-actions-runner.instances = {
+    runrun = {
+      enable = true;
+      name = "runrun";
+      url = "https://jungle.bsc.es/git/";
+      tokenFile = config.age.secrets.giteaRunnerToken.path;
+      labels = [ "native:host" ];
+      settings.runner.capacity = 8;
+    };
+  };
+
+  systemd.services.gitea-runner-runrun = {
+    path = [ "/run/current-system/sw" ];
+    serviceConfig = {
+      # DynamicUser doesn't work well with SSH
+      DynamicUser = lib.mkForce false;
+      User = "gitea-runner";
+      Group = "gitea-runner";
+    };
+  };
+
+  users.users.gitea-runner = {
+    isSystemUser = true;
+    home = "/var/lib/gitea-runner";
+    description = "Gitea Runner";
+    group = "gitea-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitea-runner = {};
+}
+

m/hut/gitlab-runner.nix

@@ -0,0 +1,112 @@
+{ pkgs, lib, config, ... }:
+
+{
+  age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age;
+  age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age;
+
+  services.gitlab-runner = {
+    enable = true;
+    settings.concurrent = 5;
+    services = let
+      common-shell = {
+        executor = "shell";
+        environmentVariables = {
+          SHELL = "${pkgs.bash}/bin/bash";
+        };
+      };
+      common-docker = {
+        executor = "docker";
+        dockerImage = "debian:stable";
+        registrationFlags = [
+          "--docker-network-mode host"
+        ];
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
+      };
+    in {
+      # For pm.bsc.es/gitlab
+      gitlab-pm-shell = common-shell // {
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path;
+      };
+      gitlab-pm-docker = common-docker // {
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path;
+      };
+
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
+        # FIXME
+        registrationFlags = [
+          "--docker-network-mode host"
+        ];
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerDisableCache = true;
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat > "$HOME/.ssh/config" << EOF
+          Host bscpm03.bsc.es gitlab-internal.bsc.es
+            User git
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          EOF
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm03.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          ${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixos-24.11 nixpkgs
+          ${pkgs.nix}/bin/nix-channel --update nixpkgs
+          ${pkgs.nix}/bin/nix-env -i ${lib.concatStringsSep " " (with pkgs; [ nix cacert git openssh netcat curl ])}
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin";
+          NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
+        };
+      };
+    };
+  };
+
+  #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
+  systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
+  systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";
+  systemd.services.gitlab-runner.serviceConfig.Group = "gitlab-runner";
+  systemd.services.gitlab-runner.serviceConfig.ExecStart = lib.mkForce
+    ''${pkgs.gitlab-runner}/bin/gitlab-runner run --config ''${HOME}/.gitlab-runner/config.toml --listen-address "127.0.0.1:9252" --working-directory ''${HOME}'';
+
+  users.users.gitlab-runner = {
+    uid = config.ids.uids.gitlab-runner;
+    #isNormalUser = true;
+    home = "/var/lib/gitlab-runner";
+    description = "Gitlab Runner";
+    group = "gitlab-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;
+}

m/hut/gpfs-probe.nix

@@ -0,0 +1,31 @@
+{ pkgs, config, lib, ... }:
+let
+  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
+    ''
+      cp ${./gpfs-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  # Use a new user to handle the SSH keys
+  users.groups.ssh-robot = { };
+  users.users.ssh-robot = {
+    description = "SSH Robot";
+    isNormalUser = true;
+    home = "/var/lib/ssh-robot";
+  };
+
+  systemd.services.gpfs-probe = {
+    description = "Daemon to report GPFS latency via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
+}

m/hut/gpfs-probe.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+N=500
+
+t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
+
+if [ -z "$t" ]; then
+  t="5.00"
+fi
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP gpfs_touch_latency Time to create $N files.
+# TYPE gpfs_touch_latency gauge
+gpfs_touch_latency $t
+EOF

m/hut/monitoring.nix

@@ -0,0 +1,266 @@
+{ config, lib, ... }:
+
+{
+  imports = [
+    ../module/slurm-exporter.nix
+    ./gpfs-probe.nix
+  ];
+
+  age.secrets.grafanaJungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    owner = "grafana";
+    mode = "400";
+  };
+
+  age.secrets.ipmiYml.file = ../../secrets/ipmi.yml.age;
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        domain = "jungle.bsc.es";
+        root_url = "%(protocol)s://%(domain)s/grafana";
+        serve_from_sub_path = true;
+        http_port = 2342;
+        http_addr = "127.0.0.1";
+      };
+      smtp = {
+        enabled = true;
+        from_address = "jungle-robot@bsc.es";
+        user = "jungle-robot";
+        # Read the password from a file, which is only readable by grafana user
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
+        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
+        host = "mail.bsc.es:465";
+        startTLS_policy = "NoStartTLS";
+      };
+      feature_toggles.publicDashboards = true;
+      "auth.anonymous".enabled = true;
+      log.level = "warn";
+    };
+  };
+
+  # Make grafana alerts also use the proxy
+  systemd.services.grafana.environment = config.networking.proxy.envVars;
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    retentionTime = "1y";
+    listenAddress = "127.0.0.1";
+  };
+
+  systemd.services.prometheus-ipmi-exporter.serviceConfig.DynamicUser = lib.mkForce false;
+  systemd.services.prometheus-ipmi-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+
+  # We need access to the devices to monitor the disk space
+  systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+  systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+  virtualisation.docker.daemon.settings = {
+    metrics-addr = "127.0.0.1:9323";
+  };
+
+  # Required to allow the smartctl exporter to read the nvme0 character device,
+  # see the commit message on:
+  # https://github.com/NixOS/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
+  services.udev.extraRules = ''
+    SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
+  '';
+
+  services.prometheus = {
+
+    exporters = {
+      ipmi = {
+        enable = true;
+        group = "root";
+        user = "root";
+        configFile = config.age.secrets.ipmiYml.path;
+        extraFlags = [ "--log.level=debug" ];
+        listenAddress = "127.0.0.1";
+      };
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" "logind" ];
+        port = 9002;
+        listenAddress = "127.0.0.1";
+      };
+      smartctl = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+      };
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = ./blackbox.yml;
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "xeon07";
+        static_configs = [{
+          targets = [
+            "127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
+            "127.0.0.1:${toString config.services.prometheus.exporters.ipmi.port}"
+            "127.0.0.1:9323"
+            "127.0.0.1:9252"
+            "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
+            "127.0.0.1:9341" # Slurm exporter
+            "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
+          ];
+        }];
+      }
+      {
+        job_name = "ceph";
+        static_configs = [{
+          targets = [
+            "10.0.40.40:9283" # Ceph statistics
+            "10.0.40.40:9002" # Node exporter
+            "10.0.40.42:9002" # Node exporter
+          ];
+        }];
+      }
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = { module = [ "http_2xx" ]; };
+        static_configs = [{
+          targets = [
+            "https://www.google.com/robots.txt"
+            "https://pm.bsc.es/"
+            "https://pm.bsc.es/gitlab/"
+            "https://jungle.bsc.es/"
+            "https://gitlab.bsc.es/"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-icmp";
+        metrics_path = "/probe";
+        params = { module = [ "icmp" ]; };
+        static_configs = [{
+          targets = [
+            "1.1.1.1"
+            "8.8.8.8"
+            "ssfhead"
+            "anella-bsc.cesca.cat"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
+          }
+        ];
+      }
+      {
+        job_name = "gitea";
+        static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
+      }
+      {
+        # Scrape the IPMI info of the hosts remotely via LAN
+        job_name = "ipmi-lan";
+        scrape_interval = "1m";
+        scrape_timeout = "30s";
+        metrics_path = "/ipmi";
+        scheme = "http";
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            separator = ";";
+            regex = "(.*)(:80)?";
+            target_label = "__param_target";
+            replacement = "\${1}";
+            action = "replace";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            separator = ";";
+            regex = "(.*)-ipmi"; # Remove "-ipm̀i" at the end
+            target_label = "instance";
+            replacement = "\${1}";
+            action = "replace";
+          }
+          {
+            # Sets the fixed "module=lan" URL param
+            separator = ";";
+            regex = "(.*)";
+            target_label = "__param_module";
+            replacement = "lan";
+            action = "replace";
+          }
+          {
+            # Sets the target to query as the localhost IPMI exporter
+            separator = ";";
+            regex = ".*";
+            target_label = "__address__";
+            replacement = "127.0.0.1:9290";
+            action = "replace";
+          }
+        ];
+
+        # Load the list of targets from another file
+        file_sd_configs = [
+          {
+            files = [ "${./targets.yml}" ];
+            refresh_interval = "30s";
+          }
+        ];
+      }
+      {
+        job_name = "ipmi-raccoon";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9291" ]; }
+        ];
+        params = {
+          target = [ "84.88.51.142" ];
+          module = [ "raccoon" ];
+        };
+      }
+      {
+        job_name = "ipmi-fox";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9290" ]; }
+        ];
+        params = {
+          target = [ "fox-ipmi" ];
+          module = [ "fox" ];
+        };
+      }
+    ];
+  };
+}

m/hut/msmtp.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+{
+  age.secrets.jungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    group = "gitea";
+    mode = "440";
+  };
+
+  programs.msmtp = {
+    enable = true;
+    accounts = {
+      default = {
+        auth = true;
+        tls = true;
+        tls_starttls = false;
+        port = 465;
+        host = "mail.bsc.es";
+        user = "jungle-robot";
+        passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
+        from = "jungle-robot@bsc.es";
+      };
+    };
+  };
+}

m/hut/nginx.nix

@@ -0,0 +1,70 @@
+{ theFlake, pkgs, ... }:
+let
+  website = pkgs.stdenv.mkDerivation {
+    name = "jungle-web";
+    src = theFlake;
+    buildInputs = [ pkgs.hugo ];
+    buildPhase = ''
+      cd web
+      rm -rf public/
+      hugo
+    '';
+    installPhase = ''
+      cp -r public $out
+    '';
+  };
+in
+{
+  services.nginx = {
+    enable = true;
+    virtualHosts."jungle.bsc.es" = {
+      root = "${website}";
+      listen = [
+        {
+          addr = "127.0.0.1";
+          port = 80;
+        }
+      ];
+      extraConfig = ''
+        set_real_ip_from 127.0.0.1;
+        set_real_ip_from 84.88.52.107;
+        real_ip_recursive on;
+        real_ip_header X-Forwarded-For;
+
+        location /git {
+          rewrite ^/git$ / break;
+          rewrite ^/git/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:3000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /cache {
+          rewrite ^/cache(.*) /$1 break;
+          proxy_pass http://127.0.0.1:5000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /lists {
+          proxy_pass http://127.0.0.1:8081;
+          proxy_redirect http:// $scheme://;
+        }
+        location /grafana {
+          proxy_pass http://127.0.0.1:2342;
+          proxy_redirect http:// $scheme://;
+          proxy_set_header Host $host;
+          # Websockets
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+        }
+        location ~ ^/~(.+?)(/.*)?$ {
+          alias /ceph/home/$1/public_html$2;
+          index  index.html index.htm;
+          autoindex on;
+          absolute_redirect off;
+        }
+        location /p/ {
+          alias /ceph/p/;
+        }
+      '';
+    };
+  };
+}

m/hut/nix-serve.nix

@@ -0,0 +1,16 @@
+{ config, ... }:
+
+{
+  age.secrets.nixServe.file = ../../secrets/nix-serve.age;
+
+  services.nix-serve = {
+    enable = true;
+    # Only listen locally, as we serve it via ssh
+    bindAddress = "127.0.0.1";
+    port = 5000;
+
+    secretKeyFile = config.age.secrets.nixServe.path;
+    # Public key:
+    # jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=
+  };
+}

m/hut/p.nix

@@ -0,0 +1,43 @@
+{ pkgs, lib, config, ... }:
+let
+  p = pkgs.writeShellScriptBin "p" ''
+    set -e
+    cd /ceph
+    pastedir="p/$USER"
+    mkdir -p "$pastedir"
+
+    ext="txt"
+
+    if [ -n "$1" ]; then
+      ext="$1"
+    fi
+
+    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
+
+    cat > "$out"
+    chmod go+r "$out"
+    echo "https://jungle.bsc.es/$out"
+  '';
+in
+{
+  environment.systemPackages = with pkgs; [ p ];
+
+  # Make sure we have a directory per user. We cannot use the nice
+  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
+  # may not be mounted when it runs.
+  systemd.services.create-paste-dirs = let
+    # Take only normal users in hut
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "remote-fs.target" ];
+    after = [ "remote-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+}

m/hut/postgresql.nix

@@ -0,0 +1,19 @@
+{ lib, ... }:
+
+{
+  services.postgresql = {
+    enable = true;
+    ensureDatabases = [ "perftestsdb" ];
+    ensureUsers = [
+      { name = "anavarro"; ensureClauses.superuser = true; }
+      { name = "rarias";   ensureClauses.superuser = true; }
+      { name = "grafana"; }
+    ];
+    authentication = ''
+      #type  database     DBuser    auth-method
+      local  perftestsdb  rarias    trust
+      local  perftestsdb  anavarro  trust
+      local  perftestsdb  grafana   trust
+    '';
+  };
+}

m/hut/public-inbox.css

@@ -0,0 +1,79 @@
+/*
+ * CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
+ * Dark color scheme using 216 web-safe colors, inspired
+ * somewhat by the default color scheme in mutt.
+ * It reduces eyestrain for me, and energy usage for all:
+ * https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
+ */
+
+* {
+	font-size: 14px;
+	font-family: monospace;
+}
+
+pre {
+	white-space: pre-wrap;
+	padding: 10px;
+	background: #f5f5f5;
+}
+
+hr {
+	margin: 30px 0;
+}
+
+body {
+	max-width: 120ex; /* 120 columns wide */
+	margin: 50px auto;
+}
+
+/*
+ * Underlined links add visual noise which make them hard-to-read.
+ * Use colors to make them stand out, instead.
+ */
+a:link {
+	color: #007;
+	text-decoration: none;
+}
+a:visited {
+	color:#504;
+}
+a:hover {
+	text-decoration: underline;
+}
+
+/* quoted text in emails gets a different color */
+*.q { color:gray }
+
+/*
+ * these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
+ * (cgit uses <div>, public-inbox uses <span>)
+ */
+*.add { color:darkgreen } /* diff post-image lines */
+*.del { color:darkred } /* diff pre-image lines */
+*.head { color:black } /* diff header (metainformation) */
+*.hunk { color:gray } /* diff hunk-header */
+
+/*
+ * highlight 3.x colors (tested 3.18) for displaying blobs.
+ * This doesn't use most of the colors available, as I find too
+ * many colors overwhelming, so the default is commented out.
+ */
+.hl.num { color:#f30 } /* number */
+.hl.esc { color:#f0f } /* escape character */
+.hl.str { color:#f30 } /* string */
+.hl.ppc { color:#f0f } /* preprocessor */
+.hl.pps { color:#f30 } /* preprocessor string */
+.hl.slc { color:#09f } /* single-line comment */
+.hl.com { color:#09f } /* multi-line comment */
+/* .hl.opt { color:#ccc } */ /* operator */
+/* .hl.ipl { color:#ccc } */ /* interpolation */
+
+/* keyword groups kw[a-z] */
+.hl.kwa { color:#ff0 }
+.hl.kwb { color:#0f0 }
+.hl.kwc { color:#ff0 }
+/* .hl.kwd { color:#ccc } */
+
+/* line-number (unused by public-inbox) */
+/* .hl.lin { color:#ccc } */
+

m/hut/public-inbox.nix

@@ -0,0 +1,47 @@
+{ lib, ... }:
+
+{
+  services.public-inbox = {
+    enable = true;
+    http = {
+      enable = true;
+      port = 8081;
+      mounts = [ "/lists" ];
+    };
+    settings.publicinbox = {
+      css = [ "${./public-inbox.css}" ];
+      wwwlisting = "all";
+    };
+    inboxes = {
+      bscpkgs = {
+        url = "https://jungle.bsc.es/lists/bscpkgs";
+        address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for bscpkgs";
+        listid = "~rodarima/bscpkgs.lists.sr.ht";
+      };
+      jungle = {
+        url = "https://jungle.bsc.es/lists/jungle";
+        address = [ "~rodarima/jungle@lists.sr.ht" ];
+        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
+        description = "Patches for jungle";
+        listid = "~rodarima/jungle.lists.sr.ht";
+      };
+    };
+  };
+
+  # We need access to the network for the watch service, as we will fetch the
+  # emails directly from the IMAP server.
+  systemd.services.public-inbox-watch.serviceConfig = {
+    PrivateNetwork = lib.mkForce false;
+    RestrictAddressFamilies = lib.mkForce [ "AF_UNIX"  "AF_INET" "AF_INET6" ];
+    KillSignal = "SIGKILL"; # Avoid slow shutdown
+
+    # Required for chmod(..., 02750) on directories by git, from
+    # systemd.exec(8):
+    # > Note that this restricts marking of any type of file system object with
+    # > these bits, including both regular files and directories (where the SGID
+    # > is a different meaning than for files, see documentation).
+    RestrictSUIDSGID = lib.mkForce false;
+  };
+}

m/hut/pxe.nix

@@ -0,0 +1,35 @@
+{ theFlake, pkgs, ... }:
+
+# This module describes a script that can launch the pixiecore daemon to serve a
+# NixOS image via PXE to a node to directly boot from there, without requiring a
+# working disk.
+
+let
+  # The host config must have the netboot-minimal.nix module too
+  host = theFlake.nixosConfigurations.lake2;
+  sys = host.config.system;
+  build = sys.build;
+  kernel = "${build.kernel}/bzImage";
+  initrd = "${build.netbootRamdisk}/initrd";
+  init = "${build.toplevel}/init";
+
+  script = pkgs.writeShellScriptBin "pixiecore-helper" ''
+    #!/usr/bin/env bash -x
+
+    ${pkgs.pixiecore}/bin/pixiecore \
+      boot ${kernel} ${initrd} --cmdline "init=${init} loglevel=4" \
+      --debug --dhcp-no-bind --port 64172 --status-port 64172 "$@"
+  '';
+in
+{
+  ## We need a DHCP server to provide the IP
+  #services.dnsmasq = {
+  #  enable = true;
+  #  settings = {
+  #    domain-needed = true;
+  #    dhcp-range = [ "192.168.0.2,192.168.0.254" ];
+  #  };
+  #};
+
+  environment.systemPackages = [ script ];
+}

m/hut/slurm-server.nix

@@ -0,0 +1,7 @@
+{ ... }:
+
+{
+  services.slurm = {
+    server.enable = true;
+  };
+}

m/hut/targets.yml

@@ -0,0 +1,15 @@
+- targets:
+  - owl1-ipmi
+  - owl2-ipmi
+  - xeon03-ipmi
+  - xeon04-ipmi
+  - koro-ipmi
+  - xeon06-ipmi
+  - hut-ipmi
+  - eudy-ipmi
+  # Storage
+  - bay-ipmi
+  - oss01-ipmi
+  - lake2-ipmi
+  labels:
+    job: ipmi-lan

m/koro/configuration.nix

@@ -0,0 +1,35 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    #(modulesPath + "/installer/netboot/netboot-minimal.nix")
+
+    ../eudy/cpufreq.nix
+    ../eudy/users.nix
+    ./kernel.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d5376d2";
+
+  # disable automatic garbage collector
+  nix.gc.automatic = lib.mkForce false;
+
+  # members of the tracing group can use the lttng-provided kernel events
+  # without root permissions
+  users.groups.tracing.members = [ "arocanon" "vlopez" ];
+
+  # set up both ethernet and infiniband ips
+  networking = {
+    hostName = "koro";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.5";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.5";
+      prefixLength = 24;
+    } ];
+  };
+}

m/koro/kernel.nix

@@ -0,0 +1,70 @@
+{ pkgs, lib, ... }:
+
+let
+  #fcs-devel = pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = /mnt/data/kernel/fcs/kernel/src;
+  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
+  #};
+
+  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
+  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
+  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
+  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
+  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = builtins.fetchGit {
+  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+  #     rev = gitCommit;
+  #     ref = "fcs";
+  #   };
+  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
+  #};
+
+  kernel = nixos-fcs;
+
+  nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+    version = "6.2.8";
+    src = builtins.fetchGit {
+      url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+      rev = gitCommit;
+      ref = branch;
+    };
+    structuredExtraConfig = with lib.kernel; {
+      # add general custom kernel options here
+    } // lib.optionalAttrs lockStat {
+      LOCK_STAT = yes;
+    } // lib.optionalAttrs preempt {
+      PREEMPT = lib.mkForce yes;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+    };
+    kernelPatches = [];
+    extraMeta.branch = lib.versions.majorMinor version;
+  }));
+
+  nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
+  nixos-fcs-lockstat = nixos-fcs.override {
+    lockStat = true;
+  };
+  nixos-fcs-lockstat-preempt = nixos-fcs.override {
+    lockStat = true;
+    preempt = true;
+  };
+  latest = pkgs.linuxPackages_latest;
+
+in {
+  imports = [
+    ../eudy/kernel/lttng.nix
+    ../eudy/kernel/perf.nix
+  ];
+  boot.kernelPackages = lib.mkForce kernel;
+
+  # disable all cpu mitigations
+  boot.kernelParams = [
+    "mitigations=off"
+  ];
+  
+  # enable memory overcommit, needed to build a taglibc system using nix after
+  # increasing the openblas memory footprint
+  boot.kernel.sysctl."vm.overcommit_memory" = 1;
+}

m/lake2/configuration.nix

@@ -0,0 +1,83 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/monitoring.nix
+  ];
+
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
+
+  boot.kernel.sysctl = {
+    "kernel.yama.ptrace_scope" = lib.mkForce "1";
+  };
+
+  environment.systemPackages = with pkgs; [
+    ceph
+  ];
+
+  services.ceph = {
+    enable = true;
+    global = {
+      fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b";
+      monHost = "10.0.40.40";
+      monInitialMembers = "bay";
+      clusterNetwork = "10.0.40.40/24"; # Use Ethernet only
+    };
+    osd = {
+      enable = true;
+      # One daemon per NVME disk
+      daemons = [ "4" "5" "6" "7" ];
+      extraConfig = {
+        "osd crush chooseleaf type" = "0";
+        "osd journal size" = "10000";
+        "osd pool default min size" = "2";
+        "osd pool default pg num" = "200";
+        "osd pool default pgp num" = "200";
+        "osd pool default size" = "3";
+      };
+    };
+  };
+
+  networking = {
+    hostName = "lake2";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.42";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.42";
+      prefixLength = 24;
+    } ];
+    firewall = {
+      extraCommands = ''
+        # Accept all incoming TCP traffic from bay
+        iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
+        # Accept monitoring requests from hut
+        iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
+        # Accept all Ceph traffic from the local network
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+      '';
+    };
+  };
+
+  # Missing service for volumes, see:
+  # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/
+  systemd.services.ceph-volume = {
+    enable = true;
+    description = "Ceph Volume activation";
+    unitConfig = {
+      Type = "oneshot";
+      After = "local-fs.target";
+      Wants = "local-fs.target";
+    };
+    path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ];
+    serviceConfig = {
+      KillMode = "none";
+      Environment = "CEPH_VOLUME_TIMEOUT=10000";
+      ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'";
+      TimeoutSec = "0";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
+}

m/module/ceph.nix

@@ -0,0 +1,24 @@
+{ config, pkgs, ... }:
+
+# Mounts the /ceph filesystem at boot
+{
+  environment.systemPackages = with pkgs; [
+    ceph-client
+    fio # For benchmarks
+  ];
+
+  # We need the ceph module loaded as the mount.ceph binary fails to run the
+  # modprobe command.
+  boot.kernelModules = [ "ceph" ];
+
+  age.secrets.cephUser.file = ../../secrets/ceph-user.age;
+
+  fileSystems."/ceph" = {
+    fsType = "ceph";
+    device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
+    options = [
+      "mon_addr=10.0.40.40"
+      "secretfile=${config.age.secrets.cephUser.path}"
+    ];
+  };
+}

m/module/debuginfod.nix

@@ -0,0 +1,3 @@
+{
+  services.nixseparatedebuginfod.enable = true;
+}

m/module/emulation.nix

@@ -0,0 +1,3 @@
+{
+  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
+}

m/module/jungle-users.nix

@@ -0,0 +1,24 @@
+{ config, lib, ... }:
+
+with lib;
+
+{
+  options = {
+    users.jungleUsers = mkOption {
+      type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
+      description = ''
+        Same as users.users but with the extra `hosts` attribute, which controls
+        access to the nodes by `networking.hostName`.
+      '';
+    };
+  };
+
+  config = let
+    allowedUser = host: userConf: builtins.elem host userConf.hosts;
+    filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
+    removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
+    currentHost = config.networking.hostName;
+  in {
+    users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
+  };
+}

m/module/monitoring.nix

@@ -0,0 +1,25 @@
+{ config, lib, ... }:
+
+{
+  # We need access to the devices to monitor the disk space
+  systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+  systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+  # Required to allow the smartctl exporter to read the nvme0 character device,
+  # see the commit message on:
+  # https://github.com/NixOS/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
+  services.udev.extraRules = ''
+    SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
+  '';
+
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+      smartctl.enable = true;
+    };
+  };
+}

m/module/slurm-client.nix

@@ -0,0 +1,145 @@
+{ config, pkgs, lib, ... }:
+
+let
+  suspendProgram = pkgs.writeScript "suspend.sh" ''
+    #!/usr/bin/env bash
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Shutting down host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
+    done
+  '';
+
+  resumeProgram = pkgs.writeScript "resume.sh" ''
+    #!/usr/bin/env bash
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Starting host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
+    done
+  '';
+
+  prolog = pkgs.writeScript "prolog.sh" ''
+    #!/usr/bin/env bash
+
+    echo "hello from the prolog"
+
+    exit 0
+  '';
+
+  epilog = pkgs.writeScript "epilog.sh" ''
+    #!/usr/bin/env bash
+
+    echo "hello from the epilog"
+
+    exit 0
+  '';
+
+in {
+  systemd.services.slurmd.serviceConfig = {
+    # Kill all processes in the control group on stop/restart. This will kill
+    # all the jobs running, so ensure that we only upgrade when the nodes are
+    # not in use. See:
+    # https://github.com/NixOS/nixpkgs/commit/ae93ed0f0d4e7be0a286d1fca86446318c0c6ffb
+    # https://bugs.schedmd.com/show_bug.cgi?id=2095#c24
+    KillMode = lib.mkForce "control-group";
+  };
+
+  services.slurm = {
+    client.enable = true;
+    controlMachine = "hut";
+    clusterName = "jungle";
+    nodeName = [
+      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
+      "fox       Sockets=2 CoresPerSocket=96 ThreadsPerCore=2 Feature=fox"
+      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
+    ];
+
+    partitionName = [
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "fox Nodes=fox          Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "all Nodes=owl[1-2],hut Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+    ];
+
+    # See slurm.conf(5) for more details about these options.
+    extraConfig = ''
+      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
+      # not with Intel MPI. For that use the compatibility shim libpmi.so
+      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
+      # library in SLURM (--mpi=pmix). See more details here:
+      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
+      MpiDefault=pmix
+
+      # When a node reboots return that node to the slurm queue as soon as it
+      # becomes operative again.
+      ReturnToService=2
+
+      # Track all processes by using a cgroup
+      ProctrackType=proctrack/cgroup
+
+      # Enable task/affinity to allow the jobs to run in a specified subset of
+      # the resources. Use the task/cgroup plugin to enable process containment.
+      TaskPlugin=task/affinity,task/cgroup
+
+      # Power off unused nodes until they are requested
+      SuspendProgram=${suspendProgram}
+      SuspendTimeout=60
+      ResumeProgram=${resumeProgram}
+      ResumeTimeout=300
+      SuspendExcNodes=hut,fox
+
+      # Turn the nodes off after 1 hour of inactivity
+      SuspendTime=3600
+
+      # Reduce port range so we can allow only this range in the firewall
+      SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
+    '';
+  };
+
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
+  age.secrets.mungeKey = {
+    file = ../../secrets/munge-key.age;
+    owner = "munge";
+    group = "munge";
+  };
+
+  services.munge = {
+    enable = true;
+    password = config.age.secrets.mungeKey.path;
+  };
+}

m/module/slurm-exporter.nix

@@ -0,0 +1,28 @@
+{ config, lib, pkgs, ... }:
+
+# See also: https://github.com/NixOS/nixpkgs/pull/112010
+# And: https://github.com/NixOS/nixpkgs/pull/115839
+
+with lib;
+
+{
+  systemd.services."prometheus-slurm-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = ''
+        ${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
+      '';
+      Environment = [
+        "PATH=${pkgs.slurm}/bin"
+        # We need to specify the slurm config to be able to talk to the slurmd
+        # daemon.
+        "SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
+      ];
+    };
+  };
+}

m/module/slurm-firewall.nix

@@ -0,0 +1,8 @@
+{ ... }:
+
+{
+  networking.firewall = {
+    # Required for PMIx in SLURM, we should find a better way
+    allowedTCPPortRanges = [ { from=1024; to=65535; } ];
+  };
+}

m/module/slurm-hut-nix-store.nix

@@ -0,0 +1,19 @@
+{ ... }:
+
+{
+  # Mount the hut nix store via NFS
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  systemd.services.slurmd.serviceConfig = {
+    # When running a job, bind the hut store in /nix/store so the paths are
+    # available too.
+    # FIXME: This doesn't keep the programs in /run/current-system/sw/bin
+    # available in the store. Ideally they should be merged but the overlay FS
+    # doesn't work when the underlying directories change.
+    BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
+  };
+}

m/owl1/configuration.nix

@@ -0,0 +1,27 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
+    ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c";
+
+  networking = {
+    hostName = "owl1";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.1";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.1";
+      prefixLength = 24;
+    } ];
+  };
+}

m/owl2/configuration.nix

@@ -0,0 +1,28 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/ceph.nix
+    ../module/emulation.nix
+    ../module/slurm-client.nix
+    ../module/slurm-firewall.nix
+    ../module/debuginfod.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629";
+
+  networking = {
+    hostName = "owl2";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.2";
+      prefixLength = 24;
+    } ];
+    # Watch out! The OmniPath device is not in the same place here:
+    interfaces.ibp129s0.ipv4.addresses = [ {
+      address = "10.0.42.2";
+      prefixLength = 24;
+    } ];
+  };
+}

m/raccoon/configuration.nix

@@ -0,0 +1,64 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+  ];
+
+  # Don't install Grub on the disk yet
+  boot.loader.grub.device = "nodev";
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS1,115200"
+  ];
+
+  networking = {
+    hostName = "raccoon";
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    defaultGateway = "84.88.51.129";
+    interfaces.eno0.ipv4.addresses = [ {
+      address = "84.88.51.152";
+      prefixLength = 25;
+    } ];
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.graphics.enable = true;
+  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  users.motd = ''
+    ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⡎⢰⣧⠀⠀⠀⠁⠈⠛⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⣴⡦⠶⠟⠓⠚⠻⡄⠀
+    ⠀⠀⠀⠀⠀⠀⣧⠀⣱⣀⣰⣧⠀⢀⠀⣘⣿⣿⣦⣶⣄⣠⡀⠀⠀⣀⣀⣤⣴⣄⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠿⠏⠁⠀⣀⣠⣶⣿⡶⣿⠀
+    ⠀⠀⠀⠀⠀⠀⣹⣆⠘⣿⣿⣿⣇⢸⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣶⣶⣦⡀⣀⣤⣠⣤⡾⠋⠀⢀⣤⣶⣿⣿⣿⣿⣿⣿⣿⡀
+    ⠀⠀⠀⠀⠀⠀⠘⢿⡄⢼⣿⣿⣿⣿⣿⡟⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣵⣾⡾⠙⣋⣩⣽⣿⣿⣿⣿⢋⡼⠁
+    ⠀⠀⠀⠀⠀⠀⠀⠈⢻⣄⠸⢿⣿⣿⠿⠷⠀⠈⠀⣭⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣾⣿⣿⣿⣿⣿⣿⠇⡼⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⣯⡀⠀⢼⡿⠀⠀⠀⢼⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣿⡿⣿⣿⣿⠿⣿⣯⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢋⡼⠁⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢻⡏⠠⣦⠁⠀⠀⠀⠀⠀⠟⠛⠛⣿⣿⣿⣿⣿⠿⠁⠀⠁⢿⠙⠁⠀⠛⠹⣿⣏⣾⣿⣿⣿⣿⣿⣿⣿⣿⠿⠃⣹⠁⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣘⣧⠀⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⡿⡿⠀⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀⢹⣿⠿⢿⣿⣿⣿⣿⣿⠋⢀⡤⠛⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⡯⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⢸⣿⣿⣿⠛⠉⠀⣰⠷⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠇⠀⠀⠀⠀⠀⢀⣿⡇⠀⠀⢻⣿⣿⠁⠀⠀⢠⣾⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠟⢿⣿⣄⡀⢸⣿⡀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⠀⠀⠀⢰⣿⣿⡛⣿⣿⡄⢠⡺⠿⡍⠁⢀⣤⣿⣿⣿⠿⣷⣮⣉⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⠀⠀⠈⣧⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⢾⠉⠃⠀⣴⣿⣟⠻⣿⣿⣿⡇⢸⣿⣶⠀⢀⣾⣿⣿⣟⠿⣷⣾⣿⣿⣿⣿⣦⣤⣤⡤⠀⠀⠀⠀⠀⠁⠀⠀⠀⣼⠗⠀⠀⠀⠀
+    ⠀⠀⠐⢄⡀⠀⠀⠀⢘⡀⠀⢶⣾⣿⣿⣿⣿⡿⠋⠁⠈⠻⠉⠀⠚⠻⣿⣿⣿⣶⣾⣿⣿⣿⣿⣿⣿⣷⣬⣤⣶⣦⡀⣾⣶⣇⠀⠀⠈⢉⣷⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠈⠓⠶⢦⡽⠄⣈⣿⣿⣿⣿⣿⠏⠀⠀⠀⠀⠀⠀⠀⠀⠀⠹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡓⠙⣿⡟⠀⠀⠀⠈⠛⣷⣶⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⢀⣬⠆⢠⣍⣛⠻⣿⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣉⣀⡀⠀⠀⠈⠛⢿⣦⡀
+    ⠐⠒⠒⠶⠶⠶⢦⣬⣟⣥⣀⡉⠛⠻⠶⢁⣤⣾⣿⣿⣿⣷⡄⠀⠀⠀⠀⠀⢸⣿⣿⣿⣿⣿⣟⡛⠿⠭⠭⠭⠭⠭⠿⠿⠿⢿⣿⣟⠃⠀⠀⠀⠹⣟⠓
+    ⠀⣀⣠⠤⠤⢤⣤⣾⣤⡄⣉⣉⣙⣓⡂⣿⣿⣭⣹⣿⣿⣿⣿⡰⣂⣀⢀⠀⠻⣿⠛⠻⠟⠡⣶⣾⣿⣿⣿⣿⣿⣿⣿⡖⠒⠒⠒⠛⠷⢤⡀⢰⣴⣿⡆
+    ⠀⠀⠀⢀⣠⡴⠾⠟⠻⣟⡉⠉⠉⠉⢁⢿⣿⣿⣿⣿⣿⣿⡿⣱⣿⣭⡌⠤⠀⠀⠐⣶⣌⡻⣶⣭⡻⢿⣿⣿⣿⣿⣿⣯⣥⣤⣦⠀⠠⣴⣶⣶⣿⡟⢿
+    ⢀⠔⠊⠉⠀⠀⠀⠀⢸⣯⣤⠀⠀⠠⣼⣮⣟⣿⣿⣿⣻⣭⣾⣿⣿⣷⣶⣦⠶⣚⣾⣿⣿⣷⣜⣿⣿⣶⣝⢿⣿⣿⣿⣿⣷⣦⣄⣰⡄⠈⢿⣿⡿⣇⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠈⢡⢇⠀⠀⣠⣿⣿⣿⣯⣟⣛⣛⣛⣛⣛⣩⣭⣴⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣻⣿⣧⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠏⠀⢹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣍⣿⣿⣿⣿⡄⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣾⡁⢈⣾⣿⡿⠛⣛⣿⣿⣿⣿ DO YOU BRING FEEDS? ⣿⣿⣿⣿⣿⣿⡏⠈⠙⠈⠁⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⡿⠛⠉⣽⣿⣷⣾⡿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠷⠌⠛⠉⠀⠁⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠹⠋⠀⢻⣿⣿⣿⣿⠿⢿⣿⣿⣿⣿⣿⣿⠿⣿⣿⣿⣿⠿⠛⠋⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
+    ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠁⠀⠀⠀⠀⠀⠈⠉⠉⠀⠀⠈⠋⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 
+  '';
+}

nixos-config.nix

@@ -0,0 +1 @@
+(builtins.getFlake (toString ./.)).nixosConfigurations

overlays-compat/overlays.nix

@@ -1,8 +0,0 @@
-self: super:
-with super.lib;
-let
-  # Load the system config and get the `nixpkgs.overlays` option
-  overlays = (import <nixpkgs/nixos> { }).config.nixpkgs.overlays;
-in
-  # Apply all overlays to the input of the current "main" overlay
-  foldl' (flip extends) (_: super) overlays self

pkgs/mpich-fix-hwtopo.patch

@@ -0,0 +1,36 @@
+diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
+index 33e88bc..ee3641c 100644
+--- a/src/util/mpir_hwtopo.c
++++ b/src/util/mpir_hwtopo.c
+@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
+ #ifdef HAVE_HWLOC
+     bindset = hwloc_bitmap_alloc();
+     hwloc_topology_init(&hwloc_topology);
+-    char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
+-    if (xmlfile != NULL) {
+-        int rc;
+-        rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
+-        if (rc == 0) {
+-            /* To have hwloc still actually call OS-specific hooks, the
+-             * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
+-             * file is really the underlying system. */
+-            hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
+-        }
+-        MPL_free(xmlfile);
+-    }
+
+     hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
+     if (!hwloc_topology_load(hwloc_topology)) 
+
+--- a/src/mpi/init/local_proc_attrs.c
++++ b/src/mpi/init/local_proc_attrs.c
+@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
+     /* Set the number of tag bits. The device may override this value. */
+     MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
+
+-    char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
+-    MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
+-    MPL_free(requested_kinds);
+-
+     return mpi_errno;
+ }

pkgs/overlay.nix

@@ -0,0 +1,57 @@
+final: prev:
+{
+  # Set MPICH as default
+  mpi = final.mpich;
+
+  # Configure the network for MPICH
+  mpich = with final; let
+    # pmix comes with the libraries in .out and headers in .dev
+    pmixAll = symlinkJoin {
+      name = "pmix-all";
+      paths = [ pmix.dev pmix.out ];
+    };
+  in prev.mpich.overrideAttrs (old: {
+    patches = (old.patches or []) ++ [
+      # See https://github.com/pmodels/mpich/issues/6946
+      ./mpich-fix-hwtopo.patch
+    ];
+    buildInput = old.buildInputs ++ [
+      libfabric
+      pmixAll
+    ];
+    configureFlags = [
+      "--enable-shared"
+      "--enable-sharedlib"
+      "--with-pm=no"
+      "--with-device=ch4:ofi"
+      "--with-pmi=pmix"
+      "--with-pmix=${pmixAll}"
+      "--with-libfabric=${libfabric}"
+      "--enable-g=log"
+    ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
+      "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
+      "FCFLAGS=-fallow-argument-mismatch"
+    ];
+  });
+
+  slurm = prev.slurm.overrideAttrs (old: {
+    patches = (old.patches or []) ++ [
+      # See https://bugs.schedmd.com/show_bug.cgi?id=19324
+      ./slurm-rank-expansion.patch
+    ];
+    # Install also the pam_slurm_adopt library to restrict users from accessing
+    # nodes with no job allocated.
+    postBuild = (old.postBuild or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security"
+      popd
+    '';
+    postInstall = (old.postInstall or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security" install
+      popd
+    '';
+  });
+
+  prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
+}

pkgs/slurm-exporter.nix

@@ -0,0 +1,22 @@
+{ buildGoModule, fetchFromGitHub, lib }:
+
+buildGoModule rec {
+  pname = "prometheus-slurm-exporter";
+  version = "0.20";
+
+  src = fetchFromGitHub {
+    rev = version;
+    owner = "vpenso";
+    repo = pname;
+    sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
+  };
+
+  vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Prometheus SLURM Exporter";
+    homepage = "https://github.com/vpenso/prometheus-slurm-exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/slurm-rank-expansion.patch

@@ -0,0 +1,11 @@
+--- a/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:05:24.815313882 +0100
++++ b/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:09:53.936900823 +0100
+@@ -314,7 +314,7 @@ static void _dmdx_req(buf_t *buf, int no
+ 	}
+ 
+ 	nsptr = pmixp_nspaces_local();
+-	if (nsptr->ntasks <= rank) {
++	if ((long) nsptr->ntasks <= (long) rank) {
+ 		char *nodename = pmixp_info_job_host(nodeid);
+ 		PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
+ 			    nodename, ns, nsptr->ntasks, rank);

rebuild.sh

@@ -0,0 +1,16 @@
+#!/bin/sh -ex
+
+if [ "$(id -u)" != 0 ]; then
+ echo "Needs root permissions"
+ exit 1
+fi
+
+if [ "$(hostname)" != "hut" ]; then
+  >&2 echo "must run from machine hut, not $(hostname)"
+  exit 1
+fi
+
+# Update all nodes
+nixos-rebuild switch --flake .
+nixos-rebuild switch --flake .#owl1 --target-host owl1
+nixos-rebuild switch --flake .#owl2 --target-host owl2

secrets/gitea-runner-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg eRVX5yndWDLg9hw7sY1Iu8pJFy47luHvdL+zZGK2u1s
+e1nXXiMW0ywkZYh2s6c7/quGMfBOJOaRhNQDjCD2Iyc
+-> ssh-ed25519 CAWG4Q gYG7GRxRpJ0/5Wz0Z0J2wfLfkMFNmcy81dQEewM7gUA
+lamdUdx+xOFWF1lmUM4x9TT0cJtKu9Sp7w9JHwm13u0
+-> ssh-ed25519 MSF3dg HEzfpR8alG6WPzhaEjAmmjOFoFcMSQUldx46dBsXri4
+OAD5H/zZGhfevYrFJzJrbNKPomKZDOS9Qx5tmTp78Jo
+--- A0sMSiNXWaEIgRXR0x6UAIaluuVH6Zlv4CJ9sI0NXOw
+ÿú6çphóÎÆ{Ñ>®F|ÅiÃvâæE}{ìruÎâÆ·‹Ý°ËÍ}^»‰>ñc6¥´j÷	ùgèGW<47>Ã:—J3ù|ø|†ZÑ

secrets/gitlab-bsc-docker-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg 4Xns3jybBuv8flzd+h3DArVBa/AlKjt1J9jAyJsasCE
+uyVjJxh5i8aGgAgCpPl6zTYeIkf9mIwURof51IKWvwE
+-> ssh-ed25519 CAWG4Q T2r6r1tyNgq1XlYXVtLJFfOfUnm6pSVlPwUqC1pkyRo
+9yDoKU0EC34QMUXYnsJvhPCLm6oD9w7NlTi2sheoBqQ
+-> ssh-ed25519 MSF3dg Bh9DekFTq+QMUEAonwcaIAJX4Js1O7cHjDniCD0gtm8
+t/Ro0URLeDUWcvb7rlkG2s03PZ+9Rr3N4TIX03tXpVc
+--- E5+/D4aK2ihKRR4YC5XOTmUbKgOqBR0Nk0gYvFOzXOI
+‰ÀÍyKF~djº˜r%¸Š'ÉÓÖPä&_-lŸ”ö&’o¥_ér¯¦r¢ÿß<C3BF>’0ï,­U7†nC·Te…÷[fˆ97ü•…šÙ˦“ÈC!D±E<C2B1>Wé*ÐLAô
x6¾#–¯
’sqôiéËÆäÏŸ“åk ,ùÝ“

secrets/gitlab-runner-docker-token.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg GdmdkW+BqqwBgu30b846jv3J7jtCM+a3rgOERuA050A
+FeGqM75jG9egesR+yyVKHm0/M+uBBp5Hclg4+qN0BR8
+-> ssh-ed25519 CAWG4Q a0wTWHgulQUYDAMZmXf3dOf6PdYgCqNtSylzWVVRNVM
+Bx+WSYaiY4ZwlSZJo2a1XPMQmbKOU7F0tKAqVRLBOPo
+-> ssh-ed25519 MSF3dg KccUvZZUbxbCrRWUWrX8KcHF6vQ5FV/BqUqI59G7dj4
+CFr7GXpZ9rPgy7HBfOyiYF9FnZUw6KcZwq9f7/0KaU8
+--- E0Rp6RR/8+o0jvB1lRdhnlabxvI6uu/IgL2ZpPXzTc8
+û#ã¶H÷$°F;Ñéù%›È6êË2†¢rfXŸ\Dn ÖшºÈ‰©x™Î>¥Ù&;÷c‘UŠI=›ÑMöÀª?Tœ¡Ç¸ÂÂ"px†Ó­\s‚ÙãbFý<46>ù¹WD¼{Ë
+AW>?U©ÙÊçÐHÔ³

secrets/gitlab-runner-shell-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg xWRxJGWSzA5aplRYCYLB6aBwrUrQQJ2MtDYaD75V5nI
+J07XF3NQiaYKKKNRcNWi9MloJD2wXHd+2K7bo6lF+QU
+-> ssh-ed25519 CAWG4Q jNWymbyCczcm8RcaIEbFQBlOMALsuxTl4+pLUi0aR20
+z5NixlrRD+Y7Z/aFPs6hiDW4/lp8CBQCeJYpbuG9yYM
+-> ssh-ed25519 MSF3dg QsUQloEKN3k1G49FQnNR/Do6ILgGpjFcw3zu5kk1Ako
+IHwyFWUEWqCStNcFprnpBa8L5J6zKIsn+7HcgGRv3sM
+--- oUia0fsL6opeYWACyXtHAu/Ld+bUIt/7S1VszYTvwgU
+™êVäœ*øtë2-Ÿ7·œ–Ž“§hÜ&‰éÍ¢_!Õ¿+”·±¯(‚ã¡nù¿	¬í(Ëê÷/}òœäáCúNÍ·|ÇNèuÎ5‰Ã¹å‹šKÀìlÆ"ÃØklOX¨yº÷æØàù¤¹ø²Aíõe„È$

secrets/secrets.nix

@@ -0,0 +1,19 @@
+let
+  keys = import ../keys.nix;
+  adminsKeys = builtins.attrValues keys.admins;
+  hut = [ keys.hosts.hut ] ++ adminsKeys;
+  # Only expose ceph keys to safe nodes and admins
+  safe = keys.hostGroup.safe ++ adminsKeys;
+in
+{
+  "gitea-runner-token.age".publicKeys = hut;
+  "gitlab-runner-docker-token.age".publicKeys = hut;
+  "gitlab-runner-shell-token.age".publicKeys = hut;
+  "gitlab-bsc-docker-token.age".publicKeys = hut;
+  "nix-serve.age".publicKeys = hut;
+  "jungle-robot-password.age".publicKeys = hut;
+  "ipmi.yml.age".publicKeys = hut;
+
+  "ceph-user.age".publicKeys = safe;
+  "munge-key.age".publicKeys = safe;
+}

web/.gitignore

@@ -0,0 +1 @@
+./public

web/archetypes/default.md

@@ -0,0 +1,6 @@
+---
+title: "{{ replace .Name "-" " " | title }}"
+date: {{ .Date }}
+draft: true
+---
+

web/content/_index.md

@@ -0,0 +1,25 @@
+![Rainforest](jungle.jpg)
+
+Welcome to the jungle, a set of machines with no imposed rules that are fully
+controlled and maintained by their users.
+
+The configuration of all the machines is written in a centralized [git
+repository][config] using the Nix language for NixOS. Changes in the
+configuration of the machines are introduced by merge requests and pass a review
+step before being deployed.
+
+[config]: https://pm.bsc.es/gitlab/rarias/jungle
+
+The machines have access to the large list of packages available in
+[Nixpkgs][nixpkgs] and a custom set of packages named [bscpkgs][bscpkgs],
+specifically tailored to our needs for HPC machines. Users can install their own
+packages and made them system-wide available by opening a merge request.
+
+[nixpkgs]: https://github.com/NixOS/nixpkgs
+[bscpkgs]: https://pm.bsc.es/gitlab/rarias/bscpkgs
+
+We have put a lot of effort to guarantee very good reproducibility properties in
+the configuration of the machines and the software they use.
+
+To enter the jungle machines follow the [instructions](access) to submit a
+request.

web/content/access/index.md

@@ -0,0 +1,22 @@
+---
+title: "Enter the jungle"
+description: "Request access to the machines"
+---
+
+![Cave](./cave.jpg)
+
+Before requesting access to the jungle machines, you must be able to access the
+`ssfhead.bsc.es` node (only available via the intranet or VPN). You can request
+access to the login machine using a resource petition in the BSC intranet.
+
+Then, to request access to the machines we will need some information about you:
+
+1. Which machines you want access to ([hut](/hut), [fox](/fox), owl1, owl2, eudy, koro...)
+1. Your user name and user id (to match the NFS permissions)
+1. Your real name and surname (for identification purposes)
+1. The salted hash of your login password, generated with `mkpasswd -m sha-512`
+1. An SSH public key of type Ed25519 (can be generated with `ssh-keygen -t ed25519`)
+
+Send an email to <jungle@bsc.es> with the details, or directly open a
+merge request in the [jungle
+repository](https://pm.bsc.es/gitlab/rarias/jungle/).

web/content/doc/_index.md

@@ -0,0 +1,10 @@
+---
+title: "Docs"
+description: "Documentation for users of jungle machines"
+date: 2023-09-15
+---
+
+If this is the first time you use any of the jungle machines with NixOS, follow
+the [quick start guide](quickstart).
+
+

web/content/doc/quickstart.md

@@ -0,0 +1,234 @@
+---
+title: "Quick start"
+date: 2023-09-15
+---
+
+This documentation will guide you on how to build custom packages of software
+and use them in the jungle machines. It has been designed to reduce the friction
+from users coming from module systems.
+
+You should be able to access the jungle machines, otherwise [request
+access](/access).
+
+## Changes from other HPC machines
+
+Users of other machines have been using the Lmod tool (module load ...) to add
+or remove programs from their environment, as well as manually building their
+own software for too many years.
+
+While we cannot prevent users from continuing to use this tedious mechanism, we
+have designed the jungle machines to be much easier to operate by using the nix
+package manager.
+
+### Freedom to install packages
+
+When a user wanted to install a package, it was forced to either do it on its
+own directory, or request a system administrator to install it in a shared
+directory, so other users can also use that package.
+
+This situation is gone, each user can install any package of software by
+themselves, without requiring any other authorization. When two users request
+the same package, the same copy will be provided.
+
+A new package will be downloaded if it is available (someone already built it)
+or will be built from source on demand.
+
+### No changes over time
+
+All users retain the same versions of the packages they request until they
+decide to update them.
+
+## Using nix to manage packages
+
+In this chapter we show how to install packages and enter a development shell to
+build new programs from source. The examples are done from the hut machine,
+read [this page](/access) to request access.
+
+### Installing binaries
+
+To temporarily install new packages, use:
+
+```text
+hut% nix shell jungle#gcc jungle#cowsay jungle#ovni
+```
+
+Notice that the packages are described as two parts divided by the `#` symbol.
+The first part defines where to take the package from and the second part is
+the name of the package. For now we will use `jungle#<package>`. You can find
+many more packages here:
+
+<https://search.nixos.org/packages>
+
+You will now enter a new shell, where those requested package **binaries are
+available in $PATH**:
+
+```text
+hut% cowsay hello world
+ _____________
+< hello world >
+ -------------
+        \   ^__^
+         \  (oo)\_______
+            (__)\       )\/\
+                ||----w |
+                ||     ||
+
+hut% ovniver
+LD_LIBRARY_PATH not set
+libovni: build v1.11.0 (a7103f8), dynamic v1.11.0 (a7103f8)
+
+hut% gcc --version
+gcc (GCC) 13.3.0
+Copyright (C) 2023 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+```
+
+### Building programs
+
+The above method only loads new binaries in the `$PATH`. If we try to build a
+program that includes headers or links with a library, it will fail to find
+them:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut% gcc test.c -lovni -o test
+test.c:1:10: fatal error: ovni.h: No such file or directory
+    1 | #include <ovni.h>
+      |          ^~~~~~~~
+compilation terminated.
+```
+
+We could manually add the full path to the ovni include directory with `-I` and
+the libraries with `-L`, but there is a tool that already perform these steps
+automatically for us, `nix develop`.
+
+Let's go back to our original shell first, where those packages are not
+available anymore:
+
+```
+hut% ps
+    PID TTY          TIME CMD
+2356260 pts/1    00:00:01 zsh
+2457268 pts/1    00:00:00 zsh
+2457297 pts/1    00:00:00 ps
+hut% exit
+hut% ovniver
+ovniver: command not found
+```
+
+### Creating a flake.nix
+
+To define which packages we want, we will write a small file that list them, a
+flake.nix file.
+
+First, we will create a new directory where we are going to be working:
+
+```
+hut% mkdir example
+hut% cd exmple
+```
+
+Then place this flake.nix file:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell {
+      pname = "devshell";
+      buildInputs = with pkgs; [
+        ovni gcc cowsay # more packages here...
+      ];
+    };
+  };
+}
+```
+
+
+Now enter the shell with:
+
+```
+hut% nix develop
+warning: creating lock file '/home/Computational/rarias/example/flake.lock':
+• Added input 'jungle':
+    'path:/nix/store/27srv8haj6vv4ywrbmw0a8vds561m8rq-source?lastModified=1739479441&narHash=sha256-Kgjs8SO1w9NbPBu8ghwzCxYJ9kvWpoQOT%2BXwPvA9DcU%3D&rev=76396c0d67ef0cf32377d5c1894bb695293bca9d' (2025-02-13)
+• Added input 'jungle/agenix':
+    'github:ryantm/agenix/f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41?narHash=sha256-b%2Buqzj%2BWa6xgMS9aNbX4I%2BsXeb5biPDi39VgvSFqFvU%3D' (2024-08-10)
+• Added input 'jungle/agenix/darwin':
+    'github:lnl7/nix-darwin/4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d?narHash=sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0%3D' (2023-11-24)
+• Added input 'jungle/agenix/darwin/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/home-manager':
+    'github:nix-community/home-manager/3bfaacf46133c037bb356193bd2f1765d9dc82c1?narHash=sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE%3D' (2023-12-20)
+• Added input 'jungle/agenix/home-manager/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/agenix/systems':
+    'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e?narHash=sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768%3D' (2023-04-09)
+• Added input 'jungle/bscpkgs':
+    'git+https://git.sr.ht/~rodarima/bscpkgs?ref=refs/heads/master&rev=6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f' (2024-11-29)
+• Added input 'jungle/bscpkgs/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/nixpkgs':
+    'github:NixOS/nixpkgs/9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc?narHash=sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8%3D' (2025-01-14)
+
+hut$ 
+```
+
+Notice that long list of messages is Nix creating a new flake.lock file with the
+current state of the packages. Next invocations will use the same packages as
+described by the lock file.
+
+### Building a program from nix develop
+
+Now let's try again building our test program:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut$ gcc test.c -o test -lovni
+hut$ ldd test
+        linux-vdso.so.1 (0x00007ffff7fc4000)
+        libovni.so.1 => /nix/store/sqk972akjv0q8dchn8ccjln2llzyyfd0-ovni-1.11.0/lib/libovni.so.1 (0x00007ffff7fab000)
+        libc.so.6 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/libc.so.6 (0x00007ffff7db2000)
+        /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/ld-linux-x86-64.so.2 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib64/ld-linux-x86-64.so.2 (0x00007ffff7fc6000)
+hut$ ./test
+```
+
+Now the ovni.h header and the libovni library are found and the program is
+successfully built, linked and executed.
+
+You can add more packages as needed in your flake.nix:
+
+```nix
+  buildInputs = with pkgs; [
+    ovni gcc cowsay # more packages here...
+  ];
+```
+
+Make sure you exit the develop shell first, and then enter again with `nix
+develop`.
+
+## Remember
+
+- `nix shell` places binaries in the `$PATH`.
+- `nix develop` enters a development shell where both binaries and the libraries
+  and includes are available so you can build new programs.

web/content/eudy/_index.md

@@ -0,0 +1,10 @@
+---
+title: "Eudy"
+description: "Linux kernel experiments"
+---
+
+[![Eudy](eudy.jpg)](https://commons.wikimedia.org/w/index.php?curid=5817408)
+
+The *eudy* machine is destined as a playground for Linux kernel experiments. The
+name is a shorthand of the Eudyptula species of little penguins found the New
+Zealand and Australia.

web/content/fox/_index.md

@@ -0,0 +1,97 @@
+---
+title: "Fox"
+description: "AMD Genoa 9684X with 2 NVIDIA RTX4000 GPUs"
+date: 2025-02-12
+---
+
+![Fox](fox.jpg)
+
+Picture by [Joanne Redwood](https://web.archive.org/web/20191109175146/https://www.inaturalist.org/photos/6568074),
+[CC0](http://creativecommons.org/publicdomain/zero/1.0/deed.en).
+
+The *fox* machine is a big GPU server that is configured to run heavy workloads.
+It has two fast AMD CPUs with large cache and 2 reasonable NVIDIA GPUs. Here are
+the detailed specifications:
+
+- 2x AMD GENOA X 9684X DP/UP 96C/192T 2.55G 1,150M 400W SP5 3D V-cach
+- 24x 32GB DDR5-4800 ECC RDIMM (total 768 GiB of RAM)
+- 1x 2.5" SSD SATA3 MICRON 5400 MAX 480GB
+- 2x 2.5" KIOXIA CM7-R 1.92TB NVMe GEN5 PCIe 5x4
+- 2x NVIDIA RTX4000 ADA Gen 20GB GDDR6 PCIe 4.0
+
+## Access
+
+To access the machine, request a SLURM session from [hut](/hut) using the `fox`
+partition:
+
+    hut% salloc -p fox
+
+Then connect via ssh:
+
+    hut% ssh fox
+    fox%
+
+Follow [these steps](/access) if you don't have access to hut or fox.
+
+## CUDA
+
+To use CUDA, you can use the following `flake.nix` placed in a new directory to
+load all the required dependencies:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+
+  outputs = { jungle, ... }: {
+    devShell.x86_64-linux = let
+      pkgs = jungle.nixosConfigurations.fox.pkgs;
+    in pkgs.mkShell {
+      name = "cuda-env-shell";
+      buildInputs = with pkgs; [
+        git gitRepo gnupg autoconf curl
+        procps gnumake util-linux m4 gperf unzip
+
+        # Cuda packages (more at https://search.nixos.org/packages)
+        cudatoolkit linuxPackages.nvidia_x11
+        cudaPackages.cuda_cudart.static
+        cudaPackages.libcusparse
+
+        libGLU libGL
+        xorg.libXi xorg.libXmu freeglut
+        xorg.libXext xorg.libX11 xorg.libXv xorg.libXrandr zlib
+        ncurses5 stdenv.cc binutils
+      ];
+      shellHook = ''
+        export CUDA_PATH=${pkgs.cudatoolkit}
+        export LD_LIBRARY_PATH=/var/run/opengl-driver/lib
+        export SMS=50
+      '';
+    };
+  };
+}
+```
+
+Then just run `nix develop` from the same directory:
+
+    % mkdir cuda
+    % cd cuda
+    % vim flake.nix
+    [...]
+    % nix develop
+    $ nvcc -V
+    nvcc: NVIDIA (R) Cuda compiler driver
+    Copyright (c) 2005-2024 NVIDIA Corporation
+    Built on Tue_Feb_27_16:19:38_PST_2024
+    Cuda compilation tools, release 12.4, V12.4.99
+    Build cuda_12.4.r12.4/compiler.33961263_0
+
+## Filesystems
+
+The machine has several file systems available.
+
+- `$HOME`: Mounted via NFS across all nodes. It is slow and has low capacity.
+  Don't abuse.
+- `/ceph/home/$USER`: Shared Ceph file system across jungle nodes. Slow but high
+  capacity. Stores three redundant copies of every file.
+- `/nvme{0,1}/$USER`: The two local NVME disks, very fast and large capacity.
+- `/tmp`: tmpfs, fast but not backed by a disk. Will be erased on reboot.