Use NixOS attributes for the install section

Add a RequiredBy dependency for remote-fs.target
Use a systemd mount directly for the nix store
2023-09-18 19:27:14 +02:00 · 2023-09-18 19:05:58 +02:00 · 2023-09-18 18:53:40 +02:00 · 2023-09-18 18:06:51 +02:00 · 2023-09-18 17:28:47 +02:00 · 2023-09-18 16:07:46 +02:00
44 changed files with 581 additions and 794 deletions
--- a/doc/R1000WF_SystemIntegration_and_ServiceGuide_Rev2_4.pdf
+++ b/doc/R1000WF_SystemIntegration_and_ServiceGuide_Rev2_4.pdf
--- a/flake.lock
+++ b/flake.lock
@@ -6,15 +6,14 @@
        "home-manager": "home-manager",
        "nixpkgs": [
          "nixpkgs"
-        ],
-        "systems": "systems"
+        ]
      },
      "locked": {
-        "lastModified": 1712079060,
-        "narHash": "sha256-/JdiT9t+zzjChc5qQiF+jhrVhRt8figYH29rZO7pFe4=",
+        "lastModified": 1690228878,
+        "narHash": "sha256-9Xe7JV0krp4RJC9W9W9WutZVlw6BlHTFMiUP/k48LQY=",
        "owner": "ryantm",
        "repo": "agenix",
-        "rev": "1381a759b205dff7a6818733118d02253340fd5e",
+        "rev": "d8c973fd228949736dedf61b7f8cc1ece3236792",
        "type": "github"
      },
      "original": {
@@ -30,17 +29,17 @@
        ]
      },
      "locked": {
-        "lastModified": 1713974364,
-        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
+        "lastModified": 1694708510,
+        "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
        "ref": "refs/heads/master",
-        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
-        "revCount": 937,
+        "rev": "3a4062ac04be6263c64a481420d8e768c2521b80",
+        "revCount": 862,
        "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
      },
      "original": {
        "type": "git",
-        "url": "https://git.sr.ht/~rodarima/bscpkgs"
+        "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
      }
    },
    "darwin": {
@@ -51,11 +50,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1700795494,
-        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
+        "lastModified": 1673295039,
+        "narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
        "owner": "lnl7",
        "repo": "nix-darwin",
-        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
+        "rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
        "type": "github"
      },
      "original": {
@@ -73,11 +72,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1703113217,
-        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
+        "lastModified": 1682203081,
+        "narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
        "owner": "nix-community",
        "repo": "home-manager",
-        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
+        "rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
        "type": "github"
      },
      "original": {
@@ -88,11 +87,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1713714899,
-        "narHash": "sha256-+z/XjO3QJs5rLE5UOf015gdVauVRQd2vZtsFkaXBq2Y=",
+        "lastModified": 1693663421,
+        "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "6143fc5eeb9c4f00163267708e26191d1e918932",
+        "rev": "e56990880811a451abd32515698c712788be5720",
        "type": "github"
      },
      "original": {
@@ -108,21 +107,6 @@
        "bscpkgs": "bscpkgs",
        "nixpkgs": "nixpkgs"
      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
    }
  },
  "root": "root",
--- a/flake.nix
+++ b/flake.nix
@@ -3,7 +3,7 @@
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    agenix.url = "github:ryantm/agenix";
    agenix.inputs.nixpkgs.follows = "nixpkgs";
-    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
+    bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
    bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
  };

--- a/m/bay/configuration.nix
+++ b/m/bay/configuration.nix
@@ -13,6 +13,10 @@
    ceph
  ];

+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+
  networking = {
    hostName = "bay";
    interfaces.eno1.ipv4.addresses = [ {
@@ -23,16 +27,6 @@
      address = "10.0.42.40";
      prefixLength = 24;
    } ];
-    firewall = {
-      extraCommands = ''
-        # Accept all incoming TCP traffic from lake2
-        iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
-        # Accept monitoring requests from hut
-        iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
-        # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
-      '';
-    };
  };

  services.ceph = {
--- a/m/common/fs.nix
+++ b/m/common/fs.nix
@@ -25,7 +25,4 @@
    device = "none";
    fsType = "tracefs";
  };
-
-  # Mount a tmpfs into /tmp
-  boot.tmp.useTmpfs = true;
 }
--- a/m/common/main.nix
+++ b/m/common/main.nix
@@ -8,6 +8,7 @@
    ./hw.nix
    ./net.nix
    ./ntp.nix
+    ./slurm.nix
    ./ssh.nix
    ./users.nix
    ./watchdog.nix
@@ -41,7 +42,7 @@
    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
    ncdu config.boot.kernelPackages.perf ldns
    # From bsckgs overlay
-    osumb
+    bsc.osumb
  ];

  programs.direnv.enable = true;
--- a/m/common/net.nix
+++ b/m/common/net.nix
@@ -23,7 +23,7 @@
      allowedTCPPorts = [ 22 ];
      extraCommands = ''
        # Prevent ssfhead from contacting our slurmd daemon
-        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse
        # But accept traffic to slurm ports from any other node in the subnet
        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
        # We also need to open the srun port range
--- a/m/module/slurm-client.nix
+++ b/m/module/slurm-client.nix
--- a/m/eudy/configuration.nix
+++ b/m/eudy/configuration.nix
@@ -9,7 +9,7 @@
    ./cpufreq.nix
    ./fs.nix
    ./users.nix
-    ../module/debuginfod.nix
+    ./slurm.nix
  ];

  # Select this using the ID to avoid mismatches
--- a/m/eudy/slurm.nix
+++ b/m/eudy/slurm.nix
@@ -0,0 +1,7 @@
+{ lib, ... }:
+
+{
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+}
--- a/m/hut/blackbox.yml
+++ b/m/hut/blackbox.yml
@@ -1,162 +0,0 @@
-modules:
-  http_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      proxy_url: "http://127.0.0.1:23080"
-      skip_resolve_phase_with_proxy: true
-      follow_redirects: true
-      valid_status_codes: []  # Defaults to 2xx
-      method: GET
-  http_with_proxy:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      skip_resolve_phase_with_proxy: true
-  http_with_proxy_and_headers:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      proxy_connect_header:
-        Proxy-Authorization:
-          - Bearer token
-  http_post_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Content-Type: application/json
-      body: '{}'
-  http_post_body_file:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      body_file: "/files/body.txt"
-  http_basic_auth_example:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Host: "login.example.com"
-      basic_auth:
-        username: "username"
-        password: "mysecret"
-  http_2xx_oauth_client_credentials:
-    prober: http
-    timeout: 5s
-    http:
-      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
-      follow_redirects: true
-      preferred_ip_protocol: "ip4"
-      valid_status_codes:
-        - 200
-        - 201
-      oauth2:
-        client_id: "client_id"
-        client_secret: "client_secret"
-        token_url: "https://api.example.com/token"
-        endpoint_params:
-          grant_type: "client_credentials"
-  http_custom_ca_example:
-    prober: http
-    http:
-      method: GET
-      tls_config:
-        ca_file: "/certs/my_cert.crt"
-  http_gzip:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-  http_gzip_with_accept_encoding:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-      headers:
-        Accept-Encoding: gzip
-  tls_connect:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      tls: true
-  tcp_connect_example:
-    prober: tcp
-    timeout: 5s
-  imap_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "OK.*STARTTLS"
-        - send: ". STARTTLS"
-        - expect: "OK"
-        - starttls: true
-        - send: ". capability"
-        - expect: "CAPABILITY IMAP4rev1"
-  smtp_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "^220 ([^ ]+) ESMTP (.+)$"
-        - send: "EHLO prober\r"
-        - expect: "^250-STARTTLS"
-        - send: "STARTTLS\r"
-        - expect: "^220"
-        - starttls: true
-        - send: "EHLO prober\r"
-        - expect: "^250-AUTH"
-        - send: "QUIT\r"
-  irc_banner_example:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - send: "NICK prober"
-        - send: "USER prober prober prober :prober"
-        - expect: "PING :([^ ]+)"
-          send: "PONG ${1}"
-        - expect: "^:[^ ]+ 001"
-  icmp:
-    prober: icmp
-    timeout: 5s
-    icmp:
-      preferred_ip_protocol: "ip4"
-  dns_udp_example:
-    prober: dns
-    timeout: 5s
-    dns:
-      query_name: "www.prometheus.io"
-      query_type: "A"
-      valid_rcodes:
-        - NOERROR
-      validate_answer_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-        fail_if_all_match_regexp:
-          - ".*127.0.0.1"
-        fail_if_not_matches_regexp:
-          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
-        fail_if_none_matches_regexp:
-          - "127.0.0.1"
-      validate_authority_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-      validate_additional_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-  dns_soa:
-    prober: dns
-    dns:
-      query_name: "prometheus.io"
-      query_type: "SOA"
-  dns_tcp_example:
-    prober: dns
-    dns:
-      transport_protocol: "tcp" # defaults to "udp"
-      preferred_ip_protocol: "ip4" # defaults to "ip6"
-      query_name: "www.prometheus.io"
--- a/m/hut/configuration.nix
+++ b/m/hut/configuration.nix
@@ -5,16 +5,11 @@
    ../common/main.nix

    ../module/ceph.nix
-    ../module/debuginfod.nix
-    ../module/slurm-client.nix
    ./gitlab-runner.nix
    ./monitoring.nix
    ./nfs.nix
-    ./slurm-server.nix
+    ./slurm-daemon.nix
    ./nix-serve.nix
-    ./public-inbox.nix
-    ./gitea.nix
-    ./msmtp.nix
    #./pxe.nix
  ];

--- a/m/hut/gitea.nix
+++ b/m/hut/gitea.nix
@@ -1,62 +0,0 @@
-{ config, lib, ... }:
-{
-  age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
-
-  services.gitea = {
-    enable = true;
-    appName = "Gitea in the jungle";
-
-    settings = {
-      server = {
-        ROOT_URL = "https://jungle.bsc.es/git/";
-        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
-        LANDING_PAGE = "explore";
-      };
-      metrics.ENABLED = true;
-      service = {
-        REGISTER_MANUAL_CONFIRM = true;
-        ENABLE_NOTIFY_MAIL = true;
-      };
-
-      mailer = {
-        ENABLED       = true;
-        FROM          = "jungle-robot@bsc.es";
-        PROTOCOL      = "sendmail";
-        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
-        SENDMAIL_ARGS = "--";
-      };
-    };
-  };
-
-  services.gitea-actions-runner.instances = {
-    runrun = {
-      enable = true;
-      name = "runrun";
-      url = "https://jungle.bsc.es/git/";
-      tokenFile = config.age.secrets.giteaRunnerToken.path;
-      labels = [ "native:host" ];
-      settings.runner.capacity = 8;
-    };
-  };
-
-  systemd.services.gitea-runner-runrun = {
-    path = [ "/run/current-system/sw" ];
-    serviceConfig = {
-      # DynamicUser doesn't work well with SSH
-      DynamicUser = lib.mkForce false;
-      User = "gitea-runner";
-      Group = "gitea-runner";
-    };
-  };
-
-  users.users.gitea-runner = {
-    isSystemUser = true;
-    home = "/var/lib/gitea-runner";
-    description = "Gitea Runner";
-    group = "gitea-runner";
-    extraGroups = [ "docker" ];
-    createHome = true;
-  };
-  users.groups.gitea-runner = {};
-}
-
--- a/m/hut/gitlab-runner.nix
+++ b/m/hut/gitlab-runner.nix
@@ -2,14 +2,14 @@

 {
  age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
-  age.secrets.gitlabToken.file = ../../secrets/gitlab-bsc-es-token.age;
  age.secrets.nosvToken.file = ../../secrets/nosv-token.age;

  services.gitlab-runner = {
    enable = true;
    settings.concurrent = 5;
-    services = let
-      common-shell = {
+    services = {
+      ovni-shell = {
+        registrationConfigFile = config.age.secrets.ovniToken.path;
        executor = "shell";
        tagList = [ "nix" "xeon" ];
        registrationFlags = [
@@ -20,7 +20,8 @@
          SHELL = "${pkgs.bash}/bin/bash";
        };
      };
-      common-docker = {
+      ovni-docker = {
+        registrationConfigFile = config.age.secrets.ovniToken.path;
        dockerImage = "debian:stable";
        tagList = [ "docker" "xeon" ];
        registrationFlags = [
@@ -32,20 +33,18 @@
          http_proxy = "http://localhost:23080";
        };
      };
-    in {
-      # For gitlab.bsc.es
-      gitlab-bsc-es-shell = common-shell // {
-        registrationConfigFile = config.age.secrets.gitlabToken.path;
-      };
-      gitlab-bsc-es-docker = common-docker // {
-        registrationConfigFile = config.age.secrets.gitlabToken.path;
-      };
-      # For pm.bsc.es/gitlab
-      gitlab-pm-shell = common-shell // {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
-      };
-      gitlab-pm-docker = common-docker // {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+      nosv-docker = {
+        registrationConfigFile = config.age.secrets.nosvToken.path;
+        dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
+        registrationFlags = [
+          "--docker-network-mode host"
+          "--docker-cpus 56"
+        ];
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
      };
    };
  };
--- a/m/hut/monitoring.nix
+++ b/m/hut/monitoring.nix
@@ -1,14 +1,6 @@
 { config, lib, ... }:

 {
-  imports = [ ../module/slurm-exporter.nix ];
-
-  age.secrets.grafanaJungleRobotPassword = {
-    file = ../../secrets/jungle-robot-password.age;
-    owner = "grafana";
-    mode = "400";
-  };
-
  services.grafana = {
    enable = true;
    settings = {
@@ -19,24 +11,10 @@
        http_port = 2342;
        http_addr = "127.0.0.1";
      };
-      smtp = {
-        enabled = true;
-        from_address = "jungle-robot@bsc.es";
-        user = "jungle-robot";
-        # Read the password from a file, which is only readable by grafana user
-        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
-        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
-        host = "mail.bsc.es:465";
-        startTLS_policy = "NoStartTLS";
-      };
      feature_toggles.publicDashboards = true;
-      "auth.anonymous".enabled = true;
    };
  };

-  # Make grafana alerts also use the proxy
-  systemd.services.grafana.environment = config.networking.proxy.envVars;
-
  services.prometheus = {
    enable = true;
    port = 9001;
@@ -83,11 +61,6 @@
        enable = true;
        listenAddress = "127.0.0.1";
      };
-      blackbox = {
-        enable = true;
-        listenAddress = "127.0.0.1";
-        configFile = ./blackbox.yml;
-      };
    };

    scrapeConfigs = [
@@ -100,8 +73,6 @@
            "127.0.0.1:9323"
            "127.0.0.1:9252"
            "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
-            "127.0.0.1:9341" # Slurm exporter
-            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
          ];
        }];
      }
@@ -115,71 +86,6 @@
          ];
        }];
      }
-      {
-        job_name = "blackbox-http";
-        metrics_path = "/probe";
-        params = { module = [ "http_2xx" ]; };
-        static_configs = [{
-          targets = [
-            "https://www.google.com/robots.txt"
-            "https://pm.bsc.es/"
-            "https://pm.bsc.es/gitlab/"
-            "https://jungle.bsc.es/"
-            "https://gitlab.bsc.es/"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "blackbox-icmp";
-        metrics_path = "/probe";
-        params = { module = [ "icmp" ]; };
-        static_configs = [{
-          targets = [
-            "1.1.1.1"
-            "8.8.8.8"
-            "ssfhead"
-            "anella-bsc.cesca.cat"
-          ];
-        }];
-        relabel_configs = [
-          {
-            # Takes the address and sets it in the "target=<xyz>" URL parameter
-            source_labels = [ "__address__" ];
-            target_label = "__param_target";
-          }
-          {
-            # Sets the "instance" label with the remote host we are querying
-            source_labels = [ "__param_target" ];
-            target_label = "instance";
-          }
-          {
-            # Shows the host target address instead of the blackbox address
-            target_label = "__address__";
-            replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
-          }
-        ];
-      }
-      {
-        job_name = "gitea";
-        static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
-      }
      {
        # Scrape the IPMI info of the hosts remotely via LAN
        job_name = "ipmi-lan";
--- a/m/hut/msmtp.nix
+++ b/m/hut/msmtp.nix
@@ -1,24 +0,0 @@
-{ config, lib, ... }:
-{
-  age.secrets.jungleRobotPassword = {
-    file = ../../secrets/jungle-robot-password.age;
-    group = "gitea";
-    mode = "440";
-  };
-
-  programs.msmtp = {
-    enable = true;
-    accounts = {
-      default = {
-        auth = true;
-        tls = true;
-        tls_starttls = false;
-        port = 465;
-        host = "mail.bsc.es";
-        user = "jungle-robot";
-        passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
-        from = "jungle-robot@bsc.es";
-      };
-    };
-  };
-}
--- a/m/hut/public-inbox.css
+++ b/m/hut/public-inbox.css
@@ -1,79 +0,0 @@
-/*
- * CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
- * Dark color scheme using 216 web-safe colors, inspired
- * somewhat by the default color scheme in mutt.
- * It reduces eyestrain for me, and energy usage for all:
- * https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
- */
-
-* {
-	font-size: 14px;
-	font-family: monospace;
-}
-
-pre {
-	white-space: pre-wrap;
-	padding: 10px;
-	background: #f5f5f5;
-}
-
-hr {
-	margin: 30px 0;
-}
-
-body {
-	max-width: 120ex; /* 120 columns wide */
-	margin: 50px auto;
-}
-
-/*
- * Underlined links add visual noise which make them hard-to-read.
- * Use colors to make them stand out, instead.
- */
-a:link {
-	color: #007;
-	text-decoration: none;
-}
-a:visited {
-	color:#504;
-}
-a:hover {
-	text-decoration: underline;
-}
-
-/* quoted text in emails gets a different color */
-*.q { color:gray }
-
-/*
- * these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
- * (cgit uses <div>, public-inbox uses <span>)
- */
-*.add { color:darkgreen } /* diff post-image lines */
-*.del { color:darkred } /* diff pre-image lines */
-*.head { color:black } /* diff header (metainformation) */
-*.hunk { color:gray } /* diff hunk-header */
-
-/*
- * highlight 3.x colors (tested 3.18) for displaying blobs.
- * This doesn't use most of the colors available, as I find too
- * many colors overwhelming, so the default is commented out.
- */
-.hl.num { color:#f30 } /* number */
-.hl.esc { color:#f0f } /* escape character */
-.hl.str { color:#f30 } /* string */
-.hl.ppc { color:#f0f } /* preprocessor */
-.hl.pps { color:#f30 } /* preprocessor string */
-.hl.slc { color:#09f } /* single-line comment */
-.hl.com { color:#09f } /* multi-line comment */
-/* .hl.opt { color:#ccc } */ /* operator */
-/* .hl.ipl { color:#ccc } */ /* interpolation */
-
-/* keyword groups kw[a-z] */
-.hl.kwa { color:#ff0 }
-.hl.kwb { color:#0f0 }
-.hl.kwc { color:#ff0 }
-/* .hl.kwd { color:#ccc } */
-
-/* line-number (unused by public-inbox) */
-/* .hl.lin { color:#ccc } */
-
--- a/m/hut/public-inbox.nix
+++ b/m/hut/public-inbox.nix
@@ -1,47 +0,0 @@
-{ lib, ... }:
-
-{
-  services.public-inbox = {
-    enable = true;
-    http = {
-      enable = true;
-      port = 8081;
-      mounts = [ "/lists" ];
-    };
-    settings.publicinbox = {
-      css = [ "${./public-inbox.css}" ];
-      wwwlisting = "all";
-    };
-    inboxes = {
-      bscpkgs = {
-        url = "https://jungle.bsc.es/lists/bscpkgs";
-        address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
-        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
-        description = "Patches for bscpkgs";
-        listid = "~rodarima/bscpkgs.lists.sr.ht";
-      };
-      jungle = {
-        url = "https://jungle.bsc.es/lists/jungle";
-        address = [ "~rodarima/jungle@lists.sr.ht" ];
-        watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
-        description = "Patches for jungle";
-        listid = "~rodarima/jungle.lists.sr.ht";
-      };
-    };
-  };
-
-  # We need access to the network for the watch service, as we will fetch the
-  # emails directly from the IMAP server.
-  systemd.services.public-inbox-watch.serviceConfig = {
-    PrivateNetwork = lib.mkForce false;
-    RestrictAddressFamilies = lib.mkForce [ "AF_UNIX"  "AF_INET" "AF_INET6" ];
-    KillSignal = "SIGKILL"; # Avoid slow shutdown
-
-    # Required for chmod(..., 02750) on directories by git, from
-    # systemd.exec(8):
-    # > Note that this restricts marking of any type of file system object with
-    # > these bits, including both regular files and directories (where the SGID
-    # > is a different meaning than for files, see documentation).
-    RestrictSUIDSGID = lib.mkForce false;
-  };
-}
--- a/m/hut/slurm-daemon.nix
+++ b/m/hut/slurm-daemon.nix
--- a/m/koro/configuration.nix
+++ b/m/koro/configuration.nix
@@ -7,6 +7,7 @@

    ../eudy/cpufreq.nix
    ../eudy/users.nix
+    ../eudy/slurm.nix
    ./users.nix
    ./kernel.nix
  ];
--- a/m/lake2/configuration.nix
+++ b/m/lake2/configuration.nix
@@ -12,6 +12,10 @@
    ceph
  ];

+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+
  services.ceph = {
    enable = true;
    global = {
@@ -45,16 +49,6 @@
      address = "10.0.42.42";
      prefixLength = 24;
    } ];
-    firewall = {
-      extraCommands = ''
-        # Accept all incoming TCP traffic from bay
-        iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
-        # Accept monitoring requests from hut
-        iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
-        # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
-      '';
-    };
  };

  # Missing service for volumes, see:
--- a/m/module/ceph.nix
+++ b/m/module/ceph.nix
@@ -3,6 +3,7 @@
 # Mounts the /ceph filesystem at boot
 {
  environment.systemPackages = with pkgs; [
+    ceph
    ceph-client
    fio # For benchmarks
  ];
--- a/m/module/debuginfod.nix
+++ b/m/module/debuginfod.nix
@@ -1,3 +0,0 @@
-{
-  services.nixseparatedebuginfod.enable = true;
-}
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -0,0 +1,69 @@
+{ ... }:
+{
+  # Don't make the nix store read-only, as this would prevent the overlay FS
+  # from being able to mount it.
+  boot.readOnlyNixStore = false;
+
+  # The nix-daemon.socket has an unnecessary dependency over the /nix/store
+  # mount point. But that mount point won't be provided until the network is
+  # ready. However, the network-address-eno1.service, has a dependency over
+  # sockets.target, causing a cycle.
+  # One solution is to make the nix-daemon.socket depend only on the socket
+  # patch (which is already covered by ConditionPathIsReadWrite =
+  # /nix/var/nix/daemon-socket), instead on the /nix/store.
+  #
+  # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor =
+  # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get
+  # added by systemd when the override config is merged with the one that Nix
+  # provides:
+  #
+  # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor
+  # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket
+  # 
+  # To fix this, the Nix package is patched to only depend on /nix/var instead.
+  # See ../../pkgs/overlay.nix for details.
+
+  # Mount the hut nix store via NFS in read-only mode.
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  # A workdir is also needed, so setup a permanent dir using tmpfiles.
+  systemd.tmpfiles.rules = [
+    "d /mnt/nix-work 0700 root root -"
+  ];
+
+  # Mount an overlay in /nix/store using as lower layer the NFS store and upper
+  # layer the disk nix store. The destination is still the nix store in
+  # /nix/store (confusing). We need rw access, as the daemon need to write the
+  # lock files to build derivations locally. Use a systemd mount unit directly
+  # so we can specify the LazyUmount option and we avoid having it mounted
+  # in the stage1 before systemd.
+  systemd.mounts = [
+    {
+      what = "overlay";
+      type = "overlay";
+      where = "/nix/store";
+      # We need the local-fs.target to be ready, so the network interfaces can
+      # be configured to the network.target is reached. So make this a netdev
+      # mount.
+      options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
+      description = "Overlay /nix/store mount";
+      mountConfig = {
+        LazyUnmount = true;
+      };
+
+      # Run the unit after remote-fs-pre.target but before the remote-fs.target
+      after = [ "remote-fs-pre.target"];
+      before = [ "umount.target" "remote-fs.target" ];
+      # Install by using wantedBy over remote-fs.target
+      wantedBy = [ "remote-fs.target" ];
+      unitConfig = {
+        # We need to wait for the NFS mount
+        RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
+      };
+    }
+  ];
+}
--- a/m/module/slurm-exporter.nix
+++ b/m/module/slurm-exporter.nix
@@ -1,28 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-# See also: https://github.com/NixOS/nixpkgs/pull/112010
-# And: https://github.com/NixOS/nixpkgs/pull/115839
-
-with lib;
-
-{
-  systemd.services."prometheus-slurm-exporter" = {
-    wantedBy = [ "multi-user.target" ];
-    after = [ "network.target" ];
-    serviceConfig = {
-      Restart = mkDefault "always";
-      PrivateTmp = mkDefault true;
-      WorkingDirectory = mkDefault "/tmp";
-      DynamicUser = mkDefault true;
-      ExecStart = ''
-        ${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
-      '';
-      Environment = [
-        "PATH=${pkgs.slurm}/bin"
-        # We need to specify the slurm config to be able to talk to the slurmd
-        # daemon.
-        "SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
-      ];
-    };
-  };
-}
--- a/m/module/slurm-hut-nix-store.nix
+++ b/m/module/slurm-hut-nix-store.nix
@@ -1,19 +0,0 @@
-{ ... }:
-
-{
-  # Mount the hut nix store via NFS
-  fileSystems."/mnt/hut-nix-store" = {
-    device = "hut:/nix/store";
-    fsType = "nfs";
-    options = [ "ro" ];
-  };
-
-  systemd.services.slurmd.serviceConfig = {
-    # When running a job, bind the hut store in /nix/store so the paths are
-    # available too.
-    # FIXME: This doesn't keep the programs in /run/current-system/sw/bin
-    # available in the store. Ideally they should be merged but the overlay FS
-    # doesn't work when the underlying directories change.
-    BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
-  };
-}
--- a/m/owl1/configuration.nix
+++ b/m/owl1/configuration.nix
@@ -4,9 +4,8 @@
  imports = [
    ../common/main.nix
    ../module/ceph.nix
-    ../module/slurm-client.nix
    ../module/slurm-firewall.nix
-    ../module/slurm-hut-nix-store.nix
+    ../module/shared-nix-store.nix
  ];

  # Select the this using the ID to avoid mismatches
--- a/m/owl2/configuration.nix
+++ b/m/owl2/configuration.nix
@@ -4,9 +4,8 @@
  imports = [
    ../common/main.nix
    ../module/ceph.nix
-    ../module/slurm-client.nix
    ../module/slurm-firewall.nix
-    ../module/slurm-hut-nix-store.nix
+    ../module/shared-nix-store.nix
  ];

  # Select the this using the ID to avoid mismatches
--- a/pkgs/ceph.nix
+++ b/pkgs/ceph.nix
@@ -0,0 +1,405 @@
+{ lib
+, stdenv
+, runCommand
+, fetchurl
+, fetchFromGitHub
+, fetchPypi
+
+# Build time
+, cmake
+, ensureNewerSourcesHook
+, fmt
+, git
+, makeWrapper
+, nasm
+, pkg-config
+, which
+
+# Tests
+, nixosTests
+
+# Runtime dependencies
+, arrow-cpp
+, babeltrace
+, boost179
+, bzip2
+, cryptsetup
+, cunit
+, doxygen
+, gperf
+, graphviz
+, gtest
+, icu
+, libcap
+, libcap_ng
+, libnl
+, libxml2
+, lttng-ust
+, lua
+, lz4
+, oath-toolkit
+, openldap
+, python310
+, rdkafka
+, rocksdb
+, snappy
+, sqlite
+, utf8proc
+, zlib
+, zstd
+
+# Optional Dependencies
+, curl ? null
+, expat ? null
+, fuse ? null
+, libatomic_ops ? null
+, libedit ? null
+, libs3 ? null
+, yasm ? null
+
+# Mallocs
+, gperftools ? null
+, jemalloc ? null
+
+# Crypto Dependencies
+, cryptopp ? null
+, nspr ? null
+, nss ? null
+
+# Linux Only Dependencies
+, linuxHeaders
+, util-linux
+, libuuid
+, udev
+, keyutils
+, rdma-core
+, rabbitmq-c
+, libaio ? null
+, libxfs ? null
+, liburing ? null
+, zfs ? null
+, ...
+}:
+
+# We must have one crypto library
+assert cryptopp != null || (nss != null && nspr != null);
+
+let
+  shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
+
+  optYasm = shouldUsePkg yasm;
+  optExpat = shouldUsePkg expat;
+  optCurl = shouldUsePkg curl;
+  optFuse = shouldUsePkg fuse;
+  optLibedit = shouldUsePkg libedit;
+  optLibatomic_ops = shouldUsePkg libatomic_ops;
+  optLibs3 = shouldUsePkg libs3;
+
+  optJemalloc = shouldUsePkg jemalloc;
+  optGperftools = shouldUsePkg gperftools;
+
+  optCryptopp = shouldUsePkg cryptopp;
+  optNss = shouldUsePkg nss;
+  optNspr = shouldUsePkg nspr;
+
+  optLibaio = shouldUsePkg libaio;
+  optLibxfs = shouldUsePkg libxfs;
+  optZfs = shouldUsePkg zfs;
+
+  # Downgrade rocksdb, 7.10 breaks ceph
+  rocksdb' = rocksdb.overrideAttrs {
+    version = "7.9.2";
+    src = fetchFromGitHub {
+      owner = "facebook";
+      repo = "rocksdb";
+      rev = "refs/tags/v7.9.2";
+      hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
+    };
+  };
+
+  hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
+
+  # Malloc implementation (can be jemalloc, tcmalloc or null)
+  malloc = if optJemalloc != null then optJemalloc else optGperftools;
+
+  # We prefer nss over cryptopp
+  cryptoStr = if optNss != null && optNspr != null then "nss" else
+    if optCryptopp != null then "cryptopp" else "none";
+
+  cryptoLibsMap = {
+    nss = [ optNss optNspr ];
+    cryptopp = [ optCryptopp ];
+    none = [ ];
+  };
+
+  getMeta = description: with lib; {
+     homepage = "https://ceph.io/en/";
+     inherit description;
+     license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+     maintainers = with maintainers; [ adev ak johanot krav ];
+     platforms = [ "x86_64-linux" "aarch64-linux" ];
+   };
+
+  ceph-common = with python.pkgs; buildPythonPackage {
+    pname = "ceph-common";
+    inherit src version;
+
+    sourceRoot = "ceph-${version}/src/python-common";
+
+    propagatedBuildInputs = [
+      pyyaml
+    ];
+
+    nativeCheckInputs = [
+      pytestCheckHook
+    ];
+
+    disabledTests = [
+      # requires network access
+      "test_valid_addr"
+    ];
+
+    meta = getMeta "Ceph common module for code shared by manager modules";
+  };
+
+  # Watch out for python <> boost compatibility
+  python = python310.override {
+    packageOverrides = self: super: {
+      sqlalchemy = super.sqlalchemy.overridePythonAttrs rec {
+        version = "1.4.46";
+        src = fetchPypi {
+          pname = "SQLAlchemy";
+          inherit version;
+          hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA=";
+        };
+        disabledTestPaths = [
+          "test/aaa_profiling"
+          "test/ext/mypy"
+        ];
+      };
+    };
+  };
+
+  boost = boost179.override {
+    enablePython = true;
+    inherit python;
+  };
+
+  # TODO: split this off in build and runtime environment
+  ceph-python-env = python.withPackages (ps: with ps; [
+    ceph-common
+
+    # build time
+    cython
+
+    # debian/control
+    bcrypt
+    cherrypy
+    influxdb
+    jinja2
+    kubernetes
+    natsort
+    numpy
+    pecan
+    prettytable
+    pyjwt
+    pyopenssl
+    python-dateutil
+    pyyaml
+    requests
+    routes
+    scikit-learn
+    scipy
+    setuptools
+    sphinx
+    virtualenv
+    werkzeug
+
+    # src/pybind/mgr/requirements-required.txt
+    cryptography
+    jsonpatch
+
+    # src/tools/cephfs/shell/setup.py
+    cmd2
+    colorama
+  ]);
+  inherit (ceph-python-env.python) sitePackages;
+
+  version = "18.2.0";
+  src = fetchurl {
+    url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
+    hash = "sha256:0k9nl6xi5brva51rr14m7ig27mmmd7vrpchcmqc40q3c2khn6ns9";
+  };
+in rec {
+  ceph = stdenv.mkDerivation {
+    pname = "ceph";
+    inherit src version;
+
+    nativeBuildInputs = [
+      cmake
+      fmt
+      git
+      makeWrapper
+      nasm
+      pkg-config
+      python
+      python.pkgs.python # for the toPythonPath function
+      python.pkgs.wrapPython
+      which
+      (ensureNewerSourcesHook { year = "1980"; })
+      # for building docs/man-pages presumably
+      doxygen
+      graphviz
+    ];
+
+    enableParallelBuilding = true;
+
+    buildInputs = cryptoLibsMap.${cryptoStr} ++ [
+      arrow-cpp
+      babeltrace
+      boost
+      bzip2
+      ceph-python-env
+      cryptsetup
+      cunit
+      gperf
+      gtest
+      icu
+      libcap
+      libnl
+      libxml2
+      lttng-ust
+      lua
+      lz4
+      malloc
+      oath-toolkit
+      openldap
+      optLibatomic_ops
+      optLibs3
+      optYasm
+      rdkafka
+      rocksdb'
+      snappy
+      sqlite
+      utf8proc
+      zlib
+      zstd
+    ] ++ lib.optionals stdenv.isLinux [
+      keyutils
+      libcap_ng
+      liburing
+      libuuid
+      linuxHeaders
+      optLibaio
+      optLibxfs
+      optZfs
+      rabbitmq-c
+      rdma-core
+      udev
+      util-linux
+    ] ++ lib.optionals hasRadosgw [
+      optCurl
+      optExpat
+      optFuse
+      optLibedit
+    ];
+
+    pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
+
+    preConfigure =''
+      substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
+      substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
+      substituteInPlace src/common/module.c --replace "/bin/grep" "grep"
+
+      # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
+      # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others
+      export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages}
+      patchShebangs src/
+    '';
+
+    cmakeFlags = [
+      "-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
+
+      "-DWITH_CEPHFS_SHELL:BOOL=ON"
+      "-DWITH_SYSTEMD:BOOL=OFF"
+      # `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
+      #     CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
+      #     Command failed: 2
+      #
+      #        'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
+      #
+      #     See also
+      #
+      #        /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
+      # and that file contains:
+      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
+      #     /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
+      #       71 |     ::THRIFT_CLOSESOCKET(socket_);
+      #          |       ^~~~~~~~~~~~~~~~~~
+      # Looks like `close()` is somehow not included.
+      # But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
+      # So it's proably not worth trying to fix that for this Ceph version,
+      # and instead just disable Ceph's Jaeger support.
+      "-DWITH_JAEGER:BOOL=OFF"
+      "-DWITH_TESTS:BOOL=OFF"
+
+      # Use our own libraries, where possible
+      "-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
+      "-DWITH_SYSTEM_BOOST:BOOL=ON"
+      "-DWITH_SYSTEM_GTEST:BOOL=ON"
+      "-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
+      "-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
+      "-DWITH_SYSTEM_ZSTD:BOOL=ON"
+
+      # TODO breaks with sandbox, tries to download stuff with npm
+      "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
+      # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
+      ''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
+    ] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON";
+
+    postFixup = ''
+      wrapPythonPrograms
+      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
+
+      # Test that ceph-volume exists since the build system has a tendency to
+      # silently drop it with misconfigurations.
+      test -f $out/bin/ceph-volume
+    '';
+
+    outputs = [ "out" "lib" "dev" "doc" "man" ];
+
+    doCheck = false; # uses pip to install things from the internet
+
+    # Takes 7+h to build with 2 cores.
+    requiredSystemFeatures = [ "big-parallel" ];
+
+    meta = getMeta "Distributed storage system";
+
+    passthru = {
+      inherit version;
+      tests = {
+        inherit (nixosTests)
+          ceph-multi-node
+          ceph-single-node
+          ceph-single-node-bluestore;
+      };
+    };
+  };
+
+  ceph-client = runCommand "ceph-client-${version}" {
+      meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
+    } ''
+      mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
+      cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
+      cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
+      cp -r ${ceph}/bin/rbd-replay* $out/bin
+      cp -r ${ceph}/sbin/mount.ceph $out/bin
+      cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
+      ln -s bin $out/sbin
+      cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
+      cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
+      # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
+      substituteInPlace $out/bin/ceph          --replace ${ceph} $out
+      substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
+   '';
+}
--- a/pkgs/mpich-fix-hwtopo.patch
+++ b/pkgs/mpich-fix-hwtopo.patch
@@ -1,36 +0,0 @@
-diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
-index 33e88bc..ee3641c 100644
--- a/src/util/mpir_hwtopo.c
-+++ b/src/util/mpir_hwtopo.c
-@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
- #ifdef HAVE_HWLOC
-     bindset = hwloc_bitmap_alloc();
-     hwloc_topology_init(&hwloc_topology);
-    char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
-    if (xmlfile != NULL) {
-        int rc;
-        rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
-        if (rc == 0) {
-            /* To have hwloc still actually call OS-specific hooks, the
-             * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
-             * file is really the underlying system. */
-            hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
-        }
-        MPL_free(xmlfile);
-    }
-
-     hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
-     if (!hwloc_topology_load(hwloc_topology)) 
-
--- a/src/mpi/init/local_proc_attrs.c
-+++ b/src/mpi/init/local_proc_attrs.c
-@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
-     /* Set the number of tag bits. The device may override this value. */
-     MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
-
-    char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
-    MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
-    MPL_free(requested_kinds);
-
-     return mpi_errno;
- }
--- a/pkgs/nix-socket.patch
+++ b/pkgs/nix-socket.patch
@@ -0,0 +1,11 @@
+--- a/misc/systemd/nix-daemon.socket.in	1970-01-01 01:00:01.000000000 +0100
+++ b/misc/systemd/nix-daemon.socket.in	2023-09-18 17:53:32.351760208 +0200
+@@ -1,7 +1,7 @@
+ [Unit]
+ Description=Nix Daemon Socket
+ Before=multi-user.target
+-RequiresMountsFor=@storedir@
+RequiresMountsFor=@localstatedir@
+ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
+ 
+ [Socket]
--- a/pkgs/overlay.nix
+++ b/pkgs/overlay.nix
@@ -1,45 +1,39 @@
 final: prev:
 {
-  # Set MPICH as default
-  mpi = final.mpich;
+  bsc = prev.bsc.extend (bscFinal: bscPrev: {
+    # Set MPICH as default
+    mpi = bscFinal.mpich;

-  # Configure the network for MPICH
-  mpich = with final; let
-    # pmix comes with the libraries in .out and headers in .dev
-    pmixAll = symlinkJoin {
-      name = "pmix-all";
-      paths = [ pmix.dev pmix.out ];
-    };
-  in prev.mpich.overrideAttrs (old: {
-    patches = [
-      # See https://github.com/pmodels/mpich/issues/6946
-      ./mpich-fix-hwtopo.patch
-    ];
-    buildInput = old.buildInputs ++ [
-      libfabric
-      pmixAll
-    ];
-    configureFlags = [
-      "--enable-shared"
-      "--enable-sharedlib"
-      "--with-pm=no"
-      "--with-device=ch4:ofi"
-      "--with-pmi=pmix"
-      "--with-pmix=${pmixAll}"
-      "--with-libfabric=${libfabric}"
-      "--enable-g=log"
-    ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
-      "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
-      "FCFLAGS=-fallow-argument-mismatch"
-    ];
+    # Configure the network for MPICH
+    mpich = with final; prev.mpich.overrideAttrs (old: {
+      buildInput = old.buildInputs ++ [
+        libfabric
+        pmix
+      ];
+      configureFlags = [
+        "--enable-shared"
+        "--enable-sharedlib"
+        "--with-pm=no"
+        "--with-device=ch4:ofi"
+        "--with-pmi=pmix"
+        "--with-pmix=${final.pmix}"
+        "--with-libfabric=${final.libfabric}"
+        "--enable-g=log"
+      ] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
+        "FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
+        "FCFLAGS=-fallow-argument-mismatch"
+      ];
+    });
  });

-  slurm = prev.slurm.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [
-      # See https://bugs.schedmd.com/show_bug.cgi?id=19324
-      ./slurm-rank-expansion.patch
-    ];
-  });
+  # Update ceph to 18.2.0 until it lands in nixpkgs, see:
+  # https://github.com/NixOS/nixpkgs/pull/247849
+  inherit (prev.callPackage ./ceph.nix {
+    lua = prev.lua5_4;
+    fmt = prev.fmt_8;
+  }) ceph ceph-client;

-  prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
+  nix = prev.nix.overrideAttrs (old: {
+    patches = old.patches ++ [ ./nix-socket.patch ];
+  });
 }
--- a/pkgs/slurm-exporter.nix
+++ b/pkgs/slurm-exporter.nix
@@ -1,22 +0,0 @@
-{ buildGoModule, fetchFromGitHub, lib }:
-
-buildGoModule rec {
-  pname = "prometheus-slurm-exporter";
-  version = "0.20";
-
-  src = fetchFromGitHub {
-    rev = version;
-    owner = "vpenso";
-    repo = pname;
-    sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
-  };
-
-  vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
-  doCheck = false;
-
-  meta = with lib; {
-    description = "Prometheus SLURM Exporter";
-    homepage = "https://github.com/vpenso/prometheus-slurm-exporter";
-    platforms = platforms.linux;
-  };
-}
--- a/pkgs/slurm-rank-expansion.patch
+++ b/pkgs/slurm-rank-expansion.patch
@@ -1,11 +0,0 @@
--- a/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:05:24.815313882 +0100
-+++ b/src/plugins/mpi/pmix/pmixp_dmdx.c	2024-03-15 13:09:53.936900823 +0100
-@@ -314,7 +314,7 @@ static void _dmdx_req(buf_t *buf, int no
- 	}
- 
- 	nsptr = pmixp_nspaces_local();
-	if (nsptr->ntasks <= rank) {
-+	if ((long) nsptr->ntasks <= (long) rank) {
- 		char *nodename = pmixp_info_job_host(nodeid);
- 		PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
- 			    nodename, ns, nsptr->ntasks, rank);
--- a/secrets/gitea-runner-token.age
+++ b/secrets/gitea-runner-token.age
@@ -1,9 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 HY2yRg DQdgCk16Yu524BsrWVf0krnwWzDM6SeaJCgQipOfwCA
-Ab9ocqra/UWJZI+QGMlxUhBu5AzqfjPgXl+ENIiHYGs
-> ssh-ed25519 CAWG4Q KF9rGCenb3nf+wyz2hyVs/EUEbsmUs5R+1fBxlCibC8
-7++Kxbr3FHVdVfnFdHYdAuR0Tgfd+sRcO6WRss6LhEw
-> ssh-ed25519 MSF3dg aUe4DhRsu4X8CFOEAnD/XM/o/0qHYSB522woCaAVh0I
-GRcs5cm2YqA/lGhUtbpboBaz7mfgiLaCr+agaB7vACU
--- 9Q7Ou+Pxq+3RZilCb2dKC/pCFjZEt4rp5KnTUUU7WJ8
-1<12>Mw4<77><34>	<09>:H<>@<40>/<2F>gLtM<74>,<2C>ƥ<>*<2A><>z<EFBFBD>NV5<56>m<EFBFBD><6D>N<EFBFBD>o<EFBFBD><6F>j1$<24>T<EFBFBD>G_<47>E{<7B>%<25><>1ǯ<17><>H<EFBFBD><EFBFBD>A<EFBFBD>p<EFBFBD>
--- a/secrets/gitlab-bsc-es-token.age
+++ b/secrets/gitlab-bsc-es-token.age
@@ -1,11 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 HY2yRg caTbx0NBmsTSmZH4HtBaxhsauWqWUDTesJqT08UsoEQ
-8ND31xuco+H8d5SKg8xsCFRPVDhU4d8UKwV1BnmKVjQ
-> ssh-ed25519 CAWG4Q 4ETYuhCwHHECkut4DWDknMMgpAvFqtzLWVC2Wi2L8FM
-BGMvRnAfd8qZG5hzLefmk32FkGvwzE9pqBUyx4JY0co
-> ssh-ed25519 MSF3dg hj5QL4ZfylN8/W/MXQHvVqtI7mRvlQOYr8HsaQEmPB0
-kvB7sljmmkswSGZDQnrwdTbTsN78EAwH3pz1pPe0Hu0
-> )Q-grease vHF} [8p1> @7z;C"/
-tgSUKFyyrf2jLXZp+pakigwB2fRO/WFj2Qnt1aPjtVPEK92JbJ4
--- xzM0AhV4gTQE0Q7inJNo9vFj+crJQxWeI7u9pl7bqAI
-<05>6nGJ<47>0B<03><>7F<37><46><EFBFBD>b<EFBFBD>ٽ2<D9BD>L<EFBFBD><4C><EFBFBD>]<18>2zl<7A><6C>&e<>K<EFBFBD>x<EFBFBD><78><EFBFBD>9SWN<19>V"Mf<4D><66><EFBFBD><EFBFBD>KHUC:1b;9St<53><74><EFBFBD><EFBFBD>Duѧ<75><D1A7>Ϣ<EFBFBD>̟<04><02><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>f<EFBFBD>7<1D><>1<>I(<28>d<><64><EFBFBD><EFBFBD><EFBFBD><0C>
--- a/secrets/jungle-robot-password.age
+++ b/secrets/jungle-robot-password.age
@@ -1,10 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 HY2yRg 3L1Y5upc5qN6fgiFAox5rD/W8n0eQUv5mT39QAdO5Ac
-XkWsmPmzRgHjsvJgsDKJRgHZ7/sBZFmd1Doppj/y390
-> ssh-ed25519 CAWG4Q v03Qr+fckdIpsxvQG/viKxlF8WNpO4XUe//QcPzH4k0
-afUwi3ccDCRfUxPDdF7ZkoL+0UX1XwqVtiyabDWjVQk
-> ssh-ed25519 MSF3dg c2hEUk4LslJpiL7v/4UpT8fK7ZiBJ8+uRhZ/vBoRUDE
-YX9EpnJpHo1eDsZtapTVY6jD+81kb588Oik4NoY9jro
--- LhUkopNtCsyHCLzEYzBFs+vekOkAR4B3VBaiMF/ZF8w
-o˝<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>C<EFBFBD><EFBFBD>H<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>y<EFBFBD>LؔItM<74><4D><EFBFBD><EFBFBD>IױsM<12>\<5C>1-<2D>K<EFBFBD><4B><10><>G:<><7F><02><>
-<EFBFBD>g<16><><EFBFBD>bpF<70>Ӷ<EFBFBD>%Y<>
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@@ -6,12 +6,9 @@ let
  safe = keys.hostGroup.safe ++ adminsKeys;
 in
 {
-  "gitlab-bsc-es-token.age".publicKeys = hut;
-  "gitea-runner-token.age".publicKeys = hut;
  "ovni-token.age".publicKeys = hut;
  "nosv-token.age".publicKeys = hut;
  "nix-serve.age".publicKeys = hut;
-  "jungle-robot-password.age".publicKeys = hut;

  "ceph-user.age".publicKeys = safe;
  "munge-key.age".publicKeys = safe;
--- a/web/content/git/_index.md
+++ b/web/content/git/_index.md
@@ -1,6 +0,0 @@
---
-title: "Git"
-description: "Gitea instance"
---
-
-If you are reading this page, the proxy to the Gitea service is not working.
--- a/web/content/lists/_index.md
+++ b/web/content/lists/_index.md
@@ -1,6 +0,0 @@
---
-title: "Lists"
-description: "Mailing lists"
---
-
-If you are reading this page, the proxy to the public-inbox service is not working.
--- a/web/content/owl/_index.md
+++ b/web/content/owl/_index.md
@@ -8,11 +8,3 @@ description: "Low system noise"
 Much like the silent flight of an owl at night, these nodes are configured to
 minimize the system noise and let programs run undisturbed. The list of nodes is
 `owl[1-2]` and are available for jobs with SLURM.
-
-The contents of the nix store of the hut node is made available in the owl nodes
-when a job is running. This allows jobs to access the same paths that are on hut
-to load dependencies.
-
-For now, only the hut node can be used to build new derivations so that they
-appear in the compute nodes. This applies to the `nix build`, `nix develop` and
-`nix shell` commands.
--- a/web/hugo.toml
+++ b/web/hugo.toml
@@ -2,27 +2,4 @@ baseURL = 'https://jungle.bsc.es/'
 languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
-
-[[menu.main]]
-identifier = "grafana"
-name = "Grafana"
-url = "/grafana/"
-weight = 10
-
-[[menu.main]]
-identifier = "Git"
-name = "Git"
-url = "/git/"
-weight = 20
-
-[[menu.main]]
-identifier = "Lists"
-name = "Lists"
-url = "/lists/"
-weight = 30
-
-[[menu.main]]
-identifier = "Posts"
-name = "Posts"
-url = "/posts/"
-weight = 40
+sectionPagesMenu = "main"
--- a/web/themes/PaperMod/layouts/partials/header.html
+++ b/web/themes/PaperMod/layouts/partials/header.html
@@ -100,9 +100,9 @@
                {{- $separator := or $label_text (not site.Params.disableThemeToggle)}}
                {{- with site.Home.AllTranslations }}
                <ul class="lang-switch">
+                    {{- if $separator }}<li>|</li>{{ end }}
                    {{- range . -}}
                    {{- if ne $lang .Lang }}
-                    {{- if $separator }}<li>|</li>{{ end }}
                    <li>
                        <a href="{{- .Permalink -}}" title="{{ .Language.Params.languageAltTitle | default (.Language.LanguageName | emojify) | default (.Lang | title) }}"
                            aria-label="{{ .Language.LanguageName | default (.Lang | title) }}">
--- a/web/themes/PaperMod/layouts/partials/post_meta.html
+++ b/web/themes/PaperMod/layouts/partials/post_meta.html
@@ -17,5 +17,5 @@
 {{- end }}

 {{- with ($scratch.Get "meta") }}
-{{- delimit . "&nbsp;·&nbsp;" | safeHTML  -}}
+{{- delimit . "&nbsp;·&nbsp;" -}}
 {{- end -}}
Author	SHA1	Message	Date
Rodrigo Arias Mallo	e065cde376	Use NixOS attributes for the install section	2023-09-18 19:27:14 +02:00
Rodrigo Arias Mallo	3bb0b550aa	Add a RequiredBy dependency for remote-fs.target	2023-09-18 19:05:58 +02:00
Rodrigo Arias Mallo	e4cbcab81c	Use a systemd mount directly for the nix store Allows the LazyUnmount option and avoids the stage1 hack with /nix//store.	2023-09-18 18:53:40 +02:00
Rodrigo Arias Mallo	77b41a90e2	Patch nix instead of using an override unit	2023-09-18 18:06:51 +02:00
Rodrigo Arias Mallo	1fc6891dc6	Remove nix-daemon.socket dependency of /nix/store The dependency causes a cycle as the nix store will be mounted after the network is ready, which itself depends on the socket.target which requires the nix-daemon.socket to be ready too.	2023-09-18 17:28:47 +02:00
Rodrigo Arias Mallo	8c11c7460a	Delay the mount until the network is ready	2023-09-18 16:07:46 +02:00
Rodrigo Arias Mallo	e6014511f5	Wait for the NFS hut store to be mounted	2023-09-18 15:50:37 +02:00
Rodrigo Arias Mallo	320c58ce48	Prevent the overlay to be mounted in stage1	2023-09-18 13:57:41 +02:00
Rodrigo Arias Mallo	d145ee9b2c	Mount the overlay in /nix/store	2023-09-18 13:02:32 +02:00
Rodrigo Arias Mallo	140178d58e	Begin the nix store overlay We need to disable the read-only bind mount, so we can directly bind mount the overlay.	2023-09-18 11:22:24 +02:00
Rodrigo Arias Mallo	d48f3b989a	Enable direnv integration	2023-09-17 22:27:51 +02:00