Replace xeon07 by hut in ssh config

The xeon07 machine has been renamed to hut.

Reviewed-by: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>

.gitignore

@@ -1,2 +1,3 @@
 *.swp
 /result
+/misc

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1720546205,
+        "lastModified": 1750173260,
-        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
+        "narHash": "sha256-9P1FziAwl5+3edkfFcr5HeGtQUtrSdk/MksX39GieoA=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
+        "rev": "531beac616433bac6f9e2a19feb8e99a22a66baf",
         "type": "github"
       },
       "original": {
@@ -30,11 +30,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1713974364,
+        "lastModified": 1749650500,
-        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
+        "narHash": "sha256-2MHfVPV6RA7qPSCtXh4+KK0F0UjN+J4z8//+n6NK7Xs=",
         "ref": "refs/heads/master",
-        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
+        "rev": "9d1944c658929b6f98b3f3803fead4d1b91c4405",
-        "revCount": 937,
+        "revCount": 961,
         "type": "git",
         "url": "https://git.sr.ht/~rodarima/bscpkgs"
       },
@@ -51,11 +51,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1700795494,
+        "lastModified": 1744478979,
-        "narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
+        "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
+        "rev": "43975d782b418ebf4969e9ccba82466728c2851b",
         "type": "github"
       },
       "original": {
@@ -73,11 +73,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1703113217,
+        "lastModified": 1745494811,
-        "narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
+        "narHash": "sha256-YZCh2o9Ua1n9uCvrvi5pRxtuVNml8X2a03qIFfRKpFs=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
+        "rev": "abfad3d2958c9e6300a883bd443512c55dfeb1be",
         "type": "github"
       },
       "original": {
@@ -88,16 +88,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1720957393,
+        "lastModified": 1752436162,
-        "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
+        "narHash": "sha256-Kt1UIPi7kZqkSc5HVj6UY5YLHHEzPBkgpNUByuyxtlw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
+        "rev": "dfcd5b901dbab46c9c6e80b265648481aafb01f8",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
+        "ref": "nixos-25.05",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,6 +1,6 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
@@ -18,6 +18,7 @@ in
   {
     nixosConfigurations = {
       hut     = mkConf "hut";
+      tent    = mkConf "tent";
       owl1    = mkConf "owl1";
       owl2    = mkConf "owl2";
       eudy    = mkConf "eudy";
@@ -25,6 +26,9 @@ in
       bay     = mkConf "bay";
       lake2   = mkConf "lake2";
       raccoon = mkConf "raccoon";
+      fox     = mkConf "fox";
+      apex    = mkConf "apex";
+      weasel  = mkConf "weasel";
     };
 
     packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

keys.nix

@@ -2,28 +2,35 @@
 # here all the public keys
 rec {
   hosts = {
-    hut   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
+    hut    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
-    owl1  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
+    owl1   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
-    owl2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
+    owl2   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
-    eudy  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
+    eudy   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
-    koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
+    koro   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
-    bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
+    bay    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
-    lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    lake2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
+    tent   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
+    apex   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
+    weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
   };
 
   hostGroup = with hosts; rec {
+    untrusted  = [ fox ];
     compute    = [ owl1 owl2 ];
-    playground = [ eudy koro ];
+    playground = [ eudy koro weasel ];
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
+    login      = [ apex ];
 
-    system     = storage ++ monitor;
+    system     = storage ++ monitor ++ login;
     safe       = system ++ compute;
     all        = safe ++ playground;
   };
 
   admins = {
-    rarias = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
+    "rarias@hut"  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIE1oZTPtlEXdGt0Ak+upeCIiBdaDQtcmuWoTUCVuSVIR rarias@hut";
-    root   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
+    "rarias@tent" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIwlWSBTZi74WTz5xn6gBvTmCoVltmtIAeM3RMmkh4QZ rarias@tent";
+    root          = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb root@hut";
   };
 }

m/apex/configuration.nix

@@ -0,0 +1,83 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../common/ssf/hosts.nix
+    ../module/ceph.nix
+    ./nfs.nix
+  ];
+
+  # Don't install grub MBR for now
+  boot.loader.grub.device = "nodev";
+
+  boot.initrd.kernelModules = [
+    "megaraid_sas" # For HW RAID
+  ];
+
+  environment.systemPackages = with pkgs; [
+    storcli # To manage HW RAID
+  ];
+
+  fileSystems."/home" = {
+    device = "/dev/disk/by-label/home";
+    fsType = "ext4";
+  };
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  networking = {
+    hostName = "apex";
+    defaultGateway = "84.88.53.233";
+    nameservers = [ "8.8.8.8" ];
+
+    # Public facing interface
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "84.88.53.236";
+      prefixLength = 29;
+    } ];
+
+    # Internal LAN to our Ethernet switch
+    interfaces.eno2.ipv4.addresses = [ {
+      address = "10.0.40.30";
+      prefixLength = 24;
+    } ];
+
+    # Infiniband over Omnipath switch (disconnected for now)
+    # interfaces.ibp5s0 = {};
+
+    nat = {
+      enable = true;
+      internalInterfaces = [ "eno2" ];
+      externalInterface = "eno1";
+    };
+  };
+
+  # Use SSH tunnel to reach internal hosts
+  programs.ssh.extraConfig = ''
+    Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+    Host raccoon
+      HostName knights3.bsc.es
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+    Host tent
+      ProxyJump raccoon
+  '';
+
+  networking.firewall = {
+    extraCommands = ''
+      # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our
+      # logs. Insert as first position so we also protect SSH.
+      iptables -I nixos-fw 1 -p tcp -s 192.168.8.16 -j nixos-fw-refuse
+      # Same with opsmonweb01.bsc.es which seems to be trying to access via SSH
+      iptables -I nixos-fw 2 -p tcp -s 84.88.52.176 -j nixos-fw-refuse
+    '';
+  };
+
+  # Use tent for cache
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+}

m/apex/nfs.nix

@@ -0,0 +1,32 @@
+{ ... }:
+
+{
+  services.nfs.server = {
+    enable = true;
+    lockdPort = 4001;
+    mountdPort = 4002;
+    statdPort = 4000;
+    exports = ''
+      /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+    '';
+  };
+  networking.firewall = {
+    # Check with `rpcinfo -p`
+    extraCommands = ''
+      # Accept NFS traffic from compute nodes but not from the outside
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      # Same but UDP
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+    '';
+  };
+}

m/bay/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/monitoring.nix
   ];
 

m/common/base/env.nix

@@ -3,8 +3,8 @@
 {
   environment.systemPackages = with pkgs; [
     vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    nix-diff ipmitool freeipmi ethtool lm_sensors cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
+    ncdu config.boot.kernelPackages.perf ldns pv
     # From bsckgs overlay
     osumb
   ];
@@ -21,6 +21,8 @@
     }
   ];
 
+  environment.enableAllTerminfo = true;
+
   environment.variables = {
     EDITOR = "vim";
     VISUAL = "vim";

m/common/base/net.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, lib, ... }:
 
 {
   networking = {
@@ -10,8 +10,11 @@
       allowedTCPPorts = [ 22 ];
     };
 
+    # Make sure we use iptables
+    nftables.enable = lib.mkForce false;
+
     hosts = {
-      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.53.236" = [ "apex" "ssfhead.bsc.es" "ssfhead" ];
       "84.88.51.152" = [ "raccoon" ];
       "84.88.51.142" = [ "raccoon-ipmi" ];
     };

m/common/base/nix.nix

@@ -6,6 +6,8 @@
     (import ../../../pkgs/overlay.nix)
   ];
 
+  nixpkgs.config.allowUnfree = true;
+
   nix = {
     nixPath = [
       "nixpkgs=${nixpkgs}"
@@ -23,6 +25,7 @@
       trusted-users = [ "@wheel" ];
       flake-registry = pkgs.writeText "global-registry.json"
         ''{"flakes":[],"version":2}'';
+      keep-outputs = true;
     };
 
     gc = {
@@ -32,6 +35,21 @@
     };
   };
 
+  # The nix-gc.service can begin its execution *before* /home is mounted,
+  # causing it to remove all gcroots considering them as stale, as it cannot
+  # access the symlink. To prevent this problem, we force the service to wait
+  # until /home is mounted as well as other remote FS like /ceph.
+  systemd.services.nix-gc = {
+    # Start remote-fs.target if not already being started and fail if it fails
+    # to start. It will also be stopped if the remote-fs.target fails after
+    # starting successfully.
+    bindsTo = [ "remote-fs.target" ];
+    # Wait until remote-fs.target fully starts before starting this one.
+    after = [ "remote-fs.target"];
+    # Ensure we can access a remote path inside /home
+    unitConfig.ConditionPathExists = "/home/Computational";
+  };
+
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions
   # on your system were taken. It‘s perfectly fine and recommended to leave

m/common/base/ssh.nix

@@ -8,15 +8,11 @@ in
   # Enable the OpenSSH daemon.
   services.openssh.enable = true;
 
-  # Connect to intranet git hosts via proxy
-  programs.ssh.extraConfig = ''
-    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
-      User git
-      ProxyCommand nc -X connect -x hut:23080 %h %p
-  '';
-
   programs.ssh.knownHosts = hostsKeys // {
     "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
     "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
+    "bscpm04.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT";
+    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
+    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
   };
 }

m/common/base/users.nix

@@ -20,6 +20,7 @@
       rarias = {
         uid = 1880;
         isNormalUser = true;
+        linger = true;
         home = "/home/Computational/rarias";
         description = "Rodrigo Arias";
         group = "Computational";
@@ -39,7 +40,7 @@
         home = "/home/Computational/arocanon";
         description = "Aleix Roca";
         group = "Computational";
-        extraGroups = [ "wheel" ];
+        extraGroups = [ "wheel" "tracing" ];
         hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
@@ -55,7 +56,7 @@
         home = "/home/Computational/rpenacob";
         description = "Raúl Peñacoba";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" ];
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "fox" ];
         hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
@@ -68,10 +69,10 @@
         home = "/home/Computational/anavarro";
         description = "Antoni Navarro";
         group = "Computational";
-        hosts = [ "hut" "raccoon" ];
+        hosts = [ "apex" "hut" "tent" "raccoon" "fox" "weasel" ];
-        hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
+        hashedPassword = "$6$EgturvVYXlKgP43g$gTN78LLHIhaF8hsrCXD.O6mKnZSASWSJmCyndTX8QBWT6wTlUhcWVAKz65lFJPXjlJA4u7G1ydYQ0GG6Wk07b1";
         openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMsbM21uepnJwPrRe6jYFz8zrZ6AYMtSEvvt4c9spmFP toni@delltoni"
         ];
       };
 
@@ -81,7 +82,7 @@
         home = "/home/Computational/abonerib";
         description = "Aleix Boné";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "raccoon" ];
+        hosts = [ "apex" "owl1" "owl2" "hut" "tent" "raccoon" "fox" "weasel" ];
         hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
@@ -94,16 +95,70 @@
         home = "/home/Computational/vlopez";
         description = "Victor López";
         group = "Computational";
-        hosts = [ "koro" ];
+        hosts = [ "apex" "koro" ];
         hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
         ];
       };
+
+      dbautist = {
+        uid = 5649;
+        isNormalUser = true;
+        home = "/home/Computational/dbautist";
+        description = "Dylan Bautista Cases";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "raccoon" ];
+        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
+        ];
+      };
+
+      dalvare1 = {
+        uid = 2758;
+        isNormalUser = true;
+        home = "/home/Computational/dalvare1";
+        description = "David Álvarez";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "fox" ];
+        hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
+        ];
+      };
+
+      varcila = {
+        uid = 5650;
+        isNormalUser = true;
+        home = "/home/Computational/varcila";
+        description = "Vincent Arcila";
+        group = "Computational";
+        hosts = [ "apex" "hut" "tent" "fox" ];
+        hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
+        ];
+      };
+
+      pmartin1 = {
+        # Arbitrary UID but large so it doesn't collide with other users on ssfhead.
+        uid = 9652;
+        isNormalUser = true;
+        home = "/home/Computational/pmartin1";
+        description = "Pedro J. Martinez-Ferrer";
+        group = "Computational";
+        hosts = [ "fox" ];
+        hashedPassword = "$6$nIgDMGnt4YIZl3G.$.JQ2jXLtDPRKsbsJfJAXdSvjDIzRrg7tNNjPkLPq3KJQhMjfDXRUvzagUHUU2TrE2hHM8/6uq8ex0UdxQ0ysl.";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
+        ];
+      };
     };
 
     groups = {
       Computational = { gid = 564; };
+      tracing = { };
     };
   };
 }

m/common/ssf.nix

@@ -0,0 +1,10 @@
+{
+  # Provides the base system for a xeon node in the SSF rack.
+  imports = [
+    ./xeon.nix
+    ./ssf/fs.nix
+    ./ssf/hosts.nix
+    ./ssf/net.nix
+    ./ssf/ssh.nix
+  ];
+}

m/common/ssf/hosts.nix

@@ -0,0 +1,23 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Login
+    "10.0.40.30" = [ "apex" ];
+
+    # Storage
+    "10.0.40.40" = [ "bay" ];   "10.0.42.40" = [ "bay-ib" ];    "10.0.40.141" = [ "bay-ipmi" ];
+    "10.0.40.41" = [ "oss01" ]; "10.0.42.41" = [ "oss01-ib0" ]; "10.0.40.142" = [ "oss01-ipmi" ];
+    "10.0.40.42" = [ "lake2" ]; "10.0.42.42" = [ "lake2-ib" ];  "10.0.40.143" = [ "lake2-ipmi" ];
+
+    # Xeon compute
+    "10.0.40.1" = [ "owl1" ];   "10.0.42.1" = [ "owl1-ib" ];   "10.0.40.101" = [ "owl1-ipmi" ];
+    "10.0.40.2" = [ "owl2" ];   "10.0.42.2" = [ "owl2-ib" ];   "10.0.40.102" = [ "owl2-ipmi" ];
+    "10.0.40.3" = [ "xeon03" ]; "10.0.42.3" = [ "xeon03-ib" ]; "10.0.40.103" = [ "xeon03-ipmi" ];
+    #"10.0.40.4" = [ "tent" ];   "10.0.42.4" = [ "tent-ib" ];   "10.0.40.104" = [ "tent-ipmi" ];
+    "10.0.40.5" = [ "koro" ];   "10.0.42.5" = [ "koro-ib" ];   "10.0.40.105" = [ "koro-ipmi" ];
+    "10.0.40.6" = [ "weasel" ]; "10.0.42.6" = [ "weasel-ib" ]; "10.0.40.106" = [ "weasel-ipmi" ];
+    "10.0.40.7" = [ "hut" ];    "10.0.42.7" = [ "hut-ib" ];    "10.0.40.107" = [ "hut-ipmi" ];
+    "10.0.40.8" = [ "eudy" ];   "10.0.42.8" = [ "eudy-ib" ];   "10.0.40.108" = [ "eudy-ipmi" ];
+  };
+}

m/common/ssf/net.nix

@@ -0,0 +1,23 @@
+{ pkgs, ... }:
+
+{
+  # Infiniband (IPoIB)
+  environment.systemPackages = [ pkgs.rdma-core ];
+  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
+
+  networking = {
+    defaultGateway = "10.0.40.30";
+    nameservers = ["8.8.8.8"];
+
+    firewall = {
+      extraCommands = ''
+        # Prevent ssfhead from contacting our slurmd daemon
+        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
+        # But accept traffic to slurm ports from any other node in the subnet
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
+        # We also need to open the srun port range
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
+      '';
+    };
+  };
+}

m/common/ssf/ssh.nix

@@ -0,0 +1,16 @@
+{
+  # Use SSH tunnel to apex to reach internal hosts
+  programs.ssh.extraConfig = ''
+    Host tent
+      ProxyJump raccoon
+
+    # Access raccoon via the HTTP proxy
+    Host raccoon knights3.bsc.es
+      HostName knights3.bsc.es
+      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
+
+    # Make sure we can reach gitlab even if we don't have SSH access to raccoon
+    Host bscpm04.bsc.es gitlab-internal.bsc.es
+      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
+  '';
+}

m/common/xeon.nix

@@ -1,9 +1,7 @@
 {
-  # Provides the base system for a xeon node.
+  # Provides the base system for a xeon node, not necessarily in the SSF rack.
   imports = [
     ./base.nix
-    ./xeon/fs.nix
     ./xeon/console.nix
-    ./xeon/net.nix
   ];
 }

m/common/xeon/net.nix

@@ -1,90 +0,0 @@
-{ pkgs, ... }:
-
-{
-  # Infiniband (IPoIB)
-  environment.systemPackages = [ pkgs.rdma-core ];
-  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
-
-  networking = {
-    defaultGateway = "10.0.40.30";
-    nameservers = ["8.8.8.8"];
-
-    proxy = {
-      default = "http://hut:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
-      # Don't set all_proxy as go complains and breaks the gitlab runner, see:
-      # https://github.com/golang/go/issues/16715
-      allProxy = null;
-    };
-
-    firewall = {
-      extraCommands = ''
-        # Prevent ssfhead from contacting our slurmd daemon
-        iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
-        # But accept traffic to slurm ports from any other node in the subnet
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
-        # We also need to open the srun port range
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
-      '';
-    };
-
-    extraHosts = ''
-      10.0.40.30              ssfhead
-      
-      # Node Entry for node: mds01 (ID=72)
-      10.0.40.40              bay mds01 mds01-eth0
-      10.0.42.40              bay-ib mds01-ib0
-      10.0.40.141             bay-ipmi mds01-ipmi0
-      
-      # Node Entry for node: oss01 (ID=73)
-      10.0.40.41              oss01 oss01-eth0
-      10.0.42.41              oss01-ib0
-      10.0.40.142             oss01-ipmi0
-      
-      # Node Entry for node: oss02 (ID=74)
-      10.0.40.42              lake2 oss02 oss02-eth0
-      10.0.42.42              lake2-ib oss02-ib0
-      10.0.40.143             lake2-ipmi oss02-ipmi0
-      
-      # Node Entry for node: xeon01 (ID=15)
-      10.0.40.1               owl1 xeon01 xeon01-eth0
-      10.0.42.1               owl1-ib xeon01-ib0
-      10.0.40.101             owl1-ipmi xeon01-ipmi0
-      
-      # Node Entry for node: xeon02 (ID=16)
-      10.0.40.2               owl2 xeon02 xeon02-eth0
-      10.0.42.2               owl2-ib xeon02-ib0
-      10.0.40.102             owl2-ipmi xeon02-ipmi0
-      
-      # Node Entry for node: xeon03 (ID=17)
-      10.0.40.3               xeon03 xeon03-eth0
-      10.0.42.3               xeon03-ib0
-      10.0.40.103             xeon03-ipmi0
-      
-      # Node Entry for node: xeon04 (ID=18)
-      10.0.40.4               xeon04 xeon04-eth0
-      10.0.42.4               xeon04-ib0
-      10.0.40.104             xeon04-ipmi0
-      
-      # Node Entry for node: xeon05 (ID=19)
-      10.0.40.5               koro xeon05 xeon05-eth0
-      10.0.42.5               koro-ib xeon05-ib0
-      10.0.40.105             koro-ipmi xeon05-ipmi0
-      
-      # Node Entry for node: xeon06 (ID=20)
-      10.0.40.6               xeon06 xeon06-eth0
-      10.0.42.6               xeon06-ib0
-      10.0.40.106             xeon06-ipmi0
-      
-      # Node Entry for node: xeon07 (ID=21)
-      10.0.40.7               hut xeon07 xeon07-eth0
-      10.0.42.7               hut-ib xeon07-ib0
-      10.0.40.107             hut-ipmi xeon07-ipmi0
-      
-      # Node Entry for node: xeon08 (ID=22)
-      10.0.40.8               eudy xeon08 xeon08-eth0
-      10.0.42.8               eudy-ib xeon08-ib0
-      10.0.40.108             eudy-ipmi xeon08-ipmi0
-    '';
-  };
-}

m/eudy/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix

m/fox/configuration.nix

@@ -0,0 +1,83 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+    ../common/xeon/console.nix
+    ../module/emulation.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.kernelModules = [ "kvm-amd" "amd_uncore" ];
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+  hardware.cpu.intel.updateMicrocode = lib.mkForce false;
+
+  # Use performance for benchmarks
+  powerManagement.cpuFreqGovernor = "performance";
+
+  # Disable NUMA balancing
+  boot.kernel.sysctl."kernel.numa_balancing" = 0;
+
+  # Expose kernel addresses
+  boot.kernel.sysctl."kernel.kptr_restrict" = 0;
+
+  services.openssh.settings.X11Forwarding = true;
+
+  networking = {
+    timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
+    hostName = "fox";
+    # UPC network (may change over time, use DHCP)
+    # Public IP configuration:
+    # - Hostname: fox.ac.upc.edu
+    # - IP: 147.83.30.141
+    # - Gateway: 147.83.30.130
+    # - NetMask: 255.255.255.192
+    # Private IP configuration for BMC:
+    # - Hostname: fox-ipmi.ac.upc.edu
+    # - IP: 147.83.35.27
+    # - Gateway: 147.83.35.2
+    # - NetMask: 255.255.255.0
+    interfaces.enp1s0f0np0.useDHCP = true;
+  };
+
+  # Use hut for cache
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.nvidia.open = true;
+  hardware.graphics.enable = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  # Mount NVME disks
+  fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
+  fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
+
+  # Make a /nvme{0,1}/$USER directory for each user.
+  systemd.services.create-nvme-dirs = let
+    # Take only normal users in fox
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /nvme{0,1}/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-nvme-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+}

m/hut/blackbox.yml

@@ -3,160 +3,12 @@ modules:
     prober: http
     timeout: 5s
     http:
-      proxy_url: "http://127.0.0.1:23080"
-      skip_resolve_phase_with_proxy: true
-      follow_redirects: true
-      valid_status_codes: []  # Defaults to 2xx
-      method: GET
-  http_with_proxy:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      skip_resolve_phase_with_proxy: true
-  http_with_proxy_and_headers:
-    prober: http
-    http:
-      proxy_url: "http://127.0.0.1:3128"
-      proxy_connect_header:
-        Proxy-Authorization:
-          - Bearer token
-  http_post_2xx:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Content-Type: application/json
-      body: '{}'
-  http_post_body_file:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      body_file: "/files/body.txt"
-  http_basic_auth_example:
-    prober: http
-    timeout: 5s
-    http:
-      method: POST
-      headers:
-        Host: "login.example.com"
-      basic_auth:
-        username: "username"
-        password: "mysecret"
-  http_2xx_oauth_client_credentials:
-    prober: http
-    timeout: 5s
-    http:
-      valid_http_versions: ["HTTP/1.1", "HTTP/2"]
       follow_redirects: true
       preferred_ip_protocol: "ip4"
-      valid_status_codes:
+      valid_status_codes: []  # Defaults to 2xx
-        - 200
-        - 201
-      oauth2:
-        client_id: "client_id"
-        client_secret: "client_secret"
-        token_url: "https://api.example.com/token"
-        endpoint_params:
-          grant_type: "client_credentials"
-  http_custom_ca_example:
-    prober: http
-    http:
       method: GET
-      tls_config:
-        ca_file: "/certs/my_cert.crt"
-  http_gzip:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-  http_gzip_with_accept_encoding:
-    prober: http
-    http:
-      method: GET
-      compression: gzip
-      headers:
-        Accept-Encoding: gzip
-  tls_connect:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      tls: true
-  tcp_connect_example:
-    prober: tcp
-    timeout: 5s
-  imap_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "OK.*STARTTLS"
-        - send: ". STARTTLS"
-        - expect: "OK"
-        - starttls: true
-        - send: ". capability"
-        - expect: "CAPABILITY IMAP4rev1"
-  smtp_starttls:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - expect: "^220 ([^ ]+) ESMTP (.+)$"
-        - send: "EHLO prober\r"
-        - expect: "^250-STARTTLS"
-        - send: "STARTTLS\r"
-        - expect: "^220"
-        - starttls: true
-        - send: "EHLO prober\r"
-        - expect: "^250-AUTH"
-        - send: "QUIT\r"
-  irc_banner_example:
-    prober: tcp
-    timeout: 5s
-    tcp:
-      query_response:
-        - send: "NICK prober"
-        - send: "USER prober prober prober :prober"
-        - expect: "PING :([^ ]+)"
-          send: "PONG ${1}"
-        - expect: "^:[^ ]+ 001"
   icmp:
     prober: icmp
     timeout: 5s
     icmp:
       preferred_ip_protocol: "ip4"
-  dns_udp_example:
-    prober: dns
-    timeout: 5s
-    dns:
-      query_name: "www.prometheus.io"
-      query_type: "A"
-      valid_rcodes:
-        - NOERROR
-      validate_answer_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-        fail_if_all_match_regexp:
-          - ".*127.0.0.1"
-        fail_if_not_matches_regexp:
-          - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
-        fail_if_none_matches_regexp:
-          - "127.0.0.1"
-      validate_authority_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-      validate_additional_rrs:
-        fail_if_matches_regexp:
-          - ".*127.0.0.1"
-  dns_soa:
-    prober: dns
-    dns:
-      query_name: "prometheus.io"
-      query_type: "SOA"
-  dns_tcp_example:
-    prober: dns
-    dns:
-      transport_protocol: "tcp" # defaults to "udp"
-      preferred_ip_protocol: "ip4" # defaults to "ip6"
-      query_name: "www.prometheus.io"

m/hut/configuration.nix

@@ -1,8 +1,8 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
 
     ../module/ceph.nix
     ../module/debuginfod.nix
@@ -23,11 +23,21 @@
   ];
 
   # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
 
-  fileSystems."/nvme" = {
+  fileSystems = {
-    fsType = "ext4";
+    "/" = lib.mkForce {
-    device = "/dev/disk/by-label/nvme";
+      device = "/dev/disk/by-label/nvme";
+      fsType = "ext4";
+      neededForBoot = true;
+      options = [ "noatime" ];
+    };
+
+    "/boot" = lib.mkForce {
+      device = "/dev/disk/by-label/nixos-boot";
+      fsType = "ext4";
+      neededForBoot = true;
+    };
   };
 
   networking = {
@@ -46,6 +56,11 @@
         iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
       '';
+      # Flush all rules and chains on stop so it won't break on start
+      extraStopCommands = ''
+        iptables -F
+        iptables -X
+      '';
     };
   };
 

m/hut/gitlab-runner.nix

@@ -1,8 +1,9 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age;
+  age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age;
-  age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age;
 
   services.gitlab-runner = {
     enable = true;
@@ -21,21 +22,90 @@
           "--docker-network-mode host"
         ];
         environmentVariables = {
-          https_proxy = "http://localhost:23080";
+          https_proxy = "http://hut:23080";
-          http_proxy = "http://localhost:23080";
+          http_proxy = "http://hut:23080";
         };
       };
     in {
       # For pm.bsc.es/gitlab
       gitlab-pm-shell = common-shell // {
-        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path;
       };
       gitlab-pm-docker = common-docker // {
-        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path;
+      };
+
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
+        tagList = [ "docker" "hut" ];
+        environmentVariables = {
+          # We cannot access the hut local interface from docker, so we connect
+          # to hut directly via the ethernet one.
+          https_proxy = "http://hut:23080";
+          http_proxy = "http://hut:23080";
+        };
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerExtraHosts = [
+          # Required to pass the proxy via hut
+          "hut:10.0.40.7"
+        ];
+        dockerDisableCache = true;
+        registrationFlags = [
+          # Increase build log length to 64 MiB
+          "--output-limit 65536"
+        ];
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat > "$HOME/.ssh/config" << EOF
+          Host bscpm04.bsc.es gitlab-internal.bsc.es
+            User git
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          EOF
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          # Required to load SSL certificate paths
+          . ${pkgs.cacert}/nix-support/setup-hook
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+        };
       };
     };
   };
 
+  # DOCKER* chains are useless, override at FORWARD and nixos-fw
+  networking.firewall.extraCommands = ''
+    # Don't forward any traffic from docker
+    iptables -I FORWARD 1 -p all -i docker0 -j nixos-fw-log-refuse
+
+    # Allow incoming traffic from docker to 23080
+    iptables -A nixos-fw -p tcp -i docker0 -d hut --dport 23080 -j ACCEPT
+  '';
+
   #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";

m/hut/gpfs-probe.nix

@@ -0,0 +1,31 @@
+{ pkgs, config, lib, ... }:
+let
+  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
+    ''
+      cp ${./gpfs-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  # Use a new user to handle the SSH keys
+  users.groups.ssh-robot = { };
+  users.users.ssh-robot = {
+    description = "SSH Robot";
+    isNormalUser = true;
+    home = "/var/lib/ssh-robot";
+  };
+
+  systemd.services.gpfs-probe = {
+    description = "Daemon to report GPFS latency via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
+}

m/hut/gpfs-probe.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+N=500
+
+t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
+
+if [ -z "$t" ]; then
+  t="5.00"
+fi
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP gpfs_touch_latency Time to create $N files.
+# TYPE gpfs_touch_latency gauge
+gpfs_touch_latency $t
+EOF

m/hut/ipmi.yml

@@ -1,13 +0,0 @@
-modules:
-        default:
-                collectors:
-                - bmc
-                - ipmi
-                - chassis
-
-        lan:
-                collectors:
-                - ipmi
-                - chassis
-                user: ""
-                pass: ""

m/hut/monitoring.nix

@@ -1,7 +1,13 @@
 { config, lib, ... }:
 
 {
-  imports = [ ../module/slurm-exporter.nix ];
+  imports = [
+    ../module/slurm-exporter.nix
+    ../module/meteocat-exporter.nix
+    ../module/upc-qaire-exporter.nix
+    ./gpfs-probe.nix
+    ../module/nix-daemon-exporter.nix
+  ];
 
   age.secrets.grafanaJungleRobotPassword = {
     file = ../../secrets/jungle-robot-password.age;
@@ -9,6 +15,8 @@
     mode = "400";
   };
 
+  age.secrets.ipmiYml.file = ../../secrets/ipmi.yml.age;
+
   services.grafana = {
     enable = true;
     settings = {
@@ -41,7 +49,7 @@
   services.prometheus = {
     enable = true;
     port = 9001;
-    retentionTime = "1y";
+    retentionTime = "5y";
     listenAddress = "127.0.0.1";
   };
 
@@ -70,13 +78,13 @@
         enable = true;
         group = "root";
         user = "root";
-        configFile = ./ipmi.yml;
+        configFile = config.age.secrets.ipmiYml.path;
-        #extraFlags = [ "--log.level=debug" ];
+        # extraFlags = [ "--log.level=debug" ];
         listenAddress = "127.0.0.1";
       };
       node = {
         enable = true;
-        enabledCollectors = [ "systemd" ];
+        enabledCollectors = [ "systemd" "logind" ];
         port = 9002;
         listenAddress = "127.0.0.1";
       };
@@ -102,6 +110,10 @@
             "127.0.0.1:9252"
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
+            "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9999" # Nix-daemon custom exporter
+            "127.0.0.1:9929" # Meteocat custom exporter
+            "127.0.0.1:9928" # UPC Qaire custom exporter
             "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];
@@ -157,6 +169,9 @@
             "8.8.8.8"
             "ssfhead"
             "anella-bsc.cesca.cat"
+            "upc-anella.cesca.cat"
+            "fox.ac.upc.edu"
+            "arenys5.ac.upc.edu"
           ];
         }];
         relabel_configs = [
@@ -202,7 +217,7 @@
             # Sets the "instance" label with the remote host we are querying
             source_labels = [ "__param_target" ];
             separator = ";";
-            regex = "(.*)";
+            regex = "(.*)-ipmi"; # Remove "-ipm̀i" at the end
             target_label = "instance";
             replacement = "\${1}";
             action = "replace";
@@ -244,6 +259,14 @@
           module = [ "raccoon" ];
         };
       }
+      {
+        job_name = "raccoon";
+        static_configs = [
+          {
+            targets = [ "127.0.0.1:19002" ]; # Node exporter
+          }
+        ];
+      }
     ];
   };
 }

m/hut/nginx.nix

@@ -1,14 +1,73 @@
+{ theFlake, pkgs, ... }:
+let
+  website = pkgs.stdenv.mkDerivation {
+    name = "jungle-web";
+    src = theFlake;
+    buildInputs = [ pkgs.hugo ];
+    buildPhase = ''
+      cd web
+      rm -rf public/
+      hugo
+    '';
+    installPhase = ''
+      cp -r public $out
+    '';
+    # Don't mess doc/
+    dontFixup = true;
+  };
+in
 {
+  networking.firewall.allowedTCPPorts = [ 80 ];
   services.nginx = {
     enable = true;
     virtualHosts."jungle.bsc.es" = {
+      root = "${website}";
       listen = [
         {
-          addr = "127.0.0.1";
+          addr = "0.0.0.0";
-          port = 8123;
+          port = 80;
         }
       ];
-      locations."/p/".alias = "/ceph/p/";
+      extraConfig = ''
+        set_real_ip_from 127.0.0.1;
+        set_real_ip_from 84.88.52.107;
+        real_ip_recursive on;
+        real_ip_header X-Forwarded-For;
+
+        location /git {
+          rewrite ^/git$ / break;
+          rewrite ^/git/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:3000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /cache {
+          rewrite ^/cache/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:5000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /lists {
+          proxy_pass http://127.0.0.1:8081;
+          proxy_redirect http:// $scheme://;
+        }
+        location /grafana {
+          proxy_pass http://127.0.0.1:2342;
+          proxy_redirect http:// $scheme://;
+          proxy_set_header Host $host;
+          # Websockets
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+        }
+        location ~ ^/~(.+?)(/.*)?$ {
+          alias /ceph/home/$1/public_html$2;
+          index  index.html index.htm;
+          autoindex on;
+          absolute_redirect off;
+        }
+        location /p/ {
+          alias /ceph/p/;
+        }
+      '';
     };
   };
 }

m/hut/p.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, lib, config, ... }:
 let
   p = pkgs.writeShellScriptBin "p" ''
     set -e
@@ -6,12 +6,14 @@ let
     pastedir="p/$USER"
     mkdir -p "$pastedir"
 
+    ext="txt"
+
     if [ -n "$1" ]; then
-            out="$pastedir/$1"
+      ext="$1"
-    else
-            out=$(mktemp "$pastedir/XXXXXXXX.txt")
     fi
 
+    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
+
     cat > "$out"
     chmod go+r "$out"
     echo "https://jungle.bsc.es/$out"
@@ -19,4 +21,23 @@ let
 in
 {
   environment.systemPackages = with pkgs; [ p ];
+
+  # Make sure we have a directory per user. We cannot use the nice
+  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
+  # may not be mounted when it runs.
+  systemd.services.create-paste-dirs = let
+    # Take only normal users in hut
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "remote-fs.target" ];
+    after = [ "remote-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
 }

m/hut/targets.yml

@@ -1,15 +1,15 @@
 - targets:
-  - 10.0.40.101
+  - owl1-ipmi
-  - 10.0.40.102
+  - owl2-ipmi
-  - 10.0.40.103
+  - xeon03-ipmi
-  - 10.0.40.104
+  - xeon04-ipmi
-  - 10.0.40.105
+  - koro-ipmi
-  - 10.0.40.106
+  - weasel-ipmi
-  - 10.0.40.107
+  - hut-ipmi
-  - 10.0.40.108
+  - eudy-ipmi
   # Storage
-  - 10.0.40.141
+  - bay-ipmi
-  - 10.0.40.142
+  - oss01-ipmi
-  - 10.0.40.143
+  - lake2-ipmi
   labels:
     job: ipmi-lan

m/koro/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix

m/lake2/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/monitoring.nix
   ];
 

m/map.nix

@@ -0,0 +1,70 @@
+{
+  # In physical order from top to bottom (see note below)
+  ssf = {
+    # Switches for Ethernet and OmniPath
+    switch-C6-S1A-05 = { pos=42; size=1; model="Dell S3048-ON"; };
+    switch-opa = { pos=41; size=1; };
+
+    # SSF login
+    apex = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="rodrigo.arias@bsc.es"; };
+
+    # Storage
+    bay   = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };
+    lake1 = { pos=37; size=1; label="OSS01"; board="S2600WT2R"; sn="BQWL64850234"; contact="rodrigo.arias@bsc.es"; };
+    lake2 = { pos=36; size=1; label="OSS02"; board="S2600WT2R"; sn="BQWL64850266"; contact="rodrigo.arias@bsc.es"; };
+
+    # Compute xeon
+    owl1   = { pos=35; size=1; label="SSF-XEON01"; board="S2600WTTR"; sn="BQWL64954172"; contact="rodrigo.arias@bsc.es"; };
+    owl2   = { pos=34; size=1; label="SSF-XEON02"; board="S2600WTTR"; sn="BQWL64756560"; contact="rodrigo.arias@bsc.es"; };
+    xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; };
+    # Slot 34 empty
+    koro   = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; };
+    weasel = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
+    hut    = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; };
+    eudy   = { pos=28; size=1; label="SSF-XEON08"; board="S2600WTTR"; sn="BQWL64756586"; contact="aleix.rocanonell@bsc.es"; };
+
+    # 16 KNL nodes, 4 per chassis
+    knl01_04 = { pos=26; size=2; label="KNL01..KNL04"; board="HNS7200APX"; };
+    knl05_08 = { pos=24; size=2; label="KNL05..KNL18"; board="HNS7200APX"; };
+    knl09_12 = { pos=22; size=2; label="KNL09..KNL12"; board="HNS7200APX"; };
+    knl13_16 = { pos=20; size=2; label="KNL13..KNL16"; board="HNS7200APX"; };
+
+    # Slot 19 empty
+
+    # EPI (hw team, guessed order)
+    epi01 = { pos=18; size=1; contact="joan.cabre@bsc.es"; };
+    epi02 = { pos=17; size=1; contact="joan.cabre@bsc.es"; };
+    epi03 = { pos=16; size=1; contact="joan.cabre@bsc.es"; };
+    anon  = { pos=14; size=2; }; # Unlabeled machine. Operative
+
+    # These are old and decommissioned (off)
+    power8    = { pos=12; size=2; label="BSCPOWER8N3";   decommissioned=true; };
+    powern1   = { pos=8;  size=4; label="BSCPOWERN1";    decommissioned=true; };
+    gustafson = { pos=7;  size=1; label="gustafson";     decommissioned=true; };
+    odap01    = { pos=3;  size=4; label="ODAP01";        decommissioned=true; };
+    amhdal    = { pos=2;  size=1; label="AMHDAL";        decommissioned=true; }; # sic
+    moore     = { pos=1;  size=1; label="moore (earth)"; decommissioned=true; };
+  };
+
+  bsc2218 = {
+    raccoon = { board="W2600CR"; sn="QSIP22500829"; contact="rodrigo.arias@bsc.es"; };
+    tent    = { label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  upc = {
+    fox = { board="H13DSG-O-CPU"; sn="UM24CS600392"; prod="AS-4125GS-TNRT"; prod_sn="E508839X5103339"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  # NOTE: Position is specified in "U" units (44.45 mm) and starts at 1 from the
+  # bottom. Example:
+  #
+  #  |   ...  | - [pos+size] <--- Label in chassis
+  #  +--------+
+  #  |  node  | - [pos+1]
+  #  |   2U   | - [pos]
+  #  +------- +
+  #  |   ...  | - [pos-1]
+  #
+  # NOTE: The board and sn refers to the FRU information (Board Product and
+  # Board Serial) via `ipmitool fru print 0`.
+}

m/module/ceph.nix

@@ -13,22 +13,10 @@
 
   age.secrets.cephUser.file = ../../secrets/ceph-user.age;
 
-  fileSystems."/ceph-slow" = {
-    fsType = "ceph";
-    device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
-    options = [
-      "mon_addr=10.0.40.40"
-      "secretfile=${config.age.secrets.cephUser.path}"
-    ];
-  };
-
-  services.cachefilesd.enable = true;
-
   fileSystems."/ceph" = {
     fsType = "ceph";
     device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
     options = [
-      "fsc"
       "mon_addr=10.0.40.40"
       "secretfile=${config.age.secrets.cephUser.path}"
     ];

m/module/hut-substituter.nix

@@ -0,0 +1,10 @@
+{ config, ... }:
+{
+  nix.settings =
+    # Don't add hut as a cache to itself
+    assert config.networking.hostName != "hut";
+    {
+      extra-substituters = [ "http://hut/cache" ];
+      extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+    };
+}

m/module/meteocat-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-meteocat-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.meteocat-exporter}/bin/meteocat-exporter";
+    };
+  };
+}

m/module/nix-daemon-builds.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Locate nix daemon pid
+nd=$(pgrep -o nix-daemon)
+
+# Locate children of nix-daemon
+pids1=$(tr ' ' '\n' < "/proc/$nd/task/$nd/children")
+
+# For each children, locate 2nd level children
+pids2=$(echo "$pids1" | xargs -I @ /bin/sh -c 'cat /proc/@/task/*/children' | tr ' ' '\n')
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP nix_daemon_build Nix daemon derivation build state.
+# TYPE nix_daemon_build gauge
+EOF
+
+for pid in $pids2; do
+  name=$(cat /proc/$pid/environ 2>/dev/null | tr '\0' '\n' | rg "^name=(.+)" - --replace '$1' | tr -dc ' [:alnum:]_\-\.')
+  user=$(ps -o uname= -p "$pid")
+  if [ -n "$name" -a -n "$user" ]; then
+    printf 'nix_daemon_build{user="%s",name="%s"} 1\n' "$user" "$name"
+  fi
+done

m/module/nix-daemon-exporter.nix

@@ -0,0 +1,23 @@
+{ pkgs, config, lib, ... }:
+let
+  script = pkgs.runCommand "nix-daemon-exporter.sh" { }
+    ''
+      cp ${./nix-daemon-builds.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  systemd.services.nix-daemon-exporter = {
+    description = "Daemon to export nix-daemon metrics";
+    path = [ pkgs.procps pkgs.ripgrep ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9999,fork EXEC:${script}";
+      # Needed root to read the environment, potentially unsafe
+      User = "root";
+      Group = "root";
+    };
+  };
+}

m/module/p.nix

@@ -0,0 +1,68 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.p;
+in
+{
+  options = {
+    services.p = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the p service.";
+      };
+      path = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/p";
+        description = "Where to save the pasted files on disk.";
+      };
+      url = lib.mkOption {
+        type = lib.types.str;
+        default = "https://jungle.bsc.es/p";
+        description = "URL prefix for the printed file.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = let 
+      p = pkgs.writeShellScriptBin "p" ''
+        set -e
+        pastedir="${cfg.path}/$USER"
+        cd "$pastedir"
+
+        ext="txt"
+        if [ -n "$1" ]; then
+          ext="$1"
+        fi
+
+        out=$(mktemp "XXXXXXXX.$ext")
+        cat > "$out"
+        chmod go+r "$out"
+        echo "${cfg.url}/$USER/$out"
+      '';
+    in [ p ];
+
+    systemd.services.p = let
+      # Take only normal users
+      users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+      # Create a directory for each user
+      commands = lib.concatLists (lib.mapAttrsToList (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 ${cfg.path}/${user.name}"
+      ]) users);
+    in {
+      description = "P service setup";
+      requires = [ "network-online.target" ];
+      #wants = [ "remote-fs.target" ];
+      #after = [ "remote-fs.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = pkgs.writeShellScript "p-init.sh" (''
+
+          install -d -o root -g root -m 0755 ${cfg.path}
+
+        '' + (lib.concatLines commands));
+      };
+    };
+  };
+}

m/module/slurm-client.nix

@@ -47,8 +47,7 @@ in {
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-      "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -91,9 +90,29 @@ in {
       # Ignore memory constraints and only use unused cores to share a node with
       # other jobs.
       SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
     '';
   };
 
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
   age.secrets.mungeKey = {
     file = ../../secrets/munge-key.age;
     owner = "munge";

m/module/ssh-hut-extern.nix

@@ -0,0 +1,8 @@
+{
+  programs.ssh.extraConfig = ''
+    Host apex ssfhead
+      HostName ssflogin.bsc.es
+    Host hut
+      ProxyJump apex
+  '';
+}

m/module/upc-qaire-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-upc-qaire-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.upc-qaire-exporter}/bin/upc-qaire-exporter";
+    };
+  };
+}

m/module/vpn-dac.nix

@@ -0,0 +1,35 @@
+{config, ...}:
+{
+  age.secrets.vpn-dac-login.file = ../../secrets/vpn-dac-login.age;
+  age.secrets.vpn-dac-client-key.file = ../../secrets/vpn-dac-client-key.age;
+
+  services.openvpn.servers = {
+    # systemctl status openvpn-dac.service
+    dac = {
+      config = ''
+        client
+        dev tun
+        proto tcp
+        remote vpn.ac.upc.edu 1194
+        remote vpn.ac.upc.edu 80
+        resolv-retry infinite
+        nobind
+        persist-key
+        persist-tun
+        ca ${./vpn-dac/ca.crt}
+        cert ${./vpn-dac/client.crt}
+        # Only key needs to be secret
+        key ${config.age.secrets.vpn-dac-client-key.path}
+        remote-cert-tls server
+        comp-lzo
+        verb 3
+        auth-user-pass ${config.age.secrets.vpn-dac-login.path}
+        reneg-sec 0
+
+        # Only route fox-ipmi
+        pull-filter ignore "route "
+        route 147.83.35.27 255.255.255.255
+      '';
+    };
+  };
+}

m/module/vpn-dac/ca.crt

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFUjCCBDqgAwIBAgIJAJH118PApk5hMA0GCSqGSIb3DQEBCwUAMIHLMQswCQYD
+VQQGEwJFUzESMBAGA1UECBMJQmFyY2Vsb25hMRIwEAYDVQQHEwlCYXJjZWxvbmEx
+LTArBgNVBAoTJFVuaXZlcnNpdGF0IFBvbGl0ZWNuaWNhIGRlIENhdGFsdW55YTEk
+MCIGA1UECxMbQXJxdWl0ZWN0dXJhIGRlIENvbXB1dGFkb3JzMRAwDgYDVQQDEwdM
+Q0FDIENBMQ0wCwYDVQQpEwRMQ0FDMR4wHAYJKoZIhvcNAQkBFg9sY2FjQGFjLnVw
+Yy5lZHUwHhcNMTYwMTEyMTI0NDIxWhcNNDYwMTEyMTI0NDIxWjCByzELMAkGA1UE
+BhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0w
+KwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAi
+BgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENB
+QyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMu
+ZWR1MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0CteSeof7Xwi51kC
+F0nQ4E9iR5Lq7wtfRuVPn6JJcIxJJ6+F9gr4R/HIHTztW4XAzReE36DYfexupx3D
+6UgQIkMLlVyGqRbulNF+RnCx20GosF7Dm4RGBVvOxBP1PGjYq/A+XhaaDAFd0cOF
+LMNkzuYP7PF0bnBEaHnxmN8bPmuyDyas7fK9AAc3scyWT2jSBPbOVFvCJwPg8MH9
+V/h+hKwL/7hRt1MVfVv2qyIuKwTki8mUt0RcVbP7oJoRY5K1+R52phIz/GL/b4Fx
+L6MKXlQxLi8vzP4QZXgCMyV7oFNdU3VqCEXBA11YIRvsOZ4QS19otIk/ZWU5x+HH
+LAIJ7wIDAQABo4IBNTCCATEwHQYDVR0OBBYEFNyezX1cH1N4QR14ebBpljqmtE7q
+MIIBAAYDVR0jBIH4MIH1gBTcns19XB9TeEEdeHmwaZY6prRO6qGB0aSBzjCByzEL
+MAkGA1UEBhMCRVMxEjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vs
+b25hMS0wKwYDVQQKEyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVu
+eWExJDAiBgNVBAsTG0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UE
+AxMHTENBQyBDQTENMAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0Bh
+Yy51cGMuZWR1ggkAkfXXw8CmTmEwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsF
+AAOCAQEAUAmOvVXIQrR+aZVO0bOTeugKBHB75eTIZSIHIn2oDUvDbAP5GXIJ56A1
+6mZXxemSMY8/9k+pRcwJhfat3IgvAN159XSqf9kRv0NHgc3FWUI1Qv/BsAn0vJO/
+oK0dbmbbRWqt86qNrCN+cUfz5aovvxN73jFfnvfDQFBk/8enj9wXxYfokjjLPR1Q
++oTkH8dY68qf71oaUB9MndppPEPSz0K1S6h1XxvJoSu9MVSXOQHiq1cdZdxRazI3
+4f7q9sTCL+khwDAuZxAYzlEYxFFa/NN8PWU6xPw6V+t/aDhOiXUPJQB/O/K7mw3Z
+TQQx5NqM7B5jjak5fauR3/oRD8XXsA==
+-----END CERTIFICATE-----

m/module/vpn-dac/client.crt

@@ -0,0 +1,100 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number: 2 (0x2)
+    Signature Algorithm: sha256WithRSAEncryption
+        Issuer: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
+        Validity
+            Not Before: Jan 12 12:45:41 2016 GMT
+            Not After : Jan 12 12:45:41 2046 GMT
+        Subject: C=ES, ST=Barcelona, L=Barcelona, O=Universitat Politecnica de Catalunya, OU=Arquitectura de Computadors, CN=client/name=LCAC/emailAddress=lcac@ac.upc.edu
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+                Public-Key: (2048 bit)
+                Modulus:
+                    00:97:99:fa:7a:0e:4d:e2:1d:a5:b1:a8:14:18:64:
+                    c7:66:bf:de:99:1d:92:3b:86:82:4d:95:39:f7:a6:
+                    56:49:97:14:4f:e3:37:00:6c:f4:d0:1d:56:79:e7:
+                    19:b5:dd:36:15:8e:1d:57:7b:59:29:d2:11:bf:58:
+                    48:e0:f7:41:3d:16:64:8d:a2:0b:4a:ac:fa:c6:83:
+                    dc:10:2a:2c:d9:97:48:ee:11:2a:bc:4b:60:dd:b9:
+                    2e:8f:45:ca:87:0b:38:65:1c:f8:a2:1d:f9:50:aa:
+                    6e:60:f9:48:df:57:12:23:e1:e7:0c:81:5c:9f:c5:
+                    b2:e6:99:99:95:30:6d:57:36:06:8c:fd:fb:f9:4f:
+                    60:d2:3c:ba:ae:28:56:2f:da:58:5c:e8:c5:7b:ec:
+                    76:d9:28:6e:fb:8c:07:f9:d7:23:c3:72:76:3c:fa:
+                    dc:20:67:8f:cc:16:e0:91:07:d5:68:f9:20:4d:7d:
+                    5c:2d:02:04:16:76:52:f3:53:be:a3:dc:0d:d5:fb:
+                    6b:55:29:f3:52:35:c8:7d:99:d1:4a:94:be:b1:8e:
+                    fd:85:18:25:eb:41:e9:56:da:af:62:84:20:0a:00:
+                    17:94:92:94:91:6a:f8:54:37:17:ee:1e:bb:fb:93:
+                    71:91:d9:e4:e9:b8:3b:18:7d:6d:7d:4c:ce:58:55:
+                    f9:41
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Basic Constraints: 
+                CA:FALSE
+            Netscape Comment: 
+                Easy-RSA Generated Certificate
+            X509v3 Subject Key Identifier: 
+                1B:88:06:D5:33:1D:5C:48:46:B5:DE:78:89:36:96:91:3A:74:43:18
+            X509v3 Authority Key Identifier: 
+                keyid:DC:9E:CD:7D:5C:1F:53:78:41:1D:78:79:B0:69:96:3A:A6:B4:4E:EA
+                DirName:/C=ES/ST=Barcelona/L=Barcelona/O=Universitat Politecnica de Catalunya/OU=Arquitectura de Computadors/CN=LCAC CA/name=LCAC/emailAddress=lcac@ac.upc.edu
+                serial:91:F5:D7:C3:C0:A6:4E:61
+
+            X509v3 Extended Key Usage: 
+                TLS Web Client Authentication
+            X509v3 Key Usage: 
+                Digital Signature
+            X509v3 Subject Alternative Name: 
+                DNS:client
+    Signature Algorithm: sha256WithRSAEncryption
+         42:e8:50:b2:e7:88:75:86:0b:bb:29:e3:aa:c6:0e:4c:e8:ea:
+         3d:0c:02:31:7f:3b:80:0c:3f:80:af:45:d6:62:27:a0:0e:e7:
+         26:09:12:97:95:f8:d9:9b:89:b5:ef:56:64:f1:de:82:74:e0:
+         31:0a:cc:90:0a:bd:50:b8:54:95:0a:ae:3b:40:df:76:b6:d1:
+         01:2e:f3:96:9f:52:d4:e9:14:6d:b7:14:9d:45:99:33:36:2a:
+         01:0b:15:1a:ed:55:dc:64:83:65:1a:06:42:d9:c7:dc:97:d4:
+         02:81:c2:58:2b:ea:e4:b7:ae:84:3a:e4:3f:f1:2e:fa:ec:f3:
+         40:5d:b8:6a:d5:5e:e1:e8:2f:e2:2f:48:a4:38:a1:4f:22:e3:
+         4f:66:94:aa:02:78:9a:2b:7a:5d:aa:aa:51:a5:e3:d0:91:e9:
+         1d:f9:08:ed:8b:51:c9:a6:af:46:85:b5:1c:ed:12:a1:28:33:
+         75:36:00:d8:5c:14:65:96:c0:28:7d:47:50:a4:89:5f:b0:72:
+         1a:4b:13:17:26:0f:f0:b8:65:3c:e9:96:36:f9:bf:90:59:33:
+         87:1f:01:03:25:f8:f0:3a:9b:33:02:d0:0a:43:b5:0a:cf:62:
+         a1:45:38:37:07:9d:9c:94:0b:31:c6:3c:34:b7:fc:5a:0c:e4:
+         bf:23:f6:7d
+-----BEGIN CERTIFICATE-----
+MIIFqjCCBJKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADCByzELMAkGA1UEBhMCRVMx
+EjAQBgNVBAgTCUJhcmNlbG9uYTESMBAGA1UEBxMJQmFyY2Vsb25hMS0wKwYDVQQK
+EyRVbml2ZXJzaXRhdCBQb2xpdGVjbmljYSBkZSBDYXRhbHVueWExJDAiBgNVBAsT
+G0FycXVpdGVjdHVyYSBkZSBDb21wdXRhZG9yczEQMA4GA1UEAxMHTENBQyBDQTEN
+MAsGA1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MB4X
+DTE2MDExMjEyNDU0MVoXDTQ2MDExMjEyNDU0MVowgcoxCzAJBgNVBAYTAkVTMRIw
+EAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNlbG9uYTEtMCsGA1UEChMk
+VW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1bnlhMSQwIgYDVQQLExtB
+cnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxDzANBgNVBAMTBmNsaWVudDENMAsG
+A1UEKRMETENBQzEeMBwGCSqGSIb3DQEJARYPbGNhY0BhYy51cGMuZWR1MIIBIjAN
+BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAl5n6eg5N4h2lsagUGGTHZr/emR2S
+O4aCTZU596ZWSZcUT+M3AGz00B1WeecZtd02FY4dV3tZKdIRv1hI4PdBPRZkjaIL
+Sqz6xoPcECos2ZdI7hEqvEtg3bkuj0XKhws4ZRz4oh35UKpuYPlI31cSI+HnDIFc
+n8Wy5pmZlTBtVzYGjP37+U9g0jy6rihWL9pYXOjFe+x22Shu+4wH+dcjw3J2PPrc
+IGePzBbgkQfVaPkgTX1cLQIEFnZS81O+o9wN1ftrVSnzUjXIfZnRSpS+sY79hRgl
+60HpVtqvYoQgCgAXlJKUkWr4VDcX7h67+5Nxkdnk6bg7GH1tfUzOWFX5QQIDAQAB
+o4IBljCCAZIwCQYDVR0TBAIwADAtBglghkgBhvhCAQ0EIBYeRWFzeS1SU0EgR2Vu
+ZXJhdGVkIENlcnRpZmljYXRlMB0GA1UdDgQWBBQbiAbVMx1cSEa13niJNpaROnRD
+GDCCAQAGA1UdIwSB+DCB9YAU3J7NfVwfU3hBHXh5sGmWOqa0TuqhgdGkgc4wgcsx
+CzAJBgNVBAYTAkVTMRIwEAYDVQQIEwlCYXJjZWxvbmExEjAQBgNVBAcTCUJhcmNl
+bG9uYTEtMCsGA1UEChMkVW5pdmVyc2l0YXQgUG9saXRlY25pY2EgZGUgQ2F0YWx1
+bnlhMSQwIgYDVQQLExtBcnF1aXRlY3R1cmEgZGUgQ29tcHV0YWRvcnMxEDAOBgNV
+BAMTB0xDQUMgQ0ExDTALBgNVBCkTBExDQUMxHjAcBgkqhkiG9w0BCQEWD2xjYWNA
+YWMudXBjLmVkdYIJAJH118PApk5hMBMGA1UdJQQMMAoGCCsGAQUFBwMCMAsGA1Ud
+DwQEAwIHgDARBgNVHREECjAIggZjbGllbnQwDQYJKoZIhvcNAQELBQADggEBAELo
+ULLniHWGC7sp46rGDkzo6j0MAjF/O4AMP4CvRdZiJ6AO5yYJEpeV+NmbibXvVmTx
+3oJ04DEKzJAKvVC4VJUKrjtA33a20QEu85afUtTpFG23FJ1FmTM2KgELFRrtVdxk
+g2UaBkLZx9yX1AKBwlgr6uS3roQ65D/xLvrs80BduGrVXuHoL+IvSKQ4oU8i409m
+lKoCeJorel2qqlGl49CR6R35CO2LUcmmr0aFtRztEqEoM3U2ANhcFGWWwCh9R1Ck
+iV+wchpLExcmD/C4ZTzpljb5v5BZM4cfAQMl+PA6mzMC0ApDtQrPYqFFODcHnZyU
+CzHGPDS3/FoM5L8j9n0=
+-----END CERTIFICATE-----

m/owl1/configuration.nix

@@ -2,12 +2,13 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/ceph.nix
     ../module/emulation.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
     ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -2,12 +2,13 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/ceph.nix
     ../module/emulation.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
     ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/raccoon/configuration.nix

@@ -3,6 +3,10 @@
 {
   imports = [
     ../common/base.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+    ../eudy/kernel/perf.nix
   ];
 
   # Don't install Grub on the disk yet
@@ -23,15 +27,44 @@
       address = "84.88.51.152";
       prefixLength = 25;
     } ];
+    interfaces.enp5s0f1.ipv4.addresses = [ {
+      address = "10.0.44.1";
+      prefixLength = 24;
+    } ];
+    nat = {
+      enable = true;
+      internalInterfaces = [ "enp5s0f1" ];
+      externalInterface = "eno0";
+    };
+    hosts = {
+      "10.0.44.4" = [ "tent" ];
+    };
   };
 
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  # Enable performance governor
+  powerManagement.cpuFreqGovernor = "performance";
+
   # Configure Nvidia driver to use with CUDA
   hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.nvidia.open = false; # Maxwell is older than Turing architecture
   hardware.graphics.enable = true;
-  nixpkgs.config.allowUnfree = true;
   nixpkgs.config.nvidia.acceptLicense = true;
   services.xserver.videoDrivers = [ "nvidia" ];
 
+  services.openssh.settings.X11Forwarding = true;
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
   users.motd = ''
     ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
     ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀

m/tent/blackbox.yml

@@ -0,0 +1,14 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      preferred_ip_protocol: "ip4"
+      follow_redirects: true
+      valid_status_codes: []  # Defaults to 2xx
+      method: GET
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: "ip4"

m/tent/configuration.nix

@@ -0,0 +1,80 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+    ./monitoring.nix
+    ./nginx.nix
+    ./nix-serve.nix
+    ./gitlab-runner.nix
+    ./gitea.nix
+    ../hut/public-inbox.nix
+    ../hut/msmtp.nix
+    ../module/p.nix
+    ../module/vpn-dac.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d537675";
+
+  networking = {
+    hostName = "tent";
+    interfaces.eno1.ipv4.addresses = [
+      {
+        address = "10.0.44.4";
+        prefixLength = 24;
+      }
+    ];
+
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    search = [ "bsc.es" "ac.upc.edu" ];
+    defaultGateway = "10.0.44.1";
+  };
+
+  services.p.enable = true;
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
+  boot.swraid = {
+    enable = true;
+    mdadmConf = ''
+      DEVICE partitions
+      ARRAY /dev/md0 metadata=1.2 UUID=496db1e2:056a92aa:a544543f:40db379d
+      MAILADDR root
+    '';
+  };
+
+  fileSystems."/vault" = {
+    device = "/dev/disk/by-label/vault";
+    fsType = "ext4";
+  };
+
+  # Make a /vault/$USER directory for each user.
+  systemd.services.create-vault-dirs = let
+    # Take only normal users in tent
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0711 /vault/home/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-vault-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+
+  # disable automatic garbage collector
+  nix.gc.automatic = lib.mkForce false;
+}

m/tent/gitea.nix

@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+{
+  services.gitea = {
+    enable = true;
+    appName = "Gitea in the jungle";
+
+    settings = {
+      server = {
+        ROOT_URL = "https://jungle.bsc.es/git/";
+        LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
+        LANDING_PAGE = "explore";
+      };
+      metrics.ENABLED = true;
+      service = {
+        DISABLE_REGISTRATION = true;
+        REGISTER_MANUAL_CONFIRM = true;
+        ENABLE_NOTIFY_MAIL = true;
+      };
+      log.LEVEL = "Warn";
+
+      mailer = {
+        ENABLED       = true;
+        FROM          = "jungle-robot@bsc.es";
+        PROTOCOL      = "sendmail";
+        SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
+        SENDMAIL_ARGS = "--";
+      };
+    };
+  };
+}

m/tent/gitlab-runner.nix

@@ -0,0 +1,93 @@
+{ pkgs, lib, config, ... }:
+
+{
+  age.secrets.tent-gitlab-runner-pm-shell.file = ../../secrets/tent-gitlab-runner-pm-shell-token.age;
+  age.secrets.tent-gitlab-runner-pm-docker.file = ../../secrets/tent-gitlab-runner-pm-docker-token.age;
+  age.secrets.tent-gitlab-runner-bsc-docker.file = ../../secrets/tent-gitlab-runner-bsc-docker-token.age;
+
+  services.gitlab-runner = let sec = config.age.secrets; in {
+    enable = true;
+    settings.concurrent = 5;
+    services = {
+      # For gitlab.pm.bsc.es
+      gitlab-pm-shell = {
+        executor = "shell";
+        environmentVariables = {
+          SHELL = "${pkgs.bash}/bin/bash";
+        };
+        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-shell.path;
+        preGetSourcesScript = pkgs.writeScript "setup" ''
+          echo "This is the preGetSources script running, brace for impact"
+          env
+        '';
+      };
+      gitlab-pm-docker = {
+        authenticationTokenConfigFile = sec.tent-gitlab-runner-pm-docker.path;
+        executor = "docker";
+        dockerImage = "debian:stable";
+      };
+
+      # For gitlab.bsc.es
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = sec.tent-gitlab-runner-bsc-docker.path;
+        tagList = [ "docker" "tent" "nix" ];
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerDisableCache = true;
+        registrationFlags = [
+          # Increase build log length to 64 MiB
+          "--output-limit 65536"
+        ];
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          # Required to load SSL certificate paths
+          . ${pkgs.cacert}/nix-support/setup-hook
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+        };
+      };
+    };
+  };
+
+  systemd.services.gitlab-runner.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    User = "gitlab-runner";
+    Group = "gitlab-runner";
+    ExecStart = lib.mkForce
+      ''${pkgs.gitlab-runner}/bin/gitlab-runner run --config ''${HOME}/.gitlab-runner/config.toml --listen-address "127.0.0.1:9252" --working-directory ''${HOME}'';
+  };
+
+  users.users.gitlab-runner = {
+    uid = config.ids.uids.gitlab-runner;
+    home = "/var/lib/gitlab-runner";
+    description = "Gitlab Runner";
+    group = "gitlab-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;
+}

m/tent/monitoring.nix

@@ -0,0 +1,217 @@
+{ config, lib, pkgs, ... }:
+
+{
+  imports = [
+    ../module/meteocat-exporter.nix
+    ../module/upc-qaire-exporter.nix
+    ../module/nix-daemon-exporter.nix
+  ];
+
+  age.secrets.grafanaJungleRobotPassword = {
+    file = ../../secrets/jungle-robot-password.age;
+    owner = "grafana";
+    mode = "400";
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        domain = "jungle.bsc.es";
+        root_url = "%(protocol)s://%(domain)s/grafana";
+        serve_from_sub_path = true;
+        http_port = 2342;
+        http_addr = "127.0.0.1";
+      };
+      smtp = {
+        enabled = true;
+        from_address = "jungle-robot@bsc.es";
+        user = "jungle-robot";
+        # Read the password from a file, which is only readable by grafana user
+        # https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
+        password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
+        host = "mail.bsc.es:465";
+        startTLS_policy = "NoStartTLS";
+      };
+      feature_toggles.publicDashboards = true;
+      "auth.anonymous".enabled = true;
+      log.level = "warn";
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    retentionTime = "5y";
+    listenAddress = "127.0.0.1";
+  };
+
+  # We need access to the devices to monitor the disk space
+  systemd.services.prometheus-node-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+  systemd.services.prometheus-node-exporter.serviceConfig.ProtectHome = lib.mkForce "read-only";
+
+  # Credentials for IPMI exporter
+  age.secrets.ipmiYml = {
+    file = ../../secrets/ipmi.yml.age;
+    owner = "ipmi-exporter";
+  };
+
+  # Create an IPMI group and assign the ipmi0 device
+  users.groups.ipmi = {};
+  services.udev.extraRules = ''
+    SUBSYSTEM=="ipmi", KERNEL=="ipmi0", GROUP="ipmi", MODE="0660"
+  '';
+
+  # Add a new ipmi-exporter user that can read the ipmi0 device
+  users.users.ipmi-exporter = {
+    isSystemUser = true;
+    group = "ipmi";
+  };
+
+  # Disable dynamic user so we have the ipmi-exporter user available for the credentials
+  systemd.services.prometheus-ipmi-exporter.serviceConfig = {
+    DynamicUser = lib.mkForce false;
+    PrivateDevices = lib.mkForce false;
+    User = lib.mkForce "ipmi-exporter";
+    Group = lib.mkForce "ipmi";
+    RestrictNamespaces = lib.mkForce false;
+    # Fake uid to 0 so it shuts up
+    ExecStart = let
+      cfg = config.services.prometheus.exporters.ipmi;
+    in lib.mkForce (lib.concatStringsSep " " ([
+      "${pkgs.util-linux}/bin/unshare --map-user 0"
+      "${pkgs.prometheus-ipmi-exporter}/bin/ipmi_exporter"
+      "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
+      "--config.file ${lib.escapeShellArg cfg.configFile}"
+    ] ++ cfg.extraFlags));
+  };
+
+  services.prometheus = {
+    exporters = {
+      ipmi = {
+        enable = true;
+        configFile = config.age.secrets.ipmiYml.path;
+        #extraFlags = [ "--log.level=debug" ];
+        listenAddress = "127.0.0.1";
+      };
+      node = {
+        enable = true;
+        enabledCollectors = [ "logind" ];
+        port = 9002;
+        listenAddress = "127.0.0.1";
+      };
+      blackbox = {
+        enable = true;
+        listenAddress = "127.0.0.1";
+        configFile = ./blackbox.yml;
+      };
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "local";
+        static_configs = [{
+          targets = [
+            "127.0.0.1:9002" # Node exporter
+            #"127.0.0.1:9115" # Blackbox exporter
+            "127.0.0.1:9290" # IPMI exporter for local node
+            "127.0.0.1:9928" # UPC Qaire custom exporter
+            "127.0.0.1:9929" # Meteocat custom exporter
+            "127.0.0.1:9999" # Nix-daemon custom exporter
+          ];
+        }];
+      }
+      {
+        job_name = "blackbox-http";
+        metrics_path = "/probe";
+        params = { module = [ "http_2xx" ]; };
+        static_configs = [{
+          targets = [
+            "https://www.google.com/robots.txt"
+            "https://pm.bsc.es/"
+            "https://pm.bsc.es/gitlab/"
+            "https://jungle.bsc.es/"
+            "https://gitlab.bsc.es/"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "blackbox-icmp";
+        metrics_path = "/probe";
+        params = { module = [ "icmp" ]; };
+        static_configs = [{
+          targets = [
+            "1.1.1.1"
+            "8.8.8.8"
+            "ssfhead"
+            "raccoon"
+            "anella-bsc.cesca.cat"
+            "upc-anella.cesca.cat"
+            "fox.ac.upc.edu"
+            "fox-ipmi.ac.upc.edu"
+            "arenys5.ac.upc.edu"
+            "arenys0-2.ac.upc.edu"
+            "epi01.bsc.es"
+            "axle.bsc.es"
+          ];
+        }];
+        relabel_configs = [
+          {
+            # Takes the address and sets it in the "target=<xyz>" URL parameter
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            # Sets the "instance" label with the remote host we are querying
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            # Shows the host target address instead of the blackbox address
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115";
+          }
+        ];
+      }
+      {
+        job_name = "ipmi-raccoon";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9290" ]; }
+        ];
+        params = {
+          target = [ "raccoon-ipmi" ];
+          module = [ "raccoon" ];
+        };
+      }
+      {
+        job_name = "ipmi-fox";
+        metrics_path = "/ipmi";
+        static_configs = [
+          { targets = [ "127.0.0.1:9290" ]; }
+        ];
+        params = {
+          target = [ "fox-ipmi.ac.upc.edu" ];
+          module = [ "fox" ];
+        };
+      }
+    ];
+  };
+}

m/tent/nginx.nix

@@ -0,0 +1,73 @@
+{ theFlake, pkgs, ... }:
+let
+  website = pkgs.stdenv.mkDerivation {
+    name = "jungle-web";
+    src = theFlake;
+    buildInputs = [ pkgs.hugo ];
+    buildPhase = ''
+      cd web
+      rm -rf public/
+      hugo
+    '';
+    installPhase = ''
+      cp -r public $out
+    '';
+    # Don't mess doc/
+    dontFixup = true;
+  };
+in
+{
+  networking.firewall.allowedTCPPorts = [ 80 ];
+  services.nginx = {
+    enable = true;
+    virtualHosts."jungle.bsc.es" = {
+      root = "${website}";
+      listen = [
+        {
+          addr = "0.0.0.0";
+          port = 80;
+        }
+      ];
+      extraConfig = ''
+        set_real_ip_from 127.0.0.1;
+        set_real_ip_from 84.88.52.107;
+        real_ip_recursive on;
+        real_ip_header X-Forwarded-For;
+
+        location /git {
+          rewrite ^/git$ / break;
+          rewrite ^/git/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:3000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /cache {
+          rewrite ^/cache/(.*) /$1 break;
+          proxy_pass http://127.0.0.1:5000;
+          proxy_redirect http:// $scheme://;
+        }
+        location /lists {
+          proxy_pass http://127.0.0.1:8081;
+          proxy_redirect http:// $scheme://;
+        }
+        location /grafana {
+          proxy_pass http://127.0.0.1:2342;
+          proxy_redirect http:// $scheme://;
+          proxy_set_header Host $host;
+          # Websockets
+          proxy_http_version 1.1;
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "upgrade";
+        }
+        location ~ ^/~(.+?)(/.*)?$ {
+          alias /vault/home/$1/public_html$2;
+          index  index.html index.htm;
+          autoindex on;
+          absolute_redirect off;
+        }
+        location /p/ {
+          alias /var/lib/p/;
+        }
+      '';
+    };
+  };
+}

m/tent/nix-serve.nix

@@ -0,0 +1,16 @@
+{ config, ... }:
+
+{
+  age.secrets.nixServe.file = ../../secrets/nix-serve.age;
+
+  services.nix-serve = {
+    enable = true;
+    # Only listen locally, as we serve it via ssh
+    bindAddress = "127.0.0.1";
+    port = 5000;
+
+    secretKeyFile = config.age.secrets.nixServe.path;
+    # Public key:
+    # jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=
+  };
+}

m/weasel/configuration.nix

@@ -0,0 +1,28 @@
+{ lib, ... }:
+
+{
+  imports = [
+    ../common/ssf.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d5356ca";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  # Users with sudo access
+  users.groups.wheel.members = [ "abonerib" "anavarro" ];
+
+  networking = {
+    hostName = "weasel";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.6";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.6";
+      prefixLength = 24;
+    } ];
+  };
+}

pkgs/meteocat-exporter/default.nix

@@ -0,0 +1,25 @@
+{ python3Packages, lib }:
+
+python3Packages.buildPythonApplication rec {
+  pname = "meteocat-exporter";
+  version = "1.0";
+
+  src = ./.;
+
+  doCheck = false;
+
+  build-system = with python3Packages; [
+    setuptools
+  ];
+
+  dependencies = with python3Packages; [
+    beautifulsoup4
+    lxml
+    prometheus-client
+  ];
+
+  meta = with lib; {
+    description = "MeteoCat Prometheus Exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/meteocat-exporter/meteocat-exporter

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import time
+from prometheus_client import start_http_server, Gauge
+from bs4 import BeautifulSoup
+from urllib import request
+
+# Configuration -------------------------------------------
+meteo_station = "X8" # Barcelona - Zona Universitària
+listening_port = 9929
+update_period = 60 * 5 # Each 5 min
+# ---------------------------------------------------------
+
+metric_tmin = Gauge('meteocat_temp_min', 'Min temperature')
+metric_tmax = Gauge('meteocat_temp_max', 'Max temperature')
+metric_tavg = Gauge('meteocat_temp_avg', 'Average temperature')
+metric_srad = Gauge('meteocat_solar_radiation', 'Solar radiation')
+
+def update(st):
+    url = 'https://www.meteo.cat/observacions/xema/dades?codi=' + st
+    response = request.urlopen(url)
+    data = response.read()
+    soup = BeautifulSoup(data, 'lxml')
+    table = soup.find("table", {"class" : "tblperiode"})
+    rows = table.find_all('tr')
+    row = rows[-1] # Take the last row
+    row_data = []
+    header = row.find('th')
+    header_text = header.text.strip()
+    row_data.append(header_text)
+    for col in row.find_all('td'):
+        row_data.append(col.text)
+    try:
+        # Sometimes it will return '(s/d)' and fail to parse
+        metric_tavg.set(float(row_data[1]))
+        metric_tmax.set(float(row_data[2]))
+        metric_tmin.set(float(row_data[3]))
+        metric_srad.set(float(row_data[10]))
+        #print("ok: temp_avg={}".format(float(row_data[1])))
+    except:
+        print("cannot parse row: {}".format(row))
+        metric_tavg.set(float("nan"))
+        metric_tmax.set(float("nan"))
+        metric_tmin.set(float("nan"))
+        metric_srad.set(float("nan"))
+
+if __name__ == '__main__':
+    start_http_server(port=listening_port, addr="localhost")
+    while True:
+        try:
+            update(meteo_station)
+        except:
+            print("update failed")
+        time.sleep(update_period)

pkgs/meteocat-exporter/setup.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(name='meteocat-exporter',
+      version='1.0',
+      # Modules to import from other scripts:
+      packages=find_packages(),
+      # Executables
+      scripts=["meteocat-exporter"],
+     )

pkgs/mpich-fix-hwtopo.patch

@@ -1,36 +0,0 @@
-diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
-index 33e88bc..ee3641c 100644
---- a/src/util/mpir_hwtopo.c
-+++ b/src/util/mpir_hwtopo.c
-@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
- #ifdef HAVE_HWLOC
-     bindset = hwloc_bitmap_alloc();
-     hwloc_topology_init(&hwloc_topology);
--    char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
--    if (xmlfile != NULL) {
--        int rc;
--        rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
--        if (rc == 0) {
--            /* To have hwloc still actually call OS-specific hooks, the
--             * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
--             * file is really the underlying system. */
--            hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
--        }
--        MPL_free(xmlfile);
--    }
-
-     hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
-     if (!hwloc_topology_load(hwloc_topology)) 
-
---- a/src/mpi/init/local_proc_attrs.c
-+++ b/src/mpi/init/local_proc_attrs.c
-@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
-     /* Set the number of tag bits. The device may override this value. */
-     MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
-
--    char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
--    MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
--    MPL_free(requested_kinds);
--
-     return mpi_errno;
- }

pkgs/overlay.nix

@@ -11,10 +11,6 @@ final: prev:
       paths = [ pmix.dev pmix.out ];
     };
   in prev.mpich.overrideAttrs (old: {
-    patches = [
-      # See https://github.com/pmodels/mpich/issues/6946
-      ./mpich-fix-hwtopo.patch
-    ];
     buildInput = old.buildInputs ++ [
       libfabric
       pmixAll
@@ -39,7 +35,21 @@ final: prev:
       # See https://bugs.schedmd.com/show_bug.cgi?id=19324
       ./slurm-rank-expansion.patch
     ];
+    # Install also the pam_slurm_adopt library to restrict users from accessing
+    # nodes with no job allocated.
+    postBuild = (old.postBuild or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security"
+      popd
+    '';
+    postInstall = (old.postInstall or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security" install
+      popd
+    '';
   });
 
   prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
+  meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { };
+  upc-qaire-exporter = prev.callPackage ./upc-qaire-exporter/default.nix { };
 }

pkgs/upc-qaire-exporter/default.nix

@@ -0,0 +1,24 @@
+{ python3Packages, lib }:
+
+python3Packages.buildPythonApplication rec {
+  pname = "upc-qaire-exporter";
+  version = "1.0";
+
+  src = ./.;
+
+  doCheck = false;
+
+  build-system = with python3Packages; [
+    setuptools
+  ];
+
+  dependencies = with python3Packages; [
+    prometheus-client
+    requests
+  ];
+
+  meta = with lib; {
+    description = "UPC Qaire Prometheus Exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/upc-qaire-exporter/setup.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(name='upc-qaire-exporter',
+      version='1.0',
+      # Modules to import from other scripts:
+      packages=find_packages(),
+      # Executables
+      scripts=["upc-qaire-exporter"],
+     )

pkgs/upc-qaire-exporter/upc-qaire-exporter

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import time
+from prometheus_client import start_http_server, Gauge
+import requests, json
+from datetime import datetime, timedelta
+
+# Configuration -------------------------------------------
+listening_port = 9928
+update_period = 60 * 5 # Each 5 min
+# ---------------------------------------------------------
+
+metric_temp = Gauge('upc_c6_s302_temp', 'UPC C6 S302 temperature sensor')
+
+def genparams():
+    d = {}
+    d['topic'] = 'TEMPERATURE'
+    d['shift_dates_to'] = ''
+    d['datapoints'] = 301
+    d['devicesAndColors'] = '1148418@@@#40ACB6'
+
+    now = datetime.now()
+
+    d['fromDate'] = now.strftime('%d/%m/%Y')
+    d['toDate'] = now.strftime('%d/%m/%Y')
+    d['serviceFrequency'] = 'NONE'
+
+    # WTF!
+    for i in range(7):
+        for j in range(48):
+            key = 'week.days[{}].hours[{}].value'.format(i, j)
+            d[key] = 'OPEN'
+
+    return d
+
+def measure():
+    # First we need to load session
+    s = requests.Session()
+    r = s.get("https://upc.edu/sirena")
+    if r.status_code != 200:
+        print("bad HTTP status code on new session: {}".format(r.status_code))
+        return
+
+    if s.cookies.get("JSESSIONID") is None:
+        print("cannot get JSESSIONID")
+        return
+
+    # Now we can pull the data
+    url = "https://upcsirena.app.dexma.com/l_12535/analysis/by_datapoints/data.json"
+    r = s.post(url, data=genparams())
+
+    if r.status_code != 200:
+        print("bad HTTP status code on data: {}".format(r.status_code))
+        return
+
+    #print(r.text)
+    j = json.loads(r.content)
+
+    # Just take the last one
+    last = j['data']['chartElementList'][-1]
+    temp = last['values']['1148418-Temperatura']
+
+    return temp
+
+if __name__ == '__main__':
+    start_http_server(port=listening_port, addr="localhost")
+    while True:
+        try:
+            metric_temp.set(measure())
+        except:
+            print("measure failed")
+            metric_temp.set(float("nan"))
+
+        time.sleep(update_period)

secrets/gitea-runner-token.age

@@ -1,9 +1,11 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg DQdgCk16Yu524BsrWVf0krnwWzDM6SeaJCgQipOfwCA
+-> ssh-ed25519 HY2yRg d7+nvfAcdC3GjJxipXFrsfGGyP5jAY+gRWRV+4FVYAM
-Ab9ocqra/UWJZI+QGMlxUhBu5AzqfjPgXl+ENIiHYGs
+CG7r0bRGgnUWcdfDnpe7HwZ3L/y7b5iuJuqvf15b3/Y
--> ssh-ed25519 CAWG4Q KF9rGCenb3nf+wyz2hyVs/EUEbsmUs5R+1fBxlCibC8
+-> ssh-ed25519 CAWG4Q X0vITOErz4wkR3VQYOcVlnrkHtwe+ytdZz1Hcrs4vVs
-7++Kxbr3FHVdVfnFdHYdAuR0Tgfd+sRcO6WRss6LhEw
+6IWYOhXLQ+BnML9YfLLHJYEO2CZ/uEc9IBqhoWvjDHI
--> ssh-ed25519 MSF3dg aUe4DhRsu4X8CFOEAnD/XM/o/0qHYSB522woCaAVh0I
+-> ssh-ed25519 xA739A p5e/0AJtZ0+zbRvkB/usLuxusY8xXRx9Ksi/LQlcIHw
-GRcs5cm2YqA/lGhUtbpboBaz7mfgiLaCr+agaB7vACU
+M4S/qlzT9POyJx4gY9lmycstUcdwG2cinN4OlV22zzo
---- 9Q7Ou+Pxq+3RZilCb2dKC/pCFjZEt4rp5KnTUUU7WJ8
+-> ssh-ed25519 MSF3dg Ydl7uBWzBx6sAaxbzC3x8qiaU3ysGqV4rUFLpHCEV30
-1<12>
Mw4<77><34>	<09>:H<>@<40>/<2F>gLtM<74>,<2C>ƥ<>*<2A><>z<EFBFBD>NV5<56>m<EFBFBD><6D>N<EFBFBD>o<EFBFBD><6F>j1$<24>T<EFBFBD>G_<47>E{<7B>%<25><>1ǯ<17><>H<EFBFBD><EFBFBD>A<EFBFBD>p<EFBFBD>
+/1AUHBhCNOs9i7LJbmzwQDHsu+ybzYf6+coztKk5E3U
+--- kYt15WxClpT7PXD1oFe9GqJU+OswjH7y9wIc8/GzZ7M
+<EFBFBD><EFBFBD>h<>ߓ<><DF93><EFBFBD>`<60><><EFBFBD>V4F<34><46>_k)^<5E>m$uj:ѳ<><D1B3><17><><EFBFBD>}<7D>Z]$U]<12>u<EFBFBD> <20>0<EFBFBD><30><EFBFBD>v8<76>?<3F>X<EFBFBD>P<EFBFBD>g%d<>#<23>d9{rAi<41><69>

secrets/gitlab-bsc-docker-token.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg 2jO/raAo/qSdOmENknW2ksMgGQDmn9re+X//ekjgRGg
++Wo4K69UucwxYtbMSn3dDB6M5/1y+PY+p071vgPOsuE
+-> ssh-ed25519 CAWG4Q WKIgDc1Bv/iHas0D7IHmbru5Jy/Iy4abUsIuTDC8+Wc
+kNbE/CBggRmgUk3OewptR3oNo/t/uOQuf8nZDfJUoLA
+-> ssh-ed25519 xA739A FaaHnwsRGA+PZgVjyqvY6ctw/frYFKQzRl92+mTXZwM
+ivABFIbzu4pQrz3yp2IiyTXFFmb/aWO+MaBwz8Bsabc
+-> ssh-ed25519 MSF3dg HHwnX2dzb4PVdwKUROo8eQS2iyVc5QCUIXanObh71FU
+dM0qRA18DdN3QFEr5etAjtCNn2U2dmkqbUkykEp4eLg
+--- x+ixidWejvGMWNz6qzelFPz+9sJ15T7oofx3G9O1ZZE
+<EFBFBD><0E>VFh-=B<><42>9<EFBFBD>
)<29>o<EFBFBD>E<<3C><>q(P<><1E><14>(/<2F>YqsU<73><55>X<EFBFBD>e<EFBFBD>䉐)

secrets/jungle-robot-password.age

@@ -1,10 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg 3L1Y5upc5qN6fgiFAox5rD/W8n0eQUv5mT39QAdO5Ac
+-> ssh-ed25519 HY2yRg rsbyYULV9S/kz4OzBLQIVfyotgKrzPzvjPNVw69coTo
-XkWsmPmzRgHjsvJgsDKJRgHZ7/sBZFmd1Doppj/y390
+i9fgGAYTPxJ4Ulft3xzwNPF8v85Ae9ePMNWp593vSfA
--> ssh-ed25519 CAWG4Q v03Qr+fckdIpsxvQG/viKxlF8WNpO4XUe//QcPzH4k0
+-> ssh-ed25519 G5LX5w mhB3iiqV2e+tT31FCREX2Bqq2F2g+vTYvjCuyGSeJxs
-afUwi3ccDCRfUxPDdF7ZkoL+0UX1XwqVtiyabDWjVQk
+Ep9zZykCGFW841S2mnllEi0oPnRiRuYIGtv6ckp+IBg
--> ssh-ed25519 MSF3dg c2hEUk4LslJpiL7v/4UpT8fK7ZiBJ8+uRhZ/vBoRUDE
+-> ssh-ed25519 CAWG4Q M0AJEZuiC6FnRy8rAJQ9T9dCXfIfLXGk0uBGhYOxRSg
-YX9EpnJpHo1eDsZtapTVY6jD+81kb588Oik4NoY9jro
+5jSRNTi0c6we/oLBdUy1am5saH/5Nh1fmVqYajXFbGc
---- LhUkopNtCsyHCLzEYzBFs+vekOkAR4B3VBaiMF/ZF8w
+-> ssh-ed25519 xA739A Zf9tUKg4S4UuWMGEtAWVg0pa6vTzKIl2Ty39IjEG2mE
-o˝<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>C<EFBFBD><EFBFBD>H<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>y<EFBFBD>LؔItM<74><4D><EFBFBD><EFBFBD>IױsM<12>\<5C>1-<2D>K<EFBFBD><4B><10><>G:<><7F><02><>
+RCSkVFyO2ZuDlAHung9bTeM91aTXxNRJ779kE0C6pK4
-<EFBFBD>g<16><><EFBFBD>bpF<70>Ӷ<EFBFBD>%Y<>
+-> ssh-ed25519 MSF3dg QLiG9s3mgfO6HnQ8/ReizkGllsjYebIL5ZthSVcD7Ao
+YdzcodBarrdg6R96Ys01aEPoeYygbT56yz90BMFfr0U
+--- fS/rGOP3IGG8b3bCDy26nBL0P1rtqC70CmKOGDsg8Tw
+;Y<><59>M<EFBFBD>_<EFBFBD><5F><EFBFBD>Zꙺ:<3A>]Ez89z
e<><65><EFBFBD>	<09><>D
<01>X<EFBFBD><19><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>9{<7B>x^<5E><><EFBFBD>L<>l<EFBFBD><6C><EFBFBD><0B>㦑9R<39>VhWs

secrets/nix-serve.age

@@ -1,12 +1,14 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg d144D+VvxhYgKtH//uD2qNuVnYX6bh74YqkyM3ZjBwU
+-> ssh-ed25519 HY2yRg tdVrzL3EryCEDJSiAjHfr3AC6rhyKLLe9ZaKKa/fyEk
-0IeVmFAf4U8Sm0d01O6ZwJ1V2jl/mSMl4wF0MP5LrIg
+kIbJjp/odUkQ9E2fXpk4zratLieyMNdNLHYGQt8+860
--> ssh-ed25519 CAWG4Q H4nKxue/Cj/3KUF5A+/ygHMjjArwgx3SIWwXcqFtyUo
+-> ssh-ed25519 G5LX5w A0wBDwowrQyByfinVVrypH5VyvyKk3O3O8+2JnVgcCI
-4k5NJkLUrueLYiPkr2LAwQLWmuaOIsDmV/86ravpleU
+kLiXfQkC+8QycLyyM/6dAKEE6SGxSZJS7PuOTQr10XE
--> ssh-ed25519 MSF3dg HpgUAFHLPs4w0cdJHqTwf8lySkTeV9O9NnBf49ClDHs
+-> ssh-ed25519 CAWG4Q HkbFgDtrbuv+KCwULZppiy88ZHl3kHcdlTVTOfMKTzM
-foPIUUgAYe1YSDy6+aMfjN7xv9xud9fDmhRlIztHoEo
+KMGdQl8Gl51gUp1bxEa41a0VBBiHWD81/9C75NX/pzA
--> vLkF\<-grease
+-> ssh-ed25519 xA739A XfYFE5jPFvcoTMXtwJgs3+HPLQxRmvz1W7yqE7jSYGE
-3GRT+W8gYSpjl/a6Ix9+g9UJnTpl1ZH/oucfR801vfE8y77DV2Jxz/XJwzxYxKG5
+497iDMqiIx1u+cBu8KZDNF2SPpGCrVqjGdUPD8kEjE4
-YEhiTGMNbXw/V7E5aVSz6Bdc
+-> ssh-ed25519 MSF3dg Vbxxsmfoywpi4W9WUMzgay3Nd1UBigliYHD7Wew9AHM
---- GtiHKCZdHByq9j0BSLd544PhbEwTN138E8TFdxipeiA
+aLt5GN8jJWbbrHfs1321tQz44lBaATe0BipT/EGc80I
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>G$S<><53><EFBFBD>RA<52><41><EFBFBD>Th]n<>8<EFBFBD>,<2C>H<EFBFBD>s<EFBFBD><73><EFBFBD>=p<><70><EFBFBD>'<27><><EFBFBD>+<2B>j<><6A><EFBFBD><EFBFBD>9<EFBFBD>)<29>:<3A>)<15><><EFBFBD>Y<EFBFBD><59><EFBFBD>8<EFBFBD>I<EFBFBD><49>8:ol<6F><6C><EFBFBD><1F><><EFBFBD>Z<EFBFBD><5A>3<>PM<50>F;<3B>rY<72><59><EFBFBD><EFBFBD><1F>$<24><>y<EFBFBD>L<>ٜ<EFBFBD>Μ<1B><>U<EFBFBD>s16Ǿ<>L<EFBFBD>b<EFBFBD><62><EFBFBD>
+--- JHESkz0eGNPo3ZEGALVH4xsQ4p1O/6ShlfOw58fjH1k
+
+<EFBFBD>AwN<EFBFBD>g<EFBFBD><EFBFBD><EFBFBD><EFBFBD>C<EFBFBD><EFBFBD><EFBFBD>Ԣְ7<EFBFBD>	ǟ4#0<><30><EFBFBD>ss<73><73><EFBFBD>-*<2A><19>$Z<><5A><EFBFBD><EFBFBD><EFBFBD>[*<2A><>
ia<69>{<7B>?=<3D><08><>v-E<EE9495>7<EFBFBD><37><10>0<EFBFBD><30>]<5D><>q0<71>)q"K<><4B><EFBFBD>{BZs<7F><73><EFBFBD><EFBFBD>*<2A>l<EFBFBD><6C>9-E+<02><>8<(<28><><EFBFBD>a*$dN<64><4E>xd

secrets/secrets.nix

@@ -2,6 +2,8 @@ let
   keys = import ../keys.nix;
   adminsKeys = builtins.attrValues keys.admins;
   hut = [ keys.hosts.hut ] ++ adminsKeys;
+  mon = [ keys.hosts.hut keys.hosts.tent ] ++ adminsKeys;
+  tent = [ keys.hosts.tent ] ++ adminsKeys;
   # Only expose ceph keys to safe nodes and admins
   safe = keys.hostGroup.safe ++ adminsKeys;
 in
@@ -9,8 +11,16 @@ in
   "gitea-runner-token.age".publicKeys = hut;
   "gitlab-runner-docker-token.age".publicKeys = hut;
   "gitlab-runner-shell-token.age".publicKeys = hut;
-  "nix-serve.age".publicKeys = hut;
+  "gitlab-bsc-docker-token.age".publicKeys = hut;
-  "jungle-robot-password.age".publicKeys = hut;
+  "nix-serve.age".publicKeys = mon;
+  "jungle-robot-password.age".publicKeys = mon;
+  "ipmi.yml.age".publicKeys = mon;
+
+  "tent-gitlab-runner-pm-docker-token.age".publicKeys = tent;
+  "tent-gitlab-runner-pm-shell-token.age".publicKeys = tent;
+  "tent-gitlab-runner-bsc-docker-token.age".publicKeys = tent;
+  "vpn-dac-login.age".publicKeys = tent;
+  "vpn-dac-client-key.age".publicKeys = tent;
 
   "ceph-user.age".publicKeys = safe;
   "munge-key.age".publicKeys = safe;

secrets/tent-gitlab-runner-bsc-docker-token.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 G5LX5w HlQ4V8lBd3im5j8KHEuQZBTuztvPj1QoWdv6FL6qzGI
+Jpt91X1UIIVFQt1X6Q//kALn+Cetp/LqBZZvTuhFthw
+-> ssh-ed25519 CAWG4Q StnngJAcuAwUnTrXDR3nJ2KFN0jNdTqSz+/1TfmWkzA
+CR4AQ6fqaJVY1mdUIX1gzaZwRs1sU8F8hHztnkN8vN0
+-> ssh-ed25519 xA739A xya5A5t63Owx+VrGgUfV/lIP8b/xV1cerMpuZBLaDVM
+w+pA583yUnFq2AvGBGzWbQIGQEY9WqW0CSLQ9v+SG0c
+-> ssh-ed25519 MSF3dg aXkLxCyYdOwVopHHmpXEI6WlAIizKdJi4IO0KEdhS3s
+WKXkTszZN66+QZdSDJ4D9q7xgYWMfliOLCubIF2Dqkc
+--- uVWoU2lMkqQ/9Z0BqKRCeUpsKi8lwmHukT/FV8wYMbg
+<EFBFBD><EFBFBD>1G+<2B>6<EFBFBD><36>g[|x]2T<32>й<EFBFBD><D0B9><EFBFBD> <20>CKu)<29><><EFBFBD>]<5D><>8֓<38><D693><EFBFBD><EFBFBD>l<EFBFBD><6C>S<EFBFBD><53><EFBFBD>Q<EFBFBD><07><>x<EFBFBD><78><EFBFBD><EFBFBD>#7r<37>k{*<2A><>3ս~C<>b<EFBFBD><62><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ڵ<EFBFBD>Np<1E><05>]J]h<>je+d%Е<>#<23>m<EFBFBD>?=6}<7D> 

secrets/tent-gitlab-runner-pm-docker-token.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 G5LX5w sg9SmahxBg35MDIxhrp4oHkaTaxsKoVQju2eNhCt0BM
+CZ64dEGqz2tbkG8KtimZvLUEMrQpVVBJP7Fu46WTMgc
+-> ssh-ed25519 CAWG4Q jzS1R14W1CWxdziMLG/yCGPLWSkiyE+9lqyCVe491ng
+acJo/nhKq3pSPoFEPaFLN1fzHHbEzstNoLtohWAHKiM
+-> ssh-ed25519 xA739A qeGJoLeSIQwLU2Yg+Gi2bikHJ3HscLfyo1msqL3JwHw
+tTwaxRBKTl/SoyY/LnxR/j/5WvCNX5VeZLKi018YMrY
+-> ssh-ed25519 MSF3dg Wym7Uyf1XvH1H6mNDERkO8opkMiN0zzXm2PjXftEOWs
+Uw8ZwwKIB5UqgVuoSLE2QajNDJZkH7/Y3Nsy+WFl7Xs
+--- 94hGVbYiCGZdMEJesCMLh7IZi+w5l/Kr1lZJHQgrc0o
+j5j磛<6A><04><>J<EFBFBD><4A><EFBFBD>a<EFBFBD>]<5D>a%dr<64><72>FDT<44><54>^<5E><>Q<EFBFBD>s/<2F>kwB<77>$<24><>$<24><>H<EFBFBD>'<27><><EFBFBD><EFBFBD><EFBFBD>w<14><?^|<7C><07>h$<24>ؗ<EFBFBD>GI<47>ĕsT2RU<52><55>*/O<>7<EFBFBD><37><EFBFBD>G<EFBFBD>pͪ<70>4<EFBFBD><34><EFBFBD>M9<4D>j<><06>

secrets/tent-gitlab-runner-pm-shell-token.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 G5LX5w 5K0mzfJGvAB2LGmoQ9ZLbWooVEX6F4+fQdo1JUoB3FM
+AKGa507bUrYjXFaMQ1MXTDBFYsdS6zbs+flmxYN0UNo
+-> ssh-ed25519 CAWG4Q 8KzLc949on8iN1pK8q11OpCIeO71t6b0zxCLHhcQ6ns
+uy7z6RdIuoUes+Uap3k5eoFFuu/DcSrEBwq4V4C/ygc
+-> ssh-ed25519 xA739A SLx5cKo0fdAHj+cLpJ4FYTWTUTyDsCqKQOufDu3xnGo
+VnS/WsiSaf6RpXuhgfij4pYu4p9hlJl1oXrfYY9rKlQ
+-> ssh-ed25519 MSF3dg c5ZXvdNxNfZU3HeWsttuhy+UC5JxWN/IFuCuCGbksn4
+vcKlIirf+VvERX71YpmwW6zp6ClhlG2PR4R8LIN7cQo
+--- pJKICDaYAlxqNnvHIuzB3Yk7tv0ZNYflGTQD+Zk/8+4
+<EFBFBD>h/\J<>J
+<EFBFBD>0?<3F> <20>p<EFBFBD><70><EFBFBD>@܉7<DC89><37>3<EFBFBD><33><EFBFBD><EFBFBD>z<EFBFBD><7A><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>a<EFBFBD><61>'<27>,ka<6B>I<EFBFBD>XXOZ<4F>I\<5C><><EFBFBD><EFBFBD><EFBFBD>	<09>BP<42><50>/cUɿ~B<><42>S'Q<><51><EFBFBD><EFBFBD>f<06><><EFBFBD>er<65><72><EFBFBD><EFBFBD>^<5E><><EFBFBD><EFBFBD>8l<38><6C>V<EFBFBD>E<EFBFBD><45><EFBFBD>

secrets/vpn-dac-login.age

@@ -0,0 +1,12 @@
+age-encryption.org/v1
+-> ssh-ed25519 G5LX5w /RF8uZ/KahUqjEFILbF3+Jin+U0SQdoQChcc9RJ9axc
+aEmPk++86nBR6d2BIa/oaUdyiLS6cH8TUoYJE3bxba4
+-> ssh-ed25519 CAWG4Q qHyh9nQi8c3z/KHby9y5vhzN0Dwz0zca98ebjJmXrzs
+ZbmwNzrSSQ3RvskE8SqcBa0vMy8pzm/HPGHLm5zuPGQ
+-> ssh-ed25519 xA739A FlGbfS4bUxA3gVDzb3yPjp4hV8a7aiNBLUctnN3bGEY
+3fI6SyVjVhh2M8uc/XV3blpdQMPMYi2qzaHNXvx0bvM
+-> ssh-ed25519 MSF3dg 0Bs/aW0nNISS+93It75o6hKZWa7S+LF5bF5ApsJ2fQ8
+y7o0KYDHEen13ndIxg/mYil3eMxxzvYF2pWqhMb+rBU
+--- Iqo75G4+02Y9nc1OOkcEx+iQlKnGYCekAx76tRH53wA
+<10>
+<EFBFBD>X<EFBFBD><EFBFBD>%f<0C><><12>hX<0B><>R<>c<EFBFBD>+z<><7A>eg<65>
& <20>d<EFBFBD><64><EFBFBD>ק<06><>A<EFBFBD><41><EFBFBD>чXM<58>1<EFBFBD>

web/content/access/index.md

@@ -5,18 +5,12 @@ description: "Request access to the machines"
 
 ![Cave](./cave.jpg)
 
-Before requesting access to the jungle machines, you must be able to access the
+To request access to the machines we will need some information:
-`ssfhead.bsc.es` node (only available via the intranet or VPN). You can request
-access to the login machine using a resource petition in the BSC intranet.
 
-Then, to request access to the machines we will need some information about you:
+1. Which machines you want access to ([hut](/hut), [fox](/fox), owl1, owl2, eudy, koro...)
-
+1. Your user name (make sure it matches the one you use for the BSC intranet)
-1. Which machines you want access to (hut, owl1, owl2, eudy, koro...)
-1. Your user name and user id (to match the NFS permissions)
 1. Your real name and surname (for identification purposes)
 1. The salted hash of your login password, generated with `mkpasswd -m sha-512`
 1. An SSH public key of type Ed25519 (can be generated with `ssh-keygen -t ed25519`)
 
-Send an email to <jungle@bsc.es> with the details, or directly open a
+Send an email to <jungle@bsc.es> with the details.
-merge request in the [jungle
-repository](https://pm.bsc.es/gitlab/rarias/jungle/).

web/content/doc/_index.md

@@ -0,0 +1,10 @@
+---
+title: "Docs"
+description: "Documentation for users of jungle machines"
+date: 2023-09-15
+---
+
+If this is the first time you use any of the jungle machines with NixOS, follow
+the [quick start guide](quickstart).
+
+

web/content/doc/quickstart.md

@@ -0,0 +1,234 @@
+---
+title: "Quick start"
+date: 2023-09-15
+---
+
+This documentation will guide you on how to build custom packages of software
+and use them in the jungle machines. It has been designed to reduce the friction
+from users coming from module systems.
+
+You should be able to access the jungle machines, otherwise [request
+access](/access).
+
+## Changes from other HPC machines
+
+Users of other machines have been using the Lmod tool (module load ...) to add
+or remove programs from their environment, as well as manually building their
+own software for too many years.
+
+While we cannot prevent users from continuing to use this tedious mechanism, we
+have designed the jungle machines to be much easier to operate by using the nix
+package manager.
+
+### Freedom to install packages
+
+When a user wanted to install a package, it was forced to either do it on its
+own directory, or request a system administrator to install it in a shared
+directory, so other users can also use that package.
+
+This situation is gone, each user can install any package of software by
+themselves, without requiring any other authorization. When two users request
+the same package, the same copy will be provided.
+
+A new package will be downloaded if it is available (someone already built it)
+or will be built from source on demand.
+
+### No changes over time
+
+All users retain the same versions of the packages they request until they
+decide to update them.
+
+## Using nix to manage packages
+
+In this chapter we show how to install packages and enter a development shell to
+build new programs from source. The examples are done from the hut machine,
+read [this page](/access) to request access.
+
+### Installing binaries
+
+To temporarily install new packages, use:
+
+```text
+hut% nix shell jungle#gcc jungle#cowsay jungle#ovni
+```
+
+Notice that the packages are described as two parts divided by the `#` symbol.
+The first part defines where to take the package from and the second part is
+the name of the package. For now we will use `jungle#<package>`. You can find
+many more packages here:
+
+<https://search.nixos.org/packages>
+
+You will now enter a new shell, where those requested package **binaries are
+available in $PATH**:
+
+```text
+hut% cowsay hello world
+ _____________
+< hello world >
+ -------------
+        \   ^__^
+         \  (oo)\_______
+            (__)\       )\/\
+                ||----w |
+                ||     ||
+
+hut% ovniver
+LD_LIBRARY_PATH not set
+libovni: build v1.11.0 (a7103f8), dynamic v1.11.0 (a7103f8)
+
+hut% gcc --version
+gcc (GCC) 13.3.0
+Copyright (C) 2023 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+```
+
+### Building programs
+
+The above method only loads new binaries in the `$PATH`. If we try to build a
+program that includes headers or links with a library, it will fail to find
+them:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut% gcc test.c -lovni -o test
+test.c:1:10: fatal error: ovni.h: No such file or directory
+    1 | #include <ovni.h>
+      |          ^~~~~~~~
+compilation terminated.
+```
+
+We could manually add the full path to the ovni include directory with `-I` and
+the libraries with `-L`, but there is a tool that already perform these steps
+automatically for us, `nix develop`.
+
+Let's go back to our original shell first, where those packages are not
+available anymore:
+
+```
+hut% ps
+    PID TTY          TIME CMD
+2356260 pts/1    00:00:01 zsh
+2457268 pts/1    00:00:00 zsh
+2457297 pts/1    00:00:00 ps
+hut% exit
+hut% ovniver
+ovniver: command not found
+```
+
+### Creating a flake.nix
+
+To define which packages we want, we will write a small file that list them, a
+flake.nix file.
+
+First, we will create a new directory where we are going to be working:
+
+```
+hut% mkdir example
+hut% cd exmple
+```
+
+Then place this flake.nix file:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell {
+      pname = "devshell";
+      buildInputs = with pkgs; [
+        ovni gcc cowsay # more packages here...
+      ];
+    };
+  };
+}
+```
+
+
+Now enter the shell with:
+
+```
+hut% nix develop
+warning: creating lock file '/home/Computational/rarias/example/flake.lock':
+• Added input 'jungle':
+    'path:/nix/store/27srv8haj6vv4ywrbmw0a8vds561m8rq-source?lastModified=1739479441&narHash=sha256-Kgjs8SO1w9NbPBu8ghwzCxYJ9kvWpoQOT%2BXwPvA9DcU%3D&rev=76396c0d67ef0cf32377d5c1894bb695293bca9d' (2025-02-13)
+• Added input 'jungle/agenix':
+    'github:ryantm/agenix/f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41?narHash=sha256-b%2Buqzj%2BWa6xgMS9aNbX4I%2BsXeb5biPDi39VgvSFqFvU%3D' (2024-08-10)
+• Added input 'jungle/agenix/darwin':
+    'github:lnl7/nix-darwin/4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d?narHash=sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0%3D' (2023-11-24)
+• Added input 'jungle/agenix/darwin/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/home-manager':
+    'github:nix-community/home-manager/3bfaacf46133c037bb356193bd2f1765d9dc82c1?narHash=sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE%3D' (2023-12-20)
+• Added input 'jungle/agenix/home-manager/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/agenix/systems':
+    'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e?narHash=sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768%3D' (2023-04-09)
+• Added input 'jungle/bscpkgs':
+    'git+https://git.sr.ht/~rodarima/bscpkgs?ref=refs/heads/master&rev=6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f' (2024-11-29)
+• Added input 'jungle/bscpkgs/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/nixpkgs':
+    'github:NixOS/nixpkgs/9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc?narHash=sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8%3D' (2025-01-14)
+
+hut$ 
+```
+
+Notice that long list of messages is Nix creating a new flake.lock file with the
+current state of the packages. Next invocations will use the same packages as
+described by the lock file.
+
+### Building a program from nix develop
+
+Now let's try again building our test program:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut$ gcc test.c -o test -lovni
+hut$ ldd test
+        linux-vdso.so.1 (0x00007ffff7fc4000)
+        libovni.so.1 => /nix/store/sqk972akjv0q8dchn8ccjln2llzyyfd0-ovni-1.11.0/lib/libovni.so.1 (0x00007ffff7fab000)
+        libc.so.6 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/libc.so.6 (0x00007ffff7db2000)
+        /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/ld-linux-x86-64.so.2 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib64/ld-linux-x86-64.so.2 (0x00007ffff7fc6000)
+hut$ ./test
+```
+
+Now the ovni.h header and the libovni library are found and the program is
+successfully built, linked and executed.
+
+You can add more packages as needed in your flake.nix:
+
+```nix
+  buildInputs = with pkgs; [
+    ovni gcc cowsay # more packages here...
+  ];
+```
+
+Make sure you exit the develop shell first, and then enter again with `nix
+develop`.
+
+## Remember
+
+- `nix shell` places binaries in the `$PATH`.
+- `nix develop` enters a development shell where both binaries and the libraries
+  and includes are available so you can build new programs.

web/content/fox/_index.md

@@ -0,0 +1,97 @@
+---
+title: "Fox"
+description: "AMD Genoa 9684X with 2 NVIDIA RTX4000 GPUs"
+date: 2025-02-12
+---
+
+![Fox](fox.jpg)
+
+Picture by [Joanne Redwood](https://web.archive.org/web/20191109175146/https://www.inaturalist.org/photos/6568074),
+[CC0](http://creativecommons.org/publicdomain/zero/1.0/deed.en).
+
+The *fox* machine is a big GPU server that is configured to run heavy workloads.
+It has two fast AMD CPUs with large cache and 2 reasonable NVIDIA GPUs. Here are
+the detailed specifications:
+
+- 2x AMD GENOA X 9684X DP/UP 96C/192T 2.55G 1,150M 400W SP5 3D V-cach
+- 24x 32GB DDR5-4800 ECC RDIMM (total 768 GiB of RAM)
+- 1x 2.5" SSD SATA3 MICRON 5400 MAX 480GB
+- 2x 2.5" KIOXIA CM7-R 1.92TB NVMe GEN5 PCIe 5x4
+- 2x NVIDIA RTX4000 ADA Gen 20GB GDDR6 PCIe 4.0
+
+## Access
+
+To access the machine, request a SLURM session from [hut](/hut) using the `fox`
+partition:
+
+    hut% salloc -p fox
+
+Then connect via ssh:
+
+    hut% ssh fox
+    fox%
+
+Follow [these steps](/access) if you don't have access to hut or fox.
+
+## CUDA
+
+To use CUDA, you can use the following `flake.nix` placed in a new directory to
+load all the required dependencies:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+
+  outputs = { jungle, ... }: {
+    devShell.x86_64-linux = let
+      pkgs = jungle.nixosConfigurations.fox.pkgs;
+    in pkgs.mkShell {
+      name = "cuda-env-shell";
+      buildInputs = with pkgs; [
+        git gitRepo gnupg autoconf curl
+        procps gnumake util-linux m4 gperf unzip
+
+        # Cuda packages (more at https://search.nixos.org/packages)
+        cudatoolkit linuxPackages.nvidia_x11
+        cudaPackages.cuda_cudart.static
+        cudaPackages.libcusparse
+
+        libGLU libGL
+        xorg.libXi xorg.libXmu freeglut
+        xorg.libXext xorg.libX11 xorg.libXv xorg.libXrandr zlib
+        ncurses5 stdenv.cc binutils
+      ];
+      shellHook = ''
+        export CUDA_PATH=${pkgs.cudatoolkit}
+        export LD_LIBRARY_PATH=/var/run/opengl-driver/lib
+        export SMS=50
+      '';
+    };
+  };
+}
+```
+
+Then just run `nix develop` from the same directory:
+
+    % mkdir cuda
+    % cd cuda
+    % vim flake.nix
+    [...]
+    % nix develop
+    $ nvcc -V
+    nvcc: NVIDIA (R) Cuda compiler driver
+    Copyright (c) 2005-2024 NVIDIA Corporation
+    Built on Tue_Feb_27_16:19:38_PST_2024
+    Cuda compilation tools, release 12.4, V12.4.99
+    Build cuda_12.4.r12.4/compiler.33961263_0
+
+## Filesystems
+
+The machine has several file systems available.
+
+- `$HOME`: Mounted via NFS across all nodes. It is slow and has low capacity.
+  Don't abuse.
+- `/ceph/home/$USER`: Shared Ceph file system across jungle nodes. Slow but high
+  capacity. Stores three redundant copies of every file.
+- `/nvme{0,1}/$USER`: The two local NVME disks, very fast and large capacity.
+- `/tmp`: tmpfs, fast but not backed by a disk. Will be erased on reboot.

web/content/hut/_index.md

@@ -13,6 +13,115 @@ which is available at `hut` or `xeon07`. It runs the following services:
 - Grafana: to plot the data in the web browser.
 - Slurmctld: to manage the SLURM nodes.
 - Gitlab runner: to run CI jobs from Gitlab.
+- Nix binary cache: to serve cached nix builds
 
 This node is prone to interruptions from all the services it runs, so it is not
 a good candidate for low noise executions.
+
+# Binary cache
+
+We provide a binary cache in `hut`, with the aim of avoiding unnecessary
+recompilation of packages.
+
+The cache should contain common packages from bscpkgs, but we don't provide
+any guarantee that of what will be available in the cache, or for how long.
+We recommend following the latest version of the `jungle` flake to avoid cache
+misses.
+
+## Usage
+
+### From NixOS
+
+In NixOS, we can add the cache through the `nix.settings` option, which will
+enable it for all builds in the system.
+
+```nix
+{ ... }: {
+  nix.settings = {
+    extra-substituters = [ "https://jungle.bsc.es/cache" ];
+    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+}
+```
+
+### Interactively
+
+The cache can also be specified in a per-command basis through the flags
+`--substituters` and `--trusted-public-keys`:
+
+```sh
+nix build --substituters "https://jungle.bsc.es/cache" --trusted-public-keys "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" <...>
+```
+
+Note: you'll have to be a trusted user.
+
+### Nix configuration file (non-nixos)
+
+If using nix outside of NixOS, you'll have to update `/etc/nix/nix.conf`
+
+```
+# echo "extra-substituters = https://jungle.bsc.es/cache" >> /etc/nix/nix.conf
+# echo "extra-trusted-public-keys = jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" >> /etc/nix/nix.conf
+```
+
+### Hint in flakes
+
+By adding the configuration below to a `flake.nix`, when someone uses the flake,
+`nix` will interactively ask to trust and use the provided binary cache:
+
+```nix
+{
+  nixConfig = {
+    extra-substituters = [
+      "https://jungle.bsc.es/cache"
+    ];
+    extra-trusted-public-keys = [
+      "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0="
+    ];
+  };
+  outputs = { ... }: {
+    ...
+  };
+}
+```
+
+### Querying the cache
+
+Check if the cache is available:
+```sh
+$ curl https://jungle.bsc.es/cache/nix-cache-info
+StoreDir: /nix/store
+WantMassQuery: 1
+Priority: 30
+```
+
+Prevent nix from building locally:
+```bash
+nix build --max-jobs 0 <...>
+```
+
+Check if a package is in cache:
+```bash
+# Do a raw eval on the <package>.outPath (this should not build the package)
+$ nix eval --raw jungle#openmp.outPath
+/nix/store/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-openmp-2024.11
+# Take the hash (everything from / to - in the basename) and curl <hash>.narinfo
+# if it exists in the cache, it will return HTTP 200 and some information
+# if not, it will return 404
+$ curl https://jungle.bsc.es/cache/dwnn4dgm1m4184l4xbi0qfrprji9wjmi.narinfo
+StorePath: /nix/store/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-openmp-2024.11
+URL: nar/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-17imkdfqzmnb013d14dx234bx17bnvws8baf3ii1xra5qi2y1wiz.nar
+Compression: none
+NarHash: sha256:17imkdfqzmnb013d14dx234bx17bnvws8baf3ii1xra5qi2y1wiz
+NarSize: 1519328
+References: 4gk773fqcsv4fh2rfkhs9bgfih86fdq8-gcc-13.3.0-lib nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36
+Deriver: vcn0x8hikc4mvxdkvrdxp61bwa5r7lr6-openmp-2024.11.drv
+Sig: jungle.bsc.es:GDTOUEs1jl91wpLbb+gcKsAZjpKdARO9j5IQqb3micBeqzX2M/NDtKvgCS1YyiudOUdcjwa3j+hyzV2njokcCA==
+# In oneline:
+$ curl "https://jungle.bsc.es/cache/$(nix eval --raw jungle#<package>.outPath | cut -d '/' -f4 | cut -d '-' -f1).narinfo"
+```
+
+#### References
+
+- https://nix.dev/guides/recipes/add-binary-cache.html
+- https://nixos.wiki/wiki/Binary_Cache

web/content/paste/_index.md

@@ -0,0 +1,68 @@
+---
+title: "Paste"
+description: "Paste service"
+author: "Rodrigo Arias Mallo"
+date: 2024-09-20
+---
+
+The tent machine provides a paste service using the program `p` (as in paste).
+
+You can use it directly from the tent machine or remotely if you have [SSH
+access](/access) to tent using the following alias:
+
+```
+alias p="ssh tent p"
+```
+
+You can add it to bashrc or zshrc for persistent installation.
+
+## Usage
+
+The `p` command reads from the standard input, uploads the content to a file
+in the local filesystem and prints the URL to access it. It only accepts an
+optional argument, which is the extension of the file that will be stored on
+disk (without the dot). By default it uses the `txt` extension, so plain text
+can be read in the browser directly.
+
+```
+p [extension]
+```
+
+To remove files, go to `/var/lib/p/$USER` and remove them manually.
+
+## Examples
+
+Share a text file, in this case the source of p itself:
+
+```
+tent% p < m/tent/p.nix
+https://jungle.bsc.es/p/rarias/okbtG130.txt
+```
+
+Paste the last dmesg lines directly from a pipe:
+
+```
+tent% dmesg | tail -5 | p
+https://jungle.bsc.es/p/rarias/luX4STm9.txt
+```
+
+Upload a PNG picture from a file:
+
+```
+hop% p png < mark-api-cpu.png
+https://jungle.bsc.es/p/rarias/oSRAMVsE.png
+```
+
+Take an screenshot and upload it as a PNG file:
+
+```
+hop% scrot -s - | p png
+https://jungle.bsc.es/p/rarias/SOgK5EV0.png
+```
+
+Upload a directory by creating a tar.gz file on the fly:
+
+```
+hop% tar c ovni | gzip | p tar.gz
+https://jungle.bsc.es/p/rarias/tkwROcTR.tar.gz
+```

web/hugo.toml

@@ -3,26 +3,38 @@ languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
 
+[[menu.main]]
+identifier = "doc"
+name = "Docs"
+url = "/doc/"
+weight = 10
+
 [[menu.main]]
 identifier = "grafana"
 name = "Grafana"
 url = "/grafana/"
-weight = 10
+weight = 20
 
 [[menu.main]]
 identifier = "Git"
 name = "Git"
 url = "/git/"
-weight = 20
+weight = 30
 
 [[menu.main]]
 identifier = "Lists"
 name = "Lists"
 url = "/lists/"
-weight = 30
+weight = 40
+
+[[menu.main]]
+identifier = "Paste"
+name = "Paste"
+url = "/paste/"
+weight = 50
 
 [[menu.main]]
 identifier = "Posts"
 name = "Posts"
 url = "/posts/"
-weight = 40
+weight = 60

web/themes/PaperMod/layouts/partials/templates/opengraph.html

@@ -49,4 +49,4 @@
 {{ end }}{{ end }}
 
 {{- /* Facebook Page Admin ID for Domain Insights */}}
-{{- with site.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}
+{{- with site.Params.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}

web/themes/PaperMod/layouts/partials/templates/twitter_cards.html

@@ -28,6 +28,6 @@
 {{- end }}
 <meta name="twitter:title" content="{{ .Title }}"/>
 <meta name="twitter:description" content="{{ with .Description }}{{ . }}{{ else }}{{if .IsPage}}{{ .Summary }}{{ else }}{{ with site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end -}}"/>
-{{ with site.Social.twitter -}}
+{{ with site.Params.Social.twitter -}}
 <meta name="twitter:site" content="@{{ . }}"/>
 {{ end -}}