Create directories in /vault for tent users

We moved the tent machine to the server room in the BSC building and is
now directly connected to the raccoon via NAT.

Fixes: #106

flake.nix

@@ -18,6 +18,7 @@ in
   {
     nixosConfigurations = {
       hut     = mkConf "hut";
+      tent    = mkConf "tent";
       owl1    = mkConf "owl1";
       owl2    = mkConf "owl2";
       eudy    = mkConf "eudy";
@@ -25,6 +26,7 @@ in
       bay     = mkConf "bay";
       lake2   = mkConf "lake2";
       raccoon = mkConf "raccoon";
+      fox     = mkConf "fox";
     };
 
     packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

keys.nix

@@ -9,9 +9,11 @@ rec {
     koro  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
     bay   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
     lake2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
   };
 
   hostGroup = with hosts; rec {
+    untrusted  = [ fox ];
     compute    = [ owl1 owl2 ];
     playground = [ eudy koro ];
     storage    = [ bay lake2 ];

m/bay/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/monitoring.nix
   ];
 

m/common/base/nix.nix

@@ -23,6 +23,7 @@
       trusted-users = [ "@wheel" ];
       flake-registry = pkgs.writeText "global-registry.json"
         ''{"flakes":[],"version":2}'';
+      keep-outputs = true;
     };
 
     gc = {

m/common/base/ssh.nix

@@ -8,20 +8,10 @@ in
   # Enable the OpenSSH daemon.
   services.openssh.enable = true;
 
-  # Connect to intranet git hosts via proxy
-  programs.ssh.extraConfig = ''
-    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
-      User git
-      ProxyCommand nc -X connect -x hut:23080 %h %p
-
-    # Connect to BSC machines via hut proxy too
-    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-      ProxyCommand nc -X connect -x hut:23080 %h %p
-  '';
-
   programs.ssh.knownHosts = hostsKeys // {
     "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
     "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
+    "bscpm04.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT";
     "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
     "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
   };

m/common/base/users.nix

@@ -20,6 +20,7 @@
       rarias = {
         uid = 1880;
         isNormalUser = true;
+        linger = true;
         home = "/home/Computational/rarias";
         description = "Rodrigo Arias";
         group = "Computational";
@@ -39,7 +40,7 @@
         home = "/home/Computational/arocanon";
         description = "Aleix Roca";
         group = "Computational";
-        extraGroups = [ "wheel" ];
+        extraGroups = [ "wheel" "tracing" ];
         hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
@@ -55,7 +56,7 @@
         home = "/home/Computational/rpenacob";
         description = "Raúl Peñacoba";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" ];
+        hosts = [ "owl1" "owl2" "hut" "tent" ];
         hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
@@ -68,7 +69,7 @@
         home = "/home/Computational/anavarro";
         description = "Antoni Navarro";
         group = "Computational";
-        hosts = [ "hut" "raccoon" ];
+        hosts = [ "hut" "tent" "raccoon" "fox" ];
         hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
@@ -81,7 +82,7 @@
         home = "/home/Computational/abonerib";
         description = "Aleix Boné";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "raccoon" ];
+        hosts = [ "owl1" "owl2" "hut" "tent" "raccoon" "fox" ];
         hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
@@ -107,16 +108,43 @@
         home = "/home/Computational/dbautist";
         description = "Dylan Bautista Cases";
         group = "Computational";
-        hosts = [ "hut" ];
+        hosts = [ "hut" "tent" "raccoon" ];
         hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
         ];
       };
+
+      dalvare1 = {
+        uid = 2758;
+        isNormalUser = true;
+        home = "/home/Computational/dalvare1";
+        description = "David Álvarez";
+        group = "Computational";
+        hosts = [ "hut" "tent" "fox" ];
+        hashedPassword = "$6$mpyIsV3mdq.rK8$FvfZdRH5OcEkUt5PnIUijWyUYZvB1SgeqxpJ2p91TTe.3eQIDTcLEQ5rxeg.e5IEXAZHHQ/aMsR5kPEujEghx0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
+        ];
+      };
+
+      varcila = {
+        uid = 5650;
+        isNormalUser = true;
+        home = "/home/Computational/varcila";
+        description = "Vincent Arcila";
+        group = "Computational";
+        hosts = [ "hut" "tent" "fox" ];
+        hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
+        ];
+      };
     };
 
     groups = {
       Computational = { gid = 564; };
+      tracing = { };
     };
   };
 }

m/common/ssf.nix

@@ -0,0 +1,9 @@
+{
+  # Provides the base system for a xeon node in the SSF rack.
+  imports = [
+    ./xeon.nix
+    ./ssf/fs.nix
+    ./ssf/net.nix
+    ./ssf/ssh.nix
+  ];
+}

m/common/ssf/net.nix

@@ -11,7 +11,7 @@
 
     proxy = {
       default = "http://hut:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
+      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40,hut";
       # Don't set all_proxy as go complains and breaks the gitlab runner, see:
       # https://github.com/golang/go/issues/16715
       allProxy = null;
@@ -34,37 +34,37 @@
       # Node Entry for node: mds01 (ID=72)
       10.0.40.40              bay mds01 mds01-eth0
       10.0.42.40              bay-ib mds01-ib0
-      10.0.40.141             bay-ipmi mds01-ipmi0
+      10.0.40.141             bay-ipmi mds01-ipmi0 mds01-ipmi
       
       # Node Entry for node: oss01 (ID=73)
       10.0.40.41              oss01 oss01-eth0
       10.0.42.41              oss01-ib0
-      10.0.40.142             oss01-ipmi0
+      10.0.40.142             oss01-ipmi0 oss01-ipmi
       
       # Node Entry for node: oss02 (ID=74)
       10.0.40.42              lake2 oss02 oss02-eth0
       10.0.42.42              lake2-ib oss02-ib0
-      10.0.40.143             lake2-ipmi oss02-ipmi0
+      10.0.40.143             lake2-ipmi oss02-ipmi0 oss02-ipmi
       
       # Node Entry for node: xeon01 (ID=15)
       10.0.40.1               owl1 xeon01 xeon01-eth0
       10.0.42.1               owl1-ib xeon01-ib0
-      10.0.40.101             owl1-ipmi xeon01-ipmi0
+      10.0.40.101             owl1-ipmi xeon01-ipmi0 xeon01-ipmi
       
       # Node Entry for node: xeon02 (ID=16)
       10.0.40.2               owl2 xeon02 xeon02-eth0
       10.0.42.2               owl2-ib xeon02-ib0
-      10.0.40.102             owl2-ipmi xeon02-ipmi0
+      10.0.40.102             owl2-ipmi xeon02-ipmi0 xeon02-ipmi
       
       # Node Entry for node: xeon03 (ID=17)
       10.0.40.3               xeon03 xeon03-eth0
       10.0.42.3               xeon03-ib0
-      10.0.40.103             xeon03-ipmi0
+      10.0.40.103             xeon03-ipmi0 xeon03-ipmi
       
       # Node Entry for node: xeon04 (ID=18)
       10.0.40.4               xeon04 xeon04-eth0
       10.0.42.4               xeon04-ib0
-      10.0.40.104             xeon04-ipmi0
+      10.0.40.104             xeon04-ipmi0 xeon04-ipmi
       
       # Node Entry for node: xeon05 (ID=19)
       10.0.40.5               koro xeon05 xeon05-eth0
@@ -74,17 +74,17 @@
       # Node Entry for node: xeon06 (ID=20)
       10.0.40.6               xeon06 xeon06-eth0
       10.0.42.6               xeon06-ib0
-      10.0.40.106             xeon06-ipmi0
+      10.0.40.106             xeon06-ipmi0 xeon06-ipmi
       
       # Node Entry for node: xeon07 (ID=21)
       10.0.40.7               hut xeon07 xeon07-eth0
       10.0.42.7               hut-ib xeon07-ib0
-      10.0.40.107             hut-ipmi xeon07-ipmi0
+      10.0.40.107             hut-ipmi xeon07-ipmi0 xeon07-ipmi
       
       # Node Entry for node: xeon08 (ID=22)
       10.0.40.8               eudy xeon08 xeon08-eth0
       10.0.42.8               eudy-ib xeon08-ib0
-      10.0.40.108             eudy-ipmi xeon08-ipmi0
+      10.0.40.108             eudy-ipmi xeon08-ipmi0 xeon08-ipmi
     '';
   };
 }

m/common/ssf/ssh.nix

@@ -0,0 +1,8 @@
+{
+  # Connect to intranet git hosts via proxy
+  programs.ssh.extraConfig = ''
+    # Connect to BSC machines via hut proxy too
+    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+      ProxyCommand nc -X connect -x hut:23080 %h %p
+  '';
+}

m/common/xeon.nix

@@ -1,9 +1,7 @@
 {
-  # Provides the base system for a xeon node.
+  # Provides the base system for a xeon node, not necessarily in the SSF rack.
   imports = [
     ./base.nix
-    ./xeon/fs.nix
     ./xeon/console.nix
-    ./xeon/net.nix
   ];
 }

m/eudy/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix

m/fox/configuration.nix

@@ -0,0 +1,76 @@
+{ lib, config, pkgs, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+    ../common/xeon/console.nix
+    ../module/emulation.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x500a07514b0c1103";
+
+  # No swap, there is plenty of RAM
+  swapDevices = lib.mkForce [];
+
+  boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ];
+  boot.kernelModules = [ "kvm-amd" ];
+
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+  hardware.cpu.intel.updateMicrocode = lib.mkForce false;
+
+  networking = {
+    defaultGateway = "147.83.30.130";
+    nameservers = [ "8.8.8.8" ];
+    hostName = "fox";
+    interfaces.enp1s0f0np0.ipv4.addresses = [
+      {
+        # UPC network
+        # Public IP configuration:
+        # - Hostname: fox.ac.upc.edu
+        # - IP: 147.83.30.141
+        # - Gateway: 147.83.30.130
+        # - NetMask: 255.255.255.192
+        # Private IP configuration for BMC:
+        # - Hostname: fox-ipmi.ac.upc.edu
+        # - IP: 147.83.35.27
+        # - Gateway: 147.83.35.2
+        # - NetMask: 255.255.255.0
+        address = "147.83.30.141";
+        prefixLength = 26; # 255.255.255.192
+      }
+    ];
+    extraHosts = ''
+      147.83.30.141   fox.ac.upc.edu
+      147.83.35.27    fox-ipmi.ac.upc.edu
+    '';
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.graphics.enable = true;
+  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+
+  # Mount NVME disks
+  fileSystems."/nvme0" = { device = "/dev/disk/by-label/nvme0"; fsType = "ext4"; };
+  fileSystems."/nvme1" = { device = "/dev/disk/by-label/nvme1"; fsType = "ext4"; };
+
+  # Make a /nvme{0,1}/$USER directory for each user.
+  systemd.services.create-nvme-dirs = let
+    # Take only normal users in fox
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0755 /nvme{0,1}/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-nvme-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+}

m/hut/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
 
     ../module/ceph.nix
     ../module/debuginfod.nix
@@ -56,6 +56,11 @@
         iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
       '';
+      # Flush all rules and chains on stop so it won't break on start
+      extraStopCommands = ''
+        iptables -F
+        iptables -X
+      '';
     };
   };
 

m/hut/gitlab-runner.nix

@@ -1,8 +1,9 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age;
-  age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age;
+  age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age;
+  age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age;
 
   services.gitlab-runner = {
     enable = true;
@@ -21,21 +22,90 @@
           "--docker-network-mode host"
         ];
         environmentVariables = {
-          https_proxy = "http://localhost:23080";
-          http_proxy = "http://localhost:23080";
+          https_proxy = "http://hut:23080";
+          http_proxy = "http://hut:23080";
         };
       };
     in {
       # For pm.bsc.es/gitlab
       gitlab-pm-shell = common-shell // {
-        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path;
       };
       gitlab-pm-docker = common-docker // {
-        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path;
+      };
+
+      gitlab-bsc-docker = {
+        # gitlab.bsc.es still uses the old token mechanism
+        registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
+        tagList = [ "docker" "hut" ];
+        environmentVariables = {
+          # We cannot access the hut local interface from docker, so we connect
+          # to hut directly via the ethernet one.
+          https_proxy = "http://hut:23080";
+          http_proxy = "http://hut:23080";
+        };
+        executor = "docker";
+        dockerImage = "alpine";
+        dockerVolumes = [
+          "/nix/store:/nix/store:ro"
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+        ];
+        dockerExtraHosts = [
+          # Required to pass the proxy via hut
+          "hut:10.0.40.7"
+        ];
+        dockerDisableCache = true;
+        registrationFlags = [
+          # Increase build log length to 64 MiB
+          "--output-limit 65536"
+        ];
+        preBuildScript = pkgs.writeScript "setup-container" ''
+          mkdir -p -m 0755 /nix/var/log/nix/drvs
+          mkdir -p -m 0755 /nix/var/nix/gcroots
+          mkdir -p -m 0755 /nix/var/nix/profiles
+          mkdir -p -m 0755 /nix/var/nix/temproots
+          mkdir -p -m 0755 /nix/var/nix/userpool
+          mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+          mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+          mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+          mkdir -p -m 0700 "$HOME/.nix-defexpr"
+          mkdir -p -m 0700 "$HOME/.ssh"
+          cat > "$HOME/.ssh/config" << EOF
+          Host bscpm04.bsc.es gitlab-internal.bsc.es
+            User git
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
+            ProxyCommand nc -X connect -x hut:23080 %h %p
+          EOF
+          cat >> "$HOME/.ssh/known_hosts" << EOF
+          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
+          EOF
+          . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
+          # Required to load SSL certificate paths
+          . ${pkgs.cacert}/nix-support/setup-hook
+        '';
+        environmentVariables = {
+          ENV = "/etc/profile";
+          USER = "root";
+          NIX_REMOTE = "daemon";
+          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+        };
       };
     };
   };
 
+  # DOCKER* chains are useless, override at FORWARD and nixos-fw
+  networking.firewall.extraCommands = ''
+    # Don't forward any traffic from docker
+    iptables -I FORWARD 1 -p all -i docker0 -j nixos-fw-log-refuse
+
+    # Allow incoming traffic from docker to 23080
+    iptables -A nixos-fw -p tcp -i docker0 -d hut --dport 23080 -j ACCEPT
+  '';
+
   #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";

m/hut/ipmi.yml

@@ -1,13 +0,0 @@
-modules:
-        default:
-                collectors:
-                - bmc
-                - ipmi
-                - chassis
-
-        lan:
-                collectors:
-                - ipmi
-                - chassis
-                user: ""
-                pass: ""

m/hut/monitoring.nix

@@ -3,7 +3,10 @@
 {
   imports = [
     ../module/slurm-exporter.nix
+    ../module/meteocat-exporter.nix
+    ../module/upc-qaire-exporter.nix
     ./gpfs-probe.nix
+    ./nix-daemon-exporter.nix
   ];
 
   age.secrets.grafanaJungleRobotPassword = {
@@ -12,6 +15,8 @@
     mode = "400";
   };
 
+  age.secrets.ipmiYml.file = ../../secrets/ipmi.yml.age;
+
   services.grafana = {
     enable = true;
     settings = {
@@ -44,7 +49,7 @@
   services.prometheus = {
     enable = true;
     port = 9001;
-    retentionTime = "1y";
+    retentionTime = "5y";
     listenAddress = "127.0.0.1";
   };
 
@@ -73,8 +78,8 @@
         enable = true;
         group = "root";
         user = "root";
-        configFile = ./ipmi.yml;
-        #extraFlags = [ "--log.level=debug" ];
+        configFile = config.age.secrets.ipmiYml.path;
+        # extraFlags = [ "--log.level=debug" ];
         listenAddress = "127.0.0.1";
       };
       node = {
@@ -106,6 +111,9 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9999" # Nix-daemon custom exporter
+            "127.0.0.1:9929" # Meteocat custom exporter
+            "127.0.0.1:9928" # UPC Qaire custom exporter
             "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];
@@ -161,6 +169,9 @@
             "8.8.8.8"
             "ssfhead"
             "anella-bsc.cesca.cat"
+            "upc-anella.cesca.cat"
+            "fox.ac.upc.edu"
+            "arenys5.ac.upc.edu"
           ];
         }];
         relabel_configs = [
@@ -206,7 +217,7 @@
             # Sets the "instance" label with the remote host we are querying
             source_labels = [ "__param_target" ];
             separator = ";";
-            regex = "(.*)";
+            regex = "(.*)-ipmi"; # Remove "-ipm̀i" at the end
             target_label = "instance";
             replacement = "\${1}";
             action = "replace";
@@ -248,6 +259,14 @@
           module = [ "raccoon" ];
         };
       }
+      {
+        job_name = "raccoon";
+        static_configs = [
+          {
+            targets = [ "127.0.0.1:19002" ]; # Node exporter
+          }
+        ];
+      }
     ];
   };
 }

m/hut/nginx.nix

@@ -12,16 +12,19 @@ let
     installPhase = ''
       cp -r public $out
     '';
+    # Don't mess doc/
+    dontFixup = true;
   };
 in
 {
+  networking.firewall.allowedTCPPorts = [ 80 ];
   services.nginx = {
     enable = true;
     virtualHosts."jungle.bsc.es" = {
       root = "${website}";
       listen = [
         {
-          addr = "127.0.0.1";
+          addr = "0.0.0.0";
           port = 80;
         }
       ];

m/hut/nix-daemon-builds.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# Locate nix daemon pid
+nd=$(pgrep -o nix-daemon)
+
+# Locate children of nix-daemon
+pids1=$(tr ' ' '\n' < "/proc/$nd/task/$nd/children")
+
+# For each children, locate 2nd level children
+pids2=$(echo "$pids1" | xargs -I @ /bin/sh -c 'cat /proc/@/task/*/children' | tr ' ' '\n')
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+# HELP nix_daemon_build Nix daemon derivation build state.
+# TYPE nix_daemon_build gauge
+EOF
+
+for pid in $pids2; do
+  name=$(cat /proc/$pid/environ 2>/dev/null | tr '\0' '\n' | rg "^name=(.+)" - --replace '$1' | tr -dc ' [:alnum:]_\-\.')
+  user=$(ps -o uname= -p "$pid")
+  if [ -n "$name" -a -n "$user" ]; then
+    printf 'nix_daemon_build{user="%s",name="%s"} 1\n' "$user" "$name"
+  fi
+done

m/hut/nix-daemon-exporter.nix

@@ -0,0 +1,23 @@
+{ pkgs, config, lib, ... }:
+let
+  script = pkgs.runCommand "nix-daemon-exporter.sh" { }
+    ''
+      cp ${./nix-daemon-builds.sh} $out;
+      chmod +x $out
+    ''
+  ;
+in
+{
+  systemd.services.nix-daemon-exporter = {
+    description = "Daemon to export nix-daemon metrics";
+    path = [ pkgs.procps pkgs.ripgrep ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9999,fork EXEC:${script}";
+      # Needed root to read the environment, potentially unsafe
+      User = "root";
+      Group = "root";
+    };
+  };
+}

m/hut/targets.yml

@@ -1,15 +1,15 @@
 - targets:
-  - 10.0.40.101
-  - 10.0.40.102
-  - 10.0.40.103
-  - 10.0.40.104
-  - 10.0.40.105
-  - 10.0.40.106
-  - 10.0.40.107
-  - 10.0.40.108
+  - owl1-ipmi
+  - owl2-ipmi
+  - xeon03-ipmi
+  - xeon04-ipmi
+  - koro-ipmi
+  - xeon06-ipmi
+  - hut-ipmi
+  - eudy-ipmi
   # Storage
-  - 10.0.40.141
-  - 10.0.40.142
-  - 10.0.40.143
+  - bay-ipmi
+  - oss01-ipmi
+  - lake2-ipmi
   labels:
     job: ipmi-lan

m/koro/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix

m/lake2/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/monitoring.nix
   ];
 

m/map.nix

@@ -0,0 +1,70 @@
+{
+  # In physical order from top to bottom (see note below)
+  ssf = {
+    # Switches for Ethernet and OmniPath
+    switch-C6-S1A-05 = { pos=42; size=1; model="Dell S3048-ON"; };
+    switch-opa = { pos=41; size=1; };
+
+    # SSF login
+    ssfhead = { pos=39; size=2; label="SSFHEAD"; board="R2208WTTYSR"; contact="operations@bsc.es"; };
+
+    # Storage
+    bay   = { pos=38; size=1; label="MDS01"; board="S2600WT2R"; sn="BQWL64850303"; contact="rodrigo.arias@bsc.es"; };
+    lake1 = { pos=37; size=1; label="OSS01"; board="S2600WT2R"; sn="BQWL64850234"; contact="rodrigo.arias@bsc.es"; };
+    lake2 = { pos=36; size=1; label="OSS02"; board="S2600WT2R"; sn="BQWL64850266"; contact="rodrigo.arias@bsc.es"; };
+
+    # Compute xeon
+    owl1   = { pos=35; size=1; label="SSF-XEON01"; board="S2600WTTR"; sn="BQWL64954172"; contact="rodrigo.arias@bsc.es"; };
+    owl2   = { pos=34; size=1; label="SSF-XEON02"; board="S2600WTTR"; sn="BQWL64756560"; contact="rodrigo.arias@bsc.es"; };
+    xeon03 = { pos=33; size=1; label="SSF-XEON03"; board="S2600WTTR"; sn="BQWL64750826"; contact="rodrigo.arias@bsc.es"; };
+    # Slot 34 empty
+    koro   = { pos=31; size=1; label="SSF-XEON05"; board="S2600WTTR"; sn="BQWL64954293"; contact="rodrigo.arias@bsc.es"; };
+    xeon06 = { pos=30; size=1; label="SSF-XEON06"; board="S2600WTTR"; sn="BQWL64750846"; contact="antoni.navarro@bsc.es"; };
+    hut    = { pos=29; size=1; label="SSF-XEON07"; board="S2600WTTR"; sn="BQWL64751184"; contact="rodrigo.arias@bsc.es"; };
+    eudy   = { pos=28; size=1; label="SSF-XEON08"; board="S2600WTTR"; sn="BQWL64756586"; contact="aleix.rocanonell@bsc.es"; };
+
+    # 16 KNL nodes, 4 per chassis
+    knl01_04 = { pos=26; size=2; label="KNL01..KNL04"; board="HNS7200APX"; };
+    knl05_08 = { pos=24; size=2; label="KNL05..KNL18"; board="HNS7200APX"; };
+    knl09_12 = { pos=22; size=2; label="KNL09..KNL12"; board="HNS7200APX"; };
+    knl13_16 = { pos=20; size=2; label="KNL13..KNL16"; board="HNS7200APX"; };
+
+    # Slot 19 empty
+
+    # EPI (hw team, guessed order)
+    epi01 = { pos=18; size=1; contact="joan.cabre@bsc.es"; };
+    epi02 = { pos=17; size=1; contact="joan.cabre@bsc.es"; };
+    epi03 = { pos=16; size=1; contact="joan.cabre@bsc.es"; };
+    anon  = { pos=14; size=2; }; # Unlabeled machine. Operative
+
+    # These are old and decommissioned (off)
+    power8    = { pos=12; size=2; label="BSCPOWER8N3";   decommissioned=true; };
+    powern1   = { pos=8;  size=4; label="BSCPOWERN1";    decommissioned=true; };
+    gustafson = { pos=7;  size=1; label="gustafson";     decommissioned=true; };
+    odap01    = { pos=3;  size=4; label="ODAP01";        decommissioned=true; };
+    amhdal    = { pos=2;  size=1; label="AMHDAL";        decommissioned=true; }; # sic
+    moore     = { pos=1;  size=1; label="moore (earth)"; decommissioned=true; };
+  };
+
+  bsc2218 = {
+    raccoon = { board="W2600CR"; sn="QSIP22500829"; contact="rodrigo.arias@bsc.es"; };
+    tent    = { label="SSF-XEON04"; board="S2600WTTR"; sn="BQWL64751229"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  upc = {
+    fox = { board="H13DSG-O-CPU"; sn="UM24CS600392"; prod="AS-4125GS-TNRT"; prod_sn="E508839X5103339"; contact="rodrigo.arias@bsc.es"; };
+  };
+
+  # NOTE: Position is specified in "U" units (44.45 mm) and starts at 1 from the
+  # bottom. Example:
+  #
+  #  |   ...  | - [pos+size] <--- Label in chassis
+  #  +--------+
+  #  |  node  | - [pos+1]
+  #  |   2U   | - [pos]
+  #  +------- +
+  #  |   ...  | - [pos-1]
+  #
+  # NOTE: The board and sn refers to the FRU information (Board Product and
+  # Board Serial) via `ipmitool fru print 0`.
+}

m/module/hut-substituter.nix

@@ -4,7 +4,7 @@
     # Don't add hut as a cache to itself
     assert config.networking.hostName != "hut";
     {
-      substituters = [ "https://jungle.bsc.es/cache" ];
+      substituters = [ "http://hut/cache" ];
       trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
     };
 }

m/module/meteocat-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-meteocat-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.meteocat-exporter}/bin/meteocat-exporter";
+    };
+  };
+}

m/module/slurm-client.nix

@@ -47,8 +47,7 @@ in {
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-      "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -91,9 +90,29 @@ in {
       # Ignore memory constraints and only use unused cores to share a node with
       # other jobs.
       SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
     '';
   };
 
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
   age.secrets.mungeKey = {
     file = ../../secrets/munge-key.age;
     owner = "munge";

m/module/ssh-hut-extern.nix

@@ -0,0 +1,9 @@
+{
+  programs.ssh.extraConfig = ''
+    Host ssfhead
+      HostName ssflogin.bsc.es
+    Host hut
+      ProxyJump ssfhead
+      HostName xeon07
+  '';
+}

m/module/upc-qaire-exporter.nix

@@ -0,0 +1,17 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+{
+  systemd.services."prometheus-upc-qaire-exporter" = {
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network.target" ];
+    serviceConfig = {
+      Restart = mkDefault "always";
+      PrivateTmp = mkDefault true;
+      WorkingDirectory = mkDefault "/tmp";
+      DynamicUser = mkDefault true;
+      ExecStart = "${pkgs.upc-qaire-exporter}/bin/upc-qaire-exporter";
+    };
+  };
+}

m/owl1/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/ceph.nix
     ../module/emulation.nix
     ../module/slurm-client.nix

m/owl2/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/xeon.nix
+    ../common/ssf.nix
     ../module/ceph.nix
     ../module/emulation.nix
     ../module/slurm-client.nix

m/raccoon/configuration.nix

@@ -3,7 +3,10 @@
 {
   imports = [
     ../common/base.nix
-    ../module/hut-substituter.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+    ../eudy/kernel/perf.nix
   ];
 
   # Don't install Grub on the disk yet
@@ -24,8 +27,28 @@
       address = "84.88.51.152";
       prefixLength = 25;
     } ];
+    interfaces.enp5s0f1.ipv4.addresses = [ {
+      address = "10.0.44.1";
+      prefixLength = 24;
+    } ];
+    nat = {
+      enable = true;
+      internalInterfaces = [ "enp5s0f1" ];
+      externalInterface = "eno0";
+    };
+    hosts = {
+      "10.0.44.4" = [ "tent" ];
+    };
   };
 
+  nix.settings = {
+    substituters = [ "https://jungle.bsc.es/cache" ];
+    trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  # Enable performance governor
+  powerManagement.cpuFreqGovernor = "performance";
+
   # Configure Nvidia driver to use with CUDA
   hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
   hardware.graphics.enable = true;
@@ -33,6 +56,18 @@
   nixpkgs.config.nvidia.acceptLicense = true;
   services.xserver.videoDrivers = [ "nvidia" ];
 
+  # Disable garbage collection for now
+  nix.gc.automatic = lib.mkForce false;
+
+  services.openssh.settings.X11Forwarding = true;
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
   users.motd = ''
     ⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
     ⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀

m/tent/configuration.nix

@@ -0,0 +1,70 @@
+{ config, pkgs, lib, ... }:
+
+{
+  imports = [
+    ../common/xeon.nix
+    ../module/emulation.nix
+    ../module/debuginfod.nix
+    ../module/ssh-hut-extern.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d537675";
+
+  networking = {
+    hostName = "tent";
+    interfaces.eno1.ipv4.addresses = [
+      {
+        address = "10.0.44.4";
+        prefixLength = 24;
+      }
+    ];
+
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    defaultGateway = "10.0.44.1";
+  };
+
+  nix.settings = {
+    substituters = [ "https://jungle.bsc.es/cache" ];
+    trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    enabledCollectors = [ "systemd" ];
+    port = 9002;
+    listenAddress = "127.0.0.1";
+  };
+
+  boot.swraid = {
+    enable = true;
+    mdadmConf = ''
+      DEVICE partitions
+      ARRAY /dev/md0 metadata=1.2 UUID=496db1e2:056a92aa:a544543f:40db379d
+      MAILADDR root
+    '';
+  };
+
+  fileSystems."/vault" = {
+    device = "/dev/disk/by-label/vault";
+    fsType = "ext4";
+  };
+
+  # Make a /vault/$USER directory for each user.
+  systemd.services.create-vault-dirs = let
+    # Take only normal users in tent
+    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
+    commands = lib.concatLists (lib.mapAttrsToList
+      (_: user: [
+        "install -d -o ${user.name} -g ${user.group} -m 0711 /vault/${user.name}"
+      ]) users);
+    script = pkgs.writeShellScript "create-vault-dirs.sh" (lib.concatLines commands);
+  in {
+    enable = true;
+    wants = [ "local-fs.target" ];
+    after = [ "local-fs.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig.ExecStart = script;
+  };
+}

pkgs/meteocat-exporter/default.nix

@@ -0,0 +1,25 @@
+{ python3Packages, lib }:
+
+python3Packages.buildPythonApplication rec {
+  pname = "meteocat-exporter";
+  version = "1.0";
+
+  src = ./.;
+
+  doCheck = false;
+
+  build-system = with python3Packages; [
+    setuptools
+  ];
+
+  dependencies = with python3Packages; [
+    beautifulsoup4
+    lxml
+    prometheus-client
+  ];
+
+  meta = with lib; {
+    description = "MeteoCat Prometheus Exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/meteocat-exporter/meteocat-exporter

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import time
+from prometheus_client import start_http_server, Gauge
+from bs4 import BeautifulSoup
+from urllib import request
+
+# Configuration -------------------------------------------
+meteo_station = "X8" # Barcelona - Zona Universitària
+listening_port = 9929
+update_period = 60 * 5 # Each 5 min
+# ---------------------------------------------------------
+
+metric_tmin = Gauge('meteocat_temp_min', 'Min temperature')
+metric_tmax = Gauge('meteocat_temp_max', 'Max temperature')
+metric_tavg = Gauge('meteocat_temp_avg', 'Average temperature')
+metric_srad = Gauge('meteocat_solar_radiation', 'Solar radiation')
+
+def update(st):
+    url = 'https://www.meteo.cat/observacions/xema/dades?codi=' + st
+    response = request.urlopen(url)
+    data = response.read()
+    soup = BeautifulSoup(data, 'lxml')
+    table = soup.find("table", {"class" : "tblperiode"})
+    rows = table.find_all('tr')
+    row = rows[-1] # Take the last row
+    row_data = []
+    header = row.find('th')
+    header_text = header.text.strip()
+    row_data.append(header_text)
+    for col in row.find_all('td'):
+        row_data.append(col.text)
+    try:
+        # Sometimes it will return '(s/d)' and fail to parse
+        metric_tavg.set(float(row_data[1]))
+        metric_tmax.set(float(row_data[2]))
+        metric_tmin.set(float(row_data[3]))
+        metric_srad.set(float(row_data[10]))
+        #print("ok: temp_avg={}".format(float(row_data[1])))
+    except:
+        print("cannot parse row: {}".format(row))
+        metric_tavg.set(float("nan"))
+        metric_tmax.set(float("nan"))
+        metric_tmin.set(float("nan"))
+        metric_srad.set(float("nan"))
+
+if __name__ == '__main__':
+    start_http_server(port=listening_port, addr="localhost")
+    while True:
+        try:
+            update(meteo_station)
+        except:
+            print("update failed")
+        time.sleep(update_period)

pkgs/meteocat-exporter/setup.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(name='meteocat-exporter',
+      version='1.0',
+      # Modules to import from other scripts:
+      packages=find_packages(),
+      # Executables
+      scripts=["meteocat-exporter"],
+     )

pkgs/overlay.nix

@@ -39,7 +39,21 @@ final: prev:
       # See https://bugs.schedmd.com/show_bug.cgi?id=19324
       ./slurm-rank-expansion.patch
     ];
+    # Install also the pam_slurm_adopt library to restrict users from accessing
+    # nodes with no job allocated.
+    postBuild = (old.postBuild or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security"
+      popd
+    '';
+    postInstall = (old.postInstall or "") + ''
+      pushd contribs/pam_slurm_adopt
+        make "PAM_DIR=$out/lib/security" install
+      popd
+    '';
   });
 
   prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
+  meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { };
+  upc-qaire-exporter = prev.callPackage ./upc-qaire-exporter/default.nix { };
 }

pkgs/upc-qaire-exporter/default.nix

@@ -0,0 +1,24 @@
+{ python3Packages, lib }:
+
+python3Packages.buildPythonApplication rec {
+  pname = "upc-qaire-exporter";
+  version = "1.0";
+
+  src = ./.;
+
+  doCheck = false;
+
+  build-system = with python3Packages; [
+    setuptools
+  ];
+
+  dependencies = with python3Packages; [
+    prometheus-client
+    requests
+  ];
+
+  meta = with lib; {
+    description = "UPC Qaire Prometheus Exporter";
+    platforms = platforms.linux;
+  };
+}

pkgs/upc-qaire-exporter/setup.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(name='upc-qaire-exporter',
+      version='1.0',
+      # Modules to import from other scripts:
+      packages=find_packages(),
+      # Executables
+      scripts=["upc-qaire-exporter"],
+     )

pkgs/upc-qaire-exporter/upc-qaire-exporter

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import time
+from prometheus_client import start_http_server, Gauge
+import requests, json
+from datetime import datetime, timedelta
+
+# Configuration -------------------------------------------
+listening_port = 9928
+update_period = 60 * 5 # Each 5 min
+# ---------------------------------------------------------
+
+metric_temp = Gauge('upc_c6_s302_temp', 'UPC C6 S302 temperature sensor')
+
+def genparams():
+    d = {}
+    d['topic'] = 'TEMPERATURE'
+    d['shift_dates_to'] = ''
+    d['datapoints'] = 301
+    d['devicesAndColors'] = '1148418@@@#40ACB6'
+
+    now = datetime.now()
+
+    d['fromDate'] = now.strftime('%d/%m/%Y')
+    d['toDate'] = now.strftime('%d/%m/%Y')
+    d['serviceFrequency'] = 'NONE'
+
+    # WTF!
+    for i in range(7):
+        for j in range(48):
+            key = 'week.days[{}].hours[{}].value'.format(i, j)
+            d[key] = 'OPEN'
+
+    return d
+
+def measure():
+    # First we need to load session
+    s = requests.Session()
+    r = s.get("https://upc.edu/sirena")
+    if r.status_code != 200:
+        print("bad HTTP status code on new session: {}".format(r.status_code))
+        return
+
+    if s.cookies.get("JSESSIONID") is None:
+        print("cannot get JSESSIONID")
+        return
+
+    # Now we can pull the data
+    url = "https://upcsirena.app.dexma.com/l_12535/analysis/by_datapoints/data.json"
+    r = s.post(url, data=genparams())
+
+    if r.status_code != 200:
+        print("bad HTTP status code on data: {}".format(r.status_code))
+        return
+
+    #print(r.text)
+    j = json.loads(r.content)
+
+    # Just take the last one
+    last = j['data']['chartElementList'][-1]
+    temp = last['values']['1148418-Temperatura']
+
+    return temp
+
+if __name__ == '__main__':
+    start_http_server(port=listening_port, addr="localhost")
+    while True:
+        try:
+            metric_temp.set(measure())
+        except:
+            print("measure failed")
+            metric_temp.set(float("nan"))
+
+        time.sleep(update_period)

secrets/gitlab-bsc-docker-token.age

@@ -0,0 +1,10 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg XPOFoZqY+AnKC77jrgNqAm1ADphurfuhO4NRrfiuUDc
+iCfMMpGHyaYHGy6ci8sqjUtcPeteLlyvLGEF79VPOEc
+-> ssh-ed25519 CAWG4Q 6OsGrnM+/c5lTN81Rvp166K+ygmSIFeSYzXxYg25KGE
+Av1zTw2zK4Gufzti9kQaye7C362GCiDRRHzCqBLR33g
+-> ssh-ed25519 MSF3dg 8CHqJ7mEDvjvqbmF+eE6Em1Wi6eHAzEUpiExC1gm7S0
+bdwzYHw3RAbdHq+RsiFUP++sQ586VUlSnAzAOhiQUjI
+--- gA5XSUfjUBol938sC5DbUf8PvQUIr2pNkS2nL95OF9c
+<EFBFBD><EFBFBD><EFBFBD>Ea1G7<47><37>ݩ[R<><52>\{~$Go<47>cQ<63>wKP&<26><><EFBFBD>w<EFBFBD><77><EFBFBD>6]
+<EFBFBD><EFBFBD>ѣ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>^z<7A>̄ 1k<31><6B><03><><EFBFBD><EFBFBD><EFBFBD>Y<EFBFBD><59>2<15>p<>2<EFBFBD><32>K<EFBFBD><4B><EFBFBD><EFBFBD>nok<>/X<><58>pt''<27><>$0co=<3D><EFBFBD>

secrets/gitlab-runner-docker-token.age

@@ -1,9 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg 0sEIUEJBJQ0k0rBfHaOEbq1pNBqsPin4Xq85v0ds9jY
-4wzjLapoOcq53nT2K3hSGED4jTDXci25GLHkl/fL4EI
--> ssh-ed25519 CAWG4Q f68ZbJGwXuCZVnqhwbh+8dh0X/MCdjEd+sVtPyBu/hU
-u2TQreyWQvP6PGuwuUNKA/AL68560flqSlaItN3k41I
--> ssh-ed25519 MSF3dg HdrtRW2j7mfkLH0/4aJK5R0cWdjf56HYtEZgzHi9EAs
-A6MF6tXmSUq2RF2bpmav0GFTRERwluSZGh2snP/KqkA
---- drsezqi7J/g8gm6N10SkfeAWnYct99WUraB5djLJqpo
-g<EFBFBD>
+-> ssh-ed25519 HY2yRg pXNTB/ailRwSEJG1pXvrzzpz5HqkDZdWVWnOH7JGeQ4
+NzA+2fxfkNRy/u+Zq96A02K1Vxy0ETYZjMkDVTKyCY8
+-> ssh-ed25519 CAWG4Q 7CLJWn+EAxoWDduXaOSrHaBFHQ4GIpYP/62FFTj3ZTI
+vSYV1pQg2qI2ngCzM0nCZAnqdz1tbT4hM5m+/TyGU2c
+-> ssh-ed25519 MSF3dg Akmp4NcZcDuaYHta/Vej6zulNSrAOCd5lmSV+OiBGC4
+qTxqVzTyywur+GjtUQdbaIUdH1fqCqPe6qPf8iHRa4w
+--- uCKNqD1TmZZThOzlpsecBKx/k+noIWhCVMr/pzNwBr8
+r'<27>Ƌs4<15>˺<EFBFBD>Aĥ<41>P<05>L<EFBFBD><4C>7`	<09><02><>)H<><48>-<2D>0<EFBFBD>AH<41>5<16><><1F><>L<EFBFBD>Qe<51><65>H2b<32><1D>B޲<42>CJG<4A>"-S<><1C>\<5C><><0C>
H<n<>V

secrets/gitlab-runner-shell-token.age

@@ -1,9 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg VY8s9s1zuHOv2axmIacwKg2ozsJnskHTQtslRZ3YI1M
-fKkJuydLOzF/ciPYSYu4ziSCozdl6sowvDMYZmxqmHY
--> ssh-ed25519 CAWG4Q 2ARFd/7RWQ/QOk47FnJFChaVBgoV4LE6EA+JHezkXgg
-MV4g4Llv8Qcd/wUgJyoNG5AXb6o3aFTrOYGC+lXlSzw
--> ssh-ed25519 MSF3dg SKoxWe8Mi8EkBjkESxStOCI5V4C0KYEXIOx7OdENgTA
-p/owKwQ4e4pcGV+hqej2AfPU5QaM2i8VfxhlkjCM4Z4
---- 0VWKU5CQiGbiOtQ2tsZZg88oZm1qcUDEnU5zDTtV+KU
-<EFBFBD><EFBFBD>u<>cl<1C><>`<19><06>M<1C><>'Vk6Y<36>!<21>=<3D>L<1E><>y<EFBFBD>-<2D><><EFBFBD>O<EFBFBD>Az<02>ƘVEK<45><4B><EFBFBD>R<EFBFBD>_<08>qL|1V<>[)<29>q<EFBFBD><71><EFBFBD>ƓL<C693><4C>Dy<><79>0<EFBFBD>_<><5F><EFBFBD>q)-T,ƪ<16>_9<5F> <20>?<3F><><EFBFBD><EFBFBD>ib<69>1
+-> ssh-ed25519 HY2yRg s6iI9f25xulF4KXt+XY07kXXPKxXo7f2Ql/OTHN55Hk
+WO4Fd2H9c+HL3+XhUF3BmEZVILlcchGxSrSmL2OEdGw
+-> ssh-ed25519 CAWG4Q TBkdpx8k8K1NvW3wcvaF7omKFwEJ2DxWJp3tIOTjwCA
+LcYgWRix23AQnw0OQ7f8+8S3J84CHUElX1vKZSETiLE
+-> ssh-ed25519 MSF3dg WzrF8kjTP7BXXDjmUp7kPCKguthAW12RPo6Vy2RMmh4
+8C3mT9ktudCTANDxhyNszUkbeDG6X4wOJdx825++dYM
+--- /w3YQ2UeTi67H1JR0GsdPz2KoLN2Y7BIZfFY+//AWjY
+<EFBFBD>ӣ-`P<>@<40><>ބ<><DE84><EFBFBD>)9<>9l<><6C><0C><>Zf<5A><66>V?I>΍<>w鉐<77>z40 <20>2{i@<40>Z<EFBFBD><5A>x<EFBFBD><78>AHn<48>%<25><><0C>ʤ<>/W<><57>Ĕ<0F>l<EFBFBD><6C><EFBFBD><EFBFBD>}<7D>&<08>얶<EFBFBD>(<28><><EFBFBD>K<EFBFBD><4B><EFBFBD>S<EFBFBD><53>o<EFBFBD>z<EFBFBD>=<1F>d

secrets/secrets.nix

@@ -9,8 +9,10 @@ in
   "gitea-runner-token.age".publicKeys = hut;
   "gitlab-runner-docker-token.age".publicKeys = hut;
   "gitlab-runner-shell-token.age".publicKeys = hut;
+  "gitlab-bsc-docker-token.age".publicKeys = hut;
   "nix-serve.age".publicKeys = hut;
   "jungle-robot-password.age".publicKeys = hut;
+  "ipmi.yml.age".publicKeys = hut;
 
   "ceph-user.age".publicKeys = safe;
   "munge-key.age".publicKeys = safe;

web/content/access/index.md

@@ -11,7 +11,7 @@ access to the login machine using a resource petition in the BSC intranet.
 
 Then, to request access to the machines we will need some information about you:
 
-1. Which machines you want access to (hut, owl1, owl2, eudy, koro...)
+1. Which machines you want access to ([hut](/hut), [fox](/fox), owl1, owl2, eudy, koro...)
 1. Your user name and user id (to match the NFS permissions)
 1. Your real name and surname (for identification purposes)
 1. The salted hash of your login password, generated with `mkpasswd -m sha-512`

web/content/doc/_index.md

@@ -0,0 +1,10 @@
+---
+title: "Docs"
+description: "Documentation for users of jungle machines"
+date: 2023-09-15
+---
+
+If this is the first time you use any of the jungle machines with NixOS, follow
+the [quick start guide](quickstart).
+
+

web/content/doc/quickstart.md

@@ -0,0 +1,234 @@
+---
+title: "Quick start"
+date: 2023-09-15
+---
+
+This documentation will guide you on how to build custom packages of software
+and use them in the jungle machines. It has been designed to reduce the friction
+from users coming from module systems.
+
+You should be able to access the jungle machines, otherwise [request
+access](/access).
+
+## Changes from other HPC machines
+
+Users of other machines have been using the Lmod tool (module load ...) to add
+or remove programs from their environment, as well as manually building their
+own software for too many years.
+
+While we cannot prevent users from continuing to use this tedious mechanism, we
+have designed the jungle machines to be much easier to operate by using the nix
+package manager.
+
+### Freedom to install packages
+
+When a user wanted to install a package, it was forced to either do it on its
+own directory, or request a system administrator to install it in a shared
+directory, so other users can also use that package.
+
+This situation is gone, each user can install any package of software by
+themselves, without requiring any other authorization. When two users request
+the same package, the same copy will be provided.
+
+A new package will be downloaded if it is available (someone already built it)
+or will be built from source on demand.
+
+### No changes over time
+
+All users retain the same versions of the packages they request until they
+decide to update them.
+
+## Using nix to manage packages
+
+In this chapter we show how to install packages and enter a development shell to
+build new programs from source. The examples are done from the hut machine,
+read [this page](/access) to request access.
+
+### Installing binaries
+
+To temporarily install new packages, use:
+
+```text
+hut% nix shell jungle#gcc jungle#cowsay jungle#ovni
+```
+
+Notice that the packages are described as two parts divided by the `#` symbol.
+The first part defines where to take the package from and the second part is
+the name of the package. For now we will use `jungle#<package>`. You can find
+many more packages here:
+
+<https://search.nixos.org/packages>
+
+You will now enter a new shell, where those requested package **binaries are
+available in $PATH**:
+
+```text
+hut% cowsay hello world
+ _____________
+< hello world >
+ -------------
+        \   ^__^
+         \  (oo)\_______
+            (__)\       )\/\
+                ||----w |
+                ||     ||
+
+hut% ovniver
+LD_LIBRARY_PATH not set
+libovni: build v1.11.0 (a7103f8), dynamic v1.11.0 (a7103f8)
+
+hut% gcc --version
+gcc (GCC) 13.3.0
+Copyright (C) 2023 Free Software Foundation, Inc.
+This is free software; see the source for copying conditions.  There is NO
+warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+```
+
+### Building programs
+
+The above method only loads new binaries in the `$PATH`. If we try to build a
+program that includes headers or links with a library, it will fail to find
+them:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut% gcc test.c -lovni -o test
+test.c:1:10: fatal error: ovni.h: No such file or directory
+    1 | #include <ovni.h>
+      |          ^~~~~~~~
+compilation terminated.
+```
+
+We could manually add the full path to the ovni include directory with `-I` and
+the libraries with `-L`, but there is a tool that already perform these steps
+automatically for us, `nix develop`.
+
+Let's go back to our original shell first, where those packages are not
+available anymore:
+
+```
+hut% ps
+    PID TTY          TIME CMD
+2356260 pts/1    00:00:01 zsh
+2457268 pts/1    00:00:00 zsh
+2457297 pts/1    00:00:00 ps
+hut% exit
+hut% ovniver
+ovniver: command not found
+```
+
+### Creating a flake.nix
+
+To define which packages we want, we will write a small file that list them, a
+flake.nix file.
+
+First, we will create a new directory where we are going to be working:
+
+```
+hut% mkdir example
+hut% cd exmple
+```
+
+Then place this flake.nix file:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell {
+      pname = "devshell";
+      buildInputs = with pkgs; [
+        ovni gcc cowsay # more packages here...
+      ];
+    };
+  };
+}
+```
+
+
+Now enter the shell with:
+
+```
+hut% nix develop
+warning: creating lock file '/home/Computational/rarias/example/flake.lock':
+• Added input 'jungle':
+    'path:/nix/store/27srv8haj6vv4ywrbmw0a8vds561m8rq-source?lastModified=1739479441&narHash=sha256-Kgjs8SO1w9NbPBu8ghwzCxYJ9kvWpoQOT%2BXwPvA9DcU%3D&rev=76396c0d67ef0cf32377d5c1894bb695293bca9d' (2025-02-13)
+• Added input 'jungle/agenix':
+    'github:ryantm/agenix/f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41?narHash=sha256-b%2Buqzj%2BWa6xgMS9aNbX4I%2BsXeb5biPDi39VgvSFqFvU%3D' (2024-08-10)
+• Added input 'jungle/agenix/darwin':
+    'github:lnl7/nix-darwin/4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d?narHash=sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0%3D' (2023-11-24)
+• Added input 'jungle/agenix/darwin/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/home-manager':
+    'github:nix-community/home-manager/3bfaacf46133c037bb356193bd2f1765d9dc82c1?narHash=sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE%3D' (2023-12-20)
+• Added input 'jungle/agenix/home-manager/nixpkgs':
+    follows 'jungle/agenix/nixpkgs'
+• Added input 'jungle/agenix/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/agenix/systems':
+    'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e?narHash=sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768%3D' (2023-04-09)
+• Added input 'jungle/bscpkgs':
+    'git+https://git.sr.ht/~rodarima/bscpkgs?ref=refs/heads/master&rev=6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f' (2024-11-29)
+• Added input 'jungle/bscpkgs/nixpkgs':
+    follows 'jungle/nixpkgs'
+• Added input 'jungle/nixpkgs':
+    'github:NixOS/nixpkgs/9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc?narHash=sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8%3D' (2025-01-14)
+
+hut$ 
+```
+
+Notice that long list of messages is Nix creating a new flake.lock file with the
+current state of the packages. Next invocations will use the same packages as
+described by the lock file.
+
+### Building a program from nix develop
+
+Now let's try again building our test program:
+
+```text
+hut$ cat test.c
+#include <ovni.h>
+
+int main()
+{
+        ovni_version_check();
+        return 0;
+}
+hut$ gcc test.c -o test -lovni
+hut$ ldd test
+        linux-vdso.so.1 (0x00007ffff7fc4000)
+        libovni.so.1 => /nix/store/sqk972akjv0q8dchn8ccjln2llzyyfd0-ovni-1.11.0/lib/libovni.so.1 (0x00007ffff7fab000)
+        libc.so.6 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/libc.so.6 (0x00007ffff7db2000)
+        /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib/ld-linux-x86-64.so.2 => /nix/store/nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36/lib64/ld-linux-x86-64.so.2 (0x00007ffff7fc6000)
+hut$ ./test
+```
+
+Now the ovni.h header and the libovni library are found and the program is
+successfully built, linked and executed.
+
+You can add more packages as needed in your flake.nix:
+
+```nix
+  buildInputs = with pkgs; [
+    ovni gcc cowsay # more packages here...
+  ];
+```
+
+Make sure you exit the develop shell first, and then enter again with `nix
+develop`.
+
+## Remember
+
+- `nix shell` places binaries in the `$PATH`.
+- `nix develop` enters a development shell where both binaries and the libraries
+  and includes are available so you can build new programs.

web/content/fox/_index.md

@@ -0,0 +1,97 @@
+---
+title: "Fox"
+description: "AMD Genoa 9684X with 2 NVIDIA RTX4000 GPUs"
+date: 2025-02-12
+---
+
+![Fox](fox.jpg)
+
+Picture by [Joanne Redwood](https://web.archive.org/web/20191109175146/https://www.inaturalist.org/photos/6568074),
+[CC0](http://creativecommons.org/publicdomain/zero/1.0/deed.en).
+
+The *fox* machine is a big GPU server that is configured to run heavy workloads.
+It has two fast AMD CPUs with large cache and 2 reasonable NVIDIA GPUs. Here are
+the detailed specifications:
+
+- 2x AMD GENOA X 9684X DP/UP 96C/192T 2.55G 1,150M 400W SP5 3D V-cach
+- 24x 32GB DDR5-4800 ECC RDIMM (total 768 GiB of RAM)
+- 1x 2.5" SSD SATA3 MICRON 5400 MAX 480GB
+- 2x 2.5" KIOXIA CM7-R 1.92TB NVMe GEN5 PCIe 5x4
+- 2x NVIDIA RTX4000 ADA Gen 20GB GDDR6 PCIe 4.0
+
+## Access
+
+To access the machine, request a SLURM session from [hut](/hut) using the `fox`
+partition:
+
+    hut% salloc -p fox
+
+Then connect via ssh:
+
+    hut% ssh fox
+    fox%
+
+Follow [these steps](/access) if you don't have access to hut or fox.
+
+## CUDA
+
+To use CUDA, you can use the following `flake.nix` placed in a new directory to
+load all the required dependencies:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+
+  outputs = { jungle, ... }: {
+    devShell.x86_64-linux = let
+      pkgs = jungle.nixosConfigurations.fox.pkgs;
+    in pkgs.mkShell {
+      name = "cuda-env-shell";
+      buildInputs = with pkgs; [
+        git gitRepo gnupg autoconf curl
+        procps gnumake util-linux m4 gperf unzip
+
+        # Cuda packages (more at https://search.nixos.org/packages)
+        cudatoolkit linuxPackages.nvidia_x11
+        cudaPackages.cuda_cudart.static
+        cudaPackages.libcusparse
+
+        libGLU libGL
+        xorg.libXi xorg.libXmu freeglut
+        xorg.libXext xorg.libX11 xorg.libXv xorg.libXrandr zlib
+        ncurses5 stdenv.cc binutils
+      ];
+      shellHook = ''
+        export CUDA_PATH=${pkgs.cudatoolkit}
+        export LD_LIBRARY_PATH=/var/run/opengl-driver/lib
+        export SMS=50
+      '';
+    };
+  };
+}
+```
+
+Then just run `nix develop` from the same directory:
+
+    % mkdir cuda
+    % cd cuda
+    % vim flake.nix
+    [...]
+    % nix develop
+    $ nvcc -V
+    nvcc: NVIDIA (R) Cuda compiler driver
+    Copyright (c) 2005-2024 NVIDIA Corporation
+    Built on Tue_Feb_27_16:19:38_PST_2024
+    Cuda compilation tools, release 12.4, V12.4.99
+    Build cuda_12.4.r12.4/compiler.33961263_0
+
+## Filesystems
+
+The machine has several file systems available.
+
+- `$HOME`: Mounted via NFS across all nodes. It is slow and has low capacity.
+  Don't abuse.
+- `/ceph/home/$USER`: Shared Ceph file system across jungle nodes. Slow but high
+  capacity. Stores three redundant copies of every file.
+- `/nvme{0,1}/$USER`: The two local NVME disks, very fast and large capacity.
+- `/tmp`: tmpfs, fast but not backed by a disk. Will be erased on reboot.

web/content/hut/_index.md

@@ -23,7 +23,7 @@ a good candidate for low noise executions.
 We provide a binary cache in `hut`, with the aim of avoiding unnecessary
 recompilation of packages.
 
-The cache should contain common packages from bsckpkgs, but we don't provide
+The cache should contain common packages from bscpkgs, but we don't provide
 any guarantee that of what will be available in the cache, or for how long.
 We recommend following the latest version of the `jungle` flake to avoid cache
 misses.
@@ -38,7 +38,7 @@ enable it for all builds in the system.
 ```nix
 { ... }: {
   nix.settings = {
-    substituters = [ "https://jungle.bsc.es" ];
+    substituters = [ "https://jungle.bsc.es/cache" ];
     trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
   };
 }
@@ -53,13 +53,15 @@ The cache can also be specified in a per-command basis through the flags
 nix build --substituters "https://jungle.bsc.es/cache" --trusted-public-keys "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" <...>
 ```
 
+Note: you'll have to be a trusted user.
+
 ### Nix configuration file (non-nixos)
 
-If using nix outside of NixOS, you'll have to update `nix.conf`
+If using nix outside of NixOS, you'll have to update `/etc/nix/nix.conf`
 
-```bash
-echo "substituters = https://jungle.bsc.es" >> /etc/nix/nix.conf
-echo "trusted-public-keys = jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" >> /etc/nix/nix.conf
+```
+# echo "substituters = https://jungle.bsc.es/cache" >> /etc/nix/nix.conf
+# echo "trusted-public-keys = jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" >> /etc/nix/nix.conf
 ```
 
 ### Hint in flakes

web/hugo.toml

@@ -3,32 +3,38 @@ languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
 
+[[menu.main]]
+identifier = "doc"
+name = "Docs"
+url = "/doc/"
+weight = 10
+
 [[menu.main]]
 identifier = "grafana"
 name = "Grafana"
 url = "/grafana/"
-weight = 10
+weight = 20
 
 [[menu.main]]
 identifier = "Git"
 name = "Git"
 url = "/git/"
-weight = 20
+weight = 30
 
 [[menu.main]]
 identifier = "Lists"
 name = "Lists"
 url = "/lists/"
-weight = 30
+weight = 40
 
 [[menu.main]]
 identifier = "Paste"
 name = "Paste"
 url = "/paste/"
-weight = 40
+weight = 50
 
 [[menu.main]]
 identifier = "Posts"
 name = "Posts"
 url = "/posts/"
-weight = 50
+weight = 60