m/common/xeon/net.nix

@@ -11,7 +11,7 @@
 
     proxy = {
       default = "http://hut:23080/";
-      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
+      noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40,hut";
       # Don't set all_proxy as go complains and breaks the gitlab runner, see:
       # https://github.com/golang/go/issues/16715
       allProxy = null;

m/hut/configuration.nix

@@ -56,6 +56,11 @@
         iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
         iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
       '';
+      # Flush all rules and chains on stop so it won't break on start
+      extraStopCommands = ''
+        iptables -F
+        iptables -X
+      '';
     };
   };
 

m/hut/nginx.nix

@@ -17,13 +17,14 @@ let
   };
 in
 {
+  networking.firewall.allowedTCPPorts = [ 80 ];
   services.nginx = {
     enable = true;
     virtualHosts."jungle.bsc.es" = {
       root = "${website}";
       listen = [
         {
-          addr = "127.0.0.1";
+          addr = "0.0.0.0";
           port = 80;
         }
       ];
@@ -40,7 +41,7 @@ in
           proxy_redirect http:// $scheme://;
         }
         location /cache {
-          rewrite ^/cache(.*) /$1 break;
+          rewrite ^/cache/(.*) /$1 break;
           proxy_pass http://127.0.0.1:5000;
           proxy_redirect http:// $scheme://;
         }

m/module/hut-substituter.nix

@@ -0,0 +1,10 @@
+{ config, ... }:
+{
+  nix.settings =
+    # Don't add hut as a cache to itself
+    assert config.networking.hostName != "hut";
+    {
+      substituters = [ "http://hut/cache" ];
+      trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+    };
+}

m/owl1/configuration.nix

@@ -8,6 +8,7 @@
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
     ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -8,6 +8,7 @@
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
     ../module/debuginfod.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/raccoon/configuration.nix

@@ -25,6 +25,11 @@
     } ];
   };
 
+  nix.settings = {
+    substituters = [ "https://jungle.bsc.es/cache" ];
+    trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+
   # Configure Nvidia driver to use with CUDA
   hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
   hardware.graphics.enable = true;

web/content/hut/_index.md

@@ -13,6 +13,115 @@ which is available at `hut` or `xeon07`. It runs the following services:
 - Grafana: to plot the data in the web browser.
 - Slurmctld: to manage the SLURM nodes.
 - Gitlab runner: to run CI jobs from Gitlab.
+- Nix binary cache: to serve cached nix builds
 
 This node is prone to interruptions from all the services it runs, so it is not
 a good candidate for low noise executions.
+
+# Binary cache
+
+We provide a binary cache in `hut`, with the aim of avoiding unnecessary
+recompilation of packages.
+
+The cache should contain common packages from bscpkgs, but we don't provide
+any guarantee that of what will be available in the cache, or for how long.
+We recommend following the latest version of the `jungle` flake to avoid cache
+misses.
+
+## Usage
+
+### From NixOS
+
+In NixOS, we can add the cache through the `nix.settings` option, which will
+enable it for all builds in the system.
+
+```nix
+{ ... }: {
+  nix.settings = {
+    substituters = [ "https://jungle.bsc.es/cache" ];
+    trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  };
+}
+```
+
+### Interactively
+
+The cache can also be specified in a per-command basis through the flags
+`--substituters` and `--trusted-public-keys`:
+
+```sh
+nix build --substituters "https://jungle.bsc.es/cache" --trusted-public-keys "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" <...>
+```
+
+Note: you'll have to be a trusted user.
+
+### Nix configuration file (non-nixos)
+
+If using nix outside of NixOS, you'll have to update `/etc/nix/nix.conf`
+
+```
+# echo "substituters = https://jungle.bsc.es/cache" >> /etc/nix/nix.conf
+# echo "trusted-public-keys = jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" >> /etc/nix/nix.conf
+```
+
+### Hint in flakes
+
+By adding the configuration below to a `flake.nix`, when someone uses the flake,
+`nix` will interactively ask to trust and use the provided binary cache:
+
+```nix
+{
+  nixConfig = {
+    extra-substituters = [
+      "https://jungle.bsc.es/cache"
+    ];
+    extra-trusted-public-keys = [
+      "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0="
+    ];
+  };
+  outputs = { ... }: {
+    ...
+  };
+}
+```
+
+### Querying the cache
+
+Check if the cache is available:
+```sh
+$ curl https://jungle.bsc.es/cache/nix-cache-info
+StoreDir: /nix/store
+WantMassQuery: 1
+Priority: 30
+```
+
+Prevent nix from building locally:
+```bash
+nix build --max-jobs 0 <...>
+```
+
+Check if a package is in cache:
+```bash
+# Do a raw eval on the <package>.outPath (this should not build the package)
+$ nix eval --raw jungle#openmp.outPath
+/nix/store/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-openmp-2024.11
+# Take the hash (everything from / to - in the basename) and curl <hash>.narinfo
+# if it exists in the cache, it will return HTTP 200 and some information
+# if not, it will return 404
+$ curl https://jungle.bsc.es/cache/dwnn4dgm1m4184l4xbi0qfrprji9wjmi.narinfo
+StorePath: /nix/store/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-openmp-2024.11
+URL: nar/dwnn4dgm1m4184l4xbi0qfrprji9wjmi-17imkdfqzmnb013d14dx234bx17bnvws8baf3ii1xra5qi2y1wiz.nar
+Compression: none
+NarHash: sha256:17imkdfqzmnb013d14dx234bx17bnvws8baf3ii1xra5qi2y1wiz
+NarSize: 1519328
+References: 4gk773fqcsv4fh2rfkhs9bgfih86fdq8-gcc-13.3.0-lib nqb2ns2d1lahnd5ncwmn6k84qfd7vx2k-glibc-2.40-36
+Deriver: vcn0x8hikc4mvxdkvrdxp61bwa5r7lr6-openmp-2024.11.drv
+Sig: jungle.bsc.es:GDTOUEs1jl91wpLbb+gcKsAZjpKdARO9j5IQqb3micBeqzX2M/NDtKvgCS1YyiudOUdcjwa3j+hyzV2njokcCA==
+# In oneline:
+$ curl "https://jungle.bsc.es/cache/$(nix eval --raw jungle#<package>.outPath | cut -d '/' -f4 | cut -d '-' -f1).narinfo"
+```
+
+#### References
+
+- https://nix.dev/guides/recipes/add-binary-cache.html
+- https://nixos.wiki/wiki/Binary_Cache