Add 10 min shutdown jitter to avoid spikes

The shutdown timer will fire at slightly different times for the
different nodes, so we slowly decrease the power consumption.

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1712079060,
+        "lastModified": 1720546205,
-        "narHash": "sha256-/JdiT9t+zzjChc5qQiF+jhrVhRt8figYH29rZO7pFe4=",
+        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "1381a759b205dff7a6818733118d02253340fd5e",
+        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
         "type": "github"
       },
       "original": {
@@ -88,11 +88,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1713714899,
+        "lastModified": 1720957393,
-        "narHash": "sha256-+z/XjO3QJs5rLE5UOf015gdVauVRQd2vZtsFkaXBq2Y=",
+        "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6143fc5eeb9c4f00163267708e26191d1e918932",
+        "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
         "type": "github"
       },
       "original": {

m/common/base.nix

@@ -3,6 +3,7 @@
   # Includes the basic configuration for an Intel server.
   imports = [
     ./base/agenix.nix
+    ./base/august-shutdown.nix
     ./base/boot.nix
     ./base/env.nix
     ./base/fs.nix

m/common/base/august-shutdown.nix

@@ -0,0 +1,14 @@
+{
+  # Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
+  # hardware from spurious electrical peaks on the yearly electrical cut for
+  # manteinance that starts on August 4th.
+  systemd.timers.august-shutdown = {
+    description = "Shutdown on August 2nd for maintenance";
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnCalendar = "*-08-02 11:00:00";
+      RandomizedDelaySec = "10min";
+      Unit = "systemd-poweroff.service";
+    };
+  };
+}

m/hut/configuration.nix

@@ -6,6 +6,7 @@
 
     ../module/ceph.nix
     ../module/debuginfod.nix
+    ../module/emulation.nix
     ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
@@ -19,8 +20,6 @@
     #./pxe.nix
   ];
 
-  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
-
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
 

m/hut/gitea.nix

@@ -17,6 +17,7 @@
         REGISTER_MANUAL_CONFIRM = true;
         ENABLE_NOTIFY_MAIL = true;
       };
+      log.LEVEL = "Warn";
 
       mailer = {
         ENABLED       = true;

m/hut/gitlab-runner.nix

@@ -1,9 +1,8 @@
 { pkgs, lib, config, ... }:
 
 {
-  age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
+  age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age;
-  age.secrets.gitlabToken.file = ../../secrets/gitlab-bsc-es-token.age;
+  age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age;
-  age.secrets.nosvToken.file = ../../secrets/nosv-token.age;
 
   services.gitlab-runner = {
     enable = true;
@@ -11,20 +10,14 @@
     services = let
       common-shell = {
         executor = "shell";
-        tagList = [ "nix" "xeon" ];
-        registrationFlags = [
-          # Using space doesn't work, and causes it to misread the next flag
-          "--locked='false'"
-        ];
         environmentVariables = {
           SHELL = "${pkgs.bash}/bin/bash";
         };
       };
       common-docker = {
+        executor = "docker";
         dockerImage = "debian:stable";
-        tagList = [ "docker" "xeon" ];
         registrationFlags = [
-          "--locked='false'"
           "--docker-network-mode host"
         ];
         environmentVariables = {
@@ -33,19 +26,12 @@
         };
       };
     in {
-      # For gitlab.bsc.es
-      gitlab-bsc-es-shell = common-shell // {
-        registrationConfigFile = config.age.secrets.gitlabToken.path;
-      };
-      gitlab-bsc-es-docker = common-docker // {
-        registrationConfigFile = config.age.secrets.gitlabToken.path;
-      };
       # For pm.bsc.es/gitlab
       gitlab-pm-shell = common-shell // {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path;
       };
       gitlab-pm-docker = common-docker // {
-        registrationConfigFile = config.age.secrets.ovniToken.path;
+        authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path;
       };
     };
   };

m/hut/monitoring.nix

@@ -31,6 +31,7 @@
       };
       feature_toggles.publicDashboards = true;
       "auth.anonymous".enabled = true;
+      log.level = "warn";
     };
   };
 

m/module/emulation.nix

@@ -0,0 +1,3 @@
+{
+  boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
+}

m/module/slurm-client.nix

@@ -47,8 +47,8 @@ in {
     ];
 
     partitionName = [
-      "owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP"
+      "owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-      "all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP"
+      "all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.
@@ -83,6 +83,14 @@ in {
 
       # Reduce port range so we can allow only this range in the firewall
       SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_CORE
     '';
   };
 

m/owl1/configuration.nix

@@ -4,9 +4,10 @@
   imports = [
     ../common/xeon.nix
     ../module/ceph.nix
+    ../module/emulation.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
-    ../module/slurm-hut-nix-store.nix
+    ../module/debuginfod.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -4,9 +4,10 @@
   imports = [
     ../common/xeon.nix
     ../module/ceph.nix
+    ../module/emulation.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix
-    ../module/slurm-hut-nix-store.nix
+    ../module/debuginfod.nix
   ];
 
   # Select the this using the ID to avoid mismatches

secrets/gitlab-bsc-es-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 HY2yRg caTbx0NBmsTSmZH4HtBaxhsauWqWUDTesJqT08UsoEQ
-8ND31xuco+H8d5SKg8xsCFRPVDhU4d8UKwV1BnmKVjQ
--> ssh-ed25519 CAWG4Q 4ETYuhCwHHECkut4DWDknMMgpAvFqtzLWVC2Wi2L8FM
-BGMvRnAfd8qZG5hzLefmk32FkGvwzE9pqBUyx4JY0co
--> ssh-ed25519 MSF3dg hj5QL4ZfylN8/W/MXQHvVqtI7mRvlQOYr8HsaQEmPB0
-kvB7sljmmkswSGZDQnrwdTbTsN78EAwH3pz1pPe0Hu0
--> )Q-grease vHF} [8p1> @7z;C"/
-tgSUKFyyrf2jLXZp+pakigwB2fRO/WFj2Qnt1aPjtVPEK92JbJ4
---- xzM0AhV4gTQE0Q7inJNo9vFj+crJQxWeI7u9pl7bqAI
-á6nGJÖ0Bˆ’7F° –bßÙ½2®L³äÇ]²2zl<7A>À&e†KÄx®àé9SWNàV"MfŽ€ëÙKHUC:1b;9St‰ëõ±Duѧç‹Ï¢žÌŸ¡<02>èÐéîÀ–<C380>ÔfÕ7¨î1§I(õdÓþô‡ïó

secrets/gitlab-runner-docker-token.age

@@ -0,0 +1,9 @@
+age-encryption.org/v1
+-> ssh-ed25519 HY2yRg WvKK6U1wQtx2pbUDfuaUIXTQiCulDkz7hgUCSwMfMzQ
+jLktUMqKuVxukqzz++pHOKvmucUQqeKYy5IwBma7KxY
+-> ssh-ed25519 CAWG4Q XKGuNNoYFl9bdZzsqYYTY7GsEt5sypLW4R+1uk78NmU
+8dIA2GzRAwTGM5CDHSM2BUBsbXzEAUssWUz2PY2PaTg
+-> ssh-ed25519 MSF3dg T630RsKuZIF/bp+KITnIIWWHsg6M/VQGqbWQZxqT+AA
+SraZcgZJVtmUzHF/XR9J7aK5t5EDNpkC/av/WJUT/G8
+--- /12G8pj9sbs591OM/ryhoLnSWWmzYcoqprk9uN/3g18
+ä·ù
¼Â‡%å]yi"ô<>»LÓâùH`ªa$Æþ)¦9ve<76>.0úmÉK<EFBFBD>vƒÀïu"|1cÞ-%ÔÕ"åWFï¡ÞA«<41>hº$•ºj<eñ¶xÅLx«ç.?œÈâ:L…¬–ƒ,ëu»|³‹F|Õi²äÔ

secrets/nosv-token.age

@@ -1,11 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 HY2yRg hrdS7Dl/j+u3XVfM79ZJpZSlre9TcD7DTQ+EEAT6kEE
-avUO96P1h7w2BYWgrQ7GpUgdaCV9AZL7eOTTcF9gfro
--> ssh-ed25519 CAWG4Q A5raRY1CAgFYZgoQ92GMyNejYNdHx/7Y6uTS+EjLPWA
-FRFqT2Jz7qRcybaxkQTKHGl797LVXoHpYG4RZSrX/70
--> ssh-ed25519 MSF3dg D+R80Bg7W9AuiOMAqtGFZQl994dRBIegYRLmmTaeZ3o
-BHvZsugRiuZ91b4jk91h30o3eF3hadSnVCwxXge95T8
--> BT/El`a-grease W{nq|Vm )bld 2Nl}4 N$#JGB4t
-oLG+0S1aGfO/ohCfgGmhDhwwLi4H
---- 2I5C+FvBG/K1ZHh7C5QD39feTSLoFGwcTeZAmeILNsI
-¹õW©o÷ ÙÄd;ËÐC¾.¹¡_(“u
G¡€‰#ìvâœgÉ<67>†õõy¹Y‰žl9ŒÈ
¡Ïµ.Œé0x<30>Þ½úN. /ü<>tB×b‡ü¼K¼ì:Q×—È\¹ÀÍT_´»Átxïm’——_JñÞž-š

secrets/secrets.nix

@@ -6,10 +6,9 @@ let
   safe = keys.hostGroup.safe ++ adminsKeys;
 in
 {
-  "gitlab-bsc-es-token.age".publicKeys = hut;
   "gitea-runner-token.age".publicKeys = hut;
-  "ovni-token.age".publicKeys = hut;
+  "gitlab-runner-docker-token.age".publicKeys = hut;
-  "nosv-token.age".publicKeys = hut;
+  "gitlab-runner-shell-token.age".publicKeys = hut;
   "nix-serve.age".publicKeys = hut;
   "jungle-robot-password.age".publicKeys = hut;