Keep compute nodes off when power comes back

When the power comes back, we don't know if the AC unit will be
operating properly or if the room will be at a safe temperature. So,
instead of powering all the machines back, only configure the login to
power on, so we can check the state of the room and power the rest of
the machines.

m/apex/configuration.nix

@@ -5,10 +5,12 @@
     ../common/xeon.nix
     ../common/ssf/hosts.nix
     ../module/ceph.nix
-    ../module/slurm-server.nix
+    ../module/power-policy.nix
     ./nfs.nix
   ];
 
+  power.policy = "always-on";
+
   # Don't install grub MBR for now
   boot.loader.grub.device = "nodev";
 

m/common/base.nix

@@ -3,7 +3,6 @@
   # Includes the basic configuration for an Intel server.
   imports = [
     ./base/agenix.nix
-    ./base/always-power-on.nix
     ./base/august-shutdown.nix
     ./base/boot.nix
     ./base/env.nix
@@ -12,6 +11,7 @@
     ./base/net.nix
     ./base/nix.nix
     ./base/ntp.nix
+    ./base/power-policy.nix
     ./base/rev.nix
     ./base/ssh.nix
     ./base/users.nix

m/common/base/always-power-on.nix

@@ -1,8 +0,0 @@
-{
-  imports = [
-    ../../module/power-policy.nix
-  ];
-
-  # Turn on as soon as we have power
-  power.policy = "always-on";
-}

m/common/base/power-policy.nix

@@ -0,0 +1,9 @@
+{
+  imports = [
+    ../../module/power-policy.nix
+  ];
+
+  # By default, keep the machines off as we don't know if the AC will be working
+  # once the electricity comes back.
+  power.policy = "always-off";
+}

m/common/base/users.nix

@@ -154,20 +154,6 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIV5LEAII5rfe1hYqDYIIrhb1gOw7RcS1p2mhOTqG+zc pedro@pedro-ThinkPad-P14s-Gen-2a"
         ];
       };
-
-      csiringo = {
-        # Arbitrary UID but large so it doesn't collide with other users on ssfhead.
-        uid = 9653;
-        isNormalUser = true;
-        home = "/home/Computational/csiringo";
-        description = "Cesare Siringo";
-        group = "Computational";
-        hosts = [ "apex" "weasel" ];
-        hashedPassword = "$6$0IsZlju8jFukLlAw$VKm0FUXbS.mVmPm3rcJeizTNU4IM5Nmmy21BvzFL.cQwvlGwFI1YWRQm6gsbd4nbg47mPDvYkr/ar0SlgF6GO1";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHA65zvvG50iuFEMf+guRwZB65jlGXfGLF4HO+THFaed csiringo@bsc.es"
-        ];
-      };
     };
 
     groups = {

m/fox/configuration.nix

@@ -6,8 +6,11 @@
     ../common/xeon/console.nix
     ../module/emulation.nix
     ../module/nvidia.nix
+    ../module/power-policy.nix
   ];
 
+  power.policy = "always-on";
+
   # Don't turn off on August as UPC has different dates.
   # Fox works fine on power cuts.
   systemd.timers.august-shutdown.enable = false;
@@ -35,16 +38,6 @@
 
   services.openssh.settings.X11Forwarding = true;
 
-  # Use SSH tunnel to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host bscpm04.bsc.es gitlab-internal.bsc.es tent
-      ProxyJump raccoon
-    Host raccoon
-      ProxyJump apex
-      HostName 127.0.0.1
-      Port 22022
-  '';
-
   networking = {
     timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
     hostName = "fox";

m/hut/configuration.nix

@@ -7,9 +7,11 @@
     ../module/ceph.nix
     ../module/debuginfod.nix
     ../module/emulation.nix
+    ../module/slurm-client.nix
     ./gitlab-runner.nix
     ./monitoring.nix
     ./nfs.nix
+    ./slurm-server.nix
     ./nix-serve.nix
     ./public-inbox.nix
     ./gitea.nix

m/hut/slurm-server.nix

@@ -0,0 +1,7 @@
+{ ... }:
+
+{
+  services.slurm = {
+    server.enable = true;
+  };
+}

m/module/slurm-client.nix

@@ -1,10 +1,33 @@
 { config, pkgs, lib, ... }:
 
-{
-  imports = [
-    ./slurm-common.nix
-  ];
+let
+  suspendProgram = pkgs.writeScript "suspend.sh" ''
+    #!/usr/bin/env bash
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Shutting down host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
+    done
+  '';
 
+  resumeProgram = pkgs.writeScript "resume.sh" ''
+    #!/usr/bin/env bash
+    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
+    set -x
+    export "PATH=/run/current-system/sw/bin:$PATH"
+    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
+    hosts=$(scontrol show hostnames $1)
+    for host in $hosts; do
+      echo Starting host: $host
+      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
+    done
+  '';
+
+in {
   systemd.services.slurmd.serviceConfig = {
     # Kill all processes in the control group on stop/restart. This will kill
     # all the jobs running, so ensure that we only upgrade when the nodes are
@@ -14,5 +37,90 @@
     KillMode = lib.mkForce "control-group";
   };
 
-  services.slurm.client.enable = true;
+  services.slurm = {
+    client.enable = true;
+    controlMachine = "hut";
+    clusterName = "jungle";
+    nodeName = [
+      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
+      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
+    ];
+
+    partitionName = [
+      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+    ];
+
+    # See slurm.conf(5) for more details about these options.
+    extraConfig = ''
+      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
+      # not with Intel MPI. For that use the compatibility shim libpmi.so
+      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
+      # library in SLURM (--mpi=pmix). See more details here:
+      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
+      MpiDefault=pmix
+
+      # When a node reboots return that node to the slurm queue as soon as it
+      # becomes operative again.
+      ReturnToService=2
+
+      # Track all processes by using a cgroup
+      ProctrackType=proctrack/cgroup
+
+      # Enable task/affinity to allow the jobs to run in a specified subset of
+      # the resources. Use the task/cgroup plugin to enable process containment.
+      TaskPlugin=task/affinity,task/cgroup
+
+      # Power off unused nodes until they are requested
+      SuspendProgram=${suspendProgram}
+      SuspendTimeout=60
+      ResumeProgram=${resumeProgram}
+      ResumeTimeout=300
+      SuspendExcNodes=hut
+
+      # Turn the nodes off after 1 hour of inactivity
+      SuspendTime=3600
+
+      # Reduce port range so we can allow only this range in the firewall
+      SrunPortRange=60000-61000
+
+      # Use cores as consumable resources. In SLURM terms, a core may have
+      # multiple hardware threads (or CPUs).
+      SelectType=select/cons_tres
+
+      # Ignore memory constraints and only use unused cores to share a node with
+      # other jobs.
+      SelectTypeParameters=CR_Core
+
+      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
+      # This sets up the "extern" step into which ssh-launched processes will be
+      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
+      # when a task runs (srun) so we can ssh early.
+      PrologFlags=Alloc,Contain,X11
+
+      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
+      # adopted by the external step, similar to tasks running in regular steps
+      # LaunchParameters=ulimit_pam_adopt
+      SlurmdDebug=debug5
+      #DebugFlags=Protocol,Cgroup
+    '';
+
+    extraCgroupConfig = ''
+      CgroupPlugin=cgroup/v2
+      #ConstrainCores=yes
+    '';
+  };
+
+  # Place the slurm config in /etc as this will be required by PAM
+  environment.etc.slurm.source = config.services.slurm.etcSlurm;
+
+  age.secrets.mungeKey = {
+    file = ../../secrets/munge-key.age;
+    owner = "munge";
+    group = "munge";
+  };
+
+  services.munge = {
+    enable = true;
+    password = config.age.secrets.mungeKey.path;
+  };
 }

m/module/slurm-common.nix

@@ -1,115 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  suspendProgram = pkgs.writeScript "suspend.sh" ''
-    #!/usr/bin/env bash
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Shutting down host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power off
-    done
-  '';
-
-  resumeProgram = pkgs.writeScript "resume.sh" ''
-    #!/usr/bin/env bash
-    exec 1>>/var/log/power_save.log 2>>/var/log/power_save.log
-    set -x
-    export "PATH=/run/current-system/sw/bin:$PATH"
-    echo "$(date) Suspend invoked $0 $*" >> /var/log/power_save.log
-    hosts=$(scontrol show hostnames $1)
-    for host in $hosts; do
-      echo Starting host: $host
-      ipmitool -I lanplus -H ''${host}-ipmi -P "" -U "" chassis power on
-    done
-  '';
-
-in {
-  services.slurm = {
-    controlMachine = "apex";
-    clusterName = "jungle";
-    nodeName = [
-      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
-    ];
-
-    partitionName = [
-      "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
-    ];
-
-    # See slurm.conf(5) for more details about these options.
-    extraConfig = ''
-      # Use PMIx for MPI by default. It works okay with MPICH and OpenMPI, but
-      # not with Intel MPI. For that use the compatibility shim libpmi.so
-      # setting I_MPI_PMI_LIBRARY=$pmix/lib/libpmi.so while maintaining the PMIx
-      # library in SLURM (--mpi=pmix). See more details here:
-      # https://pm.bsc.es/gitlab/rarias/jungle/-/issues/16
-      MpiDefault=pmix
-
-      # When a node reboots return that node to the slurm queue as soon as it
-      # becomes operative again.
-      ReturnToService=2
-
-      # Track all processes by using a cgroup
-      ProctrackType=proctrack/cgroup
-
-      # Enable task/affinity to allow the jobs to run in a specified subset of
-      # the resources. Use the task/cgroup plugin to enable process containment.
-      TaskPlugin=task/affinity,task/cgroup
-
-      # Power off unused nodes until they are requested
-      SuspendProgram=${suspendProgram}
-      SuspendTimeout=60
-      ResumeProgram=${resumeProgram}
-      ResumeTimeout=300
-      #SuspendExcNodes=
-
-      # Turn the nodes off after 1 hour of inactivity
-      SuspendTime=3600
-
-      # Reduce port range so we can allow only this range in the firewall
-      SrunPortRange=60000-61000
-
-      # Use cores as consumable resources. In SLURM terms, a core may have
-      # multiple hardware threads (or CPUs).
-      SelectType=select/cons_tres
-
-      # Ignore memory constraints and only use unused cores to share a node with
-      # other jobs.
-      SelectTypeParameters=CR_Core
-
-      # Required for pam_slurm_adopt, see https://slurm.schedmd.com/pam_slurm_adopt.html
-      # This sets up the "extern" step into which ssh-launched processes will be
-      # adopted. Alloc runs the prolog at job allocation (salloc) rather than
-      # when a task runs (srun) so we can ssh early.
-      PrologFlags=Alloc,Contain,X11
-
-      # LaunchParameters=ulimit_pam_adopt will set RLIMIT_RSS in processes
-      # adopted by the external step, similar to tasks running in regular steps
-      # LaunchParameters=ulimit_pam_adopt
-      SlurmdDebug=debug5
-      #DebugFlags=Protocol,Cgroup
-    '';
-
-    extraCgroupConfig = ''
-      CgroupPlugin=cgroup/v2
-      #ConstrainCores=yes
-    '';
-  };
-
-  # Place the slurm config in /etc as this will be required by PAM
-  environment.etc.slurm.source = config.services.slurm.etcSlurm;
-
-  age.secrets.mungeKey = {
-    file = ../../secrets/munge-key.age;
-    owner = "munge";
-    group = "munge";
-  };
-
-  services.munge = {
-    enable = true;
-    password = config.age.secrets.mungeKey.path;
-  };
-}

m/module/slurm-server.nix

@@ -1,18 +0,0 @@
-{ ... }:
-
-{
-  imports = [
-    ./slurm-common.nix
-  ];
-
-  services.slurm.server.enable = true;
-
-  networking.firewall = {
-    extraCommands = ''
-      # Accept slurm connections to controller from compute nodes
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817 -j nixos-fw-accept
-      # Accept slurm connections from compute nodes for srun
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 60000:61000 -j nixos-fw-accept
-    '';
-  };
-}

m/raccoon/configuration.nix

@@ -7,9 +7,12 @@
     ../module/debuginfod.nix
     ../module/ssh-hut-extern.nix
     ../module/nvidia.nix
+    ../module/power-policy.nix
     ../eudy/kernel/perf.nix
   ];
 
+  power.policy = "always-on";
+
   # Don't install Grub on the disk yet
   boot.loader.grub.device = "nodev";
 

m/weasel/configuration.nix

@@ -14,10 +14,6 @@
   # Users with sudo access
   users.groups.wheel.members = [ "abonerib" "anavarro" ];
 
-  # Run julia installed with juliaup using julia's own libraries:
-  # NIX_LD_LIBRARY_PATH=~/.julia/juliaup/${VERS}/lib/julia ~/.juliaup/bin/julia
-  programs.nix-ld.enable = true;
-
   networking = {
     hostName = "weasel";
     interfaces.eno1.ipv4.addresses = [ {