Enable watchdog

m/bay/configuration.nix

@@ -33,9 +33,10 @@
     enable = true;
     global = {
       fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b";
-      monHost = "10.0.40.40";
+      monHost = "10.0.42.40";
-      monInitialMembers = "bay";
+      monInitialMembers = "10.0.42.40";
-      clusterNetwork = "10.0.40.40/24"; # Use Ethernet only
+      publicNetwork = "10.0.42.40/24";
+      clusterNetwork = "10.0.42.40/24";
     };
     extraConfig = {
       # Only log to stderr so it appears in the journal
@@ -49,7 +50,7 @@
       enable = true;
       daemons = [ "mds0" "mds1" ];
       extraConfig = {
-        "host" = "bay";
+        "host" = "10.0.42.40";
       };
     };
     mgr = {

m/common/main.nix

@@ -10,6 +10,7 @@
     ./slurm.nix
     ./ssh.nix
     ./users.nix
+    ./watchdog.nix
   ];
 
   nixpkgs.overlays = [

m/common/watchdog.nix

@@ -0,0 +1,9 @@
+{ ... }:
+
+{
+  # The boards have a BMC watchdog controlled by IPMI
+  boot.kernelModules = [ "ipmi_watchdog" ];
+
+  # Enable systemd watchdog with 30 s interval
+  systemd.watchdog.runtimeTime = "30s";
+}

m/hut/ceph.nix

@@ -17,7 +17,7 @@
     fsType = "ceph";
     device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
     options = [
-      "mon_addr=10.0.40.40"
+      "mon_addr=10.0.42.40"
       "secretfile=${config.age.secrets."secrets/ceph-user".path}"
     ];
   };

m/lake2/configuration.nix

@@ -20,9 +20,10 @@
     enable = true;
     global = {
       fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b";
-      monHost = "10.0.40.40";
+      monHost = "10.0.42.40";
-      monInitialMembers = "bay";
+      monInitialMembers = "10.0.42.40";
-      clusterNetwork = "10.0.40.40/24"; # Use Ethernet only
+      publicNetwork = "10.0.42.40/24";
+      clusterNetwork = "10.0.42.40/24";
     };
     osd = {
       enable = true;
@@ -50,4 +51,24 @@
       prefixLength = 24;
     } ];
   };
+
+  # Missing service for volumes, see:
+  # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/
+  systemd.services.ceph-volume = {
+    enable = true;
+    description = "Ceph Volume activation";
+    unitConfig = {
+      Type = "oneshot";
+      After = "local-fs.target";
+      Wants = "local-fs.target";
+    };
+    path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ];
+    serviceConfig = {
+      KillMode = "none";
+      Environment = "CEPH_VOLUME_TIMEOUT=10000";
+      ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'";
+      TimeoutSec = "0";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
 }