Add remote sblame probe to prometheus

Reviewed-by: Aleix Boné <abonerib@bsc.es>

m/hut/monitoring.nix

@@ -5,7 +5,7 @@
     ../module/slurm-exporter.nix
     ../module/meteocat-exporter.nix
     ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
+    ./ssh-robot-probes.nix
     ../module/nix-daemon-exporter.nix
   ];
 
@@ -111,6 +111,7 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9967" # SLURM custom exporter
             "127.0.0.1:9999" # Nix-daemon custom exporter
             "127.0.0.1:9929" # Meteocat custom exporter
             "127.0.0.1:9928" # UPC Qaire custom exporter

m/hut/sblame-probe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+EOF
+ssh bsc015557@glogin2.bsc.es "timeout 3 command sblame -E"

m/hut/ssh-robot-probes.nix

@@ -6,6 +6,12 @@ let
       chmod +x $out
     ''
   ;
+  sblame-probe-script = pkgs.runCommand "sblame-probe.sh" { }
+    ''
+      cp ${./sblame-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
 in
 {
   # Use a new user to handle the SSH keys
@@ -28,4 +34,17 @@ in
       Group = "ssh-robot";
     };
   };
+
+  systemd.services.sblame-probe = {
+    description = "Daemon to report SLURM statistics via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9967,fork EXEC:${sblame-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
 }