From f71e807d479f1c685601ba452e81c5101203fc3a Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Wed, 5 Nov 2025 13:27:09 +0100
Subject: [PATCH] Add remote sblame probe to prometheus
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Reviewed-by: Aleix Boné <abonerib@bsc.es>
---
 m/hut/monitoring.nix                          |  3 ++-
 m/hut/sblame-probe.sh                         |  8 ++++++++
 .../{gpfs-probe.nix => ssh-robot-probes.nix}  | 19 +++++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100755 m/hut/sblame-probe.sh
 rename m/hut/{gpfs-probe.nix => ssh-robot-probes.nix} (58%)

diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix
index 0c1cb78a..76de7ce0 100644
--- a/m/hut/monitoring.nix
+++ b/m/hut/monitoring.nix
@@ -5,7 +5,7 @@
     ../module/slurm-exporter.nix
     ../module/meteocat-exporter.nix
     ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
+    ./ssh-robot-probes.nix
     ../module/nix-daemon-exporter.nix
   ];
 
@@ -111,6 +111,7 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9967" # SLURM custom exporter
             "127.0.0.1:9999" # Nix-daemon custom exporter
             "127.0.0.1:9929" # Meteocat custom exporter
             "127.0.0.1:9928" # UPC Qaire custom exporter
diff --git a/m/hut/sblame-probe.sh b/m/hut/sblame-probe.sh
new file mode 100755
index 00000000..f8c02e2d
--- /dev/null
+++ b/m/hut/sblame-probe.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+EOF
+ssh bsc015557@glogin2.bsc.es "timeout 3 command sblame -E"
diff --git a/m/hut/gpfs-probe.nix b/m/hut/ssh-robot-probes.nix
similarity index 58%
rename from m/hut/gpfs-probe.nix
rename to m/hut/ssh-robot-probes.nix
index d4a0d986..a956e8ce 100644
--- a/m/hut/gpfs-probe.nix
+++ b/m/hut/ssh-robot-probes.nix
@@ -6,6 +6,12 @@ let
       chmod +x $out
     ''
   ;
+  sblame-probe-script = pkgs.runCommand "sblame-probe.sh" { }
+    ''
+      cp ${./sblame-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
 in
 {
   # Use a new user to handle the SSH keys
@@ -28,4 +34,17 @@ in
       Group = "ssh-robot";
     };
   };
+
+  systemd.services.sblame-probe = {
+    description = "Daemon to report SLURM statistics via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9967,fork EXEC:${sblame-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
 }
-- 
2.51.2