Add remote sblame probe to prometheus

Reviewed-by: Aleix Boné <abonerib@bsc.es>

m/hut/gitlab-runner.nix

@@ -51,6 +51,7 @@
           "/nix/store:/nix/store:ro"
           "/nix/var/nix/db:/nix/var/nix/db:ro"
           "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+          "/var/run/postgresql/:/var/run/postgresql/"
         ];
         dockerExtraHosts = [
           # Required to pass the proxy via hut

m/hut/monitoring.nix

@@ -5,7 +5,7 @@
     ../module/slurm-exporter.nix
     ../module/meteocat-exporter.nix
     ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
+    ./ssh-robot-probes.nix
     ../module/nix-daemon-exporter.nix
   ];
 
@@ -111,6 +111,7 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9967" # SLURM custom exporter
             "127.0.0.1:9999" # Nix-daemon custom exporter
             "127.0.0.1:9929" # Meteocat custom exporter
             "127.0.0.1:9928" # UPC Qaire custom exporter

m/hut/postgresql.nix

@@ -8,12 +8,14 @@
       { name = "anavarro"; ensureClauses.superuser = true; }
       { name = "rarias";   ensureClauses.superuser = true; }
       { name = "grafana"; }
+      { name = "gitlab-runner"; }
     ];
     authentication = ''
-      #type  database     DBuser    auth-method
+      #type  database     DBuser         auth-method
-      local  perftestsdb  rarias    trust
+      local  perftestsdb  rarias         trust
-      local  perftestsdb  anavarro  trust
+      local  perftestsdb  anavarro       trust
-      local  perftestsdb  grafana   trust
+      local  perftestsdb  grafana        trust
+      local  perftestsdb  gitlab-runner  trust
     '';
   };
 }

m/hut/sblame-probe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+EOF
+ssh bsc015557@glogin2.bsc.es "timeout 3 command sblame -E"

m/hut/ssh-robot-probes.nix

@@ -6,6 +6,12 @@ let
       chmod +x $out
     ''
   ;
+  sblame-probe-script = pkgs.runCommand "sblame-probe.sh" { }
+    ''
+      cp ${./sblame-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
 in
 {
   # Use a new user to handle the SSH keys
@@ -28,4 +34,17 @@ in
       Group = "ssh-robot";
     };
   };
+
+  systemd.services.sblame-probe = {
+    description = "Daemon to report SLURM statistics via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9967,fork EXEC:${sblame-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
 }