diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index d4dc1e15..4f1352e0 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -4,6 +4,7 @@ imports = [ ../module/slurm-exporter.nix ./gpfs-probe.nix + ./nix-daemon-exporter.nix ]; age.secrets.grafanaJungleRobotPassword = { @@ -108,6 +109,7 @@ "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" "127.0.0.1:9341" # Slurm exporter "127.0.0.1:9966" # GPFS custom exporter + "127.0.0.1:9999" # Nix-daemon custom exporter "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }]; diff --git a/m/hut/nix-daemon-builds.sh b/m/hut/nix-daemon-builds.sh new file mode 100755 index 00000000..79ab65cd --- /dev/null +++ b/m/hut/nix-daemon-builds.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# Locate nix daemon pid +nd=$(pgrep -o nix-daemon) + +# Locate children of nix-daemon +pids1=$(tr ' ' '\n' < "/proc/$nd/task/$nd/children") + +# For each children, locate 2nd level children +pids2=$(echo "$pids1" | xargs -I @ /bin/sh -c 'cat /proc/@/task/*/children' | tr ' ' '\n') + +cat </dev/null | tr '\0' '\n' | rg "^name=(.+)" - --replace '$1' | tr -dc ' [:alnum:]_\-\.') + user=$(ps -o uname= -p "$pid") + if [ -n "$name" -a -n "$user" ]; then + printf 'nix_daemon_build{user="%s",name="%s"} 1\n' "$user" "$name" + fi +done diff --git a/m/hut/nix-daemon-exporter.nix b/m/hut/nix-daemon-exporter.nix new file mode 100644 index 00000000..9353fe7a --- /dev/null +++ b/m/hut/nix-daemon-exporter.nix @@ -0,0 +1,23 @@ +{ pkgs, config, lib, ... }: +let + script = pkgs.runCommand "nix-daemon-exporter.sh" { } + '' + cp ${./nix-daemon-builds.sh} $out; + chmod +x $out + '' + ; +in +{ + systemd.services.nix-daemon-exporter = { + description = "Daemon to export nix-daemon metrics"; + path = [ pkgs.procps pkgs.ripgrep ]; + wantedBy = [ "default.target" ]; + serviceConfig = { + Type = "simple"; + ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9999,fork EXEC:${script}"; + # Needed root to read the environment, potentially unsafe + User = "root"; + Group = "root"; + }; + }; +}