From aae6585f6644fa2ef54b8c2737cf78cb03ee982a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 23 May 2025 15:40:09 +0200 Subject: [PATCH 1/2] Add meteocat exporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allows us to track ambient temperature changes and estimate the temperature delta between the server room and exterior temperature. We should be able to predict when we would need to stop the machines due to excesive temperature as summer approaches. Reviewed-by: Aleix Boné --- m/hut/monitoring.nix | 2 + m/module/meteocat-exporter.nix | 17 ++++++++ pkgs/meteocat-exporter/default.nix | 25 +++++++++++ pkgs/meteocat-exporter/meteocat-exporter | 54 ++++++++++++++++++++++++ pkgs/meteocat-exporter/setup.py | 11 +++++ pkgs/overlay.nix | 1 + 6 files changed, 110 insertions(+) create mode 100644 m/module/meteocat-exporter.nix create mode 100644 pkgs/meteocat-exporter/default.nix create mode 100644 pkgs/meteocat-exporter/meteocat-exporter create mode 100644 pkgs/meteocat-exporter/setup.py diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 4f1352e0..8f1bea44 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -3,6 +3,7 @@ { imports = [ ../module/slurm-exporter.nix + ../module/meteocat-exporter.nix ./gpfs-probe.nix ./nix-daemon-exporter.nix ]; @@ -110,6 +111,7 @@ "127.0.0.1:9341" # Slurm exporter "127.0.0.1:9966" # GPFS custom exporter "127.0.0.1:9999" # Nix-daemon custom exporter + "127.0.0.1:9929" # Meteocat custom exporter "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }]; diff --git a/m/module/meteocat-exporter.nix b/m/module/meteocat-exporter.nix new file mode 100644 index 00000000..ffc03385 --- /dev/null +++ b/m/module/meteocat-exporter.nix @@ -0,0 +1,17 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + systemd.services."prometheus-meteocat-exporter" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + Restart = mkDefault "always"; + PrivateTmp = mkDefault true; + WorkingDirectory = mkDefault "/tmp"; + DynamicUser = mkDefault true; + ExecStart = "${pkgs.meteocat-exporter}/bin/meteocat-exporter"; + }; + }; +} diff --git a/pkgs/meteocat-exporter/default.nix b/pkgs/meteocat-exporter/default.nix new file mode 100644 index 00000000..5bc4f096 --- /dev/null +++ b/pkgs/meteocat-exporter/default.nix @@ -0,0 +1,25 @@ +{ python3Packages, lib }: + +python3Packages.buildPythonApplication rec { + pname = "meteocat-exporter"; + version = "1.0"; + + src = ./.; + + doCheck = false; + + build-system = with python3Packages; [ + setuptools + ]; + + dependencies = with python3Packages; [ + beautifulsoup4 + lxml + prometheus-client + ]; + + meta = with lib; { + description = "MeteoCat Prometheus Exporter"; + platforms = platforms.linux; + }; +} diff --git a/pkgs/meteocat-exporter/meteocat-exporter b/pkgs/meteocat-exporter/meteocat-exporter new file mode 100644 index 00000000..acc9f3e3 --- /dev/null +++ b/pkgs/meteocat-exporter/meteocat-exporter @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import time +from prometheus_client import start_http_server, Gauge +from bs4 import BeautifulSoup +from urllib import request + +# Configuration ------------------------------------------- +meteo_station = "X8" # Barcelona - Zona Universitària +listening_port = 9929 +update_period = 60 * 5 # Each 5 min +# --------------------------------------------------------- + +metric_tmin = Gauge('meteocat_temp_min', 'Min temperature') +metric_tmax = Gauge('meteocat_temp_max', 'Max temperature') +metric_tavg = Gauge('meteocat_temp_avg', 'Average temperature') +metric_srad = Gauge('meteocat_solar_radiation', 'Solar radiation') + +def update(st): + url = 'https://www.meteo.cat/observacions/xema/dades?codi=' + st + response = request.urlopen(url) + data = response.read() + soup = BeautifulSoup(data, 'lxml') + table = soup.find("table", {"class" : "tblperiode"}) + rows = table.find_all('tr') + row = rows[-1] # Take the last row + row_data = [] + header = row.find('th') + header_text = header.text.strip() + row_data.append(header_text) + for col in row.find_all('td'): + row_data.append(col.text) + try: + # Sometimes it will return '(s/d)' and fail to parse + metric_tavg.set(float(row_data[1])) + metric_tmax.set(float(row_data[2])) + metric_tmin.set(float(row_data[3])) + metric_srad.set(float(row_data[10])) + #print("ok: temp_avg={}".format(float(row_data[1]))) + except: + print("cannot parse row: {}".format(row)) + metric_tavg.set(float("nan")) + metric_tmax.set(float("nan")) + metric_tmin.set(float("nan")) + metric_srad.set(float("nan")) + +if __name__ == '__main__': + start_http_server(port=listening_port, addr="localhost") + while True: + try: + update(meteo_station) + except: + print("update failed") + time.sleep(update_period) diff --git a/pkgs/meteocat-exporter/setup.py b/pkgs/meteocat-exporter/setup.py new file mode 100644 index 00000000..9cc74d3d --- /dev/null +++ b/pkgs/meteocat-exporter/setup.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +setup(name='meteocat-exporter', + version='1.0', + # Modules to import from other scripts: + packages=find_packages(), + # Executables + scripts=["meteocat-exporter"], + ) diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 2eb92295..995ceebe 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -54,4 +54,5 @@ final: prev: }); prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { }; + meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { }; } -- 2.51.2 From 7a2f37aaa2d2fce24827f52b6e175c0d1c0632ab Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 26 May 2025 11:24:12 +0200 Subject: [PATCH 2/2] Add UPC temperature sensor monitoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These sensors are part of their air quality measurements, which just happen to be very close to our server room. Reviewed-by: Aleix Boné --- m/hut/monitoring.nix | 2 + m/module/upc-qaire-exporter.nix | 17 +++++ pkgs/overlay.nix | 1 + pkgs/upc-qaire-exporter/default.nix | 24 +++++++ pkgs/upc-qaire-exporter/setup.py | 11 ++++ pkgs/upc-qaire-exporter/upc-qaire-exporter | 74 ++++++++++++++++++++++ 6 files changed, 129 insertions(+) create mode 100644 m/module/upc-qaire-exporter.nix create mode 100644 pkgs/upc-qaire-exporter/default.nix create mode 100644 pkgs/upc-qaire-exporter/setup.py create mode 100644 pkgs/upc-qaire-exporter/upc-qaire-exporter diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 8f1bea44..7042c913 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -4,6 +4,7 @@ imports = [ ../module/slurm-exporter.nix ../module/meteocat-exporter.nix + ../module/upc-qaire-exporter.nix ./gpfs-probe.nix ./nix-daemon-exporter.nix ]; @@ -112,6 +113,7 @@ "127.0.0.1:9966" # GPFS custom exporter "127.0.0.1:9999" # Nix-daemon custom exporter "127.0.0.1:9929" # Meteocat custom exporter + "127.0.0.1:9928" # UPC Qaire custom exporter "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }]; diff --git a/m/module/upc-qaire-exporter.nix b/m/module/upc-qaire-exporter.nix new file mode 100644 index 00000000..ddb27eba --- /dev/null +++ b/m/module/upc-qaire-exporter.nix @@ -0,0 +1,17 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + systemd.services."prometheus-upc-qaire-exporter" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + Restart = mkDefault "always"; + PrivateTmp = mkDefault true; + WorkingDirectory = mkDefault "/tmp"; + DynamicUser = mkDefault true; + ExecStart = "${pkgs.upc-qaire-exporter}/bin/upc-qaire-exporter"; + }; + }; +} diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 995ceebe..2a1df4a2 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -55,4 +55,5 @@ final: prev: prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { }; meteocat-exporter = prev.callPackage ./meteocat-exporter/default.nix { }; + upc-qaire-exporter = prev.callPackage ./upc-qaire-exporter/default.nix { }; } diff --git a/pkgs/upc-qaire-exporter/default.nix b/pkgs/upc-qaire-exporter/default.nix new file mode 100644 index 00000000..b5c14cb2 --- /dev/null +++ b/pkgs/upc-qaire-exporter/default.nix @@ -0,0 +1,24 @@ +{ python3Packages, lib }: + +python3Packages.buildPythonApplication rec { + pname = "upc-qaire-exporter"; + version = "1.0"; + + src = ./.; + + doCheck = false; + + build-system = with python3Packages; [ + setuptools + ]; + + dependencies = with python3Packages; [ + prometheus-client + requests + ]; + + meta = with lib; { + description = "UPC Qaire Prometheus Exporter"; + platforms = platforms.linux; + }; +} diff --git a/pkgs/upc-qaire-exporter/setup.py b/pkgs/upc-qaire-exporter/setup.py new file mode 100644 index 00000000..e2238a7c --- /dev/null +++ b/pkgs/upc-qaire-exporter/setup.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages + +setup(name='upc-qaire-exporter', + version='1.0', + # Modules to import from other scripts: + packages=find_packages(), + # Executables + scripts=["upc-qaire-exporter"], + ) diff --git a/pkgs/upc-qaire-exporter/upc-qaire-exporter b/pkgs/upc-qaire-exporter/upc-qaire-exporter new file mode 100644 index 00000000..39697aa6 --- /dev/null +++ b/pkgs/upc-qaire-exporter/upc-qaire-exporter @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import time +from prometheus_client import start_http_server, Gauge +import requests, json +from datetime import datetime, timedelta + +# Configuration ------------------------------------------- +listening_port = 9928 +update_period = 60 * 5 # Each 5 min +# --------------------------------------------------------- + +metric_temp = Gauge('upc_c6_s302_temp', 'UPC C6 S302 temperature sensor') + +def genparams(): + d = {} + d['topic'] = 'TEMPERATURE' + d['shift_dates_to'] = '' + d['datapoints'] = 301 + d['devicesAndColors'] = '1148418@@@#40ACB6' + + now = datetime.now() + + d['fromDate'] = now.strftime('%d/%m/%Y') + d['toDate'] = now.strftime('%d/%m/%Y') + d['serviceFrequency'] = 'NONE' + + # WTF! + for i in range(7): + for j in range(48): + key = 'week.days[{}].hours[{}].value'.format(i, j) + d[key] = 'OPEN' + + return d + +def measure(): + # First we need to load session + s = requests.Session() + r = s.get("https://upc.edu/sirena") + if r.status_code != 200: + print("bad HTTP status code on new session: {}".format(r.status_code)) + return + + if s.cookies.get("JSESSIONID") is None: + print("cannot get JSESSIONID") + return + + # Now we can pull the data + url = "https://upcsirena.app.dexma.com/l_12535/analysis/by_datapoints/data.json" + r = s.post(url, data=genparams()) + + if r.status_code != 200: + print("bad HTTP status code on data: {}".format(r.status_code)) + return + + #print(r.text) + j = json.loads(r.content) + + # Just take the last one + last = j['data']['chartElementList'][-1] + temp = last['values']['1148418-Temperatura'] + + return temp + +if __name__ == '__main__': + start_http_server(port=listening_port, addr="localhost") + while True: + try: + metric_temp.set(measure()) + except: + print("measure failed") + metric_temp.set(float("nan")) + + time.sleep(update_period) -- 2.51.2