From 4acd35e0367523c90b0036bd15bcbecd9b6d5e14 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Oct 2023 08:58:07 +0200 Subject: [PATCH] Monitor PM webpage via blackbox --- m/hut/blackbox.yml | 160 +++++++++++++++++++++++++++++++++++++++++++ m/hut/monitoring.nix | 34 +++++++++ 2 files changed, 194 insertions(+) create mode 100644 m/hut/blackbox.yml diff --git a/m/hut/blackbox.yml b/m/hut/blackbox.yml new file mode 100644 index 0000000..232fb89 --- /dev/null +++ b/m/hut/blackbox.yml @@ -0,0 +1,160 @@ +modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_status_codes: [] # Defaults to 2xx + method: GET + http_with_proxy: + prober: http + http: + proxy_url: "http://127.0.0.1:3128" + skip_resolve_phase_with_proxy: true + http_with_proxy_and_headers: + prober: http + http: + proxy_url: "http://127.0.0.1:3128" + proxy_connect_header: + Proxy-Authorization: + - Bearer token + http_post_2xx: + prober: http + timeout: 5s + http: + method: POST + headers: + Content-Type: application/json + body: '{}' + http_post_body_file: + prober: http + timeout: 5s + http: + method: POST + body_file: "/files/body.txt" + http_basic_auth_example: + prober: http + timeout: 5s + http: + method: POST + headers: + Host: "login.example.com" + basic_auth: + username: "username" + password: "mysecret" + http_2xx_oauth_client_credentials: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2"] + follow_redirects: true + preferred_ip_protocol: "ip4" + valid_status_codes: + - 200 + - 201 + oauth2: + client_id: "client_id" + client_secret: "client_secret" + token_url: "https://api.example.com/token" + endpoint_params: + grant_type: "client_credentials" + http_custom_ca_example: + prober: http + http: + method: GET + tls_config: + ca_file: "/certs/my_cert.crt" + http_gzip: + prober: http + http: + method: GET + compression: gzip + http_gzip_with_accept_encoding: + prober: http + http: + method: GET + compression: gzip + headers: + Accept-Encoding: gzip + tls_connect: + prober: tcp + timeout: 5s + tcp: + tls: true + tcp_connect_example: + prober: tcp + timeout: 5s + imap_starttls: + prober: tcp + timeout: 5s + tcp: + query_response: + - expect: "OK.*STARTTLS" + - send: ". STARTTLS" + - expect: "OK" + - starttls: true + - send: ". capability" + - expect: "CAPABILITY IMAP4rev1" + smtp_starttls: + prober: tcp + timeout: 5s + tcp: + query_response: + - expect: "^220 ([^ ]+) ESMTP (.+)$" + - send: "EHLO prober\r" + - expect: "^250-STARTTLS" + - send: "STARTTLS\r" + - expect: "^220" + - starttls: true + - send: "EHLO prober\r" + - expect: "^250-AUTH" + - send: "QUIT\r" + irc_banner_example: + prober: tcp + timeout: 5s + tcp: + query_response: + - send: "NICK prober" + - send: "USER prober prober prober :prober" + - expect: "PING :([^ ]+)" + send: "PONG ${1}" + - expect: "^:[^ ]+ 001" + icmp_example: + prober: icmp + timeout: 5s + icmp: + preferred_ip_protocol: "ip4" + source_ip_address: "127.0.0.1" + dns_udp_example: + prober: dns + timeout: 5s + dns: + query_name: "www.prometheus.io" + query_type: "A" + valid_rcodes: + - NOERROR + validate_answer_rrs: + fail_if_matches_regexp: + - ".*127.0.0.1" + fail_if_all_match_regexp: + - ".*127.0.0.1" + fail_if_not_matches_regexp: + - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1" + fail_if_none_matches_regexp: + - "127.0.0.1" + validate_authority_rrs: + fail_if_matches_regexp: + - ".*127.0.0.1" + validate_additional_rrs: + fail_if_matches_regexp: + - ".*127.0.0.1" + dns_soa: + prober: dns + dns: + query_name: "prometheus.io" + query_type: "SOA" + dns_tcp_example: + prober: dns + dns: + transport_protocol: "tcp" # defaults to "udp" + preferred_ip_protocol: "ip4" # defaults to "ip6" + query_name: "www.prometheus.io" diff --git a/m/hut/monitoring.nix b/m/hut/monitoring.nix index 6c7d093..49ebf7c 100644 --- a/m/hut/monitoring.nix +++ b/m/hut/monitoring.nix @@ -64,6 +64,11 @@ enable = true; listenAddress = "127.0.0.1"; }; + blackbox = { + enable = true; + listenAddress = "127.0.0.1"; + configFile = ./blackbox.yml; + }; }; scrapeConfigs = [ @@ -77,6 +82,7 @@ "127.0.0.1:9252" "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}" "127.0.0.1:9341" # Slurm exporter + "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}" ]; }]; } @@ -90,6 +96,34 @@ ]; }]; } + { + job_name = "blackbox"; + metrics_path = "/probe"; + params = { module = [ "http_2xx" ]; }; + static_configs = [{ + targets = [ + "https://pm.bsc.es/" + "https://jungle.bsc.es/" + ]; + }]; + relabel_configs = [ + { + # Takes the address and sets it in the "target=" URL parameter + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + # Sets the "instance" label with the remote host we are querying + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + # Shows the host target address instead of the blackbox address + target_label = "__address__"; + replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"; + } + ]; + } { # Scrape the IPMI info of the hosts remotely via LAN job_name = "ipmi-lan";