Add remote sblame probe to prometheus

Reviewed-by: Aleix Boné <abonerib@bsc.es>

keys.nix

@@ -22,8 +22,9 @@ rec {
     storage    = [ bay lake2 ];
     monitor    = [ hut ];
     login      = [ apex ];
+    services   = [ tent ];
 
-    system     = storage ++ monitor ++ login;
+    system     = storage ++ monitor ++ login ++ services;
     safe       = system ++ compute;
     all        = safe ++ playground;
   };

m/apex/nfs.nix

@@ -7,7 +7,7 @@
     mountdPort = 4002;
     statdPort = 4000;
     exports = ''
-      /home 10.0.40.0/24(rw,async,no_subtree_check,no_root_squash)
+      /home 10.0.40.0/21(rw,async,no_subtree_check,no_root_squash)
       /home 10.106.0.0/24(rw,async,no_subtree_check,no_root_squash)
     '';
   };
@@ -15,19 +15,19 @@
     # Check with `rpcinfo -p`
     extraCommands = ''
       # Accept NFS traffic from compute nodes but not from the outside
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
-      iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p tcp -s 10.0.40.0/21 --dport 20048 -j nixos-fw-accept
       # Same but UDP
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 111   -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 2049  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4000  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4001  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 4002  -j nixos-fw-accept
-      iptables -A nixos-fw -p udp -s 10.0.40.0/24 --dport 20048 -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 111   -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 2049  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4000  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4001  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 4002  -j nixos-fw-accept
+      iptables -A nixos-fw -p udp -s 10.0.40.0/21 --dport 20048 -j nixos-fw-accept
 
       # Accept NFS traffic from wg0
       iptables -A nixos-fw -p tcp -i wg0 -s 10.106.0.0/24 --dport 111   -j nixos-fw-accept

m/bay/configuration.nix

@@ -35,7 +35,7 @@
         # Accept monitoring requests from hut
         iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
         # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/21 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
       '';
     };
   };

m/common/base/users.nix

@@ -207,6 +207,19 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOKZKot/Y3F5Wq9pQIXlCbyvQuVVeWMCsAC96Nd+LTcG erwin@Oreo"
         ];
       };
+
+      ssanzmar = {
+        uid = 9657;
+        isNormalUser = true;
+        home = "/home/Computational/ssanzmar";
+        description = "Sergio Sanz Martínez";
+        group = "Computational";
+        hosts = [ "apex" "fox" ];
+        hashedPassword = "$6$HUjNDJeJMmNQ6M64$laXSOZcXg6o4v2r8Jm8Xj9kmqw7veCY32po3TVDPRR4WlyxvOeqwoKr4NjlUlPPpKN55Oot3ZYHi.9iNXsH5E1";
+        openssh.authorizedKeys.keys = [
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIELrsRRHXryrdA2ZBx5XmdGxL4DC5bmJydhBeTWQ0SQ sergio.sanz.martinez@estudiantat.upc.edu"
+        ];
+      };
     };
 
     groups = {

m/hut/gitlab-runner.nix

@@ -51,6 +51,7 @@
           "/nix/store:/nix/store:ro"
           "/nix/var/nix/db:/nix/var/nix/db:ro"
           "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+          "/var/run/postgresql/:/var/run/postgresql/"
         ];
         dockerExtraHosts = [
           # Required to pass the proxy via hut

m/hut/monitoring.nix

@@ -5,7 +5,7 @@
     ../module/slurm-exporter.nix
     ../module/meteocat-exporter.nix
     ../module/upc-qaire-exporter.nix
-    ./gpfs-probe.nix
+    ./ssh-robot-probes.nix
     ../module/nix-daemon-exporter.nix
   ];
 
@@ -111,6 +111,7 @@
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
             "127.0.0.1:9966" # GPFS custom exporter
+            "127.0.0.1:9967" # SLURM custom exporter
             "127.0.0.1:9999" # Nix-daemon custom exporter
             "127.0.0.1:9929" # Meteocat custom exporter
             "127.0.0.1:9928" # UPC Qaire custom exporter

m/hut/postgresql.nix

@@ -8,12 +8,14 @@
       { name = "anavarro"; ensureClauses.superuser = true; }
       { name = "rarias";   ensureClauses.superuser = true; }
       { name = "grafana"; }
+      { name = "gitlab-runner"; }
     ];
     authentication = ''
-      #type  database     DBuser    auth-method
-      local  perftestsdb  rarias    trust
-      local  perftestsdb  anavarro  trust
-      local  perftestsdb  grafana   trust
+      #type  database     DBuser         auth-method
+      local  perftestsdb  rarias         trust
+      local  perftestsdb  anavarro       trust
+      local  perftestsdb  grafana        trust
+      local  perftestsdb  gitlab-runner  trust
     '';
   };
 }

m/hut/sblame-probe.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cat <<EOF
+HTTP/1.1 200 OK
+Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
+
+EOF
+ssh bsc015557@glogin2.bsc.es "timeout 3 command sblame -E"

m/hut/ssh-robot-probes.nix

@@ -6,6 +6,12 @@ let
       chmod +x $out
     ''
   ;
+  sblame-probe-script = pkgs.runCommand "sblame-probe.sh" { }
+    ''
+      cp ${./sblame-probe.sh} $out;
+      chmod +x $out
+    ''
+  ;
 in
 {
   # Use a new user to handle the SSH keys
@@ -28,4 +34,17 @@ in
       Group = "ssh-robot";
     };
   };
+
+  systemd.services.sblame-probe = {
+    description = "Daemon to report SLURM statistics via SSH";
+    path = [ pkgs.openssh pkgs.netcat ];
+    after = [ "network.target" ];
+    wantedBy = [ "default.target" ];
+    serviceConfig = {
+      Type = "simple";
+      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9967,fork EXEC:${sblame-probe-script}";
+      User = "ssh-robot";
+      Group = "ssh-robot";
+    };
+  };
 }

m/lake2/configuration.nix

@@ -57,7 +57,7 @@
         # Accept monitoring requests from hut
         iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
         # Accept all Ceph traffic from the local network
-        iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
+        iptables -A nixos-fw -p tcp -s 10.0.40.0/21 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
       '';
     };
   };

m/tent/configuration.nix

@@ -17,6 +17,7 @@
     ../module/vpn-dac.nix
     ../module/hut-substituter.nix
     ../module/tc1-board.nix
+    ../module/ceph.nix
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -64,6 +65,13 @@
     fsType = "ext4";
   };
 
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
   # Make a /vault/$USER directory for each user.
   systemd.services.create-vault-dirs = let
     # Take only normal users in tent

m/tent/gitea.nix

@@ -1,4 +1,7 @@
 { config, lib, ... }:
+let
+  cfg = config.services.gitea;
+in
 {
   services.gitea = {
     enable = true;
@@ -26,6 +29,52 @@
         SENDMAIL_ARGS = "--";
       };
     };
+
+    dump = {
+      enable = false; # Do not enable NixOS module, use our custom systemd script below
+      backupDir = "/vault/backup/gitea";
+    };
+  };
+
+  systemd.services.gitea-backup = let
+    exe = lib.getExe cfg.package;
+  in {
+    description = "Gitea daily backup";
+    after = [ "gitea.service" ];
+    path = [ cfg.package ];
+
+    environment = {
+      USER = cfg.user;
+      HOME = cfg.stateDir;
+      GITEA_WORK_DIR = cfg.stateDir;
+      GITEA_CUSTOM = cfg.customDir;
+    };
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = cfg.user;
+      WorkingDirectory = cfg.dump.backupDir;
+    };
+
+    script = ''
+      name="gitea-dump-$(date +%a).${cfg.dump.type}"
+      ${exe} dump --type ${cfg.dump.type} --file - >"$name.tmp"
+      mv "$name.tmp" "$name"
+      cp "$name" "/ceph/backup/gitea/$name"
+    '';
+  };
+
+  # Create also the /ceph directories if needed
+  systemd.tmpfiles.rules = [
+    "d /ceph/backup/gitea/ 0750 ${cfg.user} ${cfg.group} - -"
+    "z /ceph/backup/gitea/ 0750 ${cfg.user} ${cfg.group} - -"
+  ];
+
+  systemd.timers.gitea-backup = {
+    description = "Update timer for gitea-backup";
+    partOf = [ "gitea-backup.service" ];
+    wantedBy = [ "timers.target" ];
+    timerConfig.OnCalendar = cfg.dump.interval;
   };
 
   # Allow gitea user to send mail

m/tent/gitlab-runner.nix

@@ -43,6 +43,7 @@
         registrationFlags = [
           # Increase build log length to 64 MiB
           "--output-limit 65536"
+          "--docker-network-mode host"
         ];
         preBuildScript = pkgs.writeScript "setup-container" ''
           mkdir -p -m 0755 /nix/var/log/nix/drvs

m/weasel/configuration.nix

@@ -25,7 +25,7 @@
       address = "10.0.40.6";
       prefixLength = 24;
     } ];
-    interfaces.ibp5s0.ipv4.addresses = [ {
+    interfaces.ibs785.ipv4.addresses = [ {
       address = "10.0.42.6";
       prefixLength = 24;
     } ];

secrets/ceph-user.age

@@ -1,25 +1,29 @@
 age-encryption.org/v1
--> ssh-ed25519 AY8zKw /gmhFOFqOs8IobAImvQVKeM5Y6k0FpuR61/Cu5drVVI
-g9FXJg2oIoien0zJ70FWHwSTM8SBwbpS188S3Swj7EM
--> ssh-ed25519 sgAamA opPjlWPhSiI0Rd5l7kd204S5FXFLcQcQftyKb7MDmnU
-3XrRDVnglCP+vBwvfd1rP5gHttsGDHyXwbf10a8/kKY
--> ssh-ed25519 HY2yRg QKZbubM76C3tobPoyCFDRclA9Pzb2fC7s4WOoIgdORc
-K5kckU0KhQFTE6SikJXFJgM41Tco5+VqOsaG0qLrY1Q
--> ssh-ed25519 fw2Xhg +ohqts8dLFjvdHxrGHcOGxU0dm+V3N//giljHkobpDM
-jR/UzGrfS9lrJ/VeolKLxfzeJAf2fIB2pdIn/6ukqNk
--> ssh-ed25519 tcumPQ 3DPkDPIQQSVtXSLzIRETsIyXQ0k1o18Evn6vf+l/6R8
-bLXF62OmJjnOT1vvgq3+AcOKKSG5NonrK5EqCVc0Mwo
--> ssh-ed25519 JJ1LWg 2Wefc7eLolMU5InEmCNTq21Mf71mI0a2N1HgDrlHvy4
-qXFW9CQBnrzubZ0mzS0Io2WGRrwGBkmeYndBTcZn/fM
--> ssh-ed25519 cDBabA oiH36AoIt/fFFYgnoxtH7OoetP+2/wjtn8qo3RJDSHc
-qKmkxy1aZGP4ZwC0iH7n7hiJ0+rFQYvjQb5O1a1Z0r4
--> ssh-ed25519 cK5kHw bX3RtO5StMejUYWAaA37fjHA5nO7Xs1vWDQk3yOjs2o
-Egxmcf8FKAd+E5hMLmhV1yQsCo5rJyUazf1szOvpTAM
--> ssh-ed25519 CAWG4Q oKqqRDJH0w8lsoQBQk0w8PO+z5gFNmSaGBUSumvDp1I
-m1zWp9MfViAmtpbJhqOHraIokDaPKb0DvvO4vAGCTWI
--> ssh-ed25519 xA739A G26kPOz6sbFATs+KAr7gbDvji13eA1smFusQAOJXMwA
-Sppvz7A103kZoNxoGsd6eXeCvVh7mBE2MRwLFj9O1dY
--> ssh-ed25519 MSF3dg 55ekNcp+inbUd+GQ/VZ7BoBASaJ8YDqF74CVXy1PUxQ
-aTHLLAbzQPWWld/OT3BKebc6FcmsqMTaWCPBGm1UHic
---- mVkAMnI9XQhS3fMiFuuXP/yLR9wEG9+Rr8pA4Uc0avY
-<04>DU <20><>s<EFBFBD><73><EFBFBD><EFBFBD>j<EFBFBD><6A>M<EFBFBD><4D>$<24>[<5B>M<EFBFBD><4D><EFBFBD><03>[_<>K7s<37>ju<>v<EFBFBD>D<EFBFBD>4<EFBFBD>g<EFBFBD><67>܄3<>Gn<47><6E><EFBFBD> ɽ<>P<EFBFBD>7~rZs<><73>
+-> ssh-ed25519 AY8zKw Crgof1PMHzv3jBw8VeJAst6FKSoyqPFdANFpf79CAgo
+7fagE5BmlWdTsdY/i3RbExu1KBcjW1LQXbYwu6chxlk
+-> ssh-ed25519 sgAamA tGRCaK8mjvz65YziXjRcjMOHIRoyGNJFzBEEbivXPDo
+YLzE5a3J81r+gzkfZIeh9gS+mXzMooC82tBbZ+C3C8o
+-> ssh-ed25519 HY2yRg +vhO1/vdGPM1JnZRsvVnViFWaFWUZ7MIqvWdePivkxA
+2K+JdN82DTeGh9QwZBTaghg8C5BCLoEsOgTCM64PU28
+-> ssh-ed25519 fw2Xhg NHDn0dq32I/AVdUZlpzBX6retlEYEUipde7A9R90qW4
+SJO78ooqEwfHlBRW+YCzgSQJb1JHNo8jz37t3qvLClE
+-> ssh-ed25519 G5LX5w d4HfLzI2623artkR2FIfRJgr5yb2BKZJUWqPnwOWDCk
+Kh50QESJZSjaJPyp3xroHGn0fD5pPNEYgKkDdqxGpjs
+-> ssh-ed25519 tcumPQ wQyOKtT15Qezs3cyv5/xxIPVD7Jyk6N6ZLkfxxBHLTo
+rKlRBjJdfDVT6U8211+ssFF8yY9yRs1u3GhCSvsw2oE
+-> ssh-ed25519 JJ1LWg 98tF1MdA244xNny4w3RnMFuubf4WcuQaZf2bN2Uq8Qc
+MA1Xh1H9vHisVYdqkxNeBkngtn8cYuT2eSimvooIXYo
+-> ssh-ed25519 cDBabA imJ0rXLQETELP7yo3sArhqA9nJwY+S6gkC7tA7CJsQA
+pKMHW/KDAoEj5ZD64VKekg6et9hlS2PKSgDw3eB3eu8
+-> ssh-ed25519 WY7yGw +2g5021/02HvLxLqq42ynr6qKgOKJ3J5GgB1a1bmFXg
+fYvj52R6bM6ngPOZ2lwVezTJnx+8LJBbdnaapKKbyd0
+-> ssh-ed25519 cK5kHw fLZ6yF3NggJ724rjYqhs5ZZh1xUExuK+ITAyqONluzk
+NS9OMX70XEHrbPQnmC4KB/eoiHChIb8DwDLYJiwOLUU
+-> ssh-ed25519 CAWG4Q tVduE/wMzdfS+DjNbU3Q4blNhL/A63IehNSZGJkJjD0
+jEBB5zG+gLA/88YF+KqWQsNH7lfCsWNvAkrgfbescFs
+-> ssh-ed25519 xA739A ZhFvev77I+YOl1YSHKn2ZcEvGoLjWOILufjd4q/k8HM
+YXEtHHtjPQlgZW60zHgHm7CLI6vYiRo+AM8QERL9tCg
+-> ssh-ed25519 MSF3dg 9DvLNheBU1vlfW2zNNxBrGnJ6k4P5ox7s+OGKlgRdyQ
+wseHfLGHz0huNi5sZsNOfeNkm6Kjjx0SZ8lK4/oXtUQ
+--- bnJE+14onuSla0XmckD4z/wChWGZh6exbkcbyhcmNYU
+<EFBFBD><EFBFBD>t<>N猈<><10>U<EFBFBD>w▮i2<69><32>-<2D>iV'(<1E>IF<49><46>	S<><53>xs/s<><73>	<09><>NDm<44>Q<EFBFBD><51><EFBFBD>o<EFBFBD><6F><EFBFBD><EFBFBD>wZv<7F><76>.\

secrets/gitlab-runner-docker-token.age

@@ -1,13 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg U2KQWviZIVNemm9e8h7H+eOzoYNxXgLLS3hsZLMAuGk
-6n5dH1McNzk3rscP4v2pqZYDWtUFMd15rZsEd/mqIFM
--> ssh-ed25519 cK5kHw Ebrj/cpz1cFWAYAV9OxgyyH85OEMUnfUIV66p7jaoFY
-6J7hWqODtS/fIF4BpxhxbrxZq5vbolvbLqRKqazT02M
--> ssh-ed25519 CAWG4Q mXqoQH9ycHF7u0y8mazCgynHxNLxTnrmQHke+2a5QCc
-mq6PdSF+KOqthuXwzTCsOQsi5KG0z1wHUck+bSTyOBY
--> ssh-ed25519 xA739A TADeswueqDEroZWLjMw3RDNwVQ2xRD+JUMVZENovn0M
-KFlnSjVFbjc+ZsbY8Ed7edC5B01TJGzd/dSryiLArPc
--> ssh-ed25519 MSF3dg Pq+ZD8AqJGDHDbd4PO1ngNFST8+6C2ghZkO/knKzzEc
-wyiL/u38hdQMokmfTsBrY7CtYwc+31FG4EDaqVEn31U
---- 1z4cOipayh0zYkvasEVEvGreajegE/dqBV7b6E7aFh0
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD>R<EFBFBD>@<40>/i<>I'<27><><EFBFBD>Nx<4E>r"<1D>`<1E>O<EFBFBD><4F><EFBFBD>y<><79>8<EFBFBD><38> \/<2F><>I<19><17>D<EFBFBD>`<60>ߓ<EFBFBD><DF93><EFBFBD><1E><04>uy<75><79><EFBFBD>:9Lt<4C><1D><><EFBFBD>؋<EFBFBD><D88B><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>AU<41><55><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>`<60>;<3B>q8<71>GLU#<23>i<EFBFBD>y<EFBFBD><79>i<03>ڜ
+-> ssh-ed25519 HY2yRg eHM55QsHK1ca9b5nP3EoVUZYu0w2d4B5tkilNK0j/lw
+6Na6lkMe0fOd7+vNP1fLIaVEQDUw5m65Wh8jUH1I6C0
+-> ssh-ed25519 cK5kHw 0ekhoBYwF7OSWwn4P5f/J4gXb9UHJAWGKV0yI7HCzzE
+2Q+Tt5jXAB9ip9jf1z+jeM4FSiqd1w5DNtbqtacuOcM
+-> ssh-ed25519 CAWG4Q Jmw4v9efOFXHjjNky96q/d6vGBP5dNM4wK9zoGrwOh8
+u5I17wcIq8/2ARWckDXsYckhfX0jWE4AEm5mip/KHws
+-> ssh-ed25519 xA739A 10pPeC2YG9DJzaQlt7p+fGo27VDiL2dN6JmvY2npcUw
+4aRV8DekYeL9HagGWgOSjlYnPKmYdKZH8Aw4lRdm+r8
+-> ssh-ed25519 MSF3dg hDwIE3Su6cN3sq2E5v/oy6vTNfxTT1ZPts85//gIhwY
+aoiaGjQYJB1ededhIuVBCKDRLIOVThWz1pSTvg65J3Y
+--- OYPAGb5U/nwLOIV5VchSvxhChjNnwzbEgU9glSkWCl4
+<EFBFBD>=<3D><><EFBFBD>c<EFBFBD>WȟJSaІ&<26><1F>ቧ)E<><0B> C<><43>J~u<>c<63><7F>2<EFBFBD><32>v<EFBFBD><76><EFBFBD><03><>s<EFBFBD><73><EFBFBD>vf<76><10><>X7(<28>~<7E><1A>=XCi;<3B>״<EFBFBD>\ߢ<><DFA2><EFBFBD>ܣ<EFBFBD><10><><07>ɳCe<43>D;;X*<2A>3<EFBFBD>i<EFBFBD><69>r<EFBFBD>Em<45><6D><

secrets/vpn-dac-login.age

@@ -1,14 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 G5LX5w SRJhNenoQXbT1FgX3TMPnVH5P6oe2eHot+M1YsEjsEk
-hfTSLgKi98Eh7JK5o7x2POpTEtQlQCpEa3keUFYCuME
--> ssh-ed25519 cK5kHw z5TwWJTkvx7HztjXHJW/aCOtOfPrQaLP0gyIT7rXcyU
-b4NCpHfasgvkLLr+6LcWUl60p59aSNnfp3bl2OFYXo0
--> ssh-ed25519 CAWG4Q 4VpS1/OnFe8nxcQbRTKNhjsh/ZQ5cbhSMXwK/jjQ+3o
-WF9wvOkqVml4UcEzyzeumKuUwCwwr2zvKLMg+PCB8nk
--> ssh-ed25519 xA739A 67FhuJ070jBVMt/xbKHWhfri6iIm0FyaFvzQabsvFBM
-1G5/913dDv/r/6p1x/c5YiUnZzrX/LvIj33KW+PN0KU
--> ssh-ed25519 MSF3dg Bj/yB4N2wkyHCHC22tcjjJAA4ebSamN0Z4UVX3ZnryI
-6D/ZgTs+j+MGDAbPU5zyK0i9zN6tQy68IcOnQZ27mYg
---- 169erk3ICSYLs4FPEuXCn7QlekWhsmSn0Lr+/R14I5Q
-<EFBFBD><EFBFBD><EFBFBD><EFBFBD><05>ҽ3<D2BD>s<EFBFBD>
-w<EFBFBD><EFBFBD>4D<EFBFBD><EFBFBD>b.<2E><><EFBFBD>"|<7C><><EFBFBD>)"<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>;<3B>.<2E>ɫ7)<29>LeC<05>=S؟
+-> ssh-ed25519 G5LX5w /9lcJOXC9CN02+XLswUaJ0H7jU6Xhjd8Xg4+KY0l1Vc
+fCLzsLc9zrocM8SHOKyZwt6eUEr8r1WLug9RLi63KU0
+-> ssh-ed25519 cK5kHw 1qza6h2NRSs4g8LYdFU7E+Dn1CgdtCU7DPdYInP1GwM
+/6uk7pTFkNTRTI7nA+x4y4CyOBVQVXX2lnpOg3ktPe4
+-> ssh-ed25519 CAWG4Q o+vyzcejSaNVYPSGzzOdzaqPByZ6zA1uaJf4KOg+wQA
+wfZmWrDSfRV8C+Hu+SeZDcomf/qigBqxuQK77SfnuEo
+-> ssh-ed25519 xA739A +rBsOC+IBE3lmc/pfrziftLIqMSyaGMsggRjC5Pqwl0
+xa7ulLz2+YC3g2hu7e9XhRYDIUb2sriaaigJRYF2oB8
+-> ssh-ed25519 MSF3dg TK6PmKjjQt8ni0mJLCt7P41lUsgimlj3o5Q6n3N+DE4
+ne+s3ctcg8cBjY06LY2lrW7wcxomvKHxu6MlirEA8Kg
+--- eorg2ckkUZ1Ogi4iTTg2MoiVBwl1F0RCmH2D8N1d1So
+<EFBFBD><EFBFBD><EFBFBD>8<1C><><EFBFBD><EFBFBD><EFBFBD><12>i<17>$]K<>J=2Z<1D><>ӼF<D3BC>][<14><><EFBFBD>8<EFBFBD><38>ޤ<12>	<09>=<3D><>
LD/<2F>gz

secrets/wg-fox.age

@@ -1,14 +1,13 @@
 age-encryption.org/v1
--> ssh-ed25519 cDBabA heyW9/cxgwFX9IexQIXjAQDWGQPNcMXcArQp2Rxsqx4
-o9MQ7EH8PDDjsJdpH9F3Xq2zUoaDAJQlfFmYucSFs6Y
--> ssh-ed25519 cK5kHw Sza4pos7K3qW3omEeyidI/jszJNf9smemSZnUJfCIww
-D6vazXki7hIYraIuSiGPS+FPbkFUwHhHWDf52OhEIMg
--> ssh-ed25519 CAWG4Q YexIHueOIMmIN8JIDyNUOKBkyz/k18HqV3hTXh48KlM
-xh8UJzzWT6ByN+Dpn4JrMNsjGC/uc/v6LynwjBDz9NQ
--> ssh-ed25519 xA739A KySG3TXdqfCMUkVEDGa74B0op745s3XGYxFLyAXSQAc
-5EI/yb5ctW9Qu18bHm3/sK97kwGcKzzmWvPSCWm89XA
--> ssh-ed25519 MSF3dg MNxnNj0fHmri8ophexXPNjRUBUWrzcuk5S1mucxUMTE
-GVFWXtISEU8ZmlwL4nh4weAgfGrt2GHX0DTzbpS6zg8
---- UdrqkYG2ZApAuwdZeNhC50NP2rkD/Ol6y8nJa4RHx7Y
-<EFBFBD>ܻ<EFBFBD>m(<28><><EFBFBD>><3E>H<48>Y87<><37>G<0F>+*<12><><EFBFBD><EFBFBD>9V<>.<2E><><EFBFBD><EFBFBD><03><><EFBFBD>p<EFBFBD>Oo<4F>=+哇<>P0<50><30>{<7B>)<29><17><><EFBFBD><EFBFBD>><3E>z3P^
-u
+-> ssh-ed25519 cDBabA So/Tqwdwd7G0PbE4RwH2qDrNcdqTkhFjF4IJrLKKpkM
+MEA5dzlUeFXm3pa+ndxrcE0ZWdO00Xf98+Q8U9LZ+cQ
+-> ssh-ed25519 cK5kHw sCHD/hHBOfMBUQXkLG3MBPNC4ebLOXW37OlF/C8FEjU
+4TFbKoy23Ic2vteXZ02fMrFxyb4NxyWaSo5I8dn48mI
+-> ssh-ed25519 CAWG4Q KYGPAXTx8H5cBC3YIBxi5B7OeF15C9rEIPFCcG0vEDw
+9LC2Zvp1Oiau1/hfPf+nJknl6BUSr+lzTn6TozZNxJg
+-> ssh-ed25519 xA739A hpvNBHPgYRtUx0HyUAdCW8s7QTmGyPXwzRHb8qYoeG0
+QkUZINY7Fr7HpyY6lbIMcP+hGO3oCmLL6N+yDN4weyk
+-> ssh-ed25519 MSF3dg P9TmEfXS+hyxsbVKja58UWAFpad0ZS3LhwrMkLnSNAY
+hiHuh7HhoYwHi2KFbCczXJoF3On9eqjD1Wsp9Q1NW/w
+--- SN3peoDvjXuD/Q4DdebQFam1CE22NyGZlMmnKyCTuX8
+s<0F><><14><>&׳֦<D7B3><D6A6><EFBFBD><EFBFBD>
}<7D>#In0&<26><1F>{<7B>1<EFBFBD><31>.