Add p command to paste files

Use /ceph for cached ceph and /ceph-slow for uncached ceph.

.gitignore

@@ -1,3 +1,2 @@
 *.swp
 /result
-/misc

flake.lock

@@ -10,11 +10,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1723293904,
+        "lastModified": 1720546205,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
         "type": "github"
       },
       "original": {
@@ -30,11 +30,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1732868163,
+        "lastModified": 1713974364,
-        "narHash": "sha256-qck4h298AgcNI6BnGhEwl26MTLXjumuJVr+9kak7uPo=",
+        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
         "ref": "refs/heads/master",
-        "rev": "6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f",
+        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
-        "revCount": 952,
+        "revCount": 937,
         "type": "git",
         "url": "https://git.sr.ht/~rodarima/bscpkgs"
       },
@@ -88,16 +88,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1736867362,
+        "lastModified": 1720957393,
-        "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=",
+        "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc",
+        "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-24.11",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,6 +1,6 @@
 {
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";

m/common/base/nix.nix

@@ -32,21 +32,6 @@
     };
   };
 
-  # The nix-gc.service can begin its execution *before* /home is mounted,
-  # causing it to remove all gcroots considering them as stale, as it cannot
-  # access the symlink. To prevent this problem, we force the service to wait
-  # until /home is mounted as well as other remote FS like /ceph.
-  systemd.services.nix-gc = {
-    # Start remote-fs.target if not already being started and fail if it fails
-    # to start. It will also be stopped if the remote-fs.target fails after
-    # starting successfully.
-    bindsTo = [ "remote-fs.target" ];
-    # Wait until remote-fs.target fully starts before starting this one.
-    after = [ "remote-fs.target"];
-    # Ensure we can access a remote path inside /home
-    unitConfig.ConditionPathExists = "/home/Computational";
-  };
-
   # This value determines the NixOS release from which the default
   # settings for stateful data, like file locations and database versions
   # on your system were taken. It‘s perfectly fine and recommended to leave

m/common/base/ssh.nix

@@ -13,16 +13,10 @@ in
     Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
       User git
       ProxyCommand nc -X connect -x hut:23080 %h %p
-
-    # Connect to BSC machines via hut proxy too
-    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-      ProxyCommand nc -X connect -x hut:23080 %h %p
   '';
 
   programs.ssh.knownHosts = hostsKeys // {
     "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
     "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
-    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
-    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
   };
 }

m/common/base/users.nix

@@ -100,19 +100,6 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
         ];
       };
-
-      dbautist = {
-        uid = 5649;
-        isNormalUser = true;
-        home = "/home/Computational/dbautist";
-        description = "Dylan Bautista Cases";
-        group = "Computational";
-        hosts = [ "hut" ];
-        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
-        ];
-      };
     };
 
     groups = {

m/hut/configuration.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, ... }:
 
 {
   imports = [
@@ -23,21 +23,11 @@
   ];
 
   # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
+  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
 
-  fileSystems = {
+  fileSystems."/nvme" = {
-    "/" = lib.mkForce {
+    fsType = "ext4";
-      device = "/dev/disk/by-label/nvme";
+    device = "/dev/disk/by-label/nvme";
-      fsType = "ext4";
-      neededForBoot = true;
-      options = [ "noatime" ];
-    };
-
-    "/boot" = lib.mkForce {
-      device = "/dev/disk/by-label/nixos-boot";
-      fsType = "ext4";
-      neededForBoot = true;
-    };
   };
 
   networking = {

m/hut/gpfs-probe.nix

@@ -1,31 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
-    ''
-      cp ${./gpfs-probe.sh} $out;
-      chmod +x $out
-    ''
-  ;
-in
-{
-  # Use a new user to handle the SSH keys
-  users.groups.ssh-robot = { };
-  users.users.ssh-robot = {
-    description = "SSH Robot";
-    isNormalUser = true;
-    home = "/var/lib/ssh-robot";
-  };
-
-  systemd.services.gpfs-probe = {
-    description = "Daemon to report GPFS latency via SSH";
-    path = [ pkgs.openssh pkgs.netcat ];
-    after = [ "network.target" ];
-    wantedBy = [ "default.target" ];
-    serviceConfig = {
-      Type = "simple";
-      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
-      User = "ssh-robot";
-      Group = "ssh-robot";
-    };
-  };
-}

m/hut/gpfs-probe.sh

@@ -1,18 +0,0 @@
-#!/bin/sh
-
-N=500
-
-t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
-
-if [ -z "$t" ]; then
-  t="5.00"
-fi
-
-cat <<EOF
-HTTP/1.1 200 OK
-Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
-
-# HELP gpfs_touch_latency Time to create $N files.
-# TYPE gpfs_touch_latency gauge
-gpfs_touch_latency $t
-EOF

m/hut/monitoring.nix

@@ -1,10 +1,7 @@
 { config, lib, ... }:
 
 {
-  imports = [
+  imports = [ ../module/slurm-exporter.nix ];
-    ../module/slurm-exporter.nix
-    ./gpfs-probe.nix
-  ];
 
   age.secrets.grafanaJungleRobotPassword = {
     file = ../../secrets/jungle-robot-password.age;
@@ -79,7 +76,7 @@
       };
       node = {
         enable = true;
-        enabledCollectors = [ "systemd" "logind" ];
+        enabledCollectors = [ "systemd" ];
         port = 9002;
         listenAddress = "127.0.0.1";
       };
@@ -105,7 +102,6 @@
             "127.0.0.1:9252"
             "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
             "127.0.0.1:9341" # Slurm exporter
-            "127.0.0.1:9966" # GPFS custom exporter
             "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
           ];
         }];

m/hut/nginx.nix

@@ -1,70 +1,14 @@
-{ theFlake, pkgs, ... }:
-let
-  website = pkgs.stdenv.mkDerivation {
-    name = "jungle-web";
-    src = theFlake;
-    buildInputs = [ pkgs.hugo ];
-    buildPhase = ''
-      cd web
-      rm -rf public/
-      hugo
-    '';
-    installPhase = ''
-      cp -r public $out
-    '';
-  };
-in
 {
   services.nginx = {
     enable = true;
     virtualHosts."jungle.bsc.es" = {
-      root = "${website}";
       listen = [
         {
           addr = "127.0.0.1";
-          port = 80;
+          port = 8123;
         }
       ];
-      extraConfig = ''
+      locations."/p/".alias = "/ceph/p/";
-        set_real_ip_from 127.0.0.1;
-        set_real_ip_from 84.88.52.107;
-        real_ip_recursive on;
-        real_ip_header X-Forwarded-For;
-
-        location /git {
-          rewrite ^/git$ / break;
-          rewrite ^/git/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:3000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /cache {
-          rewrite ^/cache(.*) /$1 break;
-          proxy_pass http://127.0.0.1:5000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /lists {
-          proxy_pass http://127.0.0.1:8081;
-          proxy_redirect http:// $scheme://;
-        }
-        location /grafana {
-          proxy_pass http://127.0.0.1:2342;
-          proxy_redirect http:// $scheme://;
-          proxy_set_header Host $host;
-          # Websockets
-          proxy_http_version 1.1;
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection "upgrade";
-        }
-        location ~ ^/~(.+?)(/.*)?$ {
-          alias /ceph/home/$1/public_html$2;
-          index  index.html index.htm;
-          autoindex on;
-          absolute_redirect off;
-        }
-        location /p/ {
-          alias /ceph/p/;
-        }
-      '';
     };
   };
 }

m/hut/p.nix

@@ -1,4 +1,4 @@
-{ pkgs, lib, config, ... }:
+{ pkgs, ... }:
 let
   p = pkgs.writeShellScriptBin "p" ''
     set -e
@@ -6,14 +6,12 @@ let
     pastedir="p/$USER"
     mkdir -p "$pastedir"
 
-    ext="txt"
-
     if [ -n "$1" ]; then
-      ext="$1"
+            out="$pastedir/$1"
+    else
+            out=$(mktemp "$pastedir/XXXXXXXX.txt")
     fi
 
-    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
-
     cat > "$out"
     chmod go+r "$out"
     echo "https://jungle.bsc.es/$out"
@@ -21,23 +19,4 @@ let
 in
 {
   environment.systemPackages = with pkgs; [ p ];
-
-  # Make sure we have a directory per user. We cannot use the nice
-  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
-  # may not be mounted when it runs.
-  systemd.services.create-paste-dirs = let
-    # Take only normal users in hut
-    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-    commands = lib.concatLists (lib.mapAttrsToList
-      (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
-      ]) users);
-    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
-  in {
-    enable = true;
-    wants = [ "remote-fs.target" ];
-    after = [ "remote-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig.ExecStart = script;
-  };
 }

m/module/ceph.nix

@@ -13,7 +13,7 @@
 
   age.secrets.cephUser.file = ../../secrets/ceph-user.age;
 
-  fileSystems."/ceph" = {
+  fileSystems."/ceph-slow" = {
     fsType = "ceph";
     device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
     options = [
@@ -21,4 +21,16 @@
       "secretfile=${config.age.secrets.cephUser.path}"
     ];
   };
+
+  services.cachefilesd.enable = true;
+
+  fileSystems."/ceph" = {
+    fsType = "ceph";
+    device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
+    options = [
+      "fsc"
+      "mon_addr=10.0.40.40"
+      "secretfile=${config.age.secrets.cephUser.path}"
+    ];
+  };
 }

pkgs/overlay.nix

@@ -11,7 +11,7 @@ final: prev:
       paths = [ pmix.dev pmix.out ];
     };
   in prev.mpich.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [
+    patches = [
       # See https://github.com/pmodels/mpich/issues/6946
       ./mpich-fix-hwtopo.patch
     ];

secrets/gitlab-runner-docker-token.age

@@ -1,9 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg 0sEIUEJBJQ0k0rBfHaOEbq1pNBqsPin4Xq85v0ds9jY
+-> ssh-ed25519 HY2yRg WvKK6U1wQtx2pbUDfuaUIXTQiCulDkz7hgUCSwMfMzQ
-4wzjLapoOcq53nT2K3hSGED4jTDXci25GLHkl/fL4EI
+jLktUMqKuVxukqzz++pHOKvmucUQqeKYy5IwBma7KxY
--> ssh-ed25519 CAWG4Q f68ZbJGwXuCZVnqhwbh+8dh0X/MCdjEd+sVtPyBu/hU
+-> ssh-ed25519 CAWG4Q XKGuNNoYFl9bdZzsqYYTY7GsEt5sypLW4R+1uk78NmU
-u2TQreyWQvP6PGuwuUNKA/AL68560flqSlaItN3k41I
+8dIA2GzRAwTGM5CDHSM2BUBsbXzEAUssWUz2PY2PaTg
--> ssh-ed25519 MSF3dg HdrtRW2j7mfkLH0/4aJK5R0cWdjf56HYtEZgzHi9EAs
+-> ssh-ed25519 MSF3dg T630RsKuZIF/bp+KITnIIWWHsg6M/VQGqbWQZxqT+AA
-A6MF6tXmSUq2RF2bpmav0GFTRERwluSZGh2snP/KqkA
+SraZcgZJVtmUzHF/XR9J7aK5t5EDNpkC/av/WJUT/G8
---- drsezqi7J/g8gm6N10SkfeAWnYct99WUraB5djLJqpo
+--- /12G8pj9sbs591OM/ryhoLnSWWmzYcoqprk9uN/3g18
-gÔ
(ìÐJ!M6¬É3e¸AÜæÃ?\1y÷eüFN\‘<>/MêòªN`K
^€+"¤«Y^å>dÒH÷°‡¸†]P…ÓûJ‘`xôã»{Ú±ô„y°ÅÎøSˆéy
PX{w‰Sï
ž^5X¶JPô;v‰
+ä·ù
¼Â‡%å]yi"ô<>»LÓâùH`ªa$Æþ)¦9ve<76>.0úmÉK<EFBFBD>vƒÀïu"|1cÞ-%ÔÕ"åWFï¡ÞA«<41>hº$•ºj<eñ¶xÅLx«ç.?œÈâ:L…¬–ƒ,ëu»|³‹F|Õi²äÔ

web/content/paste/_index.md

@@ -1,68 +0,0 @@
----
-title: "Paste"
-description: "Paste service"
-author: "Rodrigo Arias Mallo"
-date: 2024-09-20
----
-
-The hut machine provides a paste service using the program `p` (as in paste).
-
-You can use it directly from the hut machine or remotely if you have [SSH
-access](/access) to hut using the following alias:
-
-```
-alias p="ssh hut p"
-```
-
-You can add it to bashrc or zshrc for persistent installation.
-
-## Usage
-
-The `p` command reads from the standard input, uploads the content to a file
-in the ceph filesystem and prints the URL to access it. It only accepts an
-optional argument, which is the extension of the file that will be stored on
-disk (without the dot). By default it uses the `txt` extension, so plain text
-can be read in the browser directly.
-
-```
-p [extension]
-```
-
-To remove files, go to `/ceph/p/$USER` and remove them manually.
-
-## Examples
-
-Share a text file, in this case the source of p itself:
-
-```
-hut% p < m/hut/p.nix
-https://jungle.bsc.es/p/rarias/okbtG130.txt
-```
-
-Paste the last dmesg lines directly from a pipe:
-
-```
-hut% dmesg | tail -5 | p
-https://jungle.bsc.es/p/rarias/luX4STm9.txt
-```
-
-Upload a PNG picture from a file:
-
-```
-hop% p png < mark-api-cpu.png
-https://jungle.bsc.es/p/rarias/oSRAMVsE.png
-```
-
-Take an screenshot and upload it as a PNG file:
-
-```
-hop% scrot -s - | p png
-https://jungle.bsc.es/p/rarias/SOgK5EV0.png
-```
-
-Upload a directory by creating a tar.gz file on the fly:
-
-```
-hop% tar c ovni | gzip | p tar.gz
-https://jungle.bsc.es/p/rarias/tkwROcTR.tar.gz
-```

web/hugo.toml

@@ -21,14 +21,8 @@ name = "Lists"
 url = "/lists/"
 weight = 30
 
-[[menu.main]]
-identifier = "Paste"
-name = "Paste"
-url = "/paste/"
-weight = 40
-
 [[menu.main]]
 identifier = "Posts"
 name = "Posts"
 url = "/posts/"
-weight = 50
+weight = 40

web/themes/PaperMod/layouts/partials/templates/opengraph.html

@@ -49,4 +49,4 @@
 {{ end }}{{ end }}
 
 {{- /* Facebook Page Admin ID for Domain Insights */}}
-{{- with site.Params.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}
+{{- with site.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}

web/themes/PaperMod/layouts/partials/templates/twitter_cards.html

@@ -28,6 +28,6 @@
 {{- end }}
 <meta name="twitter:title" content="{{ .Title }}"/>
 <meta name="twitter:description" content="{{ with .Description }}{{ . }}{{ else }}{{if .IsPage}}{{ .Summary }}{{ else }}{{ with site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end -}}"/>
-{{ with site.Params.Social.twitter -}}
+{{ with site.Social.twitter -}}
 <meta name="twitter:site" content="@{{ . }}"/>
 {{ end -}}