Add p command to paste files

Enable nginx
Mount the NVME disk in /nvme
2024-09-16 16:33:42 +02:00 · 2024-09-16 16:33:34 +02:00 · 2024-09-12 09:54:55 +02:00 · 2024-09-12 09:54:55 +02:00 · 2024-09-12 09:54:55 +02:00
20 changed files with 50 additions and 287 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,2 @@
 *.swp
 /result
-/misc
--- a/flake.lock
+++ b/flake.lock
@ -10,11 +10,11 @@
        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1723293904,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "lastModified": 1720546205,
+        "narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
        "owner": "ryantm",
        "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
        "type": "github"
      },
      "original": {
@ -30,11 +30,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1732868163,
-        "narHash": "sha256-qck4h298AgcNI6BnGhEwl26MTLXjumuJVr+9kak7uPo=",
+        "lastModified": 1713974364,
+        "narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
        "ref": "refs/heads/master",
-        "rev": "6782fc6c5b5a29e84a7f2c2d1064f4bcb1288c0f",
-        "revCount": 952,
+        "rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
+        "revCount": 937,
        "type": "git",
        "url": "https://git.sr.ht/~rodarima/bscpkgs"
      },
@ -88,16 +88,16 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1736867362,
-        "narHash": "sha256-i/UJ5I7HoqmFMwZEH6vAvBxOrjjOJNU739lnZnhUln8=",
+        "lastModified": 1720957393,
+        "narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "9c6b49aeac36e2ed73a8c472f1546f6d9cf1addc",
+        "rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixos-24.11",
+        "ref": "nixos-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
--- a/flake.nix
+++ b/flake.nix
@ -1,6 +1,6 @@
 {
  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    agenix.url = "github:ryantm/agenix";
    agenix.inputs.nixpkgs.follows = "nixpkgs";
    bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
--- a/m/common/base/nix.nix
+++ b/m/common/base/nix.nix
@ -32,21 +32,6 @@
    };
  };

-  # The nix-gc.service can begin its execution *before* /home is mounted,
-  # causing it to remove all gcroots considering them as stale, as it cannot
-  # access the symlink. To prevent this problem, we force the service to wait
-  # until /home is mounted as well as other remote FS like /ceph.
-  systemd.services.nix-gc = {
-    # Start remote-fs.target if not already being started and fail if it fails
-    # to start. It will also be stopped if the remote-fs.target fails after
-    # starting successfully.
-    bindsTo = [ "remote-fs.target" ];
-    # Wait until remote-fs.target fully starts before starting this one.
-    after = [ "remote-fs.target"];
-    # Ensure we can access a remote path inside /home
-    unitConfig.ConditionPathExists = "/home/Computational";
-  };
-
  # This value determines the NixOS release from which the default
  # settings for stateful data, like file locations and database versions
  # on your system were taken. It‘s perfectly fine and recommended to leave
--- a/m/common/base/ssh.nix
+++ b/m/common/base/ssh.nix
@ -13,16 +13,10 @@ in
    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
      User git
      ProxyCommand nc -X connect -x hut:23080 %h %p
-
-    # Connect to BSC machines via hut proxy too
-    Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
-      ProxyCommand nc -X connect -x hut:23080 %h %p
  '';

  programs.ssh.knownHosts = hostsKeys // {
    "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
    "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
-    "glogin1.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
-    "glogin2.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFsHsZGCrzpd4QDVn5xoDOtrNBkb0ylxKGlyBt6l9qCz";
  };
 }
--- a/m/common/base/users.nix
+++ b/m/common/base/users.nix
@ -100,19 +100,6 @@
          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
        ];
      };
-
-      dbautist = {
-        uid = 5649;
-        isNormalUser = true;
-        home = "/home/Computational/dbautist";
-        description = "Dylan Bautista Cases";
-        group = "Computational";
-        hosts = [ "hut" ];
-        hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
-        ];
-      };
    };

    groups = {
--- a/m/hut/configuration.nix
+++ b/m/hut/configuration.nix
@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, ... }:

 {
  imports = [
@ -23,21 +23,11 @@
  ];

  # Select the this using the ID to avoid mismatches
-  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53567f";
+  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";

-  fileSystems = {
-    "/" = lib.mkForce {
-      device = "/dev/disk/by-label/nvme";
-      fsType = "ext4";
-      neededForBoot = true;
-      options = [ "noatime" ];
-    };
-
-    "/boot" = lib.mkForce {
-      device = "/dev/disk/by-label/nixos-boot";
-      fsType = "ext4";
-      neededForBoot = true;
-    };
+  fileSystems."/nvme" = {
+    fsType = "ext4";
+    device = "/dev/disk/by-label/nvme";
  };

  networking = {
--- a/m/hut/gpfs-probe.nix
+++ b/m/hut/gpfs-probe.nix
@ -1,31 +0,0 @@
-{ pkgs, config, lib, ... }:
-let
-  gpfs-probe-script = pkgs.runCommand "gpfs-probe.sh" { }
-    ''
-      cp ${./gpfs-probe.sh} $out;
-      chmod +x $out
-    ''
-  ;
-in
-{
-  # Use a new user to handle the SSH keys
-  users.groups.ssh-robot = { };
-  users.users.ssh-robot = {
-    description = "SSH Robot";
-    isNormalUser = true;
-    home = "/var/lib/ssh-robot";
-  };
-
-  systemd.services.gpfs-probe = {
-    description = "Daemon to report GPFS latency via SSH";
-    path = [ pkgs.openssh pkgs.netcat ];
-    after = [ "network.target" ];
-    wantedBy = [ "default.target" ];
-    serviceConfig = {
-      Type = "simple";
-      ExecStart = "${pkgs.socat}/bin/socat TCP4-LISTEN:9966,fork EXEC:${gpfs-probe-script}";
-      User = "ssh-robot";
-      Group = "ssh-robot";
-    };
-  };
-}
--- a/m/hut/gpfs-probe.sh
+++ b/m/hut/gpfs-probe.sh
@ -1,18 +0,0 @@
-#!/bin/sh
-
-N=500
-
-t=$(timeout 5 ssh bsc015557@glogin2.bsc.es "timeout 3 command time -f %e touch /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N} 2>&1; rm -f /gpfs/projects/bsc15/bsc015557/gpfs.{1..$N}")
-
-if [ -z "$t" ]; then
-  t="5.00"
-fi
-
-cat <<EOF
-HTTP/1.1 200 OK
-Content-Type: text/plain; version=0.0.4; charset=utf-8; escaping=values
-
-# HELP gpfs_touch_latency Time to create $N files.
-# TYPE gpfs_touch_latency gauge
-gpfs_touch_latency $t
-EOF
--- a/m/hut/monitoring.nix
+++ b/m/hut/monitoring.nix
@ -1,10 +1,7 @@
 { config, lib, ... }:

 {
-  imports = [
-    ../module/slurm-exporter.nix
-    ./gpfs-probe.nix
-  ];
+  imports = [ ../module/slurm-exporter.nix ];

  age.secrets.grafanaJungleRobotPassword = {
    file = ../../secrets/jungle-robot-password.age;
@ -79,7 +76,7 @@
      };
      node = {
        enable = true;
-        enabledCollectors = [ "systemd" "logind" ];
+        enabledCollectors = [ "systemd" ];
        port = 9002;
        listenAddress = "127.0.0.1";
      };
@ -105,7 +102,6 @@
            "127.0.0.1:9252"
            "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
            "127.0.0.1:9341" # Slurm exporter
-            "127.0.0.1:9966" # GPFS custom exporter
            "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
          ];
        }];
--- a/m/hut/nginx.nix
+++ b/m/hut/nginx.nix
@ -1,70 +1,14 @@
-{ theFlake, pkgs, ... }:
-let
-  website = pkgs.stdenv.mkDerivation {
-    name = "jungle-web";
-    src = theFlake;
-    buildInputs = [ pkgs.hugo ];
-    buildPhase = ''
-      cd web
-      rm -rf public/
-      hugo
-    '';
-    installPhase = ''
-      cp -r public $out
-    '';
-  };
-in
 {
  services.nginx = {
    enable = true;
    virtualHosts."jungle.bsc.es" = {
-      root = "${website}";
      listen = [
        {
          addr = "127.0.0.1";
-          port = 80;
+          port = 8123;
        }
      ];
-      extraConfig = ''
-        set_real_ip_from 127.0.0.1;
-        set_real_ip_from 84.88.52.107;
-        real_ip_recursive on;
-        real_ip_header X-Forwarded-For;
-
-        location /git {
-          rewrite ^/git$ / break;
-          rewrite ^/git/(.*) /$1 break;
-          proxy_pass http://127.0.0.1:3000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /cache {
-          rewrite ^/cache(.*) /$1 break;
-          proxy_pass http://127.0.0.1:5000;
-          proxy_redirect http:// $scheme://;
-        }
-        location /lists {
-          proxy_pass http://127.0.0.1:8081;
-          proxy_redirect http:// $scheme://;
-        }
-        location /grafana {
-          proxy_pass http://127.0.0.1:2342;
-          proxy_redirect http:// $scheme://;
-          proxy_set_header Host $host;
-          # Websockets
-          proxy_http_version 1.1;
-          proxy_set_header Upgrade $http_upgrade;
-          proxy_set_header Connection "upgrade";
-        }
-        location ~ ^/~(.+?)(/.*)?$ {
-          alias /ceph/home/$1/public_html$2;
-          index  index.html index.htm;
-          autoindex on;
-          absolute_redirect off;
-        }
-        location /p/ {
-          alias /ceph/p/;
-        }
-      '';
+      locations."/p/".alias = "/ceph/p/";
    };
  };
 }
--- a/m/hut/p.nix
+++ b/m/hut/p.nix
@ -1,4 +1,4 @@
-{ pkgs, lib, config, ... }:
+{ pkgs, ... }:
 let
  p = pkgs.writeShellScriptBin "p" ''
    set -e
@ -6,14 +6,12 @@ let
    pastedir="p/$USER"
    mkdir -p "$pastedir"

-    ext="txt"
-
    if [ -n "$1" ]; then
-      ext="$1"
+            out="$pastedir/$1"
+    else
+            out=$(mktemp "$pastedir/XXXXXXXX.txt")
    fi

-    out=$(mktemp "$pastedir/XXXXXXXX.$ext")
-
    cat > "$out"
    chmod go+r "$out"
    echo "https://jungle.bsc.es/$out"
@ -21,23 +19,4 @@ let
 in
 {
  environment.systemPackages = with pkgs; [ p ];
-
-  # Make sure we have a directory per user. We cannot use the nice
-  # systemd-tmpfiles-setup.service service because this is a remote FS, and it
-  # may not be mounted when it runs.
-  systemd.services.create-paste-dirs = let
-    # Take only normal users in hut
-    users = lib.filterAttrs (_: v: v.isNormalUser) config.users.users;
-    commands = lib.concatLists (lib.mapAttrsToList
-      (_: user: [
-        "install -d -o ${user.name} -g ${user.group} -m 0755 /ceph/p/${user.name}"
-      ]) users);
-    script = pkgs.writeShellScript "create-paste-dirs.sh" (lib.concatLines commands);
-  in {
-    enable = true;
-    wants = [ "remote-fs.target" ];
-    after = [ "remote-fs.target" ];
-    wantedBy = [ "multi-user.target" ];
-    serviceConfig.ExecStart = script;
-  };
 }
--- a/m/module/ceph.nix
+++ b/m/module/ceph.nix
@ -13,7 +13,7 @@

  age.secrets.cephUser.file = ../../secrets/ceph-user.age;

-  fileSystems."/ceph" = {
+  fileSystems."/ceph-slow" = {
    fsType = "ceph";
    device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
    options = [
@ -21,4 +21,16 @@
      "secretfile=${config.age.secrets.cephUser.path}"
    ];
  };
+
+  services.cachefilesd.enable = true;
+
+  fileSystems."/ceph" = {
+    fsType = "ceph";
+    device = "user@9c8d06e0-485f-4aaf-b16b-06d6daf1232b.cephfs=/";
+    options = [
+      "fsc"
+      "mon_addr=10.0.40.40"
+      "secretfile=${config.age.secrets.cephUser.path}"
+    ];
+  };
 }
--- a/pkgs/overlay.nix
+++ b/pkgs/overlay.nix
@ -11,7 +11,7 @@ final: prev:
      paths = [ pmix.dev pmix.out ];
    };
  in prev.mpich.overrideAttrs (old: {
-    patches = (old.patches or []) ++ [
+    patches = [
      # See https://github.com/pmodels/mpich/issues/6946
      ./mpich-fix-hwtopo.patch
    ];
--- a/secrets/gitlab-runner-docker-token.age
+++ b/secrets/gitlab-runner-docker-token.age
@ -1,9 +1,9 @@
 age-encryption.org/v1
-> ssh-ed25519 HY2yRg 0sEIUEJBJQ0k0rBfHaOEbq1pNBqsPin4Xq85v0ds9jY
-4wzjLapoOcq53nT2K3hSGED4jTDXci25GLHkl/fL4EI
-> ssh-ed25519 CAWG4Q f68ZbJGwXuCZVnqhwbh+8dh0X/MCdjEd+sVtPyBu/hU
-u2TQreyWQvP6PGuwuUNKA/AL68560flqSlaItN3k41I
-> ssh-ed25519 MSF3dg HdrtRW2j7mfkLH0/4aJK5R0cWdjf56HYtEZgzHi9EAs
-A6MF6tXmSUq2RF2bpmav0GFTRERwluSZGh2snP/KqkA
--- drsezqi7J/g8gm6N10SkfeAWnYct99WUraB5djLJqpo
-gÔ
(ìÐJ!M6¬É3e¸AÜæÃ?\1y÷eüFN\‘<>/MêòªN`K^€+"¤«Y^å>dÒH÷°‡¸†]P…ÓûJ‘`xôã»{Ú±ô„y°ÅÎøSˆéyPX{w‰Sï
ž^5X¶JPô;v‰
+-> ssh-ed25519 HY2yRg WvKK6U1wQtx2pbUDfuaUIXTQiCulDkz7hgUCSwMfMzQ
+jLktUMqKuVxukqzz++pHOKvmucUQqeKYy5IwBma7KxY
+-> ssh-ed25519 CAWG4Q XKGuNNoYFl9bdZzsqYYTY7GsEt5sypLW4R+1uk78NmU
+8dIA2GzRAwTGM5CDHSM2BUBsbXzEAUssWUz2PY2PaTg
+-> ssh-ed25519 MSF3dg T630RsKuZIF/bp+KITnIIWWHsg6M/VQGqbWQZxqT+AA
+SraZcgZJVtmUzHF/XR9J7aK5t5EDNpkC/av/WJUT/G8
+--- /12G8pj9sbs591OM/ryhoLnSWWmzYcoqprk9uN/3g18
+ä·ù¼Â‡%å]yi"ô<>»LÓâùH`ªa$Æþ)¦9ve<76>.0úmÉK<EFBFBD>vƒÀïu"|1cÞ-%ÔÕ"åWFï¡ÞA«<41>hº$•ºj<eñ¶xÅLx«ç.?œÈâ:L…¬–ƒ,ëu»|³‹F|Õi²äÔ
--- a/secrets/gitlab-runner-shell-token.age
+++ b/secrets/gitlab-runner-shell-token.age
--- a/web/content/paste/_index.md
+++ b/web/content/paste/_index.md
@ -1,68 +0,0 @@
---
-title: "Paste"
-description: "Paste service"
-author: "Rodrigo Arias Mallo"
-date: 2024-09-20
---
-
-The hut machine provides a paste service using the program `p` (as in paste).
-
-You can use it directly from the hut machine or remotely if you have [SSH
-access](/access) to hut using the following alias:
-
-```
-alias p="ssh hut p"
-```
-
-You can add it to bashrc or zshrc for persistent installation.
-
-## Usage
-
-The `p` command reads from the standard input, uploads the content to a file
-in the ceph filesystem and prints the URL to access it. It only accepts an
-optional argument, which is the extension of the file that will be stored on
-disk (without the dot). By default it uses the `txt` extension, so plain text
-can be read in the browser directly.
-
-```
-p [extension]
-```
-
-To remove files, go to `/ceph/p/$USER` and remove them manually.
-
-## Examples
-
-Share a text file, in this case the source of p itself:
-
-```
-hut% p < m/hut/p.nix
-https://jungle.bsc.es/p/rarias/okbtG130.txt
-```
-
-Paste the last dmesg lines directly from a pipe:
-
-```
-hut% dmesg | tail -5 | p
-https://jungle.bsc.es/p/rarias/luX4STm9.txt
-```
-
-Upload a PNG picture from a file:
-
-```
-hop% p png < mark-api-cpu.png
-https://jungle.bsc.es/p/rarias/oSRAMVsE.png
-```
-
-Take an screenshot and upload it as a PNG file:
-
-```
-hop% scrot -s - | p png
-https://jungle.bsc.es/p/rarias/SOgK5EV0.png
-```
-
-Upload a directory by creating a tar.gz file on the fly:
-
-```
-hop% tar c ovni | gzip | p tar.gz
-https://jungle.bsc.es/p/rarias/tkwROcTR.tar.gz
-```
--- a/web/hugo.toml
+++ b/web/hugo.toml
@ -21,14 +21,8 @@ name = "Lists"
 url = "/lists/"
 weight = 30

-[[menu.main]]
-identifier = "Paste"
-name = "Paste"
-url = "/paste/"
-weight = 40
-
 [[menu.main]]
 identifier = "Posts"
 name = "Posts"
 url = "/posts/"
-weight = 50
+weight = 40
--- a/web/themes/PaperMod/layouts/partials/templates/opengraph.html
+++ b/web/themes/PaperMod/layouts/partials/templates/opengraph.html
@ -49,4 +49,4 @@
 {{ end }}{{ end }}

 {{- /* Facebook Page Admin ID for Domain Insights */}}
-{{- with site.Params.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}
+{{- with site.Social.facebook_admin }}<meta property="fb:admins" content="{{ . }}" />{{ end }}
--- a/web/themes/PaperMod/layouts/partials/templates/twitter_cards.html
+++ b/web/themes/PaperMod/layouts/partials/templates/twitter_cards.html
@ -28,6 +28,6 @@
 {{- end }}
 <meta name="twitter:title" content="{{ .Title }}"/>
 <meta name="twitter:description" content="{{ with .Description }}{{ . }}{{ else }}{{if .IsPage}}{{ .Summary }}{{ else }}{{ with site.Params.description }}{{ . }}{{ end }}{{ end }}{{ end -}}"/>
-{{ with site.Params.Social.twitter -}}
+{{ with site.Social.twitter -}}
 <meta name="twitter:site" content="@{{ . }}"/>
 {{ end -}}
Author	SHA1	Message	Date
Rodrigo Arias Mallo	9b1391a9f6	Add p command to paste files	2024-09-16 16:33:42 +02:00
Rodrigo Arias Mallo	c8ca5adf84	Enable nginx	2024-09-16 16:33:34 +02:00
Rodrigo Arias Mallo	43e4c60dd5	Mount the NVME disk in /nvme	2024-09-12 09:54:55 +02:00
Rodrigo Arias Mallo	f5d6f32ca8	Rename ceph mount points Use /ceph for cached ceph and /ceph-slow for uncached ceph.	2024-09-12 09:54:55 +02:00
Rodrigo Arias Mallo	8fccb40a7a	Add cached ceph FS mount point in /cache	2024-09-12 09:54:55 +02:00