Add quickstart guide

Use agenix to store the credentials safely.

m/common/base/users.nix

@@ -81,7 +81,7 @@
         home = "/home/Computational/abonerib";
         description = "Aleix Boné";
         group = "Computational";
-        hosts = [ "owl1" "owl2" "hut" "raccoon" "fox" ];
+        hosts = [ "owl1" "owl2" "hut" "raccoon" ];
         hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
@@ -126,19 +126,6 @@
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGEfy6F4rF80r4Cpo2H5xaWqhuUZzUsVsILSKGJzt5jF dalvare1@ssfhead"
         ];
       };
-
-      varcila = {
-        uid = 5650;
-        isNormalUser = true;
-        home = "/home/Computational/varcila";
-        description = "Vincent Arcila";
-        group = "Computational";
-        hosts = [ "hut" "fox" ];
-        hashedPassword = "$6$oB0Tcn99DcM4Ch$Vn1A0ulLTn/8B2oFPi9wWl/NOsJzaFAWjqekwcuC9sMC7cgxEVb.Nk5XSzQ2xzYcNe5MLtmzkVYnRS1CqP39Y0";
-        openssh.authorizedKeys.keys = [
-          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGt0ESYxekBiHJQowmKpfdouw0hVm3N7tUMtAaeLejK vincent@varch"
-        ];
-      };
     };
 
     groups = {

m/hut/gitlab-runner.nix

@@ -22,8 +22,8 @@
           "--docker-network-mode host"
         ];
         environmentVariables = {
-          https_proxy = "http://hut:23080";
+          https_proxy = "http://localhost:23080";
-          http_proxy = "http://hut:23080";
+          http_proxy = "http://localhost:23080";
         };
       };
     in {
@@ -38,13 +38,14 @@
       gitlab-bsc-docker = {
         # gitlab.bsc.es still uses the old token mechanism
         registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path;
-        tagList = [ "docker" "hut" ];
         environmentVariables = {
-          # We cannot access the hut local interface from docker, so we connect
+          https_proxy = "http://localhost:23080";
-          # to hut directly via the ethernet one.
+          http_proxy = "http://localhost:23080";
-          https_proxy = "http://hut:23080";
-          http_proxy = "http://hut:23080";
         };
+        # FIXME
+        registrationFlags = [
+          "--docker-network-mode host"
+        ];
         executor = "docker";
         dockerImage = "alpine";
         dockerVolumes = [
@@ -52,15 +53,7 @@
           "/nix/var/nix/db:/nix/var/nix/db:ro"
           "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
         ];
-        dockerExtraHosts = [
-          # Required to pass the proxy via hut
-          "hut:10.0.40.7"
-        ];
         dockerDisableCache = true;
-        registrationFlags = [
-          # Increase build log length to 64 MiB
-          "--output-limit 65536"
-        ];
         preBuildScript = pkgs.writeScript "setup-container" ''
           mkdir -p -m 0755 /nix/var/log/nix/drvs
           mkdir -p -m 0755 /nix/var/nix/gcroots
@@ -73,38 +66,32 @@
           mkdir -p -m 0700 "$HOME/.nix-defexpr"
           mkdir -p -m 0700 "$HOME/.ssh"
           cat > "$HOME/.ssh/config" << EOF
-          Host bscpm04.bsc.es gitlab-internal.bsc.es
+          Host bscpm03.bsc.es gitlab-internal.bsc.es
             User git
             ProxyCommand nc -X connect -x hut:23080 %h %p
           Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es
             ProxyCommand nc -X connect -x hut:23080 %h %p
           EOF
           cat >> "$HOME/.ssh/known_hosts" << EOF
-          bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT
+          bscpm03.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS
           gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3
           EOF
           . ${pkgs.nix}/etc/profile.d/nix-daemon.sh
-          # Required to load SSL certificate paths
+          ${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixos-24.11 nixpkgs
-          . ${pkgs.cacert}/nix-support/setup-hook
+          ${pkgs.nix}/bin/nix-channel --update nixpkgs
+          ${pkgs.nix}/bin/nix-env -i ${lib.concatStringsSep " " (with pkgs; [ nix cacert git openssh netcat curl ])}
         '';
         environmentVariables = {
           ENV = "/etc/profile";
           USER = "root";
           NIX_REMOTE = "daemon";
-          PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin";
+          PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin";
+          NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
         };
       };
     };
   };
 
-  # DOCKER* chains are useless, override at FORWARD
-  networking.firewall.extraCommands = ''
-    # Allow docker to use our proxy
-    iptables -I FORWARD 1 -p tcp -i docker0 -d hut --dport 23080 -j nixos-fw-accept
-    # Block anything else coming from docker
-    iptables -I FORWARD 2 -p all -i docker0 -j nixos-fw-log-refuse
-  '';
-
   #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";

m/hut/monitoring.nix

@@ -76,7 +76,7 @@
         group = "root";
         user = "root";
         configFile = config.age.secrets.ipmiYml.path;
-        # extraFlags = [ "--log.level=debug" ];
+        extraFlags = [ "--log.level=debug" ];
         listenAddress = "127.0.0.1";
       };
       node = {

m/hut/nginx.nix

@@ -12,8 +12,6 @@ let
     installPhase = ''
       cp -r public $out
     '';
-    # Don't mess doc/
-    dontFixup = true;
   };
 in
 {

m/module/slurm-client.nix

@@ -27,6 +27,22 @@ let
     done
   '';
 
+  prolog = pkgs.writeScript "prolog.sh" ''
+    #!/usr/bin/env bash
+
+    echo "hello from the prolog"
+
+    exit 0
+  '';
+
+  epilog = pkgs.writeScript "epilog.sh" ''
+    #!/usr/bin/env bash
+
+    echo "hello from the epilog"
+
+    exit 0
+  '';
+
 in {
   systemd.services.slurmd.serviceConfig = {
     # Kill all processes in the control group on stop/restart. This will kill
@@ -43,13 +59,14 @@ in {
     clusterName = "jungle";
     nodeName = [
       "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
-      "fox       Sockets=2 CoresPerSocket=96 ThreadsPerCore=1 Feature=fox"
+      "fox       Sockets=2 CoresPerSocket=96 ThreadsPerCore=2 Feature=fox"
       "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
     ];
 
     partitionName = [
       "owl Nodes=owl[1-2]     Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
       "fox Nodes=fox          Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
+      "all Nodes=owl[1-2],hut Default=NO  DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
     ];
 
     # See slurm.conf(5) for more details about these options.

secrets/gitlab-bsc-docker-token.age

@@ -1,11 +1,9 @@
 age-encryption.org/v1
--> ssh-ed25519 HY2yRg WSdjyQPzBJ4JbzQpGeq1AAYpWKoXmLI1ZtmNmM5QOzs
+-> ssh-ed25519 HY2yRg 4Xns3jybBuv8flzd+h3DArVBa/AlKjt1J9jAyJsasCE
-qGDlDT31DQF1DdHen0+5+52DdsQlabJdA2pOB5O1I6g
+uyVjJxh5i8aGgAgCpPl6zTYeIkf9mIwURof51IKWvwE
--> ssh-ed25519 CAWG4Q wioWMDxQjN+d4JdIbCwZg0DLQu1OH2mV6gukRprjuAs
+-> ssh-ed25519 CAWG4Q T2r6r1tyNgq1XlYXVtLJFfOfUnm6pSVlPwUqC1pkyRo
-670fE61hidOEh20hHiQAhP0+CjDF0WMBNzgwkGT8Yqg
+9yDoKU0EC34QMUXYnsJvhPCLm6oD9w7NlTi2sheoBqQ
--> ssh-ed25519 MSF3dg DN19uvAEtqq4708P6HpuX9i/o/qAvHX6dj69dCF2H1o
+-> ssh-ed25519 MSF3dg Bh9DekFTq+QMUEAonwcaIAJX4Js1O7cHjDniCD0gtm8
-4Lu9GnjiFLMeXJ2C7aVPJsCHCQVlhylNWJi896Av92s
+t/Ro0URLeDUWcvb7rlkG2s03PZ+9Rr3N4TIX03tXpVc
---- 7cKBwOYNOUZ2h3/kAY09aSMASZSxX7hZIT4kvlIiT6w
+--- E5+/D4aK2ihKRR4YC5XOTmUbKgOqBR0Nk0gYvFOzXOI
-³6—çà•äfQF5=¦bX+‡v e`Ï7/øªA~PÎÖѦ7<15>Ì
+‰ÀÍyKF~djº˜r%¸Š'ÉÓÖPä&_-lŸ”ö&’o¥_ér¯¦r¢ÿß<C3BF>’0ï,­U7†nC·Te…÷[fˆ97ü•…šÙ˦“ÈC!D±E<C2B1>Wé*ÐLAô
x6¾#–¯
’sqôiéËÆäÏŸ“åk ,ùÝ“
-´ÖA÷)·h³ù=oZ¸$é^´V0ñ/Ü…µr
-k¸uœbĶ:R‘<52>>^gŒõ¼ik_*%<0B>a7ùKGæ<47>ÐÖçâ&­PI¶£n