Use NixOS attributes for the install section

Allows the LazyUnmount option and avoids the stage1 hack with
/nix//store.

flake.lock

@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1694077645,
+        "lastModified": 1694708510,
         "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
         "ref": "refs/heads/master",
-        "rev": "6122fef92701701e1a0622550ac0fc5c2beb5906",
+        "rev": "3a4062ac04be6263c64a481420d8e768c2521b80",
-        "revCount": 860,
+        "revCount": 862,
         "type": "git",
         "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
       },

flake.nix

@@ -26,6 +26,9 @@ in
       lake2 = mkConf "lake2";
     };
 
-    packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs;
+    packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
+      bscpkgs = bscpkgs.packages.x86_64-linux;
+      nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
+    };
   };
 }

m/common/main.nix

@@ -28,12 +28,13 @@
 
   nix.nixPath = [
     "nixpkgs=${nixpkgs}"
-    "bscpkgs=${bscpkgs}"
     "jungle=${theFlake.outPath}"
   ];
 
+  nix.settings.flake-registry =
+    pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
+
   nix.registry.nixpkgs.flake = nixpkgs;
-  nix.registry.bscpkgs.flake = bscpkgs;
   nix.registry.jungle.flake = theFlake;
 
   environment.systemPackages = with pkgs; [
@@ -44,6 +45,8 @@
     bsc.osumb
   ];
 
+  programs.direnv.enable = true;
+
   systemd.services."serial-getty@ttyS0" = {
     enable = true;
     wantedBy = [ "getty.target" ];

m/common/zsh.nix

@@ -2,7 +2,6 @@
 
 {
   environment.systemPackages = with pkgs; [
-    direnv
     zsh-completions
     nix-zsh-completions
   ];

m/module/shared-nix-store.nix

@@ -0,0 +1,69 @@
+{ ... }:
+{
+  # Don't make the nix store read-only, as this would prevent the overlay FS
+  # from being able to mount it.
+  boot.readOnlyNixStore = false;
+
+  # The nix-daemon.socket has an unnecessary dependency over the /nix/store
+  # mount point. But that mount point won't be provided until the network is
+  # ready. However, the network-address-eno1.service, has a dependency over
+  # sockets.target, causing a cycle.
+  # One solution is to make the nix-daemon.socket depend only on the socket
+  # patch (which is already covered by ConditionPathIsReadWrite =
+  # /nix/var/nix/daemon-socket), instead on the /nix/store.
+  #
+  # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor =
+  # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get
+  # added by systemd when the override config is merged with the one that Nix
+  # provides:
+  #
+  # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor
+  # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket
+  # 
+  # To fix this, the Nix package is patched to only depend on /nix/var instead.
+  # See ../../pkgs/overlay.nix for details.
+
+  # Mount the hut nix store via NFS in read-only mode.
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  # A workdir is also needed, so setup a permanent dir using tmpfiles.
+  systemd.tmpfiles.rules = [
+    "d /mnt/nix-work 0700 root root -"
+  ];
+
+  # Mount an overlay in /nix/store using as lower layer the NFS store and upper
+  # layer the disk nix store. The destination is still the nix store in
+  # /nix/store (confusing). We need rw access, as the daemon need to write the
+  # lock files to build derivations locally. Use a systemd mount unit directly
+  # so we can specify the LazyUmount option and we avoid having it mounted
+  # in the stage1 before systemd.
+  systemd.mounts = [
+    {
+      what = "overlay";
+      type = "overlay";
+      where = "/nix/store";
+      # We need the local-fs.target to be ready, so the network interfaces can
+      # be configured to the network.target is reached. So make this a netdev
+      # mount.
+      options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
+      description = "Overlay /nix/store mount";
+      mountConfig = {
+        LazyUnmount = true;
+      };
+
+      # Run the unit after remote-fs-pre.target but before the remote-fs.target
+      after = [ "remote-fs-pre.target"];
+      before = [ "umount.target" "remote-fs.target" ];
+      # Install by using wantedBy over remote-fs.target
+      wantedBy = [ "remote-fs.target" ];
+      unitConfig = {
+        # We need to wait for the NFS mount
+        RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
+      };
+    }
+  ];
+}

m/owl1/configuration.nix

@@ -5,6 +5,7 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -5,6 +5,7 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

pkgs/nix-socket.patch

@@ -0,0 +1,11 @@
+--- a/misc/systemd/nix-daemon.socket.in	1970-01-01 01:00:01.000000000 +0100
++++ b/misc/systemd/nix-daemon.socket.in	2023-09-18 17:53:32.351760208 +0200
+@@ -1,7 +1,7 @@
+ [Unit]
+ Description=Nix Daemon Socket
+ Before=multi-user.target
+-RequiresMountsFor=@storedir@
++RequiresMountsFor=@localstatedir@
+ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
+ 
+ [Socket]

pkgs/overlay.nix

@@ -32,4 +32,8 @@ final: prev:
     lua = prev.lua5_4;
     fmt = prev.fmt_8;
   }) ceph ceph-client;
+
+  nix = prev.nix.overrideAttrs (old: {
+    patches = old.patches ++ [ ./nix-socket.patch ];
+  });
 }

web/content/intro-nix/_index.md

@@ -1,7 +0,0 @@
----
-title: "Intro to nix"
-date: 2023-09-15
----
-
-Basic introduction to Nix for users of the jungle machines. You should be able
-to access the jungle machines, otherwise [request access](/access).

web/content/intro-nix/ch1.md

@@ -1,100 +0,0 @@
----
-title: "Chapter 1: Packages"
-description: "Here we show where packages come from"
-date: 2023-06-13T19:36:57+02:00
-weight: 1
----
-
-In this chapter we describe where the packages available in the cluster come
-from and how to load them.
-
-## Where packages come from
-
-The packages in the jungle cluster are constructed by *layers*. Each layer
-applies some changes over the previous one:
-
-1. The first layer is [nixpkgs][1], a large repository of packages maintained by
-   the NixOS community. It provides packages like gcc, bash, gcc or the linux
-   kernel.
-
-[1]: https://github.com/NixOS/nixpkgs/
-
-2. The second layer is [bscpkgs][2], it takes the nixpkgs set of packages and
-   expands it by adding custom packages from the BSC such as Nanos6, nOS-V,
-   NODES, ovni or wxparaver.
-
-[2]: https://pm.bsc.es/gitlab/rarias/bscpkgs
-
-3. The third layer is [jungle][3], it takes the extended packages from bscpkgs
-   and configures them for the jungle cluster. For example, we configure MPICH
-   to use the OmniPath network and set it as the default implementation.
-
-[3]: https://pm.bsc.es/gitlab/rarias/jungle
-
-These layers are called *overlays* in Nix and they are the default mechanism
-used to modify the packages. Generally you will use the packages defined in the
-last layer (jungle) but you can define your own additional layer to specify
-custom changes. For example, instead of choosing MPICH, you may want to use
-Intel MPI instead by default.
-
-## Loading packages in an ephemeral shell
-
-You can manually load packages in a *new* shell with `nix shell jungle#<pkg>`,
-for example:
-
-```
-hut% which ovniemu
-ovniemu not found
-hut% nix shell jungle#bsc.ovni
-hut% which ovniemu
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
-hut% exit
-hut%
-```
-
-You can also specify multiple packages by listing them as parameters of `nix
-shell`:
-
-```
-hut% nix shell jungle#bsc.ovni jungle#bsc.osumb
-hut% which osu_bw
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
-hut% exit
-```
-
-Or make the bash (zsh in this case) shell expand them:
-
-```
-hut% echo nix shell jungle#bsc.{ovni,osumb}
-nix shell jungle#bsc.ovni jungle#bsc.osumb
-hut% nix shell jungle#bsc.{ovni,osumb}
-hut% which osu_bw
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
-hut% exit
-```
-
-You can use TAB to see which packages are available:
-
-```
-hut% nix shell jungle#bsc.n<TAB>
-jungle\#bsc.nanos6              jungle\#bsc.nixtools
-jungle\#bsc.nanos6Debug         jungle\#bsc.nix-wrap
-jungle\#bsc.nanos6Git           jungle\#bsc.nodes
-jungle\#bsc.nanos6GlibcxxDebug  jungle\#bsc.nodesGit
-jungle\#bsc.nanos6-icc          jungle\#bsc.nodesRelease
-jungle\#bsc.nanos6-icx          jungle\#bsc.nodesWithOvni
-jungle\#bsc.nanos6Release       jungle\#bsc.nosv
-jungle\#bsc.nix-mn4
-```
-
-Notice that these packages are evaluated at the moment the command is invoked.
-So if you come back a month later and run the same command, you may find that
-the packages have been updated and that could be problematic.
-
-In the next section we will create a new flake that defines the packages of the
-shell and also records the exact version of the packages that we used at the
-evaluation time for future use.
-
-In the [next chapter](../ch2) we will see how to create a permanent shell that
-will retain the same packages even if they are upgraded in the cluster, until we
-decide to upgrade them.

web/content/intro-nix/ch2.md

@@ -1,155 +0,0 @@
----
-title: "Chapter 2: Your first shell"
-date: 2023-09-15
-weight: 2
----
-
-## Creating a shell with flake.nix
-
-First, create an empty git repository where your shells will live:
-
-```txt
-hut% mkdir jungle-examples
-hut% cd jungle-examples
-hut% git init
-Initialized empty Git repository in /home/Computational/rarias/jungle-examples/.git/
-```
-
-And then, place a file named `flake.nix` on the repo with this content:
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovni osumb # other packages here...
-      ];
-    };
-  };
-}
-```
-
-This file defines a how to create a shell in the Nix language with the
-`pkgs.mkShell` function using the packages listed in `buildInputs`. It also
-requests the packages to be taken from the *jungle* input, which corresponds to
-the set of packages that [we defined earlier](../ch1#where-packages-come-from),
-tuned for the cluster. We will describe it in more detail later.
-
-The tool `nix develop` tries to find a flake.nix in the current directory and
-enter the shell described by `devShells.x86_64-linux.default` (or the
-corresponding architecture).
-
-Now, **it is important that all the files of the repository are committed in
-git**, as nix will only read what is in the index of git. If we try to enter the
-shell with the `nix develop` command, it will complain and fail:
-
-```txt
-hut% nix develop
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-error: getting status of '/nix/store/0ccnxa25whszw7mgbgyzdm4nqc0zwnm8-source/flake.nix': No such file or directory
-```
-
-The first warning states that the git directory has modified files not added to
-the index. Then the error occurs because the flake.nix is not in the index of
-git, so `nix develop` doesn't see it. So let's add it to a commit and try again:
-
-```txt
-hut% git add flake.nix
-hut% git commit flake.nix -m 'First shell'
-[master (root-commit) eb8a4ac] First shell
- 1 file changed, 13 insertions(+)
- create mode 100644 flake.nix
-hut% nix develop
-warning: creating lock file '/home/Computational/rarias/jungle-examples/flake.lock'
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-[nix-develop]$
-```
-
-In the `flake.nix` we have set the shell prompt to `[nix-develop]` so we can
-easily spot that we are inside a `nix develop` shell. To exit:
-
-```txt
-[nix-develop]$ exit
-hut%
-```
-
-## Using the flake.lock file
-
-Now we see the `creating lock file` message and the git tree becomes dirty
-again (however, we enter the shell successfully).
-
-This `flake.lock` file that has been created collects the current state of the
-jungle packages in a file, so future invocations will use the same versions. We
-can see more details with `nix flake metadata`:
-
-```txt
-hut% nix flake metadata
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-Resolved URL:  git+file:///home/Computational/rarias/jungle-examples
-Locked URL:    git+file:///home/Computational/rarias/jungle-examples
-Path:          /nix/store/bckxqjkkv52hy4pzgb96r7fchhmvmql8-source
-Revision:      eb8a4ac544a74e3995d859c751e9ff4339de6509-dirty
-Last modified: 2023-09-15 13:06:12
-Inputs:
-└───jungle: path:/nix/store/3wv6q0f3pkgw840nnkn4jsp9xi650dyj-source?lastModified=1694772033&narHash=sha256-7a09O0Jb8WncxeB32ywmQEMqJdEFLrOG/XVT9bdII6I%3D&rev=653d411b9e46076a7878be9574ed6b3bd627cff1&revCount=195
-    ├───agenix: github:ryantm/agenix/d8c973fd228949736dedf61b7f8cc1ece3236792
-    │   ├───darwin: github:lnl7/nix-darwin/87b9d090ad39b25b2400029c64825fc2a8868943
-    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
-    │   ├───home-manager: github:nix-community/home-manager/32d3e39c491e2f91152c84f8ad8b003420eab0a1
-    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
-    │   └───nixpkgs follows input 'jungle/nixpkgs'
-    ├───bscpkgs: git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=3a4062ac04be6263c64a481420d8e768c2521b80
-    │   └───nixpkgs follows input 'jungle/nixpkgs'
-    └───nixpkgs: github:NixOS/nixpkgs/e56990880811a451abd32515698c712788be5720
-```
-
-Now, as long as we keep these two files `flake.nix` and `flake.lock`, we can
-reproduce the same shell in the future, so let's add the lock file into git too.
-
-```txt
-hut% git commit -m 'Add flake.lock file'
-[master d3725ec] Add flake.lock file
- 1 file changed, 135 insertions(+)
- create mode 100644 flake.lock
-hut% git status
-On branch master
-nothing to commit, working tree clean
-```
-
-## Using the shell with nix develop
-
-Now, the invocations of `nix develop` won't complain that the git tree is dirty
-anymore and will enter the shell:
-
-```txt
-hut% nix develop
-[nix-develop]$
-```
-
-And the requested packages are now available:
-
-```txt
-[nix-develop]$ which ovniemu
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
-```
-
-The packages of the shell are listed in the `$buildInputs` variable, in case you
-need to examine them:
-
-```txt
-[nix-develop]$ printf '%s\n' $buildInputs
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
-[nix-develop]$ exit
-hut%
-```
-
-In the [next chapter](../ch3) we will see how to add more packages and also how to modify
-them.

web/content/intro-nix/ch3.md

@@ -1,160 +0,0 @@
----
-title: "Chapter 3: Custom packages"
-date: 2023-09-15
-weight: 3
----
-
-## Adding more packages
-
-So far we have define all the packages using:
-
-```nix
-pkgs.mkShell rec {
-  pname = "my-shell";
-  buildInputs = with pkgs.bsc; [
-    ovni osumb # other packages here...
-  ];
-};
-```
-
-This line specifies that all packages come from the `pkgs.bsc` set. We can add
-additional packages adding them to the list:
-
-```nix
-pkgs.mkShell rec {
-  pname = "my-shell";
-  buildInputs = with pkgs.bsc; [
-    ovni osumb sonar
-  ];
-};
-```
-
-And running `nix develop` again:
-
-```txt
-hut% nix develop
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-[nix-develop]$ printf '%s\n' $buildInputs
-/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
-/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
-/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0
-```
-
-In the jungle cluster, the default MPI implementation is currently set to MPICH,
-as it can be shown with ldd:
-
-```txt
-[nix-develop]$ ldd $(which ovnisync) | grep mpi
-        libmpi.so.12 => /nix/store/nnnaly6hgylravdrmqkhpx1ndg5p79nc-mpich-4.1.2/lib/libmpi.so.12 (0x00007ffff5200000)
-```
-
-Now, what if we want to replace the MPI implementation by another one?
-
-## Modifying a package
-
-You notice that the packages we are using are coming directly from the ones
-specified in jungle. However, what if we need to modify some option at build
-time or change a dependency?
-
-The Nix language is used to describe how to build each package, and can be
-extended to create derived versions very easily.
-
-Let's focus on the `ovni` package. First, to load the definition we can use the
-`nix edit` command, which opens the definition file using the editor defined in
-`$EDITOR`:
-
-```txt
-hut% nix edit jungle#bsc.ovni
-...
-```
-
-This particular package has several inputs that can be modified directly:
-
-```txt
-{
-  stdenv
-, lib
-, cmake
-, mpi
-, fetchFromGitHub
-, useGit ? false
-, gitBranch ? "master"
-, gitUrl ? "ssh://git@bscpm03.bsc.es/rarias/ovni.git"
-, gitCommit ? "d0a47783f20f8b177a48418966dae45454193a6a"
-, enableDebug ? false
-}:
-...
-```
-
-For example, the `enableDebug` flag, currently set to false, affects how the
-build is configured:
-
-```txt
-cmakeBuildType = if (enableDebug) then "Debug" else "Release";
-```
-
-Now, to change this option we could replace `ovni` for our version:
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovniDebug osumb sonar
-      ];
-    };
-  };
-}
-```
-
-And then, when we now enter the develop shell we can see that ovni gets build
-with the Debug option:
-
-```txt
-hut% nix develop -L
-warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
-ovni> unpacking sources
-ovni> unpacking source archive /nix/store/cz4si0vsw85r9s6dyiqr5ybngh9aympi-source
-ovni> source root is source
-ovni> patching sources
-ovni> updateAutotoolsGnuConfigScriptsPhase
-ovni> configuring
-ovni> fixing cmake files...
-ovni> cmake flags: ... -DCMAKE_BUILD_TYPE=Debug ...
-...
-[nix-develop]$ which ovniver
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver
-
-[nix-develop]$ file $(which ovniver) | fold
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver: ELF 64-bit L
-SB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /nix/st
-ore/9la894yvmmksqlapd4v16wvxpaw3rg70-glibc-2.37-8/lib/ld-linux-x86-64.so.2, for
-GNU/Linux 3.10.0, with debug_info, not stripped
-```
-
-And we see that the ovniver program is now compiled with debug symbols.
-
-However, this *only* replaces the ovni package that we specify in the shell. The
-sonar library also depends on ovni, but that package is still using the old one:
-
-```txt
-[nix-develop]$ find $buildInputs -name 'libovni.so.1'
-/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/lib/libovni.so.1
-
-[nix-develop]$ find $buildInputs -name 'libsonar-mpi.so'
-/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so
-
-[nix-develop]$ ldd /nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so | grep ovni
-        libovni.so.1 => /nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/lib/libovni.so.1 (0x00007ffff7f8d000)
-```
-
-In the [next chapter](../ch4) we will see how to replace packages in such a way
-that all the dependences are automatically updated too.

web/content/intro-nix/ch4.md

@@ -1,29 +0,0 @@
----
-title: "Chapter 4: Adding an overlay"
-date: 2023-09-15
-weight: 4
----
-
-NOTE: We shouldn't be instructing users to use an overlay to replace packages in
-`bsc.` until we have determined if we move them to the root attribute set
-first!
-
-```nix
-{
-  inputs.jungle.url = "jungle";
-  nixConfig.bash-prompt = "\[nix-develop\]$ ";
-
-  outputs = { self, jungle }:  
-  let
-    pkgs = jungle.outputs.packages.x86_64-linux;
-    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
-  in {
-    devShells.x86_64-linux.default = pkgs.mkShell rec {
-      pname = "my-shell";
-      buildInputs = with pkgs.bsc; [
-        ovniDebug osumb sonar
-      ];
-    };
-  };
-}
-```

web/hugo.toml

@@ -3,6 +3,3 @@ languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
 sectionPagesMenu = "main"
-
-[params]
-ShowBreadCrumbs = true