Show breadcrumbs in the web

flake.lock

@@ -29,11 +29,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1694708510,
+        "lastModified": 1694077645,
         "narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
         "ref": "refs/heads/master",
-        "rev": "3a4062ac04be6263c64a481420d8e768c2521b80",
-        "revCount": 862,
+        "rev": "6122fef92701701e1a0622550ac0fc5c2beb5906",
+        "revCount": 860,
         "type": "git",
         "url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
       },

flake.nix

@@ -26,9 +26,6 @@ in
       lake2 = mkConf "lake2";
     };
 
-    packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
-      bscpkgs = bscpkgs.packages.x86_64-linux;
-      nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
-    };
+    packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs;
   };
 }

m/common/main.nix

@@ -28,13 +28,12 @@
 
   nix.nixPath = [
     "nixpkgs=${nixpkgs}"
+    "bscpkgs=${bscpkgs}"
     "jungle=${theFlake.outPath}"
   ];
 
-  nix.settings.flake-registry =
-    pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
-
   nix.registry.nixpkgs.flake = nixpkgs;
+  nix.registry.bscpkgs.flake = bscpkgs;
   nix.registry.jungle.flake = theFlake;
 
   environment.systemPackages = with pkgs; [
@@ -45,8 +44,6 @@
     bsc.osumb
   ];
 
-  programs.direnv.enable = true;
-
   systemd.services."serial-getty@ttyS0" = {
     enable = true;
     wantedBy = [ "getty.target" ];

m/common/zsh.nix

@@ -2,6 +2,7 @@
 
 {
   environment.systemPackages = with pkgs; [
+    direnv
     zsh-completions
     nix-zsh-completions
   ];

m/module/shared-nix-store.nix

@@ -1,69 +0,0 @@
-{ ... }:
-{
-  # Don't make the nix store read-only, as this would prevent the overlay FS
-  # from being able to mount it.
-  boot.readOnlyNixStore = false;
-
-  # The nix-daemon.socket has an unnecessary dependency over the /nix/store
-  # mount point. But that mount point won't be provided until the network is
-  # ready. However, the network-address-eno1.service, has a dependency over
-  # sockets.target, causing a cycle.
-  # One solution is to make the nix-daemon.socket depend only on the socket
-  # patch (which is already covered by ConditionPathIsReadWrite =
-  # /nix/var/nix/daemon-socket), instead on the /nix/store.
-  #
-  # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor =
-  # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get
-  # added by systemd when the override config is merged with the one that Nix
-  # provides:
-  #
-  # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor
-  # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket
-  # 
-  # To fix this, the Nix package is patched to only depend on /nix/var instead.
-  # See ../../pkgs/overlay.nix for details.
-
-  # Mount the hut nix store via NFS in read-only mode.
-  fileSystems."/mnt/hut-nix-store" = {
-    device = "hut:/nix/store";
-    fsType = "nfs";
-    options = [ "ro" ];
-  };
-
-  # A workdir is also needed, so setup a permanent dir using tmpfiles.
-  systemd.tmpfiles.rules = [
-    "d /mnt/nix-work 0700 root root -"
-  ];
-
-  # Mount an overlay in /nix/store using as lower layer the NFS store and upper
-  # layer the disk nix store. The destination is still the nix store in
-  # /nix/store (confusing). We need rw access, as the daemon need to write the
-  # lock files to build derivations locally. Use a systemd mount unit directly
-  # so we can specify the LazyUmount option and we avoid having it mounted
-  # in the stage1 before systemd.
-  systemd.mounts = [
-    {
-      what = "overlay";
-      type = "overlay";
-      where = "/nix/store";
-      # We need the local-fs.target to be ready, so the network interfaces can
-      # be configured to the network.target is reached. So make this a netdev
-      # mount.
-      options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
-      description = "Overlay /nix/store mount";
-      mountConfig = {
-        LazyUnmount = true;
-      };
-
-      # Run the unit after remote-fs-pre.target but before the remote-fs.target
-      after = [ "remote-fs-pre.target"];
-      before = [ "umount.target" "remote-fs.target" ];
-      # Install by using wantedBy over remote-fs.target
-      wantedBy = [ "remote-fs.target" ];
-      unitConfig = {
-        # We need to wait for the NFS mount
-        RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
-      };
-    }
-  ];
-}

m/owl1/configuration.nix

@@ -5,7 +5,6 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
-    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/owl2/configuration.nix

@@ -5,7 +5,6 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
-    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches

pkgs/nix-socket.patch

@@ -1,11 +0,0 @@
---- a/misc/systemd/nix-daemon.socket.in	1970-01-01 01:00:01.000000000 +0100
-+++ b/misc/systemd/nix-daemon.socket.in	2023-09-18 17:53:32.351760208 +0200
-@@ -1,7 +1,7 @@
- [Unit]
- Description=Nix Daemon Socket
- Before=multi-user.target
--RequiresMountsFor=@storedir@
-+RequiresMountsFor=@localstatedir@
- ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
- 
- [Socket]

pkgs/overlay.nix

@@ -32,8 +32,4 @@ final: prev:
     lua = prev.lua5_4;
     fmt = prev.fmt_8;
   }) ceph ceph-client;
-
-  nix = prev.nix.overrideAttrs (old: {
-    patches = old.patches ++ [ ./nix-socket.patch ];
-  });
 }

web/content/intro-nix/_index.md

@@ -0,0 +1,7 @@
+---
+title: "Intro to nix"
+date: 2023-09-15
+---
+
+Basic introduction to Nix for users of the jungle machines. You should be able
+to access the jungle machines, otherwise [request access](/access).

web/content/intro-nix/ch1.md

@@ -0,0 +1,100 @@
+---
+title: "Chapter 1: Packages"
+description: "Here we show where packages come from"
+date: 2023-06-13T19:36:57+02:00
+weight: 1
+---
+
+In this chapter we describe where the packages available in the cluster come
+from and how to load them.
+
+## Where packages come from
+
+The packages in the jungle cluster are constructed by *layers*. Each layer
+applies some changes over the previous one:
+
+1. The first layer is [nixpkgs][1], a large repository of packages maintained by
+   the NixOS community. It provides packages like gcc, bash, gcc or the linux
+   kernel.
+
+[1]: https://github.com/NixOS/nixpkgs/
+
+2. The second layer is [bscpkgs][2], it takes the nixpkgs set of packages and
+   expands it by adding custom packages from the BSC such as Nanos6, nOS-V,
+   NODES, ovni or wxparaver.
+
+[2]: https://pm.bsc.es/gitlab/rarias/bscpkgs
+
+3. The third layer is [jungle][3], it takes the extended packages from bscpkgs
+   and configures them for the jungle cluster. For example, we configure MPICH
+   to use the OmniPath network and set it as the default implementation.
+
+[3]: https://pm.bsc.es/gitlab/rarias/jungle
+
+These layers are called *overlays* in Nix and they are the default mechanism
+used to modify the packages. Generally you will use the packages defined in the
+last layer (jungle) but you can define your own additional layer to specify
+custom changes. For example, instead of choosing MPICH, you may want to use
+Intel MPI instead by default.
+
+## Loading packages in an ephemeral shell
+
+You can manually load packages in a *new* shell with `nix shell jungle#<pkg>`,
+for example:
+
+```
+hut% which ovniemu
+ovniemu not found
+hut% nix shell jungle#bsc.ovni
+hut% which ovniemu
+/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
+hut% exit
+hut%
+```
+
+You can also specify multiple packages by listing them as parameters of `nix
+shell`:
+
+```
+hut% nix shell jungle#bsc.ovni jungle#bsc.osumb
+hut% which osu_bw
+/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
+hut% exit
+```
+
+Or make the bash (zsh in this case) shell expand them:
+
+```
+hut% echo nix shell jungle#bsc.{ovni,osumb}
+nix shell jungle#bsc.ovni jungle#bsc.osumb
+hut% nix shell jungle#bsc.{ovni,osumb}
+hut% which osu_bw
+/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
+hut% exit
+```
+
+You can use TAB to see which packages are available:
+
+```
+hut% nix shell jungle#bsc.n<TAB>
+jungle\#bsc.nanos6              jungle\#bsc.nixtools
+jungle\#bsc.nanos6Debug         jungle\#bsc.nix-wrap
+jungle\#bsc.nanos6Git           jungle\#bsc.nodes
+jungle\#bsc.nanos6GlibcxxDebug  jungle\#bsc.nodesGit
+jungle\#bsc.nanos6-icc          jungle\#bsc.nodesRelease
+jungle\#bsc.nanos6-icx          jungle\#bsc.nodesWithOvni
+jungle\#bsc.nanos6Release       jungle\#bsc.nosv
+jungle\#bsc.nix-mn4
+```
+
+Notice that these packages are evaluated at the moment the command is invoked.
+So if you come back a month later and run the same command, you may find that
+the packages have been updated and that could be problematic.
+
+In the next section we will create a new flake that defines the packages of the
+shell and also records the exact version of the packages that we used at the
+evaluation time for future use.
+
+In the [next chapter](../ch2) we will see how to create a permanent shell that
+will retain the same packages even if they are upgraded in the cluster, until we
+decide to upgrade them.

web/content/intro-nix/ch2.md

@@ -0,0 +1,155 @@
+---
+title: "Chapter 2: Your first shell"
+date: 2023-09-15
+weight: 2
+---
+
+## Creating a shell with flake.nix
+
+First, create an empty git repository where your shells will live:
+
+```txt
+hut% mkdir jungle-examples
+hut% cd jungle-examples
+hut% git init
+Initialized empty Git repository in /home/Computational/rarias/jungle-examples/.git/
+```
+
+And then, place a file named `flake.nix` on the repo with this content:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  nixConfig.bash-prompt = "\[nix-develop\]$ ";
+
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell rec {
+      pname = "my-shell";
+      buildInputs = with pkgs.bsc; [
+        ovni osumb # other packages here...
+      ];
+    };
+  };
+}
+```
+
+This file defines a how to create a shell in the Nix language with the
+`pkgs.mkShell` function using the packages listed in `buildInputs`. It also
+requests the packages to be taken from the *jungle* input, which corresponds to
+the set of packages that [we defined earlier](../ch1#where-packages-come-from),
+tuned for the cluster. We will describe it in more detail later.
+
+The tool `nix develop` tries to find a flake.nix in the current directory and
+enter the shell described by `devShells.x86_64-linux.default` (or the
+corresponding architecture).
+
+Now, **it is important that all the files of the repository are committed in
+git**, as nix will only read what is in the index of git. If we try to enter the
+shell with the `nix develop` command, it will complain and fail:
+
+```txt
+hut% nix develop
+warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
+error: getting status of '/nix/store/0ccnxa25whszw7mgbgyzdm4nqc0zwnm8-source/flake.nix': No such file or directory
+```
+
+The first warning states that the git directory has modified files not added to
+the index. Then the error occurs because the flake.nix is not in the index of
+git, so `nix develop` doesn't see it. So let's add it to a commit and try again:
+
+```txt
+hut% git add flake.nix
+hut% git commit flake.nix -m 'First shell'
+[master (root-commit) eb8a4ac] First shell
+ 1 file changed, 13 insertions(+)
+ create mode 100644 flake.nix
+hut% nix develop
+warning: creating lock file '/home/Computational/rarias/jungle-examples/flake.lock'
+warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
+[nix-develop]$
+```
+
+In the `flake.nix` we have set the shell prompt to `[nix-develop]` so we can
+easily spot that we are inside a `nix develop` shell. To exit:
+
+```txt
+[nix-develop]$ exit
+hut%
+```
+
+## Using the flake.lock file
+
+Now we see the `creating lock file` message and the git tree becomes dirty
+again (however, we enter the shell successfully).
+
+This `flake.lock` file that has been created collects the current state of the
+jungle packages in a file, so future invocations will use the same versions. We
+can see more details with `nix flake metadata`:
+
+```txt
+hut% nix flake metadata
+warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
+Resolved URL:  git+file:///home/Computational/rarias/jungle-examples
+Locked URL:    git+file:///home/Computational/rarias/jungle-examples
+Path:          /nix/store/bckxqjkkv52hy4pzgb96r7fchhmvmql8-source
+Revision:      eb8a4ac544a74e3995d859c751e9ff4339de6509-dirty
+Last modified: 2023-09-15 13:06:12
+Inputs:
+└───jungle: path:/nix/store/3wv6q0f3pkgw840nnkn4jsp9xi650dyj-source?lastModified=1694772033&narHash=sha256-7a09O0Jb8WncxeB32ywmQEMqJdEFLrOG/XVT9bdII6I%3D&rev=653d411b9e46076a7878be9574ed6b3bd627cff1&revCount=195
+    ├───agenix: github:ryantm/agenix/d8c973fd228949736dedf61b7f8cc1ece3236792
+    │   ├───darwin: github:lnl7/nix-darwin/87b9d090ad39b25b2400029c64825fc2a8868943
+    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
+    │   ├───home-manager: github:nix-community/home-manager/32d3e39c491e2f91152c84f8ad8b003420eab0a1
+    │   │   └───nixpkgs follows input 'jungle/agenix/nixpkgs'
+    │   └───nixpkgs follows input 'jungle/nixpkgs'
+    ├───bscpkgs: git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=3a4062ac04be6263c64a481420d8e768c2521b80
+    │   └───nixpkgs follows input 'jungle/nixpkgs'
+    └───nixpkgs: github:NixOS/nixpkgs/e56990880811a451abd32515698c712788be5720
+```
+
+Now, as long as we keep these two files `flake.nix` and `flake.lock`, we can
+reproduce the same shell in the future, so let's add the lock file into git too.
+
+```txt
+hut% git commit -m 'Add flake.lock file'
+[master d3725ec] Add flake.lock file
+ 1 file changed, 135 insertions(+)
+ create mode 100644 flake.lock
+hut% git status
+On branch master
+nothing to commit, working tree clean
+```
+
+## Using the shell with nix develop
+
+Now, the invocations of `nix develop` won't complain that the git tree is dirty
+anymore and will enter the shell:
+
+```txt
+hut% nix develop
+[nix-develop]$
+```
+
+And the requested packages are now available:
+
+```txt
+[nix-develop]$ which ovniemu
+/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
+```
+
+The packages of the shell are listed in the `$buildInputs` variable, in case you
+need to examine them:
+
+```txt
+[nix-develop]$ printf '%s\n' $buildInputs
+/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
+/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
+[nix-develop]$ exit
+hut%
+```
+
+In the [next chapter](../ch3) we will see how to add more packages and also how to modify
+them.

web/content/intro-nix/ch3.md

@@ -0,0 +1,160 @@
+---
+title: "Chapter 3: Custom packages"
+date: 2023-09-15
+weight: 3
+---
+
+## Adding more packages
+
+So far we have define all the packages using:
+
+```nix
+pkgs.mkShell rec {
+  pname = "my-shell";
+  buildInputs = with pkgs.bsc; [
+    ovni osumb # other packages here...
+  ];
+};
+```
+
+This line specifies that all packages come from the `pkgs.bsc` set. We can add
+additional packages adding them to the list:
+
+```nix
+pkgs.mkShell rec {
+  pname = "my-shell";
+  buildInputs = with pkgs.bsc; [
+    ovni osumb sonar
+  ];
+};
+```
+
+And running `nix develop` again:
+
+```txt
+hut% nix develop
+warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
+[nix-develop]$ printf '%s\n' $buildInputs
+/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
+/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
+/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0
+```
+
+In the jungle cluster, the default MPI implementation is currently set to MPICH,
+as it can be shown with ldd:
+
+```txt
+[nix-develop]$ ldd $(which ovnisync) | grep mpi
+        libmpi.so.12 => /nix/store/nnnaly6hgylravdrmqkhpx1ndg5p79nc-mpich-4.1.2/lib/libmpi.so.12 (0x00007ffff5200000)
+```
+
+Now, what if we want to replace the MPI implementation by another one?
+
+## Modifying a package
+
+You notice that the packages we are using are coming directly from the ones
+specified in jungle. However, what if we need to modify some option at build
+time or change a dependency?
+
+The Nix language is used to describe how to build each package, and can be
+extended to create derived versions very easily.
+
+Let's focus on the `ovni` package. First, to load the definition we can use the
+`nix edit` command, which opens the definition file using the editor defined in
+`$EDITOR`:
+
+```txt
+hut% nix edit jungle#bsc.ovni
+...
+```
+
+This particular package has several inputs that can be modified directly:
+
+```txt
+{
+  stdenv
+, lib
+, cmake
+, mpi
+, fetchFromGitHub
+, useGit ? false
+, gitBranch ? "master"
+, gitUrl ? "ssh://git@bscpm03.bsc.es/rarias/ovni.git"
+, gitCommit ? "d0a47783f20f8b177a48418966dae45454193a6a"
+, enableDebug ? false
+}:
+...
+```
+
+For example, the `enableDebug` flag, currently set to false, affects how the
+build is configured:
+
+```txt
+cmakeBuildType = if (enableDebug) then "Debug" else "Release";
+```
+
+Now, to change this option we could replace `ovni` for our version:
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  nixConfig.bash-prompt = "\[nix-develop\]$ ";
+
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell rec {
+      pname = "my-shell";
+      buildInputs = with pkgs.bsc; [
+        ovniDebug osumb sonar
+      ];
+    };
+  };
+}
+```
+
+And then, when we now enter the develop shell we can see that ovni gets build
+with the Debug option:
+
+```txt
+hut% nix develop -L
+warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
+ovni> unpacking sources
+ovni> unpacking source archive /nix/store/cz4si0vsw85r9s6dyiqr5ybngh9aympi-source
+ovni> source root is source
+ovni> patching sources
+ovni> updateAutotoolsGnuConfigScriptsPhase
+ovni> configuring
+ovni> fixing cmake files...
+ovni> cmake flags: ... -DCMAKE_BUILD_TYPE=Debug ...
+...
+[nix-develop]$ which ovniver
+/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver
+
+[nix-develop]$ file $(which ovniver) | fold
+/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver: ELF 64-bit L
+SB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /nix/st
+ore/9la894yvmmksqlapd4v16wvxpaw3rg70-glibc-2.37-8/lib/ld-linux-x86-64.so.2, for
+GNU/Linux 3.10.0, with debug_info, not stripped
+```
+
+And we see that the ovniver program is now compiled with debug symbols.
+
+However, this *only* replaces the ovni package that we specify in the shell. The
+sonar library also depends on ovni, but that package is still using the old one:
+
+```txt
+[nix-develop]$ find $buildInputs -name 'libovni.so.1'
+/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/lib/libovni.so.1
+
+[nix-develop]$ find $buildInputs -name 'libsonar-mpi.so'
+/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so
+
+[nix-develop]$ ldd /nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so | grep ovni
+        libovni.so.1 => /nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/lib/libovni.so.1 (0x00007ffff7f8d000)
+```
+
+In the [next chapter](../ch4) we will see how to replace packages in such a way
+that all the dependences are automatically updated too.

web/content/intro-nix/ch4.md

@@ -0,0 +1,29 @@
+---
+title: "Chapter 4: Adding an overlay"
+date: 2023-09-15
+weight: 4
+---
+
+NOTE: We shouldn't be instructing users to use an overlay to replace packages in
+`bsc.` until we have determined if we move them to the root attribute set
+first!
+
+```nix
+{
+  inputs.jungle.url = "jungle";
+  nixConfig.bash-prompt = "\[nix-develop\]$ ";
+
+  outputs = { self, jungle }:  
+  let
+    pkgs = jungle.outputs.packages.x86_64-linux;
+    ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
+  in {
+    devShells.x86_64-linux.default = pkgs.mkShell rec {
+      pname = "my-shell";
+      buildInputs = with pkgs.bsc; [
+        ovniDebug osumb sonar
+      ];
+    };
+  };
+}
+```

web/hugo.toml

@@ -3,3 +3,6 @@ languageCode = 'en-us'
 title = 'The jungle'
 theme = 'PaperMod'
 sectionPagesMenu = "main"
+
+[params]
+ShowBreadCrumbs = true