From d48f3b989a9cbf2ddd5e331903872f7abc1029a6 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Sun, 17 Sep 2023 22:27:51 +0200
Subject: [PATCH 01/11] Enable direnv integration

---
 m/common/main.nix | 2 ++
 m/common/zsh.nix  | 1 -
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/m/common/main.nix b/m/common/main.nix
index 025a7c5..999f043 100644
--- a/m/common/main.nix
+++ b/m/common/main.nix
@@ -45,6 +45,8 @@
     bsc.osumb
   ];
 
+  programs.direnv.enable = true;
+
   systemd.services."serial-getty@ttyS0" = {
     enable = true;
     wantedBy = [ "getty.target" ];
diff --git a/m/common/zsh.nix b/m/common/zsh.nix
index 666e34b..5cfcb7f 100644
--- a/m/common/zsh.nix
+++ b/m/common/zsh.nix
@@ -2,7 +2,6 @@
 
 {
   environment.systemPackages = with pkgs; [
-    direnv
     zsh-completions
     nix-zsh-completions
   ];
-- 
2.45.2


From 140178d58ec12df71e31747850337cc64d70ae9c Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 11:22:24 +0200
Subject: [PATCH 02/11] Begin the nix store overlay

We need to disable the read-only bind mount, so we can directly bind
mount the overlay.
---
 m/module/shared-nix-store.nix | 18 ++++++++++++++++++
 m/owl1/configuration.nix      |  1 +
 m/owl2/configuration.nix      |  1 +
 3 files changed, 20 insertions(+)
 create mode 100644 m/module/shared-nix-store.nix

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
new file mode 100644
index 0000000..1c19fd9
--- /dev/null
+++ b/m/module/shared-nix-store.nix
@@ -0,0 +1,18 @@
+{ ... }:
+{
+  # Don't make the nix store read-only, as this would prevent the overlay FS
+  # from being able to mount it.
+  boot.readOnlyNixStore = false;
+
+  # Mount the hut nix store via NFS
+  fileSystems."/mnt/hut-nix-store" = {
+    device = "hut:/nix/store";
+    fsType = "nfs";
+    options = [ "ro" ];
+  };
+
+  # Create the work directory
+  #systemd.tmpfiles.rules = [
+  #  "d /mnt/nix-overlay/work 0550 1000 root root -"
+  #];
+}
diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix
index 5afda72..052ea8a 100644
--- a/m/owl1/configuration.nix
+++ b/m/owl1/configuration.nix
@@ -5,6 +5,7 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches
diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix
index 9636760..ce95fa2 100644
--- a/m/owl2/configuration.nix
+++ b/m/owl2/configuration.nix
@@ -5,6 +5,7 @@
     ../common/main.nix
     ../module/ceph.nix
     ../module/slurm-firewall.nix
+    ../module/shared-nix-store.nix
   ];
 
   # Select the this using the ID to avoid mismatches
-- 
2.45.2


From d145ee9b2cdef779ff1ca9511a1d90857f4c153d Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 13:02:32 +0200
Subject: [PATCH 03/11] Mount the overlay in /nix/store

---
 m/module/shared-nix-store.nix | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 1c19fd9..72ece72 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -4,15 +4,26 @@
   # from being able to mount it.
   boot.readOnlyNixStore = false;
 
-  # Mount the hut nix store via NFS
+  # Mount the hut nix store via NFS in read-only mode.
   fileSystems."/mnt/hut-nix-store" = {
     device = "hut:/nix/store";
     fsType = "nfs";
     options = [ "ro" ];
   };
 
-  # Create the work directory
-  #systemd.tmpfiles.rules = [
-  #  "d /mnt/nix-overlay/work 0550 1000 root root -"
-  #];
+  # A workdir is also needed, so setup a permanent dir using tmpfiles.
+  systemd.tmpfiles.rules = [
+    "d /mnt/nix-work 0700 root root -"
+  ];
+
+  # Mount an overlay in /nix/store using as lower layer the NFS store and upper
+  # layer the disk nix store. The destination is still the nix store in
+  # /nix/store (confusing). We need rw access, as the daemon need to write the
+  # lock files to build derivations locally.
+  fileSystems."/nix/store" = {
+    device = "overlay";
+    fsType = "overlay";
+    options = [ "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work" ];
+    depends = [ "/nix/store" "/mnt/hut-nix-store" "/mnt/nix-work" ];
+  };
 }
-- 
2.45.2


From 320c58ce483f23da692106d8c4eddf3ac8756912 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 13:57:41 +0200
Subject: [PATCH 04/11] Prevent the overlay to be mounted in stage1

---
 m/module/shared-nix-store.nix | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 72ece72..225bdc0 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -20,7 +20,9 @@
   # layer the disk nix store. The destination is still the nix store in
   # /nix/store (confusing). We need rw access, as the daemon need to write the
   # lock files to build derivations locally.
-  fileSystems."/nix/store" = {
+  # HACK: Use /nix//store to prevent the overlay to be mounted on boot, see:
+  # https://github.com/NixOS/nixpkgs/blob/17a46d09ac123d0da3a26855bf3af7db01f9c751/nixos/lib/utils.nix#L14
+  fileSystems."/nix//store" = {
     device = "overlay";
     fsType = "overlay";
     options = [ "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work" ];
-- 
2.45.2


From e6014511f58b986344423c360b17206a64a42920 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 15:50:37 +0200
Subject: [PATCH 05/11] Wait for the NFS hut store to be mounted

---
 m/module/shared-nix-store.nix | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 225bdc0..cde993b 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -25,7 +25,27 @@
   fileSystems."/nix//store" = {
     device = "overlay";
     fsType = "overlay";
-    options = [ "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work" ];
+    options = [
+      "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work"
+      "x-systemd.requires-mounts-for=/nix/store"
+      # We need to wait for the NFS mount
+      "x-systemd.requires-mounts-for=/mnt/hut-nix-store"
+    ];
     depends = [ "/nix/store" "/mnt/hut-nix-store" "/mnt/nix-work" ];
   };
+
+  # Maybe we should move it to a systemd mount, so we avoid the /nix//store
+  # hack. Example (not tested):
+  # systemd.mounts = [
+  #   {
+  #     what = "overlay";
+  #     type = "overlay";
+  #     where = "/nix/store";
+  #     options = "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
+  #     description = "Overlay nix store mount";
+  #     requires = [ "hut-nix-store.mount" ];
+  #     after = [ "mnt-hut\\x2dnix\\x2dstore.mount" ];
+  #     before = [ "nix-daemon.service" ];
+  #   }
+  # ];
 }
-- 
2.45.2


From 8c11c7460ac75bd939b71276dd8a8f5922fce9f4 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 16:07:46 +0200
Subject: [PATCH 06/11] Delay the mount until the network is ready

---
 m/module/shared-nix-store.nix | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index cde993b..0b17fea 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -26,6 +26,10 @@
     device = "overlay";
     fsType = "overlay";
     options = [
+      # We need the local-fs.target to be ready, so the network interfaces can
+      # be configured to the network.target is reached. So make this a netdev
+      # mount.
+      "_netdev"
       "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work"
       "x-systemd.requires-mounts-for=/nix/store"
       # We need to wait for the NFS mount
-- 
2.45.2


From 1fc6891dc6fd73cbdebdd88ae7978a7d5020a562 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 17:19:28 +0200
Subject: [PATCH 07/11] Remove nix-daemon.socket dependency of /nix/store

The dependency causes a cycle as the nix store will be mounted after the
network is ready, which itself depends on the socket.target which
requires the nix-daemon.socket to be ready too.
---
 m/module/shared-nix-store.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 0b17fea..af9c147 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -4,6 +4,17 @@
   # from being able to mount it.
   boot.readOnlyNixStore = false;
 
+  # The nix-daemon.socket has an unnecessary dependency over the /nix/store
+  # mount point. But that mount point won't be provided until the network is
+  # ready. However, the network-address-eno1.service, has a dependency over
+  # sockets.target, causing a cycle.
+  # One solution is to make the nix-daemon.socket depend only on the socket
+  # patch (which is already covered by ConditionPathIsReadWrite =
+  # /nix/var/nix/daemon-socket), instead on the /nix/store.
+  systemd.sockets.nix-daemon.unitConfig = {
+    RequiresMountsFor = "/nix/var/nix/daemon-socket";
+  };
+
   # Mount the hut nix store via NFS in read-only mode.
   fileSystems."/mnt/hut-nix-store" = {
     device = "hut:/nix/store";
-- 
2.45.2


From 77b41a90e242e4a3bb12ba357067c4b0221bc732 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 18:06:51 +0200
Subject: [PATCH 08/11] Patch nix instead of using an override unit

---
 m/module/shared-nix-store.nix | 14 +++++++++++---
 pkgs/nix-socket.patch         | 11 +++++++++++
 pkgs/overlay.nix              |  4 ++++
 3 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100644 pkgs/nix-socket.patch

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index af9c147..72eab92 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -11,9 +11,17 @@
   # One solution is to make the nix-daemon.socket depend only on the socket
   # patch (which is already covered by ConditionPathIsReadWrite =
   # /nix/var/nix/daemon-socket), instead on the /nix/store.
-  systemd.sockets.nix-daemon.unitConfig = {
-    RequiresMountsFor = "/nix/var/nix/daemon-socket";
-  };
+  #
+  # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor =
+  # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get
+  # added by systemd when the override config is merged with the one that Nix
+  # provides:
+  #
+  # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor
+  # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket
+  # 
+  # To fix this, the Nix package is patched to only depend on /nix/var instead.
+  # See ../../pkgs/overlay.nix for details.
 
   # Mount the hut nix store via NFS in read-only mode.
   fileSystems."/mnt/hut-nix-store" = {
diff --git a/pkgs/nix-socket.patch b/pkgs/nix-socket.patch
new file mode 100644
index 0000000..44a66f9
--- /dev/null
+++ b/pkgs/nix-socket.patch
@@ -0,0 +1,11 @@
+--- a/misc/systemd/nix-daemon.socket.in	1970-01-01 01:00:01.000000000 +0100
++++ b/misc/systemd/nix-daemon.socket.in	2023-09-18 17:53:32.351760208 +0200
+@@ -1,7 +1,7 @@
+ [Unit]
+ Description=Nix Daemon Socket
+ Before=multi-user.target
+-RequiresMountsFor=@storedir@
++RequiresMountsFor=@localstatedir@
+ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket
+ 
+ [Socket]
diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix
index 03208c5..f731209 100644
--- a/pkgs/overlay.nix
+++ b/pkgs/overlay.nix
@@ -32,4 +32,8 @@ final: prev:
     lua = prev.lua5_4;
     fmt = prev.fmt_8;
   }) ceph ceph-client;
+
+  nix = prev.nix.overrideAttrs (old: {
+    patches = old.patches ++ [ ./nix-socket.patch ];
+  });
 }
-- 
2.45.2


From e4cbcab81c44367eb4680c58a4802de8830ef6a6 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 18:53:40 +0200
Subject: [PATCH 09/11] Use a systemd mount directly for the nix store

Allows the LazyUnmount option and avoids the stage1 hack with
/nix//store.
---
 m/module/shared-nix-store.nix | 50 ++++++++++++++---------------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 72eab92..95df035 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -38,37 +38,27 @@
   # Mount an overlay in /nix/store using as lower layer the NFS store and upper
   # layer the disk nix store. The destination is still the nix store in
   # /nix/store (confusing). We need rw access, as the daemon need to write the
-  # lock files to build derivations locally.
-  # HACK: Use /nix//store to prevent the overlay to be mounted on boot, see:
-  # https://github.com/NixOS/nixpkgs/blob/17a46d09ac123d0da3a26855bf3af7db01f9c751/nixos/lib/utils.nix#L14
-  fileSystems."/nix//store" = {
-    device = "overlay";
-    fsType = "overlay";
-    options = [
+  # lock files to build derivations locally. Use a systemd mount unit directly
+  # so we can specify the LazyUmount option and we avoid having it mounted
+  # in the stage1 before systemd.
+  systemd.mounts = [
+    {
+      what = "overlay";
+      type = "overlay";
+      where = "/nix/store";
       # We need the local-fs.target to be ready, so the network interfaces can
       # be configured to the network.target is reached. So make this a netdev
       # mount.
-      "_netdev"
-      "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work"
-      "x-systemd.requires-mounts-for=/nix/store"
-      # We need to wait for the NFS mount
-      "x-systemd.requires-mounts-for=/mnt/hut-nix-store"
-    ];
-    depends = [ "/nix/store" "/mnt/hut-nix-store" "/mnt/nix-work" ];
-  };
-
-  # Maybe we should move it to a systemd mount, so we avoid the /nix//store
-  # hack. Example (not tested):
-  # systemd.mounts = [
-  #   {
-  #     what = "overlay";
-  #     type = "overlay";
-  #     where = "/nix/store";
-  #     options = "lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
-  #     description = "Overlay nix store mount";
-  #     requires = [ "hut-nix-store.mount" ];
-  #     after = [ "mnt-hut\\x2dnix\\x2dstore.mount" ];
-  #     before = [ "nix-daemon.service" ];
-  #   }
-  # ];
+      options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work";
+      description = "Overlay /nix/store mount";
+      mountConfig = {
+        LazyUnmount = true;
+      };
+      unitConfig = {
+        # We need to wait for the NFS mount
+        RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
+        Before = "remote-fs.target";
+      };
+    }
+  ];
 }
-- 
2.45.2


From 3bb0b550aaf6d9cabcd62a28f0e7c59dbdfe4f37 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 19:05:58 +0200
Subject: [PATCH 10/11] Add a RequiredBy dependency for remote-fs.target

---
 m/module/shared-nix-store.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index 95df035..a078a14 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -57,6 +57,7 @@
       unitConfig = {
         # We need to wait for the NFS mount
         RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
+        RequiredBy = "remote-fs.target";
         Before = "remote-fs.target";
       };
     }
-- 
2.45.2


From e065cde3760230dcd654139f6d616525872483ce Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 18 Sep 2023 19:27:14 +0200
Subject: [PATCH 11/11] Use NixOS attributes for the install section

---
 m/module/shared-nix-store.nix | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix
index a078a14..aa3420a 100644
--- a/m/module/shared-nix-store.nix
+++ b/m/module/shared-nix-store.nix
@@ -54,11 +54,15 @@
       mountConfig = {
         LazyUnmount = true;
       };
+
+      # Run the unit after remote-fs-pre.target but before the remote-fs.target
+      after = [ "remote-fs-pre.target"];
+      before = [ "umount.target" "remote-fs.target" ];
+      # Install by using wantedBy over remote-fs.target
+      wantedBy = [ "remote-fs.target" ];
       unitConfig = {
         # We need to wait for the NFS mount
         RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; 
-        RequiredBy = "remote-fs.target";
-        Before = "remote-fs.target";
       };
     }
   ];
-- 
2.45.2