diff --git a/m/common/main.nix b/m/common/main.nix index 025a7c5..999f043 100644 --- a/m/common/main.nix +++ b/m/common/main.nix @@ -45,6 +45,8 @@ bsc.osumb ]; + programs.direnv.enable = true; + systemd.services."serial-getty@ttyS0" = { enable = true; wantedBy = [ "getty.target" ]; diff --git a/m/common/zsh.nix b/m/common/zsh.nix index 666e34b..5cfcb7f 100644 --- a/m/common/zsh.nix +++ b/m/common/zsh.nix @@ -2,7 +2,6 @@ { environment.systemPackages = with pkgs; [ - direnv zsh-completions nix-zsh-completions ]; diff --git a/m/module/shared-nix-store.nix b/m/module/shared-nix-store.nix new file mode 100644 index 0000000..aa3420a --- /dev/null +++ b/m/module/shared-nix-store.nix @@ -0,0 +1,69 @@ +{ ... }: +{ + # Don't make the nix store read-only, as this would prevent the overlay FS + # from being able to mount it. + boot.readOnlyNixStore = false; + + # The nix-daemon.socket has an unnecessary dependency over the /nix/store + # mount point. But that mount point won't be provided until the network is + # ready. However, the network-address-eno1.service, has a dependency over + # sockets.target, causing a cycle. + # One solution is to make the nix-daemon.socket depend only on the socket + # patch (which is already covered by ConditionPathIsReadWrite = + # /nix/var/nix/daemon-socket), instead on the /nix/store. + # + # Using systemd.sockets.nix-daemon.unitConfig.RequiresMountsFor = + # "/nix/var/nix/daemon-socket" doesn't work, as the the mount options get + # added by systemd when the override config is merged with the one that Nix + # provides: + # + # owl2% sudo systemctl show nix-daemon.socket | grep RequiresMountsFor + # RequiresMountsFor=/nix/store /nix/var/nix/daemon-socket/socket /nix/var/nix/daemon-socket + # + # To fix this, the Nix package is patched to only depend on /nix/var instead. + # See ../../pkgs/overlay.nix for details. + + # Mount the hut nix store via NFS in read-only mode. + fileSystems."/mnt/hut-nix-store" = { + device = "hut:/nix/store"; + fsType = "nfs"; + options = [ "ro" ]; + }; + + # A workdir is also needed, so setup a permanent dir using tmpfiles. + systemd.tmpfiles.rules = [ + "d /mnt/nix-work 0700 root root -" + ]; + + # Mount an overlay in /nix/store using as lower layer the NFS store and upper + # layer the disk nix store. The destination is still the nix store in + # /nix/store (confusing). We need rw access, as the daemon need to write the + # lock files to build derivations locally. Use a systemd mount unit directly + # so we can specify the LazyUmount option and we avoid having it mounted + # in the stage1 before systemd. + systemd.mounts = [ + { + what = "overlay"; + type = "overlay"; + where = "/nix/store"; + # We need the local-fs.target to be ready, so the network interfaces can + # be configured to the network.target is reached. So make this a netdev + # mount. + options = "_netdev,lowerdir=/mnt/hut-nix-store,upperdir=/nix/store,workdir=/mnt/nix-work"; + description = "Overlay /nix/store mount"; + mountConfig = { + LazyUnmount = true; + }; + + # Run the unit after remote-fs-pre.target but before the remote-fs.target + after = [ "remote-fs-pre.target"]; + before = [ "umount.target" "remote-fs.target" ]; + # Install by using wantedBy over remote-fs.target + wantedBy = [ "remote-fs.target" ]; + unitConfig = { + # We need to wait for the NFS mount + RequiresMountsFor = "/nix/store /mnt/hut-nix-store"; + }; + } + ]; +} diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index 5afda72..052ea8a 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -5,6 +5,7 @@ ../common/main.nix ../module/ceph.nix ../module/slurm-firewall.nix + ../module/shared-nix-store.nix ]; # Select the this using the ID to avoid mismatches diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index 9636760..ce95fa2 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -5,6 +5,7 @@ ../common/main.nix ../module/ceph.nix ../module/slurm-firewall.nix + ../module/shared-nix-store.nix ]; # Select the this using the ID to avoid mismatches diff --git a/pkgs/nix-socket.patch b/pkgs/nix-socket.patch new file mode 100644 index 0000000..44a66f9 --- /dev/null +++ b/pkgs/nix-socket.patch @@ -0,0 +1,11 @@ +--- a/misc/systemd/nix-daemon.socket.in 1970-01-01 01:00:01.000000000 +0100 ++++ b/misc/systemd/nix-daemon.socket.in 2023-09-18 17:53:32.351760208 +0200 +@@ -1,7 +1,7 @@ + [Unit] + Description=Nix Daemon Socket + Before=multi-user.target +-RequiresMountsFor=@storedir@ ++RequiresMountsFor=@localstatedir@ + ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket + + [Socket] diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 03208c5..f731209 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -32,4 +32,8 @@ final: prev: lua = prev.lua5_4; fmt = prev.fmt_8; }) ceph ceph-client; + + nix = prev.nix.overrideAttrs (old: { + patches = old.patches ++ [ ./nix-socket.patch ]; + }); }