fix race with getStore() (#45)

by calling it once before starting the server

closes: #44

README.md

@@ -4,40 +4,77 @@
 
 ## Quick start
 
-There are two main approaches you can use to upgrade a NixOS system to replace
+There are three main approaches you can use to configure a NixOS system to replace
-the old `nix-serve` with `nix-serve-ng`.
+the old `nix-serve` with `nix-serve-ng`:
 
-If you specify your desired NixOS system within `flake.nix` then you can do
+- **A**: Set `services.nix-serve.package = pkgs.nix-serve-ng;` in your NixOS configuration
-something like this:
+  - `nix-serve-ng` is [packaged in nixpkgs](https://search.nixos.org/packages) already
+  - There is no need to consume this repository directly
+- **B**: Include `nix-serve-ng.nixosModules.default` in your NixOS configuration
+  - `nix-serve-ng` refers to this repository being a flake input
+  - Requires consume this repository / this flake
+  - Overlays `pkgs.nix-serve` with `pkgs.nix-serve-ng`
+- **C**: Like **B** but not requiring a flake
+
+We recommend approach **A**. Only use **B** or **C** if you need a bleeding edge
+upstream version of the project.
+
+### Variant A: 
+
+_The code snippet below shows a `flake.nix`._
 
 ```nix
-{ inputs = {
+{ 
-    nixpkgs.url = github:NixOS/nixpkgs;
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs";
 
-    nix-serve-ng.url = github:aristanetworks/nix-serve-ng;
+  outputs = { nixpkgs, ... }: {
-  };
-
-  outputs = { nixpkgs, nix-serve-ng, ... }: {
     nixosConfigurations.default = nixpkgs.lib.nixosSystem {
       modules = [
-        nix-serve-ng.nixosModules.default
+        /* ... */
-
+        { 
-        { services.nix-serve.enable = true;
+          services.nix-serve.enable = true;
-
+          services.nix-serve.package = pkgs.nix-serve-ng;
-          …
+          /* ... */
         }
+        /* ... */
       ];
-
-      system = "x86_64-linux";
     };
   };
 }
 ```
 
-If you don't use `flake.nix` then you can instead define your NixOS module:
+### Variant B: 
-like this:
+
+_The code snippet below shows a `flake.nix`._
 
 ```nix
+{ 
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs";
+  inputs.nix-serve-ng.url = "github:aristanetworks/nix-serve-ng";
+
+  outputs = { nixpkgs, nix-serve-ng, ... }: {
+    nixosConfigurations.default = nixpkgs.lib.nixosSystem {
+      modules = [
+        nix-serve-ng.nixosModules.default
+        /* ... */
+        { 
+          services.nix-serve.enable = true;
+          /* ... */
+        }
+        /* ... */
+      ];
+    };
+  };
+}
+```
+
+### Variant C: 
+
+_The code snippet below shows a NixOS module file._
+
+```nix
+{ config, pkgs, lib, ... }:
+
 let 
   nix-serve-ng-src = builtins.fetchTarball {
     # Replace the URL and hash with whatever you actually need
@@ -47,15 +84,24 @@ let
   };
 
   nix-serve-ng = import nix-serve-ng-src;
-
 in
-  { ... }: {
+{ 
-    imports = [ nix-serve-ng.nixosModules.default ];
+  /* ... */
+  imports = [ nix-serve-ng.nixosModules.default ];
   
-    …
+  config = {
-  }
+    services.nix-serve.enable = true;
+  };
+  /* ... */
+}
 ```
 
+## Lix compatability
+
+The default `nix-serve-ng` should work on top of lix, but if you want to build
+it against lix for development or to remove the default nix dependency, you can
+pass `-f lix` to cabal or use the `lix-serve-ng` package from the flake.
+
 ## Motivation
 
 Our requirements for this project were:
@@ -107,7 +153,7 @@ Did we satisfy those requirements?
   * [`eris`](https://github.com/thoughtpolice/eris) - A Perl rewrite of
     `nix-serve`
 
-  * [`harmonia`](https://github.com/helsinki-systems/harmonia) - A Rust rewrite
+  * [`harmonia`](https://github.com/nix-community/harmonia) - A Rust rewrite
     of `nix-serve`
 
   See the Benchmarks section below for more details

cbits/nix.cpp

@@ -1,7 +1,16 @@
 #include <cstddef>
 #include <cstdlib>
-#include <nix/store-api.hh>
+
-#include <nix/log-store.hh>
+#ifndef LIX
+    #include <nix/store/store-api.hh>
+    #include <nix/store/log-store.hh>
+    #include <nix/main/shared.hh>
+#else
+    #include <lix/libstore/store-api.hh>
+    #include <lix/libstore/log-store.hh>
+    #include <lix/libmain/shared.hh>
+#endif
+
 #include "nix.hh"
 
 using namespace nix;
@@ -14,8 +23,11 @@ static ref<Store> getStore()
     static std::shared_ptr<Store> _store;
 
     if (!_store) {
-        initLibStore();
+#ifndef LIX
-        loadConfFile();
+        initLibStore(true);
+#else
+        initNix();
+#endif
 
         _store = openStore();
     }
@@ -25,6 +37,12 @@ static ref<Store> getStore()
 
 extern "C" {
 
+// Must be called once before the server is stated to avoid races
+void initStore()
+{
+    getStore();
+}
+
 void freeString(struct string * const input)
 {
     free((void *) input->data);
@@ -120,7 +138,11 @@ void queryPathInfo
         output->deriver = emptyString;
     };
 
-    copyString(validPathInfo->narHash.to_string(Base32, true), &output->narHash);
+#ifndef LIX
+    copyString(validPathInfo->narHash.to_string(nix::HashFormat::Nix32, true), &output->narHash);
+#else
+    copyString(validPathInfo->narHash.to_string(nix::Base::Base32, true), &output->narHash);
+#endif
 
     output->narSize = validPathInfo->narSize;
 
@@ -183,7 +205,11 @@ bool dumpPath
         });
 
         try {
+#ifndef LIX
             store->narFromPath(storePath.value(), sink);
+#else
+            sink << store->narFromPath(storePath.value());
+#endif
         } catch (const std::runtime_error & e) {
             // Intentionally do nothing.  We're only using the exception as a
             // short-circuiting mechanism.

default.nix

@@ -1,9 +1,14 @@
-(import (
+(import
-  let
+  (
-    lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+    let
-  in fetchTarball {
+      lock = builtins.fromJSON (builtins.readFile ./flake.lock);
-    url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+    in
-    sha256 = lock.nodes.flake-compat.locked.narHash; }
+    fetchTarball {
-) {
+      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
-  src =  ./.;
+      sha256 = lock.nodes.flake-compat.locked.narHash;
-}).defaultNix
+    }
+  )
+  {
+    src = ./.;
+  }
+).defaultNix

flake.lock

@@ -16,26 +16,129 @@
         "type": "github"
       }
     },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "lix": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "nix2container": "nix2container",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-regression": "nixpkgs-regression",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1737234286,
+        "narHash": "sha256-CCKIAE84dzkrnlxJCKFyffAxP3yfsOAbdvydUGqq24g=",
+        "rev": "2837da71ec1588c1187d2e554719b15904a46c8b",
+        "revCount": 16631,
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/lix"
+      },
+      "original": {
+        "rev": "2837da71ec1588c1187d2e554719b15904a46c8b",
+        "type": "git",
+        "url": "https://git.lix.systems/lix-project/lix"
+      }
+    },
+    "nix2container": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1724996935,
+        "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=",
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "nix2container",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1688403656,
+        "lastModified": 1733348545,
-        "narHash": "sha256-zmNai3dKWUCKpKubPWsEJ1Q7od96KebWVDJNCnk+fr0=",
+        "narHash": "sha256-b4JrUmqT0vFNx42aEN9LTWOHomkTKL/ayLopflVf81U=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "453da3c28f7a95374b73d1f3fd665dd40e6049e9",
+        "rev": "9ecb50d2fae8680be74c08bb0a995c5383747f89",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "nixos-24.11-small",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1744174375,
+        "narHash": "sha256-oxI9TLgnQbQ/WL0tIwVSIooLbXq4PW1QUhf5aQmXFgk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ef3a956f697525883b77192cbe208233ea0f8f79",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "master",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1733318908,
+        "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "6f4e2a2112050951a314d2733a994fbab94864c6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
-        "nixpkgs": "nixpkgs",
+        "lix": "lix",
+        "nixpkgs": "nixpkgs_2",
         "utils": "utils"
       }
     },

flake.nix

@@ -1,5 +1,9 @@
-{ inputs = {
+{
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+  inputs = {
+    # Temporary, until Nixpkgs master has Nix 2.28
+    nixpkgs.url = "github:NixOS/nixpkgs/master";
+
+    lix.url = "git+https://git.lix.systems/lix-project/lix?rev=2837da71ec1588c1187d2e554719b15904a46c8b";
 
     utils.url = "github:numtide/flake-utils";
 
@@ -9,11 +13,14 @@
     };
   };
 
-  outputs = { nixpkgs, utils, ... }:
+  outputs =
+    { nixpkgs, lix, utils, ... }:
     let
-      compiler = "ghc92";
+      compiler = "ghc94";
 
       overlay = final: prev: {
+        lix = lix.packages.${final.system}.default;
+
         cabal2nix-unwrapped =
           final.haskell.lib.justStaticExecutables
             final.haskell.packages."${compiler}".cabal2nix;
@@ -21,22 +28,30 @@
         haskell = prev.haskell // {
           packages = prev.haskell.packages // {
             "${compiler}" = prev.haskell.packages."${compiler}".override (old: {
-              overrides =
+              overrides = final.lib.fold final.lib.composeExtensions (old.overrides or (_: _: { })) [
-                final.lib.fold final.lib.composeExtensions (old.overrides or (_: _: { })) [
+                (final.haskell.lib.packageSourceOverrides {
-                  (final.haskell.lib.packageSourceOverrides {
+                  nix-serve-ng = ./.;
-                    nix-serve-ng = ./.;
+                  lix-serve-ng = ./.;
-                  })
+
-                  (haskellPackagesNew: haskellPackagesOld: {
+                  base16 = "1.0";
-                    nix-serve-ng =
+                })
-                      final.haskell.lib.overrideCabal
+                (haskellPackagesNew: haskellPackagesOld: {
-                        haskellPackagesOld.nix-serve-ng
+                  nix-serve-ng = final.haskell.lib.overrideCabal haskellPackagesOld.nix-serve-ng (old: {
-                        (old: {
+                    executableSystemDepends = (old.executableSystemDepends or [ ]) ++ [
-                          executableSystemDepends = (old.executableSystemDepends or []) ++ [
+                      final.boost.dev
-                            final.boost.dev
+                      final.nixVersions.nix_2_28
-                          ];
+                    ];
-                        });
+                  });
-                  })
+                  lix-serve-ng = final.haskell.lib.overrideCabal haskellPackagesOld.lix-serve-ng (old: {
-                ];
+                    pname = "lix-serve-ng";
+                    configureFlags = (old.configureFlags or [ ]) ++ [ "-flix" ];
+                    executableSystemDepends = (old.executableSystemDepends or [ ]) ++ [
+                      final.boost.dev
+                      final.lix
+                    ];
+                  });
+                })
+              ];
             });
           };
         };
@@ -44,61 +59,81 @@
         nix-serve-ng =
           final.haskell.lib.justStaticExecutables
             final.haskell.packages."${compiler}".nix-serve-ng;
+
+        lix-serve-ng =
+          final.haskell.lib.justStaticExecutables
+            final.haskell.packages."${compiler}".lix-serve-ng;
       };
 
     in
-      utils.lib.eachDefaultSystem (system:
+    utils.lib.eachDefaultSystem (
-        let
+      system:
-          pkgs = import nixpkgs {
+      let
-            config = { };
+        pkgs = import nixpkgs {
+          config = { };
 
-            overlays = [ overlay ];
+          overlays = [ overlay ];
 
-            inherit system;
+          inherit system;
+        };
+
+        inherit (pkgs) nix-serve-ng lix-serve-ng;
+
+      in
+      rec {
+        packages = {
+          inherit nix-serve-ng lix-serve-ng;
+          default = nix-serve-ng;
+        };
+
+        defaultPackage = packages.default;
+
+        apps = rec {
+          default = nix-serve-ng;
+          nix-serve-ng = {
+            type = "app";
+            program = "${nix-serve-ng}/bin/nix-serve";
           };
-
+          lix-serve-ng = {
-          inherit (pkgs) nix-serve-ng;
+            type = "app";
-
+            program = "${lix-serve-ng}/bin/nix-serve";
-        in
-          rec {
-            packages.default = nix-serve-ng;
-
-            defaultPackage = packages.default;
-
-            apps.default = {
-              type = "app";
-
-              program = "${nix-serve-ng}/bin/nix-serve";
-            };
-
-            defaultApp = apps.default;
-
-            devShells.default =
-              (pkgs.haskell.lib.doBenchmark
-                pkgs.haskell.packages."${compiler}".nix-serve-ng
-              ).env;
-
-            devShell = devShells.default;
-          }) // rec {
-        overlays = {
-          # The default overlay only adds the exports for
-          # `pkgs.haskell.packages."${compiler}".nix-serve-ng` and
-          # `pkgs.nix-serve-ng`
-          default = overlay;
-
-          # This overlay additionally overrides `pkgs.nix-serve` to refer to
-          # `pkgs.nix-serve-ng`
-          override = final: prev: {
-            nix-serve = final.nix-serve-ng;
           };
         };
 
-        # The NixOS module is a thin wrapper around the overlay to replace
+        defaultApp = apps.default;
-        # `nix-serve` with `nix-serve-ng`
+
-        #
+        devShells = rec {
-        # You can continue to use the old `services.nix-serve` NixOS options.
+          default = nix-serve-ng;
-        nixosModules.default = {
+          nix-serve-ng = (pkgs.haskell.lib.doBenchmark pkgs.haskell.packages."${compiler}".nix-serve-ng).env;
-          nixpkgs.overlays = [ overlays.default overlays.override ];
+          lix-serve-ng = (pkgs.haskell.lib.doBenchmark pkgs.haskell.packages."${compiler}".lix-serve-ng).env;
+        };
+
+        devShell = devShells.default;
+      }
+    )
+    // rec {
+      overlays = {
+        # The default overlay only adds the exports for
+        # `pkgs.haskell.packages."${compiler}".nix-serve-ng` and
+        # `pkgs.nix-serve-ng`
+        default = overlay;
+
+        # This overlay additionally overrides `pkgs.nix-serve` to refer to
+        # `pkgs.nix-serve-ng`
+        override = final: prev: {
+          nix-serve = final.nix-serve-ng;
         };
       };
+
+      # The NixOS module is a thin wrapper around the overlay to replace
+      # `nix-serve` with `nix-serve-ng`
+      #
+      # You can continue to use the old `services.nix-serve` NixOS options.
+      nixosModules.default = {
+        nixpkgs.overlays = [
+          overlays.default
+          overlays.override
+        ];
+      };
+    };
 }

nix-serve-ng.cabal

@@ -11,6 +11,11 @@ maintainer:         opensource@awakesecurity.com
 copyright:          2022 Arista Networks
 extra-source-files: CHANGELOG.md
 
+Flag Lix
+  Description: Build against Lix
+  Default:     False
+  Manual:      True
+
 executable nix-serve
     hs-source-dirs:   src
 
@@ -36,10 +41,10 @@ executable nix-serve
     cxx-sources:      cbits/nix.cpp
                     , cbits/nix.hh
 
-    cxx-options:      -std=c++17
+    cxx-options:      -std=c++20
 
     build-depends:    base < 5
-                    , base16
+                    , base16 >= 1.0
                     , base32
                     , bytestring
                     , charset
@@ -55,8 +60,14 @@ executable nix-serve
                     , warp
                     , warp-tls
 
-    pkgconfig-depends:
+    if flag(lix)
-                      nix-store
+      cxx-options:      -DLIX
+      pkgconfig-depends:
+                      , lix-main
+                      , lix-store
+    else
+      pkgconfig-depends:
+                        nix-store
 
     if os(darwin)
       extra-libraries: c++

shell.nix

@@ -1,9 +1,14 @@
-(import (
+(import
-  let
+  (
-    lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+    let
-  in fetchTarball {
+      lock = builtins.fromJSON (builtins.readFile ./flake.lock);
-    url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+    in
-    sha256 = lock.nodes.flake-compat.locked.narHash; }
+    fetchTarball {
-) {
+      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
-  src =  ./.;
+      sha256 = lock.nodes.flake-compat.locked.narHash;
-}).shellNix
+    }
+  )
+  {
+    src = ./.;
+  }
+).shellNix

src/Main.hs

@@ -341,6 +341,8 @@ main = do
     options@Options{ priority, timeout, verbosity } <- do
         Options.execParser Options.parserInfo
 
+    Nix.initStore
+
     storeDirectory <- Nix.getStoreDir
 
     secretKeyFile <- Environment.lookupEnv "NIX_SECRET_KEY_FILE"

src/Nix.hsc

@@ -34,6 +34,8 @@ import qualified Foreign
 
 #include "nix.hh"
 
+foreign import ccall "initStore" initStore :: IO ()
+
 foreign import ccall "freeString" freeString :: Ptr String_ -> IO ()
 
 data String_ = String_ { data_ :: Ptr CChar, size  :: CSize }
@@ -162,8 +164,9 @@ data PathInfo = PathInfo
 
 fromCPathInfo :: CPathInfo -> IO PathInfo
 fromCPathInfo CPathInfo{ deriver, narHash, narSize, references, sigs } = do
+    let String_{ data_ } = deriver
     deriver_ <-
-        if data_ (deriver :: String_) == Foreign.nullPtr
+        if data_ == Foreign.nullPtr
         then return Nothing
         else fmap Just (fromString_ deriver)
 
@@ -230,7 +233,7 @@ fingerprintPath storePath PathInfo{ narHash, narSize, references } = do
 
     base32Suffix <- if
         | ByteString.length suffix == 64
-        , Right digest <- Base16.decodeBase16 suffix ->
+        , Right digest <- Base16.decodeBase16Untyped suffix ->
             return (Base32.encodeBase32' digest)
         | ByteString.length suffix == 52 ->
             return suffix