diff --git a/README.md b/README.md index c5913eb..0769969 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,15 @@ Did we satisfy those requirements? * [`harmonia`](https://github.com/helsinki-systems/harmonia) - A Rust rewrite of `nix-serve` + See the Benchmarks section below for more details + * Backwards-compatibility - We have excellent backwards-compatibility. In particular, in the vast - majority of cases, you can simply replace `pkgs.nix-serve` with - `pkgs.nix-serve-ng` and make no other changes. + We have excellent backwards-compatibility with one major exception: + `nix-serve-ng` does not support MacOS (whereas `nix-serve` does). + + Other than that, in the vast majority of cases, you can simply replace + `pkgs.nix-serve` with `pkgs.nix-serve-ng` and make no other changes. * Our executable shares the same name (`nix-serve`) as the original program @@ -173,3 +177,52 @@ Did we satisfy those requirements? You don't need to define or use any new NixOS options. You continue to use the old `services.nix-serve` options hierarchy to configure the upgraded service. + +## Benchmarks + +The test environment is a large server machine: + +* CPU: 24 × Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz +* RAM: 384 GB (24 × 16 GB @ 2133 MT/s) +* Disk (`/nix/store`): ≈4 TB SSD + +Legend: + +* **Fetch present NAR info ×10**: Time to fetch the NAR info for 10 different files that are present +* **Fetch absent NAR info ×1**: Time to fetch the NAR info a single file that is absent +* **Fetch empty NAR ×10**: Time to fetch the NAR for the same empty file 10 times +* **Fetch 10 MB NAR ×10**: Time to fetch the NAR for the same 10 MB file 10 times + +Raw numbers: + +| Benchmark | `nix-serve` | `eris` | `harmonia` | `nix-serve-ng` | +|----------------------------|------------------|------------------|------------------|------------------| +| Fetch present NAR info ×10 | 2.09 ms ± 66 μs | 41.5 ms ± 426 μs | 1.57 ms ± 91 μs | 1.32 ms ± 33 μs | +| Fetch absent NAR info ×1 | 212 μs ± 18 μs | 3.42 ms ± 113 μs | 139 μs ± 11 μs | 115 μs ± 6.2 μs | +| Fetch empty NAR ×10 | 164 ms ± 8.5 ms | 246 ms ± 20 ms | 279 ms ± 10 ms | 5.16 ms ± 368 μs | +| Fetch 10 MB NAR ×10 | 291 ms ± 8.7 ms | 453 ms ± 19 ms | 487 ms ± 41 ms | 86.9 ms ± 3.0 ms | + +Speedups (compared to `nix-serve`): + +| Benchmark | `nix-serve` | `eris` | `harmonia` | `nix-serve-ng` | +|----------------------------|------------------|------------------|------------------|------------------| +| Fetch present NAR info ×10 | 1.0 | 0.05 | 1.33 | 1.58 | +| Fetch absent NAR info ×1 | 1.0 | 0.06 | 1.53 | 1.84 | +| Fetch empty NAR ×10 | 1.0 | 0.67 | 0.59 | 31.80 | +| Fetch 10 MB NAR ×10 | 1.0 | 0.64 | 0.60 | 3.35 | + +We can summarize `nix-serve-ng`'s performance like this: + +* Time to handle a NAR info request: ≈ 100 μs +* Time to serve a NAR: ≈ 500 μs + 800 μs / MB + +You can reproduce these benchmarks using the benchmark suite. See the +instructions in [`./benchmark/Main.hs`](./benchmark/Main.hs) for running your +own benchmarks. + +Caveats: + +* We haven't used any of these services' tuning options, including: + * Tuning garbage collection (for `nix-serve-ng`) + * Tuning concurrency/parallelism/workers +* We haven't benchmarked memory utilization diff --git a/benchmark/Main.hs b/benchmark/Main.hs index d1f79d5..90d68da 100644 --- a/benchmark/Main.hs +++ b/benchmark/Main.hs @@ -30,8 +30,8 @@ import qualified System.IO.Temp as Temp import qualified Test.Tasty.Bench as Bench import qualified Turtle -prepare :: Natural -> IO (Vector FilePath) -prepare maxSize = +prepareMixed :: Natural -> IO (Vector FilePath) +prepareMixed maxSize = Temp.withSystemTempFile "zeros" \tempFile handle -> do IO.hClose handle @@ -54,6 +54,22 @@ prepare maxSize = Vector.generateM (fromIntegral numFiles) generate +prepareConstant :: Natural -> IO (Vector FilePath) +prepareConstant size = + Temp.withSystemTempFile "zeros" \tempFile handle -> do + IO.hClose handle + + ByteString.writeFile tempFile (ByteString.replicate (fromIntegral size) 0) + + text <- Turtle.single do + Turtle.inproc "nix-store" + [ "--add", Text.pack tempFile ] + empty + + let path = Text.unpack (Turtle.lineToText text) + + return (Vector.replicate (fromIntegral numFiles) path) + port :: Int port = 8000 @@ -100,12 +116,18 @@ main = do return (HTTP.responseBody response) Bench.defaultMain - [ Bench.env (prepare 1000000) \files -> do + [ Bench.env (prepareMixed 10000000) \files -> do Bench.bench ("fetch present NAR info ×" <> show numFiles) (Bench.nfAppIO (traverse_ fetchNarInfo) files) , Bench.bench "fetch absent NAR info ×1" (Bench.nfAppIO fetchNarInfo "/nix/store/00000000000000000000000000000000") - , Bench.env (prepare 1000000) \files -> do - Bench.bench ("fetch present NAR ×" <> show numFiles) + , Bench.env (prepareConstant 0) \files -> do + Bench.bench ("fetch empty NAR ×" <> show numFiles) + (Bench.nfAppIO (traverse_ fetchNar) files) + , Bench.env (prepareConstant 10000000) \files -> do + Bench.bench ("fetch 10 MB NAR ×" <> show numFiles) + (Bench.nfAppIO (traverse_ fetchNar) files) + , Bench.env (prepareMixed 10000000) \files -> do + Bench.bench ("fetch NARs of mixed sizes up to 10 MB ×" <> show numFiles) (Bench.nfAppIO (traverse_ fetchNar) files) ]