New design with overlays

bsc/garlic/default.nix

@@ -1,27 +1,26 @@
 {
   pkgs
-, bsc
+, callPackage
+, callPackages
 }:
 
 let
-  callPackage = pkgs.lib.callPackageWith (pkgs // bsc // garlic);
-  callPackages = pkgs.lib.callPackagesWith (pkgs // bsc // garlic);
 
-  garlic = rec {
+  garlic = {
 
     # Load some helper functions to generate app variants
     inherit (import ./gen.nix) genApps genApp genConfigs;
-    inherit bsc;
 
     mpptest = callPackage ./mpptest { };
 
     ppong = callPackage ./ppong {
-      mpi = bsc.mpi;
+      mpi = pkgs.mpi;
     };
 
     nbody = callPackage ./nbody {
-      cc = bsc.icc;
+      cc = pkgs.icc;
-      mpi = bsc.impi;
+      mpi = pkgs.impi;
+      tampi = pkgs.tampi;
       gitBranch = "garlic/seq";
     };
 
@@ -38,15 +37,17 @@ let
     };
 
     # Perf is tied to a linux kernel specific version
-    linuxPackages = bsc.linuxPackages_4_4;
+    linuxPackages = pkgs.linuxPackages_4_4;
     perfWrapper = callPackage ./perf.nix {
-      perf = linuxPackages.perf;
+      perf = pkgs.linuxPackages.perf;
     };
 
     exp = {
       noise = callPackage ./exp/noise.nix { };
       nbody = {
-        bs = callPackage ./exp/nbody/bs.nix { };
+        bs = callPackage ./exp/nbody/bs.nix {
+          pkgs = pkgs // garlic;
+        };
         mpi = callPackage ./exp/nbody/mpi.nix { };
       };
       osu = rec {

bsc/garlic/exp/nbody/bs.nix

@@ -1,7 +1,7 @@
 {
-  bsc
+  stdenv
-, stdenv
+, nixpkgs
-, nbody
+, pkgs
 , genApp
 , genConfigs
 , runWrappers
@@ -12,7 +12,7 @@ with stdenv.lib;
 let
   # Set variable configuration for the experiment
   varConfig = {
-    cc = [ bsc.icc ];
+    cc = [ pkgs.bsc.icc ];
     blocksize = [ 1024 ];
   };
 
@@ -20,7 +20,7 @@ let
   common = {
     # Compile time nbody config
     gitBranch = "garlic/mpi+send";
-    mpi = bsc.impi;
+    mpi = pkgs.bsc.impi;
 
     # nbody runtime options
     particles = 1024*128;
@@ -97,7 +97,28 @@ let
       -p ${toString conf.particles} )'';
   };
 
-  nbodyFn = {stage, conf, ...}: with conf; nbody.override {
+  bscOverlay = import ../../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  nbodyFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.nbody.override {
       inherit cc blocksize mpi gitBranch;
     };
 

bsc/garlic/nbody/default.nix

@@ -1,6 +1,7 @@
 {
   stdenv
 , cc
+, tampi ? null
 , mpi ? null
 , cflags ? null
 , gitBranch
@@ -37,6 +38,7 @@ stdenv.mkDerivation rec {
   dontPatchShebangs = true;
 
   installPhase = ''
+    echo ${tampi}
     mkdir -p $out/bin
     cp nbody* $out/bin/${name}
   '';

bsc/intel-compiler/default.nix

@@ -2,27 +2,27 @@
   stdenv
 , gcc
 , nanos6
-, icc-unwrapped
+, iccUnwrapped
 , wrapCCWith
-, intel-license
+, intelLicense
 }:
 
 let
   targetConfig = stdenv.targetPlatform.config;
   inherit gcc;
 in wrapCCWith rec {
-  cc = icc-unwrapped;
+  cc = iccUnwrapped;
   extraBuildCommands = ''
     echo "-B${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-cflags
-    echo "-isystem ${icc-unwrapped}/include" >> $out/nix-support/cc-cflags
+    echo "-isystem ${iccUnwrapped}/include" >> $out/nix-support/cc-cflags
-    echo "-isystem ${icc-unwrapped}/include/intel64" >> $out/nix-support/cc-cflags
+    echo "-isystem ${iccUnwrapped}/include/intel64" >> $out/nix-support/cc-cflags
     echo "-L${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-ldflags
     echo "-L${gcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags
 
-    cat "${icc-unwrapped}/nix-support/propagated-build-inputs" >> \
+    cat "${iccUnwrapped}/nix-support/propagated-build-inputs" >> \
       $out/nix-support/propagated-build-inputs
 
-    echo "export INTEL_LICENSE_FILE=${intel-license}" \
+    echo "export INTEL_LICENSE_FILE=${intelLicense}" \
       >> $out/nix-support/setup-hook
 
     # Create the wrappers for icc and icpc

default.nix

@@ -1,229 +1,7 @@
-{ pkgs ? import <nixpkgs> {} }:
-
 let
-  inherit (pkgs.lib) callPackageWith;
+  bscOverlay = import ./overlay.nix;
-  inherit (pkgs.lib) callPackagesWith;
+  pkgs = import <nixpkgs> {
-  callPackage = callPackageWith (pkgs // self.bsc);
+    overlays = [ bscOverlay ];
-  callPackageStatic = callPackageWith (pkgs.pkgsStatic);
-  callPackages = callPackagesWith (pkgs // self.bsc);
-
-  self.bsc = rec {
-
-    nixpkgs = pkgs;
-
-
-    # Load the default implementation
-    #mpi = mpich;
-    #mpi = openmpi;
-    mpi = intel-mpi;
-
-    # Load the default compiler
-    #stdenv = pkgs.gcc10Stdenv;
-    stdenv = pkgs.gcc9Stdenv;
-    #stdenv = pkgs.gcc7Stdenv;
-    #stdenv = pkgs.llvmPackages_10.stdenv;
-    #stdenv = pkgs.llvmPackages_9.stdenv;
-    #stdenv = pkgs.llvmPackages_8.stdenv;
-    #stdenv = pkgs.llvmPackages_7.stdenv;
-
-    binutils = pkgs.binutils;
-    coreutils = pkgs.coreutils;
-    gcc = stdenv.cc;
-
-    nanos6 = nanos6-git;
-
-    # --------------------------------------------------------- #
-    #  BSC Packages
-    # --------------------------------------------------------- #
-
-    perf = callPackage ./bsc/perf/default.nix {
-      kernel = pkgs.linuxPackages_4_9.kernel;
-      systemtap = pkgs.linuxPackages_4_9.systemtap;
   };
 
-    # ParaStation MPI
+in pkgs
-    pscom = callPackage ./bsc/parastation/pscom.nix { };
-    psmpi = callPackage ./bsc/parastation/psmpi.nix { };
-
-    osumb = callPackage ./bsc/osu/default.nix { };
-
-    mpich = callPackage ./bsc/mpich/default.nix { };
-    mpichDbg = callPackage ./bsc/mpich/default.nix { enableDebug = true; };
-
-    # Default Intel MPI version is 2019 (the last one)
-    impi = intel-mpi;
-    intel-mpi = intel-mpi-2019;
-    intel-mpi-2019 = callPackage ./bsc/intel-mpi/default.nix {
-      # Intel MPI provides a debug version of the MPI library, but
-      # by default we use the release variant for performance
-      enableDebug = false;
-    };
-
-    # By default we use Intel compiler 2020 update 1
-    icc-unwrapped = icc2020-unwrapped;
-    icc2020-unwrapped = callPackage ./bsc/intel-compiler/icc2020.nix {
-      intel-mpi = intel-mpi-2019;
-    };
-
-    # A wrapper script that puts all the flags and environment vars properly and
-    # calls the intel compiler binary
-    icc = callPackage bsc/intel-compiler/default.nix {
-      inherit icc-unwrapped intel-license;
-    };
-
-    intel-license = callPackage bsc/intel-compiler/license.nix {
-    };
-
-    pmix2 = callPackage ./bsc/pmix/pmix2.nix { };
-
-    slurm17 = callPackage ./bsc/slurm/default.nix {
-      pmix = pmix2;
-    };
-
-    slurm17-libpmi2 = callPackage ./bsc/slurm/pmi2.nix {
-      pmix = pmix2;
-    };
-
-    openmpi-mn4 = callPackage ./bsc/openmpi/default.nix {
-      pmix = pmix2;
-      pmi2 = slurm17-libpmi2;
-      enableCxx = true;
-    };
-
-    openmpi = openmpi-mn4;
-
-    fftw = callPackage ./bsc/fftw/default.nix {
-      mpi = mpi;
-    };
-
-    extrae = callPackage ./bsc/extrae/default.nix {
-      mpi = mpi;
-    };
-
-    tampi = callPackage ./bsc/tampi/default.nix {
-      mpi = mpi;
-    };
-
-    mcxx = callPackage ./bsc/mcxx/default.nix {
-    };
-
-    mcxx-rarias = callPackage ./bsc/mcxx/rarias.nix {
-    };
-
-    nanos6-latest = callPackage ./bsc/nanos6/default.nix {
-      extrae = extrae;
-    };
-
-    nanos6-git = callPackage ./bsc/nanos6/git.nix {
-      extrae = extrae;
-    };
-
-    vtk = callPackage ./bsc/vtk/default.nix {
-      mpi = mpi;
-      inherit (pkgs.xorg) libX11 xorgproto libXt;
-    };
-
-    dummy = callPackage ./bsc/dummy/default.nix {
-    };
-
-    clang-ompss2-unwrapped = callPackage ./bsc/llvm-ompss2/clang.nix {
-      stdenv = pkgs.llvmPackages_10.stdenv;
-      enableDebug = false;
-    };
-
-    clang-ompss2 = callPackage bsc/llvm-ompss2/default.nix {
-      inherit clang-ompss2-unwrapped;
-    };
-
-    stdenv-nanos6 = pkgs.clangStdenv.override {
-      cc = clang-ompss2;
-    };
-
-    cpic = callPackage ./bsc/apps/cpic/default.nix {
-      stdenv = stdenv-nanos6;
-      inherit mpi tampi;
-    };
-
-    mpptest = callPackage ./bsc/mpptest/default.nix {
-    };
-
-    # Apps for Garlic
-    nbody = callPackage ./bsc/apps/nbody/default.nix {
-      stdenv = pkgs.gcc9Stdenv;
-      mpi = intel-mpi;
-      tampi = tampi;
-    };
-
-    heat = callPackage ./bsc/apps/heat/default.nix {
-      stdenv = pkgs.gcc7Stdenv;
-      mpi = intel-mpi;
-      tampi = tampi;
-    };
-
-    saiph = callPackage ./bsc/apps/saiph/default.nix {
-      stdenv = stdenv-nanos6;
-      mpi = intel-mpi;
-      tampi = tampi;
-      inherit vtk;
-      boost = pkgs.boost;
-    };
-
-    creams = callPackage ./bsc/apps/creams/default.nix {
-      stdenv = pkgs.gcc9Stdenv;
-      mpi = intel-mpi;
-      tampi = tampi.override {
-        mpi = intel-mpi;
-      };
-    };
-
-    lulesh = callPackage ./bsc/apps/lulesh/default.nix {
-      mpi = intel-mpi;
-    };
-
-    hpcg = callPackage ./bsc/apps/hpcg/default.nix {
-    };
-
-    hpccg = callPackage ./bsc/apps/hpccg/default.nix {
-    };
-
-    fwi = callPackage ./bsc/apps/fwi/default.nix {
-    };
-
-    garlic = callPackage ./bsc/garlic/default.nix {
-      pkgs = pkgs;
-      bsc = self.bsc;
-    };
-
-    # Patched nix for deep cluster
-    inherit (callPackage ./bsc/nix/default.nix {
-        storeDir = "/nix/store";
-        stateDir = "/nix/var";
-        boehmgc = pkgs.boehmgc.override { enableLargeConfig = true; };
-        })
-      nix
-      nixUnstable
-      nixFlakes;
-
-    clsync = callPackage ./bsc/clsync/default.nix { };
-
-    nixStatic = (callPackageStatic ./bsc/nix/static.nix {
-        callPackage = callPackageWith (pkgs.pkgsStatic);
-        storeDir = "/nix/store";
-        stateDir = "/nix/var";
-        sandbox-shell = "/bin/sh";
-        boehmgc = pkgs.pkgsStatic.boehmgc.override { enableLargeConfig = true; };
-        }).nix;
-
-    test = {
-      chroot = callPackage ./test/chroot.nix { };
-
-      internet = callPackage ./test/security/internet.nix { };
-
-      clang-ompss2 = callPackage ./test/compilers/clang-ompss2.nix {
-        stdenv = stdenv-nanos6;
-        inherit clang-ompss2;
-      };
-    };
-  };
-
-in pkgs // self

overlay.nix

@@ -0,0 +1,196 @@
+self:  /* Future last stage */
+super: /* Previous stage */
+
+let
+  #callPackage = self.callPackage;
+  inherit (self.lib) callPackageWith;
+  inherit (self.lib) callPackagesWith;
+  callPackage = callPackageWith (self // self.bsc);
+
+  bsc = {
+    # Default MPI implementation to use. Will be overwritten by the
+    # experiments.
+    mpi = self.bsc.openmpi;
+
+    # --------------------------------------------------------- #
+    #  BSC Packages
+    # --------------------------------------------------------- #
+
+    perf = callPackage ./bsc/perf/default.nix {
+      kernel = self.linuxPackages_4_9.kernel;
+      systemtap = self.linuxPackages_4_9.systemtap;
+    };
+
+    # ParaStation MPI
+    pscom = callPackage ./bsc/parastation/pscom.nix { };
+    psmpi = callPackage ./bsc/parastation/psmpi.nix { };
+
+    osumb = callPackage ./bsc/osu/default.nix { };
+
+    mpich = callPackage ./bsc/mpich/default.nix { };
+
+    mpichDebug = self.mpich.override { enableDebug = true; };
+
+    # Default Intel MPI version is 2019 (the last one)
+    impi = self.bsc.intelMpi;
+
+    intelMpi = self.bsc.intelMpi2019;
+
+    intelMpi2019 = callPackage ./bsc/intel-mpi/default.nix {
+      # Intel MPI provides a debug version of the MPI library, but
+      # by default we use the release variant for performance
+      enableDebug = false;
+    };
+
+    # By default we use Intel compiler 2020 update 1
+    iccUnwrapped = self.bsc.icc2020Unwrapped;
+
+    icc2020Unwrapped = callPackage ./bsc/intel-compiler/icc2020.nix {
+      intel-mpi = self.bsc.intelMpi;
+    };
+
+    # A wrapper script that puts all the flags and environment vars properly and
+    # calls the intel compiler binary
+    icc = callPackage ./bsc/intel-compiler/default.nix {
+      iccUnwrapped = self.bsc.iccUnwrapped;
+      intelLicense = self.bsc.intelLicense;
+    };
+
+    intelLicense = callPackage ./bsc/intel-compiler/license.nix { };
+
+    pmix2 = callPackage ./bsc/pmix/pmix2.nix { };
+
+    slurm17 = callPackage ./bsc/slurm/default.nix {
+      pmix = self.bsc.pmix2;
+    };
+
+    slurm17-libpmi2 = callPackage ./bsc/slurm/pmi2.nix {
+      pmix = self.bsc.pmix2;
+    };
+
+    openmpi-mn4 = callPackage ./bsc/openmpi/default.nix {
+      pmix = self.bsc.pmix2;
+      pmi2 = self.bsc.slurm17-libpmi2;
+      enableCxx = true;
+    };
+
+    openmpi = self.bsc.openmpi-mn4;
+
+    fftw = callPackage ./bsc/fftw/default.nix {
+      mpi = self.bsc.mpi;
+    };
+
+    extrae = callPackage ./bsc/extrae/default.nix {
+      mpi = self.bsc.mpi;
+    };
+
+    tampi = callPackage ./bsc/tampi/default.nix {
+      mpi = self.bsc.mpi;
+    };
+
+    mcxx = callPackage ./bsc/mcxx/default.nix { };
+
+    mcxx-rarias = callPackage ./bsc/mcxx/rarias.nix { };
+
+    # Use nanos6 git by default
+    nanos6 = self.nanos6-git;
+    nanos6-latest = callPackage ./bsc/nanos6/default.nix {
+      extrae = self.bsc.extrae;
+    };
+
+    nanos6-git = callPackage ./bsc/nanos6/git.nix {
+      extrae = self.bsc.extrae;
+    };
+
+    vtk = callPackage ./bsc/vtk/default.nix {
+      mpi = self.bsc.mpi;
+      inherit (self.xorg) libX11 xorgproto libXt;
+    };
+
+    dummy = callPackage ./bsc/dummy/default.nix { };
+
+    clang-ompss2-unwrapped = callPackage ./bsc/llvm-ompss2/clang.nix {
+      stdenv = self.llvmPackages_10.stdenv;
+      enableDebug = false;
+    };
+
+    clang-ompss2 = callPackage bsc/llvm-ompss2/default.nix {
+      clang-ompss2-unwrapped = self.bsc.clang-ompss2-unwrapped;
+    };
+
+    stdenvOmpss2 = self.clangStdenv.override {
+      cc = self.bsc.clang-ompss2;
+    };
+
+    cpic = callPackage ./bsc/apps/cpic/default.nix {
+      stdenv = self.bsc.stdenvOmpss2;
+      mpi = self.bsc.mpi;
+      tampi = self.bsc.tampi;
+    };
+
+    mpptest = callPackage ./bsc/mpptest/default.nix { };
+
+
+    garlic = {
+
+      # Load some helper functions to generate app variants
+      inherit (import ./bsc/garlic/gen.nix) genApps genApp genConfigs;
+
+      mpptest = callPackage ./bsc/garlic/mpptest { };
+
+      ppong = callPackage ./bsc/garlic/ppong {
+        mpi = self.bsc.mpi;
+      };
+
+      nbody = callPackage ./bsc/garlic/nbody {
+        cc = self.bsc.icc;
+        mpi = self.bsc.impi;
+        tampi = self.bsc.tampi;
+        gitBranch = "garlic/seq";
+      };
+
+      runWrappers = {
+        sbatch  = callPackage ./bsc/garlic/sbatch.nix { };
+        srun    = callPackage ./bsc/garlic/srun.nix { };
+        launch  = callPackage ./bsc/garlic/launcher.nix { };
+        control = callPackage ./bsc/garlic/control.nix { };
+        nixsetup= callPackage ./bsc/garlic/nix-setup.nix { };
+        argv    = callPackage ./bsc/garlic/argv.nix { };
+        statspy = callPackage ./bsc/garlic/statspy.nix { };
+        extrae  = callPackage ./bsc/garlic/extrae.nix { };
+        stagen  = callPackage ./bsc/garlic/stagen.nix { };
+      };
+
+      # Perf is tied to a linux kernel specific version
+      linuxPackages = self.linuxPackages_4_4;
+      perfWrapper = callPackage ./bsc/garlic/perf.nix {
+        perf = self.linuxPackages.perf;
+      };
+
+      exp = {
+        noise = callPackage ./bsc/garlic/exp/noise.nix { };
+        nbody = {
+          bs = callPackage ./bsc/garlic/exp/nbody/bs.nix {
+            pkgs = self // self.bsc.garlic;
+            nixpkgs = import <nixpkgs>;
+            genApp = self.bsc.garlic.genApp;
+            genConfigs = self.bsc.garlic.genConfigs;
+            runWrappers = self.bsc.garlic.runWrappers;
+          };
+#          mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { };
+        };
+        osu = rec {
+          latency-internode = callPackage ./bsc/garlic/exp/osu/latency.nix { };
+          latency-intranode = callPackage ./bsc/garlic/exp/osu/latency.nix {
+            interNode = false;
+          };
+          latency = latency-internode;
+        };
+      };
+    };
+  };
+
+in
+  {
+    bsc = bsc;
+  }