diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix index ec6c1f3..f5d727e 100644 --- a/bsc/garlic/default.nix +++ b/bsc/garlic/default.nix @@ -1,27 +1,26 @@ { pkgs -, bsc +, callPackage +, callPackages }: let - callPackage = pkgs.lib.callPackageWith (pkgs // bsc // garlic); - callPackages = pkgs.lib.callPackagesWith (pkgs // bsc // garlic); - garlic = rec { + garlic = { # Load some helper functions to generate app variants inherit (import ./gen.nix) genApps genApp genConfigs; - inherit bsc; mpptest = callPackage ./mpptest { }; ppong = callPackage ./ppong { - mpi = bsc.mpi; + mpi = pkgs.mpi; }; nbody = callPackage ./nbody { - cc = bsc.icc; - mpi = bsc.impi; + cc = pkgs.icc; + mpi = pkgs.impi; + tampi = pkgs.tampi; gitBranch = "garlic/seq"; }; @@ -38,15 +37,17 @@ let }; # Perf is tied to a linux kernel specific version - linuxPackages = bsc.linuxPackages_4_4; + linuxPackages = pkgs.linuxPackages_4_4; perfWrapper = callPackage ./perf.nix { - perf = linuxPackages.perf; + perf = pkgs.linuxPackages.perf; }; exp = { noise = callPackage ./exp/noise.nix { }; nbody = { - bs = callPackage ./exp/nbody/bs.nix { }; + bs = callPackage ./exp/nbody/bs.nix { + pkgs = pkgs // garlic; + }; mpi = callPackage ./exp/nbody/mpi.nix { }; }; osu = rec { diff --git a/bsc/garlic/exp/nbody/bs.nix b/bsc/garlic/exp/nbody/bs.nix index e1591a8..718eb1b 100644 --- a/bsc/garlic/exp/nbody/bs.nix +++ b/bsc/garlic/exp/nbody/bs.nix @@ -1,7 +1,7 @@ { - bsc -, stdenv -, nbody + stdenv +, nixpkgs +, pkgs , genApp , genConfigs , runWrappers @@ -12,7 +12,7 @@ with stdenv.lib; let # Set variable configuration for the experiment varConfig = { - cc = [ bsc.icc ]; + cc = [ pkgs.bsc.icc ]; blocksize = [ 1024 ]; }; @@ -20,7 +20,7 @@ let common = { # Compile time nbody config gitBranch = "garlic/mpi+send"; - mpi = bsc.impi; + mpi = pkgs.bsc.impi; # nbody runtime options particles = 1024*128; @@ -97,10 +97,31 @@ let -p ${toString conf.particles} )''; }; - nbodyFn = {stage, conf, ...}: with conf; nbody.override { - inherit cc blocksize mpi gitBranch; + bscOverlay = import ../../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; }; + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + nbodyFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.nbody.override { + inherit cc blocksize mpi gitBranch; + }; + stages = with common; [] # Use sbatch to request resources first ++ optional enableSbatch sbatch diff --git a/bsc/garlic/nbody/default.nix b/bsc/garlic/nbody/default.nix index 4955187..b592391 100644 --- a/bsc/garlic/nbody/default.nix +++ b/bsc/garlic/nbody/default.nix @@ -1,6 +1,7 @@ { stdenv , cc +, tampi ? null , mpi ? null , cflags ? null , gitBranch @@ -37,6 +38,7 @@ stdenv.mkDerivation rec { dontPatchShebangs = true; installPhase = '' + echo ${tampi} mkdir -p $out/bin cp nbody* $out/bin/${name} ''; diff --git a/bsc/intel-compiler/default.nix b/bsc/intel-compiler/default.nix index 14ae4b8..17296ad 100644 --- a/bsc/intel-compiler/default.nix +++ b/bsc/intel-compiler/default.nix @@ -2,27 +2,27 @@ stdenv , gcc , nanos6 -, icc-unwrapped +, iccUnwrapped , wrapCCWith -, intel-license +, intelLicense }: let targetConfig = stdenv.targetPlatform.config; inherit gcc; in wrapCCWith rec { - cc = icc-unwrapped; + cc = iccUnwrapped; extraBuildCommands = '' echo "-B${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-cflags - echo "-isystem ${icc-unwrapped}/include" >> $out/nix-support/cc-cflags - echo "-isystem ${icc-unwrapped}/include/intel64" >> $out/nix-support/cc-cflags + echo "-isystem ${iccUnwrapped}/include" >> $out/nix-support/cc-cflags + echo "-isystem ${iccUnwrapped}/include/intel64" >> $out/nix-support/cc-cflags echo "-L${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-ldflags echo "-L${gcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags - cat "${icc-unwrapped}/nix-support/propagated-build-inputs" >> \ + cat "${iccUnwrapped}/nix-support/propagated-build-inputs" >> \ $out/nix-support/propagated-build-inputs - echo "export INTEL_LICENSE_FILE=${intel-license}" \ + echo "export INTEL_LICENSE_FILE=${intelLicense}" \ >> $out/nix-support/setup-hook # Create the wrappers for icc and icpc diff --git a/default.nix b/default.nix index 9ff9f0c..0077d5f 100644 --- a/default.nix +++ b/default.nix @@ -1,229 +1,7 @@ -{ pkgs ? import {} }: - let - inherit (pkgs.lib) callPackageWith; - inherit (pkgs.lib) callPackagesWith; - callPackage = callPackageWith (pkgs // self.bsc); - callPackageStatic = callPackageWith (pkgs.pkgsStatic); - callPackages = callPackagesWith (pkgs // self.bsc); - - self.bsc = rec { - - nixpkgs = pkgs; - - - # Load the default implementation - #mpi = mpich; - #mpi = openmpi; - mpi = intel-mpi; - - # Load the default compiler - #stdenv = pkgs.gcc10Stdenv; - stdenv = pkgs.gcc9Stdenv; - #stdenv = pkgs.gcc7Stdenv; - #stdenv = pkgs.llvmPackages_10.stdenv; - #stdenv = pkgs.llvmPackages_9.stdenv; - #stdenv = pkgs.llvmPackages_8.stdenv; - #stdenv = pkgs.llvmPackages_7.stdenv; - - binutils = pkgs.binutils; - coreutils = pkgs.coreutils; - gcc = stdenv.cc; - - nanos6 = nanos6-git; - - # --------------------------------------------------------- # - # BSC Packages - # --------------------------------------------------------- # - - perf = callPackage ./bsc/perf/default.nix { - kernel = pkgs.linuxPackages_4_9.kernel; - systemtap = pkgs.linuxPackages_4_9.systemtap; - }; - - # ParaStation MPI - pscom = callPackage ./bsc/parastation/pscom.nix { }; - psmpi = callPackage ./bsc/parastation/psmpi.nix { }; - - osumb = callPackage ./bsc/osu/default.nix { }; - - mpich = callPackage ./bsc/mpich/default.nix { }; - mpichDbg = callPackage ./bsc/mpich/default.nix { enableDebug = true; }; - - # Default Intel MPI version is 2019 (the last one) - impi = intel-mpi; - intel-mpi = intel-mpi-2019; - intel-mpi-2019 = callPackage ./bsc/intel-mpi/default.nix { - # Intel MPI provides a debug version of the MPI library, but - # by default we use the release variant for performance - enableDebug = false; - }; - - # By default we use Intel compiler 2020 update 1 - icc-unwrapped = icc2020-unwrapped; - icc2020-unwrapped = callPackage ./bsc/intel-compiler/icc2020.nix { - intel-mpi = intel-mpi-2019; - }; - - # A wrapper script that puts all the flags and environment vars properly and - # calls the intel compiler binary - icc = callPackage bsc/intel-compiler/default.nix { - inherit icc-unwrapped intel-license; - }; - - intel-license = callPackage bsc/intel-compiler/license.nix { - }; - - pmix2 = callPackage ./bsc/pmix/pmix2.nix { }; - - slurm17 = callPackage ./bsc/slurm/default.nix { - pmix = pmix2; - }; - - slurm17-libpmi2 = callPackage ./bsc/slurm/pmi2.nix { - pmix = pmix2; - }; - - openmpi-mn4 = callPackage ./bsc/openmpi/default.nix { - pmix = pmix2; - pmi2 = slurm17-libpmi2; - enableCxx = true; - }; - - openmpi = openmpi-mn4; - - fftw = callPackage ./bsc/fftw/default.nix { - mpi = mpi; - }; - - extrae = callPackage ./bsc/extrae/default.nix { - mpi = mpi; - }; - - tampi = callPackage ./bsc/tampi/default.nix { - mpi = mpi; - }; - - mcxx = callPackage ./bsc/mcxx/default.nix { - }; - - mcxx-rarias = callPackage ./bsc/mcxx/rarias.nix { - }; - - nanos6-latest = callPackage ./bsc/nanos6/default.nix { - extrae = extrae; - }; - - nanos6-git = callPackage ./bsc/nanos6/git.nix { - extrae = extrae; - }; - - vtk = callPackage ./bsc/vtk/default.nix { - mpi = mpi; - inherit (pkgs.xorg) libX11 xorgproto libXt; - }; - - dummy = callPackage ./bsc/dummy/default.nix { - }; - - clang-ompss2-unwrapped = callPackage ./bsc/llvm-ompss2/clang.nix { - stdenv = pkgs.llvmPackages_10.stdenv; - enableDebug = false; - }; - - clang-ompss2 = callPackage bsc/llvm-ompss2/default.nix { - inherit clang-ompss2-unwrapped; - }; - - stdenv-nanos6 = pkgs.clangStdenv.override { - cc = clang-ompss2; - }; - - cpic = callPackage ./bsc/apps/cpic/default.nix { - stdenv = stdenv-nanos6; - inherit mpi tampi; - }; - - mpptest = callPackage ./bsc/mpptest/default.nix { - }; - - # Apps for Garlic - nbody = callPackage ./bsc/apps/nbody/default.nix { - stdenv = pkgs.gcc9Stdenv; - mpi = intel-mpi; - tampi = tampi; - }; - - heat = callPackage ./bsc/apps/heat/default.nix { - stdenv = pkgs.gcc7Stdenv; - mpi = intel-mpi; - tampi = tampi; - }; - - saiph = callPackage ./bsc/apps/saiph/default.nix { - stdenv = stdenv-nanos6; - mpi = intel-mpi; - tampi = tampi; - inherit vtk; - boost = pkgs.boost; - }; - - creams = callPackage ./bsc/apps/creams/default.nix { - stdenv = pkgs.gcc9Stdenv; - mpi = intel-mpi; - tampi = tampi.override { - mpi = intel-mpi; - }; - }; - - lulesh = callPackage ./bsc/apps/lulesh/default.nix { - mpi = intel-mpi; - }; - - hpcg = callPackage ./bsc/apps/hpcg/default.nix { - }; - - hpccg = callPackage ./bsc/apps/hpccg/default.nix { - }; - - fwi = callPackage ./bsc/apps/fwi/default.nix { - }; - - garlic = callPackage ./bsc/garlic/default.nix { - pkgs = pkgs; - bsc = self.bsc; - }; - - # Patched nix for deep cluster - inherit (callPackage ./bsc/nix/default.nix { - storeDir = "/nix/store"; - stateDir = "/nix/var"; - boehmgc = pkgs.boehmgc.override { enableLargeConfig = true; }; - }) - nix - nixUnstable - nixFlakes; - - clsync = callPackage ./bsc/clsync/default.nix { }; - - nixStatic = (callPackageStatic ./bsc/nix/static.nix { - callPackage = callPackageWith (pkgs.pkgsStatic); - storeDir = "/nix/store"; - stateDir = "/nix/var"; - sandbox-shell = "/bin/sh"; - boehmgc = pkgs.pkgsStatic.boehmgc.override { enableLargeConfig = true; }; - }).nix; - - test = { - chroot = callPackage ./test/chroot.nix { }; - - internet = callPackage ./test/security/internet.nix { }; - - clang-ompss2 = callPackage ./test/compilers/clang-ompss2.nix { - stdenv = stdenv-nanos6; - inherit clang-ompss2; - }; - }; + bscOverlay = import ./overlay.nix; + pkgs = import { + overlays = [ bscOverlay ]; }; -in pkgs // self +in pkgs diff --git a/overlay.nix b/overlay.nix new file mode 100644 index 0000000..ab77d56 --- /dev/null +++ b/overlay.nix @@ -0,0 +1,196 @@ +self: /* Future last stage */ +super: /* Previous stage */ + +let + #callPackage = self.callPackage; + inherit (self.lib) callPackageWith; + inherit (self.lib) callPackagesWith; + callPackage = callPackageWith (self // self.bsc); + + bsc = { + # Default MPI implementation to use. Will be overwritten by the + # experiments. + mpi = self.bsc.openmpi; + + # --------------------------------------------------------- # + # BSC Packages + # --------------------------------------------------------- # + + perf = callPackage ./bsc/perf/default.nix { + kernel = self.linuxPackages_4_9.kernel; + systemtap = self.linuxPackages_4_9.systemtap; + }; + + # ParaStation MPI + pscom = callPackage ./bsc/parastation/pscom.nix { }; + psmpi = callPackage ./bsc/parastation/psmpi.nix { }; + + osumb = callPackage ./bsc/osu/default.nix { }; + + mpich = callPackage ./bsc/mpich/default.nix { }; + + mpichDebug = self.mpich.override { enableDebug = true; }; + + # Default Intel MPI version is 2019 (the last one) + impi = self.bsc.intelMpi; + + intelMpi = self.bsc.intelMpi2019; + + intelMpi2019 = callPackage ./bsc/intel-mpi/default.nix { + # Intel MPI provides a debug version of the MPI library, but + # by default we use the release variant for performance + enableDebug = false; + }; + + # By default we use Intel compiler 2020 update 1 + iccUnwrapped = self.bsc.icc2020Unwrapped; + + icc2020Unwrapped = callPackage ./bsc/intel-compiler/icc2020.nix { + intel-mpi = self.bsc.intelMpi; + }; + + # A wrapper script that puts all the flags and environment vars properly and + # calls the intel compiler binary + icc = callPackage ./bsc/intel-compiler/default.nix { + iccUnwrapped = self.bsc.iccUnwrapped; + intelLicense = self.bsc.intelLicense; + }; + + intelLicense = callPackage ./bsc/intel-compiler/license.nix { }; + + pmix2 = callPackage ./bsc/pmix/pmix2.nix { }; + + slurm17 = callPackage ./bsc/slurm/default.nix { + pmix = self.bsc.pmix2; + }; + + slurm17-libpmi2 = callPackage ./bsc/slurm/pmi2.nix { + pmix = self.bsc.pmix2; + }; + + openmpi-mn4 = callPackage ./bsc/openmpi/default.nix { + pmix = self.bsc.pmix2; + pmi2 = self.bsc.slurm17-libpmi2; + enableCxx = true; + }; + + openmpi = self.bsc.openmpi-mn4; + + fftw = callPackage ./bsc/fftw/default.nix { + mpi = self.bsc.mpi; + }; + + extrae = callPackage ./bsc/extrae/default.nix { + mpi = self.bsc.mpi; + }; + + tampi = callPackage ./bsc/tampi/default.nix { + mpi = self.bsc.mpi; + }; + + mcxx = callPackage ./bsc/mcxx/default.nix { }; + + mcxx-rarias = callPackage ./bsc/mcxx/rarias.nix { }; + + # Use nanos6 git by default + nanos6 = self.nanos6-git; + nanos6-latest = callPackage ./bsc/nanos6/default.nix { + extrae = self.bsc.extrae; + }; + + nanos6-git = callPackage ./bsc/nanos6/git.nix { + extrae = self.bsc.extrae; + }; + + vtk = callPackage ./bsc/vtk/default.nix { + mpi = self.bsc.mpi; + inherit (self.xorg) libX11 xorgproto libXt; + }; + + dummy = callPackage ./bsc/dummy/default.nix { }; + + clang-ompss2-unwrapped = callPackage ./bsc/llvm-ompss2/clang.nix { + stdenv = self.llvmPackages_10.stdenv; + enableDebug = false; + }; + + clang-ompss2 = callPackage bsc/llvm-ompss2/default.nix { + clang-ompss2-unwrapped = self.bsc.clang-ompss2-unwrapped; + }; + + stdenvOmpss2 = self.clangStdenv.override { + cc = self.bsc.clang-ompss2; + }; + + cpic = callPackage ./bsc/apps/cpic/default.nix { + stdenv = self.bsc.stdenvOmpss2; + mpi = self.bsc.mpi; + tampi = self.bsc.tampi; + }; + + mpptest = callPackage ./bsc/mpptest/default.nix { }; + + + garlic = { + + # Load some helper functions to generate app variants + inherit (import ./bsc/garlic/gen.nix) genApps genApp genConfigs; + + mpptest = callPackage ./bsc/garlic/mpptest { }; + + ppong = callPackage ./bsc/garlic/ppong { + mpi = self.bsc.mpi; + }; + + nbody = callPackage ./bsc/garlic/nbody { + cc = self.bsc.icc; + mpi = self.bsc.impi; + tampi = self.bsc.tampi; + gitBranch = "garlic/seq"; + }; + + runWrappers = { + sbatch = callPackage ./bsc/garlic/sbatch.nix { }; + srun = callPackage ./bsc/garlic/srun.nix { }; + launch = callPackage ./bsc/garlic/launcher.nix { }; + control = callPackage ./bsc/garlic/control.nix { }; + nixsetup= callPackage ./bsc/garlic/nix-setup.nix { }; + argv = callPackage ./bsc/garlic/argv.nix { }; + statspy = callPackage ./bsc/garlic/statspy.nix { }; + extrae = callPackage ./bsc/garlic/extrae.nix { }; + stagen = callPackage ./bsc/garlic/stagen.nix { }; + }; + + # Perf is tied to a linux kernel specific version + linuxPackages = self.linuxPackages_4_4; + perfWrapper = callPackage ./bsc/garlic/perf.nix { + perf = self.linuxPackages.perf; + }; + + exp = { + noise = callPackage ./bsc/garlic/exp/noise.nix { }; + nbody = { + bs = callPackage ./bsc/garlic/exp/nbody/bs.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; +# mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { }; + }; + osu = rec { + latency-internode = callPackage ./bsc/garlic/exp/osu/latency.nix { }; + latency-intranode = callPackage ./bsc/garlic/exp/osu/latency.nix { + interNode = false; + }; + latency = latency-internode; + }; + }; + }; + }; + +in + { + bsc = bsc; + }