diff --git a/bsc/garlic/nbody/default.nix b/bsc/garlic/nbody/default.nix deleted file mode 100644 index b592391..0000000 --- a/bsc/garlic/nbody/default.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, cc -, tampi ? null -, mpi ? null -, cflags ? null -, gitBranch -, gitURL ? "ssh://git@bscpm02.bsc.es/garlic/apps/nbody.git" -, blocksize ? 2048 -}: - -with stdenv.lib; -stdenv.mkDerivation rec { - name = "nbody"; - - src = /home/Computational/rarias/bscpkgs/manual/nbody; - - #src = builtins.fetchGit { - # url = "${gitURL}"; - # ref = "${gitBranch}"; - #}; - programPath = "/bin/nbody"; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) [ mpi ]; - - preBuild = (if cflags != null then '' - makeFlagsArray+=(CFLAGS="${cflags}") - '' else ""); - - makeFlags = [ - "CC=${cc.cc.CC}" - "BS=${toString blocksize}" - ]; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp nbody* $out/bin/${name} - ''; - -} diff --git a/garlic/exp/default.nix b/garlic/exp/default.nix new file mode 100644 index 0000000..79e775b --- /dev/null +++ b/garlic/exp/default.nix @@ -0,0 +1,64 @@ +{ + pkgs +, callPackage +, callPackages +}: + +let + + garlic = { + + # Load some helper functions to generate app variants + inherit (import ./gen.nix) genApps genApp genConfigs; + + mpptest = callPackage ./mpptest { }; + + ppong = callPackage ./ppong { + mpi = pkgs.mpi; + }; + + nbody = callPackage ./nbody { + cc = pkgs.icc; + mpi = pkgs.impi; + tampi = pkgs.tampi; + gitBranch = "garlic/seq"; + }; + + runWrappers = { + sbatch = callPackage ./stages/sbatch.nix { }; + srun = callPackage ./stages/srun.nix { }; + launch = callPackage ./stages/launcher.nix { }; + control = callPackage ./stages/control.nix { }; + nixsetup= callPackage ./stages/nix-setup.nix { }; + argv = callPackage ./stages/argv.nix { }; + statspy = callPackage ./stages/statspy.nix { }; + extrae = callPackage ./stages/extrae.nix { }; + stagen = callPackage ./stages/stagen.nix { }; + }; + + # Perf is tied to a linux kernel specific version + linuxPackages = pkgs.linuxPackages_4_4; + perfWrapper = callPackage ./perf.nix { + perf = pkgs.linuxPackages.perf; + }; + + exp = { + noise = callPackage ./exp/noise.nix { }; + nbody = { + bs = callPackage ./exp/nbody/bs.nix { + pkgs = pkgs // garlic; + }; + mpi = callPackage ./exp/nbody/mpi.nix { }; + }; + osu = rec { + latency-internode = callPackage ./exp/osu/latency.nix { }; + latency-intranode = callPackage ./exp/osu/latency.nix { + interNode = false; + }; + latency = latency-internode; + }; + }; + }; + +in + garlic diff --git a/bsc/garlic/exp/nbody/bs.nix b/garlic/exp/nbody/bs.nix similarity index 98% rename from bsc/garlic/exp/nbody/bs.nix rename to garlic/exp/nbody/bs.nix index 718eb1b..be956fd 100644 --- a/bsc/garlic/exp/nbody/bs.nix +++ b/garlic/exp/nbody/bs.nix @@ -97,7 +97,7 @@ let -p ${toString conf.particles} )''; }; - bscOverlay = import ../../../../overlay.nix; + bscOverlay = import ../../../overlay.nix; genPkgs = newOverlay: nixpkgs { overlays = [ diff --git a/bsc/garlic/exp/nbody/extrae.xml b/garlic/exp/nbody/extrae.xml similarity index 100% rename from bsc/garlic/exp/nbody/extrae.xml rename to garlic/exp/nbody/extrae.xml diff --git a/bsc/garlic/exp/nbody/mpi.nix b/garlic/exp/nbody/mpi.nix similarity index 100% rename from bsc/garlic/exp/nbody/mpi.nix rename to garlic/exp/nbody/mpi.nix diff --git a/garlic/exp/nbody/tampi.nix b/garlic/exp/nbody/tampi.nix new file mode 100644 index 0000000..4ec5d49 --- /dev/null +++ b/garlic/exp/nbody/tampi.nix @@ -0,0 +1,165 @@ +{ + stdenv +, nixpkgs +, pkgs +, genApp +, genConfigs +, runWrappers +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Set variable configuration for the experiment + varConfig = { + cc = [ bsc.icc ]; + mpi = [ bsc.impi bsc.openmpi ]; + blocksize = [ 1024 ]; + }; + + # Common configuration + common = { + # Compile time nbody config + gitBranch = "garlic/tampi+send+oss+task"; + + # nbody runtime options + particles = 1024*128; + timesteps = 20; + + # Resources + ntasksPerNode = "48"; + nodes = "1"; + + # Stage configuration + enableSbatch = true; + enableControl = true; + enableExtrae = false; + enablePerf = false; + enableCtf = false; + + # MN4 path + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // common) (genConfigs varConfig); + + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" else "${stage}"; + + w = runWrappers; + + sbatch = {stage, conf, ...}: with conf; w.sbatch { + program = stageProgram stage; + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-bs"; + inherit nixPrefix nodes ntasksPerNode; + }; + + control = {stage, conf, ...}: with conf; w.control { + program = stageProgram stage; + }; + + srun = {stage, conf, ...}: with conf; w.srun { + program = stageProgram stage; + srunOptions = "--cpu-bind=verbose,rank"; + inherit nixPrefix; + }; + + statspy = {stage, conf, ...}: with conf; w.statspy { + program = stageProgram stage; + }; + + perf = {stage, conf, ...}: with conf; w.perf { + program = stageProgram stage; + perfArgs = "sched record -a"; + }; + + nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + program = stageProgram stage; + }; + + extrae = {stage, conf, ...}: w.extrae { + program = stageProgram stage; + traceLib = "mpi"; # mpi -> libtracempi.so + configFile = ./extrae.xml; + }; + + ctf = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + export NANOS6=ctf + export NANOS6_CTF2PRV=0 + ''; + }; + + argv = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + set -e + export I_MPI_THREAD_SPLIT=1 + ''; + argv = ''( -t ${toString conf.timesteps} + -p ${toString conf.particles} )''; + }; + + bscOverlay = import ../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + nbodyFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.nbody.override { + inherit cc blocksize mpi gitBranch; + }; + + stages = with common; [] + # Use sbatch to request resources first + ++ optional enableSbatch sbatch + + # Repeats the next stages N times + ++ optionals enableControl [ nixsetup control ] + + # Executes srun to launch the program in the requested nodes, and + # immediately after enters the nix environment again, as slurmstepd launches + # the next stages from outside the namespace. + ++ [ srun nixsetup ] + + # Intrumentation with extrae + ++ optional enableExtrae extrae + + # Optionally profile the next stages with perf + ++ optional enablePerf perf + + # Optionally profile nanos6 with the new ctf + ++ optional enableCtf ctf + + # Execute the nbody app with the argv and env vars + ++ [ argv nbodyFn ]; + + # List of actual programs to be executed + jobs = map (conf: w.stagen { inherit conf stages; }) configs; + +in + # We simply run each program one after another + w.launch jobs diff --git a/bsc/garlic/exp/noise.nix b/garlic/exp/noise.nix similarity index 100% rename from bsc/garlic/exp/noise.nix rename to garlic/exp/noise.nix diff --git a/bsc/garlic/exp/osu/latency.nix b/garlic/exp/osu/latency.nix similarity index 100% rename from bsc/garlic/exp/osu/latency.nix rename to garlic/exp/osu/latency.nix diff --git a/bsc/garlic/gen.nix b/garlic/gen.nix similarity index 100% rename from bsc/garlic/gen.nix rename to garlic/gen.nix diff --git a/bsc/garlic/mpptest/default.nix b/garlic/mpptest/default.nix similarity index 100% rename from bsc/garlic/mpptest/default.nix rename to garlic/mpptest/default.nix diff --git a/bsc/garlic/nbody/argv.nix b/garlic/nbody/argv.nix similarity index 100% rename from bsc/garlic/nbody/argv.nix rename to garlic/nbody/argv.nix diff --git a/garlic/nbody/default.nix b/garlic/nbody/default.nix new file mode 100644 index 0000000..b45a852 --- /dev/null +++ b/garlic/nbody/default.nix @@ -0,0 +1,68 @@ +{ + stdenv +, cc +, mpi ? null +, tampi ? null +, mcxx ? null +, cflags ? null +, gitBranch +, gitURL ? "ssh://git@bscpm02.bsc.es/garlic/apps/nbody.git" +, blocksize ? 2048 +}: + +assert !(tampi != null && mcxx == null); + +with stdenv.lib; +stdenv.mkDerivation rec { + name = "nbody"; + + #src = /home/Computational/rarias/bscpkgs/manual/nbody; + + src = builtins.fetchGit { + url = "${gitURL}"; + ref = "${gitBranch}"; + }; + programPath = "/bin/nbody"; + + buildInputs = [ + cc + ] + ++ optional (mpi != null) mpi + ++ optional (tampi != null) tampi + ++ optional (mcxx != null) mcxx; + + preBuild = (if cflags != null then '' + makeFlagsArray+=(CFLAGS="${cflags}") + '' else ""); + + postPatch = "" + + # This should be fixed in the Makefile as well. + + ''sed -i 's/libtampi.a/libtampi-c.a/g' Makefile + '' + # Dirty HACK until the nbody issue at: + # https://pm.bsc.es/gitlab/garlic/apps/nbody/-/issues/1 + # is properly fixed. + + + (if (mpi.pname or "unknown") == "openmpi" then + ''sed -i 's/-lstdc++/-lstdc++ -lmpi_cxx/g' Makefile + '' + else + "" + ); + + makeFlags = [ + "CC=${cc.cc.CC}" + "BS=${toString blocksize}" + ] + ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; + + dontPatchShebangs = true; + + installPhase = '' + echo ${tampi} + mkdir -p $out/bin + cp nbody* $out/bin/${name} + ''; + +} diff --git a/bsc/garlic/ppong/default.nix b/garlic/ppong/default.nix similarity index 100% rename from bsc/garlic/ppong/default.nix rename to garlic/ppong/default.nix diff --git a/bsc/garlic/argv.nix b/garlic/stages/argv.nix similarity index 79% rename from bsc/garlic/argv.nix rename to garlic/stages/argv.nix index 071bfd5..5060661 100644 --- a/bsc/garlic/argv.nix +++ b/garlic/stages/argv.nix @@ -6,7 +6,9 @@ { program , env ? "" -, argv # bash array as string, example: argv=''(-f "file with spaces" -t 10)'' + +# bash array as string, example: argv=''(-f "file with spaces" -t 10)'' +, argv ? "()" }: stdenv.mkDerivation { diff --git a/bsc/garlic/control.nix b/garlic/stages/control.nix similarity index 100% rename from bsc/garlic/control.nix rename to garlic/stages/control.nix diff --git a/bsc/garlic/default.nix b/garlic/stages/default.nix similarity index 100% rename from bsc/garlic/default.nix rename to garlic/stages/default.nix diff --git a/bsc/garlic/extrae.nix b/garlic/stages/extrae.nix similarity index 100% rename from bsc/garlic/extrae.nix rename to garlic/stages/extrae.nix diff --git a/garlic/stages/gen.nix b/garlic/stages/gen.nix new file mode 100644 index 0000000..0c20f9c --- /dev/null +++ b/garlic/stages/gen.nix @@ -0,0 +1,34 @@ +let + lib = import ; + + gen = rec { + # genAttrSets "a" ["hello" "world"] + # [ { a = "hello"; } { a = "world"; } ] + genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); + + # addAttrSets "a" [1 2] {e=4;} + # [ { a = 1; e = 4; } { a = 2; e = 4; } ] + addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); + + # attrToList {a=1;} + # [ { name = "a"; value = 1; } ] + attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); + + # mergeConfig [{e=1;}] {name="a"; value=[1 2] + # [ { a = 1; e = 1; } { a = 2; e = 1; } ] + mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr)); + + # genConfigs {a=[1 2]; b=[3 4];} + # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] + genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config)); + + # Generate multiple app versions by override with each config + genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs); + + # Generate app version from an array of apps + genApps = (apps: configs: + lib.flatten (map (app: genApp app configs) apps)); + + }; +in + gen diff --git a/bsc/garlic/launcher.nix b/garlic/stages/launcher.nix similarity index 100% rename from bsc/garlic/launcher.nix rename to garlic/stages/launcher.nix diff --git a/bsc/garlic/nix-setup.nix b/garlic/stages/nix-setup.nix similarity index 100% rename from bsc/garlic/nix-setup.nix rename to garlic/stages/nix-setup.nix diff --git a/bsc/garlic/perf.nix b/garlic/stages/perf.nix similarity index 100% rename from bsc/garlic/perf.nix rename to garlic/stages/perf.nix diff --git a/bsc/garlic/sbatch.nix b/garlic/stages/sbatch.nix similarity index 100% rename from bsc/garlic/sbatch.nix rename to garlic/stages/sbatch.nix diff --git a/bsc/garlic/srun.nix b/garlic/stages/srun.nix similarity index 100% rename from bsc/garlic/srun.nix rename to garlic/stages/srun.nix diff --git a/bsc/garlic/stagen.nix b/garlic/stages/stagen.nix similarity index 100% rename from bsc/garlic/stagen.nix rename to garlic/stages/stagen.nix diff --git a/bsc/garlic/statspy.nix b/garlic/stages/statspy.nix similarity index 100% rename from bsc/garlic/statspy.nix rename to garlic/stages/statspy.nix diff --git a/overlay.nix b/overlay.nix index ab77d56..81f5d50 100644 --- a/overlay.nix +++ b/overlay.nix @@ -2,20 +2,19 @@ self: /* Future last stage */ super: /* Previous stage */ let - #callPackage = self.callPackage; inherit (self.lib) callPackageWith; inherit (self.lib) callPackagesWith; callPackage = callPackageWith (self // self.bsc); + # --------------------------------------------------------- # + # BSC Packages + # --------------------------------------------------------- # + bsc = { # Default MPI implementation to use. Will be overwritten by the # experiments. mpi = self.bsc.openmpi; - # --------------------------------------------------------- # - # BSC Packages - # --------------------------------------------------------- # - perf = callPackage ./bsc/perf/default.nix { kernel = self.linuxPackages_4_9.kernel; systemtap = self.linuxPackages_4_9.systemtap; @@ -76,34 +75,29 @@ let openmpi = self.bsc.openmpi-mn4; - fftw = callPackage ./bsc/fftw/default.nix { - mpi = self.bsc.mpi; + fftw = callPackage ./bsc/fftw/default.nix { }; + + extrae = callPackage ./bsc/extrae/default.nix { }; + + tampi = callPackage ./bsc/tampi/default.nix { }; + + mcxxGit = callPackage ./bsc/mcxx/default.nix { + bison = self.bison_3_5; }; - extrae = callPackage ./bsc/extrae/default.nix { - mpi = self.bsc.mpi; + mcxxRarias = callPackage ./bsc/mcxx/rarias.nix { + bison = self.bison_3_5; }; - tampi = callPackage ./bsc/tampi/default.nix { - mpi = self.bsc.mpi; - }; - - mcxx = callPackage ./bsc/mcxx/default.nix { }; - - mcxx-rarias = callPackage ./bsc/mcxx/rarias.nix { }; + mcxx = self.bsc.mcxxGit; # Use nanos6 git by default - nanos6 = self.nanos6-git; - nanos6-latest = callPackage ./bsc/nanos6/default.nix { - extrae = self.bsc.extrae; - }; + nanos6 = self.bsc.nanos6-git; + nanos6-latest = callPackage ./bsc/nanos6/default.nix { }; - nanos6-git = callPackage ./bsc/nanos6/git.nix { - extrae = self.bsc.extrae; - }; + nanos6-git = callPackage ./bsc/nanos6/git.nix { }; vtk = callPackage ./bsc/vtk/default.nix { - mpi = self.bsc.mpi; inherit (self.xorg) libX11 xorgproto libXt; }; @@ -130,47 +124,57 @@ let mpptest = callPackage ./bsc/mpptest/default.nix { }; - garlic = { # Load some helper functions to generate app variants - inherit (import ./bsc/garlic/gen.nix) genApps genApp genConfigs; + inherit (import ./garlic/gen.nix) genApps genApp genConfigs; - mpptest = callPackage ./bsc/garlic/mpptest { }; + mpptest = callPackage ./garlic/mpptest { }; - ppong = callPackage ./bsc/garlic/ppong { + ppong = callPackage ./garlic/ppong { mpi = self.bsc.mpi; }; - nbody = callPackage ./bsc/garlic/nbody { + nbody = callPackage ./garlic/nbody { cc = self.bsc.icc; - mpi = self.bsc.impi; + mpi = self.bsc.mpi; tampi = self.bsc.tampi; + mcxx = self.bsc.mcxx; gitBranch = "garlic/seq"; }; + # Execution wrappers runWrappers = { - sbatch = callPackage ./bsc/garlic/sbatch.nix { }; - srun = callPackage ./bsc/garlic/srun.nix { }; - launch = callPackage ./bsc/garlic/launcher.nix { }; - control = callPackage ./bsc/garlic/control.nix { }; - nixsetup= callPackage ./bsc/garlic/nix-setup.nix { }; - argv = callPackage ./bsc/garlic/argv.nix { }; - statspy = callPackage ./bsc/garlic/statspy.nix { }; - extrae = callPackage ./bsc/garlic/extrae.nix { }; - stagen = callPackage ./bsc/garlic/stagen.nix { }; + sbatch = callPackage ./garlic/stages/sbatch.nix { }; + srun = callPackage ./garlic/stages/srun.nix { }; + launch = callPackage ./garlic/stages/launcher.nix { }; + control = callPackage ./garlic/stages/control.nix { }; + nixsetup = callPackage ./garlic/stages/nix-setup.nix { }; + argv = callPackage ./garlic/stages/argv.nix { }; + statspy = callPackage ./garlic/stages/statspy.nix { }; + extrae = callPackage ./garlic/stages/extrae.nix { }; + stagen = callPackage ./garlic/stages/stagen.nix { }; + perf = callPackage ./garlic/stages/perf.nix { }; }; # Perf is tied to a linux kernel specific version - linuxPackages = self.linuxPackages_4_4; - perfWrapper = callPackage ./bsc/garlic/perf.nix { - perf = self.linuxPackages.perf; - }; + #linuxPackages = self.linuxPackages_4_4; + #perfWrapper = callPackage ./garlic/perf.nix { + # perf = self.linuxPackages.perf; + #}; exp = { - noise = callPackage ./bsc/garlic/exp/noise.nix { }; + noise = callPackage ./garlic/exp/noise.nix { }; nbody = { - bs = callPackage ./bsc/garlic/exp/nbody/bs.nix { + bs = callPackage ./garlic/exp/nbody/bs.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; + + tampi = callPackage ./garlic/exp/nbody/tampi.nix { pkgs = self // self.bsc.garlic; nixpkgs = import ; genApp = self.bsc.garlic.genApp; @@ -180,8 +184,8 @@ let # mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { }; }; osu = rec { - latency-internode = callPackage ./bsc/garlic/exp/osu/latency.nix { }; - latency-intranode = callPackage ./bsc/garlic/exp/osu/latency.nix { + latency-internode = callPackage ./garlic/exp/osu/latency.nix { }; + latency-intranode = callPackage ./garlic/exp/osu/latency.nix { interNode = false; }; latency = latency-internode; @@ -193,4 +197,10 @@ let in { bsc = bsc; + + # Alias + garlic = bsc.garlic; + + # Alias + exp = bsc.garlic.exp; }