diff --git a/garlic/exp/nbody/common.nix b/garlic/exp/nbody/common.nix new file mode 100644 index 0000000..eb73c09 --- /dev/null +++ b/garlic/exp/nbody/common.nix @@ -0,0 +1,74 @@ +{ + stdenv +, nixpkgs +, pkgs +, stages +, machineConf +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + w = runWrappers; +in +{ + /* Returns the path of the executable of a stage */ + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" + else "${stage}"; + + /* Takes a list of units and builds an experiment, after executing the + trebuchet and the isolate stages. Returns the trebuchet stage. */ + buildExperiment = {units, conf, ...}: stage.trebuchet { + inherit (machineConf) nixPrefix; + nextStage = stage.isolate { + inherit (machineConf) nixPrefix; + nextStage = stage.experiment { + inherit units; + } + } + }; + + sbatch = {nextStage, conf, ...}: with conf; w.sbatch ( + # Allow a user to define a custom reservation for the job in MareNostrum4, + # by setting the garlic.sbatch.reservation attribute in the + # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no + # reservation is used. The user reservation may be overwritten by the + # experiment, if the reservation is set like with nodes or ntasksPerNode. + optionalAttrs (pkgs.config ? garlic.sbatch.reservation) { + inherit (pkgs.config.garlic.sbatch) reservation; + } // { + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-tampi"; + inherit nextStage nixPrefix nodes ntasksPerNode; + } + ); + + control = {nextStage, conf, ...}: stages.control { + inherit (conf) loops; + inherit nextStage; + }; + + srun = {nextStage, conf, ...}: stages.srun { + inherit (conf) nixPrefix cpuBind; + inherit nextStage; + }; + + isolate = {nextStage, conf, ...}: stages.isolate { + clusterName = machineConf.name; + inherit (conf) nixPrefix; + inherit nextStage; + }; + + stdStages = [ + sbatch + isolate + control + srun + isolate + ]; +} diff --git a/garlic/exp/nbody/test.nix b/garlic/exp/nbody/test.nix new file mode 100644 index 0000000..e5e72ed --- /dev/null +++ b/garlic/exp/nbody/test.nix @@ -0,0 +1,75 @@ +{ + stdenv +, stdexp +, pkgs +, targetMachine +, stages +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Configurations for each unit (using the cartesian product) + confUnit = with bsc; { + blocksize = [ 1024 2048 ]; + }; + + # Configuration for the complete experiment + confExperiment = with bsc; { + # nbody options + particles = 1024 * 4; + timesteps = 10; + cc = icc; + mpi = impi; + gitBranch = "garlic/mpi+send"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + ntasksPerNode = 2; + nodes = 1; + cpuBind = "sockets,verbose"; + }; + + confMachine = targetMachine.config; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + var = confUnit; + fixed = confMachine // confExperiment; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + argv = [ "-t" timesteps "-p" particles ]; + env = ""; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + program = {nextStage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = stdexp.genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.garlic.nbody.override { + inherit cc blocksize mpi gitBranch; + }; + + # Generate the experimental units + units = map (c: stages.unit { + conf = c; + stages = stdexp.stdStages ++ [ exec program ]; + }) configs; + +in + + elemAt units 0 + #buildExperiment units diff --git a/garlic/exp/stdexp.nix b/garlic/exp/stdexp.nix new file mode 100644 index 0000000..ef2b05a --- /dev/null +++ b/garlic/exp/stdexp.nix @@ -0,0 +1,84 @@ +{ + stdenv +, config +, stages +, targetMachine +, garlicTools +}: + +with stdenv.lib; +with garlicTools; + +let + machineConf = targetMachine.config; +in +rec { + /* Takes a list of units and builds an experiment, after executing the + trebuchet and the isolate stages. Returns the trebuchet stage. */ + buildExperiment = {units, conf, ...}: stage.trebuchet { + inherit (machineConf) nixPrefix; + nextStage = stage.isolate { + inherit (machineConf) nixPrefix; + nextStage = stage.experiment { + inherit units; + }; + }; + }; + + /* Given an attrset of lists `var` and an attrset `fixed`, computes the + cartesian product of all combinations of `var` and prepends `fixed` + to each. */ + buildConfigs = {fixed, var}: + map (c: fixed // c) (genConfigs var); + + sbatch = {nextStage, conf, ...}: with conf; stages.sbatch ( + # Allow a user to define a custom reservation for the job in MareNostrum4, + # by setting the garlic.sbatch.reservation attribute in the + # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no + # reservation is used. The user reservation may be overwritten by the + # experiment, if the reservation is set like with nodes or ntasksPerNode. + optionalAttrs (config ? garlic.sbatch.reservation) { + inherit (config.garlic.sbatch) reservation; + } // { + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-tampi"; + inherit nextStage nixPrefix nodes ntasksPerNode; + } + ); + + control = {nextStage, conf, ...}: stages.control { + inherit (conf) loops; + inherit nextStage; + }; + + srun = {nextStage, conf, ...}: stages.srun { + inherit (conf) nixPrefix cpuBind; + inherit nextStage; + }; + + isolate = {nextStage, conf, ...}: stages.isolate { + clusterName = machineConf.name; + inherit (conf) nixPrefix; + inherit nextStage; + }; + + stdStages = [ + sbatch + isolate + control + srun + isolate + ]; + + # FIXME: Remove this hack and allow custom nixpkgs + bscOverlay = import ../../overlay.nix; + nixpkgs = import ; + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; +} diff --git a/garlic/machines.nix b/garlic/machines.nix new file mode 100644 index 0000000..197f49d --- /dev/null +++ b/garlic/machines.nix @@ -0,0 +1,17 @@ +{ + stdenv +}: + +{ + # MareNostrum 4 configuration + mn4 = { + config = { + name = "mn4"; + nixPrefix = "/gpfs/projects/bsc15/nix"; + cachelineBytes = 64; + march = "skylake-avx512"; + mtune = "skylake-avx512"; + }; + # TODO: Add the specific details for SLURM and the interconection here + }; +} diff --git a/garlic/stages/argv.nix b/garlic/stages/argv.nix deleted file mode 100644 index e879851..0000000 --- a/garlic/stages/argv.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - stdenv -, bash -}: - -{ - program -, env ? "" - -# bash array as string, example: argv=''(-f "file with spaces" -t 10)'' -, argv ? "()" -}: - -stdenv.mkDerivation { - name = "argv"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out < $out < $out < $out/${name} < $out <; +{ + stdenv +}: +with stdenv.lib; + +let gen = rec { # genAttrSets "a" ["hello" "world"] # [ { a = "hello"; } { a = "world"; } ] @@ -16,18 +20,24 @@ let # mergeConfig [{e=1;}] {name="a"; value=[1 2] # [ { a = 1; e = 1; } { a = 2; e = 1; } ] - mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr)); + mergeConfig = (arr: new: flatten ( map (x: addAttrSets new.name new.value x) arr)); # genConfigs {a=[1 2]; b=[3 4];} # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] - genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config)); + genConfigs = (config: foldl mergeConfig [{}] (attrToList config)); # Generate multiple app versions by override with each config genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs); # Generate app version from an array of apps genApps = (apps: configs: - lib.flatten (map (app: genApp app configs) apps)); + flatten (map (app: genApp app configs) apps)); + + /* Returns the path of the executable of a stage */ + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" + else "${stage}"; }; in diff --git a/overlay.nix b/overlay.nix index b329277..fbba8e3 100644 --- a/overlay.nix +++ b/overlay.nix @@ -140,13 +140,20 @@ let nixPrefix = "/gpfs/projects/bsc15/nix"; }; + garlicTools = callPackage ./garlic/tools.nix {}; + garlic = { + # Configuration for the machines + machines = callPackage ./garlic/machines.nix {}; + + # Use the configuration for the following target machine + targetMachine = self.garlic.machines.mn4; # Load some helper functions to generate app variants - inherit (import ./garlic/gen.nix) genApps genApp genConfigs; - # Override the hardening flags and parallel build by default (TODO) - #mkDerivation = callPackage ./garlic/mkDerivation.nix { }; + stdexp = callPackage ./garlic/exp/stdexp.nix { + inherit (self.garlic) targetMachine stages; + }; # Apps for Garlic # heat = callPackage ./garlic/heat { @@ -187,30 +194,30 @@ let gitBranch = "garlic/seq"; }; - saiph = callPackage ./garlic/saiph { + saiph = callPackage ./garlic/saiph/default.nix { cc = self.bsc.clangOmpss2; }; - # Execution wrappers - runWrappers = { - sbatch = callPackage ./garlic/stages/sbatch.nix { }; - srun = callPackage ./garlic/stages/srun.nix { }; - launch = callPackage ./garlic/stages/launcher { }; - control = callPackage ./garlic/stages/control.nix { }; - nixsetup = callPackage ./garlic/stages/nix-setup.nix { }; - argv = callPackage ./garlic/stages/argv.nix { }; - statspy = callPackage ./garlic/stages/statspy.nix { }; - extrae = callPackage ./garlic/stages/extrae.nix { }; - stagen = callPackage ./garlic/stages/stagen.nix { }; - perf = callPackage ./garlic/stages/perf.nix { }; - broom = callPackage ./garlic/stages/broom.nix { }; - envRecord = callPackage ./garlic/stages/envRecord.nix { }; - valgrind = callPackage ./garlic/stages/valgrind.nix { }; - isolate = callPackage ./garlic/stages/isolate { }; - trebuchet = callPackage ./garlic/stages/trebuchet.nix { }; - strace = callPackage ./garlic/stages/strace.nix { }; - unit = callPackage ./garlic/stages/unit.nix { }; - experiment= callPackage ./garlic/stages/experiment.nix { }; + # Execution stages + stages = { + sbatch = callPackage ./garlic/stages/sbatch.nix { }; + srun = callPackage ./garlic/stages/srun.nix { }; + launch = callPackage ./garlic/stages/launcher { }; + control = callPackage ./garlic/stages/control.nix { }; + nixsetup = callPackage ./garlic/stages/nix-setup.nix { }; + exec = callPackage ./garlic/stages/exec.nix { }; + statspy = callPackage ./garlic/stages/statspy.nix { }; + extrae = callPackage ./garlic/stages/extrae.nix { }; + stagen = callPackage ./garlic/stages/stagen.nix { }; + perf = callPackage ./garlic/stages/perf.nix { }; + broom = callPackage ./garlic/stages/broom.nix { }; + envRecord = callPackage ./garlic/stages/envRecord.nix { }; + valgrind = callPackage ./garlic/stages/valgrind.nix { }; + isolate = callPackage ./garlic/stages/isolate { }; + trebuchet = callPackage ./garlic/stages/trebuchet.nix { }; + strace = callPackage ./garlic/stages/strace.nix { }; + unit = callPackage ./garlic/stages/unit.nix { }; + experiment = callPackage ./garlic/stages/experiment.nix { }; }; # Tests (move to bsc ?) @@ -231,7 +238,7 @@ let nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; }; tampi = callPackage ./garlic/exp/nbody/tampi.nix { @@ -239,7 +246,12 @@ let nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; + }; + + test = callPackage ./garlic/exp/nbody/test.nix { + pkgs = self // self.bsc.garlic; + inherit (self.garlic) stdexp targetMachine stages; }; # mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { }; }; @@ -250,7 +262,7 @@ let nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; }; }; @@ -261,14 +273,14 @@ let nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; }; hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { pkgs = self // self.bsc.garlic; nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; }; }; }; @@ -287,12 +299,18 @@ let nixpkgs = import ; genApp = self.bsc.garlic.genApp; genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + stages = self.bsc.garlic.stages; }; # mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { }; }; }; }; + + test = { + exec = callPackage ./test/garlic/exec.nix { + exec = self.bsc.garlic.stages.exec; + }; + }; }; in