diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix index 7ba8545..257f02e 100644 --- a/garlic/apps/hpcg/default.nix +++ b/garlic/apps/hpcg/default.nix @@ -41,4 +41,6 @@ stdenv.mkDerivation rec { cp bin/* $out/bin/ ''; + programPath = "/bin/xhpcg"; + } diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix index 9adc093..91b0429 100644 --- a/garlic/exp/hpcg/mpi.nix +++ b/garlic/exp/hpcg/mpi.nix @@ -1,169 +1,65 @@ { stdenv -, nixpkgs -, pkgs -, genApp -, genConfigs -, runWrappers +, stdexp +, bsc +, targetMachine +, stages }: with stdenv.lib; let - bsc = pkgs.bsc; - - # Set variable configuration for the experiment - varConfig = { - cc = [ bsc.icc ]; - mpi = [ bsc.impi ]; - gitBranch = [ "garlic/mpi" ]; - makefileName = [ "MPI" ]; + # Initial variable configuration + varConf = with bsc; { n = [ 104 64 ]; }; - # Common configuration - common = { + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + cc = icc; + mpi = impi; + gitBranch = "garlic/mpi"; + # FIXME: fix this option in the garlic/mpi git branch + makefileName = "MPI"; + + # Repeat the execution of each unit 30 times + loops = 30; + # Resources - ntasksPerNode = "48"; - nodes = "1"; - - # Stage configuration - enableSbatch = true; - enableControl = true; - enableExtrae = false; - enablePerf = false; - enableCtf = false; - - # MN4 path - nixPrefix = "/gpfs/projects/bsc15/nix"; + qos = "debug"; + ntasksPerNode = 48; + nodes = 1; + time = "02:00:00"; + cpuBind = "sockets,verbose"; + jobName = "hpcg-${toString n}-${gitBranch}"; }; - # Compute the cartesian product of all configurations - configs = map (conf: conf // common) (genConfigs varConfig); - - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" else "${stage}"; - - w = runWrappers; - - sbatch = {stage, conf, ...}: with conf; w.sbatch ( - # Allow a user to define a custom reservation for the job in MareNostrum4, - # by setting the garlic.sbatch.reservation attribute in the - # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no - # reservation is used. The user reservation may be overwritten by the - # experiment, if the reservation is set like with nodes or ntasksPerNode. - optionalAttrs (pkgs.config ? garlic.sbatch.reservation) { - inherit (pkgs.config.garlic.sbatch) reservation; - } // { - program = stageProgram stage; - exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "hpcg"; - inherit nixPrefix nodes ntasksPerNode; - } - ); - - control = {stage, conf, ...}: with conf; w.control { - program = stageProgram stage; + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; }; - srun = {stage, conf, ...}: with conf; w.srun { - program = stageProgram stage; - srunOptions = "--cpu-bind=verbose,rank"; - inherit nixPrefix; - }; - - statspy = {stage, conf, ...}: with conf; w.statspy { - program = stageProgram stage; - }; - - perf = {stage, conf, ...}: with conf; w.perf { - program = stageProgram stage; - perfArgs = "sched record -a"; - }; - - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { - program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; - }; - - extrae = {stage, conf, ...}: w.extrae { - program = stageProgram stage; - traceLib = "mpi"; # mpi -> libtracempi.so - configFile = ./extrae.xml; - }; - - ctf = {stage, conf, ...}: w.argv { - program = stageProgram stage; - env = '' - export NANOS6=ctf - export NANOS6_CTF2PRV=0 - ''; - }; - - argv = {stage, conf, ...}: with conf; w.argv { - program = stageProgram stage; - argv = ''( - --nx=${toString n} - --ny=${toString n} - --nz=${toString n} - )''; - }; - - bscOverlay = import ../../../overlay.nix; - - genPkgs = newOverlay: nixpkgs { - overlays = [ - bscOverlay - newOverlay + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + argv = [ + "--nx=${toString n}" + "--ny=${toString n}" + "--nz=${toString n}" ]; }; - # We may be able to use overlays by invoking the fix function directly, but we - # have to get the definition of the bsc packages and the garlic ones as - # overlays. - - hpcgFn = {stage, conf, ...}: with conf; - let - # We set the mpi implementation to the one specified in the conf, so all - # packages in bsc will use that one. - customPkgs = genPkgs (self: super: { - bsc = super.bsc // { mpi = conf.mpi; }; - }); - in - customPkgs.bsc.garlic.hpcg.override { + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { inherit cc mpi gitBranch makefileName; }; - stages = with common; [] - # Use sbatch to request resources first - ++ optional enableSbatch sbatch - - # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] - - # Executes srun to launch the program in the requested nodes, and - # immediately after enters the nix environment again, as slurmstepd launches - # the next stages from outside the namespace. - ++ [ srun nixsetup ] - - # Intrumentation with extrae - ++ optional enableExtrae extrae - - # Optionally profile the next stages with perf - ++ optional enablePerf perf - - # Optionally profile nanos6 with the new ctf - ++ optional enableCtf ctf - - # Execute the hpcg app with the argv and env vars - ++ [ argv hpcgFn ]; - - # List of actual programs to be executed - jobs = map (conf: w.stagen { inherit conf stages; }) configs; + pipeline = stdexp.stdPipeline ++ [ exec program ]; in - # We simply run each program one after another - w.launch jobs + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix index 79e8c21..0d5d855 100644 --- a/garlic/exp/hpcg/serial.nix +++ b/garlic/exp/hpcg/serial.nix @@ -1,167 +1,63 @@ { stdenv -, nixpkgs -, pkgs -, genApp -, genConfigs -, runWrappers +, stdexp +, bsc +, targetMachine +, stages }: with stdenv.lib; let - bsc = pkgs.bsc; - - # Set variable configuration for the experiment - varConfig = { - cc = [ bsc.icc ]; - gitBranch = [ "garlic/seq" ]; + # Initial variable configuration + varConf = with bsc; { n = [ 104 64 ]; }; - # Common configuration - common = { + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + cc = icc; + mpi = impi; + gitBranch = "garlic/seq"; + + # Repeat the execution of each unit 30 times + loops = 30; + # Resources - ntasksPerNode = "48"; - nodes = "1"; - - # Stage configuration - enableSbatch = true; - enableControl = true; - enableExtrae = false; - enablePerf = false; - enableCtf = false; - - # MN4 path - nixPrefix = "/gpfs/projects/bsc15/nix"; + qos = "debug"; + ntasksPerNode = 48; + nodes = 1; + time = "02:00:00"; + cpuBind = "sockets,verbose"; + jobName = "hpcg-${toString n}-${gitBranch}"; }; - # Compute the cartesian product of all configurations - configs = map (conf: conf // common) (genConfigs varConfig); - - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" else "${stage}"; - - w = runWrappers; - - sbatch = {stage, conf, ...}: with conf; w.sbatch ( - # Allow a user to define a custom reservation for the job in MareNostrum4, - # by setting the garlic.sbatch.reservation attribute in the - # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no - # reservation is used. The user reservation may be overwritten by the - # experiment, if the reservation is set like with nodes or ntasksPerNode. - optionalAttrs (pkgs.config ? garlic.sbatch.reservation) { - inherit (pkgs.config.garlic.sbatch) reservation; - } // { - program = stageProgram stage; - exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "hpcg"; - inherit nixPrefix nodes ntasksPerNode; - } - ); - - control = {stage, conf, ...}: with conf; w.control { - program = stageProgram stage; + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; }; - srun = {stage, conf, ...}: with conf; w.srun { - program = stageProgram stage; - srunOptions = "--cpu-bind=verbose,rank"; - inherit nixPrefix; - }; - - statspy = {stage, conf, ...}: with conf; w.statspy { - program = stageProgram stage; - }; - - perf = {stage, conf, ...}: with conf; w.perf { - program = stageProgram stage; - perfArgs = "sched record -a"; - }; - - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { - program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; - }; - - extrae = {stage, conf, ...}: w.extrae { - program = stageProgram stage; - traceLib = "mpi"; # mpi -> libtracempi.so - configFile = ./extrae.xml; - }; - - ctf = {stage, conf, ...}: w.argv { - program = stageProgram stage; - env = '' - export NANOS6=ctf - export NANOS6_CTF2PRV=0 - ''; - }; - - argv = {stage, conf, ...}: with conf; w.argv { - program = stageProgram stage; - argv = ''( - --nx=${toString n} - --ny=${toString n} - --nz=${toString n} - )''; - }; - - bscOverlay = import ../../../overlay.nix; - - genPkgs = newOverlay: nixpkgs { - overlays = [ - bscOverlay - newOverlay + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + argv = [ + "--nx=${toString n}" + "--ny=${toString n}" + "--nz=${toString n}" ]; }; - # We may be able to use overlays by invoking the fix function directly, but we - # have to get the definition of the bsc packages and the garlic ones as - # overlays. - - hpcgFn = {stage, conf, ...}: with conf; - let - # We set the mpi implementation to the one specified in the conf, so all - # packages in bsc will use that one. - customPkgs = genPkgs (self: super: { - bsc = super.bsc // { mpi = conf.mpi; }; - }); - in - customPkgs.bsc.garlic.hpcg.override { + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { inherit cc gitBranch; }; - stages = with common; [] - # Use sbatch to request resources first - ++ optional enableSbatch sbatch - - # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] - - # Executes srun to launch the program in the requested nodes, and - # immediately after enters the nix environment again, as slurmstepd launches - # the next stages from outside the namespace. - ++ [ srun nixsetup ] - - # Intrumentation with extrae - ++ optional enableExtrae extrae - - # Optionally profile the next stages with perf - ++ optional enablePerf perf - - # Optionally profile nanos6 with the new ctf - ++ optional enableCtf ctf - - # Execute the hpcg app with the argv and env vars - ++ [ argv hpcgFn ]; - - # List of actual programs to be executed - jobs = map (conf: w.stagen { inherit conf stages; }) configs; + pipeline = stdexp.stdPipeline ++ [ exec program ]; in - # We simply run each program one after another - w.launch jobs + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/tampi.nix b/garlic/exp/hpcg/tampi.nix deleted file mode 100644 index d2a0394..0000000 --- a/garlic/exp/hpcg/tampi.nix +++ /dev/null @@ -1,171 +0,0 @@ -{ - stdenv -, nixpkgs -, pkgs -, genApp -, genConfigs -, runWrappers -}: - -with stdenv.lib; - -let - bsc = pkgs.bsc; - - # Set variable configuration for the experiment - varConfig = { - cc = [ bsc.icc ]; - mpi = [ bsc.impi ]; - #gitBranch = [ "garlic/tampi+send+oss+task" ]; - n = [ 104 64 ]; - nodes = [1 2 4]; - }; - - # Common configuration - common = { - # Compile time nbody config - gitBranch = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete"; - - # Resources - ntasksPerNode = "48"; - - # Stage configuration - enableSbatch = true; - enableControl = true; - enableExtrae = false; - enablePerf = false; - enableCtf = false; - - # MN4 path - nixPrefix = "/gpfs/projects/bsc15/nix"; - }; - - # Compute the cartesian product of all configurations - configs = map (conf: conf // common) (genConfigs varConfig); - - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" else "${stage}"; - - w = runWrappers; - - sbatch = {stage, conf, ...}: with conf; w.sbatch ( - # Allow a user to define a custom reservation for the job in MareNostrum4, - # by setting the garlic.sbatch.reservation attribute in the - # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no - # reservation is used. The user reservation may be overwritten by the - # experiment, if the reservation is set like with nodes or ntasksPerNode. - optionalAttrs (pkgs.config ? garlic.sbatch.reservation) { - inherit (pkgs.config.garlic.sbatch) reservation; - } // { - program = stageProgram stage; - exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "nbody-tampi"; - inherit nixPrefix nodes ntasksPerNode; - } - ); - - control = {stage, conf, ...}: with conf; w.control { - program = stageProgram stage; - }; - - srun = {stage, conf, ...}: with conf; w.srun { - program = stageProgram stage; - srunOptions = "--cpu-bind=verbose,rank"; - inherit nixPrefix; - }; - - statspy = {stage, conf, ...}: with conf; w.statspy { - program = stageProgram stage; - }; - - perf = {stage, conf, ...}: with conf; w.perf { - program = stageProgram stage; - perfArgs = "sched record -a"; - }; - - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { - program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; - }; - - extrae = {stage, conf, ...}: w.extrae { - program = stageProgram stage; - traceLib = "mpi"; # mpi -> libtracempi.so - configFile = ./extrae.xml; - }; - - ctf = {stage, conf, ...}: w.argv { - program = stageProgram stage; - env = '' - export NANOS6=ctf - export NANOS6_CTF2PRV=0 - ''; - }; - - argv = {stage, conf, ...}: with conf; w.argv { - program = stageProgram stage; - argv = ''( - --nx=${toString n} - --ny=${toString n} - --nz=${toString n} - )''; - }; - - bscOverlay = import ../../../overlay.nix; - - genPkgs = newOverlay: nixpkgs { - overlays = [ - bscOverlay - newOverlay - ]; - }; - - # We may be able to use overlays by invoking the fix function directly, but we - # have to get the definition of the bsc packages and the garlic ones as - # overlays. - - hpcgFn = {stage, conf, ...}: with conf; - let - # We set the mpi implementation to the one specified in the conf, so all - # packages in bsc will use that one. - customPkgs = genPkgs (self: super: { - bsc = super.bsc // { mpi = conf.mpi; }; - }); - in - customPkgs.bsc.garlic.hpcg.override { - inherit cc mpi gitBranch; - }; - - stages = with common; [] - # Use sbatch to request resources first - ++ optional enableSbatch sbatch - - # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] - - # Executes srun to launch the program in the requested nodes, and - # immediately after enters the nix environment again, as slurmstepd launches - # the next stages from outside the namespace. - ++ [ srun nixsetup ] - - # Intrumentation with extrae - ++ optional enableExtrae extrae - - # Optionally profile the next stages with perf - ++ optional enablePerf perf - - # Optionally profile nanos6 with the new ctf - ++ optional enableCtf ctf - - # Execute the nbody app with the argv and env vars - ++ [ argv hpcgFn ]; - - # List of actual programs to be executed - jobs = map (conf: w.stagen { inherit conf stages; }) configs; - -in - # We simply run each program one after another - w.launch jobs diff --git a/overlay.nix b/overlay.nix index 8143eb4..3ce3368 100644 --- a/overlay.nix +++ b/overlay.nix @@ -197,7 +197,7 @@ let gitBranch = "garlic/mpi+send+seq"; }; - hpcg = callPackage ./garlic/hpcg { + hpcg = callPackage ./garlic/apps/hpcg/default.nix { cc = self.bsc.icc; gitBranch = "garlic/seq"; };