From a55019c6ef4d5b1acb5d70f85e203604197cd86c Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Fri, 5 Mar 2021 14:46:48 +0100 Subject: [PATCH] creams: add more nodes for granularity experiments --- garlic/exp/creams/gran+node1.nix | 17 ++++- garlic/exp/creams/gran+node16.nix | 16 ++--- garlic/exp/creams/gran+node2.nix | 116 ++++++++++++++++++++++++++++++ garlic/exp/creams/gran+node4.nix | 116 ++++++++++++++++++++++++++++++ garlic/exp/creams/gran+node8.nix | 116 ++++++++++++++++++++++++++++++ garlic/exp/creams/ss+hybrid.nix | 15 +++- garlic/exp/creams/ss+pure.nix | 15 +++- garlic/exp/index.nix | 7 +- garlic/fig/index.nix | 2 +- 9 files changed, 404 insertions(+), 16 deletions(-) create mode 100644 garlic/exp/creams/gran+node2.nix create mode 100644 garlic/exp/creams/gran+node4.nix create mode 100644 garlic/exp/creams/gran+node8.nix diff --git a/garlic/exp/creams/gran+node1.nix b/garlic/exp/creams/gran+node1.nix index 35a2c1c..5bd2726 100644 --- a/garlic/exp/creams/gran+node1.nix +++ b/garlic/exp/creams/gran+node1.nix @@ -21,8 +21,6 @@ let { nodes=1 ; nprocz=2 ; granul= 9; time= "02:00:00"; } { nodes=1 ; nprocz=2 ; granul= 5; time= "02:00:00"; } { nodes=1 ; nprocz=2 ; granul= 4; time= "02:00:00"; } - { nodes=1 ; nprocz=2 ; granul= 2; time= "02:00:00"; } - { nodes=1 ; nprocz=2 ; granul= 1; time= "02:00:00"; } ]; gitBranch = [ @@ -81,9 +79,22 @@ let preSrun = '' cp -r ${input}/SodTubeBenchmark/* . chmod +w -R . + rm -f nanos6.toml ''; }; + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + # Creams program creams = {nextStage, conf, ...}: with conf; let @@ -98,7 +109,7 @@ let # Replace the stdandard srun stage with our own srun = customSrun; }; - } ++ [ creams ]; + } ++ [ exec creams ]; in diff --git a/garlic/exp/creams/gran+node16.nix b/garlic/exp/creams/gran+node16.nix index 4818dc8..7322175 100644 --- a/garlic/exp/creams/gran+node16.nix +++ b/garlic/exp/creams/gran+node16.nix @@ -12,23 +12,23 @@ let # Initial variable configuration varConf = { input = [ -# { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; } + { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul=64; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul=32; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul=16; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul= 9; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul= 5; time= "02:00:00"; } { nodes=16 ; nprocz=32 ; granul= 4; time= "02:00:00"; } -# { nodes=16 ; nprocz=32 ; granul= 2; time= "02:00:00"; } -# { nodes=16 ; nprocz=32 ; granul= 1; time= "02:00:00"; } + { nodes=16 ; nprocz=32 ; granul= 2; time= "02:00:00"; } + { nodes=16 ; nprocz=32 ; granul= 1; time= "02:00:00"; } ]; gitBranch = [ -# "garlic/mpi+send+omp+fork" -# "garlic/mpi+send+omp+task" -# "garlic/mpi+send+oss+task" -# "garlic/mpi+isend+omp+task" -# "garlic/mpi+isend+oss+task" + "garlic/mpi+send+omp+fork" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + "garlic/mpi+isend+omp+task" + "garlic/mpi+isend+oss+task" "garlic/tampi+isend+oss+task" ]; }; diff --git a/garlic/exp/creams/gran+node2.nix b/garlic/exp/creams/gran+node2.nix new file mode 100644 index 0000000..0e022f0 --- /dev/null +++ b/garlic/exp/creams/gran+node2.nix @@ -0,0 +1,116 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = { + input = [ + { nodes=2 ; nprocz=4 ; granul=256; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul=128; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul=64; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul=37; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul=32; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul=16; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul= 9; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul= 5; time= "02:00:00"; } + { nodes=2 ; nprocz=4 ; granul= 4; time= "02:00:00"; } + ]; + + gitBranch = [ + "garlic/mpi+send+omp+fork" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + "garlic/mpi+isend+omp+task" + "garlic/mpi+isend+oss+task" + "garlic/tampi+isend+oss+task" + ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "creams-gran-node2"; + inherit (targetMachine.config) hw; + # Options for creams + cc = icc; + mpi = impi; + inherit (c.input) granul time nodes; + inherit (c) gitBranch; + unitName = "${expName}-${toString nodes}-${gitBranch}"; + + # Repeat the execution of each unit 10 times + loops = 10; + + # Resources + qos = "debug"; + ntasksPerNode = hw.socketsPerNode; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; + + nprocz = ntasksPerNode * nodes; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + # Custom srun stage to copy the creams input dataset + customSrun = {nextStage, conf, ...}: + let + input = bsc.garlic.apps.creamsInput.override { + inherit (conf) gitBranch granul nprocz; + }; + in + stages.srun { + # These are part of the stdndard srun stage: + inherit (conf) nixPrefix; + inherit nextStage; + cpuBind = "cores,verbose"; + + # Now we add some commands to execute before calling srun. These will + # only run in one rank (the first in the list of allocated nodes) + preSrun = '' + cp -r ${input}/SodTubeBenchmark/* . + chmod +w -R . + rm -f nanos6.toml + ''; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + + # Creams program + creams = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.creams.override { + inherit cc mpi gitBranch; + }; + + pipeline = stdexp.stdPipelineOverride { + overrides = { + # Replace the stdandard srun stage with our own + srun = customSrun; + }; + } ++ [ exec creams ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/gran+node4.nix b/garlic/exp/creams/gran+node4.nix new file mode 100644 index 0000000..628ae1d --- /dev/null +++ b/garlic/exp/creams/gran+node4.nix @@ -0,0 +1,116 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = { + input = [ + { nodes=4 ; nprocz=8 ; granul=64; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul=37; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul=32; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul=16; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul= 9; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul= 5; time= "02:00:00"; } + { nodes=4 ; nprocz=4 ; granul= 4; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul= 2; time= "02:00:00"; } + { nodes=4 ; nprocz=8 ; granul= 1; time= "02:00:00"; } + ]; + + gitBranch = [ + "garlic/mpi+send+omp+fork" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + "garlic/mpi+isend+omp+task" + "garlic/mpi+isend+oss+task" + "garlic/tampi+isend+oss+task" + ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "creams-gran-node4"; + inherit (targetMachine.config) hw; + # Options for creams + cc = icc; + mpi = impi; + inherit (c.input) granul time nodes; + inherit (c) gitBranch; + unitName = "${expName}-${toString nodes}-${gitBranch}"; + + # Repeat the execution of each unit 10 times + loops = 10; + + # Resources + qos = "debug"; + ntasksPerNode = hw.socketsPerNode; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; + + nprocz = ntasksPerNode * nodes; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + # Custom srun stage to copy the creams input dataset + customSrun = {nextStage, conf, ...}: + let + input = bsc.garlic.apps.creamsInput.override { + inherit (conf) gitBranch granul nprocz; + }; + in + stages.srun { + # These are part of the stdndard srun stage: + inherit (conf) nixPrefix; + inherit nextStage; + cpuBind = "cores,verbose"; + + # Now we add some commands to execute before calling srun. These will + # only run in one rank (the first in the list of allocated nodes) + preSrun = '' + cp -r ${input}/SodTubeBenchmark/* . + chmod +w -R . + rm -f nanos6.toml + ''; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + + # Creams program + creams = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.creams.override { + inherit cc mpi gitBranch; + }; + + pipeline = stdexp.stdPipelineOverride { + overrides = { + # Replace the stdandard srun stage with our own + srun = customSrun; + }; + } ++ [ exec creams ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/gran+node8.nix b/garlic/exp/creams/gran+node8.nix new file mode 100644 index 0000000..d0fec23 --- /dev/null +++ b/garlic/exp/creams/gran+node8.nix @@ -0,0 +1,116 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = { + input = [ + { nodes=8 ; nprocz=16 ; granul=128; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul=64; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul=32; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul=16; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul= 9; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul= 5; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul= 4; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul= 2; time= "02:00:00"; } + { nodes=8 ; nprocz=16 ; granul= 1; time= "02:00:00"; } + ]; + + gitBranch = [ + "garlic/mpi+send+omp+fork" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + "garlic/mpi+isend+omp+task" + "garlic/mpi+isend+oss+task" + "garlic/tampi+isend+oss+task" + ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "creams-gran-node8"; + inherit (targetMachine.config) hw; + # Options for creams + cc = icc; + mpi = impi; + inherit (c.input) granul time nodes; + inherit (c) gitBranch; + unitName = "${expName}-${toString nodes}-${gitBranch}"; + + # Repeat the execution of each unit 10 times + loops = 10; + + # Resources + qos = "debug"; + ntasksPerNode = hw.socketsPerNode; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; + + nprocz = ntasksPerNode * nodes; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + # Custom srun stage to copy the creams input dataset + customSrun = {nextStage, conf, ...}: + let + input = bsc.garlic.apps.creamsInput.override { + inherit (conf) gitBranch granul nprocz; + }; + in + stages.srun { + # These are part of the stdndard srun stage: + inherit (conf) nixPrefix; + inherit nextStage; + cpuBind = "cores,verbose"; + + # Now we add some commands to execute before calling srun. These will + # only run in one rank (the first in the list of allocated nodes) + preSrun = '' + cp -r ${input}/SodTubeBenchmark/* . + chmod +w -R . + rm -f nanos6.toml + ''; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + + # Creams program + creams = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.creams.override { + inherit cc mpi gitBranch; + }; + + pipeline = stdexp.stdPipelineOverride { + overrides = { + # Replace the stdandard srun stage with our own + srun = customSrun; + }; + } ++ [ exec creams ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/ss+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix index 5b20531..c78376b 100644 --- a/garlic/exp/creams/ss+hybrid.nix +++ b/garlic/exp/creams/ss+hybrid.nix @@ -75,9 +75,22 @@ let preSrun = '' cp -r ${input}/SodTubeBenchmark/* . chmod +w -R . + rm -f nanos6.toml ''; }; + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + # Creams program creams = {nextStage, conf, ...}: with conf; let @@ -92,7 +105,7 @@ let # Replace the stdandard srun stage with our own srun = customSrun; }; - } ++ [ creams ]; + } ++ [ exec creams ]; in diff --git a/garlic/exp/creams/ss+pure.nix b/garlic/exp/creams/ss+pure.nix index 7585576..5d36e5e 100644 --- a/garlic/exp/creams/ss+pure.nix +++ b/garlic/exp/creams/ss+pure.nix @@ -70,9 +70,22 @@ let preSrun = '' cp -r ${input}/SodTubeBenchmark/* . chmod +w -R . + rm -f nanos6.toml ''; }; + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + # Creams program creams = {nextStage, conf, ...}: with conf; let @@ -87,7 +100,7 @@ let # Replace the stdandard srun stage with our own srun = customSrun; }; - } ++ [ creams ]; + } ++ [ exec creams ]; in diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 6b93c92..ec90363 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -42,8 +42,11 @@ hybrid = callPackage ./creams/ss+hybrid.nix { }; }; gran = { - pure = callPackage ./creams/gran+node1.nix { }; - hybrid = callPackage ./creams/gran+node16.nix { }; + node1 = callPackage ./creams/gran+node1.nix { }; + node2 = callPackage ./creams/gran+node2.nix { }; + node4 = callPackage ./creams/gran+node4.nix { }; + node8 = callPackage ./creams/gran+node8.nix { }; + node16 = callPackage ./creams/gran+node16.nix { }; }; }; diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index c51d328..d2e4d5b 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -53,7 +53,7 @@ in creams = with exp.creams; { ss = stdPlot ./creams/ss.R [ ss.hybrid ss.pure ]; - gran = stdPlot ./creams/gran.R [ gran.hybrid ]; + gran = stdPlot ./creams/gran.R [ gran.node1 gran.node2 gran.node4 gran.node8 gran.node16 ]; }; osu = with exp.osu; {