creams: add more nodes for granularity experiments

garlic/exp/creams/gran+node1.nix

@@ -21,8 +21,6 @@ let
       { nodes=1 ; nprocz=2 ; granul= 9;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul= 5;  time= "02:00:00"; }
       { nodes=1 ; nprocz=2 ; granul= 4;  time= "02:00:00"; }
-      { nodes=1 ; nprocz=2 ; granul= 2;  time= "02:00:00"; }
-      { nodes=1 ; nprocz=2 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
@@ -81,6 +79,19 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
     '';
   };
 
@@ -98,7 +109,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  

garlic/exp/creams/gran+node16.nix

@@ -12,23 +12,23 @@ let
   # Initial variable configuration
   varConf = {
     input = [
-#      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul=128; time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=64;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=32;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul=16;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 9;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 5;  time= "02:00:00"; }
       { nodes=16 ; nprocz=32 ; granul= 4;  time= "02:00:00"; }
-#      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 2;  time= "02:00:00"; }
-#      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
+      { nodes=16 ; nprocz=32 ; granul= 1;  time= "02:00:00"; }
     ];
 
     gitBranch = [
-#      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+fork"
-#      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+omp+task"
-#      "garlic/mpi+send+oss+task"
+      "garlic/mpi+send+oss+task"
-#      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+omp+task"
-#      "garlic/mpi+isend+oss+task"
+      "garlic/mpi+isend+oss+task"
       "garlic/tampi+isend+oss+task"
     ];
   };

garlic/exp/creams/gran+node2.nix

@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=2 ; nprocz=4 ; granul=256; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=128; time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=64;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=37;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=32;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul=16;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 9;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 5;  time= "02:00:00"; }
+      { nodes=2 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node2";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/creams/gran+node4.nix

@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=4 ; nprocz=8 ; granul=64;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=37;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=32;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul=16;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 9;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 5;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=4 ; granul= 4;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 2;  time= "02:00:00"; }
+      { nodes=4 ; nprocz=8 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node4";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/creams/gran+node8.nix

@@ -0,0 +1,116 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    input = [
+      { nodes=8 ; nprocz=16 ; granul=128; time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=64;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=32;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul=16;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 9;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 5;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 4;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 2;  time= "02:00:00"; }
+      { nodes=8 ; nprocz=16 ; granul= 1;  time= "02:00:00"; }
+    ];
+
+    gitBranch = [
+      "garlic/mpi+send+omp+fork"
+      "garlic/mpi+send+omp+task"
+      "garlic/mpi+send+oss+task"
+      "garlic/mpi+isend+omp+task"
+      "garlic/mpi+isend+oss+task"
+      "garlic/tampi+isend+oss+task"
+    ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "creams-gran-node8";
+    inherit (targetMachine.config) hw;
+    # Options for creams
+    cc = icc;
+    mpi = impi;
+    inherit (c.input) granul time nodes;
+    inherit (c) gitBranch;
+    unitName = "${expName}-${toString nodes}-${gitBranch}";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    cpusPerTask = hw.cpusPerSocket;
+    jobName = unitName;
+    
+    nprocz = ntasksPerNode * nodes;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  # Custom srun stage to copy the creams input dataset
+  customSrun = {nextStage, conf, ...}:
+    let
+      input = bsc.garlic.apps.creamsInput.override {
+        inherit (conf) gitBranch granul nprocz;
+      };
+    in
+      stages.srun {
+        # These are part of the stdndard srun stage:
+        inherit (conf) nixPrefix;
+        inherit nextStage;
+        cpuBind = "cores,verbose";
+
+        # Now we add some commands to execute before calling srun. These will
+        # only run in one rank (the first in the list of allocated nodes)
+        preSrun = ''
+          cp -r ${input}/SodTubeBenchmark/* .
+          chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
+    '';
+  };
+
+  # Creams program
+  creams = {nextStage, conf, ...}: with conf;
+    let
+      customPkgs = stdexp.replaceMpi conf.mpi;
+    in
+      customPkgs.apps.creams.override {
+        inherit cc mpi gitBranch;
+      };
+
+  pipeline = stdexp.stdPipelineOverride {
+    overrides = {
+      # Replace the stdandard srun stage with our own
+      srun = customSrun;
+    };
+  } ++ [ exec creams ];
+
+in
+ 
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/creams/ss+hybrid.nix

@@ -75,6 +75,19 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
     '';
   };
 
@@ -92,7 +105,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  

garlic/exp/creams/ss+pure.nix

@@ -70,6 +70,19 @@ let
         preSrun = ''
           cp -r ${input}/SodTubeBenchmark/* .
           chmod +w -R .
+          rm -f nanos6.toml
+        '';
+      };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions"
+    '';
+
+    # Remove restarts as is not needed and is huge
+    post = ''
+      rm -rf restarts || true
     '';
   };
 
@@ -87,7 +100,7 @@ let
       # Replace the stdandard srun stage with our own
       srun = customSrun;
     };
-  } ++ [ creams ];
+  } ++ [ exec creams ];
 
 in
  

garlic/exp/index.nix

@@ -42,8 +42,11 @@
       hybrid = callPackage ./creams/ss+hybrid.nix { };
     };
     gran = {
-      pure = callPackage ./creams/gran+node1.nix { };
+      node1 = callPackage ./creams/gran+node1.nix { };
-      hybrid = callPackage ./creams/gran+node16.nix { };
+      node2 = callPackage ./creams/gran+node2.nix { };
+      node4 = callPackage ./creams/gran+node4.nix { };
+      node8 = callPackage ./creams/gran+node8.nix { };
+      node16 = callPackage ./creams/gran+node16.nix { };
     };
   };
 

garlic/fig/index.nix

@@ -53,7 +53,7 @@ in
 
   creams = with exp.creams; {
     ss = stdPlot ./creams/ss.R [ ss.hybrid ss.pure ];
-    gran = stdPlot ./creams/gran.R [ gran.hybrid ];
+    gran = stdPlot ./creams/gran.R [ gran.node1 gran.node2 gran.node4 gran.node8 gran.node16 ];
   };
 
   osu = with exp.osu; {