saiph: simplify granularity and ss experiments

garlic/exp/index.nix

@@ -32,8 +32,8 @@
   };
 
   saiph = {
-    granularity-saiph = callPackage ./saiph/granularity-saiph.nix { };
+    granularity = callPackage ./saiph/granularity.nix { };
-    scalability-saiph = callPackage ./saiph/scalability-saiph.nix { };
+    ss = callPackage ./saiph/ss.nix { };
   };
 
   creams = rec {

garlic/exp/saiph/granularity.nix

@@ -1,4 +1,4 @@
-######################################################################################
+#######################################################################
 # Saiph, granularity experiment:
 #
 # App:Heat 3D - garlic/tampi+isend+oss+task+simd branch
@@ -15,12 +15,13 @@
 # Granularity experiment configuration:
 #   Single-core run
 #   MPI binded to sockets: MPI procs = 2
-#   Mesh distributed across third dimension to ensure contiguous communications
+#   Mesh distributed across third dimension to ensure contiguous
+#   communications
 #     --> nbgx = 1, nbgy = 1
 #   First dimension cannot be locally blocked (simd reasons)
 #   Second and third dimension local blocking limited by local mesh size 
 # 
-######################################################################################
+#######################################################################
 
 # Common packages, tools and options
 { 
@@ -29,81 +30,102 @@
 , bsc
 , targetMachine
 , stages
+, garlicTools
 }:
 
 with stdenv.lib;
+with garlicTools;
 
 let
 
-  #*** Variable configurations ***
+  # Variable configurations
-  varConf = with bsc; {
+  varConf = with targetMachine.config; {
     # Local blocks per dimension
-    nbl1 = [ 1 2 3 4 6 12 24 48 96 ];
+    nblx = [ 1 ]; # SIMD
-    nbl2 = [ 1 2 3 4 6 12 24 48 96 ];
+    nbly = range2 1 (hw.cpusPerNode * 8);
+    nblz = [ 8 ];
+    sizex = [ 3 ];
+    gitBranch = [ "garlic/tampi+isend+oss+task+simd" ];
   };
 
-  #*** Generate the complete configuration for each unit ***
+  # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
+  genConf = c: targetMachine.config // rec {
 
     # Experiment, units and job names 
     expName = "saiph-granularity";
-    unitName = "${expName}-N${toString nodes}" + "nbg_${toString nbgx}-${toString nbgy}-${toString nbgz}" + "nbl_1-${toString nbly}-${toString nblz}";
+    unitName = "${expName}"
-    jobName = "${unitName}";
+      + "-N${toString nodes}"
+      + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}"
+      + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}";
+
+    jobName = unitName;
 
     # saiph options
+    totalTasks = ntasksPerNode * nodes;
     nodes = 1;
     enableManualDist = true; # allows to manually set nbg{x-y-z}
     nbgx = 1;
     nbgy = 1;
-    nbgz = nodes*2;    # forcing distribution by last dim
+    nbgz = totalTasks; # forcing distribution by last dim
-    nblx = 1;          # simd reasons
-    nbly = c.nbl1;     # takes values from varConf
-    nblz = c.nbl2;     # takes values from varConf
-    mpi = impi;
-    gitBranch = "garlic/tampi+isend+oss+task+simd";
-    gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6";  # fix a specific commit
-    rev = "0"; 
 
-    # Repeat the execution of each unit 30 times
+    inherit (c) nblx nbly nblz gitBranch sizex;
-    loops = 30;
+
+    blocksPerTask = nblx * nbly * nblz * 1.0;
+    blocksPerCpu = blocksPerTask / cpusPerTask;
+
+    # fix a specific commit
+    gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6";
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     inherit (targetMachine.config) hw;
     qos = "debug";
-    ntasksPerNode = hw.socketsPerNode;   # MPI binded to sockets
+    ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets
-    cpusPerTask = hw.cpusPerSocket;      # Using the 24 CPUs of each socket
+    cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket
   };
 
   #*** Compute the final set of configurations ***
-  # Compute the array of configurations: cartesian product of all factors
+  # Compute the array of configurations: cartesian product of all
+  # factors
   allConfigs = stdexp.buildConfigs {
     inherit varConf genConf;
   };
+
   # Filter to remove non-desired configurations:
   #   --> tasks/proc < 0.5
   #   --> nblz > 50
-  configs = filter (el: if ((builtins.mul el.nbly el.nblz) < (builtins.mul 0.5 el.cpusPerTask) || el.nblz > 50) then false else true) allConfigs;
+  isGoodConfig = c:
+  let
+    maxNblz = c.cpusPerTask * 2;
+  in
+    ! (c.blocksPerCpu < 0.5 || c.nblz > maxNblz);
+
+  configs = filter (isGoodConfig) allConfigs;
 
   #*** Sets the env/argv of the program ***
-  exec = {nextStage, conf, ...}: with conf; stages.exec {
+  exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
     env = ''
-      export OMP_NUM_THREADS=${toString hw.cpusPerSocket}
+      export OMP_NUM_THREADS=${toString conf.cpusPerTask}
     '';
   };
 
   #*** Configure the program according to the app ***
-  program = {nextStage, conf, ...}:
+  program = {nextStage, conf, ...}: bsc.apps.saiph.override {
-  let
+    inherit (conf) enableManualDist
-    customPkgs = stdexp.replaceMpi conf.mpi;
+      nbgx nbgy nbgz nblx nbly nblz
-  in
+      sizex
-    customPkgs.apps.saiph.override {
+      gitBranch gitCommit;
-      inherit (conf) enableManualDist nbgx nbgy nbgz nblx nbly nblz mpi gitBranch gitCommit;
+
-    };
+    L3SizeKB = conf.hw.cacheSizeKB.L3;
+    cachelineBytes = conf.hw.cachelineBytes;
+  };
 
   #*** Add stages to the pipeline ***
   pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
-	stdexp.genExperiment { inherit configs pipeline; }
 
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/saiph/ss.nix

@@ -31,62 +31,68 @@
 , bsc
 , targetMachine
 , stages
+, garlicTools
 }:
 
 with stdenv.lib;
+with garlicTools;
 
 let
 
   #*** Variable configurations ***
-  varConf = with bsc; {
+  varConf = with targetMachine.config; {
+    # FIXME: None of those selected nbl* and problem size is able to give good
+    # efficiency when testing strong scaling. We should find better values.
     # Local blocks per dimension
-    nbl1 = [ 1 2 3 4 6 12 24 48 96 ];
+    nblx = [ 1 ]; # SIMD
-    nbl2 = [ 1 2 3 4 6 12 24 48 96 ];
+    nbly = [ 32 ];
-    # Number of nodes
+    nblz = [ 8 ];
-    nodes = [ 1 2 4 8 ];
+    sizex = [ 3 6 ];
+    gitBranch = [ "garlic/tampi+isend+oss+task+simd" ];
+    nodes = range2 1 8;
   };
 
   #*** Generate the complete configuration for each unit ***
-  genConf = with bsc; c: targetMachine.config // rec {
+  genConf = c: targetMachine.config // rec {
 
     # Experiment, units and job names 
-    expName = "saiph-scalability";
+    expName = "saiph-ss";
-    unitName = "${expName}-N${toString nodes}" + "nbg_${toString nbgx}-${toString nbgy}-${toString nbgz}" + "nbl_1-${toString nbly}-${toString nblz}";
+    unitName = "${expName}"
-    jobName = "${unitName}";
+      + "-N${toString nodes}"
+      + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}"
+      + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}";
+    jobName = unitName;
 
     # saiph options
-    nodes = c.nodes;            # takes values from varConf
     enableManualDist = true;    # allows to manually set nbg{x-y-z}
     nbgx = 1;
     nbgy = 1;
-    nbgz = nodes*2;    # forcing distribution by last dim
+    nbgz = nodes * ntasksPerNode;    # forcing distribution by last dim
-    nblx = 1;          # simd reasons
+
-    nbly = c.nbl1;     # takes values from varConf
+    inherit (c) nblx nbly nblz nodes sizex;
-    nblz = c.nbl2;     # takes values from varConf
+
-    mpi = impi;
     gitBranch = "garlic/tampi+isend+oss+task+simd";
     gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6";  # fix a specific commit 
-    rev = "0";
+
-    # Repeat the execution of each unit 30 times
+    blocksPerTask = nblx * nbly * nblz * 1.0;
-    loops = 30;
+    blocksPerCpu = blocksPerTask / cpusPerTask;
+
+    # Repeat the execution of each unit 10 times
+    loops = 10;
 
     # Resources
     inherit (targetMachine.config) hw;
-    qos = "bsc_cs";
+
+    qos = "debug";
     ntasksPerNode = hw.socketsPerNode;   # MPI binded to sockets
     cpusPerTask = hw.cpusPerSocket;      # Using the 24 CPUs of each socket
   };
 
   #*** Compute the final set of configurations ***
   # Compute the array of configurations: cartesian product of all factors
-  allConfigs = stdexp.buildConfigs {
+  configs = stdexp.buildConfigs {
     inherit varConf genConf;
   };
-  # Filter to remove non-desired configurations:
-  #   --> tasks/proc < 3
-  #   --> nblz > 25
-  configs = filter (el: if ((builtins.mul el.nbly el.nblz) < (builtins.mul 3 el.cpusPerTask) || el.nblz > 25) then false else true) allConfigs;
-
 
   #*** Sets the env/argv of the program ***
   exec = {nextStage, conf, ...}: with conf; stages.exec {
@@ -97,17 +103,19 @@ let
   };
 
   #*** Configure the program according to the app ***
-  program = {nextStage, conf, ...}:
+  program = {nextStage, conf, ...}: bsc.apps.saiph.override {
-  let
+    inherit (conf) enableManualDist
-    customPkgs = stdexp.replaceMpi conf.mpi;
+      nbgx nbgy nbgz nblx nbly nblz
-  in
+      sizex
-    customPkgs.apps.saiph.override {
+      gitBranch gitCommit;
-      inherit (conf) enableManualDist nbgx nbgy nbgz nblx nbly nblz mpi gitBranch gitCommit;
+
-    };
+    L3SizeKB = conf.hw.cacheSizeKB.L3;
+    cachelineBytes = conf.hw.cachelineBytes;
+  };
 
   #*** Add stages to the pipeline ***
   pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
-	stdexp.genExperiment { inherit configs pipeline; }
 
+  stdexp.genExperiment { inherit configs pipeline; }