fwi: merge forkjoin ss experiment into one file

Additional options are only active with enableExtended = true

garlic/exp/fwi/ss.nix

@@ -8,56 +8,54 @@
 , bsc
 , targetMachine
 , stages
+, garlicTools
+, enableExtended ? false
 }:
 
 with stdenv.lib;
+with garlicTools;
 
 let
 
   inherit (targetMachine) fs;
 
+  # We split these into a separate group so we can remove the blocksize
+  # later.
+  forkJoinBranches = [
+       "garlic/mpi+send+omp+fork"
+  ];
+
   # Initial variable configuration
   varConf = {
     gitBranch = [
-       "garlic/tampi+send+oss+task"
+      "garlic/tampi+isend+oss+task"
-       "garlic/tampi+isend+oss+task"
+    ] ++ optionals (enableExtended) ([
-       "garlic/mpi+send+omp+task"
+      "garlic/tampi+send+oss+task"
-       "garlic/mpi+send+oss+task"
+      "garlic/mpi+send+omp+task"
-    ];
+      "garlic/mpi+send+oss+task"
+    ] ++ forkJoinBranches);
 
-    blocksize = [ 1 2 4 8 16 ];
+    blocksize = if (enableExtended)
+      then range2 1 16
+      else [ 2 ];
 
-    n = [
+    n = [ {nx=100; nz=100; ny=8000;} ];
-        {nx=100; nz=100; ny=8000;}
-    ];
-
-    nodes = [ 1 2 4 8 16 ];
 
+    nodes = range2 1 16;
   };
 
-# The c value contains something like:
-# {
-#   n = { nx=500; ny=500; nz=500; }
-#   blocksize = 1;
-#   gitBranch = "garlic/tampi+send+oss+task";
-# }
-
   machineConfig = targetMachine.config;
 
   # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
+  genConf = c: targetMachine.config // rec {
-    expName = "fwi";
+    expName = "fwi-ss";
-    unitName = "${expName}-test";
+    unitName = "${expName}"
+    + "-nodes${toString nodes}"
+    + "-bs${toString blocksize}"
+    + "-${toString gitBranch}";
+
     inherit (machineConfig) hw;
-
-    cc = icc;
     inherit (c) gitBranch blocksize;
-
-    #nx = c.n.nx;
-    #ny = c.n.ny;
-    #nz = c.n.nz;
-
-    # Same but shorter:
     inherit (c.n) nx ny nz;
 
     fwiInput = bsc.apps.fwi.input.override {
@@ -69,53 +67,62 @@ let
 
     # Repeat the execution of each unit several times
     loops = 10;
-    #loops = 1;
 
     # Resources
     cpusPerTask = hw.cpusPerSocket;
-    ntasksPerNode = 2;
+    ntasksPerNode = hw.socketsPerNode;
     nodes = c.nodes;
     qos = "debug";
     time = "02:00:00";
     jobName = unitName;
 
-    tracing = "no";
+    enableCTF = false;
 
     # Enable permissions to write in the local storage
     extraMounts = [ fs.local.temp ];
 
   };
 
+  # Returns true if the given config is in the forkJoinBranches list
+  isForkJoin = c: any (e: c.gitBranch == e) forkJoinBranches;
+
+  # Set the blocksize to null for the fork join branch
+  fixBlocksize = c: if (isForkJoin c) then (c // { blocksize = null; }) else c;
+
   # Compute the array of configurations
-  configs = stdexp.buildConfigs {
+  allConfigs = stdexp.buildConfigs {
     inherit varConf genConf;
   };
 
+  # The unique function ensures that we only run one config for the fork
+  # join branch, even if we have multiple blocksizes.
+  configs = unique (map fixBlocksize allConfigs);
+
   exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
     pre = ''
-      CDIR=$PWD
+      CDIR=$(pwd)
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
-      fi
       EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
-      mkdir -p $EXECDIR
+      mkdir -p "$EXECDIR"
-      cd $EXECDIR
+      cd "$EXECDIR"
       ln -fs ${conf.fwiInput}/InputModels InputModels || true
+    '' + optionalString (conf.enableCTF) ''
+      export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
     '';
     argv = [
       "${conf.fwiInput}/fwi_params.txt"
       "${conf.fwiInput}/fwi_frequencies.txt"
-      conf.blocksize
+    ]
+    ++ optional (isForkJoin conf) conf.blocksize
+    ++ [
       "-1" # Fordward steps
       "-1" # Backward steps
       conf.ioFreq # Write/read frequency
     ];
     post = ''
       rm -rf Results || true
-      if [[ "${conf.tracing}" == "yes" ]]; then
+    '' + optionalString (conf.enableCTF) ''
-          mv trace_* $CDIR
+      mv trace_* "$CDIR"
-      fi
     '';
   };
 
@@ -123,7 +130,7 @@ let
 
   # FWI program
   program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) cc gitBranch fwiInput;
+    inherit (conf) gitBranch fwiInput;
   };
 
   pipeline = stdexp.stdPipeline ++ [ exec program ];

garlic/exp/fwi/strong_scaling_forkjoin.nix

@@ -1,128 +0,0 @@
-# Strong scaling test for FWI variants based on forkjoint. This
-# experiment does not rely on block sizes.
-
-{
-  stdenv
-, stdexp
-, bsc
-, targetMachine
-, stages
-}:
-
-with stdenv.lib;
-
-let
-
-  inherit (targetMachine) fs;
-
-  # Initial variable configuration
-  varConf = {
-    gitBranch = [
-       "garlic/mpi+send+omp+fork"
-    ];
-
-    blocksize = [ 0 ];
-
-    n = [
-        {nx=100; nz=100; ny=8000;}
-    ];
-
-    nodes = [ 1 2 4 8 16 ];
-
-  };
-
-# The c value contains something like:
-# {
-#   n = { nx=500; ny=500; nz=500; }
-#   blocksize = 1;
-#   gitBranch = "garlic/tampi+send+oss+task";
-# }
-
-  machineConfig = targetMachine.config;
-
-  # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
-    expName = "fwi";
-    unitName = "${expName}-test";
-    inherit (machineConfig) hw;
-
-    cc = icc;
-    inherit (c) gitBranch blocksize;
-
-    #nx = c.n.nx;
-    #ny = c.n.ny;
-    #nz = c.n.nz;
-
-    # Same but shorter:
-    inherit (c.n) nx ny nz;
-
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
-    # Other FWI parameters
-    ioFreq = -1;
-
-    # Repeat the execution of each unit several times
-    loops = 10;
-    #loops = 1;
-
-    # Resources
-    cpusPerTask = hw.cpusPerSocket;
-    ntasksPerNode = 2;
-    nodes = c.nodes;
-    qos = "debug";
-    time = "02:00:00";
-    jobName = unitName;
-
-    tracing = "no";
-
-    # Enable permissions to write in the local storage
-    extraMounts = [ fs.local.temp ];
-
-  };
-
-  # Compute the array of configurations
-  configs = stdexp.buildConfigs {
-    inherit varConf genConf;
-  };
-
-  exec = {nextStage, conf, ...}: stages.exec {
-    inherit nextStage;
-    pre = ''
-      CDIR=$PWD
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
-      fi
-      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
-      mkdir -p $EXECDIR
-      cd $EXECDIR
-      ln -fs ${conf.fwiInput}/InputModels InputModels || true
-    '';
-    argv = [
-      "${conf.fwiInput}/fwi_params.txt"
-      "${conf.fwiInput}/fwi_frequencies.txt"
-      "-1" # Fordward steps
-      "-1" # Backward steps
-      conf.ioFreq # Write/read frequency
-    ];
-    post = ''
-      rm -rf Results || true
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          mv trace_* $CDIR
-      fi
-    '';
-  };
-
-  apps = bsc.garlic.apps;
-
-  # FWI program
-  program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) cc gitBranch fwiInput;
-  };
-
-  pipeline = stdexp.stdPipeline ++ [ exec program ];
-
-in
- 
-  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/index.nix

@@ -98,9 +98,9 @@
   };
 
   fwi = {
-    granularity             = callPackage ./fwi/granularity.nix { };
+    granularity = callPackage ./fwi/granularity.nix { };
-    strong_scaling_task     = callPackage ./fwi/strong_scaling_task.nix { };
+    ss = callPackage ./fwi/ss.nix { };
-    strong_scaling_forkjoin = callPackage ./fwi/strong_scaling_forkjoin.nix { };
+
     strong_scaling_mpionly  = callPackage ./fwi/strong_scaling_mpionly.nix { };
     data_reuse              = callPackage ./fwi/data_reuse.nix { };
     strong_scaling_io       = callPackage ./fwi/strong_scaling_io.nix { };