From 788dd13ebd1f8f6757177436c818fbe5796f95dd Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 12 Apr 2021 15:37:39 +0200
Subject: [PATCH] fwi: merge mpi pure experiment

The getResources function is used to assign the proper cpu binding
depending on the version. However, additional contraints are required to
ensure that we have enough points in Y.

By default the mpi+send+seq branch is disabled.
---
 garlic/exp/fwi/common.nix                 |   9 ++
 garlic/exp/fwi/ss.nix                     |  16 +--
 garlic/exp/fwi/strong_scaling_mpionly.nix | 130 ----------------------
 3 files changed, 18 insertions(+), 137 deletions(-)
 delete mode 100644 garlic/exp/fwi/strong_scaling_mpionly.nix

diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix
index ebaf371..b220df2 100644
--- a/garlic/exp/fwi/common.nix
+++ b/garlic/exp/fwi/common.nix
@@ -31,6 +31,15 @@ rec {
     # join branch, even if we have multiple blocksizes.
     unique (map fixBlocksize allConfigs);
 
+  getResources = {gitBranch, hw}:
+  if (gitBranch == "garlic/mpi+send+seq") then {
+    cpusPerTask = hw.cpusPerSocket;
+    ntasksPerNode = hw.socketsPerNode;
+  } else {
+    cpusPerTask = 1;
+    ntasksPerNode = hw.cpusPerNode;
+  };
+
   exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
 
diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix
index 25ff81d..2302d43 100644
--- a/garlic/exp/fwi/ss.nix
+++ b/garlic/exp/fwi/ss.nix
@@ -17,6 +17,8 @@ with stdenv.lib;
 with garlicTools;
 
 let
+  common = callPackage ./common.nix {};
+  inherit (common) getConfigs getResources pipeline;
 
   inherit (targetMachine) fs;
 
@@ -29,13 +31,16 @@ let
       "garlic/mpi+send+omp+task"
       "garlic/mpi+send+oss+task"
       "garlic/mpi+send+omp+fork"
+      # FIXME: the mpi pure version has additional constraints with the
+      # number of planes in Y. By now is disabled.
+      #"garlic/mpi+send+seq"
     ];
 
     blocksize = if (enableExtended)
       then range2 1 16
       else [ 2 ];
 
-    n = [ {nx=100; nz=100; ny=8000;} ];
+    n = [ { nx=100; ny=8000; nz=100; } ];
 
     nodes = range2 1 16;
   };
@@ -65,8 +70,9 @@ let
     loops = 10;
 
     # Resources
-    cpusPerTask = hw.cpusPerSocket;
-    ntasksPerNode = hw.socketsPerNode;
+    inherit (getResources { inherit gitBranch hw; })
+      cpusPerTask ntasksPerNode;
+
     nodes = c.nodes;
     qos = "debug";
     time = "02:00:00";
@@ -79,10 +85,6 @@ let
     tempDir = fs.local.temp;
   };
 
-  common = callPackage ./common.nix {};
-
-  inherit (common) getConfigs pipeline;
-
   configs = getConfigs {
     inherit varConf genConf;
   };
diff --git a/garlic/exp/fwi/strong_scaling_mpionly.nix b/garlic/exp/fwi/strong_scaling_mpionly.nix
deleted file mode 100644
index 94ffd8d..0000000
--- a/garlic/exp/fwi/strong_scaling_mpionly.nix
+++ /dev/null
@@ -1,130 +0,0 @@
-# Strong scaling test for FWI variants based exclusively on MPI. This
-# experiment does not rely on block sizes. An MPI process is instantiated per
-# core.
-
-{
-  stdenv
-, stdexp
-, bsc
-, targetMachine
-, stages
-}:
-
-with stdenv.lib;
-
-let
-
-  inherit (targetMachine) fs;
-
-  # Initial variable configuration
-  varConf = {
-    gitBranch = [
-       "garlic/mpi+send+seq"
-    ];
-
-    blocksize = [ 0 ];
-
-    n = [
-        {nx=100; nz=100; ny=8000;}
-    ];
-
-    # Not enough planes for 4, 8 and 16 nodes
-    nodes = [ 1 2 ];
-
-  };
-
-# The c value contains something like:
-# {
-#   n = { nx=500; ny=500; nz=500; }
-#   blocksize = 1;
-#   gitBranch = "garlic/tampi+send+oss+task";
-# }
-
-  machineConfig = targetMachine.config;
-
-  # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
-    expName = "fwi";
-    unitName = "${expName}-test";
-    inherit (machineConfig) hw;
-
-    cc = icc;
-    inherit (c) gitBranch blocksize;
-
-    #nx = c.n.nx;
-    #ny = c.n.ny;
-    #nz = c.n.nz;
-
-    # Same but shorter:
-    inherit (c.n) nx ny nz;
-
-    fwiInput = bsc.apps.fwi.input.override {
-      inherit (c.n) nx ny nz;
-    };
-
-    # Other FWI parameters
-    ioFreq = -1;
-
-    # Repeat the execution of each unit several times
-    loops = 10;
-    #loops = 1;
-
-    # Resources
-    cpusPerTask = 1;
-    ntasksPerNode = hw.cpusPerNode;
-    nodes = c.nodes;
-    qos = "debug";
-    time = "02:00:00";
-    jobName = unitName;
-
-    tracing = "no";
-
-    # Enable permissions to write in the local storage
-    extraMounts = [ fs.local.temp ];
-
-  };
-
-  # Compute the array of configurations
-  configs = stdexp.buildConfigs {
-    inherit varConf genConf;
-  };
-
-  exec = {nextStage, conf, ...}: stages.exec {
-    inherit nextStage;
-    pre = ''
-      CDIR=$PWD
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf"
-      fi
-      EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"
-      mkdir -p $EXECDIR
-      cd $EXECDIR
-      ln -fs ${conf.fwiInput}/InputModels InputModels || true
-    '';
-    argv = [
-      "${conf.fwiInput}/fwi_params.txt"
-      "${conf.fwiInput}/fwi_frequencies.txt"
-      "-1" # Fordward steps
-      "-1" # Backward steps
-      conf.ioFreq # Write/read frequency
-    ];
-    post = ''
-      rm -rf Results || true
-      if [[ "${conf.tracing}" == "yes" ]]; then
-          mv trace_* $CDIR
-      fi
-    '';
-  };
-
-  apps = bsc.garlic.apps;
-
-  # FWI program
-  program = {nextStage, conf, ...}: apps.fwi.solver.override {
-    inherit (conf) cc gitBranch fwiInput;
-  };
-
-  pipeline = stdexp.stdPipeline ++ [ exec program ];
-
-in
- 
-  stdexp.genExperiment { inherit configs pipeline; }