From a44042615a14436cc0c499f47bfd99393b2ffd8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Thu, 8 Oct 2020 17:42:17 +0200
Subject: [PATCH 01/14] WIP

---
 garlic/apps/hpcg/default.nix | 17 ++++++-------
 overlay.nix                  | 48 ++++++++++++++++++++++--------------
 2 files changed, 36 insertions(+), 29 deletions(-)

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index f311804..8ab7bec 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -2,40 +2,37 @@
   stdenv
 , nanos6
 , mpi
-, mcxx
 , tampi
-, icc
+, cc
+, gitBranch ? "garlic/seq"
+, makefileName ? "Linux_Serial"
 }:
 
 stdenv.mkDerivation rec {
   name = "hpcg";
 
   src = builtins.fetchGit {
-    url = "ssh://git@bscpm02.bsc.es/rpenacob/hpcg.git";
-    ref = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete";
+    url = "ssh://git@bscpm02.bsc.es/rpenacob/garlic-hpcg.git";
+    ref = "${gitBranch}";
   };
 
   prePatch = ''
     #export NIX_DEBUG=6
   '';
 
-  patches = [ ./tampi.patch ];
-
   buildInputs = [
     nanos6
     mpi
-    icc
     tampi
-    mcxx
+    cc
   ];
 
   enableParallelBuilding = true;
 
   configurePhase = ''
-    export TAMPI_HOME=${tampi}
     mkdir build
     cd build
-    ../configure MPI_ICPC_OSS
+    ../configure ${makefileName}
   '';
 
   installPhase = ''
diff --git a/overlay.nix b/overlay.nix
index 69b9a3e..d117f2b 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -172,19 +172,6 @@ let
       # Apps for Garlic
 
       apps = {
-        creams = callPackage ./garlic/apps/creams/default.nix {
-          gnuDef   = self.gfortran10 ; # Default GNU   compiler version
-          intelDef = self.bsc.icc    ; # Default Intel compiler version
-
-          gitBranch = "garlic/mpi+send+seq";
-
-          cc  = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
-          mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
-        };
-
-        creamsInput = callPackage ./garlic/apps/creams/input.nix {
-          gitBranch = "garlic/mpi+send+seq";
-        };
 
         nbody = callPackage ./garlic/apps/nbody/default.nix {
           cc = self.bsc.icc;
@@ -198,6 +185,23 @@ let
           cc = self.bsc.clangOmpss2;
         };
 
+        creams = callPackage ./garlic/apps/creams/default.nix {
+          gnuDef   = self.gfortran10 ; # Default GNU   compiler version
+          intelDef = self.bsc.icc    ; # Default Intel compiler version
+          gitBranch = "garlic/mpi+send+seq";
+          cc  = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
+          mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
+        };
+
+        creamsInput = callPackage ./garlic/apps/creams/input.nix {
+          gitBranch = "garlic/mpi+send+seq";
+        };
+
+        hpcg = callPackage ./garlic/hpcg {
+          cc = self.bsc.icc;
+          gitBranch = "garlic/seq";
+        };
+
 #        heat = callPackage ./garlic/apps/heat {
 #          stdenv = pkgs.gcc7Stdenv;
 #          mpi = intel-mpi;
@@ -208,8 +212,6 @@ let
 #          mpi = intel-mpi;
 #        };
 #  
-#        hpcg = callPackage ./garlic/apps/hpcg { };
-#  
 #        hpccg = callPackage ./garlic/apps/hpccg { };
 #  
 #        fwi = callPackage ./garlic/apps/fwi { };
@@ -257,11 +259,19 @@ let
           };
         };
       };
-    };
 
-    test = {
-      exec = callPackage ./test/garlic/exec.nix {
-        exec = self.bsc.garlic.stages.exec;
+      test = {
+        exec = callPackage ./test/garlic/exec.nix {
+          exec = self.bsc.garlic.stages.exec;
+        };
+
+        osu = rec {
+          latency-internode = callPackage ./garlic/exp/osu/latency.nix { };
+          latency-intranode = callPackage ./garlic/exp/osu/latency.nix {
+            interNode = false;
+          };
+          latency = latency-internode;
+        };
       };
     };
   };

From b5fb3730ace902d0eb8694fdd769da82eb33e4d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Thu, 8 Oct 2020 18:24:56 +0200
Subject: [PATCH 02/14] WIP: first serial experiment. Don't know how to add gcc
 to compile

---
 garlic/apps/hpcg/default.nix | 11 +++++------
 overlay.nix                  | 22 +++++++++++++---------
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index 8ab7bec..f9e7f89 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -1,8 +1,5 @@
 {
   stdenv
-, nanos6
-, mpi
-, tampi
 , cc
 , gitBranch ? "garlic/seq"
 , makefileName ? "Linux_Serial"
@@ -21,12 +18,14 @@ stdenv.mkDerivation rec {
   '';
 
   buildInputs = [
-    nanos6
-    mpi
-    tampi
     cc
   ];
 
+  makeFlags = [
+    "CC=${cc.cc.CC}"
+    "CXX=${cc.cc.CXX}"
+  ];
+
   enableParallelBuilding = true;
 
   configurePhase = ''
diff --git a/overlay.nix b/overlay.nix
index d117f2b..5191ecc 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -258,19 +258,23 @@ let
             hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { };
           };
         };
-      };
 
-      test = {
-        exec = callPackage ./test/garlic/exec.nix {
-          exec = self.bsc.garlic.stages.exec;
+        hpcg = {
+          serial = callPackage ./garlic/exp/hpcg/serial.nix { };
         };
 
-        osu = rec {
-          latency-internode = callPackage ./garlic/exp/osu/latency.nix { };
-          latency-intranode = callPackage ./garlic/exp/osu/latency.nix {
-            interNode = false;
+        test = {
+          exec = callPackage ./test/garlic/exec.nix {
+            exec = self.bsc.garlic.stages.exec;
+          };
+
+          osu = rec {
+            latency-internode = callPackage ./garlic/exp/osu/latency.nix { };
+            latency-intranode = callPackage ./garlic/exp/osu/latency.nix {
+              interNode = false;
+            };
+            latency = latency-internode;
           };
-          latency = latency-internode;
         };
       };
     };

From 6bd7e12cffd81212bd13666d8ded0e799b341e26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Thu, 8 Oct 2020 18:26:28 +0200
Subject: [PATCH 03/14] WIP: forgot to add the folder

---
 garlic/exp/hpcg/extrae.xml | 211 +++++++++++++++++++++++++++++++++++++
 garlic/exp/hpcg/serial.nix | 169 +++++++++++++++++++++++++++++
 garlic/exp/hpcg/tampi.nix  | 171 ++++++++++++++++++++++++++++++
 3 files changed, 551 insertions(+)
 create mode 100644 garlic/exp/hpcg/extrae.xml
 create mode 100644 garlic/exp/hpcg/serial.nix
 create mode 100644 garlic/exp/hpcg/tampi.nix

diff --git a/garlic/exp/hpcg/extrae.xml b/garlic/exp/hpcg/extrae.xml
new file mode 100644
index 0000000..b9af29b
--- /dev/null
+++ b/garlic/exp/hpcg/extrae.xml
@@ -0,0 +1,211 @@
+<?xml version='1.0'?>
+
+<!-- Here comes the Extrae configuration.
+     As a general rule, "enabled" means that the feature is enabled :)  If
+     it's not enabled, then the value can be set to some default.
+-->
+
+<!-- Must we activate the tracing? Which is the tracing mode? (detail/bursts) Where is it located? Which kind of trace? Version of the XML parser?-->
+<trace enabled="yes"
+ home="/nix/store/j80mlqa12d1baifg30jsx2smv90akzvc-extrae"
+ initial-mode="detail"
+ type="paraver"
+>
+
+  <!-- Configuration of some MPI dependant values -->
+  <mpi enabled="yes">
+    <!-- Gather counters in the MPI routines? -->
+    <counters enabled="yes" />
+  </mpi>
+
+  <!-- Emit information of the callstack -->
+  <callers enabled="yes">
+    <!-- At MPI calls, select depth level -->
+    <mpi enabled="yes">1-3</mpi>
+    <!-- At sampling points, select depth level -->
+		<sampling enabled="yes">1-5</sampling>
+		<!-- At dynamic memory system calls -->
+		<dynamic-memory enabled="no">1-3</dynamic-memory>
+		<!-- At I/O system calls -->
+		<input-output enabled="no">1-3</input-output>
+		<!-- At other system calls -->
+	  <syscall enabled="no">1-3</syscall>
+  </callers>
+
+  <!-- Configuration of some OpenMP dependant values -->
+  <openmp enabled="no" ompt="no">
+    <!-- If the library instruments OpenMP, shall we gather info about locks?
+         Obtaining such information can make the final trace quite large.
+    -->
+		<locks enabled="no" />
+		<!-- Gather info about taskloops? -->
+		<taskloop enabled="no" />
+    <!-- Gather counters in the OpenMP routines? -->
+    <counters enabled="yes" />
+  </openmp>
+
+  <!-- Configuration of some pthread dependant values -->
+  <pthread enabled="no">
+    <!-- If the library instruments pthread, shall we gather info about locks,
+	       mutexs and conds?
+         Obtaining such information can make the final trace quite large.
+    -->
+    <locks enabled="no" />
+    <!-- Gather counters in the pthread routines? -->
+    <counters enabled="yes" />
+  </pthread>
+
+  <!-- Configuration of User Functions -->
+  <user-functions enabled="no" list="/home/bsc41/bsc41273/user-functions.dat" exclude-automatic-functions="no">
+    <!-- Gather counters on the UF routines? -->
+    <counters enabled="yes" />
+  </user-functions>
+
+  <!-- Configure which software/hardware counters must be collected -->
+  <counters enabled="yes">
+    <!-- Configure the CPU hardware counters. You can define here as many sets
+         as you want. You can also define if MPI/OpenMP calls must report such
+         counters.
+         Starting-set property defines which set is chosen from every task.
+         Possible values are:
+           - cyclic : The sets are distributed in a cyclic fashion among all
+           tasks. So Task 0 takes set 1, Task 1 takes set 2,...
+           - block  : The sets are distributed in block fashion among all tasks.
+           Task [0..i-1] takes set 1, Task [i..2*i-1] takes set 2, ...
+           - Number : All the tasks will start with the given set 
+           (from 1..N).
+    -->
+    <cpu enabled="yes" starting-set-distribution="1">
+      <!-- In this example, we configure two sets of counters. The first will 
+           be changed into the second after 5 calls to some collective
+           operation on MPI_COMM_WORLD. Once the second is activated, it will
+           turn to the first after 5seconds (aprox. depending on the MPI calls
+           granularity)
+           If you want that any set be counting forever, just don't set
+           changeat-globalops, or, changeat-time.
+
+           Each set has it's own properties.
+           domain -> in which domain must PAPI obtain the information (see
+                       PAPI info)
+           changeat-globalops=num -> choose the next set after num
+                       MPI_COMM_WORLD operations
+           changeat-time=numTime -> choose the next set after num Time
+                       (for example 5s, 15m (for ms), 10M (for minutes),..)
+      -->
+      <set enabled="yes" domain="all">
+        PAPI_TOT_INS,PAPI_TOT_CYC
+      </set>
+    </cpu>
+
+    <!-- Do we want to gather information of the network counters?
+         Nowadays we can gather information about MX/GM cards.
+     -->
+    <network enabled="no" />
+
+    <!-- Obtain resource usage information -->
+    <resource-usage enabled="no" />
+
+    <!-- Obtain malloc statistics -->
+    <memory-usage enabled="no" />
+  </counters>
+
+  <!-- Define the characteristics of the tracing storage. If not defined,
+       or set, the tracing will send the traces to the current directory
+       with a default output name.
+  -->
+  <storage enabled="no">
+    <!-- The intermediate files will take the name of the application -->
+    <trace-prefix enabled="yes">TRACE</trace-prefix>
+    <!-- Stop the tracing when the intermediate file reaches this amount of MBs -->
+    <size enabled="no">5</size>
+    <!-- Where must we store the MPIT files while the app runs? -->
+    <temporal-directory enabled="yes">/scratch</temporal-directory>
+    <!-- Where must we store the MPIT files once the app ends? -->
+    <final-directory enabled="yes">/gpfs/scratch/bsc41/bsc41273</final-directory>
+  </storage>
+
+  <!-- Buffer configuration -->
+  <buffer enabled="yes">
+    <!-- How many events can we handle before any flush -->
+    <size enabled="yes">5000000</size>
+    <!-- Use the event buffer in a circular manner? You can use this option to
+         trace the last set of events. Needs MPI global routines operating on
+         MPI_COMM_WORLD communicator to be merged
+    -->
+    <circular enabled="no" />
+  </buffer>
+
+  <!-- Control tracing -->
+  <trace-control enabled="no">
+    <!-- We can start the application with a "latent tracing" and wake it up
+         once a control file is created. Use the property 'frequency' to
+         choose at which frequency this check must be done. If not supplied,
+         it will be checked every 100 global operations on MPI_COMM_WORLD.
+    -->
+    <file enabled="no" frequency="5M">/gpfs/scratch/bsc41/bsc41273/control</file>
+    <!-- 
+    -->
+    <global-ops enabled="no"></global-ops>
+  </trace-control>
+
+  <others enabled="yes">
+    <!-- Want to force a minimum amount of time of tracing? Here we force 10
+         minutes -->
+    <minimum-time enabled="no">10M</minimum-time>
+	<!-- Capture the following signals to finish cleanly -->
+    <finalize-on-signal enabled="yes" 
+      SIGUSR1="no" SIGUSR2="no" SIGINT="yes"
+      SIGQUIT="yes" SIGTERM="yes" SIGXCPU="yes"
+      SIGFPE="yes" SIGSEGV="yes" SIGABRT="yes"
+    />
+	<!-- Use instrumentation poitns to flush sampling buffer -->
+    <flush-sampling-buffer-at-instrumentation-point enabled="yes" />
+  </others>
+
+
+  <!-- Bursts library enabled? This requires an special library! -->
+  <bursts enabled="no">
+    <!-- Specify the threshold. This is mandatory! In this example, the
+         threshold is limitted to 500 microseconds
+     -->
+    <threshold enabled="yes">500u</threshold>
+    <!-- Report MPI statistics? -->
+    <mpi-statistics enabled="yes" />
+  </bursts>
+
+  <!-- Enable sampling capabilities using system clock.
+       Type may refer to: default, real, prof and virtual.
+       Period stands for the sampling period (50ms here)
+       plus a variability of 10ms, which means periods from 
+       45 to 55ms.
+  -->
+  <sampling enabled="no" type="default" period="50m" variability="10m" />
+
+  <!-- Enable dynamic memory instrumentation (experimental) -->
+  <dynamic-memory enabled="no" />
+
+  <!-- Enable I/O (read, write) instrumentation (experimental) -->
+  <input-output enabled="no" internals="no"/>
+
+	<!-- Enable system calls instrumentation (experimental) -->
+  <syscall enabled="no" />
+
+  <!-- Do merge the intermediate tracefiles into the final tracefile?
+       Named according to the binary name
+       options: 
+       synchronization = { default, task, node, no } (default is node)
+       max-memory = Number (in Mbytes) max memory used in merge step
+       joint-states = { yes, no } generate joint states?
+       keep-mpits = { yes, no } keep mpit files after merge?
+  -->
+  <merge enabled="yes" 
+    synchronization="default"
+    tree-fan-out="16"
+    max-memory="512"
+    joint-states="yes"
+    keep-mpits="yes"
+    sort-addresses="yes"
+    overwrite="yes"
+  />
+
+</trace>
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
new file mode 100644
index 0000000..51da987
--- /dev/null
+++ b/garlic/exp/hpcg/serial.nix
@@ -0,0 +1,169 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    cc = [ bsc.icc ];
+    gitBranch = [ "garlic/seq" ];
+    nx = [ 104 64 ];
+    ny = [ 104 64 ];
+    nz = [ 104 64 ];
+  };
+
+  # Common configuration
+  common = {
+    # Resources
+    ntasksPerNode = "48";
+    nodes = "1";
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch (
+    # Allow a user to define a custom reservation for the job in MareNostrum4,
+    # by setting the garlic.sbatch.reservation attribute in the 
+    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
+    # reservation is used. The user reservation may be overwritten by the
+    # experiment, if the reservation is set like with nodes or ntasksPerNode.
+    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
+      inherit (pkgs.config.garlic.sbatch) reservation;
+    } // {
+      program = stageProgram stage;
+      exclusive = true;
+      time = "02:00:00";
+      qos = "debug";
+      jobName = "hpcg";
+      inherit nixPrefix nodes ntasksPerNode;
+    }
+  );
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  argv = {stage, conf, ...}: with conf; w.argv {
+    program = stageProgram stage;
+    argv = ''(
+      --nx=${toString nx}
+      --ny=${toString ny}
+      --nz=${toString nz}
+    )'';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  hpcgFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.hpcg.override {
+      inherit cc gitBranch;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ argv hpcgFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs
diff --git a/garlic/exp/hpcg/tampi.nix b/garlic/exp/hpcg/tampi.nix
new file mode 100644
index 0000000..d2a0394
--- /dev/null
+++ b/garlic/exp/hpcg/tampi.nix
@@ -0,0 +1,171 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    cc = [ bsc.icc ];
+    mpi = [ bsc.impi ];
+    #gitBranch = [ "garlic/tampi+send+oss+task" ];
+    n = [ 104 64 ];
+    nodes = [1 2 4];
+  };
+
+  # Common configuration
+  common = {
+    # Compile time nbody config
+    gitBranch = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete";
+
+    # Resources
+    ntasksPerNode = "48";
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch (
+    # Allow a user to define a custom reservation for the job in MareNostrum4,
+    # by setting the garlic.sbatch.reservation attribute in the 
+    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
+    # reservation is used. The user reservation may be overwritten by the
+    # experiment, if the reservation is set like with nodes or ntasksPerNode.
+    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
+      inherit (pkgs.config.garlic.sbatch) reservation;
+    } // {
+      program = stageProgram stage;
+      exclusive = true;
+      time = "02:00:00";
+      qos = "debug";
+      jobName = "nbody-tampi";
+      inherit nixPrefix nodes ntasksPerNode;
+    }
+  );
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  argv = {stage, conf, ...}: with conf; w.argv {
+    program = stageProgram stage;
+    argv = ''(
+      --nx=${toString n}
+      --ny=${toString n}
+      --nz=${toString n}
+    )'';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  hpcgFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.hpcg.override {
+      inherit cc mpi gitBranch;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ argv hpcgFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs

From 7bf3e812338673c96dc7d3c0ccbbc9ef6d3bbef5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Tue, 13 Oct 2020 14:59:49 +0200
Subject: [PATCH 04/14] WIP: trying to make mpi branch working

---
 garlic/apps/hpcg/default.nix |   2 +
 garlic/exp/hpcg/mpi.nix      | 169 +++++++++++++++++++++++++++++++++++
 garlic/exp/hpcg/serial.nix   |  12 ++-
 overlay.nix                  |   3 +
 4 files changed, 179 insertions(+), 7 deletions(-)
 create mode 100644 garlic/exp/hpcg/mpi.nix

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index f9e7f89..7ba8545 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -1,6 +1,7 @@
 {
   stdenv
 , cc
+, mpi
 , gitBranch ? "garlic/seq"
 , makefileName ? "Linux_Serial"
 }:
@@ -19,6 +20,7 @@ stdenv.mkDerivation rec {
 
   buildInputs = [
     cc
+    mpi
   ];
 
   makeFlags = [
diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix
new file mode 100644
index 0000000..9adc093
--- /dev/null
+++ b/garlic/exp/hpcg/mpi.nix
@@ -0,0 +1,169 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    cc = [ bsc.icc ];
+    mpi = [ bsc.impi ];
+    gitBranch = [ "garlic/mpi" ];
+    makefileName = [ "MPI" ];
+    n = [ 104 64 ];
+  };
+
+  # Common configuration
+  common = {
+    # Resources
+    ntasksPerNode = "48";
+    nodes = "1";
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch (
+    # Allow a user to define a custom reservation for the job in MareNostrum4,
+    # by setting the garlic.sbatch.reservation attribute in the 
+    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
+    # reservation is used. The user reservation may be overwritten by the
+    # experiment, if the reservation is set like with nodes or ntasksPerNode.
+    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
+      inherit (pkgs.config.garlic.sbatch) reservation;
+    } // {
+      program = stageProgram stage;
+      exclusive = true;
+      time = "02:00:00";
+      qos = "debug";
+      jobName = "hpcg";
+      inherit nixPrefix nodes ntasksPerNode;
+    }
+  );
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  argv = {stage, conf, ...}: with conf; w.argv {
+    program = stageProgram stage;
+    argv = ''(
+      --nx=${toString n}
+      --ny=${toString n}
+      --nz=${toString n}
+    )'';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  hpcgFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.hpcg.override {
+      inherit cc mpi gitBranch makefileName;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the hpcg app with the argv and env vars
+    ++ [ argv hpcgFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
index 51da987..79e8c21 100644
--- a/garlic/exp/hpcg/serial.nix
+++ b/garlic/exp/hpcg/serial.nix
@@ -16,9 +16,7 @@ let
   varConfig = {
     cc = [ bsc.icc ];
     gitBranch = [ "garlic/seq" ];
-    nx = [ 104 64 ];
-    ny = [ 104 64 ];
-    nz = [ 104 64 ];
+    n = [ 104 64 ];
   };
 
   # Common configuration
@@ -106,9 +104,9 @@ let
   argv = {stage, conf, ...}: with conf; w.argv {
     program = stageProgram stage;
     argv = ''(
-      --nx=${toString nx}
-      --ny=${toString ny}
-      --nz=${toString nz}
+      --nx=${toString n}
+      --ny=${toString n}
+      --nz=${toString n}
     )'';
   };
 
@@ -158,7 +156,7 @@ let
     # Optionally profile nanos6 with the new ctf
     ++ optional enableCtf ctf
 
-    # Execute the nbody app with the argv and env vars
+    # Execute the hpcg app with the argv and env vars
     ++ [ argv hpcgFn ];
 
   # List of actual programs to be executed
diff --git a/overlay.nix b/overlay.nix
index 5191ecc..8143eb4 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -261,6 +261,9 @@ let
 
         hpcg = {
           serial = callPackage ./garlic/exp/hpcg/serial.nix { };
+          mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
+          # omp = callPackage ./garlic/exp/hpcg/omp.nix { };
+          # mpi+omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
         };
 
         test = {

From 01b25846880749736310486e86748ed15dab61fd Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Tue, 13 Oct 2020 17:05:36 +0200
Subject: [PATCH 05/14] Update hpcg experiments

---
 garlic/apps/hpcg/default.nix |   2 +
 garlic/exp/hpcg/mpi.nix      | 188 ++++++++---------------------------
 garlic/exp/hpcg/serial.nix   | 184 ++++++++--------------------------
 garlic/exp/hpcg/tampi.nix    | 171 -------------------------------
 overlay.nix                  |   2 +-
 5 files changed, 85 insertions(+), 462 deletions(-)
 delete mode 100644 garlic/exp/hpcg/tampi.nix

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index 7ba8545..257f02e 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -41,4 +41,6 @@ stdenv.mkDerivation rec {
     cp bin/* $out/bin/
   '';
 
+  programPath = "/bin/xhpcg";
+
 }
diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix
index 9adc093..91b0429 100644
--- a/garlic/exp/hpcg/mpi.nix
+++ b/garlic/exp/hpcg/mpi.nix
@@ -1,169 +1,65 @@
 {
   stdenv
-, nixpkgs
-, pkgs
-, genApp
-, genConfigs
-, runWrappers
+, stdexp
+, bsc
+, targetMachine
+, stages
 }:
 
 with stdenv.lib;
 
 let
-  bsc = pkgs.bsc;
-
-  # Set variable configuration for the experiment
-  varConfig = {
-    cc = [ bsc.icc ];
-    mpi = [ bsc.impi ];
-    gitBranch = [ "garlic/mpi" ];
-    makefileName = [ "MPI" ];
+  # Initial variable configuration
+  varConf = with bsc; {
     n = [ 104 64 ];
   };
 
-  # Common configuration
-  common = {
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = icc;
+    mpi = impi;
+    gitBranch = "garlic/mpi";
+    # FIXME: fix this option in the garlic/mpi git branch
+    makefileName = "MPI";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
     # Resources
-    ntasksPerNode = "48";
-    nodes = "1";
-
-    # Stage configuration
-    enableSbatch = true;
-    enableControl = true;
-    enableExtrae = false;
-    enablePerf = false;
-    enableCtf = false;
-
-    # MN4 path
-    nixPrefix = "/gpfs/projects/bsc15/nix";
+    qos = "debug";
+    ntasksPerNode = 48;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "sockets,verbose";
+    jobName = "hpcg-${toString n}-${gitBranch}";
   };
 
-  # Compute the cartesian product of all configurations
-  configs = map (conf: conf // common) (genConfigs varConfig);
-
-  stageProgram = stage:
-    if stage ? programPath
-    then "${stage}${stage.programPath}" else "${stage}";
-
-  w = runWrappers;
-
-  sbatch = {stage, conf, ...}: with conf; w.sbatch (
-    # Allow a user to define a custom reservation for the job in MareNostrum4,
-    # by setting the garlic.sbatch.reservation attribute in the 
-    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
-    # reservation is used. The user reservation may be overwritten by the
-    # experiment, if the reservation is set like with nodes or ntasksPerNode.
-    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
-      inherit (pkgs.config.garlic.sbatch) reservation;
-    } // {
-      program = stageProgram stage;
-      exclusive = true;
-      time = "02:00:00";
-      qos = "debug";
-      jobName = "hpcg";
-      inherit nixPrefix nodes ntasksPerNode;
-    }
-  );
-
-  control = {stage, conf, ...}: with conf; w.control {
-    program = stageProgram stage;
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
   };
 
-  srun = {stage, conf, ...}: with conf; w.srun {
-    program = stageProgram stage;
-    srunOptions = "--cpu-bind=verbose,rank";
-    inherit nixPrefix;
-  };
-
-  statspy = {stage, conf, ...}: with conf; w.statspy {
-    program = stageProgram stage;
-  };
-
-  perf = {stage, conf, ...}: with conf; w.perf {
-    program = stageProgram stage;
-    perfArgs = "sched record -a";
-  };
-
-  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
-    program = stageProgram stage;
-    nixsetup = "${nixPrefix}/bin/nix-setup";
-  };
-
-  extrae = {stage, conf, ...}: w.extrae {
-    program = stageProgram stage;
-    traceLib = "mpi"; # mpi -> libtracempi.so
-    configFile = ./extrae.xml;
-  };
-
-  ctf = {stage, conf, ...}: w.argv {
-    program = stageProgram stage;
-    env = ''
-      export NANOS6=ctf
-      export NANOS6_CTF2PRV=0
-    '';
-  };
-
-  argv = {stage, conf, ...}: with conf; w.argv {
-    program = stageProgram stage;
-    argv = ''(
-      --nx=${toString n}
-      --ny=${toString n}
-      --nz=${toString n}
-    )'';
-  };
-
-  bscOverlay = import ../../../overlay.nix;
-
-  genPkgs = newOverlay: nixpkgs {
-    overlays = [
-      bscOverlay
-      newOverlay
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n}"
+      "--ny=${toString n}"
+      "--nz=${toString n}"
     ];
   };
 
-  # We may be able to use overlays by invoking the fix function directly, but we
-  # have to get the definition of the bsc packages and the garlic ones as
-  # overlays.
-
-  hpcgFn = {stage, conf, ...}: with conf;
-    let
-      # We set the mpi implementation to the one specified in the conf, so all
-      # packages in bsc will use that one.
-      customPkgs = genPkgs (self: super: {
-        bsc = super.bsc // { mpi = conf.mpi; };
-      });
-    in
-    customPkgs.bsc.garlic.hpcg.override {
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
       inherit cc mpi gitBranch makefileName;
     };
 
-  stages = with common; []
-    # Use sbatch to request resources first
-    ++ optional enableSbatch sbatch
-
-    # Repeats the next stages N times
-    ++ optionals enableControl [ nixsetup control ]
-
-    # Executes srun to launch the program in the requested nodes, and
-    # immediately after enters the nix environment again, as slurmstepd launches
-    # the next stages from outside the namespace.
-    ++ [ srun nixsetup ]
-
-    # Intrumentation with extrae
-    ++ optional enableExtrae extrae
-
-    # Optionally profile the next stages with perf
-    ++ optional enablePerf perf
-
-    # Optionally profile nanos6 with the new ctf
-    ++ optional enableCtf ctf
-
-    # Execute the hpcg app with the argv and env vars
-    ++ [ argv hpcgFn ];
-
-  # List of actual programs to be executed
-  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
-  # We simply run each program one after another
-  w.launch jobs
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
index 79e8c21..0d5d855 100644
--- a/garlic/exp/hpcg/serial.nix
+++ b/garlic/exp/hpcg/serial.nix
@@ -1,167 +1,63 @@
 {
   stdenv
-, nixpkgs
-, pkgs
-, genApp
-, genConfigs
-, runWrappers
+, stdexp
+, bsc
+, targetMachine
+, stages
 }:
 
 with stdenv.lib;
 
 let
-  bsc = pkgs.bsc;
-
-  # Set variable configuration for the experiment
-  varConfig = {
-    cc = [ bsc.icc ];
-    gitBranch = [ "garlic/seq" ];
+  # Initial variable configuration
+  varConf = with bsc; {
     n = [ 104 64 ];
   };
 
-  # Common configuration
-  common = {
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = icc;
+    mpi = impi;
+    gitBranch = "garlic/seq";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
     # Resources
-    ntasksPerNode = "48";
-    nodes = "1";
-
-    # Stage configuration
-    enableSbatch = true;
-    enableControl = true;
-    enableExtrae = false;
-    enablePerf = false;
-    enableCtf = false;
-
-    # MN4 path
-    nixPrefix = "/gpfs/projects/bsc15/nix";
+    qos = "debug";
+    ntasksPerNode = 48;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "sockets,verbose";
+    jobName = "hpcg-${toString n}-${gitBranch}";
   };
 
-  # Compute the cartesian product of all configurations
-  configs = map (conf: conf // common) (genConfigs varConfig);
-
-  stageProgram = stage:
-    if stage ? programPath
-    then "${stage}${stage.programPath}" else "${stage}";
-
-  w = runWrappers;
-
-  sbatch = {stage, conf, ...}: with conf; w.sbatch (
-    # Allow a user to define a custom reservation for the job in MareNostrum4,
-    # by setting the garlic.sbatch.reservation attribute in the 
-    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
-    # reservation is used. The user reservation may be overwritten by the
-    # experiment, if the reservation is set like with nodes or ntasksPerNode.
-    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
-      inherit (pkgs.config.garlic.sbatch) reservation;
-    } // {
-      program = stageProgram stage;
-      exclusive = true;
-      time = "02:00:00";
-      qos = "debug";
-      jobName = "hpcg";
-      inherit nixPrefix nodes ntasksPerNode;
-    }
-  );
-
-  control = {stage, conf, ...}: with conf; w.control {
-    program = stageProgram stage;
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
   };
 
-  srun = {stage, conf, ...}: with conf; w.srun {
-    program = stageProgram stage;
-    srunOptions = "--cpu-bind=verbose,rank";
-    inherit nixPrefix;
-  };
-
-  statspy = {stage, conf, ...}: with conf; w.statspy {
-    program = stageProgram stage;
-  };
-
-  perf = {stage, conf, ...}: with conf; w.perf {
-    program = stageProgram stage;
-    perfArgs = "sched record -a";
-  };
-
-  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
-    program = stageProgram stage;
-    nixsetup = "${nixPrefix}/bin/nix-setup";
-  };
-
-  extrae = {stage, conf, ...}: w.extrae {
-    program = stageProgram stage;
-    traceLib = "mpi"; # mpi -> libtracempi.so
-    configFile = ./extrae.xml;
-  };
-
-  ctf = {stage, conf, ...}: w.argv {
-    program = stageProgram stage;
-    env = ''
-      export NANOS6=ctf
-      export NANOS6_CTF2PRV=0
-    '';
-  };
-
-  argv = {stage, conf, ...}: with conf; w.argv {
-    program = stageProgram stage;
-    argv = ''(
-      --nx=${toString n}
-      --ny=${toString n}
-      --nz=${toString n}
-    )'';
-  };
-
-  bscOverlay = import ../../../overlay.nix;
-
-  genPkgs = newOverlay: nixpkgs {
-    overlays = [
-      bscOverlay
-      newOverlay
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n}"
+      "--ny=${toString n}"
+      "--nz=${toString n}"
     ];
   };
 
-  # We may be able to use overlays by invoking the fix function directly, but we
-  # have to get the definition of the bsc packages and the garlic ones as
-  # overlays.
-
-  hpcgFn = {stage, conf, ...}: with conf;
-    let
-      # We set the mpi implementation to the one specified in the conf, so all
-      # packages in bsc will use that one.
-      customPkgs = genPkgs (self: super: {
-        bsc = super.bsc // { mpi = conf.mpi; };
-      });
-    in
-    customPkgs.bsc.garlic.hpcg.override {
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
       inherit cc gitBranch;
     };
 
-  stages = with common; []
-    # Use sbatch to request resources first
-    ++ optional enableSbatch sbatch
-
-    # Repeats the next stages N times
-    ++ optionals enableControl [ nixsetup control ]
-
-    # Executes srun to launch the program in the requested nodes, and
-    # immediately after enters the nix environment again, as slurmstepd launches
-    # the next stages from outside the namespace.
-    ++ [ srun nixsetup ]
-
-    # Intrumentation with extrae
-    ++ optional enableExtrae extrae
-
-    # Optionally profile the next stages with perf
-    ++ optional enablePerf perf
-
-    # Optionally profile nanos6 with the new ctf
-    ++ optional enableCtf ctf
-
-    # Execute the hpcg app with the argv and env vars
-    ++ [ argv hpcgFn ];
-
-  # List of actual programs to be executed
-  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
-  # We simply run each program one after another
-  w.launch jobs
+ 
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/hpcg/tampi.nix b/garlic/exp/hpcg/tampi.nix
deleted file mode 100644
index d2a0394..0000000
--- a/garlic/exp/hpcg/tampi.nix
+++ /dev/null
@@ -1,171 +0,0 @@
-{
-  stdenv
-, nixpkgs
-, pkgs
-, genApp
-, genConfigs
-, runWrappers
-}:
-
-with stdenv.lib;
-
-let
-  bsc = pkgs.bsc;
-
-  # Set variable configuration for the experiment
-  varConfig = {
-    cc = [ bsc.icc ];
-    mpi = [ bsc.impi ];
-    #gitBranch = [ "garlic/tampi+send+oss+task" ];
-    n = [ 104 64 ];
-    nodes = [1 2 4];
-  };
-
-  # Common configuration
-  common = {
-    # Compile time nbody config
-    gitBranch = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete";
-
-    # Resources
-    ntasksPerNode = "48";
-
-    # Stage configuration
-    enableSbatch = true;
-    enableControl = true;
-    enableExtrae = false;
-    enablePerf = false;
-    enableCtf = false;
-
-    # MN4 path
-    nixPrefix = "/gpfs/projects/bsc15/nix";
-  };
-
-  # Compute the cartesian product of all configurations
-  configs = map (conf: conf // common) (genConfigs varConfig);
-
-  stageProgram = stage:
-    if stage ? programPath
-    then "${stage}${stage.programPath}" else "${stage}";
-
-  w = runWrappers;
-
-  sbatch = {stage, conf, ...}: with conf; w.sbatch (
-    # Allow a user to define a custom reservation for the job in MareNostrum4,
-    # by setting the garlic.sbatch.reservation attribute in the 
-    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
-    # reservation is used. The user reservation may be overwritten by the
-    # experiment, if the reservation is set like with nodes or ntasksPerNode.
-    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
-      inherit (pkgs.config.garlic.sbatch) reservation;
-    } // {
-      program = stageProgram stage;
-      exclusive = true;
-      time = "02:00:00";
-      qos = "debug";
-      jobName = "nbody-tampi";
-      inherit nixPrefix nodes ntasksPerNode;
-    }
-  );
-
-  control = {stage, conf, ...}: with conf; w.control {
-    program = stageProgram stage;
-  };
-
-  srun = {stage, conf, ...}: with conf; w.srun {
-    program = stageProgram stage;
-    srunOptions = "--cpu-bind=verbose,rank";
-    inherit nixPrefix;
-  };
-
-  statspy = {stage, conf, ...}: with conf; w.statspy {
-    program = stageProgram stage;
-  };
-
-  perf = {stage, conf, ...}: with conf; w.perf {
-    program = stageProgram stage;
-    perfArgs = "sched record -a";
-  };
-
-  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
-    program = stageProgram stage;
-    nixsetup = "${nixPrefix}/bin/nix-setup";
-  };
-
-  extrae = {stage, conf, ...}: w.extrae {
-    program = stageProgram stage;
-    traceLib = "mpi"; # mpi -> libtracempi.so
-    configFile = ./extrae.xml;
-  };
-
-  ctf = {stage, conf, ...}: w.argv {
-    program = stageProgram stage;
-    env = ''
-      export NANOS6=ctf
-      export NANOS6_CTF2PRV=0
-    '';
-  };
-
-  argv = {stage, conf, ...}: with conf; w.argv {
-    program = stageProgram stage;
-    argv = ''(
-      --nx=${toString n}
-      --ny=${toString n}
-      --nz=${toString n}
-    )'';
-  };
-
-  bscOverlay = import ../../../overlay.nix;
-
-  genPkgs = newOverlay: nixpkgs {
-    overlays = [
-      bscOverlay
-      newOverlay
-    ];
-  };
-
-  # We may be able to use overlays by invoking the fix function directly, but we
-  # have to get the definition of the bsc packages and the garlic ones as
-  # overlays.
-
-  hpcgFn = {stage, conf, ...}: with conf;
-    let
-      # We set the mpi implementation to the one specified in the conf, so all
-      # packages in bsc will use that one.
-      customPkgs = genPkgs (self: super: {
-        bsc = super.bsc // { mpi = conf.mpi; };
-      });
-    in
-    customPkgs.bsc.garlic.hpcg.override {
-      inherit cc mpi gitBranch;
-    };
-
-  stages = with common; []
-    # Use sbatch to request resources first
-    ++ optional enableSbatch sbatch
-
-    # Repeats the next stages N times
-    ++ optionals enableControl [ nixsetup control ]
-
-    # Executes srun to launch the program in the requested nodes, and
-    # immediately after enters the nix environment again, as slurmstepd launches
-    # the next stages from outside the namespace.
-    ++ [ srun nixsetup ]
-
-    # Intrumentation with extrae
-    ++ optional enableExtrae extrae
-
-    # Optionally profile the next stages with perf
-    ++ optional enablePerf perf
-
-    # Optionally profile nanos6 with the new ctf
-    ++ optional enableCtf ctf
-
-    # Execute the nbody app with the argv and env vars
-    ++ [ argv hpcgFn ];
-
-  # List of actual programs to be executed
-  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
-
-in
-  # We simply run each program one after another
-  w.launch jobs
diff --git a/overlay.nix b/overlay.nix
index 8143eb4..3ce3368 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -197,7 +197,7 @@ let
           gitBranch = "garlic/mpi+send+seq";
         };
 
-        hpcg = callPackage ./garlic/hpcg {
+        hpcg = callPackage ./garlic/apps/hpcg/default.nix {
           cc = self.bsc.icc;
           gitBranch = "garlic/seq";
         };

From e20061254bec561428622cfe9516b74307bdc28f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Tue, 13 Oct 2020 19:20:24 +0200
Subject: [PATCH 06/14] WIP: Add mpi, omp and mpi+omp experiments. See more.

Seems that gcc compilation with OpenMP throws an error. Investigate.
I think I've forgot to add an override of mpicxx compiler backend
---
 garlic/apps/hpcg/default.nix | 16 +++------
 garlic/exp/hpcg/mpi+omp.nix  | 63 ++++++++++++++++++++++++++++++++++++
 garlic/exp/hpcg/mpi.nix      |  6 ++--
 garlic/exp/hpcg/omp.nix      | 63 ++++++++++++++++++++++++++++++++++++
 garlic/exp/hpcg/serial.nix   |  4 +--
 overlay.nix                  |  7 ++--
 6 files changed, 139 insertions(+), 20 deletions(-)
 create mode 100644 garlic/exp/hpcg/mpi+omp.nix
 create mode 100644 garlic/exp/hpcg/omp.nix

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index 257f02e..6dc697c 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -1,11 +1,11 @@
 {
   stdenv
 , cc
-, mpi
-, gitBranch ? "garlic/seq"
-, makefileName ? "Linux_Serial"
+, mpi ? null
+, gitBranch
 }:
 
+with stdenv.lib;
 stdenv.mkDerivation rec {
   name = "hpcg";
 
@@ -20,8 +20,8 @@ stdenv.mkDerivation rec {
 
   buildInputs = [
     cc
-    mpi
-  ];
+  ]
+  ++ optional (mpi != null) mpi;
 
   makeFlags = [
     "CC=${cc.cc.CC}"
@@ -30,12 +30,6 @@ stdenv.mkDerivation rec {
 
   enableParallelBuilding = true;
 
-  configurePhase = ''
-    mkdir build
-    cd build
-    ../configure ${makefileName}
-  '';
-
   installPhase = ''
     mkdir -p $out/bin
     cp bin/* $out/bin/
diff --git a/garlic/exp/hpcg/mpi+omp.nix b/garlic/exp/hpcg/mpi+omp.nix
new file mode 100644
index 0000000..e4facc5
--- /dev/null
+++ b/garlic/exp/hpcg/mpi+omp.nix
@@ -0,0 +1,63 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ 104 64 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = icc;
+    mpi = impi;
+    gitBranch = "garlic/mpi+omp";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 48;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "sockets,verbose";
+    jobName = "hpcg-${toString n}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n}"
+      "--ny=${toString n}"
+      "--nz=${toString n}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc mpi gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix
index 91b0429..331bd91 100644
--- a/garlic/exp/hpcg/mpi.nix
+++ b/garlic/exp/hpcg/mpi.nix
@@ -21,8 +21,6 @@ let
     cc = icc;
     mpi = impi;
     gitBranch = "garlic/mpi";
-    # FIXME: fix this option in the garlic/mpi git branch
-    makefileName = "MPI";
 
     # Repeat the execution of each unit 30 times
     loops = 30;
@@ -55,11 +53,11 @@ let
     customPkgs = stdexp.replaceMpi conf.mpi;
   in
     customPkgs.apps.hpcg.override {
-      inherit cc mpi gitBranch makefileName;
+      inherit cc mpi gitBranch;
     };
 
   pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
- 
+
   stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/hpcg/omp.nix b/garlic/exp/hpcg/omp.nix
new file mode 100644
index 0000000..bf3d59d
--- /dev/null
+++ b/garlic/exp/hpcg/omp.nix
@@ -0,0 +1,63 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ 104 64 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = icc;
+    mpi = null; # TODO: Remove this for omp
+    gitBranch = "garlic/seq";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 48;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "sockets,verbose";
+    jobName = "hpcg-${toString n}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n}"
+      "--ny=${toString n}"
+      "--nz=${toString n}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
index 0d5d855..91ce793 100644
--- a/garlic/exp/hpcg/serial.nix
+++ b/garlic/exp/hpcg/serial.nix
@@ -19,7 +19,7 @@ let
     # hpcg options
     n = c.n;
     cc = icc;
-    mpi = impi;
+    mpi = null; # TODO: Remove this for serial
     gitBranch = "garlic/seq";
 
     # Repeat the execution of each unit 30 times
@@ -59,5 +59,5 @@ let
   pipeline = stdexp.stdPipeline ++ [ exec program ];
 
 in
- 
+
   stdexp.genExperiment { inherit configs pipeline; }
diff --git a/overlay.nix b/overlay.nix
index 3ce3368..3e782c9 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -199,7 +199,8 @@ let
 
         hpcg = callPackage ./garlic/apps/hpcg/default.nix {
           cc = self.bsc.icc;
-          gitBranch = "garlic/seq";
+          mpi = self.bsc.impi;
+          gitBranch = "garlic/mpi+omp";
         };
 
 #        heat = callPackage ./garlic/apps/heat {
@@ -262,8 +263,8 @@ let
         hpcg = {
           serial = callPackage ./garlic/exp/hpcg/serial.nix { };
           mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
-          # omp = callPackage ./garlic/exp/hpcg/omp.nix { };
-          # mpi+omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
+          omp = callPackage ./garlic/exp/hpcg/omp.nix { };
+          mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
         };
 
         test = {

From ea0272c2122244a9d47301fb102323fc1dbecf04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Thu, 22 Oct 2020 13:32:20 +0200
Subject: [PATCH 07/14] Add OmpSs-2 (no mpi) version

---
 garlic/apps/hpcg/default.nix |  4 +++
 garlic/exp/hpcg/oss.nix      | 68 ++++++++++++++++++++++++++++++++++++
 overlay.nix                  | 21 +++++++++--
 3 files changed, 91 insertions(+), 2 deletions(-)
 create mode 100644 garlic/exp/hpcg/oss.nix

diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix
index 6dc697c..3d62904 100644
--- a/garlic/apps/hpcg/default.nix
+++ b/garlic/apps/hpcg/default.nix
@@ -1,6 +1,8 @@
 {
   stdenv
 , cc
+, nanos6 ? null
+, mcxx ? null
 , mpi ? null
 , gitBranch
 }:
@@ -21,6 +23,8 @@ stdenv.mkDerivation rec {
   buildInputs = [
     cc
   ]
+  ++ optional (mcxx != null) mcxx
+  ++ optional (nanos6 != null) nanos6
   ++ optional (mpi != null) mpi;
 
   makeFlags = [
diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
new file mode 100644
index 0000000..e30c6de
--- /dev/null
+++ b/garlic/exp/hpcg/oss.nix
@@ -0,0 +1,68 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ 200 104 64 ];
+    nblocks = [ 128 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    nblocks = c.nblocks;
+    cc = bsc.icc;
+    mcxx = bsc.mcxx;
+    nanos6 = bsc.nanos6;
+    mpi = null; # TODO: Remove this for oss
+    gitBranch = "garlic/oss";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 48;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "sockets,verbose";
+    jobName = "hpcg-${toString n}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n}"
+      "--ny=${toString n}"
+      "--nz=${toString n}"
+      "--nblocks=${toString nblocks}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc nanos6 mcxx gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }
diff --git a/overlay.nix b/overlay.nix
index 3e782c9..152ddf9 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -199,8 +199,24 @@ let
 
         hpcg = callPackage ./garlic/apps/hpcg/default.nix {
           cc = self.bsc.icc;
-          mpi = self.bsc.impi;
-          gitBranch = "garlic/mpi+omp";
+          mcxx = self.bsc.mcxx;
+          nanos6 = self.bsc.nanos6;
+          gitBranch = "garlic/oss";
+
+          # cc = self.bsc.icc;
+          # gitBranch = "garlic/seq";
+
+          # cc = self.bsc.icc;
+          # mpi = self.bsc.mpi;
+          # gitBranch = "garlic/mpi";
+
+          # cc = self.bsc.icc;
+          # gitBranch = "garlic/omp";
+
+          # cc = self.bsc.icc;
+          # mpi = self.bsc.mpi;
+          # gitBranch = "garlic/mpi+omp";
+
         };
 
 #        heat = callPackage ./garlic/apps/heat {
@@ -265,6 +281,7 @@ let
           mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
           omp = callPackage ./garlic/exp/hpcg/omp.nix { };
           mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
+          oss = callPackage ./garlic/exp/hpcg/oss.nix { };
         };
 
         test = {

From 22a294f9ccc79d0df9c87aee13971aafaad70401 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 23 Oct 2020 10:06:23 +0200
Subject: [PATCH 08/14] Forgot to set one task per node

---
 garlic/exp/hpcg/oss.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
index e30c6de..0cfc395 100644
--- a/garlic/exp/hpcg/oss.nix
+++ b/garlic/exp/hpcg/oss.nix
@@ -31,7 +31,7 @@ let
 
     # Resources
     qos = "debug";
-    ntasksPerNode = 48;
+    ntasksPerNode = 1;
     nodes = 1;
     time = "02:00:00";
     cpuBind = "sockets,verbose";

From b856e2147a31f7bad19dfec8b47fa12fef1c4a8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Tue, 27 Oct 2020 18:13:21 +0100
Subject: [PATCH 09/14] Use discrete deps in nanos6. Pass nblocks to omp
 version and use the same experiments as oss

---
 garlic/exp/hpcg/omp.nix | 9 ++++++---
 garlic/exp/hpcg/oss.nix | 1 +
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/garlic/exp/hpcg/omp.nix b/garlic/exp/hpcg/omp.nix
index bf3d59d..84f682c 100644
--- a/garlic/exp/hpcg/omp.nix
+++ b/garlic/exp/hpcg/omp.nix
@@ -11,23 +11,25 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 104 64 ];
+    n = [ 200 104 64 ];
+    nblocks = [ 128 ];
   };
 
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
     # hpcg options
     n = c.n;
+    nblocks = c.nblocks;
     cc = icc;
     mpi = null; # TODO: Remove this for omp
-    gitBranch = "garlic/seq";
+    gitBranch = "garlic/omp";
 
     # Repeat the execution of each unit 30 times
     loops = 30;
 
     # Resources
     qos = "debug";
-    ntasksPerNode = 48;
+    ntasksPerNode = 1;
     nodes = 1;
     time = "02:00:00";
     cpuBind = "sockets,verbose";
@@ -45,6 +47,7 @@ let
       "--nx=${toString n}"
       "--ny=${toString n}"
       "--nz=${toString n}"
+      "--nblocks=${toString nblocks}"
     ];
   };
 
diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
index 0cfc395..4f353bb 100644
--- a/garlic/exp/hpcg/oss.nix
+++ b/garlic/exp/hpcg/oss.nix
@@ -45,6 +45,7 @@ let
 
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
+    env = "NANOS6_DEPENDENCIES=discrete";
     argv = [
       "--nx=${toString n}"
       "--ny=${toString n}"

From 58e3d48a16ad5b1b6f9f254fad3b1ae9fcabded3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 30 Oct 2020 14:05:40 +0100
Subject: [PATCH 10/14] Use mask_cpu and n.x n.y n.z instead of n

---
 garlic/exp/hpcg/mpi+omp.nix | 26 ++++++++++++++++----------
 garlic/exp/hpcg/mpi.nix     | 21 +++++++++++----------
 garlic/exp/hpcg/omp.nix     | 21 +++++++++++++--------
 garlic/exp/hpcg/oss.nix     | 15 ++++++++-------
 garlic/exp/hpcg/serial.nix  | 14 +++++++-------
 5 files changed, 55 insertions(+), 42 deletions(-)

diff --git a/garlic/exp/hpcg/mpi+omp.nix b/garlic/exp/hpcg/mpi+omp.nix
index e4facc5..261979f 100644
--- a/garlic/exp/hpcg/mpi+omp.nix
+++ b/garlic/exp/hpcg/mpi+omp.nix
@@ -11,15 +11,16 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 104 64 ];
+    n = [ { x = 64; y = 64; z = 88; } ];
+    nblocks = [ 12 24 48 96 192 384 ];
   };
 
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
     # hpcg options
     n = c.n;
-    cc = icc;
-    mpi = impi;
+    cc = bsc.icc;
+    mpi = bsc.impi;
     gitBranch = "garlic/mpi+omp";
 
     # Repeat the execution of each unit 30 times
@@ -27,11 +28,12 @@ let
 
     # Resources
     qos = "debug";
-    ntasksPerNode = 48;
-    nodes = 1;
+    ntasksPerNode = 1;
+    nodes = 2;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
-    jobName = "hpcg-${toString n}-${gitBranch}";
+    # Each task in different socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
   # Compute the array of configurations
@@ -41,10 +43,14 @@ let
 
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
+    env = ''
+      OMP_PROC_BIND=true
+      OMP_NUM_THREADS=12
+    '';
     argv = [
-      "--nx=${toString n}"
-      "--ny=${toString n}"
-      "--nz=${toString n}"
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
     ];
   };
 
diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix
index 331bd91..d57dc88 100644
--- a/garlic/exp/hpcg/mpi.nix
+++ b/garlic/exp/hpcg/mpi.nix
@@ -11,15 +11,15 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 104 64 ];
+    n = [ { x = 64; y = 64; z = 88; } ];
   };
 
   # Generate the complete configuration for each unit
   genConf = with bsc; c: targetMachine.config // rec {
     # hpcg options
     n = c.n;
-    cc = icc;
-    mpi = impi;
+    cc = bsc.icc;
+    mpi = bsc.impi;
     gitBranch = "garlic/mpi";
 
     # Repeat the execution of each unit 30 times
@@ -27,11 +27,12 @@ let
 
     # Resources
     qos = "debug";
-    ntasksPerNode = 48;
-    nodes = 1;
+    ntasksPerNode = 1;
+    nodes = 24;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
-    jobName = "hpcg-${toString n}-${gitBranch}";
+    # Each task in different socket
+    cpuBind = "verbose,mask_cpu:0x1";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
   # Compute the array of configurations
@@ -42,9 +43,9 @@ let
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
     argv = [
-      "--nx=${toString n}"
-      "--ny=${toString n}"
-      "--nz=${toString n}"
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
     ];
   };
 
diff --git a/garlic/exp/hpcg/omp.nix b/garlic/exp/hpcg/omp.nix
index 84f682c..aca3a5b 100644
--- a/garlic/exp/hpcg/omp.nix
+++ b/garlic/exp/hpcg/omp.nix
@@ -11,8 +11,8 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 200 104 64 ];
-    nblocks = [ 128 ];
+    n = [ { x = 128; y = 256; z = 264; } ];
+    nblocks = [ 12 24 48 96 192 384 ];
   };
 
   # Generate the complete configuration for each unit
@@ -20,7 +20,7 @@ let
     # hpcg options
     n = c.n;
     nblocks = c.nblocks;
-    cc = icc;
+    cc = bsc.icc;
     mpi = null; # TODO: Remove this for omp
     gitBranch = "garlic/omp";
 
@@ -32,8 +32,9 @@ let
     ntasksPerNode = 1;
     nodes = 1;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
-    jobName = "hpcg-${toString n}-${gitBranch}";
+    # task in one socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
   # Compute the array of configurations
@@ -43,10 +44,14 @@ let
 
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
+    env = ''
+      OMP_PROC_BIND=true
+      OMP_NUM_THREADS=24
+    '';
     argv = [
-      "--nx=${toString n}"
-      "--ny=${toString n}"
-      "--nz=${toString n}"
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
       "--nblocks=${toString nblocks}"
     ];
   };
diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
index 4f353bb..6695023 100644
--- a/garlic/exp/hpcg/oss.nix
+++ b/garlic/exp/hpcg/oss.nix
@@ -11,8 +11,8 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 200 104 64 ];
-    nblocks = [ 128 ];
+    n = [ { x = 128; y = 256; z = 264; } ];
+    nblocks = [ 12 24 48 96 192 384 ];
   };
 
   # Generate the complete configuration for each unit
@@ -34,8 +34,9 @@ let
     ntasksPerNode = 1;
     nodes = 1;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
-    jobName = "hpcg-${toString n}-${gitBranch}";
+    # task in one socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
   # Compute the array of configurations
@@ -47,9 +48,9 @@ let
     inherit nextStage;
     env = "NANOS6_DEPENDENCIES=discrete";
     argv = [
-      "--nx=${toString n}"
-      "--ny=${toString n}"
-      "--nz=${toString n}"
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
       "--nblocks=${toString nblocks}"
     ];
   };
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
index 91ce793..90356ef 100644
--- a/garlic/exp/hpcg/serial.nix
+++ b/garlic/exp/hpcg/serial.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ 104 64 ];
+    n = [ { x = 128; y = 256; z = 264; } ];
   };
 
   # Generate the complete configuration for each unit
@@ -27,11 +27,11 @@ let
 
     # Resources
     qos = "debug";
-    ntasksPerNode = 48;
+    ntasksPerNode = 1;
     nodes = 1;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
-    jobName = "hpcg-${toString n}-${gitBranch}";
+    cpuBind = "verbose,mask_cpu:0x1";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
   # Compute the array of configurations
@@ -42,9 +42,9 @@ let
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
     argv = [
-      "--nx=${toString n}"
-      "--ny=${toString n}"
-      "--nz=${toString n}"
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
     ];
   };
 

From d757332448734e361e313390920d9f540e87683f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 30 Oct 2020 14:35:09 +0100
Subject: [PATCH 11/14] Remove extrae home

---
 garlic/exp/hpcg/extrae.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/garlic/exp/hpcg/extrae.xml b/garlic/exp/hpcg/extrae.xml
index b9af29b..45a7574 100644
--- a/garlic/exp/hpcg/extrae.xml
+++ b/garlic/exp/hpcg/extrae.xml
@@ -7,7 +7,6 @@
 
 <!-- Must we activate the tracing? Which is the tracing mode? (detail/bursts) Where is it located? Which kind of trace? Version of the XML parser?-->
 <trace enabled="yes"
- home="/nix/store/j80mlqa12d1baifg30jsx2smv90akzvc-extrae"
  initial-mode="detail"
  type="paraver"
 >

From 6a1375726f9b3371d70771e72701a9941df0dbe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 30 Oct 2020 14:44:33 +0100
Subject: [PATCH 12/14] Fix problem sizes to be equivalent between versions

---
 garlic/exp/hpcg/mpi+omp.nix | 8 ++++----
 garlic/exp/hpcg/mpi.nix     | 2 +-
 garlic/exp/hpcg/omp.nix     | 2 +-
 garlic/exp/hpcg/oss.nix     | 2 +-
 garlic/exp/hpcg/serial.nix  | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/garlic/exp/hpcg/mpi+omp.nix b/garlic/exp/hpcg/mpi+omp.nix
index 261979f..5605901 100644
--- a/garlic/exp/hpcg/mpi+omp.nix
+++ b/garlic/exp/hpcg/mpi+omp.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ { x = 64; y = 64; z = 88; } ];
+    n = [ { x = 128; y = 192; z = 192; } ];
     nblocks = [ 12 24 48 96 192 384 ];
   };
 
@@ -29,10 +29,10 @@ let
     # Resources
     qos = "debug";
     ntasksPerNode = 1;
-    nodes = 2;
+    nodes = 4;
     time = "02:00:00";
     # Each task in different socket
-    cpuBind = "verbose,mask_cpu:0xffffff";
+    cpuBind = "verbose,mask_cpu:0x3f";
     jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
   };
 
@@ -45,7 +45,7 @@ let
     inherit nextStage;
     env = ''
       OMP_PROC_BIND=true
-      OMP_NUM_THREADS=12
+      OMP_NUM_THREADS=6
     '';
     argv = [
       "--nx=${toString n.x}"
diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix
index d57dc88..a2a2ec4 100644
--- a/garlic/exp/hpcg/mpi.nix
+++ b/garlic/exp/hpcg/mpi.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ { x = 64; y = 64; z = 88; } ];
+    n = [ { x = 96; y = 96; z = 96; } ];
   };
 
   # Generate the complete configuration for each unit
diff --git a/garlic/exp/hpcg/omp.nix b/garlic/exp/hpcg/omp.nix
index aca3a5b..9960302 100644
--- a/garlic/exp/hpcg/omp.nix
+++ b/garlic/exp/hpcg/omp.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ { x = 128; y = 256; z = 264; } ];
+    n = [ { x = 256; y = 288; z = 288; } ];
     nblocks = [ 12 24 48 96 192 384 ];
   };
 
diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
index 6695023..5084504 100644
--- a/garlic/exp/hpcg/oss.nix
+++ b/garlic/exp/hpcg/oss.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ { x = 128; y = 256; z = 264; } ];
+    n = [ { x = 256; y = 288; z = 288; } ];
     nblocks = [ 12 24 48 96 192 384 ];
   };
 
diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix
index 90356ef..3c58b42 100644
--- a/garlic/exp/hpcg/serial.nix
+++ b/garlic/exp/hpcg/serial.nix
@@ -11,7 +11,7 @@ with stdenv.lib;
 let
   # Initial variable configuration
   varConf = with bsc; {
-    n = [ { x = 128; y = 256; z = 264; } ];
+    n = [ { x = 256; y = 288; z = 288; } ];
   };
 
   # Generate the complete configuration for each unit

From 56584c9e97e373baf9942b1d01ab5315c1c81917 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 30 Oct 2020 15:01:44 +0100
Subject: [PATCH 13/14] Remove the osu set of tests

---
 overlay.nix | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/overlay.nix b/overlay.nix
index 152ddf9..c695def 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -288,14 +288,6 @@ let
           exec = callPackage ./test/garlic/exec.nix {
             exec = self.bsc.garlic.stages.exec;
           };
-
-          osu = rec {
-            latency-internode = callPackage ./garlic/exp/osu/latency.nix { };
-            latency-intranode = callPackage ./garlic/exp/osu/latency.nix {
-              interNode = false;
-            };
-            latency = latency-internode;
-          };
         };
       };
     };

From 9c20537f911b5a4244f4ed2412e0a2b77fe64d7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= <rpenacob@bsc.es>
Date: Fri, 30 Oct 2020 15:10:47 +0100
Subject: [PATCH 14/14] Since mpi+omp version uses 6 threads, change nblocks
 values

---
 garlic/exp/hpcg/mpi+omp.nix | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/garlic/exp/hpcg/mpi+omp.nix b/garlic/exp/hpcg/mpi+omp.nix
index 5605901..dfe4696 100644
--- a/garlic/exp/hpcg/mpi+omp.nix
+++ b/garlic/exp/hpcg/mpi+omp.nix
@@ -12,7 +12,7 @@ let
   # Initial variable configuration
   varConf = with bsc; {
     n = [ { x = 128; y = 192; z = 192; } ];
-    nblocks = [ 12 24 48 96 192 384 ];
+    nblocks = [ 6 12 24 48 96 192 ];
   };
 
   # Generate the complete configuration for each unit
@@ -51,6 +51,7 @@ let
       "--nx=${toString n.x}"
       "--ny=${toString n.y}"
       "--nz=${toString n.z}"
+      "--nblocks=${toString nblocks}"
     ];
   };