From 966606b62dab93ac4a8c818b8a77cea10808f5bd Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Mon, 9 Nov 2020 17:47:55 +0100
Subject: [PATCH] hpcg: precompute the input

---
 garlic/exp/hpcg/gen.nix | 103 ++++++++++++++++++++++++++++++++++++++++
 garlic/exp/hpcg/oss.nix |  13 +++--
 overlay.nix             |   7 ++-
 3 files changed, 119 insertions(+), 4 deletions(-)
 create mode 100644 garlic/exp/hpcg/gen.nix

diff --git a/garlic/exp/hpcg/gen.nix b/garlic/exp/hpcg/gen.nix
new file mode 100644
index 0000000..4542b78
--- /dev/null
+++ b/garlic/exp/hpcg/gen.nix
@@ -0,0 +1,103 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+, garlicTools
+, resultFromTrebuchet
+}:
+
+with stdenv.lib;
+with builtins;
+with garlicTools;
+
+let
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    expName = "${c.expName}.gen";
+    unitName = "${expName}.n${toString n.x}";
+
+    # hpcg options
+    cc = bsc.icc;
+    mcxx = bsc.mcxx;
+    nanos6 = bsc.nanos6;
+    mpi = null; # TODO: Remove this for oss
+
+    # Only the n and gitBranch options are inherited
+    inherit (c) n gitBranch;
+
+    # Repeat the execution of each unit 30 times
+    loops = 1;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 1;
+    time = "02:00:00";
+    # task in one socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = unitName;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = "NANOS6_DEPENDENCIES=discrete";
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+      # The nblocks is ignored
+      #"--nblocks=${toString nblocks}"
+      # Store the results in the same directory
+      "--store=."
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc nanos6 mcxx gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+  genExp = configs: stdexp.genExperiment { inherit configs pipeline; };
+    
+  genInputLink = inputConfigs: {nextStage, conf, ...}:
+  let
+    # Compute the experiment that produces HPCG input matrix from the
+    # configuration of this unit:
+    configs = map genConf inputConfigs;
+    inputTre = genExp configs;
+    #inputExp = getExperimentStage inputTrebuchet;
+    #inputExp = trace inputTrebuchet inputTrebuchet.nextStage;
+    inputExp = trace (inputTre.name) (getExperimentStage inputTre);
+    # Then load the result. This is only used to ensure that we have the
+    # results, so it has been executed.
+    inputRes = resultFromTrebuchet inputTre;
+    # We also need the unit, to compute the path.
+    inputUnit = stages.unit {
+      conf = genConf conf;
+      stages = pipeline;
+    };
+    # Build the path:
+    expName = baseNameOf (toString inputExp);
+    unitName = baseNameOf (toString inputUnit);
+    relPath = "../../${expName}/${unitName}/1";
+  in stages.exec {
+    inherit nextStage;
+    env = ''
+      # This line ensures that the results of the HPCG generation are complete:
+      # ${inputRes}
+
+      # Then we simply link the input result directory in "input"
+      ln -s ${relPath} input
+    '';
+  };
+
+in
+  #{ inherit genConf genExp genInputLink; }
+  genInputLink
diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix
index fce3d2c..002bf12 100644
--- a/garlic/exp/hpcg/oss.nix
+++ b/garlic/exp/hpcg/oss.nix
@@ -4,6 +4,7 @@
 , bsc
 , targetMachine
 , stages
+, genInput
 }:
 
 with stdenv.lib;
@@ -13,8 +14,8 @@ let
   varConf = with bsc; {
     # FIXME: Temporally reduce the input size until we can load a precomputed
     # input in each run, otherwise the execution time is very large.
-    n = [ { x = 104; y = 104; z = 104; } ];
-    #n = [ { x = 256; y = 288; z = 288; } ];
+    #n = [ { x = 104; y = 104; z = 104; } ];
+    n = [ { x = 256; y = 288; z = 288; } ];
     nblocks = [ 12 24 48 96 192 384 ];
   };
 
@@ -50,6 +51,8 @@ let
     inherit varConf genConf;
   };
 
+  input = genInput configs;
+
   exec = {nextStage, conf, ...}: with conf; stages.exec {
     inherit nextStage;
     env = "NANOS6_DEPENDENCIES=discrete";
@@ -58,6 +61,9 @@ let
       "--ny=${toString n.y}"
       "--nz=${toString n.z}"
       "--nblocks=${toString nblocks}"
+      # The input symlink is generated by the input stage, which is generated by
+      # the genInput function.
+      "--load=input"
     ];
   };
 
@@ -69,8 +75,9 @@ let
       inherit cc nanos6 mcxx gitBranch;
     };
 
-  pipeline = stdexp.stdPipeline ++ [ exec program ];
+  pipeline = stdexp.stdPipeline ++ [ input exec program ];
 
 in
 
+  #{ inherit configs pipeline; }
   stdexp.genExperiment { inherit configs pipeline; }
diff --git a/overlay.nix b/overlay.nix
index 03a2fa7..fb87688 100644
--- a/overlay.nix
+++ b/overlay.nix
@@ -338,7 +338,12 @@ let
           mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
           omp = callPackage ./garlic/exp/hpcg/omp.nix { };
           mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
-          oss = callPackage ./garlic/exp/hpcg/oss.nix { };
+          input = callPackage ./garlic/exp/hpcg/gen.nix {
+            inherit (self.bsc.garlic.pp) resultFromTrebuchet;
+          };
+          oss = callPackage ./garlic/exp/hpcg/oss.nix {
+            genInput = self.bsc.garlic.exp.hpcg.input;
+          };
         };
 
         heat = {