From 6973f48638c437b40c250286d2c8448d1c308719 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Wed, 3 Mar 2021 12:33:59 +0100
Subject: [PATCH] osu: add an experiment for Intel MPI tunning

---
 garlic/exp/index.nix    |  1 +
 garlic/exp/osu/impi.nix | 68 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 garlic/exp/osu/impi.nix

diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix
index 6d4c2a5..955f735 100644
--- a/garlic/exp/index.nix
+++ b/garlic/exp/index.nix
@@ -101,6 +101,7 @@
     latencyMt = latency.override { enableMultithread = true; };
     latencyMtShm = latency.override { enableMultithread = true; interNode = true; };
     bw = callPackage ./osu/bw.nix { };
+    impi = callPackage ./osu/impi.nix { };
     bwShm = bw.override { interNode = false; };
   };
 }
diff --git a/garlic/exp/osu/impi.nix b/garlic/exp/osu/impi.nix
new file mode 100644
index 0000000..7123560
--- /dev/null
+++ b/garlic/exp/osu/impi.nix
@@ -0,0 +1,68 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+
+# Should we test the network (true) or the shared memory (false)?
+, interNode ? true
+}:
+
+with builtins;
+with stdenv.lib;
+
+let
+
+  machineConfig = targetMachine.config;
+
+  # Initial variable configuration
+  varConf = with bsc; {
+    threshold = [ 8000 16000 32000 64000 ];
+    #threshold = [ 4096 8192 10240 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    inherit (machineConfig) hw;
+    nodes = if interNode then 2 else 1;
+    ntasksPerNode = if interNode then 1 else 2;
+    mpi = impi;
+    cpusPerTask = 1;
+    time = "00:10:00";
+    qos = "debug";
+    loops = 10;
+    expName = "osu-impi-rndv";
+    unitName = expName + "-${toString threshold}";
+    jobName = expName;
+    inherit (c) threshold;
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    env = ''
+      export PSM2_MQ_RNDV_SHM_THRESH=${toString conf.threshold}
+      export PSM2_MQ_RNDV_HFI_THRESH=${toString conf.threshold}
+      export PSM2_MQ_EAGER_SDMA_SZ=${toString conf.threshold}
+      #export PSM2_MTU=${toString conf.threshold}
+      export PSM2_TRACEMASK=0x101
+    '';
+
+    program = "${nextStage}/bin/osu_bw";
+  };
+
+  program = {nextStage, conf, ...}: bsc.osumb.override {
+    # Use the specified MPI implementation
+    inherit (conf) mpi;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }