From 14684040a526fd9b3a257feb85062d16202dc08f Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Wed, 19 Aug 2020 11:07:21 +0200
Subject: [PATCH] Intra/inter node latency tests

---
 bsc/garlic/default.nix         |  8 ++++++--
 bsc/garlic/exp/osu/latency.nix | 14 ++++++++++----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix
index 38d7e73..8e96d65 100644
--- a/bsc/garlic/default.nix
+++ b/bsc/garlic/default.nix
@@ -36,8 +36,12 @@ let
         bs = callPackage ./exp/nbody/bs.nix { };
         mpi = callPackage ./exp/nbody/mpi.nix { };
       };
-      osu = {
-        latency = callPackage ./exp/osu/latency.nix { };
+      osu = rec {
+        latency-internode = callPackage ./exp/osu/latency.nix { };
+        latency-intranode = callPackage ./exp/osu/latency.nix {
+          interNode = false;
+        };
+        latency = latency-internode;
       };
     };
   };
diff --git a/bsc/garlic/exp/osu/latency.nix b/bsc/garlic/exp/osu/latency.nix
index f3a0145..3a3a777 100644
--- a/bsc/garlic/exp/osu/latency.nix
+++ b/bsc/garlic/exp/osu/latency.nix
@@ -10,6 +10,12 @@
 , argvWrapper
 , controlWrapper
 , nixsetupWrapper
+
+# Should we test the network (true) or the shared memory (false)?
+, interNode ? true
+
+# Enable multiple threads?
+, multiThread ? false
 }:
 
 let
@@ -19,8 +25,8 @@ let
   };
 
   extraConfig = {
-    ntasksPerNode = 1;
-    nodes = 2;
+    nodes = if interNode then 2 else 1;
+    ntasksPerNode = if interNode then 1 else 2;
     time = "00:10:00";
     qos = "debug";
   };
@@ -31,7 +37,7 @@ let
   sbatch = conf: app: sbatchWrapper {
     app = app;
     nixPrefix = "/gpfs/projects/bsc15/nix";
-    exclusive = false;
+    exclusive = true;
     ntasksPerNode = "${toString conf.ntasksPerNode}";
     nodes = "${toString conf.nodes}";
     time = conf.time;
@@ -58,7 +64,7 @@ let
     with conf;
     bsc.osumb.override { inherit mpi; };
 
-  pipeline = conf: srun (nixsetupWrapper (argv (osumbFn conf)));
+  pipeline = conf: sbatch conf (srun (nixsetupWrapper (argv (osumbFn conf))));
   #pipeline = conf: sbatch conf (srun (nixsetupWrapper (argv bsc.osumb)));
 
   # Ideally it should look like this: