From 09c2b9005a6d19dfbe545ee73c7c2271149708e7 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Tue, 25 Aug 2020 18:39:31 +0200
Subject: [PATCH] Testing nbody blocksize with impi

Weird run times with srun: Two exceed 20%. Relative times:

0.998649        0.998936        0.999409        1.00018         1.00191
0.998684        0.998936        0.999432        1.00041         1.00222
0.998776        0.999065        0.999527        1.00126         1.0024
0.998786        0.999084        0.999558        1.00138         1.00242
0.998856        0.999102        0.999727        1.00155         1.25585
0.998895        0.9992          0.999849        1.0018          1.27138
---
 bsc/garlic/exp/nbody/bs.nix | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/bsc/garlic/exp/nbody/bs.nix b/bsc/garlic/exp/nbody/bs.nix
index 0b41d34..088d49a 100644
--- a/bsc/garlic/exp/nbody/bs.nix
+++ b/bsc/garlic/exp/nbody/bs.nix
@@ -17,14 +17,18 @@ let
   # Set the configuration for the experiment
   config = {
     cc = [ bsc.icc ];
-    blocksize = [ 1024 2048 4096 8192 ];
+    blocksize = [ 1024 ];
   };
 
   extraConfig = {
-    particles = 16384;
+    gitBranch = "garlic/mpi+send";
+    mpi = bsc.impi;
+    particles = 1024*128;
     timesteps = 10;
-    ntasks = 1;
-    nnodes = 1;
+    ntasksPerNode = "48";
+    nodes = "1";
+    time = "02:00:00";
+    qos = "debug";
   };
 
   # Compute the cartesian product of all configurations
@@ -32,11 +36,11 @@ let
   filteredConfigs = with builtins; filter (c: c.blocksize <= 4096) allConfigs;
   configs = map (conf: conf // extraConfig) filteredConfigs;
 
-  sbatch = conf: app: sbatchWrapper {
+  sbatch = conf: app: with conf; sbatchWrapper {
     app = app;
     nixPrefix = "/gpfs/projects/bsc15/nix";
-    exclusive = false;
-    ntasks = "${toString conf.ntasks}";
+    exclusive = true;
+    inherit ntasksPerNode nodes time qos;
   };
 
   srun = app: srunWrapper {
@@ -48,20 +52,25 @@ let
     with conf;
     argvWrapper {
       app = app;
+      env = ''
+        set -e
+        export I_MPI_THREAD_SPLIT=1
+      '';
       argv = ''(-t ${toString timesteps} -p ${toString particles})'';
     };
 
   nbodyFn = conf:
     with conf;
-    nbody.override { inherit cc blocksize; };
+    nbody.override { inherit cc mpi blocksize gitBranch; };
 
   pipeline = conf:
     sbatch conf (
-      srun (
-        nixsetupWrapper (
-          controlWrapper (
-            argv conf (
-              nbodyFn conf)))));
+      nixsetupWrapper (
+        controlWrapper (
+          srun (
+            nixsetupWrapper (
+              argv conf (
+                nbodyFn conf))))));
 
   # Ideally it should look like this:
   #pipeline = sbatch nixsetup control argv nbodyFn;