From 847b5b3e0a3df859770e4f6cc85cda8767935b99 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Thu, 3 Sep 2020 16:19:52 +0200
Subject: [PATCH] Add noise experiment with nbody

---
 bsc/garlic/default.nix   |   1 +
 bsc/garlic/exp/noise.nix | 132 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 133 insertions(+)
 create mode 100644 bsc/garlic/exp/noise.nix

diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix
index 29cc8b3..ec6c1f3 100644
--- a/bsc/garlic/default.nix
+++ b/bsc/garlic/default.nix
@@ -44,6 +44,7 @@ let
     };
 
     exp = {
+      noise = callPackage ./exp/noise.nix { };
       nbody = {
         bs = callPackage ./exp/nbody/bs.nix { };
         mpi = callPackage ./exp/nbody/mpi.nix { };
diff --git a/bsc/garlic/exp/noise.nix b/bsc/garlic/exp/noise.nix
new file mode 100644
index 0000000..b3e9444
--- /dev/null
+++ b/bsc/garlic/exp/noise.nix
@@ -0,0 +1,132 @@
+{
+  bsc
+, stdenv
+, nbody
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  # Set variable configuration for the experiment
+  varConfig = {
+    cc = [ bsc.icc ];
+    blocksize = [ 1024 ];
+  };
+
+  # Common configuration
+  common = {
+    # Compile time nbody config
+    gitBranch = "garlic/mpi+send";
+    mpi = bsc.impi;
+
+    # nbody runtime options
+    particles = 1024*128;
+    timesteps = 20;
+    loops = 1000;
+
+    # Resources
+    ntasksPerNode = "48";
+    nodes = "1";
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch {
+    program = stageProgram stage;
+    exclusive = true;
+    time = "02:00:00";
+    qos = "debug";
+    jobName = "nbody-bs";
+    inherit nixPrefix nodes ntasksPerNode;
+  };
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+    inherit loops;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  argv = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      set -e
+      export I_MPI_THREAD_SPLIT=1
+    '';
+    argv = ''( -t ${toString conf.timesteps}
+      -p ${toString conf.particles} )'';
+  };
+
+  nbodyFn = {stage, conf, ...}: with conf; nbody.override {
+    inherit cc blocksize mpi gitBranch;
+  };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ argv nbodyFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs