WIP stage redesign

garlic/exp/nbody/common.nix

@@ -0,0 +1,74 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, stages
+, machineConf
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+  w = runWrappers;
+in
+{
+  /* Returns the path of the executable of a stage */
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}"
+    else "${stage}";
+
+  /* Takes a list of units and builds an experiment, after executing the
+  trebuchet and the isolate stages. Returns the trebuchet stage. */
+  buildExperiment = {units, conf, ...}: stage.trebuchet {
+    inherit (machineConf) nixPrefix;
+    nextStage = stage.isolate {
+      inherit (machineConf) nixPrefix;
+      nextStage = stage.experiment {
+        inherit units;
+      }
+    }
+  };
+
+  sbatch = {nextStage, conf, ...}: with conf; w.sbatch (
+    # Allow a user to define a custom reservation for the job in MareNostrum4,
+    # by setting the garlic.sbatch.reservation attribute in the 
+    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
+    # reservation is used. The user reservation may be overwritten by the
+    # experiment, if the reservation is set like with nodes or ntasksPerNode.
+    optionalAttrs (pkgs.config ? garlic.sbatch.reservation) {
+      inherit (pkgs.config.garlic.sbatch) reservation;
+    } // {
+      exclusive = true;
+      time = "02:00:00";
+      qos = "debug";
+      jobName = "nbody-tampi";
+      inherit nextStage nixPrefix nodes ntasksPerNode;
+    }
+  );
+
+  control = {nextStage, conf, ...}: stages.control {
+    inherit (conf) loops;
+    inherit nextStage;
+  };
+
+  srun = {nextStage, conf, ...}: stages.srun {
+    inherit (conf) nixPrefix cpuBind;
+    inherit nextStage;
+  };
+
+  isolate = {nextStage, conf, ...}: stages.isolate {
+    clusterName = machineConf.name;
+    inherit (conf) nixPrefix;
+    inherit nextStage;
+  };
+
+  stdStages = [
+    sbatch
+    isolate
+    control
+    srun
+    isolate
+  ];
+}

garlic/exp/nbody/test.nix

@@ -0,0 +1,75 @@
+{
+  stdenv
+, stdexp
+, pkgs
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Configurations for each unit (using the cartesian product)
+  confUnit = with bsc; {
+    blocksize = [ 1024 2048 ];
+  };
+
+  # Configuration for the complete experiment
+  confExperiment = with bsc; {
+    # nbody options
+    particles = 1024 * 4;
+    timesteps = 10;
+    cc = icc;
+    mpi = impi;
+    gitBranch = "garlic/mpi+send";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    ntasksPerNode = 2;
+    nodes = 1;
+    cpuBind = "sockets,verbose";
+  };
+
+  confMachine = targetMachine.config;
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    var = confUnit;
+    fixed = confMachine // confExperiment;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [ "-t" timesteps "-p" particles ];
+    env = "";
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+  program = {nextStage, conf, ...}: with conf;
+  let
+    # We set the mpi implementation to the one specified in the conf, so all
+    # packages in bsc will use that one.
+    customPkgs = stdexp.genPkgs (self: super: {
+      bsc = super.bsc // { mpi = conf.mpi; };
+    });
+  in
+  customPkgs.garlic.nbody.override {
+    inherit cc blocksize mpi gitBranch;
+  };
+
+  # Generate the experimental units
+  units = map (c: stages.unit {
+    conf = c;
+    stages = stdexp.stdStages ++ [ exec program ];
+  }) configs;
+
+in
+ 
+  elemAt units 0
+  #buildExperiment units

garlic/exp/stdexp.nix

@@ -0,0 +1,84 @@
+{
+  stdenv
+, config
+, stages
+, targetMachine
+, garlicTools
+}:
+
+with stdenv.lib;
+with garlicTools;
+
+let
+  machineConf = targetMachine.config;
+in
+rec {
+  /* Takes a list of units and builds an experiment, after executing the
+  trebuchet and the isolate stages. Returns the trebuchet stage. */
+  buildExperiment = {units, conf, ...}: stage.trebuchet {
+    inherit (machineConf) nixPrefix;
+    nextStage = stage.isolate {
+      inherit (machineConf) nixPrefix;
+      nextStage = stage.experiment {
+        inherit units;
+      };
+    };
+  };
+
+  /* Given an attrset of lists `var` and an attrset `fixed`, computes the
+    cartesian product of all combinations of `var` and prepends `fixed`
+    to each. */
+  buildConfigs = {fixed, var}:
+    map (c: fixed // c) (genConfigs var);
+
+  sbatch = {nextStage, conf, ...}: with conf; stages.sbatch (
+    # Allow a user to define a custom reservation for the job in MareNostrum4,
+    # by setting the garlic.sbatch.reservation attribute in the 
+    # ~/.config/nixpkgs/config.nix file. If the attribute is not set, no
+    # reservation is used. The user reservation may be overwritten by the
+    # experiment, if the reservation is set like with nodes or ntasksPerNode.
+    optionalAttrs (config ? garlic.sbatch.reservation) {
+      inherit (config.garlic.sbatch) reservation;
+    } // {
+      exclusive = true;
+      time = "02:00:00";
+      qos = "debug";
+      jobName = "nbody-tampi";
+      inherit nextStage nixPrefix nodes ntasksPerNode;
+    }
+  );
+
+  control = {nextStage, conf, ...}: stages.control {
+    inherit (conf) loops;
+    inherit nextStage;
+  };
+
+  srun = {nextStage, conf, ...}: stages.srun {
+    inherit (conf) nixPrefix cpuBind;
+    inherit nextStage;
+  };
+
+  isolate = {nextStage, conf, ...}: stages.isolate {
+    clusterName = machineConf.name;
+    inherit (conf) nixPrefix;
+    inherit nextStage;
+  };
+
+  stdStages = [
+    sbatch
+    isolate
+    control
+    srun
+    isolate
+  ];
+
+  # FIXME: Remove this hack and allow custom nixpkgs
+  bscOverlay = import ../../overlay.nix;
+  nixpkgs = import <nixpkgs>;
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+}

garlic/machines.nix

@@ -0,0 +1,17 @@
+{
+  stdenv
+}:
+
+{
+  # MareNostrum 4 configuration
+  mn4 = {
+    config = {
+      name = "mn4";
+      nixPrefix = "/gpfs/projects/bsc15/nix";
+      cachelineBytes = 64;
+      march = "skylake-avx512";
+      mtune = "skylake-avx512";
+    };
+    # TODO: Add the specific details for SLURM and the interconection here
+  };
+}

garlic/stages/argv.nix

@@ -1,30 +0,0 @@
-{
-  stdenv
-, bash
-}:
-
-{
-  program
-, env ? ""
-
-# bash array as string, example: argv=''(-f "file with spaces" -t 10)''
-, argv ? "()"
-}:
-
-stdenv.mkDerivation {
-  name = "argv";
-  preferLocalBuild = true;
-  phases = [ "installPhase" ];
-  installPhase = ''
-    cat > $out <<EOF
-    #!${bash}/bin/bash --norc
-    # Requires /nix to use bash
-    
-    ${env}
-
-    argv=${argv}
-    exec ${program} \''${argv[@]}
-    EOF
-    chmod +x $out
-  '';
-}

garlic/stages/control.nix

@@ -1,12 +1,15 @@
 {
   stdenv
+, garlicTools
 }:
 
 {
-  program
+  nextStage
 , loops ? 30
 }:
 
+with garlicTools;
+
 stdenv.mkDerivation {
   name = "control";
   preferLocalBuild = true;
@@ -16,7 +19,7 @@ stdenv.mkDerivation {
     cat > $out <<EOF
     #!/bin/sh
     for n in \$(seq 1 ${toString loops}); do
-      ${program}
+      ${stageProgram nextStage}
     done
     EOF
     chmod +x $out

garlic/stages/exec.nix

@@ -0,0 +1,31 @@
+{
+  stdenv
+, garlicTools
+}:
+
+{
+  nextStage
+, env ? ""
+, argv ? []
+}:
+
+with builtins;
+with garlicTools;
+
+let
+  argvString = concatStringsSep " " (map (e: toString e) argv);
+in
+stdenv.mkDerivation {
+  name = "exec";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+    ${env}
+
+    exec ${stageProgram nextStage} ${argvString}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/experiment.nix

@@ -1,9 +1,10 @@
 {
   stdenv
-, nixPrefix ? ""
 }:
 
-units:
+{
+  units:
+}
 
 with stdenv.lib;
 

garlic/stages/isolate/default.nix

@@ -3,15 +3,17 @@
 , nixtools
 , busybox
 , strace
+, garlicTools
 }:
 
 {
-  program
-, stage
+  nextStage
 , nixPrefix
 , clusterName
 }:
 
+with garlicTools;
+
 stdenv.mkDerivation {
   name = "isolate";
   preferLocalBuild = true;
@@ -23,8 +25,9 @@ stdenv.mkDerivation {
   src = ./.;
   dontPatchShebangs = true;
   programPath = "/bin/stage1";
-  inherit program nixPrefix clusterName nixtools busybox;
-  desc = "#  $out\n" + (if builtins.hasAttr "desc" stage then stage.desc else "");
+  inherit nixPrefix clusterName nixtools busybox;
+  program = stageProgram nextStage;
+  desc = "#  $out\n" + (if builtins.hasAttr "desc" nextStage then nextStage.desc else "");
   out = "$out";
   installPhase = ''
 

garlic/stages/sbatch.nix

@@ -2,10 +2,11 @@
   stdenv
 , numactl
 , slurm
+, garlicTools
 }:
 
 {
-  program
+  nextStage
 , jobName
 , chdirPrefix ? "."
 , nixPrefix ? ""
@@ -26,6 +27,8 @@
 }:
 
 with stdenv.lib;
+with garlicTools;
+
 let
 
   sbatchOpt = name: value: optionalString (value!=null)
@@ -69,7 +72,7 @@ stdenv.mkDerivation rec {
     + optionalString (extra!=null) extra
     +
     ''
-    exec ${nixPrefix}${program}
+    exec ${nixPrefix}${stageProgram nextStage}
     EOF
     
     cat > $out/${name} <<EOF

garlic/stages/srun.nix

@@ -1,23 +1,30 @@
 {
   stdenv
 , slurm
+, garlicTools
 }:
 {
-  program
-, nixPrefix ? ""
+  nextStage
+, cpuBind
+, nixPrefix
 , srunOptions ? ""
 }:
 
+with garlicTools;
+
 stdenv.mkDerivation rec {
   name = "srun";
-  preferLocalBuild = true;
   phases = [ "installPhase" ];
+  preferLocalBuild = true;
   dontPatchShebangs = true;
   installPhase = ''
     cat > $out <<EOF
     #!/bin/sh -ex
-    exec ${slurm}/bin/srun --mpi=pmi2 ${srunOptions} \
-      ${nixPrefix}${program}
+    exec ${slurm}/bin/srun \
+      --mpi=pmi2 \
+      --cpu-bind=${cpuBind} \
+      ${srunOptions} \
+      ${nixPrefix}${stageProgram nextStage}
     EOF
     chmod +x $out
   '';

garlic/stages/unit.nix

@@ -14,8 +14,8 @@ let
 
   dStages = foldr (stageFn: {conf, prevStage, stages}: {
     conf = conf;
-    prevStage = stageFn {stage=prevStage; conf=conf;};
-    stages = [ (stageFn {stage=prevStage; conf=conf;}) ] ++ stages;
+    prevStage = stageFn {nextStage=prevStage; conf=conf;};
+    stages = [ (stageFn {nextStage=prevStage; conf=conf;}) ] ++ stages;
   })
     {conf=conf; stages=[]; prevStage=null;} stages;
 

garlic/tools.nix

@@ -1,6 +1,10 @@
-let
-  lib = import <nixpkgs/lib>;
+{
+  stdenv
+}:
 
+with stdenv.lib;
+
+let
   gen = rec {
     # genAttrSets "a" ["hello" "world"]
     # [ { a = "hello"; } { a = "world"; } ]
@@ -16,18 +20,24 @@ let
 
     # mergeConfig [{e=1;}] {name="a"; value=[1 2]
     # [ { a = 1; e = 1; } { a = 2; e = 1; } ]
-    mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr));
+    mergeConfig = (arr: new: flatten ( map (x: addAttrSets new.name new.value x) arr));
 
     # genConfigs {a=[1 2]; b=[3 4];}
     # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ]
-    genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config));
+    genConfigs = (config: foldl mergeConfig [{}] (attrToList config));
 
     # Generate multiple app versions by override with each config
     genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs);
 
     # Generate app version from an array of apps
     genApps = (apps: configs:
-      lib.flatten (map (app: genApp app configs) apps));
+      flatten (map (app: genApp app configs) apps));
+
+    /* Returns the path of the executable of a stage */
+    stageProgram = stage:
+      if stage ? programPath
+      then "${stage}${stage.programPath}"
+      else "${stage}";
 
   };
 in

overlay.nix

@@ -140,13 +140,20 @@ let
       nixPrefix = "/gpfs/projects/bsc15/nix";
     };
 
+    garlicTools = callPackage ./garlic/tools.nix {};
+
     garlic = {
+      # Configuration for the machines
+      machines = callPackage ./garlic/machines.nix {};
+
+      # Use the configuration for the following target machine
+      targetMachine = self.garlic.machines.mn4;
 
       # Load some helper functions to generate app variants
-      inherit (import ./garlic/gen.nix) genApps genApp genConfigs;
 
-      # Override the hardening flags and parallel build by default (TODO)
-      #mkDerivation = callPackage ./garlic/mkDerivation.nix { };
+      stdexp = callPackage ./garlic/exp/stdexp.nix {
+        inherit (self.garlic) targetMachine stages;
+      };
 
       # Apps for Garlic
 #      heat = callPackage ./garlic/heat {
@@ -187,18 +194,18 @@ let
         gitBranch = "garlic/seq";
       };
 
-      saiph = callPackage ./garlic/saiph {
+      saiph = callPackage ./garlic/saiph/default.nix {
         cc = self.bsc.clangOmpss2;
       };
 
-      # Execution wrappers
-      runWrappers = {
+      # Execution stages
+      stages = {
         sbatch     = callPackage ./garlic/stages/sbatch.nix { };
         srun       = callPackage ./garlic/stages/srun.nix { };
         launch     = callPackage ./garlic/stages/launcher { };
         control    = callPackage ./garlic/stages/control.nix { };
         nixsetup   = callPackage ./garlic/stages/nix-setup.nix { };
-        argv      = callPackage ./garlic/stages/argv.nix { };
+        exec       = callPackage ./garlic/stages/exec.nix { };
         statspy    = callPackage ./garlic/stages/statspy.nix { };
         extrae     = callPackage ./garlic/stages/extrae.nix { };
         stagen     = callPackage ./garlic/stages/stagen.nix { };
@@ -210,7 +217,7 @@ let
         trebuchet  = callPackage ./garlic/stages/trebuchet.nix { };
         strace     = callPackage ./garlic/stages/strace.nix { };
         unit       = callPackage ./garlic/stages/unit.nix { };
-        experiment= callPackage ./garlic/stages/experiment.nix { };
+        experiment = callPackage ./garlic/stages/experiment.nix { };
       };
 
       # Tests (move to bsc ?)
@@ -231,7 +238,7 @@ let
             nixpkgs = import <nixpkgs>;
             genApp = self.bsc.garlic.genApp;
             genConfigs = self.bsc.garlic.genConfigs;
-            runWrappers = self.bsc.garlic.runWrappers;
+            stages = self.bsc.garlic.stages;
           };
 
           tampi = callPackage ./garlic/exp/nbody/tampi.nix {
@@ -239,7 +246,12 @@ let
             nixpkgs = import <nixpkgs>;
             genApp = self.bsc.garlic.genApp;
             genConfigs = self.bsc.garlic.genConfigs;
-            runWrappers = self.bsc.garlic.runWrappers;
+            stages = self.bsc.garlic.stages;
+          };
+
+          test = callPackage ./garlic/exp/nbody/test.nix {
+            pkgs = self // self.bsc.garlic;
+            inherit (self.garlic) stdexp targetMachine stages;
           };
 #          mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { };
         };
@@ -250,7 +262,7 @@ let
             nixpkgs = import <nixpkgs>;
             genApp = self.bsc.garlic.genApp;
             genConfigs = self.bsc.garlic.genConfigs;
-            runWrappers = self.bsc.garlic.runWrappers;
+            stages = self.bsc.garlic.stages;
           };
         };
 
@@ -261,14 +273,14 @@ let
               nixpkgs = import <nixpkgs>;
               genApp = self.bsc.garlic.genApp;
               genConfigs = self.bsc.garlic.genConfigs;
-              runWrappers = self.bsc.garlic.runWrappers;
+              stages = self.bsc.garlic.stages;
             };
             hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix {
               pkgs = self // self.bsc.garlic;
               nixpkgs = import <nixpkgs>;
               genApp = self.bsc.garlic.genApp;
               genConfigs = self.bsc.garlic.genConfigs;
-              runWrappers = self.bsc.garlic.runWrappers;
+              stages = self.bsc.garlic.stages;
             };
           };
         };
@@ -287,12 +299,18 @@ let
             nixpkgs = import <nixpkgs>;
             genApp = self.bsc.garlic.genApp;
             genConfigs = self.bsc.garlic.genConfigs;
-            runWrappers = self.bsc.garlic.runWrappers;
+            stages = self.bsc.garlic.stages;
           };
 #          mpi = callPackage ./bsc/garlic/exp/nbody/mpi.nix { };
         };
       };
     };
+
+    test = {
+      exec = callPackage ./test/garlic/exec.nix {
+        exec = self.bsc.garlic.stages.exec;
+      };
+    };
   };
 
 in