First attempt to create an experiment with CREAMS: strong scaling from 1 to 16 nodes using the pure MPI version

garlic/creams/default.nix

@@ -2,23 +2,39 @@
   stdenv
 , nanos6
 , mpi
+, openmpi
+, impi
 , tampi
 , mcxx
-, icc
+, cc
+, gitBranch
 }:
 
+assert (mpi == impi || mpi == openmpi);
+
+let
+  mpiName = (if mpi == openmpi then
+    "OpenMPI"
+  else
+    "IntelMPI");
+
+in
 stdenv.mkDerivation rec {
   name = "creams";
 
+  # src = /home/Computational/pmartin1/creams-simplified;
+
   src = builtins.fetchGit {
     url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git";
-    ref = "MPI+OmpSs-2+TAMPI";
+    ref = "${gitBranch}";
   };
 
+  programPath = "/bin/creams.exe";
+
   buildInputs = [
     nanos6
     mpi
-    icc
+    cc
     tampi
     mcxx
   ];
@@ -27,12 +43,18 @@ stdenv.mkDerivation rec {
 
   configurePhase = ''
     export TAMPI_HOME=${tampi}
+
     . etc/bashrc
+    #export FORTRAN_COMPILER=GNU # GCC compiler
+    export MPI_LIB=${mpiName}
+
+    echo
+
+    CREAMS_UPDATE_ENVIRONMENT
   '';
 
   installPhase = ''
     mkdir -p $out/bin
     cp -a build/* $out/bin
   '';
-
 }

garlic/creams/input.nix

@@ -0,0 +1,27 @@
+{
+  stdenv
+, nodes
+, gitBranch
+}:
+
+stdenv.mkDerivation rec {
+  name = "creams-input";
+
+  # src = /home/Computational/pmartin1/creams-simplified;
+
+  src = builtins.fetchGit {
+    url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git";
+    ref = "${gitBranch}";
+  };
+
+  phases = [ "unpackPhase" "installPhase" ];
+  
+  installPhase = ''
+    pushd SodTubeBenchmark
+      bash gridScript.sh 0 0 $((${toString nodes}*48)) 0
+    popd
+
+    mkdir -p $out
+    cp -a SodTubeBenchmark $out/
+  '';
+}

garlic/exp/creams/SS+mpi+send+seq.nix

@@ -0,0 +1,173 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    # cc  = [ self.gcc10  ]; # Does not work
+    # mpi = [ bsc.openmpi ]; # Does not work
+    # cc  = [ bsc.icc  ];
+    # mpi = [ bsc.impi ];
+
+    nodes = [ 1 2 4 8 16 ];
+  };
+
+  # Common configuration
+  common = {
+    gitBranch = "garlic/mpi+send+seq";
+
+    cc  = bsc.icc ;
+    mpi = bsc.impi;
+
+    # Resources
+    ntasksPerNode   = 48;
+    #ntasksPerSocket = 24; // Add this variable to nix
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch {
+    program = stageProgram stage;
+    exclusive = true;
+    time = "02:00:00";
+    qos = "debug";
+    jobName = "nbody-bs";
+    inherit nixPrefix nodes ntasksPerNode;
+  };
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  inputDataset = {stage, conf, ...}: with conf;
+  let
+    input = bsc.garlic.creamsInput.override {
+      inherit gitBranch nodes;
+    };
+  in w.argv
+  {
+    program = stageProgram stage;
+    env = ''
+      cp -r ${input}/SodTubeBenchmark/* .
+
+      pwd
+      ls -l
+    '';
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  creamsFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.creams.override {
+      inherit cc mpi gitBranch;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ inputDataset creamsFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs

garlic/exp/creams/test.nix

@@ -0,0 +1,173 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    # cc  = [ self.gcc10  ]; # Does not work
+    # mpi = [ bsc.openmpi ]; # Does not work
+    # cc  = [ bsc.icc  ];
+    # mpi = [ bsc.impi ];
+
+    nodes = [ 1 2 4 8 16 ];
+  };
+
+  # Common configuration
+  common = {
+    gitBranch = "garlic/mpi+send+seq";
+
+    cc  = bsc.icc ;
+    mpi = bsc.impi;
+
+    # Resources
+    ntasksPerNode   = 48;
+    #ntasksPerSocket = 24; // Add this variable to nix
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch {
+    program = stageProgram stage;
+    exclusive = true;
+    time = "02:00:00";
+    qos = "debug";
+    jobName = "nbody-bs";
+    inherit nixPrefix nodes ntasksPerNode;
+  };
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  inputDataset = {stage, conf, ...}: with conf;
+  let
+    input = bsc.garlic.creamsInput.override {
+      inherit gitBranch nodes;
+    };
+  in w.argv
+  {
+    program = stageProgram stage;
+    env = ''
+      cp -r ${input}/SodTubeBenchmark/* .
+
+      pwd
+      ls -l
+    '';
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  creamsFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.creams.override {
+      inherit cc mpi gitBranch;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ inputDataset creamsFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs

garlic/exp/creams/test.nix~

@@ -0,0 +1,169 @@
+{
+  stdenv
+, nixpkgs
+, pkgs
+, genApp
+, genConfigs
+, runWrappers
+}:
+
+with stdenv.lib;
+
+let
+  bsc = pkgs.bsc;
+
+  # Set variable configuration for the experiment
+  varConfig = {
+    mpi = [ bsc.impi bsc.openmpi ];
+    nodes = [ 1 ];
+  };
+
+  # Common configuration
+  common = {
+    # Compile time nbody config
+    gitBranch = "garlic/mpi+send+seq";
+
+    cc = bsc.icc;
+
+    # Resources
+    ntasksPerNode = 48;
+    nodes = 1;
+
+    # Stage configuration
+    enableSbatch = true;
+    enableControl = true;
+    enableExtrae = false;
+    enablePerf = false;
+    enableCtf = false;
+
+    # MN4 path
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+  };
+
+  # Compute the cartesian product of all configurations
+  configs = map (conf: conf // common) (genConfigs varConfig);
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  w = runWrappers;
+
+  sbatch = {stage, conf, ...}: with conf; w.sbatch {
+    program = stageProgram stage;
+    exclusive = true;
+    time = "02:00:00";
+    qos = "debug";
+    jobName = "nbody-bs";
+    inherit nixPrefix nodes ntasksPerNode;
+  };
+
+  control = {stage, conf, ...}: with conf; w.control {
+    program = stageProgram stage;
+  };
+
+  srun = {stage, conf, ...}: with conf; w.srun {
+    program = stageProgram stage;
+    srunOptions = "--cpu-bind=verbose,rank";
+    inherit nixPrefix;
+  };
+
+  statspy = {stage, conf, ...}: with conf; w.statspy {
+    program = stageProgram stage;
+  };
+
+  perf = {stage, conf, ...}: with conf; w.perf {
+    program = stageProgram stage;
+    perfArgs = "sched record -a";
+  };
+
+  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+    program = stageProgram stage;
+    nixsetup = "${nixPrefix}/bin/nix-setup";
+  };
+
+  extrae = {stage, conf, ...}: w.extrae {
+    program = stageProgram stage;
+    traceLib = "mpi"; # mpi -> libtracempi.so
+    configFile = ./extrae.xml;
+  };
+
+  ctf = {stage, conf, ...}: w.argv {
+    program = stageProgram stage;
+    env = ''
+      export NANOS6=ctf
+      export NANOS6_CTF2PRV=0
+    '';
+  };
+
+  bscOverlay = import ../../../overlay.nix;
+
+  genPkgs = newOverlay: nixpkgs {
+    overlays = [
+      bscOverlay
+      newOverlay
+    ];
+  };
+
+  inputDataset = {stage, conf, ...}: with conf;
+  let
+    input = bsc.garlic.creamsInput.override {
+      inherit gitBranch nodes;
+    };
+  in w.argv
+  {
+    program = stageProgram stage;
+    env = ''
+      cp -r ${input}/SodTubeBenchmark/* .
+
+      pwd
+      ls -l
+    '';
+  };
+
+  # We may be able to use overlays by invoking the fix function directly, but we
+  # have to get the definition of the bsc packages and the garlic ones as
+  # overlays.
+
+  creamsFn = {stage, conf, ...}: with conf;
+    let
+      # We set the mpi implementation to the one specified in the conf, so all
+      # packages in bsc will use that one.
+      customPkgs = genPkgs (self: super: {
+        bsc = super.bsc // { mpi = conf.mpi; };
+      });
+    in
+    customPkgs.bsc.garlic.creams.override {
+      inherit cc mpi gitBranch;
+    };
+
+  stages = with common; []
+    # Use sbatch to request resources first
+    ++ optional enableSbatch sbatch
+
+    # Repeats the next stages N times
+    ++ optionals enableControl [ nixsetup control ]
+
+    # Executes srun to launch the program in the requested nodes, and
+    # immediately after enters the nix environment again, as slurmstepd launches
+    # the next stages from outside the namespace.
+    ++ [ srun nixsetup ]
+
+    # Intrumentation with extrae
+    ++ optional enableExtrae extrae
+
+    # Optionally profile the next stages with perf
+    ++ optional enablePerf perf
+
+    # Optionally profile nanos6 with the new ctf
+    ++ optional enableCtf ctf
+
+    # Execute the nbody app with the argv and env vars
+    ++ [ inputDataset creamsFn ];
+
+  # List of actual programs to be executed
+  jobs = map (conf: w.stagen { inherit conf stages; }) configs;
+
+in
+  # We simply run each program one after another
+  w.launch jobs

garlic/stages/sbatch.nix

@@ -28,7 +28,7 @@ with stdenv.lib;
 let
 
   sbatchOpt = name: value: optionalString (value!=null)
-    "#SBATCH --${name}=${value}\n";
+    "#SBATCH --${name}=${toString value}\n";
   sbatchEnable = name: value: optionalString (value!=null)
     "#SBATCH --${name}\n";
 

overlay.nix

@@ -143,14 +143,21 @@ let
 #        tampi = tampi;
 #      };
 #
-#      creams = callPackage ./garlic/creams {
-#        stdenv = pkgs.gcc9Stdenv;
-#        mpi = intel-mpi;
-#        tampi = tampi.override {
-#          mpi = intel-mpi;
-#        };
-#      };
-#
+      creams = callPackage ./garlic/creams {
+        gitBranch = "garlic/mpi+send+seq";
+
+        #cc = self.gcc10;            # Does not work
+        #mpi = self.bsc.openmpi-mn4; # Does not work
+
+        cc  = self.bsc.icc;
+        mpi = self.bsc.mpi;
+      };
+
+      creamsInput = callPackage ./garlic/creams/input.nix {
+        gitBranch = "garlic/mpi+send+seq";
+        nodes = 1;
+      };
+
 #      lulesh = callPackage ./garlic/lulesh {
 #        mpi = intel-mpi;
 #      };
@@ -231,6 +238,16 @@ let
           };
         };
 
+        creams = {
+          SS_mpi_send_seq = callPackage ./garlic/exp/creams/SS+mpi+send+seq.nix {
+            pkgs = self // self.bsc.garlic;
+            nixpkgs = import <nixpkgs>;
+            genApp = self.bsc.garlic.genApp;
+            genConfigs = self.bsc.garlic.genConfigs;
+            runWrappers = self.bsc.garlic.runWrappers;
+          };
+        };
+
         osu = rec {
           latency-internode = callPackage ./garlic/exp/osu/latency.nix { };
           latency-intranode = callPackage ./garlic/exp/osu/latency.nix {