From df18435dfc59781aebd2ca76524cdde4e281d5ff Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Wed, 12 Aug 2020 14:00:04 +0200
Subject: [PATCH] Provide argvWrapper

---
 bsc/garlic/argv.nix          | 25 +++++++++++++
 bsc/garlic/default.nix       |  9 +++--
 bsc/garlic/exp/nbody/bs.nix  | 71 ++++++++++++++++++++++++------------
 bsc/garlic/launcher.nix      | 13 ++++++-
 bsc/garlic/nbody/argv.nix    | 29 +++++++++++++++
 bsc/garlic/nbody/default.nix | 17 ++-------
 6 files changed, 122 insertions(+), 42 deletions(-)
 create mode 100644 bsc/garlic/argv.nix
 create mode 100644 bsc/garlic/nbody/argv.nix

diff --git a/bsc/garlic/argv.nix b/bsc/garlic/argv.nix
new file mode 100644
index 0000000..5978cb3
--- /dev/null
+++ b/bsc/garlic/argv.nix
@@ -0,0 +1,25 @@
+{
+  stdenv
+}:
+
+{
+  app
+, argv # bash array as string, example: argv=''(-f "file with spaces" -t 10)''
+}:
+
+stdenv.mkDerivation {
+  inherit argv;
+  name = "${app.name}-argv";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    mkdir -p $out/bin
+    cat > $out/bin/run <<EOF
+    #!/bin/sh
+    argv=${argv}
+    exec ${app}/bin/run \''${argv[@]}
+    EOF
+    chmod +x $out/bin/run
+  '';
+}
diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix
index 626ad70..4e222df 100644
--- a/bsc/garlic/default.nix
+++ b/bsc/garlic/default.nix
@@ -23,10 +23,11 @@ let
       gitBranch = "garlic/seq";
     };
 
-    sbatch = callPackage ./sbatch.nix { };
-    launcher = callPackage ./launcher.nix { };
-    control = callPackage ./control.nix { };
-    nixsetup = callPackage ./nix-setup.nix { };
+    sbatchWrapper = callPackage ./sbatch.nix { };
+    launcherWrapper = callPackage ./launcher.nix { };
+    controlWrapper = callPackage ./control.nix { };
+    nixsetupWrapper = callPackage ./nix-setup.nix { };
+    argvWrapper = callPackage ./argv.nix { };
 
     exp = {
       nbody = {
diff --git a/bsc/garlic/exp/nbody/bs.nix b/bsc/garlic/exp/nbody/bs.nix
index 5d0670a..2ffc8fe 100644
--- a/bsc/garlic/exp/nbody/bs.nix
+++ b/bsc/garlic/exp/nbody/bs.nix
@@ -3,39 +3,64 @@
 , nbody
 , genApp
 , genConfigs
-, sbatch
-, launcher
-, control
-, nixsetup
+
+# Wrappers
+, launcherWrapper
+, sbatchWrapper
+, argvWrapper
+, controlWrapper
+, nixsetupWrapper
 }:
 
 let
   # Set the configuration for the experiment
   config = {
     cc = [ bsc.icc ];
-    blocksize = [ 1024 2048 4096 ];
+    blocksize = [ 1024 2048 4096 8192 ];
+  };
+
+  extraConfig = {
+    particles = 16384;
+    timesteps = 10;
+    ntasks = 1;
+    nnodes = 1;
   };
 
   # Compute the cartesian product of all configurations
-  configList = genConfigs config;
-  # Generate each app variant via override
-  appList = genApp nbody configList;
+  allConfigs = genConfigs config;
 
-  # Job generator helper function
-  genJobs = map (app:
-    sbatch {
-      app = (nixsetup (control app));
-      nixPrefix = "/gpfs/projects/bsc15/nix";
-      exclusive = false;
-      ntasks = "1";
-      chdirPrefix = "/home/bsc15/bsc15557/bsc-nixpkgs/out";
-    }
-  );
+  configs = with builtins; filter (c: c.blocksize <= 4096) allConfigs;
 
-  # Generate one job for each app variant
-  jobList = genJobs appList;
+  sbatch = conf: app: sbatchWrapper {
+    app = app;
+    nixPrefix = "/gpfs/projects/bsc15/nix";
+    exclusive = false;
+    ntasks = "${toString conf.ntasks}";
+    chdirPrefix = "/home/bsc15/bsc15557/bsc-nixpkgs/out";
+  };
+
+  argv = conf: app:
+    with conf;
+    argvWrapper {
+      app = app;
+      argv = ''(-t ${toString timesteps} -p ${toString particles})'';
+    };
+
+  nbodyFn = conf:
+    with conf;
+    nbody.override { inherit cc blocksize; };
+
+  pipeline = conf:
+    sbatch conf (
+      nixsetupWrapper (
+        controlWrapper (
+          argv conf (
+            nbodyFn conf))));
+
+  # Ideally it should look like this:
+  #pipeline = sbatch nixsetup control argv nbodyFn;
+
+  jobs = map (conf: pipeline (conf // extraConfig)) configs;
 
-  # And execute them all
-  main = launcher jobList;
 in
-  main
+  launcherWrapper jobs
diff --git a/bsc/garlic/launcher.nix b/bsc/garlic/launcher.nix
index 034b814..6f4e463 100644
--- a/bsc/garlic/launcher.nix
+++ b/bsc/garlic/launcher.nix
@@ -2,7 +2,7 @@
   stdenv
 }:
 
-apps:
+apps: # Each app must be unique
 
 stdenv.mkDerivation {
   name = "launcher";
@@ -16,7 +16,16 @@ stdenv.mkDerivation {
   installPhase = ''
     mkdir -p $out/apps
     for j in $apps; do
-      ln -s $j $out/apps/$(basename $j)
+      target=$out/apps/$(basename $j)
+      if [ -e $target ]; then
+        echo "Duplicated app: $j"
+        echo
+        echo "Provided apps: "
+        printf "%s\n" $apps
+        echo
+        exit 1
+      fi
+      ln -s $j $target
     done
 
     mkdir -p $out/bin
diff --git a/bsc/garlic/nbody/argv.nix b/bsc/garlic/nbody/argv.nix
new file mode 100644
index 0000000..c4528c6
--- /dev/null
+++ b/bsc/garlic/nbody/argv.nix
@@ -0,0 +1,29 @@
+{
+  stdenv
+, particles
+, timestamps
+, program
+, config
+}:
+
+stdenv.mkDerivation {
+  inherit program;
+
+  passthru = {
+    inherit config;
+  };
+
+  name = "${program.name}-argv";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    mkdir -p $out/bin
+    cat > $out/bin/run <<EOF
+    #!/bin/sh
+    exec ${program}/bin/run -p ${toString config.particles} -t ${toString config.timesteps}
+    EOF
+
+    chmod +x $out/bin/run
+  '';
+}
diff --git a/bsc/garlic/nbody/default.nix b/bsc/garlic/nbody/default.nix
index f7430e4..e6d5468 100644
--- a/bsc/garlic/nbody/default.nix
+++ b/bsc/garlic/nbody/default.nix
@@ -4,16 +4,14 @@
 , cflags ? null
 , gitBranch
 , blocksize ? 2048
-, particles ? 16384
-, timesteps ? 10
 }:
 
-stdenv.mkDerivation {
+stdenv.mkDerivation rec {
   name = "nbody";
 
   src = builtins.fetchGit {
     url = "ssh://git@bscpm02.bsc.es/rarias/nbody.git";
-    ref = gitBranch;
+    ref = "${gitBranch}";
   };
 
   buildInputs = [
@@ -33,15 +31,8 @@ stdenv.mkDerivation {
 
   installPhase = ''
     mkdir -p $out/bin
-    cp nbody $out/bin/
-
-    cat > $out/bin/run <<EOF
-    #!/bin/sh
-
-    exec $out/bin/nbody -p ${toString particles} -t ${toString timesteps}
-    EOF
-
-    chmod +x $out/bin/run
+    cp nbody* $out/bin/${name}
+    ln -s $out/bin/${name} $out/bin/run
   '';
 
 }