From ef1aeb2cfa4341a4d6386e492945e066aa1635b3 Mon Sep 17 00:00:00 2001
From: Rodrigo Arias <rodrigo.arias@bsc.es>
Date: Mon, 10 Aug 2020 18:25:53 +0200
Subject: [PATCH] Run each experiment in a unique directory

---
 bsc/garlic/default.nix                        |  2 +-
 bsc/garlic/exp/nbody/bs.nix                   | 11 +++++----
 .../{sbatch-launcher.nix => launcher.nix}     | 18 +++++++-------
 bsc/garlic/sbatch.nix                         | 24 +++++++++++++++----
 4 files changed, 35 insertions(+), 20 deletions(-)
 rename bsc/garlic/{sbatch-launcher.nix => launcher.nix} (55%)

diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix
index 0485cc0..191da96 100644
--- a/bsc/garlic/default.nix
+++ b/bsc/garlic/default.nix
@@ -24,7 +24,7 @@ let
     };
 
     sbatch = callPackage ./sbatch.nix { };
-    sbatchLauncher = callPackage ./sbatch-launcher.nix { };
+    launcher = callPackage ./launcher.nix { };
 
     exp = {
       nbody = {
diff --git a/bsc/garlic/exp/nbody/bs.nix b/bsc/garlic/exp/nbody/bs.nix
index e34453d..9a32b26 100644
--- a/bsc/garlic/exp/nbody/bs.nix
+++ b/bsc/garlic/exp/nbody/bs.nix
@@ -4,7 +4,7 @@
 , genApp
 , genConfigs
 , sbatch
-, sbatchLauncher
+, launcher
 }:
 
 let
@@ -23,16 +23,17 @@ let
   genJobs = map (app:
     sbatch {
       app = app;
-      prefix = "/gpfs/projects/bsc15/nix";
+      nixPrefix = "/gpfs/projects/bsc15/nix";
       exclusive = false;
       ntasks = "1";
+      chdirPrefix = "/home/bsc15/bsc15557/bsc-nixpkgs/out";
     }
   );
 
   # Generate one job for each app variant
   jobList = genJobs appList;
 
-  # And merge all jobs in a script to lauch them all with sbatch
-  launcher = sbatchLauncher jobList;
+  # And execute them all
+  main = launcher jobList;
 in
-  launcher
+  main
diff --git a/bsc/garlic/sbatch-launcher.nix b/bsc/garlic/launcher.nix
similarity index 55%
rename from bsc/garlic/sbatch-launcher.nix
rename to bsc/garlic/launcher.nix
index e85557f..034b814 100644
--- a/bsc/garlic/sbatch-launcher.nix
+++ b/bsc/garlic/launcher.nix
@@ -2,29 +2,29 @@
   stdenv
 }:
 
-jobs:
+apps:
 
 stdenv.mkDerivation {
   name = "launcher";
   preferLocalBuild = true;
 
-  buildInputs = [] ++ jobs;
-  jobs = jobs;
+  buildInputs = [] ++ apps;
+  apps = apps;
   phases = [ "installPhase" ];
+  dontPatchShebangs = true;
 
   installPhase = ''
-    mkdir -p $out/jobs
-    for j in $jobs; do
-      ln -s $j/job $out/jobs/$(basename $j)
+    mkdir -p $out/apps
+    for j in $apps; do
+      ln -s $j $out/apps/$(basename $j)
     done
 
     mkdir -p $out/bin
     cat > $out/bin/run <<EOF
     #!/bin/sh
 
-    for j in $out/jobs/*; do
-      echo "sbatch \$j"
-      sbatch \$j
+    for j in $out/apps/*; do
+      \$j/bin/run
     done
     EOF
 
diff --git a/bsc/garlic/sbatch.nix b/bsc/garlic/sbatch.nix
index 17a7454..dc1603f 100644
--- a/bsc/garlic/sbatch.nix
+++ b/bsc/garlic/sbatch.nix
@@ -5,12 +5,12 @@
 
 {
   app
-, prefix ? ""
+, chdirPrefix
+, nixPrefix ? ""
 , argv ? ""
 , binary ? "/bin/run"
 , ntasks ? null
 , exclusive ? true # By default we run in exclusive mode
-, chdir ? "."
 , qos ? null
 , time ? null
 , output ? "job_%j.out"
@@ -41,15 +41,16 @@ stdenv.mkDerivation rec {
   #SBATCH --ntasks-per-socket=24
   #SBATCH --cpus-per-task=1
   dontBuild = true;
+  dontPatchShebangs = true;
 
   installPhase = ''
     mkdir -p $out
     cat > $out/job <<EOF
-    #!/bin/bash
+    #!/bin/sh
     #SBATCH --job-name="${name}"
     ''
     + sbatchOpt "ntasks" ntasks
-    + sbatchOpt "chdir" chdir
+    + sbatchOpt "chdir" "${chdirPrefix}/$(basename $out)"
     + sbatchOpt "output" output
     + sbatchOpt "error" error
     + sbatchEnable "exclusive" exclusive
@@ -58,7 +59,20 @@ stdenv.mkDerivation rec {
     + optionalString (extra!=null) extra
     +
     ''
-    srun ${prefix}${app}${binary} ${argv}
+    srun ${nixPrefix}${app}${binary} ${argv}
     EOF
+    
+    mkdir -p $out/bin
+    cat > $out/bin/run <<EOF
+    #!/bin/sh
+    if [ -e "${chdirPrefix}/$(basename $out)" ]; then
+      >&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists"
+      exit 1
+    fi
+    mkdir -p "${chdirPrefix}/$(basename $out)"
+    echo sbatch ${nixPrefix}$out/job
+    sbatch ${nixPrefix}$out/job
+    EOF
+    chmod +x $out/bin/run
   '';
 }