Introduce the runexp stage

garlic/stages/experiment.nix

@@ -25,6 +25,21 @@ stdenv.mkDerivation {
     cat > $out << EOF
     #!/bin/sh
 
+    if [ -z "\$GARLIC_OUT" ]; then
+      >&2 echo "GARLIC_OUT not defined, aborting"
+      exit 1
+    fi
+
+    export GARLIC_EXPERIMENT=$(basename $out)
+
+    if [ -e "\$GARLIC_EXPERIMENT" ]; then
+      >&2 echo "Already exists \$GARLIC_EXPERIMENT, aborting"
+      exit 1
+    fi
+
+    mkdir -p "\$GARLIC_EXPERIMENT"
+    cd "\$GARLIC_EXPERIMENT"
+
     # This is an experiment formed by the following units:
     ${unitsString}
     EOF

garlic/stages/runexp/default.nix

@@ -0,0 +1,29 @@
+{
+  stdenv
+, garlicTools
+}:
+
+{
+  nextStage
+, nixPrefix
+}:
+
+with garlicTools;
+
+stdenv.mkDerivation {
+  name = "runexp";
+  preferLocalBuild = true;
+  phases = [ "unpackPhase" "installPhase" ];
+  src = ./.;
+  dontPatchShebangs = true;
+  programPath = "/bin/runexp";
+  inherit nixPrefix nextStage;
+  program = stageProgram nextStage;
+  installPhase = ''
+    substituteAllInPlace runexp
+
+    mkdir -p $out/bin
+    cp runexp $out/bin/
+    chmod +x $out/bin/runexp
+  '';
+}

garlic/stages/runexp/runexp

@@ -0,0 +1,18 @@
+#!/bin/sh -ex
+
+if [ -e /nix ]; then
+    >&2 echo "Cannot use runexp inside nix environment!"
+    exit 1
+fi
+
+>&2 echo Running runexp for MN4
+>&2 echo PATH=$PATH
+
+user=$(id -un)
+group=$(id -gn)
+
+export GARLIC_OUT="/gpfs/projects/$group/$user/garlic-out"
+mkdir -p "$GARLIC_OUT"
+cd "$GARLIC_OUT"
+
+exec @nixPrefix@@program@

garlic/stages/sbatch.nix

@@ -8,7 +8,7 @@
 {
   nextStage
 , jobName
-, chdirPrefix ? "."
+, chdir ? "."
 , nixPrefix ? ""
 , binary ? "/bin/run"
 , ntasks ? null
@@ -49,7 +49,7 @@ stdenv.mkDerivation rec {
   #SBATCH --cpus-per-task=1
   dontBuild = true;
   dontPatchShebangs = true;
-  programPath = "/${name}";
+  programPath = "/run";
 
   installPhase = ''
     mkdir -p $out
@@ -61,7 +61,7 @@ stdenv.mkDerivation rec {
     + sbatchOpt "ntasks-per-node" ntasksPerNode
     + sbatchOpt "ntasks-per-socket" ntasksPerSocket
     + sbatchOpt "nodes" nodes
-    + sbatchOpt "chdir" "${chdirPrefix}/$(basename $out)"
+    + sbatchOpt "chdir" chdir
     + sbatchOpt "output" output
     + sbatchOpt "error" error
     + sbatchEnable "exclusive" exclusive
@@ -75,16 +75,10 @@ stdenv.mkDerivation rec {
     exec ${nixPrefix}${stageProgram nextStage}
     EOF
     
-    cat > $out/${name} <<EOF
+    cat > $out/run <<EOF
     #!/bin/sh -ex
-    if [ -e "${chdirPrefix}/$(basename $out)" ]; then
-      >&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists"
-      exit 1
-    fi
-    mkdir -p "${chdirPrefix}/$(basename $out)"
-    echo ${slurm}/bin/sbatch ${nixPrefix}$out/job
     ${slurm}/bin/sbatch ${nixPrefix}$out/job
     EOF
-    chmod +x $out/${name}
+    chmod +x $out/run
   '';
 }

garlic/stages/trebuchet.nix

@@ -1,6 +1,5 @@
 {
   stdenv
-, nixtools
 , garlicTools
 }:
 
@@ -29,18 +28,15 @@ stdenv.mkDerivation {
     #!/bin/sh -e
     # Using the token @upload-to-mn@ we instruct the post-build hook to upload
     # this script and it's closure to the MN4 cluster, so it can run there.
+
+    # This trebuchet launches:
+    #  ${nextStage}
+    #  ${nextStage.nextStage}
+    #  ${nextStage.nextStage.nextStage}
+
     # Take a look at ${program}
     # to see what is being executed.
-
+    ssh ${sshHost} ${nixPrefix}${program}
-    # This trebuchet launches the following experiment in an isolated
-    # environment:
-    #  ${nextStage.nextStage}
-
-    nixtools=${nixPrefix}${nixtools}/bin
-    runexp=\$nixtools/${targetCluster}/runexp
-
-    >&2 echo "Launching \"\$runexp ${program}\" in MN4"
-    ssh ${sshHost} \$runexp ${program}
     EOF
     chmod +x $out
   '';

garlic/stdexp.nix

@@ -14,8 +14,10 @@ let
 in
 rec {
   /* Takes a list of units and builds an experiment, after executing the
-  trebuchet and the isolate stages. Returns the trebuchet stage. */
+  trebuchet, runexp and isolate stages. Returns the trebuchet stage. */
   buildTrebuchet = units: stages.trebuchet {
+    inherit (machineConf) nixPrefix;
+    nextStage = stages.runexp {
       inherit (machineConf) nixPrefix;
       nextStage = stages.isolate {
         inherit (machineConf) nixPrefix;
@@ -24,6 +26,7 @@ rec {
         };
       };
     };
+  };
 
   /* Given an attrset of lists `varConf` and a function `genConf` that accepts a
   attrset, computes the cartesian product of all combinations of `varConf` calls

overlay.nix

@@ -212,6 +212,7 @@ let
         extrae     = callPackage ./garlic/stages/extrae.nix { };
         valgrind   = callPackage ./garlic/stages/valgrind.nix { };
         isolate    = callPackage ./garlic/stages/isolate { };
+        runexp     = callPackage ./garlic/stages/runexp { };
         trebuchet  = callPackage ./garlic/stages/trebuchet.nix { };
         strace     = callPackage ./garlic/stages/strace.nix { };
         unit       = callPackage ./garlic/stages/unit.nix { };