WIP nix-isolate

garlic/exp/nbody/tampi.nix

@@ -15,7 +15,7 @@ let
   # Set variable configuration for the experiment
   varConfig = {
     cc = [ bsc.icc ];
-    mpi = [ bsc.impi bsc.openmpi ];
+    mpi = [ bsc.impi ];
     blocksize = [ 1024 ];
   };
 
@@ -89,9 +89,9 @@ let
     perfArgs = "sched record -a";
   };
 
-  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+  nixsetup = {stage, conf, ...}: with conf; w.nix-isolate {
     program = stageProgram stage;
-    nixsetup = "${nixPrefix}/bin/nix-setup";
+    clusterName = "mn4";
   };
 
   extrae = {stage, conf, ...}: w.extrae {
@@ -143,6 +143,10 @@ let
       inherit cc blocksize mpi gitBranch;
     };
 
+  launch = w.launch.override {
+    nixPrefix = common.nixPrefix;
+  };
+
   stages = with common; []
     # Use sbatch to request resources first
     ++ optional enableSbatch sbatch
@@ -172,4 +176,4 @@ let
 
 in
   # We simply run each program one after another
-  w.launch jobs
+  launch jobs

garlic/stages/launcher/default.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, nixPrefix ? ""
 }:
 
 apps: # Each app must be unique
@@ -10,9 +11,18 @@ stdenv.mkDerivation {
 
   buildInputs = [] ++ apps;
   apps = apps;
-  phases = [ "installPhase" ];
+  phases = [ "unpackPhase" "patchPhase" "installPhase" ];
   dontPatchShebangs = true;
 
+  src = ./.;
+
+  inherit nixPrefix;
+
+  patchPhase = ''
+    substituteAllInPlace run
+    substituteAllInPlace stage2
+  '';
+
   installPhase = ''
     mkdir -p $out/apps
     for j in $apps; do
@@ -29,17 +39,13 @@ stdenv.mkDerivation {
     done
 
     mkdir -p $out/bin
-    cat > $out/bin/run <<EOF
-    #!/bin/sh
-
-    for j in $out/apps/*; do
-      \$j/bin/run
-    done
-    EOF
-
-    chmod +x $out/bin/run
+    install -m755 run $out/bin/run
+    install -m755 stage2 $out/bin/stage2
+    chmod +x $out/bin/*
 
     # Mark the launcher for upload
     touch $out/.upload-to-mn
+    # And mark it as an experiment
+    touch $out/.experiment
   '';
 }

garlic/stages/launcher/run

@@ -0,0 +1,19 @@
+#!/bin/sh -ex
+
+>&2 echo "Running launcher run stage"
+env
+
+if [ -e /nix ]; then
+  >&2 echo "Cannot use the launcher inside nix environment!"
+  exit 1
+fi
+
+if [ -z "$GARLIC_OUT" ]; then
+  >&2 echo "GARLIC_OUT is not set"
+  exit 1
+fi
+  
+mkdir -p "$GARLIC_OUT"
+cd "$GARLIC_OUT"
+
+exec nix-isolate @nixPrefix@@out@/bin/stage2

garlic/stages/launcher/stage2

@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+>&2 echo "Running launcher stage2"
+
+if [ ! -e "/nix" ]; then
+  >&2 echo "Cannot execute stage2 outside from nix environment"
+  exit 1
+fi
+
+for j in @out@/apps/*; do
+  $j/bin/run
+done

garlic/stages/nix-isolate.nix

@@ -0,0 +1,40 @@
+{
+  stdenv
+, nixtools
+}:
+
+{
+  program
+, clusterName
+}:
+
+stdenv.mkDerivation {
+  name = "nix-isolate";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh -ex
+
+    >&2 echo Running nix-isolate stage
+    >&2 echo PATH=$PATH
+    >&2 echo Running env:
+    env
+
+    # We need to enter the nix namespace first, in order to have /nix
+    # available, so we use this hack:
+    if [ ! -e /nix ]; then
+      exec ${nixtools}/bin/${clusterName}/nix-isolate \$0
+    fi
+
+    if [ -e /usr ]; then
+      >&2 echo "Environment not isolated, aborting"
+      exit 1
+    fi
+
+    exec ${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/nix-setup.nix

@@ -14,7 +14,10 @@ stdenv.mkDerivation {
   dontPatchShebangs = true;
   installPhase = ''
     cat > $out <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
+
+    >&2 echo Running nixsetup stage
+    env
 
     # We need to enter the nix namespace first, in order to have /nix
     # available, so we use this hack:

garlic/stages/sbatch.nix

@@ -1,6 +1,7 @@
 {
   stdenv
 , numactl
+, slurm
 }:
 
 {
@@ -50,7 +51,7 @@ stdenv.mkDerivation rec {
   installPhase = ''
     mkdir -p $out
     cat > $out/job <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
     #SBATCH --job-name="${jobName}"
     ''
     + sbatchOpt "ntasks" ntasks
@@ -72,14 +73,14 @@ stdenv.mkDerivation rec {
     EOF
     
     cat > $out/${name} <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
     if [ -e "${chdirPrefix}/$(basename $out)" ]; then
       >&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists"
       exit 1
     fi
     mkdir -p "${chdirPrefix}/$(basename $out)"
-    echo sbatch ${nixPrefix}$out/job
-    sbatch ${nixPrefix}$out/job
+    echo ${slurm}/bin/sbatch ${nixPrefix}$out/job
+    ${slurm}/bin/sbatch ${nixPrefix}$out/job
     EOF
     chmod +x $out/${name}
   '';

garlic/stages/srun.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, slurm
 }:
 {
   program
@@ -14,8 +15,9 @@ stdenv.mkDerivation rec {
   dontPatchShebangs = true;
   installPhase = ''
     cat > $out <<EOF
-    #!/bin/sh
-    exec srun --mpi=pmi2 ${srunOptions} ${nixPrefix}${program}
+    #!/bin/sh -ex
+    exec ${slurm}/bin/srun --mpi=pmi2 ${srunOptions} \
+      ${nixPrefix}${program}
     EOF
     chmod +x $out
   '';