WIP isolate execution

bsc/nixtools/default.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, glibc
 , targetCluster
 , nixPrefix
 }:
@@ -8,6 +9,7 @@ stdenv.mkDerivation rec {
   name = "nixtools-${targetCluster}";
   #version = "${src.shortRev}";
   src = ~/nixtools;
+  buildInputs = [ glibc.static ];
   makeFlags = [ "DESTDIR=$(out)" ];
   preBuild = "env";
   dontPatchShebangs = true;

garlic/exp/nbody/tampi.nix

@@ -33,6 +33,7 @@ let
     nodes = "2";
 
     # Stage configuration
+    enableRunexp = true;
     enableSbatch = true;
     enableControl = true;
     enableExtrae = false;
@@ -89,9 +90,10 @@ let
     perfArgs = "sched record -a";
   };
 
-  nixsetup = {stage, conf, ...}: with conf; w.nix-isolate {
+  isolate = {stage, conf, ...}: with conf; w.isolate {
     program = stageProgram stage;
     clusterName = "mn4";
+    inherit nixPrefix;
   };
 
   extrae = {stage, conf, ...}: w.extrae {
@@ -148,16 +150,19 @@ let
   };
 
   stages = with common; []
+    # Launch the experiment remotely
+    #++ optional enableRunexp runexp
+
     # Use sbatch to request resources first
     ++ optional enableSbatch sbatch
 
     # Repeats the next stages N times
-    ++ optionals enableControl [ nixsetup control ]
+    ++ optionals enableControl [ isolate control ]
 
     # Executes srun to launch the program in the requested nodes, and
     # immediately after enters the nix environment again, as slurmstepd launches
     # the next stages from outside the namespace.
-    ++ [ srun nixsetup ]
+    ++ [ srun isolate ]
 
     # Intrumentation with extrae
     ++ optional enableExtrae extrae
@@ -174,6 +179,22 @@ let
   # List of actual programs to be executed
   jobs = map (conf: w.stagen { inherit conf stages; }) configs;
 
+  launcher = launch jobs;
+
+  runexp = stage: w.runexp {
+    program = stageProgram stage;
+    nixPrefix = common.nixPrefix;
+  };
+
+  isolatedRun = stage: isolate {
+    inherit stage;
+    conf = common;
+  };
+
+  final = runexp (isolatedRun launcher);
+  
+
 in
   # We simply run each program one after another
-  launch jobs
+  #launch jobs
+  final

garlic/stages/isolate/default.nix

@@ -0,0 +1,30 @@
+{
+  stdenv
+, nixtools
+, busybox
+}:
+
+{
+  program
+, nixPrefix
+, clusterName
+}:
+
+stdenv.mkDerivation {
+  name = "isolate";
+  preferLocalBuild = true;
+  phases = [ "unpackPhase" "installPhase" ];
+  src = ./.;
+  dontPatchShebangs = true;
+  programPath = "/bin/stage1";
+  inherit program nixPrefix clusterName nixtools busybox;
+  out = "$out";
+  installPhase = ''
+    substituteAllInPlace stage1
+    substituteAllInPlace stage2
+
+    mkdir -p $out/bin
+    cp stage* $out/bin/
+    chmod +x $out/bin/stage*
+  '';
+}

garlic/stages/isolate/stage1

@@ -0,0 +1,23 @@
+#!/bin/sh -ex
+
+>&2 echo Running isolate stage1
+>&2 echo PATH=$PATH
+
+if [ -e /nix ]; then
+  >&2 echo "/nix found, aborting"
+  exit 1
+fi
+
+nixhome="@nixPrefix@/nix"
+shell="@busybox@/bin/sh"
+nixjoin="@nixPrefix@@nixtools@/bin/nix-join"
+
+
+#-m @nixPrefix@ \
+join_flags="
+	-m /etc \
+	-m /var/run/munge \
+	-m /gpfs/projects/bsc15 \
+	-m /bin:@nixPrefix@@busybox@/bin"
+
+exec $nixjoin -i $join_flags $nixhome -- @out@/bin/stage2

garlic/stages/isolate/stage2

@@ -0,0 +1,16 @@
+#!/bin/sh -ex
+
+>&2 echo Running isolate stage2
+>&2 echo PATH=$PATH
+
+if [ ! -e /nix ]; then
+  >&2 echo "/nix not found, aborting" 
+  exit 1
+fi
+
+if [ -e /usr ]; then
+  >&2 echo "Environment not isolated, aborting"
+  exit 1
+fi
+
+exec @program@

garlic/stages/launcher/default.nix

@@ -13,6 +13,7 @@ stdenv.mkDerivation {
   apps = apps;
   phases = [ "unpackPhase" "patchPhase" "installPhase" ];
   dontPatchShebangs = true;
+  programPath = "/bin/run";
 
   src = ./.;
 
@@ -20,7 +21,6 @@ stdenv.mkDerivation {
 
   patchPhase = ''
     substituteAllInPlace run
-    substituteAllInPlace stage2
   '';
 
   installPhase = ''
@@ -40,7 +40,6 @@ stdenv.mkDerivation {
 
     mkdir -p $out/bin
     install -m755 run $out/bin/run
-    install -m755 stage2 $out/bin/stage2
     chmod +x $out/bin/*
 
     # Mark the launcher for upload

garlic/stages/launcher/run

@@ -1,10 +1,9 @@
 #!/bin/sh -ex
 
->&2 echo "Running launcher run stage"
+>&2 echo "Running launcher"
-env
 
-if [ -e /nix ]; then
+if [ ! -e "/nix" ]; then
-  >&2 echo "Cannot use the launcher inside nix environment!"
+  >&2 echo "missing /nix"
   exit 1
 fi
 
@@ -16,4 +15,6 @@ fi
 mkdir -p "$GARLIC_OUT"
 cd "$GARLIC_OUT"
 
-exec nix-isolate @nixPrefix@@out@/bin/stage2
+for j in @out@/apps/*; do
+  $j/bin/run
+done

garlic/stages/launcher/stage2

@@ -1,12 +0,0 @@
-#!/bin/sh -ex
-
->&2 echo "Running launcher stage2"
-
-if [ ! -e "/nix" ]; then
-  >&2 echo "Cannot execute stage2 outside from nix environment"
-  exit 1
-fi
-
-for j in @out@/apps/*; do
-  $j/bin/run
-done

garlic/stages/nix-isolate.nix

@@ -1,40 +0,0 @@
-{
-  stdenv
-, nixtools
-}:
-
-{
-  program
-, clusterName
-}:
-
-stdenv.mkDerivation {
-  name = "nix-isolate";
-  preferLocalBuild = true;
-  phases = [ "installPhase" ];
-  dontPatchShebangs = true;
-  installPhase = ''
-    cat > $out <<EOF
-    #!/bin/sh -ex
-
-    >&2 echo Running nix-isolate stage
-    >&2 echo PATH=$PATH
-    >&2 echo Running env:
-    env
-
-    # We need to enter the nix namespace first, in order to have /nix
-    # available, so we use this hack:
-    if [ ! -e /nix ]; then
-      exec ${nixtools}/bin/${clusterName}/nix-isolate \$0
-    fi
-
-    if [ -e /usr ]; then
-      >&2 echo "Environment not isolated, aborting"
-      exit 1
-    fi
-
-    exec ${program}
-    EOF
-    chmod +x $out
-  '';
-}

garlic/stages/runexp/default.nix

@@ -0,0 +1,25 @@
+{
+  stdenv
+, nixtools
+}:
+
+{
+  program
+, nixPrefix
+, sshHost ? "mn"
+, targetCluster ? "mn4"
+}:
+
+stdenv.mkDerivation {
+  name = "runexp";
+  preferLocalBuild = true;
+  phases = [ "unpackPhase" "installPhase" ];
+  dontPatchShebangs = true;
+  src = ./.;
+  inherit sshHost nixPrefix nixtools targetCluster program;
+  installPhase = ''
+    substituteAllInPlace runexp
+    cp runexp $out
+    chmod +x $out
+  '';
+}

garlic/stages/runexp/runexp

@@ -0,0 +1,14 @@
+#!/bin/sh
+# @upload-to-mn@
+
+# This program runs the current experiment in the ./result symlink in
+# MareNostrum. Requires that you define a "mn" host in the ssh config file
+# (usually in ~/.ssh/config).
+
+nixPrefix=@nixPrefix@
+nixtools=$nixPrefix@nixtools@/bin
+runexp=$nixtools/@targetCluster@/runexp
+
+>&2 echo "Launching \"$runexp @program@\" in MN"
+
+ssh mn $runexp @program@

overlay.nix

@@ -202,7 +202,8 @@ let
         broom     = callPackage ./garlic/stages/broom.nix { };
         envRecord = callPackage ./garlic/stages/envRecord.nix { };
         valgrind  = callPackage ./garlic/stages/valgrind.nix { };
-        nix-isolate = callPackage ./garlic/stages/nix-isolate.nix { };
+        isolate   = callPackage ./garlic/stages/isolate { };
+        runexp    = callPackage ./garlic/stages/runexp { };
       };
 
       # Tests (move to bsc ?)