WIP nix-isolate

bsc/nixtools/default.nix

@@ -0,0 +1,23 @@
+{
+  stdenv
+, targetCluster
+, nixPrefix
+}:
+
+stdenv.mkDerivation rec {
+  name = "nixtools-${targetCluster}";
+  #version = "${src.shortRev}";
+  src = ~/nixtools;
+  makeFlags = [ "DESTDIR=$(out)" ];
+  preBuild = "env";
+  dontPatchShebangs = true;
+  inherit nixPrefix targetCluster;
+  postPatch = ''
+    substituteAllInPlace scripts/cobi/runexp
+    sed -i s:@nixtools@:$out:g scripts/cobi/runexp
+  '';
+  #src = builtins.fetchGit {
+  #  url = "ssh://git@bscpm02.bsc.es/rarias/nixtools";
+  #  ref = "master";
+  #};
+}

garlic/exp/nbody/tampi.nix

@@ -15,7 +15,7 @@ let
   # Set variable configuration for the experiment
   varConfig = {
     cc = [ bsc.icc ];
-    mpi = [ bsc.impi bsc.openmpi ];
+    mpi = [ bsc.impi ];
     blocksize = [ 1024 ];
   };
 
@@ -89,9 +89,9 @@ let
     perfArgs = "sched record -a";
   };
 
-  nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
+  nixsetup = {stage, conf, ...}: with conf; w.nix-isolate {
     program = stageProgram stage;
-    nixsetup = "${nixPrefix}/bin/nix-setup";
+    clusterName = "mn4";
   };
 
   extrae = {stage, conf, ...}: w.extrae {
@@ -143,6 +143,10 @@ let
       inherit cc blocksize mpi gitBranch;
     };
 
+  launch = w.launch.override {
+    nixPrefix = common.nixPrefix;
+  };
+
   stages = with common; []
     # Use sbatch to request resources first
     ++ optional enableSbatch sbatch
@@ -172,4 +176,4 @@ let
 
 in
   # We simply run each program one after another
-  w.launch jobs
+  launch jobs

garlic/stages/launcher/default.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, nixPrefix ? ""
 }:
 
 apps: # Each app must be unique
@@ -10,9 +11,18 @@ stdenv.mkDerivation {
 
   buildInputs = [] ++ apps;
   apps = apps;
-  phases = [ "installPhase" ];
+  phases = [ "unpackPhase" "patchPhase" "installPhase" ];
   dontPatchShebangs = true;
 
+  src = ./.;
+
+  inherit nixPrefix;
+
+  patchPhase = ''
+    substituteAllInPlace run
+    substituteAllInPlace stage2
+  '';
+
   installPhase = ''
     mkdir -p $out/apps
     for j in $apps; do
@@ -29,17 +39,13 @@ stdenv.mkDerivation {
     done
 
     mkdir -p $out/bin
-    cat > $out/bin/run <<EOF
+    install -m755 run $out/bin/run
-    #!/bin/sh
+    install -m755 stage2 $out/bin/stage2
-
+    chmod +x $out/bin/*
-    for j in $out/apps/*; do
-      \$j/bin/run
-    done
-    EOF
-
-    chmod +x $out/bin/run
 
     # Mark the launcher for upload
     touch $out/.upload-to-mn
+    # And mark it as an experiment
+    touch $out/.experiment
   '';
 }

garlic/stages/launcher/run

@@ -0,0 +1,19 @@
+#!/bin/sh -ex
+
+>&2 echo "Running launcher run stage"
+env
+
+if [ -e /nix ]; then
+  >&2 echo "Cannot use the launcher inside nix environment!"
+  exit 1
+fi
+
+if [ -z "$GARLIC_OUT" ]; then
+  >&2 echo "GARLIC_OUT is not set"
+  exit 1
+fi
+  
+mkdir -p "$GARLIC_OUT"
+cd "$GARLIC_OUT"
+
+exec nix-isolate @nixPrefix@@out@/bin/stage2

garlic/stages/launcher/stage2

@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+>&2 echo "Running launcher stage2"
+
+if [ ! -e "/nix" ]; then
+  >&2 echo "Cannot execute stage2 outside from nix environment"
+  exit 1
+fi
+
+for j in @out@/apps/*; do
+  $j/bin/run
+done

garlic/stages/nix-isolate.nix

@@ -0,0 +1,40 @@
+{
+  stdenv
+, nixtools
+}:
+
+{
+  program
+, clusterName
+}:
+
+stdenv.mkDerivation {
+  name = "nix-isolate";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh -ex
+
+    >&2 echo Running nix-isolate stage
+    >&2 echo PATH=$PATH
+    >&2 echo Running env:
+    env
+
+    # We need to enter the nix namespace first, in order to have /nix
+    # available, so we use this hack:
+    if [ ! -e /nix ]; then
+      exec ${nixtools}/bin/${clusterName}/nix-isolate \$0
+    fi
+
+    if [ -e /usr ]; then
+      >&2 echo "Environment not isolated, aborting"
+      exit 1
+    fi
+
+    exec ${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/nix-setup.nix

@@ -14,7 +14,10 @@ stdenv.mkDerivation {
   dontPatchShebangs = true;
   installPhase = ''
     cat > $out <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
+
+    >&2 echo Running nixsetup stage
+    env
 
     # We need to enter the nix namespace first, in order to have /nix
     # available, so we use this hack:

garlic/stages/sbatch.nix

@@ -1,6 +1,7 @@
 {
   stdenv
 , numactl
+, slurm
 }:
 
 {
@@ -50,7 +51,7 @@ stdenv.mkDerivation rec {
   installPhase = ''
     mkdir -p $out
     cat > $out/job <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
     #SBATCH --job-name="${jobName}"
     ''
     + sbatchOpt "ntasks" ntasks
@@ -72,14 +73,14 @@ stdenv.mkDerivation rec {
     EOF
     
     cat > $out/${name} <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
     if [ -e "${chdirPrefix}/$(basename $out)" ]; then
       >&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists"
       exit 1
     fi
     mkdir -p "${chdirPrefix}/$(basename $out)"
-    echo sbatch ${nixPrefix}$out/job
+    echo ${slurm}/bin/sbatch ${nixPrefix}$out/job
-    sbatch ${nixPrefix}$out/job
+    ${slurm}/bin/sbatch ${nixPrefix}$out/job
     EOF
     chmod +x $out/${name}
   '';

garlic/stages/srun.nix

@@ -1,5 +1,6 @@
 {
   stdenv
+, slurm
 }:
 {
   program
@@ -14,8 +15,9 @@ stdenv.mkDerivation rec {
   dontPatchShebangs = true;
   installPhase = ''
     cat > $out <<EOF
-    #!/bin/sh
+    #!/bin/sh -ex
-    exec srun --mpi=pmi2 ${srunOptions} ${nixPrefix}${program}
+    exec ${slurm}/bin/srun --mpi=pmi2 ${srunOptions} \
+      ${nixPrefix}${program}
     EOF
     chmod +x $out
   '';

overlay.nix

@@ -70,6 +70,9 @@ let
       pmix = self.bsc.pmix2;
     };
 
+    # Use a slurm compatible with MN4
+    slurm = self.bsc.slurm17;
+
     openmpi-mn4 = callPackage ./bsc/openmpi/default.nix {
       pmix = self.bsc.pmix2;
       pmi2 = self.bsc.slurm17-libpmi2;
@@ -128,6 +131,11 @@ let
 
     mpptest = callPackage ./bsc/mpptest/default.nix { };
 
+    nixtools = callPackage ./bsc/nixtools/default.nix {
+      targetCluster = "mn4";
+      nixPrefix = "/gpfs/projects/bsc15/nix";
+    };
+
     garlic = {
 
       # Load some helper functions to generate app variants
@@ -183,7 +191,7 @@ let
       runWrappers = {
         sbatch    = callPackage ./garlic/stages/sbatch.nix { };
         srun      = callPackage ./garlic/stages/srun.nix { };
-        launch    = callPackage ./garlic/stages/launcher.nix { };
+        launch    = callPackage ./garlic/stages/launcher { };
         control   = callPackage ./garlic/stages/control.nix { };
         nixsetup  = callPackage ./garlic/stages/nix-setup.nix { };
         argv      = callPackage ./garlic/stages/argv.nix { };
@@ -194,6 +202,7 @@ let
         broom     = callPackage ./garlic/stages/broom.nix { };
         envRecord = callPackage ./garlic/stages/envRecord.nix { };
         valgrind  = callPackage ./garlic/stages/valgrind.nix { };
+        nix-isolate = callPackage ./garlic/stages/nix-isolate.nix { };
       };
 
       # Tests (move to bsc ?)
@@ -207,7 +216,7 @@ let
       hist = callPackage ./garlic/postprocess/hist { };
 
       exp = {
-        noise = callPackage ./garlic/exp/noise.nix { };
+        #noise = callPackage ./garlic/exp/noise.nix { };
         nbody = {
           bs = callPackage ./garlic/exp/nbody/bs.nix {
             pkgs = self // self.bsc.garlic;