diff --git a/bsc/nixtools/default.nix b/bsc/nixtools/default.nix index dbf9770..6cfc702 100644 --- a/bsc/nixtools/default.nix +++ b/bsc/nixtools/default.nix @@ -1,5 +1,6 @@ { stdenv +, glibc , targetCluster , nixPrefix }: @@ -8,6 +9,7 @@ stdenv.mkDerivation rec { name = "nixtools-${targetCluster}"; #version = "${src.shortRev}"; src = ~/nixtools; + buildInputs = [ glibc.static ]; makeFlags = [ "DESTDIR=$(out)" ]; preBuild = "env"; dontPatchShebangs = true; diff --git a/garlic/exp/nbody/tampi.nix b/garlic/exp/nbody/tampi.nix index 71f70e5..153c7c4 100644 --- a/garlic/exp/nbody/tampi.nix +++ b/garlic/exp/nbody/tampi.nix @@ -33,6 +33,7 @@ let nodes = "2"; # Stage configuration + enableRunexp = true; enableSbatch = true; enableControl = true; enableExtrae = false; @@ -89,9 +90,10 @@ let perfArgs = "sched record -a"; }; - nixsetup = {stage, conf, ...}: with conf; w.nix-isolate { + isolate = {stage, conf, ...}: with conf; w.isolate { program = stageProgram stage; clusterName = "mn4"; + inherit nixPrefix; }; extrae = {stage, conf, ...}: w.extrae { @@ -148,16 +150,19 @@ let }; stages = with common; [] + # Launch the experiment remotely + #++ optional enableRunexp runexp + # Use sbatch to request resources first ++ optional enableSbatch sbatch # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] + ++ optionals enableControl [ isolate control ] # Executes srun to launch the program in the requested nodes, and # immediately after enters the nix environment again, as slurmstepd launches # the next stages from outside the namespace. - ++ [ srun nixsetup ] + ++ [ srun isolate ] # Intrumentation with extrae ++ optional enableExtrae extrae @@ -174,6 +179,22 @@ let # List of actual programs to be executed jobs = map (conf: w.stagen { inherit conf stages; }) configs; + launcher = launch jobs; + + runexp = stage: w.runexp { + program = stageProgram stage; + nixPrefix = common.nixPrefix; + }; + + isolatedRun = stage: isolate { + inherit stage; + conf = common; + }; + + final = runexp (isolatedRun launcher); + + in # We simply run each program one after another - launch jobs + #launch jobs + final diff --git a/garlic/stages/isolate/default.nix b/garlic/stages/isolate/default.nix new file mode 100644 index 0000000..a8d0756 --- /dev/null +++ b/garlic/stages/isolate/default.nix @@ -0,0 +1,30 @@ +{ + stdenv +, nixtools +, busybox +}: + +{ + program +, nixPrefix +, clusterName +}: + +stdenv.mkDerivation { + name = "isolate"; + preferLocalBuild = true; + phases = [ "unpackPhase" "installPhase" ]; + src = ./.; + dontPatchShebangs = true; + programPath = "/bin/stage1"; + inherit program nixPrefix clusterName nixtools busybox; + out = "$out"; + installPhase = '' + substituteAllInPlace stage1 + substituteAllInPlace stage2 + + mkdir -p $out/bin + cp stage* $out/bin/ + chmod +x $out/bin/stage* + ''; +} diff --git a/garlic/stages/isolate/stage1 b/garlic/stages/isolate/stage1 new file mode 100644 index 0000000..3f2b546 --- /dev/null +++ b/garlic/stages/isolate/stage1 @@ -0,0 +1,23 @@ +#!/bin/sh -ex + +>&2 echo Running isolate stage1 +>&2 echo PATH=$PATH + +if [ -e /nix ]; then + >&2 echo "/nix found, aborting" + exit 1 +fi + +nixhome="@nixPrefix@/nix" +shell="@busybox@/bin/sh" +nixjoin="@nixPrefix@@nixtools@/bin/nix-join" + + +#-m @nixPrefix@ \ +join_flags=" + -m /etc \ + -m /var/run/munge \ + -m /gpfs/projects/bsc15 \ + -m /bin:@nixPrefix@@busybox@/bin" + +exec $nixjoin -i $join_flags $nixhome -- @out@/bin/stage2 diff --git a/garlic/stages/isolate/stage2 b/garlic/stages/isolate/stage2 new file mode 100644 index 0000000..a4ae039 --- /dev/null +++ b/garlic/stages/isolate/stage2 @@ -0,0 +1,16 @@ +#!/bin/sh -ex + +>&2 echo Running isolate stage2 +>&2 echo PATH=$PATH + +if [ ! -e /nix ]; then + >&2 echo "/nix not found, aborting" + exit 1 +fi + +if [ -e /usr ]; then + >&2 echo "Environment not isolated, aborting" + exit 1 +fi + +exec @program@ diff --git a/garlic/stages/launcher/default.nix b/garlic/stages/launcher/default.nix index 4b7fef8..763bdb4 100644 --- a/garlic/stages/launcher/default.nix +++ b/garlic/stages/launcher/default.nix @@ -13,6 +13,7 @@ stdenv.mkDerivation { apps = apps; phases = [ "unpackPhase" "patchPhase" "installPhase" ]; dontPatchShebangs = true; + programPath = "/bin/run"; src = ./.; @@ -20,7 +21,6 @@ stdenv.mkDerivation { patchPhase = '' substituteAllInPlace run - substituteAllInPlace stage2 ''; installPhase = '' @@ -40,7 +40,6 @@ stdenv.mkDerivation { mkdir -p $out/bin install -m755 run $out/bin/run - install -m755 stage2 $out/bin/stage2 chmod +x $out/bin/* # Mark the launcher for upload diff --git a/garlic/stages/launcher/run b/garlic/stages/launcher/run index 225cbda..3c98c2c 100644 --- a/garlic/stages/launcher/run +++ b/garlic/stages/launcher/run @@ -1,10 +1,9 @@ #!/bin/sh -ex ->&2 echo "Running launcher run stage" -env +>&2 echo "Running launcher" -if [ -e /nix ]; then - >&2 echo "Cannot use the launcher inside nix environment!" +if [ ! -e "/nix" ]; then + >&2 echo "missing /nix" exit 1 fi @@ -12,8 +11,10 @@ if [ -z "$GARLIC_OUT" ]; then >&2 echo "GARLIC_OUT is not set" exit 1 fi - + mkdir -p "$GARLIC_OUT" cd "$GARLIC_OUT" -exec nix-isolate @nixPrefix@@out@/bin/stage2 +for j in @out@/apps/*; do + $j/bin/run +done diff --git a/garlic/stages/launcher/stage2 b/garlic/stages/launcher/stage2 deleted file mode 100644 index dc0622b..0000000 --- a/garlic/stages/launcher/stage2 +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -ex - ->&2 echo "Running launcher stage2" - -if [ ! -e "/nix" ]; then - >&2 echo "Cannot execute stage2 outside from nix environment" - exit 1 -fi - -for j in @out@/apps/*; do - $j/bin/run -done diff --git a/garlic/stages/nix-isolate.nix b/garlic/stages/nix-isolate.nix deleted file mode 100644 index 1139477..0000000 --- a/garlic/stages/nix-isolate.nix +++ /dev/null @@ -1,40 +0,0 @@ -{ - stdenv -, nixtools -}: - -{ - program -, clusterName -}: - -stdenv.mkDerivation { - name = "nix-isolate"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - dontPatchShebangs = true; - installPhase = '' - cat > $out <&2 echo Running nix-isolate stage - >&2 echo PATH=$PATH - >&2 echo Running env: - env - - # We need to enter the nix namespace first, in order to have /nix - # available, so we use this hack: - if [ ! -e /nix ]; then - exec ${nixtools}/bin/${clusterName}/nix-isolate \$0 - fi - - if [ -e /usr ]; then - >&2 echo "Environment not isolated, aborting" - exit 1 - fi - - exec ${program} - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/runexp/default.nix b/garlic/stages/runexp/default.nix new file mode 100644 index 0000000..f576273 --- /dev/null +++ b/garlic/stages/runexp/default.nix @@ -0,0 +1,25 @@ +{ + stdenv +, nixtools +}: + +{ + program +, nixPrefix +, sshHost ? "mn" +, targetCluster ? "mn4" +}: + +stdenv.mkDerivation { + name = "runexp"; + preferLocalBuild = true; + phases = [ "unpackPhase" "installPhase" ]; + dontPatchShebangs = true; + src = ./.; + inherit sshHost nixPrefix nixtools targetCluster program; + installPhase = '' + substituteAllInPlace runexp + cp runexp $out + chmod +x $out + ''; +} diff --git a/garlic/stages/runexp/runexp b/garlic/stages/runexp/runexp new file mode 100755 index 0000000..9e69548 --- /dev/null +++ b/garlic/stages/runexp/runexp @@ -0,0 +1,14 @@ +#!/bin/sh +# @upload-to-mn@ + +# This program runs the current experiment in the ./result symlink in +# MareNostrum. Requires that you define a "mn" host in the ssh config file +# (usually in ~/.ssh/config). + +nixPrefix=@nixPrefix@ +nixtools=$nixPrefix@nixtools@/bin +runexp=$nixtools/@targetCluster@/runexp + +>&2 echo "Launching \"$runexp @program@\" in MN" + +ssh mn $runexp @program@ diff --git a/overlay.nix b/overlay.nix index 3486e8f..a099497 100644 --- a/overlay.nix +++ b/overlay.nix @@ -202,7 +202,8 @@ let broom = callPackage ./garlic/stages/broom.nix { }; envRecord = callPackage ./garlic/stages/envRecord.nix { }; valgrind = callPackage ./garlic/stages/valgrind.nix { }; - nix-isolate = callPackage ./garlic/stages/nix-isolate.nix { }; + isolate = callPackage ./garlic/stages/isolate { }; + runexp = callPackage ./garlic/stages/runexp { }; }; # Tests (move to bsc ?)