diff --git a/bsc/nixtools/default.nix b/bsc/nixtools/default.nix new file mode 100644 index 0000000..dbf9770 --- /dev/null +++ b/bsc/nixtools/default.nix @@ -0,0 +1,23 @@ +{ + stdenv +, targetCluster +, nixPrefix +}: + +stdenv.mkDerivation rec { + name = "nixtools-${targetCluster}"; + #version = "${src.shortRev}"; + src = ~/nixtools; + makeFlags = [ "DESTDIR=$(out)" ]; + preBuild = "env"; + dontPatchShebangs = true; + inherit nixPrefix targetCluster; + postPatch = '' + substituteAllInPlace scripts/cobi/runexp + sed -i s:@nixtools@:$out:g scripts/cobi/runexp + ''; + #src = builtins.fetchGit { + # url = "ssh://git@bscpm02.bsc.es/rarias/nixtools"; + # ref = "master"; + #}; +} diff --git a/garlic/exp/nbody/tampi.nix b/garlic/exp/nbody/tampi.nix index 7fe7c7e..71f70e5 100644 --- a/garlic/exp/nbody/tampi.nix +++ b/garlic/exp/nbody/tampi.nix @@ -15,7 +15,7 @@ let # Set variable configuration for the experiment varConfig = { cc = [ bsc.icc ]; - mpi = [ bsc.impi bsc.openmpi ]; + mpi = [ bsc.impi ]; blocksize = [ 1024 ]; }; @@ -89,9 +89,9 @@ let perfArgs = "sched record -a"; }; - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + nixsetup = {stage, conf, ...}: with conf; w.nix-isolate { program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; + clusterName = "mn4"; }; extrae = {stage, conf, ...}: w.extrae { @@ -143,6 +143,10 @@ let inherit cc blocksize mpi gitBranch; }; + launch = w.launch.override { + nixPrefix = common.nixPrefix; + }; + stages = with common; [] # Use sbatch to request resources first ++ optional enableSbatch sbatch @@ -172,4 +176,4 @@ let in # We simply run each program one after another - w.launch jobs + launch jobs diff --git a/garlic/stages/launcher.nix b/garlic/stages/launcher/default.nix similarity index 62% rename from garlic/stages/launcher.nix rename to garlic/stages/launcher/default.nix index 8b32d1e..4b7fef8 100644 --- a/garlic/stages/launcher.nix +++ b/garlic/stages/launcher/default.nix @@ -1,5 +1,6 @@ { stdenv +, nixPrefix ? "" }: apps: # Each app must be unique @@ -10,9 +11,18 @@ stdenv.mkDerivation { buildInputs = [] ++ apps; apps = apps; - phases = [ "installPhase" ]; + phases = [ "unpackPhase" "patchPhase" "installPhase" ]; dontPatchShebangs = true; + src = ./.; + + inherit nixPrefix; + + patchPhase = '' + substituteAllInPlace run + substituteAllInPlace stage2 + ''; + installPhase = '' mkdir -p $out/apps for j in $apps; do @@ -29,17 +39,13 @@ stdenv.mkDerivation { done mkdir -p $out/bin - cat > $out/bin/run <&2 echo "Running launcher run stage" +env + +if [ -e /nix ]; then + >&2 echo "Cannot use the launcher inside nix environment!" + exit 1 +fi + +if [ -z "$GARLIC_OUT" ]; then + >&2 echo "GARLIC_OUT is not set" + exit 1 +fi + +mkdir -p "$GARLIC_OUT" +cd "$GARLIC_OUT" + +exec nix-isolate @nixPrefix@@out@/bin/stage2 diff --git a/garlic/stages/launcher/stage2 b/garlic/stages/launcher/stage2 new file mode 100644 index 0000000..dc0622b --- /dev/null +++ b/garlic/stages/launcher/stage2 @@ -0,0 +1,12 @@ +#!/bin/sh -ex + +>&2 echo "Running launcher stage2" + +if [ ! -e "/nix" ]; then + >&2 echo "Cannot execute stage2 outside from nix environment" + exit 1 +fi + +for j in @out@/apps/*; do + $j/bin/run +done diff --git a/garlic/stages/nix-isolate.nix b/garlic/stages/nix-isolate.nix new file mode 100644 index 0000000..1139477 --- /dev/null +++ b/garlic/stages/nix-isolate.nix @@ -0,0 +1,40 @@ +{ + stdenv +, nixtools +}: + +{ + program +, clusterName +}: + +stdenv.mkDerivation { + name = "nix-isolate"; + preferLocalBuild = true; + phases = [ "installPhase" ]; + dontPatchShebangs = true; + installPhase = '' + cat > $out <&2 echo Running nix-isolate stage + >&2 echo PATH=$PATH + >&2 echo Running env: + env + + # We need to enter the nix namespace first, in order to have /nix + # available, so we use this hack: + if [ ! -e /nix ]; then + exec ${nixtools}/bin/${clusterName}/nix-isolate \$0 + fi + + if [ -e /usr ]; then + >&2 echo "Environment not isolated, aborting" + exit 1 + fi + + exec ${program} + EOF + chmod +x $out + ''; +} diff --git a/garlic/stages/nix-setup.nix b/garlic/stages/nix-setup.nix index 600984c..b340b19 100644 --- a/garlic/stages/nix-setup.nix +++ b/garlic/stages/nix-setup.nix @@ -14,7 +14,10 @@ stdenv.mkDerivation { dontPatchShebangs = true; installPhase = '' cat > $out <&2 echo Running nixsetup stage + env # We need to enter the nix namespace first, in order to have /nix # available, so we use this hack: diff --git a/garlic/stages/sbatch.nix b/garlic/stages/sbatch.nix index d2a936d..2ba465a 100644 --- a/garlic/stages/sbatch.nix +++ b/garlic/stages/sbatch.nix @@ -1,6 +1,7 @@ { stdenv , numactl +, slurm }: { @@ -50,7 +51,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out cat > $out/job < $out/${name} <&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists" exit 1 fi mkdir -p "${chdirPrefix}/$(basename $out)" - echo sbatch ${nixPrefix}$out/job - sbatch ${nixPrefix}$out/job + echo ${slurm}/bin/sbatch ${nixPrefix}$out/job + ${slurm}/bin/sbatch ${nixPrefix}$out/job EOF chmod +x $out/${name} ''; diff --git a/garlic/stages/srun.nix b/garlic/stages/srun.nix index 48558b9..7c1d37e 100644 --- a/garlic/stages/srun.nix +++ b/garlic/stages/srun.nix @@ -1,5 +1,6 @@ { stdenv +, slurm }: { program @@ -14,8 +15,9 @@ stdenv.mkDerivation rec { dontPatchShebangs = true; installPhase = '' cat > $out <