From 3eae92bdc46a335b34b1321081c1686d95803165 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 29 Oct 2020 17:52:03 +0100 Subject: [PATCH] Remove old pp stages --- garlic/pp/check.nix | 33 --------------- garlic/pp/fetch.nix | 64 ---------------------------- garlic/pp/result.nix | 43 ------------------- garlic/pp/result2.nix | 91 ---------------------------------------- garlic/pp/timeResult.nix | 30 ------------- 5 files changed, 261 deletions(-) delete mode 100644 garlic/pp/check.nix delete mode 100644 garlic/pp/fetch.nix delete mode 100644 garlic/pp/result.nix delete mode 100644 garlic/pp/result2.nix delete mode 100644 garlic/pp/timeResult.nix diff --git a/garlic/pp/check.nix b/garlic/pp/check.nix deleted file mode 100644 index dc67860..0000000 --- a/garlic/pp/check.nix +++ /dev/null @@ -1,33 +0,0 @@ -{ - stdenv -}: - -resultTree: - -stdenv.mkDerivation { - name = "check"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - echo "checking result tree: ${resultTree}" - cd ${resultTree} - for exp in *-experiment; do - cd ${resultTree}/$exp - echo "$exp: checking units" - for unit in *-unit; do - cd ${resultTree}/$exp/$unit - if [ ! -e status ]; then - echo "missing $unit/status file, aborting" - exit 1 - fi - st=$(cat status) - if [ "$st" != "completed" ]; then - echo "unit $unit is not complete yet, aborting" - exit 1 - fi - done - echo "$exp: execution complete" - done - ln -s $out ${resultTree} - ''; -} diff --git a/garlic/pp/fetch.nix b/garlic/pp/fetch.nix deleted file mode 100644 index d7f42a8..0000000 --- a/garlic/pp/fetch.nix +++ /dev/null @@ -1,64 +0,0 @@ -{ - stdenv -, rsync -, openssh -, nix -, curl -, garlicTools -}: - -{ - sshHost -, prefix -, experimentStage -, trebuchetStage -, garlicTemp -# We only fetch the config, stdout and stderr by default -, fetchAll ? false -}: - -with garlicTools; - -let - experimentName = baseNameOf (toString experimentStage); - rsyncFilter = if (fetchAll) then "" else '' - --include='*/*/garlic_config.json' \ - --include='*/*/std*.log' \ - --include='*/*/*/std*.log' \ - --exclude='*/*/*/*' ''; -in - stdenv.mkDerivation { - name = "fetch"; - preferLocalBuild = true; - - buildInputs = [ rsync openssh curl nix ]; - phases = [ "installPhase" ]; - # This doesn't work when multiple users have different directories where the - # results are stored. - #src = /. + "${prefix}${experimentName}"; - - installPhase = '' - cat > $out << EOF - #!/bin/sh -e - mkdir -p ${garlicTemp} - export PATH=$PATH - rsync -av \ - --copy-links \ - ${rsyncFilter} \ - '${sshHost}:${prefix}/${experimentName}' ${garlicTemp} - - res=\$(nix-build -E '(with import ./default.nix; garlic.pp.getExpResult { \ - experimentStage = "${experimentStage}"; \ - trebuchetStage = "${trebuchetStage}"; \ - garlicTemp = "${garlicTemp}"; \ - })') - - rm -rf ${garlicTemp}/${experimentName} - - echo "The results for experiment ${experimentName} are at:" - echo " \$res" - - EOF - chmod +x $out - ''; - } diff --git a/garlic/pp/result.nix b/garlic/pp/result.nix deleted file mode 100644 index e6beeb4..0000000 --- a/garlic/pp/result.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ - stdenv -, garlicTools -, fetchExperiment -}: - -{ - trebuchetStage -, experimentStage -, garlicTemp -}: - -with garlicTools; - -let - experimentName = baseNameOf (toString experimentStage); - fetcher = fetchExperiment { - sshHost = "mn1"; - prefix = "/gpfs/projects/\\\$(id -gn)/\\\$(id -un)/garlic-out"; - garlicTemp = "/tmp/garlic-temp"; - inherit experimentStage trebuchetStage; - }; -in - stdenv.mkDerivation { - name = "result"; - preferLocalBuild = true; - __noChroot = true; - - phases = [ "installPhase" ]; - - installPhase = '' - expPath=${garlicTemp}/${experimentName} - if [ ! -e $expPath ]; then - echo "The experiment ${experimentName} is missing in ${garlicTemp}." - echo "Please fetch it and try again." - echo "You can execute ${trebuchetStage} to run the experiment." - echo "And then ${fetcher} to get the results." - exit 1 - fi - mkdir -p $out - cp -a ${garlicTemp}/${experimentName} $out - ''; - } diff --git a/garlic/pp/result2.nix b/garlic/pp/result2.nix deleted file mode 100644 index 0a5d321..0000000 --- a/garlic/pp/result2.nix +++ /dev/null @@ -1,91 +0,0 @@ -{ - stdenv -, garlicTools -, rsync -}: - -{ - trebuchetStage -, experimentStage -, garlicTemp -# We only fetch the config, stdout and stderr by default -, fetchAll ? false -}: - -with garlicTools; - -let - experimentName = baseNameOf (toString experimentStage); - garlicOut = "/mnt/garlic-out"; - rsyncFilter = if (fetchAll) then "" else '' - --include='*/*/garlic_config.json' \ - --include='*/*/std*.log' \ - --include='*/*/*/std*.log' \ - --exclude='*/*/*/*' ''; -in - stdenv.mkDerivation { - name = "result"; - preferLocalBuild = true; - __noChroot = true; - - buildInputs = [ rsync ]; - phases = [ "installPhase" ]; - - installPhase = '' - expList=$(find ${garlicOut} -maxdepth 2 -name ${experimentName}) - - if [ -z "$expList" ]; then - echo "ERROR: missing results for ${experimentName}" - echo "Execute it by running:" - echo - echo -e " \e[30;48;5;2m${trebuchetStage}\e[0m" - echo - echo "cannot continue building $out, aborting" - exit 1 - fi - - N=$(echo $expList | wc -l) - echo "Found $N results: $expList" - - if [ $N -gt 1 ]; then - echo - echo "ERROR: multiple results for ${experimentName}:" - echo "$expList" - echo - echo "cannot continue building $out, aborting" - exit 1 - fi - - exp=$expList - repeat=1 - while [ 1 ]; do - repeat=0 - cd $exp - echo "$exp: checking units" - for unit in *-unit; do - cd $exp/$unit - if [ ! -e status ]; then - echo "$unit: no status" - repeat=1 - else - st=$(cat status) - echo "$unit: $st" - if [ "$st" != "completed" ]; then - repeat=1 - fi - fi - done - - if [ $repeat -eq 0 ]; then - break - fi - echo "waiting 10 seconds to try again" - sleep 10 - done - echo "$exp: execution complete, fething results" - - mkdir -p $out - #cp -aL $exp $out - rsync -P -rt --copy-links ${rsyncFilter} $exp $out - ''; - } diff --git a/garlic/pp/timeResult.nix b/garlic/pp/timeResult.nix deleted file mode 100644 index ec1b02a..0000000 --- a/garlic/pp/timeResult.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - stdenv -}: - -inputResult: - -stdenv.mkDerivation { - name = "timeResult"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - mkdir -p $out - cd ${inputResult} - for unit in *-experiment/*-unit; do - outunit=$out/$unit - mkdir -p $outunit - - # Copy the unit config - conf="$unit/garlic_config.json" - cp "$conf" "$outunit/garlic_config.json" - - # Merge all runs in one single CSV file - echo "run time" > $outunit/data.csv - for r in $(cd $unit; ls -d [0-9]* | sort -n); do - log="$unit/$r/stdout.log" - awk "/^time /{print \"$r\", \$2}" $log >> $outunit/data.csv - done - done - ''; -}