Remove old pp stages
This commit is contained in:
parent
8bc0dc202d
commit
3eae92bdc4
@ -1,33 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
}:
|
||||
|
||||
resultTree:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "check";
|
||||
preferLocalBuild = true;
|
||||
phases = [ "installPhase" ];
|
||||
installPhase = ''
|
||||
echo "checking result tree: ${resultTree}"
|
||||
cd ${resultTree}
|
||||
for exp in *-experiment; do
|
||||
cd ${resultTree}/$exp
|
||||
echo "$exp: checking units"
|
||||
for unit in *-unit; do
|
||||
cd ${resultTree}/$exp/$unit
|
||||
if [ ! -e status ]; then
|
||||
echo "missing $unit/status file, aborting"
|
||||
exit 1
|
||||
fi
|
||||
st=$(cat status)
|
||||
if [ "$st" != "completed" ]; then
|
||||
echo "unit $unit is not complete yet, aborting"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "$exp: execution complete"
|
||||
done
|
||||
ln -s $out ${resultTree}
|
||||
'';
|
||||
}
|
||||
@ -1,64 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
, rsync
|
||||
, openssh
|
||||
, nix
|
||||
, curl
|
||||
, garlicTools
|
||||
}:
|
||||
|
||||
{
|
||||
sshHost
|
||||
, prefix
|
||||
, experimentStage
|
||||
, trebuchetStage
|
||||
, garlicTemp
|
||||
# We only fetch the config, stdout and stderr by default
|
||||
, fetchAll ? false
|
||||
}:
|
||||
|
||||
with garlicTools;
|
||||
|
||||
let
|
||||
experimentName = baseNameOf (toString experimentStage);
|
||||
rsyncFilter = if (fetchAll) then "" else ''
|
||||
--include='*/*/garlic_config.json' \
|
||||
--include='*/*/std*.log' \
|
||||
--include='*/*/*/std*.log' \
|
||||
--exclude='*/*/*/*' '';
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
name = "fetch";
|
||||
preferLocalBuild = true;
|
||||
|
||||
buildInputs = [ rsync openssh curl nix ];
|
||||
phases = [ "installPhase" ];
|
||||
# This doesn't work when multiple users have different directories where the
|
||||
# results are stored.
|
||||
#src = /. + "${prefix}${experimentName}";
|
||||
|
||||
installPhase = ''
|
||||
cat > $out << EOF
|
||||
#!/bin/sh -e
|
||||
mkdir -p ${garlicTemp}
|
||||
export PATH=$PATH
|
||||
rsync -av \
|
||||
--copy-links \
|
||||
${rsyncFilter} \
|
||||
'${sshHost}:${prefix}/${experimentName}' ${garlicTemp}
|
||||
|
||||
res=\$(nix-build -E '(with import ./default.nix; garlic.pp.getExpResult { \
|
||||
experimentStage = "${experimentStage}"; \
|
||||
trebuchetStage = "${trebuchetStage}"; \
|
||||
garlicTemp = "${garlicTemp}"; \
|
||||
})')
|
||||
|
||||
rm -rf ${garlicTemp}/${experimentName}
|
||||
|
||||
echo "The results for experiment ${experimentName} are at:"
|
||||
echo " \$res"
|
||||
|
||||
EOF
|
||||
chmod +x $out
|
||||
'';
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
, garlicTools
|
||||
, fetchExperiment
|
||||
}:
|
||||
|
||||
{
|
||||
trebuchetStage
|
||||
, experimentStage
|
||||
, garlicTemp
|
||||
}:
|
||||
|
||||
with garlicTools;
|
||||
|
||||
let
|
||||
experimentName = baseNameOf (toString experimentStage);
|
||||
fetcher = fetchExperiment {
|
||||
sshHost = "mn1";
|
||||
prefix = "/gpfs/projects/\\\$(id -gn)/\\\$(id -un)/garlic-out";
|
||||
garlicTemp = "/tmp/garlic-temp";
|
||||
inherit experimentStage trebuchetStage;
|
||||
};
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
name = "result";
|
||||
preferLocalBuild = true;
|
||||
__noChroot = true;
|
||||
|
||||
phases = [ "installPhase" ];
|
||||
|
||||
installPhase = ''
|
||||
expPath=${garlicTemp}/${experimentName}
|
||||
if [ ! -e $expPath ]; then
|
||||
echo "The experiment ${experimentName} is missing in ${garlicTemp}."
|
||||
echo "Please fetch it and try again."
|
||||
echo "You can execute ${trebuchetStage} to run the experiment."
|
||||
echo "And then ${fetcher} to get the results."
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p $out
|
||||
cp -a ${garlicTemp}/${experimentName} $out
|
||||
'';
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
, garlicTools
|
||||
, rsync
|
||||
}:
|
||||
|
||||
{
|
||||
trebuchetStage
|
||||
, experimentStage
|
||||
, garlicTemp
|
||||
# We only fetch the config, stdout and stderr by default
|
||||
, fetchAll ? false
|
||||
}:
|
||||
|
||||
with garlicTools;
|
||||
|
||||
let
|
||||
experimentName = baseNameOf (toString experimentStage);
|
||||
garlicOut = "/mnt/garlic-out";
|
||||
rsyncFilter = if (fetchAll) then "" else ''
|
||||
--include='*/*/garlic_config.json' \
|
||||
--include='*/*/std*.log' \
|
||||
--include='*/*/*/std*.log' \
|
||||
--exclude='*/*/*/*' '';
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
name = "result";
|
||||
preferLocalBuild = true;
|
||||
__noChroot = true;
|
||||
|
||||
buildInputs = [ rsync ];
|
||||
phases = [ "installPhase" ];
|
||||
|
||||
installPhase = ''
|
||||
expList=$(find ${garlicOut} -maxdepth 2 -name ${experimentName})
|
||||
|
||||
if [ -z "$expList" ]; then
|
||||
echo "ERROR: missing results for ${experimentName}"
|
||||
echo "Execute it by running:"
|
||||
echo
|
||||
echo -e " \e[30;48;5;2m${trebuchetStage}\e[0m"
|
||||
echo
|
||||
echo "cannot continue building $out, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
N=$(echo $expList | wc -l)
|
||||
echo "Found $N results: $expList"
|
||||
|
||||
if [ $N -gt 1 ]; then
|
||||
echo
|
||||
echo "ERROR: multiple results for ${experimentName}:"
|
||||
echo "$expList"
|
||||
echo
|
||||
echo "cannot continue building $out, aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exp=$expList
|
||||
repeat=1
|
||||
while [ 1 ]; do
|
||||
repeat=0
|
||||
cd $exp
|
||||
echo "$exp: checking units"
|
||||
for unit in *-unit; do
|
||||
cd $exp/$unit
|
||||
if [ ! -e status ]; then
|
||||
echo "$unit: no status"
|
||||
repeat=1
|
||||
else
|
||||
st=$(cat status)
|
||||
echo "$unit: $st"
|
||||
if [ "$st" != "completed" ]; then
|
||||
repeat=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $repeat -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
echo "waiting 10 seconds to try again"
|
||||
sleep 10
|
||||
done
|
||||
echo "$exp: execution complete, fething results"
|
||||
|
||||
mkdir -p $out
|
||||
#cp -aL $exp $out
|
||||
rsync -P -rt --copy-links ${rsyncFilter} $exp $out
|
||||
'';
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
}:
|
||||
|
||||
inputResult:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "timeResult";
|
||||
preferLocalBuild = true;
|
||||
phases = [ "installPhase" ];
|
||||
installPhase = ''
|
||||
mkdir -p $out
|
||||
cd ${inputResult}
|
||||
for unit in *-experiment/*-unit; do
|
||||
outunit=$out/$unit
|
||||
mkdir -p $outunit
|
||||
|
||||
# Copy the unit config
|
||||
conf="$unit/garlic_config.json"
|
||||
cp "$conf" "$outunit/garlic_config.json"
|
||||
|
||||
# Merge all runs in one single CSV file
|
||||
echo "run time" > $outunit/data.csv
|
||||
for r in $(cd $unit; ls -d [0-9]* | sort -n); do
|
||||
log="$unit/$r/stdout.log"
|
||||
awk "/^time /{print \"$r\", \$2}" $log >> $outunit/data.csv
|
||||
done
|
||||
done
|
||||
'';
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user