WIP: postprocessing pipeline

Now each run is executed in a independent folder
This commit is contained in:
2020-10-21 18:18:43 +02:00
parent 1321b6a888
commit 4beb069627
20 changed files with 232 additions and 279 deletions

58
garlic/pp/fetch.nix Normal file
View File

@@ -0,0 +1,58 @@
{
stdenv
, rsync
, openssh
, nix
, curl
, garlicTools
}:
{
sshHost
, prefix
, experimentStage
, trebuchetStage
, garlicTemp
# We only fetch the config, stdout and stderr by default
, fetchAll ? false
}:
with garlicTools;
let
experimentName = baseNameOf (toString experimentStage);
rsyncFilter = if (fetchAll) then "" else ''
--include='*/*/garlic_config.json' \
--include='*/*/std*.log' \
--include='*/*/*/std*.log' \
--exclude='*/*/*/*' '';
in
stdenv.mkDerivation {
name = "fetch";
preferLocalBuild = true;
buildInputs = [ rsync openssh curl ];
phases = [ "installPhase" ];
installPhase = ''
cat > $out << EOF
#!/bin/sh -e
mkdir -p ${garlicTemp}
export PATH=${rsync}/bin:${openssh}/bin:${nix}/bin
rsync -av \
--copy-links \
${rsyncFilter} \
'${sshHost}:${prefix}/${experimentName}' ${garlicTemp}
res=\$(nix-build -E '(with import ./default.nix; garlic.pp.getExpResult { \
experimentStage = "${experimentStage}"; \
trebuchetStage = "${trebuchetStage}"; \
garlicTemp = "${garlicTemp}"; \
})')
echo "The results for experiment ${experimentName} are at:"
echo " \$res"
EOF
chmod +x $out
'';
}