bscpkgs/garlic/pp/fetch.nix
Rodrigo Arias Mallo 4beb069627 WIP: postprocessing pipeline
Now each run is executed in a independent folder
2020-11-03 19:09:59 +01:00

59 lines
1.2 KiB
Nix

{
stdenv
, rsync
, openssh
, nix
, curl
, garlicTools
}:
{
sshHost
, prefix
, experimentStage
, trebuchetStage
, garlicTemp
# We only fetch the config, stdout and stderr by default
, fetchAll ? false
}:
with garlicTools;
let
experimentName = baseNameOf (toString experimentStage);
rsyncFilter = if (fetchAll) then "" else ''
--include='*/*/garlic_config.json' \
--include='*/*/std*.log' \
--include='*/*/*/std*.log' \
--exclude='*/*/*/*' '';
in
stdenv.mkDerivation {
name = "fetch";
preferLocalBuild = true;
buildInputs = [ rsync openssh curl ];
phases = [ "installPhase" ];
installPhase = ''
cat > $out << EOF
#!/bin/sh -e
mkdir -p ${garlicTemp}
export PATH=${rsync}/bin:${openssh}/bin:${nix}/bin
rsync -av \
--copy-links \
${rsyncFilter} \
'${sshHost}:${prefix}/${experimentName}' ${garlicTemp}
res=\$(nix-build -E '(with import ./default.nix; garlic.pp.getExpResult { \
experimentStage = "${experimentStage}"; \
trebuchetStage = "${trebuchetStage}"; \
garlicTemp = "${garlicTemp}"; \
})')
echo "The results for experiment ${experimentName} are at:"
echo " \$res"
EOF
chmod +x $out
'';
}