From 4533c94b4f367580b978803418965b3a760572b8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Oct 2023 12:24:58 +0200 Subject: [PATCH] Remove garlic from bscpkgs --- garlic/apps/bigsort/default.nix | 54 - garlic/apps/bigsort/genseq.nix | 46 - garlic/apps/bigsort/shuffle.nix | 46 - garlic/apps/cpic/default.nix | 43 - garlic/apps/creams/default.nix | 57 - garlic/apps/creams/gen_grid.py | 49 - garlic/apps/creams/git-table.nix | 12 - garlic/apps/creams/input.nix | 48 - garlic/apps/fwi/default.nix | 77 - garlic/apps/fwi/git-table.nix | 15 - garlic/apps/fwi/params.nix | 65 - garlic/apps/heat/default.nix | 39 - garlic/apps/heat/git-table.nix | 9 - garlic/apps/heat/print-times.patch | 13 - garlic/apps/hpccg/default.nix | 55 - garlic/apps/hpccg/git-table.nix | 9 - garlic/apps/hpcg/default.nix | 44 - garlic/apps/hpcg/git-table.nix | 18 - garlic/apps/hpcg/tampi.patch | 13 - garlic/apps/ifsker/default.nix | 60 - garlic/apps/ifsker/git-table.nix | 10 - garlic/apps/index.nix | 55 - garlic/apps/lulesh/default.nix | 48 - garlic/apps/lulesh/git-table.nix | 12 - garlic/apps/miniamr/default.nix | 48 - garlic/apps/nbody/default.nix | 59 - garlic/apps/nbody/git-table.nix | 13 - garlic/apps/ppong/default.nix | 35 - garlic/apps/saiph/default.nix | 106 -- garlic/apps/saiph/git-table.nix | 10 - garlic/bundleReport.nix | 31 - garlic/default.nix | 1 - garlic/develop/default.nix | 58 - garlic/develop/inputrc | 37 - garlic/doc/.gitignore | 6 - garlic/doc/Makefile | 42 - garlic/doc/blackbox.ms | 40 - garlic/doc/branch.ms | 22 - garlic/doc/execution.ms | 254 --- garlic/doc/gitbranch.pic | 152 -- garlic/doc/pp.ms | 256 --- garlic/doc/ref.i | 9 - garlic/doc/s.css | 19 - garlic/doc/slides/2.mm | 1468 ---------------- garlic/doc/slides/3.mm | 181 -- garlic/doc/slides/3.pdf | Bin 122774 -> 0 bytes garlic/doc/slides/Makefile | 29 - garlic/doc/slides/overview.svg | 479 ------ garlic/doc/slides/sat.png | Bin 79137 -> 0 bytes garlic/doc/slides/scalability.svg | 265 --- garlic/doc/slides/scaling-region.svg | 806 --------- garlic/doc/ug.ms | 1691 ------------------- garlic/ds/ctf-mode.py | 83 - garlic/ds/index.nix | 18 - garlic/ds/osu-bw.py | 64 - garlic/ds/osu-latency.py | 64 - garlic/ds/perf-stat.py | 90 - garlic/ds/py.nix | 34 - garlic/ds/std-timetable.py | 77 - garlic/exp/bigsort/genseq.nix | 73 - garlic/exp/bigsort/shuffle.nix | 102 -- garlic/exp/bigsort/sort.nix | 126 -- garlic/exp/cn6/nbody.nix | 164 -- garlic/exp/cn6/timediff.nix | 171 -- garlic/exp/creams/granularity.nix | 126 -- garlic/exp/creams/granularity16.nix | 132 -- garlic/exp/creams/size.nix | 131 -- garlic/exp/creams/ss.nix | 126 -- garlic/exp/examples/granularity.nix | 189 --- garlic/exp/fwi/common.nix | 128 -- garlic/exp/fwi/granularity.nix | 71 - garlic/exp/fwi/io.nix | 75 - garlic/exp/fwi/reuse.nix | 91 - garlic/exp/fwi/ss.nix | 90 - garlic/exp/heat/granularity.nix | 175 -- garlic/exp/hpcg/common.nix | 73 - garlic/exp/hpcg/gen.nix | 52 - garlic/exp/hpcg/granularity.nix | 76 - garlic/exp/hpcg/scaling.nix | 78 - garlic/exp/hpcg/size.nix | 65 - garlic/exp/index.nix | 108 -- garlic/exp/lulesh/test.nix | 85 - garlic/exp/nbody/common.nix | 37 - garlic/exp/nbody/granularity.nix | 60 - garlic/exp/nbody/numa.nix | 64 - garlic/exp/nbody/old/granularity-oss.nix | 60 - garlic/exp/nbody/old/nblocks.nix | 104 -- garlic/exp/nbody/old/scaling.nix | 110 -- garlic/exp/nbody/old/strong-scaling-oss.nix | 72 - garlic/exp/nbody/old/weak-scaling-mpi.nix | 63 - garlic/exp/nbody/old/weak-scaling-oss.nix | 63 - garlic/exp/nbody/ss.nix | 59 - garlic/exp/osu/bw.nix | 60 - garlic/exp/osu/eager.nix | 84 - garlic/exp/osu/impi.nix | 69 - garlic/exp/osu/latency.nix | 69 - garlic/exp/osu/mtu.nix | 84 - garlic/exp/saiph/granularity.nix | 132 -- garlic/exp/saiph/ss.nix | 122 -- garlic/exp/slurm/cpu.nix | 63 - garlic/exp/slurm/exit1.nix | 69 - garlic/exp/slurm/sigsegv.nix | 60 - garlic/fig/creams/granularity.R | 83 - garlic/fig/creams/granularity16.R | 136 -- garlic/fig/creams/size.R | 96 -- garlic/fig/creams/ss.R | 128 -- garlic/fig/dev/shell.nix | 14 - garlic/fig/examples/granularity.R | 195 --- garlic/fig/fwi/granularity.R | 74 - garlic/fig/fwi/io.R | 120 -- garlic/fig/fwi/ss.R | 95 -- garlic/fig/heat/cache.R | 78 - garlic/fig/heat/granularity.R | 68 - garlic/fig/heat/mode.R | 122 -- garlic/fig/hpcg/granularity.R | 63 - garlic/fig/hpcg/size.R | 71 - garlic/fig/hpcg/ss.R | 82 - garlic/fig/hpcg/ws.R | 71 - garlic/fig/index.nix | 97 -- garlic/fig/nbody/granularity.R | 62 - garlic/fig/nbody/numa.R | 86 - garlic/fig/nbody/old/baseline.R | 212 --- garlic/fig/nbody/old/freeCpu.R | 112 -- garlic/fig/nbody/old/jemalloc.R | 113 -- garlic/fig/nbody/old/scaling.R | 111 -- garlic/fig/nbody/ss.R | 66 - garlic/fig/osu/bw.R | 59 - garlic/fig/osu/eager.R | 63 - garlic/fig/osu/impi.R | 68 - garlic/fig/osu/latency.R | 77 - garlic/fig/osu/mtu.R | 69 - garlic/fig/saiph/granularity.R | 95 -- garlic/fig/saiph/ss.R | 110 -- garlic/garlicd/default.nix | 28 - garlic/garlicd/garlicd | 105 -- garlic/index.nix | 138 -- garlic/machines.nix | 63 - garlic/mpptest/default.nix | 16 - garlic/pp/hist/default.nix | 25 - garlic/pp/hist/hist.sh | 79 - garlic/pp/merge.nix | 24 - garlic/pp/rplot.nix | 165 -- garlic/pp/store.nix | 89 - garlic/report.nix | 24 - garlic/report/Makefile | 4 - garlic/report/report.tex | 31 - garlic/reportTar.nix | 28 - garlic/sedReport.nix | 19 - garlic/sh/default.nix | 41 - garlic/sh/fix-figure-subtitle.sh | 6 - garlic/sh/garlic | 298 ---- garlic/sh/garlic-add-copyright | 49 - garlic/sh/garlic-git-table | 27 - garlic/sh/garlic-propagate-commit | 63 - garlic/sh/garlic-query | 102 -- garlic/sh/garlic.1 | 61 - garlic/shell.nix | 29 - garlic/stages/baywatch.nix | 26 - garlic/stages/control.nix | 52 - garlic/stages/exec.nix | 42 - garlic/stages/experiment.nix | 60 - garlic/stages/extrae.nix | 34 - garlic/stages/isolate/default.nix | 44 - garlic/stages/isolate/stage1 | 58 - garlic/stages/isolate/stage2 | 16 - garlic/stages/perf.nix | 30 - garlic/stages/pgdb.nix | 30 - garlic/stages/runexp/default.nix | 29 - garlic/stages/runexp/runexp | 21 - garlic/stages/sbatch.nix | 105 -- garlic/stages/script.nix | 28 - garlic/stages/srun.nix | 58 - garlic/stages/strace.nix | 29 - garlic/stages/trebuchet.nix | 43 - garlic/stages/unit.nix | 107 -- garlic/stages/valgrind.nix | 29 - garlic/stdexp.nix | 148 -- garlic/tools.nix | 140 -- 178 files changed, 16994 deletions(-) delete mode 100644 garlic/apps/bigsort/default.nix delete mode 100644 garlic/apps/bigsort/genseq.nix delete mode 100644 garlic/apps/bigsort/shuffle.nix delete mode 100644 garlic/apps/cpic/default.nix delete mode 100644 garlic/apps/creams/default.nix delete mode 100755 garlic/apps/creams/gen_grid.py delete mode 100644 garlic/apps/creams/git-table.nix delete mode 100644 garlic/apps/creams/input.nix delete mode 100644 garlic/apps/fwi/default.nix delete mode 100644 garlic/apps/fwi/git-table.nix delete mode 100644 garlic/apps/fwi/params.nix delete mode 100644 garlic/apps/heat/default.nix delete mode 100644 garlic/apps/heat/git-table.nix delete mode 100644 garlic/apps/heat/print-times.patch delete mode 100644 garlic/apps/hpccg/default.nix delete mode 100644 garlic/apps/hpccg/git-table.nix delete mode 100644 garlic/apps/hpcg/default.nix delete mode 100644 garlic/apps/hpcg/git-table.nix delete mode 100644 garlic/apps/hpcg/tampi.patch delete mode 100644 garlic/apps/ifsker/default.nix delete mode 100644 garlic/apps/ifsker/git-table.nix delete mode 100644 garlic/apps/index.nix delete mode 100644 garlic/apps/lulesh/default.nix delete mode 100644 garlic/apps/lulesh/git-table.nix delete mode 100644 garlic/apps/miniamr/default.nix delete mode 100644 garlic/apps/nbody/default.nix delete mode 100644 garlic/apps/nbody/git-table.nix delete mode 100644 garlic/apps/ppong/default.nix delete mode 100644 garlic/apps/saiph/default.nix delete mode 100644 garlic/apps/saiph/git-table.nix delete mode 100644 garlic/bundleReport.nix delete mode 100644 garlic/default.nix delete mode 100644 garlic/develop/default.nix delete mode 100644 garlic/develop/inputrc delete mode 100644 garlic/doc/.gitignore delete mode 100644 garlic/doc/Makefile delete mode 100644 garlic/doc/blackbox.ms delete mode 100644 garlic/doc/branch.ms delete mode 100644 garlic/doc/execution.ms delete mode 100644 garlic/doc/gitbranch.pic delete mode 100644 garlic/doc/pp.ms delete mode 100644 garlic/doc/ref.i delete mode 100644 garlic/doc/s.css delete mode 100644 garlic/doc/slides/2.mm delete mode 100644 garlic/doc/slides/3.mm delete mode 100644 garlic/doc/slides/3.pdf delete mode 100644 garlic/doc/slides/Makefile delete mode 100644 garlic/doc/slides/overview.svg delete mode 100644 garlic/doc/slides/sat.png delete mode 100644 garlic/doc/slides/scalability.svg delete mode 100644 garlic/doc/slides/scaling-region.svg delete mode 100644 garlic/doc/ug.ms delete mode 100644 garlic/ds/ctf-mode.py delete mode 100644 garlic/ds/index.nix delete mode 100644 garlic/ds/osu-bw.py delete mode 100644 garlic/ds/osu-latency.py delete mode 100644 garlic/ds/perf-stat.py delete mode 100644 garlic/ds/py.nix delete mode 100644 garlic/ds/std-timetable.py delete mode 100644 garlic/exp/bigsort/genseq.nix delete mode 100644 garlic/exp/bigsort/shuffle.nix delete mode 100644 garlic/exp/bigsort/sort.nix delete mode 100644 garlic/exp/cn6/nbody.nix delete mode 100644 garlic/exp/cn6/timediff.nix delete mode 100644 garlic/exp/creams/granularity.nix delete mode 100644 garlic/exp/creams/granularity16.nix delete mode 100644 garlic/exp/creams/size.nix delete mode 100644 garlic/exp/creams/ss.nix delete mode 100644 garlic/exp/examples/granularity.nix delete mode 100644 garlic/exp/fwi/common.nix delete mode 100644 garlic/exp/fwi/granularity.nix delete mode 100644 garlic/exp/fwi/io.nix delete mode 100644 garlic/exp/fwi/reuse.nix delete mode 100644 garlic/exp/fwi/ss.nix delete mode 100644 garlic/exp/heat/granularity.nix delete mode 100644 garlic/exp/hpcg/common.nix delete mode 100644 garlic/exp/hpcg/gen.nix delete mode 100644 garlic/exp/hpcg/granularity.nix delete mode 100644 garlic/exp/hpcg/scaling.nix delete mode 100644 garlic/exp/hpcg/size.nix delete mode 100644 garlic/exp/index.nix delete mode 100644 garlic/exp/lulesh/test.nix delete mode 100644 garlic/exp/nbody/common.nix delete mode 100644 garlic/exp/nbody/granularity.nix delete mode 100644 garlic/exp/nbody/numa.nix delete mode 100644 garlic/exp/nbody/old/granularity-oss.nix delete mode 100644 garlic/exp/nbody/old/nblocks.nix delete mode 100644 garlic/exp/nbody/old/scaling.nix delete mode 100644 garlic/exp/nbody/old/strong-scaling-oss.nix delete mode 100644 garlic/exp/nbody/old/weak-scaling-mpi.nix delete mode 100644 garlic/exp/nbody/old/weak-scaling-oss.nix delete mode 100644 garlic/exp/nbody/ss.nix delete mode 100644 garlic/exp/osu/bw.nix delete mode 100644 garlic/exp/osu/eager.nix delete mode 100644 garlic/exp/osu/impi.nix delete mode 100644 garlic/exp/osu/latency.nix delete mode 100644 garlic/exp/osu/mtu.nix delete mode 100644 garlic/exp/saiph/granularity.nix delete mode 100644 garlic/exp/saiph/ss.nix delete mode 100644 garlic/exp/slurm/cpu.nix delete mode 100644 garlic/exp/slurm/exit1.nix delete mode 100644 garlic/exp/slurm/sigsegv.nix delete mode 100644 garlic/fig/creams/granularity.R delete mode 100644 garlic/fig/creams/granularity16.R delete mode 100644 garlic/fig/creams/size.R delete mode 100644 garlic/fig/creams/ss.R delete mode 100644 garlic/fig/dev/shell.nix delete mode 100644 garlic/fig/examples/granularity.R delete mode 100644 garlic/fig/fwi/granularity.R delete mode 100644 garlic/fig/fwi/io.R delete mode 100644 garlic/fig/fwi/ss.R delete mode 100644 garlic/fig/heat/cache.R delete mode 100644 garlic/fig/heat/granularity.R delete mode 100644 garlic/fig/heat/mode.R delete mode 100644 garlic/fig/hpcg/granularity.R delete mode 100644 garlic/fig/hpcg/size.R delete mode 100644 garlic/fig/hpcg/ss.R delete mode 100644 garlic/fig/hpcg/ws.R delete mode 100644 garlic/fig/index.nix delete mode 100644 garlic/fig/nbody/granularity.R delete mode 100644 garlic/fig/nbody/numa.R delete mode 100644 garlic/fig/nbody/old/baseline.R delete mode 100644 garlic/fig/nbody/old/freeCpu.R delete mode 100644 garlic/fig/nbody/old/jemalloc.R delete mode 100644 garlic/fig/nbody/old/scaling.R delete mode 100644 garlic/fig/nbody/ss.R delete mode 100644 garlic/fig/osu/bw.R delete mode 100644 garlic/fig/osu/eager.R delete mode 100644 garlic/fig/osu/impi.R delete mode 100644 garlic/fig/osu/latency.R delete mode 100644 garlic/fig/osu/mtu.R delete mode 100644 garlic/fig/saiph/granularity.R delete mode 100644 garlic/fig/saiph/ss.R delete mode 100644 garlic/garlicd/default.nix delete mode 100755 garlic/garlicd/garlicd delete mode 100644 garlic/index.nix delete mode 100644 garlic/machines.nix delete mode 100644 garlic/mpptest/default.nix delete mode 100644 garlic/pp/hist/default.nix delete mode 100755 garlic/pp/hist/hist.sh delete mode 100644 garlic/pp/merge.nix delete mode 100644 garlic/pp/rplot.nix delete mode 100644 garlic/pp/store.nix delete mode 100644 garlic/report.nix delete mode 100644 garlic/report/Makefile delete mode 100644 garlic/report/report.tex delete mode 100644 garlic/reportTar.nix delete mode 100644 garlic/sedReport.nix delete mode 100644 garlic/sh/default.nix delete mode 100755 garlic/sh/fix-figure-subtitle.sh delete mode 100755 garlic/sh/garlic delete mode 100755 garlic/sh/garlic-add-copyright delete mode 100755 garlic/sh/garlic-git-table delete mode 100755 garlic/sh/garlic-propagate-commit delete mode 100755 garlic/sh/garlic-query delete mode 100644 garlic/sh/garlic.1 delete mode 100644 garlic/shell.nix delete mode 100644 garlic/stages/baywatch.nix delete mode 100644 garlic/stages/control.nix delete mode 100644 garlic/stages/exec.nix delete mode 100644 garlic/stages/experiment.nix delete mode 100644 garlic/stages/extrae.nix delete mode 100644 garlic/stages/isolate/default.nix delete mode 100644 garlic/stages/isolate/stage1 delete mode 100644 garlic/stages/isolate/stage2 delete mode 100644 garlic/stages/perf.nix delete mode 100644 garlic/stages/pgdb.nix delete mode 100644 garlic/stages/runexp/default.nix delete mode 100755 garlic/stages/runexp/runexp delete mode 100644 garlic/stages/sbatch.nix delete mode 100644 garlic/stages/script.nix delete mode 100644 garlic/stages/srun.nix delete mode 100644 garlic/stages/strace.nix delete mode 100644 garlic/stages/trebuchet.nix delete mode 100644 garlic/stages/unit.nix delete mode 100644 garlic/stages/valgrind.nix delete mode 100644 garlic/stdexp.nix delete mode 100644 garlic/tools.nix diff --git a/garlic/apps/bigsort/default.nix b/garlic/apps/bigsort/default.nix deleted file mode 100644 index 4e7dd3d..0000000 --- a/garlic/apps/bigsort/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - stdenv -, lib -, cc -, nanos6 ? null -, mcxx ? null -, mpi -, gitBranch -}: - -with lib; -stdenv.mkDerivation rec { - name = "bigsort"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "${gitBranch}"; - }; - - #sourceRoot = "./BigSort"; - - preBuild = '' - cd BigSort - export I_MPI_CXX=${cc.CXX} - ''; - - buildInputs = [ - cc - mpi - ] - ++ optional (mcxx != null) mcxx - ++ optional (nanos6 != null) nanos6; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - "CLUSTER=MareNostrum4" - "OPENMP=yes" - "Debug=no" - "OPENMP_FLAGS=-qopenmp" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bigsort $out/bin/BigSort - ''; - - programPath = "/bin/BigSort"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/genseq.nix b/garlic/apps/bigsort/genseq.nix deleted file mode 100644 index 6036467..0000000 --- a/garlic/apps/bigsort/genseq.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "genseq"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/GenSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp genseq $out/bin/genseq - ''; - - programPath = "/bin/genseq"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/shuffle.nix b/garlic/apps/bigsort/shuffle.nix deleted file mode 100644 index 977d911..0000000 --- a/garlic/apps/bigsort/shuffle.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "shuffle"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/ShuffleSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp shuffle $out/bin/shuffle - ''; - - programPath = "/bin/shuffle"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/cpic/default.nix b/garlic/apps/cpic/default.nix deleted file mode 100644 index 284c1e9..0000000 --- a/garlic/apps/cpic/default.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ - stdenv -, libconfig -, nanos6 -, mpi -, uthash -, fftw -, tampi -, hdf5 -}: - -stdenv.mkDerivation rec { - name = "cpic"; - - # Use my current cpic version, so I can test changes without commits - #src = /home/Computational/rarias/cpic; - - src = builtins.fetchGit { - url = "https://github.com/rodarima/cpic"; -# rev = "73bd70448587f0925b89e24c8f17e412ea3958e6"; - ref = "simd"; - }; - - enableParallelBuilding = true; - dontStrip = true; - - buildInputs = [ - libconfig - nanos6 - mpi - uthash - fftw - tampi - hdf5 - ]; - - installPhase = '' - mkdir -p $out/bin - cp cpic $out/bin/cpic - ''; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/creams/default.nix b/garlic/apps/creams/default.nix deleted file mode 100644 index c083886..0000000 --- a/garlic/apps/creams/default.nix +++ /dev/null @@ -1,57 +0,0 @@ -{ - stdenv -, nanos6 -, mpi -, openmpi -, impi -, tampi -, mcxx -, gnuDef -, intelDef -, cc -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -assert (mpi == impi || mpi == openmpi); - -let - # FIXME: We should find a better way to specify the MPI implementation - # and the compiler. - mpiName = if mpi == openmpi then "OpenMPI" else "IntelMPI"; - compName = if cc == intelDef then "Intel" else "GNU"; - - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "creams"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/creams.exe"; - - buildInputs = [ nanos6 mpi cc tampi mcxx ]; - - configurePhase = '' - export TAMPI_HOME=${tampi} - - . etc/bashrc - - export FORTRAN_COMPILER=${compName} - export MPI_LIB=${mpiName} - - CREAMS_UPDATE_ENVIRONMENT - ''; - - installPhase = '' - mkdir -p $out/bin - cp -a build/* $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/creams/gen_grid.py b/garlic/apps/creams/gen_grid.py deleted file mode 100755 index 723988b..0000000 --- a/garlic/apps/creams/gen_grid.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -parser = argparse.ArgumentParser(description="Generate a grid.dat input file for CREAMS") - -parser.add_argument('--npx', type=int, help='number of processes in X', default=1) -parser.add_argument('--npy', type=int, help='number of processes in Y', default=1) -parser.add_argument('--npz', type=int, help='number of processes in Z', default=32) -parser.add_argument('--grain', type=int, help='granularity', default=9) -parser.add_argument('--nx', type=int, help='number of points in X', default=20) -parser.add_argument('--ny', type=int, help='number of points in Y', default=20) -parser.add_argument('--nz', type=int, help='number of points in Z', default=7000) -parser.add_argument('--dx', type=float, help='grid spacing in X', default=0.0025062657) -parser.add_argument('--dy', type=float, help='grid spacing in Y', default=0.0025062657) -parser.add_argument('--dz', type=float, help='grid spacing in Z', default=0.0025062657) - -args = parser.parse_args() - -grain_str = "%d %d" % (args.grain, args.grain) -boundary = "extrapolation" - -# Print -print(' %-49d number of processes in x-direction (0 if automatic)' % args.npx) -print(' %-49d number of processes in y-direction (0 if automatic)' % args.npy) -print(' %-49d number of processes in z-direction (0 if automatic)' % args.npz) -print(' ') -print(' %-49s subdomain granularity' % grain_str) -print(' ') -print(' %-49s -x boundary' % boundary) -print(' %-49s +x boundary' % boundary) -print(' %-49s -y boundary' % boundary) -print(' %-49s +y boundary' % boundary) -print(' %-49s -z boundary' % boundary) -print(' %-49s +z boundary' % boundary) -print(' ') -print(' x-direction') -for i in range(args.nx): - print("%.9e" % (i * args.dx)) -print(' ') -print(' y-direction') -for i in range(args.ny): - print("%.9e" % (i * args.dy)) -print(' ') -print(' z-direction') -for i in range(args.nz): - print("%.9e" % (i * args.dz)) -print(' ') -print(' END') diff --git a/garlic/apps/creams/git-table.nix b/garlic/apps/creams/git-table.nix deleted file mode 100644 index 9600240..0000000 --- a/garlic/apps/creams/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-03-31 for repo: - # ssh://git@bscpm03.bsc.es/pmartin1/creams-simplified.git - - "garlic/mpi+isend+omp+task" = "e6aa540820ee12d3d45d0eef8e7eeb2f0f1daea2"; - "garlic/mpi+isend+oss+task" = "016f33b8bec996a4546e8f08b1b6b1709f00499b"; - "garlic/mpi+send+omp+fork" = "e56e059264ad1bfe5e0c96a8b9303d21dd7fa20a"; - "garlic/mpi+send+omp+task" = "919580213de34bc5b6ba60c768c5dde5e501a1f6"; - "garlic/mpi+send+oss+task" = "adab8b66f27317d51445648302e7b133edf4837d"; - "garlic/mpi+send+seq" = "956125f9334493d31ceee3fa7024efa65bee9ca5"; - "garlic/tampi+isend+oss+task" = "14a121627679a251909d4b8103d260e27eac1d29"; -} diff --git a/garlic/apps/creams/input.nix b/garlic/apps/creams/input.nix deleted file mode 100644 index 5a12f91..0000000 --- a/garlic/apps/creams/input.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, python3 -, granul ? 9 -, nprocx ? 1 -, nprocy ? 1 -, nprocz ? 1 -, nx ? 20 -, ny ? 20 -, nz ? 7000 -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; - - gen = ./gen_grid.py; -in - stdenv.mkDerivation rec { - name = "creams-input"; - - buildInputs = [ python3 ]; - - inherit (gitSource) src gitBranch gitCommit; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -a SodTubeBenchmark $out/ - - python3 ${gen} \ - --npx ${toString nprocx} \ - --npy ${toString nprocy} \ - --npz ${toString nprocz} \ - --grain ${toString granul} \ - --nx ${toString nx} \ - --ny ${toString ny} \ - --nz ${toString nz} \ - > $out/SodTubeBenchmark/grid.dat - ''; - } diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix deleted file mode 100644 index 681509a..0000000 --- a/garlic/apps/fwi/default.nix +++ /dev/null @@ -1,77 +0,0 @@ -{ - stdenv -, lib -, mpi ? null -, tampi ? null -, mcxx ? null -, cc -, gitBranch ? "garlic/tampi+send+oss+task" -, gitCommit ? null -, fwiParams -, garlicTools -}: - -with lib; - -assert !(tampi != null && mcxx == null); - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # FIXME: Correct this on the Makefile so we can just type "make fwi" - # FIXME: Allow multiple MPI implementations - postPatch = '' - sed -i 's/= OPENMPI$/= INTEL/g' Makefile - sed -i 's/USE_O_DIRECT ?= NO/USE_O_DIRECT ?= YES/g' Makefile || true - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - - make depend - - cp ${fwiParams}/generated_model_params.h src/ - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - makeFlags = [ - #"COMPILER=GNU" - #"CC=${cc.cc.CC}" - "fwi" - ]; - - installPhase = '' - mkdir -p $out/bin - cp fwi $out/bin - ''; - - programPath = "/bin/fwi"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/fwi/git-table.nix b/garlic/apps/fwi/git-table.nix deleted file mode 100644 index c20adff..0000000 --- a/garlic/apps/fwi/git-table.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git - - "garlic/mpi+send+omp+fork" = "ea1ed53f20858dc082f9cbbe0e7e8fb28a6fe58a"; - "garlic/mpi+send+omp+task" = "aa8881056fb3fa98832d203899beacfb8fa702f6"; - "garlic/mpi+send+oss+task" = "c184484af8498fd939761575b34bb46ba3be0dde"; - "garlic/mpi+send+oss+task+NOREUSE" = "0062093ef744c694d69673d1881719958bbed353"; - "garlic/mpi+send+seq" = "cc184ad77f143481e2506933d0cdc038c349f071"; - "garlic/omp+task" = "1fe23690d74ae89ace5383ab165a6ce6346c2afd"; - "garlic/oss+task" = "68e6e8f17ee03addb460745acea2a38241ca89ee"; - "garlic/seq" = "aa6b6c5857125796c65fbf23018d557e4693f1ae"; - "garlic/tampi+isend+oss+task" = "7c98194e13786c4e8ecfa8a144587e5a95e09205"; - "garlic/tampi+send+oss+task" = "e08d66f7453c4034a363bb2d22c5248fe86ed740"; -} diff --git a/garlic/apps/fwi/params.nix b/garlic/apps/fwi/params.nix deleted file mode 100644 index 7fad46b..0000000 --- a/garlic/apps/fwi/params.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, nz ? 200 -, nx ? 200 -, ny ? 500 -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -with lib; -with builtins; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi-params"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - # Set the input size with the weird order (nz,nx,ny). - postPatch = '' - sed -i 1c${toString nz} SetupParams/fwi_params.txt - sed -i 2c${toString nx} SetupParams/fwi_params.txt - sed -i 3c${toString ny} SetupParams/fwi_params.txt - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - # Also, we need to compile it with the builder platform as target, as is going - # to be executed during the build to generate the src/generated_model_params.h - # header. - makeFlags = [ "COMPILER=GNU" "params" ]; - - installPhase = '' - mkdir -p $out/ - cp src/generated_model_params.h $out/ - cp SetupParams/fwi_params.txt $out/ - cp SetupParams/fwi_frequencies.txt $out/ - - mkdir -p $out/bin - cp ModelGenerator $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/default.nix b/garlic/apps/heat/default.nix deleted file mode 100644 index 02e8b76..0000000 --- a/garlic/apps/heat/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ - stdenv -, mpi -, tampi -, mcxx -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "heat"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - - name = "heat"; - - inherit (gitSource) src gitBranch gitCommit; - - patches = [ ./print-times.patch ]; - - buildInputs = [ mpi mcxx tampi ]; - - programPath = "/bin/${name}"; - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp heat.conf $out/etc/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/git-table.nix b/garlic/apps/heat/git-table.nix deleted file mode 100644 index c7670ac..0000000 --- a/garlic/apps/heat/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-01 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/heat.git - - "garlic/mpi+send+oss+task" = "947c80070d4c53e441df54b8bfac8928b10c5fb2"; - "garlic/mpi+send+seq" = "f41e1433808d0cbecd88a869b451c927747e5d42"; - "garlic/tampi+isend+oss+task" = "b1273f9b4db32ba6e15e3d41343e67407ce2f54f"; - "garlic/tampi+send+oss+task" = "554bec249f9aa23dd92edcfa2ada1e03e05e121d"; -} diff --git a/garlic/apps/heat/print-times.patch b/garlic/apps/heat/print-times.patch deleted file mode 100644 index bf5d28a..0000000 --- a/garlic/apps/heat/print-times.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/mpi/main.c b/src/mpi/main.c -index 44f4a99..08a1f5c 100644 ---- a/src/mpi/main.c -+++ b/src/mpi/main.c -@@ -83,6 +83,8 @@ int main(int argc, char **argv) - conf.rows, conf.cols, conf.rows/nranks, totalElements, totalElements/nranks, - conf.rbs, conf.cbs, nranks, threads, conf.timesteps, end-start, throughput); - printf("time %e\n", end - start); -+ printf("start_time %.9f\n", start); -+ printf("end_time %.9f\n", end); - } - - if (conf.generateImage) { diff --git a/garlic/apps/hpccg/default.nix b/garlic/apps/hpccg/default.nix deleted file mode 100644 index bb827a7..0000000 --- a/garlic/apps/hpccg/default.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - stdenv -, lib -, icc -, mpi ? null -, tampi ? null -, mcxx ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpccg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpccg"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/test_HPCCG-mpi.exe"; - - buildInputs = [ - icc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # The hpccg app fails to compile in parallel. Makefile must be fixed before. - enableParallelBuilding = false; - - makeFlags = [ - "USE_MPI=-DUSING_MPI" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp test_HPCCG-mpi.exe $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpccg/git-table.nix b/garlic/apps/hpccg/git-table.nix deleted file mode 100644 index 507dab2..0000000 --- a/garlic/apps/hpccg/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpccg.git - - "garlic/mpi+isend+omp+fork" = "c84af0480d231961201f2904ee4e3fced9d5f9be"; - "garlic/mpi+isend+seq" = "d1b47cd459440700de1b68233ec4fe794343dbd4"; - "garlic/tampi+isend+oss+task" = "7238e9be2e4a7b028abc05d40b476462eaa3de6a"; - "garlic/tampi+isend+oss+taskfor" = "02ec60f43b8d68d74575ea0563a9029fd441f1f1"; -} diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix deleted file mode 100644 index 245012e..0000000 --- a/garlic/apps/hpcg/default.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ - stdenv -, cc -, nanos6 -, mcxx -, mpi -, tampi -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpcg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpcg"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ - cc nanos6 mcxx mpi tampi - ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bin/* $out/bin/ - ''; - - programPath = "/bin/xhpcg"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpcg/git-table.nix b/garlic/apps/hpcg/git-table.nix deleted file mode 100644 index 0aa1fc8..0000000 --- a/garlic/apps/hpcg/git-table.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpcg.git - - "garlic/mpi" = "8c94ccfd97518ed947bd6be3386260b72fdcdff2"; - "garlic/mpi+SAVEMAT" = "5dd2ad9eba13dba67086f46c6e8519804d837383"; - "garlic/mpi+omp" = "d24c372dd9fda584e711efb612f172e5c3602804"; - "garlic/mpi+oss" = "519a867bb3a3e07440df05e60a62abad764524e5"; - "garlic/mpi+send+omp+fork" = "a08d31aedbc108e1c0081cdc5021827ac9022688"; - "garlic/omp" = "dcc8a40831cda884b9240af47e883ac997150ed3"; - "garlic/omp+SAVEMAT" = "40dbac86c905e192ecc8146e0e65e4c3a3c6dbf8"; - "garlic/omp+fork" = "042752b3dbcd9b0f4db524b6cdc911278ee1a51b"; - "garlic/omp+initsplit" = "5370e7ee26fb72ef100a79624f73ed2baa6bcc79"; - "garlic/oss" = "7e6e2d969b7904572f2475bf471e637651337761"; - "garlic/oss+task" = "034940756ccab88876609c3cba4dea0a0f5c944b"; - "garlic/seq" = "dee225571ab2572d7aa51df9846b01237ee941a1"; - "garlic/tampi+isend+oss+task" = "449a3980a767f91ca65d429490080961dcfba498"; -} diff --git a/garlic/apps/hpcg/tampi.patch b/garlic/apps/hpcg/tampi.patch deleted file mode 100644 index c233dfc..0000000 --- a/garlic/apps/hpcg/tampi.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- a/setup/Make.MPI_ICPC_OSS 2020-07-13 16:02:33.272257865 +0200 -+++ b/setup/Make.MPI_ICPC_OSS 2020-07-13 16:04:34.344413390 +0200 -@@ -91,8 +91,8 @@ - # - HPCG includes / libraries / specifics ------------------------------- - # ---------------------------------------------------------------------- - # --HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I{TAMPI_HOME}/include --HPCG_LIBS = ${TAMPI_HOME}/lib/libtampi.a -+HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I$(TAMPI_HOME)/include -+HPCG_LIBS = -l:libtampi.a - # - # - Compile time options ----------------------------------------------- - # diff --git a/garlic/apps/ifsker/default.nix b/garlic/apps/ifsker/default.nix deleted file mode 100644 index b575364..0000000 --- a/garlic/apps/ifsker/default.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, mpi -, gfortran -, tampi -, nanos6 -, mcxx -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "ifsker"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "ifsker"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ tampi mpi nanos6 mcxx gfortran ]; - - # Mercurium seems to fail when building with fortran in parallel - enableParallelBuilding = false; - - # FIXME: Patch mcxx to use other directory than $HOME for the lock - # files. - preConfigure = '' - export TAMPI_HOME=${tampi} - - # $HOME is required for the lock files by mcxx to compile fortran. - # So we use the $TMPDIR to store them. - export HOME=$TMPDIR - ''; - - makeFlags = [ - "-f" "Makefile.gcc" - ]; - - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp -r data $out/etc/ - cp nanos6.toml $out/etc - ''; - - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/ifsker/git-table.nix b/garlic/apps/ifsker/git-table.nix deleted file mode 100644 index 0242708..0000000 --- a/garlic/apps/ifsker/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/ifsker.git - - "garlic/mpi+isend+oss+fork" = "a6a20303101cb140571ddc1166e66843fbe83017"; - "garlic/mpi+isend+oss+task" = "a7bdb6c9b57aafbc50cdc790eb42e5bdd850f213"; - "garlic/mpi+isend+seq" = "bc97cf30835bbf6a825209485bb96fc8314e5bcb"; - "garlic/tampi+isend+oss+task" = "33408215bc231b70b60733fddea3d1b7431bb0d1"; - "garlic/tampi+send+oss+task" = "19dae956b4ef69916c0e8ad15bb6ced0085275cd"; -} diff --git a/garlic/apps/index.nix b/garlic/apps/index.nix deleted file mode 100644 index c0e2d4d..0000000 --- a/garlic/apps/index.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = callPackage ./nbody/default.nix { }; - - saiph = callPackage ./saiph/default.nix { - cc = bsc.clangOmpss2; - L3SizeKB = garlic.targetMachine.config.hw.cacheSizeKB.L3; - cachelineBytes = garlic.targetMachine.config.hw.cachelineBytes; - }; - - creams = callPackage ./creams/default.nix { - gnuDef = self.gfortran10 ; # Default GNU compiler version - intelDef = bsc.icc ; # Default Intel compiler version - }; - - creamsInput = callPackage ./creams/input.nix { }; - - hpcg = callPackage ./hpcg/default.nix { }; - - bigsort = { - sort = callPackage ./bigsort/default.nix { - gitBranch = "garlic/mpi+send+omp+task"; - }; - - genseq = callPackage ./bigsort/genseq.nix { }; - - shuffle = callPackage ./bigsort/shuffle.nix { }; - }; - - heat = callPackage ./heat/default.nix { }; - - miniamr = callPackage ./miniamr/default.nix { - variant = "ompss-2"; - }; - - ifsker = callPackage ./ifsker/default.nix { }; - - lulesh = callPackage ./lulesh/default.nix { }; - - hpccg = callPackage ./hpccg/default.nix { }; - - fwi = rec { - params = callPackage ./fwi/params.nix { }; - solver = callPackage ./fwi/default.nix { - fwiParams = params; - }; - }; -} diff --git a/garlic/apps/lulesh/default.nix b/garlic/apps/lulesh/default.nix deleted file mode 100644 index 8f64682..0000000 --- a/garlic/apps/lulesh/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, impi -, mcxx -, icc -, tampi ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "lulesh"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "lulesh"; - - inherit (gitSource) src gitBranch gitCommit; - - dontConfigure = true; - - preBuild = optionalString (tampi != null) "export TAMPI_HOME=${tampi}"; - - #TODO: Allow multiple MPI implementations and compilers - buildInputs = [ - impi - icc - mcxx - ]; - - enableParallelBuilding = true; - - #TODO: Can we build an executable named "lulesh" in all branches? - installPhase = '' - mkdir -p $out/bin - find . -name 'lulesh*' -type f -executable -exec cp \{\} $out/bin/${name} \; - ''; - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/lulesh/git-table.nix b/garlic/apps/lulesh/git-table.nix deleted file mode 100644 index 49c2bba..0000000 --- a/garlic/apps/lulesh/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/lulesh.git - - "garlic/mpi+isend+omp+fork" = "6cc85c55cb4840d6cde12bb285f5ab1ae7878618"; - "garlic/mpi+isend+oss+task" = "0a9e2cd1d64ab4fcf1860ace02866278ad289637"; - "garlic/mpi+isend+seq" = "9df5475c7dd2b345559fae5bd07ceea38f2e7b91"; - "garlic/tampi+isend+oss+task" = "28ce0cd69f9b4e65eff8141ec455d5f60e9b98b3"; - "garlic/tampi+isend+oss+taskfor" = "928f315ea426585a32231d950da651399e48d762"; - "garlic/tampi+isend+oss+taskloop" = "7957c1a2c84ae80edddcec9eafe7efdeefa68d58"; - "garlic/tampi+isend+oss+taskloopfor" = "7efa0535130a6726f5a46669cf171412d21adc9b"; -} diff --git a/garlic/apps/miniamr/default.nix b/garlic/apps/miniamr/default.nix deleted file mode 100644 index a014a73..0000000 --- a/garlic/apps/miniamr/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, tampi -, clangOmpss2 -, mpi -, nanos6 -, mcxx -, variant -}: - -with lib; - -assert (assertOneOf "variant" variant [ "openmp" "openmp-tasks" "ompss-2" ]); - -let - cc=mcxx; -in -stdenv.mkDerivation rec { - name = "miniamr"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/ksala/miniamr.git"; - ref = "master"; - }; - - postUnpack = '' - sourceRoot=$sourceRoot/${variant} - ''; - - buildInputs = [ tampi clangOmpss2 mpi nanos6 mcxx ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp miniAMR.x $out/bin/ - ''; - - programPath = "/bin/miniAMR.x"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/nbody/default.nix b/garlic/apps/nbody/default.nix deleted file mode 100644 index d5de8ce..0000000 --- a/garlic/apps/nbody/default.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi ? null -, tampi ? null -, mcxx ? null -, cflags ? null -, gitBranch ? "garlic/seq" -, gitCommit ? null -, blocksize ? 2048 -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "nbody"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "nbody"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/nbody"; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - preBuild = (if cflags != null then '' - makeFlagsArray+=(CFLAGS="${cflags}") - '' else ""); - - makeFlags = [ - "CC=${cc.CC}" - "BS=${toString blocksize}" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp nbody* $out/bin/${name} - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/nbody/git-table.nix b/garlic/apps/nbody/git-table.nix deleted file mode 100644 index e824251..0000000 --- a/garlic/apps/nbody/git-table.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/nbody.git - - "garlic/mpi+send+oss+task" = "20aa856baa3268d99262588807911ad0b3318d09"; - "garlic/mpi+send+seq" = "3be64af0f949db5fd60fcd0334cf2cd8c9fa25c3"; - "garlic/omp+fork" = "9c869272df7c775f467a2220211a414e33321c00"; - "garlic/oss+task" = "13ab26fbad8662a1052cc94410386080bbf6a2ba"; - "garlic/seq" = "9dfea29189d14477bd75e6f741f0518e7e4e5e72"; - "garlic/seq+BLOCK" = "99408705628b374df4308dcf1cdbe2d21d1451c2"; - "garlic/tampi+isend+oss+task" = "653d26e4a0913d36ea18d4e72e65a04838bb138a"; - "garlic/tampi+send+oss+task" = "b1440ebc5f79165e5dfaa6a4ce7916eda410ec9a"; -} diff --git a/garlic/apps/ppong/default.nix b/garlic/apps/ppong/default.nix deleted file mode 100644 index 09a72a4..0000000 --- a/garlic/apps/ppong/default.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "ppong"; - - src = fetchurl { - url = "http://www.csl.mtu.edu/cs4331/common/PPong.c"; - sha256 = "0d1w72gq9627448cb7ykknhgp2wszwd117dlbalbrpf7d0la8yc0"; - }; - - unpackCmd = '' - mkdir src - cp $src src/ppong.c - ''; - - dontConfigure = true; - - buildPhase = '' - echo mpicc -include stdlib.h ppong.c -o ppong - mpicc -include stdlib.h ppong.c -o ppong - ''; - - installPhase = '' - mkdir -p $out/bin - cp ppong $out/bin/ppong - ln -s $out/bin/ppong $out/bin/run - ''; - - buildInputs = [ mpi ]; - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/saiph/default.nix b/garlic/apps/saiph/default.nix deleted file mode 100644 index bb81c89..0000000 --- a/garlic/apps/saiph/default.nix +++ /dev/null @@ -1,106 +0,0 @@ -{ - stdenv -, lib -, nanos6 -, mpi -, tampi -, cc -, vtk -, boost -, gitBranch ? "master" -, gitCommit ? null -, enableManualDist ? false -, nbgx ? null -, nbgy ? null -, nbgz ? null -, nblx ? null -, nbly ? null -, nblz ? null -, nsteps ? null -, numComm ? null -, enableVectFlags ? false -, enableDebugFlags ? false -, enableAsanFlags ? false -, cachelineBytes ? null -, L3SizeKB ? null -# Problem size: -, sizex ? 3 -, sizey ? 4 -, sizez ? 4 -, garlicTools -}: - -assert enableManualDist -> (nbgx != null); -assert enableManualDist -> (nbgy != null); -assert enableManualDist -> (nbgz != null); - -with lib; -with lib.versions; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "saiph"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "saiph"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/Heat3D_vect"; - - enableParallelBuilding = true; - dontStrip = true; - enableDebugging = true; - - buildInputs = [ - nanos6 - mpi - tampi - cc - vtk - boost - ]; - - preBuild = '' - cd saiphv2/cpp/src - export VTK_VERSION=${majorMinor (getVersion vtk.name)} - export VTK_HOME=${vtk} - make clean - - sed -i '/SIZEX =/s/3/${toString sizex}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEY =/s/4/${toString sizey}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEZ =/s/4/${toString sizez}/g' testApp/Heat3D_vect.cpp - ''; - - makeFlags = [ - "-f" "Makefile.${cc.CC}" - "apps" - "APP=Heat3D_vect" - ] ++ optional (cachelineBytes != null) "ROW_ALIGNMENT=${toString cachelineBytes}" - ++ optional (L3SizeKB != null) "L3_SIZE_K=${toString L3SizeKB}" - ++ optional (enableManualDist) "DIST_SET=1" - ++ optional (enableManualDist) "NBG_X=${toString nbgx}" - ++ optional (enableManualDist) "NBG_Y=${toString nbgy}" - ++ optional (enableManualDist) "NBG_Z=${toString nbgz}" - ++ optional (nblx != null) "NBL_X=${toString nblx}" - ++ optional (nbly != null) "NBL_Y=${toString nbly}" - ++ optional (nblz != null) "NBL_Z=${toString nblz}" - ++ optional (nsteps != null) "NSTEPS=${toString nsteps}" - ++ optional (numComm != null) "NUM_COMM=${toString numComm}" - ++ optional (enableVectFlags) "VECT_CHECKS=1" - ++ optional (enableDebugFlags) "DEBUG_CHECKS=1" - ++ optional (enableAsanFlags) "SANITIZE_CHECKS=1" - ; - - installPhase = '' - mkdir -p $out/lib - mkdir -p $out/bin - cp obj/libsaiphv2.so $out/lib/ - cp bin/Heat3D_vect $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/saiph/git-table.nix b/garlic/apps/saiph/git-table.nix deleted file mode 100644 index 13cc477..0000000 --- a/garlic/apps/saiph/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/saiph.git - - "garlic/mpi+isend+omp+fork+simd" = "96823846b327b6860f05d428f0cd5ed8ca537a0e"; - "garlic/mpi+isend+omp+task+simd" = "de0346a559120f561bff554aa86b34d01214b714"; - "garlic/mpi+isend+seq+simd" = "1411dad765231f5d3cec9f621526583974232d42"; - "garlic/tampi+isend+omp+task+simd" = "587a7651df8eb69cae4a79bdfc5cb7f50723f3ce"; - "garlic/tampi+isend+oss+task+simd" = "3731197d3e35df248fa6bdb7e4cb05c5dd4f2597"; -} diff --git a/garlic/bundleReport.nix b/garlic/bundleReport.nix deleted file mode 100644 index fd84775..0000000 --- a/garlic/bundleReport.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - stdenv -, fig -}: - -stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildPhase = '' - pwd - ls -l - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - for line in $(cat list); do - localPath=$(echo $line | tr '.' '/') - echo "mkdir -p $localPath" >> fun.nix - echo "cp -r \''${$line}/* $localPath" >> fun.nix - echo "sed -i 's;@$line@;$localPath;g' report.tex" >> fun.nix - done - echo "'''" >> fun.nix - - echo " ---------- this is the fun.nix -------------" - cat fun.nix - echo " --------------------------------------------" - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/default.nix b/garlic/default.nix deleted file mode 100644 index af0a194..0000000 --- a/garlic/default.nix +++ /dev/null @@ -1 +0,0 @@ -import ../default.nix diff --git a/garlic/develop/default.nix b/garlic/develop/default.nix deleted file mode 100644 index 6ea0930..0000000 --- a/garlic/develop/default.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, bash -, bashInteractive -, busybox -, extraInputs ? [] -}: - -let - inputrc = ./inputrc; -in -stdenv.mkDerivation { - name = "develop"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = extraInputs ++ [ busybox ]; - installPhase = '' - cat > $out < $@ - -%.html: %.ms Makefile - mkdir -p img - REFER=ref.i groff -ms -mwww $(HTML_OPT) $< > $@ - echo $(HTML_OPT) - sed -i '/<\/head>/i' $@ - sed -i 's/^<\/a>/\§<\/a>/g' $@ - #sed -i '/

/s/^ $@ - -killall -HUP mupdf - -%.utf8: %.ms - REFER=ref.i groff -ms -mwww $(PREPROC) $(TTYOPT) -Tutf8 $^ > $@ - -%.ascii: %.ms - REFER=ref.i groff -ms -mwww -c $(PREPROC) $(TTYOPT) -Tascii $^ > $@ - -doc.tar.gz: ug.pdf ug.html s.css - tar czf $@ $^ img s.css - -clean: - rm -rf img ug.pdf ug.html doc.tar.gz diff --git a/garlic/doc/blackbox.ms b/garlic/doc/blackbox.ms deleted file mode 100644 index 313a0f5..0000000 --- a/garlic/doc/blackbox.ms +++ /dev/null @@ -1,40 +0,0 @@ -.\" Use helvetica family -.fam H -.PS -moveht=0.1 -boxwid=1 -sht=boxht + 0.1 -hspace = boxwid + 0.2 -right -G: [ - boxwid=1 - extrawid=1.8 - right - A: box "nix"; arrow; - B1: box wid extrawid "App source code" "PM branch, defines..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B2: box wid extrawid "App run config" "Input size, algorithm..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B3: box wid extrawid "Build config" "MPI impl, O3, CC version..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B4: box wid extrawid "Run config" "Nodes, affinity"; - move to (A.sw.x, A.y - boxht - moveht) - A: box "MN4"; arrow; - B5: box wid extrawid "Hardware" "Cache size, intercomm..."; -] -movewid=1 -move -circlerad=0.4 -E: circle "Execution" -arrow -box "Result" -rspline = 0.5 -arrow from G.B1.e to E chop 0 chop circlerad -arrow from G.B2.e to E chop 0 chop circlerad -arrow from G.B3.e to E chop 0 chop circlerad -arrow from G.B4.e to E chop 0 chop circlerad -arrow from G.B5.e to E chop 0 chop circlerad -.PE diff --git a/garlic/doc/branch.ms b/garlic/doc/branch.ms deleted file mode 100644 index b4e6216..0000000 --- a/garlic/doc/branch.ms +++ /dev/null @@ -1,22 +0,0 @@ -.\".fam CMU -.\".TL -.\"Garlic: git branch name -.\".AU -.\"Rodrigo Arias Mallo -.\".AI -.\"Barcelona Supercomputing Center -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -\".NH 1 -\"Instructions -\".LP -\"To name a branch of a program, please follow the flowchart below. -.ps -2 -.PS 5/25.4 -copy "gitbranch.pic" -.PE diff --git a/garlic/doc/execution.ms b/garlic/doc/execution.ms deleted file mode 100644 index 81742d3..0000000 --- a/garlic/doc/execution.ms +++ /dev/null @@ -1,254 +0,0 @@ -.TL -Garlic: the execution pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the execution of experiments in the Garlic -benchmark, which are performed under strict conditions. The several -stages of the execution are documented so the experimenter can have a -global overview of how the benchmark runs under the hood. -The results of the experiment are stored in a known path to be used in -posterior processing steps. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -.NH 1 -Introduction -.LP -Every experiment in the Garlic -benchmark is controlled by a single -.I nix -file placed in the -.CW garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 1 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CW /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CW /usr -or -.CW /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 1 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.NH 2 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CW /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.NH 2 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 2 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 2 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.KF -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Target Safe Copies User Std -_ -trebuchet xeon no no yes yes -runexp login no no yes yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate comp no no no yes -control comp yes no no yes -srun comp yes no no yes -isolate comp no yes no yes -_ -exec comp yes yes no no -program comp yes yes no no -_ -.TE -.QS -.B "Table 1" : -The stages of a complete execution pipeline. The -.B target -column determines where the stage is running, -.B safe -states if the stage begins the execution inside the sandbox, -.B user -if it can be executed directly by the user, -.B copies -if there are several instances running in parallel and -.B std -if is part of the standard execution pipeline. -.QE -.KE diff --git a/garlic/doc/gitbranch.pic b/garlic/doc/gitbranch.pic deleted file mode 100644 index 939e926..0000000 --- a/garlic/doc/gitbranch.pic +++ /dev/null @@ -1,152 +0,0 @@ -#.ps -3 -#.fam CMU -#.PS 4.5/25.4 # Scale drawing to 20/25.4 in = -# = 20/25.4[in]/25.4[mm/in] = 20 mm -# FLOWCHART - Basic flow chart blocks. -scale=25.4 #Scale units from inches to mm -csize=2.0 #Cell size in mm -pstricks=0 -dx=0; dy=2; -define process -{[ - box $1; -]} -# decision(): rhomboid -> if block -define decision {[ - boxwid=boxwid*1.2 - boxht=boxht*1.2 - B: box invis $1; - line from B.n to B.e to B.s to B.w to B.n; -]} - -#--- END OF MACROS --- -boxwid=30 -fillval=1 -circlerad=10 -down -START: circle "Start" -arrow -D_MPIB: decision("MPI-based?") -arrow " Yes" ljust -D_TAMPI: decision("TAMPI?") -arrow " Yes" ljust -TAMPI: process("\fB+tampi\fP") -right -move to D_TAMPI.e -arrow "No" above -D_MPI: decision("MPI?") -down -move to D_MPI.s -arrow " Yes" ljust -MPI: process("\fB+mpi\fP") -move to TAMPI.s -A_TAMPI: arrow linewid -line from MPI.s to MPI.c - (0,boxht) \ - to A_TAMPI.c -circle at A_TAMPI.c rad 0.7 filled -move at A_TAMPI -D_ISEND: decision("MPI_Isend()?") -arrow " Yes" ljust -ISEND: process("\fB+isend\fP") -A_ISEND: arrow -right -move to D_ISEND.e -arrow "No" above -D_SEND: decision("MPI_Send()?") -down -move to D_SEND.s -arrow " Yes" ljust -SEND: process("\fB+send\fP") -right -move to D_SEND.e -arrow "No" above -D_RMA: decision("MPI_Get()?") -down -move to D_RMA.s -arrow " Yes" ljust -RMA: process("\fB+rma\fP") -line "No" above from D_MPIB.w to D_MPIB.w - (boxwid,0) -line to (D_MPIB.w.x-boxwid, A_ISEND.c.y) \ - to A_ISEND.c -line from SEND.s to SEND.c - (0,boxht) \ - to A_ISEND.c -line from RMA.s to RMA.c - (0,boxht) \ - to SEND.c - (0,boxht) -circle at A_ISEND.c rad 0.7 filled -move at A_ISEND -D_MT: decision("multithread?") -arrow " Yes" ljust -D_OMP: decision("OpenMP?") -arrow " Yes" ljust -OMP: process("\fB+omp\fP") -right -move to D_OMP.e -arrow "No" above -D_OSS: decision("OmpSs-2?") -down -move to D_OSS.s -arrow " Yes" ljust -OSS: process("\fB+oss\fP") -down -move to OMP.s -A_OMP: arrow -circle at A_OMP.c rad 0.7 filled -line from OSS.s to OSS.c - (0,boxht) \ - to A_OMP.c -move to A_OMP.s -D_FJ: decision("fork-join?") -arrow " Yes" ljust -FJ: process("\fB+fork\fP") -right -move to D_FJ.e -arrow "No" above -D_TASKFOR: decision("task for?") -arrow "No" above -down -move to D_TASKFOR.s -arrow " Yes" ljust -TASKFOR: process("\fB+taskfor\fP") -right -move to D_TASKFOR.e -arrow "No" above -D_TASK: decision("task model?") -down -move to D_TASK.s -arrow " Yes" ljust -TASK: process("\fB+task\fP") -move to FJ.s -A_FJ: arrow -circle at A_FJ.c rad 0.7 filled -line from TASKFOR.s to TASKFOR.c - (0,boxht) \ - to A_FJ.c -line from TASK.s to TASK.c - (0,boxht) \ - to TASKFOR.c - (0,boxht) -left -move to OMP.c - (boxwid,0) -SEQ: process("\fB+seq\fP") -line "No" above from D_MT.w to (SEQ.x, D_MT.w.y) -arrow to SEQ.n -line from SEQ.s to (SEQ.s.x, A_FJ.c.y) to A_FJ.c -down -move to A_FJ.s -D_SIMD: decision("SIMD opt.?") -move to D_SIMD.e -right -arrow "Yes" above -SIMD: process("\fB+simd\fP") -down -move to D_SIMD.s -arrow " No" ljust -END: circle "End" -circle radius circlerad*0.9 at END -arrow from SIMD.s to (SIMD.x, END.y) to END.e - -# Error lines -ERR: circle "Error" at (TASK.x+boxwid, END.y) -circle radius circlerad*0.9 at ERR -line "No" above from D_TASK.e to (ERR.n.x,D_TASK.e.y) -line "No" above from D_OSS.e to (ERR.n.x,D_OSS.e.y) -line "No" above from D_RMA.e to (ERR.n.x,D_RMA.e.y) -line "No" above from D_MPI.e to (ERR.n.x,D_MPI.e.y) -arrow to ERR.n -#.PE diff --git a/garlic/doc/pp.ms b/garlic/doc/pp.ms deleted file mode 100644 index 700057a..0000000 --- a/garlic/doc/pp.ms +++ /dev/null @@ -1,256 +0,0 @@ -.TL -Garlic: the postprocess pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the format used to store the results of the -execution of experiments and the postprocess steps used to generate a -set of figures from the results to present the data. The several stages -of the postprocess pipeline are documented to provide a general picture. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -.\".2C -.R1 -bracket-label " [" ] ", " -accumulate -move-punctuation -.R2 -.\"##################################################################### -.NH 1 -Introduction -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 1 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 1 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.ie t \{\ -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -.\} -.el \{\ -.nf - - Prog ---> EP ---> Result ---> PP ---> Plot - -.fi -.\} -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 1 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.SH 1 -Appendix A: Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. diff --git a/garlic/doc/ref.i b/garlic/doc/ref.i deleted file mode 100644 index 1bd3e70..0000000 --- a/garlic/doc/ref.i +++ /dev/null @@ -1,9 +0,0 @@ -%A Rodrigo Arias Mallo -%D 2020 -%T Garlic: the execution pipeline - -%A Winston Chang -%T R Graphics Cookbook: Practical Recipes for Visualizing Data -%D 2020 -%I O'Reilly Media -%O 2nd edition diff --git a/garlic/doc/s.css b/garlic/doc/s.css deleted file mode 100644 index 324f1ce..0000000 --- a/garlic/doc/s.css +++ /dev/null @@ -1,19 +0,0 @@ -html { - line-height: 1.6; - margin-bottom: 50px; - padding-bottom: 80px; -} - -body { - max-width: 700px; - text-align: justify; - margin:0 auto; -} - -pre { - overflow: auto; - display: block; - padding: 3px 3px; - line-height: 1.4; - background-color: #eeeeee; -} diff --git a/garlic/doc/slides/2.mm b/garlic/doc/slides/2.mm deleted file mode 100644 index 7cb81ce..0000000 --- a/garlic/doc/slides/2.mm +++ /dev/null @@ -1,1468 +0,0 @@ -.\"usage: NS title -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "November 24, 2020" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic update -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Changelog" -Important changes since the last meeting (2020-09-23) -.BL -.LI -Execution of experiments is now \fBisolated\fP: no $HOME or /usr at run time -.LI -Added a \fBpostprocess\fP pipeline -.LI -New \fBgarlic(1)\fP helper tool (manual included) -.LI -A plot has an experiment result as \fBdependency\fP -.LI -Experiments run on demand based on article \fBfigures\fP -.LI -Fast pkg overrides (MPI) -.LE 1 -.\"================================================================== -.NS "Overview" -Dependency graph of a complete experiment that produces a figure. Each box -is a derivation and arrows represent \fBbuild dependencies\fP. -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The \fBResult\fP is not covered by nix (yet). This is what it looks like -when executed: -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -.PE -.S P P -.DE -.P -Notice dependency order is not the same as execution order. -.\"================================================================== -.NS "Building the execution pipeline (EP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" fill -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Run nix-build with the experiment name: -.P -.VERBON -xeon07$ nix-build -A exp.nbody.baseline -\&... -/nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.VERBOFF -.P -Outputs the first stage (the trebuchet). All other stages -are built as dependencies, as they are required to build the trebuchet. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: circle at B + (0,-1.3) "trebu." -arrow -circle "runexp" -arrow -circle "isolate" -arrow -circle "exp." -arrow -circle "..." -arrow -circle "exec" -arrow -P: circle "program" -line from R.sw to T.nw dashed -line from R.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ -"Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.SP 1m -.P -The stages are launched sequentially. Let see what happens in each one. -.\"================================================================== -.NS "Execution pipeline" -.2C -List of stages required to run the program of the experiment: -.BL -.S -1 -.LI -The -.B target -column determines where the stage is running. -.LI -.B Safe -states if the stage begins the execution inside the isolated namespace -.LI -.B User -if it can be executed directly by the user -.LI -.B Copies -if there are several instances running in parallel and -.LI -.B Std -if is part of the standard execution pipeline. -.LE -.S P P -.P -Sorted by the \fBexecution order\fP. -.\" Go to the next column -.NCOL -.KF -.defcolor white rgb #FFFFFF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\m[white]\(rh\m[]\ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBtrebuchet\fP: connects via ssh to the target machine and executes the -next stage there. -.P -The target machine is set to MN4, which by default uses the host -\fBmn1\fP -.P -Literally: -.P -.VERBON -ssh mn1 /path/to/next/stage -.VERBOFF -.P -You need to define the ssh config to be able to connect to mn1. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBrunexp\fP: sets a few \fCGARLIC_*\fP environment variables used by the -benchmark and changes the current directory to the \fBout\fP directory. -.P -At build time, next stages don't know these values (cyclic dependency), -so they are populated at execution time. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes -\(rh \fBrunexp\fP login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: once on the target machine, we enter an isolated -namespace to load the nix store. -.P -Notice that this and the previous stages require the \fBsh\fP shell to be -available on the target machine -.P -They are not \fBsafe\fP as we run target machine code -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexperiment\fP: runs several units sequentially. -.P -Defines the \fCGARLIC_EXPERIMENT\fP environment variable. -.P -Creates a directory for the experiment and changes the current directory -there. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes -\(rh \fBexperiment\fP login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBunit\fP: creates an index entry for the unit and the experiment. -.P -Creates a directory for the unit and changes the current directory -there. -.P -Copies the unit configuration in the \fCgarlic_config.json\fP file -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes -\(rh \fBunit\fP login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsbatch\fP: allocates resources and executes the next stage in the -first node. -.P -The execve call is performed by a SLURM daemon, so is \fBout\fP of the -isolated environment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes -\(rh \fBsbatch\fP login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enters the isolated namespace again, with the nix store. -.P -Notice that we are now running in the compute node allocated by SLURM. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBcontrol\fP: runs the next stage several times -.P -Is controlled by the \fCloops\fP attribute, which specifies the number -of runs. -.P -Creates a directory with the number of the run and enters it. -.P -Generated results are placed in this directory. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes -\(rh \fBcontrol\fP comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsrun\fP: launches the tasks in the compute nodes and sets the -affinity. -.P -From here on, all stages are executed in parallel for each task. -.P -The srun program also forks from a SLURM daemon, exiting the -previous isolated namespace. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes -\(rh \fBsrun\fP comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enter the isolated namespace again. -.P -Now we are ready to execute the program of the experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexec\fP: sets the environment variables and argv of the program. -.P -Additional commands can be specified in the \fCpre\fP and \fCpost\fP -attributes. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ -\(rh \fBexec\fP comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBprogram\fP: the path to the program itself. -.P -This stage can be used to do some changes: -.BL -.LI -Set the mpi implementation of all dependencies. -.LI -Pass build options -.LI -Custom packages (nanos6 with jemalloc) -.LE -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no -\(rh \fBprogram\fP comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -The \fCstdexp.nix\fP file defines standard pipeline. The last two stages -are usually added to complete the pipeline: -.P -.VERBON -pipeline = stdPipeline ++ - [ exec program ]; -.VERBOFF -.P -Any stage can be modified to fit a custom experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes \fByes\fP -\(rh \fBrunexp\fP login no no yes \fByes\fP -\(rh \fBisolate\fP login no no no \fByes\fP -\(rh \fBexperiment\fP login yes no no \fByes\fP -\(rh \fBunit\fP login yes no no \fByes\fP -\(rh \fBsbatch\fP login yes no no \fByes\fP - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no \fByes\fP -\(rh \fBcontrol\fP comp yes no no \fByes\fP -\(rh \fBsrun\fP comp yes no no \fByes\fP -\(rh \fBisolate\fP comp no yes no \fByes\fP - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Isolated execution" -.2C -The filesystem is \fBnow\fP isolated to prevent irreproducible -scenarios. -.P -The nix store is mounted at /nix and only some other paths are -available like: -.BL -.S -1 1m -.LI -/var/run/munge (required for SLURM) -.LI -/dev, /sys, /proc for MPI comm -.LI -/etc for hosts (FIXME) -.LI -/gpfs/projects/bsc15 to store data -.LE -.S P P -.P -Additional mounts can be requested by using the \fCextraMounts\fP -attribute. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -We cannot access MN4 from nix, as it doesn't has the SSH keys nor -network access when building derivations. -.P -The garlic(1) tool is used to run experiments and fetch the results. See -the manual for details. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To launch the EP use \fBgarlic -R\fP and provide the trebuchet path: -.P -.VERBON -.S -2 -xeon07$ garlic -Rv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -Running experiment 1qcc...9w5-experiment -sbatch: error: spank: x11.so: Plugin file not found -Submitted batch job 12719522 -\&... -xeon07$ -.S P P -.VERBOFF -.P -Once the jobs are submited, you can leave the session: it will run -in MN4 automatically at some point. - -.\"================================================================== -.NS "Execution complete" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -When the EP is complete, the generated results are stored in MN4. -.P -As stated previously, nix cannot access MN4 (yet), so we need to manually -fetch the results. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" fill -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To fetch the results, use \fBgarlic -F\fP: -.P -.VERBON -.S -3.5 -xeon07$ garlic -Fv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -/mnt/garlic/bsc15557/out/1qc...9w5-experiment: checking units -3qnm6drx5y95kxrr43gnwqz8v4x641c7-unit: running 7 of 10 -awd3jzbcw0cwwvjrcrxzjvii3mgj663d-unit: completed -bqnnrwcbcixag0dfflk1zz34zidk97nf-unit: no status -\&... -/mn...w5-experiment: \f[CB]execution complete, fetching results\fP -these derivations will be built: - /nix/store/mqdr...q4z-resultTree.drv -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -Notice that if the experiments are still running, it waits for the -completion of all units first. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -.VERBON -.S -3.5 -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -When the fetch operation success, the \fBresultTree\fP derivation is -built, with the \fBlogs\fP of the execution. -.P -All other generated data is \fBignored by now\fP, as we don't want to -store large files in the nix store of xeon07. -.\"================================================================== -.NS "Running and fetching" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" fill -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -You can run an experiment and fetch the results with \fBgarlic -RF\fP in -one go: -.P -.VERBON -.S -2 -xeon07$ garlic -RF /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.S P P -.VERBOFF -.P -Remember that you can interrupt the fetching while is waiting, and come -later if the experiment takes too long. -.P -If nix tries to build \fBResultTree\fP and doesn't find the experiment -results, it will tell you to run this command to run and fetch the -experiment. Example: building the figure before running the experiment: -.P -.VERBON -.S -2 -xeon07$ nix-build -A fig.nbody.baseline -.S P P -.VERBOFF -.\"================================================================== -.NS "Postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" fill -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Once the \fBresultTree\fP derivation is built, multiple figures can be created -without re-running the experiment. -.P -The postprocess pipeline is formed of several stages as well, but is -considered \fBexperimental\fP; there is no standard yet. -.P -It only needs to be built, as nix can perform all tasks to create the -figures (no manual intervention) -.\"================================================================== -.NS "Building the postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" fill -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: box at R + (-0.02,-0.8) "timetable" -arrow -box "merge" -arrow -P: box "rPlot" -line from PP.sw to T.n dashed -line from PP.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ - "Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -To build the figure, only three stages are required: timetable, merge -and rPlot. -.\"================================================================== -.NS "PP stages: timetable" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" fill -arrow -box "merge" -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The timetable transforms the logs of the execution into a NDJSON file, -which contains all the unit configuration and the execution time in one -line in JSON: -.P -.VERBON -.S -2 -{ "unit":"...", "experiment":"...", "run":1, "config":{...}, "time":1.2345 } -{ "unit":"...", "experiment":"...", "run":2, "config":{...}, "time":1.2333 } -{ "unit":"...", "experiment":"...", "run":3, "config":{...}, "time":1.2323 } -.S P P -.VERBOFF -.P -This format allows R (and possibly other programs) to load \fBall\fP -information regarding the experiment configuration into a table. -.P -It requires the execution logs to contain a line with the time: -.P -.VERBON -.S -2 -time 1.2345 -.S P P -.VERBOFF -.\"================================================================== -.NS "PP stages: merge" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" fill -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The merge stage allows multiple results of several experiments to be -merged in one dataset. -.P -In this way, multiple results can be presented in one figure. -.P -It simple concatenates all the NDJSON files together. -.P -This stage can be build directly with: -.P -.VERBON -$ nix-build ds.nbody.baseline -.VERBOFF -.P -So you can inspect the dataset and play with it before generating the -plots (is automatically built by nix as a dependency). -.\"================================================================== -.NS "PP stages: rPlot" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" -arrow -P: box "rPlot" fill -.PE -.S P P -.DE -.P -Finally, the rPlot stage runs a R script that loads the NDJSON dataset -and generates some plots. -.\"================================================================== -.NS "Building the figures" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" -arrow -F: box "Figure" fill -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -The complete PP and the figures can be build by using: -.P -.VERBON -xeon07$ nix-build -A fig.nbody.baseline -.VERBOFF -.P -A interactive R shell can be used to play with the presentation of the -plots: -.P -.VERBON -xeon07$ nix-shell garlic/fig/dev/shell.nix -$ cp /nix/store/...-merge.json input.json -$ R -> source("garlic/fig/nbody/baseline.R") -.VERBOFF -.P -More about this later. -.\"================================================================== -.NS "Figure dependencies" -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" fill -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The figure contains as dependencies all the EP, results and PP. -.P -Any change in any of the stages (or dependencies) will lead to a new -figure, \fBautomatically\fP. -.P -Figures contain the hash of the dataset in the title, so they can -be tracked. -.\"================================================================== -.NS "Article with figures" -.P -An example LaTeX document uses the name of the figures in nix: -.P -.VERBON - \\includegraphics[]{@fig.nbody.small@/scatter.png} -.VERBOFF -.P -Then, nix will extract all figure references, build them (re-running the -experiment if required) and build the report: \fC$ nix-build -garlic.report\fP -.P -We also have \fBreportTar\fP that puts the figures, LaTeX sources and -a Makefile required to build the report into a self-contained tar.gz. -.P -It can be compiled with \fBmake\fP (no nix required) so it can be sent -to a journal for further changes in the LaTeX source. -.\"================================================================== -.NS "Other changes" -.DL -.LI -We can provide the complete benchmark and BSC packages as a simple -overlay. This allows others to load their own changes on top or below our -benchmark. -.LI -We now avoid reevaluation of nixpkgs when setting the MPI -implementation (allows faster evaluations: 2 s/unit \(-> 2 s total). -.LI -Dependencies between experiments results are posible (experimental): -allows generation of a dataset + computation with dependencies. -.LE -.\"================================================================== -.NS "Questions?" -.defcolor gray rgb #bbbbbb -\m[gray] -.P -Example questions: -.DL -.LI -What software was used to build this presentation? -.LI -I used groff. -.LI -And the diagrams? -.LI -Same :-D -.LI -How long takes to build? -.LI -0,39s user 0,02s system 129% cpu 0,316 total -.LE -\m[] diff --git a/garlic/doc/slides/3.mm b/garlic/doc/slides/3.mm deleted file mode 100644 index 39b07d7..0000000 --- a/garlic/doc/slides/3.mm +++ /dev/null @@ -1,181 +0,0 @@ -.\"usage: NS title -.EQ -delim $$ -.EN -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "January 14, 2021" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic experiments -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Approach 1" -This was the approach proposed for hybrids PM -.BL -.LI -Perform a granularity experiment with a \fIreasonable\fP problem size. -.LI -Take the best blocksize -.LI -Analyze strong and weak scaling with that blocksize. -.LI -Plot speedup and efficiency comparing multiple PM. -.LE 1 -The main problem is that it may lead to \fBbogus comparisons\fP. -Additionally, there is no guarantee that the best blocksize is the one -that performs better with more resources. -.\"================================================================== -.NS "Approach 2" -We want to measure scalability of the application \fBonly\fP, not mixed -with runtime overhead or lack of parallelism. -.P -We define \fBsaturation\fP as the state of an execution that allows a -program to potentially use all the resources (the name comes from the -transistor state, when current flows freely). -.P -Design a new experiment which tests multiple blocksizes and multiple -input sizes to find these states: \fBthe saturation experiment\fP. -.P -Begin with small problems and increase the size, so you get to the -answer quickly. -.\"================================================================== -.NS "Saturation experiment" -.2C -\X'pdf: pdfpic sat.png.tk.pdf -R 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -The objetive is to find the minimum input size that allows us to get -meaningful scalability results. -.LI -More precisely, a unit is in \fBsaturation state\fP if the median time -is below the \fBsaturation time limit\fP, currently set to 110% the minimum -median time (red dashed lines). -.LI -An input size is in \fBsaturation zone\fP if it allows at least K=3 -consecutive points in the saturation state. -.LI -With less than 512 particles/CPU (green line) we cannot be sure that the -performance is not impacted by the runtime overhead or lack of -parallelism. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space" -.2C -\X'pdf: pdfpic scaling-region.svg.tk.pdf -L 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -\fBSaturation limit\fP: small tasks cannot be solved without overhead -from the runtime, no matter the blocksize. -.LI -Different limits for OmpSs-2 and OpenMP. -.LI -Experiment A will show the scaling of the app while in the saturation -zone. -.LI -Experiment B will show that OpenMP scales bad in the last 2 points. -.LI -Experiment C will show that at some point both OpenMP and OmpSs-2 scale -bad. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space: experiment C" -.2C -\X'pdf: pdfpic scalability.svg.tk.pdf -L 7c' -.NCOL -.BL 1m -.LI -The experiment C will show a difference in performance when approached -to the saturation limit. -.LI -We could say that OmpSs-2 introduces less overhead, therefore allows -better scalability. -.LE -.1C -.\"================================================================== -.NS "Reproducibility" -How easy can we get the same results? Three properties R0 < R1 < R2 (no common nomenclature yet!): -.BL 1m -.LI -R0: \fBSame\fP humans on the \fBsame\fP machine obtain the same result -.LI -R1: \fBDifferent\fP humans on the \fBsame\fP machine obtain the same result -.LI -R2: \fBDifferent\fP humans on a \fBdifferent\fP machine obtain same result -.LE -.P -Garlic provides 2 types of properties: for software and for experimental -results: -.BL 1m -.LI -Software is R2: you can get the exact same software by any one, in any -machine -.LI -Experimental results are R1: you cannot change the machine MN4 (yet) -.LE -.P -Same experimental result means that the mean of your results is in the confidence -interval of our results \fBand the relative std is < 1%\fP. diff --git a/garlic/doc/slides/3.pdf b/garlic/doc/slides/3.pdf deleted file mode 100644 index 0bbb7dcf0c3e7bda53062758fcf21c0e2a67f93e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 122774 zcmeFa2UHZ>wk~W&QBf2GCACNvC~^)eG*NPrBobOOP0ok`B`HW2Bq$(ARwPJPRFEtg zB#PuHNRDq+Hz50n_n!Oyd&cSOWpR(?BywThNByp{mj7!f z#;3`T2$FoWx>eM#c~DZ4A2}kmvJgdlB{(aY;lRA3*n*ah{cXhX*<$sYlV@fvp7i4S zUA^?)o`ro}fci9f^5OU|0xIXO*0im?iTB&{^YV8Z5)LZ@;l2^33 zI#T{ZK;fI{Tl2W(H)MzX_}0-Y74HJ=ZE_5HhA(+O0LtFf+@{)DId|y&mz`O;{$qy5 z4E^s|nwx{|AC^v2)kHsKA-FZ97Aa1d7LC5)I391V_(C%6qh>Y^PNYYuCaXeN@uZ!w z^C1zR)3M281k?f34t{kDQNB+5_O-c=qcxllSX3{(|KRoEjpw(5g#$XL&{wR^Ax?Q( z-p4<3fa=8^v!c2ly8U<0w?9Dz`i(u)k$b>#<)P=L?|XC~raix`SK$1>Wh3ACt40Dp zA|>W=XLVFG-;o2jp;1%?S|8rn-S~2M;ejNVr^N9t**lSb?CvsYDu+l+#po`(zE8K% z<)*ws@W$qyZdZD*&$YX^qK)6KvBsUM3IDO@py25<`BzgDdoJF%Gv~^1AF1X2nbfKv zv%};0wX4&!E5#<<7bVCAuADqAt$g*F?Xj!_M>I)jzaPf&i?g@Zn$Vy^!;E{yt(hbZ?$b} zOY<3n5Sb{7i1?33<(Cfgg&-}yh}hZjJ2hSKJL$b1Mn1)aOWEdv;%m5TMND9aldr(g z!F}(H$xqrEl@M}7T2pFY&>Y7&Mdt9k-HI~^h znZx3{`|N$nbXq%#qO*(8;X^kShnvGa67I*= zpL}l@I-qSqBc(t}gIZ+F%g}Wl)l5=x>s2XAJ$`zqeKyy{IFa64q43$;RK6FfXA+A{ ztO?vxxrWvPJC43eFKdig&v{bo-Cf9>btG*}*;Mmo>LY@*hPh&y5b}ZEV^*)_HxkkK z9Afx};!$Q6FB>VSuIwGS^!8qsY(;G2Qi}JT;@4}Oclf7jSC01VJ&*Hyd~)v|pFID$ z_=Jc4cjA+WmG>X<8KNqK?qE4qw4(Oo$-Xq23VYJ299@cc76Cj`f9bE!B9^E)7ddW7^f-h&xj)xE= zm{LEv65Mw^q>}vktvxP}5fKT>_g^KqSSI`X%`NRS(at}`)Ad+JfZ8e{j=kD~qsr}g zevW(+Ed`hN5t;qmZI?5~_kO+jR+s#PN@7|vRnz*7oM`PX)Y?EM4V5%eN2Z{}YnvRK z8>fp8%u~B}U7^Q~@uu~_ky_=w(d^a2|7p6qVt^1uFO6b2b+xA5eZiJCCcE&KNC)uLSJ1cB%%B*p-#}LlD6mzJyeG*$^(eZX#CFFCCMICO)B$HF(H|k|W4!uO3 zDBZV@bKW*8NM`+O0XJe#bEdGu;IYw`7o`(C@8(RNIV)*;&%Rd{AAU&MY}Il)zgQGU zV920eO3A_e{Bc*38~d3aKSpYQD-+RFt1b6?*c%*R_blKF35xMl_5KOh#>kJW*<+lu zI7!ckzn)3)6o#7Q_Ir$bQpPj-_oPewxN8=4J--*nxasy{ghp2$ zuGtqUa%QAU`BLfOc^!z1YW1hdw`?7@2+vAzODRYCY? z%V)`=D(e71E8&Swi>@0g^-+dDPaWw{kW@)7zSgVRA6S^p>r7d~gU;67uf2-b9EX zsZg;KnkVDV-cLFk8ZCS8g)J@+%TxA}$}gUGJg!6Zo`Bl%>b2;1$Eg`*pTA`7KmT?u zTKpXk0gfwU=R zT~t;+>$B35XO3qNU+y#WneN%JQGC1pV5y&9`=Y*xWqs?OzTl7Sd&n8I?>Q4LY46Q= zd`yZbk-1@mTdX|qD6{F?weN-bgRmywhy6~2-9+VWcFSoO1>>lsV+l{r(=mNcHgRpCyPzHl_0^Ki z;!WJkl)q@4<-{}Fd?=7EPjn_S@gj8#|jiycd(l2Vx*Pb)0MNhdexNHlq; z62m4E6I3isRFe2r1Rp*7a&$WIA$0i)I*SbV<#CTjA=^$vfzccMhD@_~a_s2nTB(#$ zQ=_AYMriIW_cq7QYn*#!RJOOQC<6D%kmwgj^-H-Xgrf%=E;FdzxW=6G?(M`yZBr79 zz(=+rZXJ4cFQcD{EIsofj?+1IyJhZ!{Dw5Oi~F%XPNLTN90XtWYb(3bRb1MtCcV%h zT*(DTD~=5&zr02%l=$*G@zgb9v*D+gMPupI(T6BfUgSPfcr#>b5OjS|dN$TdF0$M- zvY@z^17COLn)%T-K?Z96qr$fcFSkFD4O#PSDVHzj&>nM36eAh)D`eXLx*<%EkX%XD zI@F@`X1C$Gm_AiydwJHGGPTc(wuGzc6Zn#&`On+Wc}+75S9}vAyhZl)<;M(k$bGo$+6 zkxz^_HS8%$&I%8XWLhQ9zw9TEIK!asM?T17%7|R!=D<d>pmMhpKqn#_qkc&j|7RQNDkejdaz#CYH`SgaP_F3tvBIs*@SKPr9In! zZfTDh`rpYWTs)lr$Rx3cb2y!qK< zmlOWsuH#2W=uPD%r4){*8y@|7qJ+2v^DkZH1qf&NI7Pn;VLBT3xD0P3L&KJijECu4 z|6m|4Pa|H}Y2CwFsYPF)B(KXfk(AZu4-cPEG&bE}$2sH7q?Hu9$7P=bHQCSW4A%}{ z#-9+pVV_prC>E}zCf|>;7$iHvA1SzUSSHD5-$N7fa}p{l#`|+UDSr}6o-LSCD19SF zR5JLu6UY9w$4Ps!%yG}pp|5IcG=$H7iEeFRsnB@#{l@`P1$@6DbW6BKsY#qP%h<8y zV#nl%*$)gu@Jp}Gs|zn?;icTDf8`S>IuGtD|vHSha{b(+IX?597bb{FxV zkXi3Eejj_;TU}F+^tiR?}q46VW#p8#4%eUO-iO-#< zIV4mc!hP~J-dFDSl#kSXWFH0Hir!Xc*;`Q>*5F%xi0kYA+Vw0XTmI2r-0a$mji)}H zlAK96b77V2z_KgdNfCi_=+LxF^UcF-Kkt*_dyx}KJhe-FJyx5e6xDrWKt!Z+5|Mc7 zQzRK_Y!-C3XF@FaL+yRR_}1_fIUPyvIA@k)IwpCFao6|pPA*sc7G~_bGbQZ*IjRIR z^uH5kob3Mu?GWYnexQOb{7E=P9A9QiLSpG}o*UwsRg=^`by_@2EMM5k&Lns!X8P6)9sU<*0wJfmAI#HY!M%Yjgr%XUXf8-d zd941h5SrOT>}lXxMMc(u#=Q7QBy~W%;QRgH=Mz0b2O8SHeh(L?@V9%>SXA*&^+tV~Scj6}4|#8O zh`W_50}l>nZ+E{Nf+FYE(GLVmKP@yn#HdR&Md&_OO{5g5k$v#kqg=Y}C|3egXGl3z z6B<{Nlb)$$5@A>HM6Y$6;cDYMmea$<&QXa%Juz-??}}P8brjXNKdtTi`mmiXRy83x z`S@@wan2E*BWfCG5=V}fh-Il0^*$3%X`1ifs`l+H=1C&msPR4H^|Y=%ubdE1mxW(o zk;CG>9x378t6B)MBrWPF@`S@{RYU8<`3viNp5Vz|_|2m^V1(H6uV2eh4!9Kk50QU6 ztK9#J{o`T(Cnq$ZDr4Gl30{U?J#p~J=)|PS9O1tE4II>8u8?*Gk`p+QUA(49nelzO z)70HoT%#gjJUfaj?8*^^Ycuvj-POh6;x{f$i@P^=>RH|?SvW(zPxZ`wn=yQ$GF8u0 zofjBqbVZN++~;NE(WyQqY(qrs>F86=V_tXNZIXHB@!L`yawiYPWQg*V@WofQ5O7^roHny4yI!}YeKs^k&k!F#Es#7EpdsfKI4ds0gOP<`#2r9KgL zJSsZuzI_&1QSXQ07&jsmv#)LROBA(k8hYit?k%F5K`)ij7Lh@Qex*W4X6ow`9&faL z(kEi@-bUNE5l>mA;>RIA#+MP)_&c7&du)3*Jdrb8XEY|*OzNE zUGHa*YfxAHd?!OQ!~kd2TrkK|bgabDSb3A;Vz6pKflQWSq(D*vx6xwnnX-@jB zqm$arn9L)%v^d$f-jxhArKZxdm~&>4^RK6eXWm8fs-(pi-1Rslf7c{}AgDH!1pf@N zOhrq$9OZ|@OV7WLWme0lsIX1ZeDt=7x6%Ia#o21;I;W(Gm+xr{xzMwIHt8#WdK@pj;!xo%(zd!V~$2UzCsj`yTS9gHUmkYt{ zRb`Jqg}BJ2>Uim8y-e$>xXIm+^$!tdk++$v+rOpXjyO%Qnof07*me3wk2Qtylky;riAA0g2eZHf^*{LsxN5-atVDC=gn_4IyQX?|H+qM z>|$8=P9%zlC5yqkN`0P#)WF2@0?FYGM zyNBkGgn1M9xd~|Uu;9$p8)D`(=`n7D7rlU!|5rA6N8rQ!ybYGJM{IHSBD%J^W@tlr z?FRaS6oJdJlPwYfoV=|u+Cmi2_6RyrJ~kFMRu)!Pb~Y9kUJllaEUdIFEVT5v5T+g4 z8bN1hjlOvkVUM!50lg5c%xrAT9N@eR+Tgz%Lf#r}V5bjm62N0v8vH)UFA;YxgRkh? zpv1u43olpeAwyPVs34~H#k^<&d@3r#_-Jx1eUGa(jq(E>^zu*FbnYS{>#SAgJ1_M6Ao@L zI6NTu@bK^=xY^hcTj${O?BE!D@vUR<7XEDY`R}%vF~B%mlfX;_OoEr26LT&2dRw2Z z337w+x8#7whxg%$^MG+MZNYiC?N+<(esDW*oR?!$S(^%jW!RD*-iOEBn!}btx3=(k zcpv`koGZK!>u2lS))u25Fy2-_jJ!Mb1MCC0VdKGW;da1cJ6f=FvS1`)1=3>7xl^T> z1Hjaq62aDm`+~8+IDh)U&H;a5Jz%Z`_Avv&!(hAayaq-cKs%_Ny!?xYp-?`m+ZFqmn61#1;eEVO|!Q;Wl;H~zK)VOmM zC2D;0CJMeE3R0>j3j)%$(Kj~MH@4QdGvEASYitG*1*D{}YYkE%q_MfR+1xsBZq*Hp z!8kU?HVDYpTGs$&u4`?IfUNA$w&0+Jp&1wgX=jZ_K+G0sTLaY1|HpU0U1WVQC6uGS znJ&;2W}XImW(df_&Rh?M4MU(S1GJeLNYjud%32@3%?#<9gIyb43xmxb*2acL;4tQn zv>EE=*6!wEki${d#x_8WmS%Q9uli_nb6o^vdo;0t9B_eNnaZD9;|&_|oWwzWZ-16aW9nxSmK z6;W1px?oO*Fnq%Iy{+s}HZYEYnE+WYBcLnb9z3|o4r~ia0AXx+?2kxpPAO!@Z2pkFAhDSgu2uKwHsUe_i2uK}x6#~+;*3~yf*<#h82lhAT zwRwC~Cs^0KX@&wbv9mA$+SEr|gBf59fb9!Zqz9%1)DI>ArViu>N<%=#2*?Bh0d0aA z0h57A04-U9t_TQB4fb0wBm%NWKn@594DSSViWLjBB@}1}ivtvovD;>7YnR4hhK8{6v0IFj zfi(`s<|uRQc3Xm*V3Ie%{I(Bl?-;;R*w}#c+lLITbz!eCva>MMwYD=i)3pP_fOfk2 zSZ~#{2Cf1$i4A-pBG4n+5O~;DU;|mOn%Q#y&55BwFSAYaZJokuL=FkOV)M+l9M~W- zKsx{(V6O~2z8U6v`gYcsr!{~%V6WA)Mw_B6^x&YfL2WAQSK`q}TRLrtU~O;{cr(_$ zVZ~#T5LQ~SPz1{>GmwXXBN$u2p=FISG=?vV0zPA|tB=Wa1}Nb7U?y`DT&kbp1E51l`SW#zHOzap5?I@fI787_`NJEsV8TsKp{J1ZN^F&i;Z~2+qh@ zkln)A?LY}EPFo;`6%dWu zM7QlmFt|Z-$W3V5bV@iCWAF^xf-`6f&0sJCSKh?3EhvMw;t}}CCXhh@8Mbij7cO9A z`h{Xx5Zi%an-B(J5$qR$!H&8GI{$<(2owJRynbQVzd#q52`1s~1TI**ZQS}L-lmqe zQ48(@gVsMWYunQ?dcqkAS^yXSa^bVpc8}zCrEDFSX?`JK|5dx+D0V^ z3rY|MlXgJTCL(PD5-JwXED%9Ea0uFlAqa~? z5Eg`>9T)^{Ll6WIXcK@yFkoT_3qQ~f^ntM80|9CZ3vj@BVd&XJ9taCO5Egf!9k2s! zqYi`x9pDuJb0DYffly{<#+IOv1biFJY{7*S6B`>3;JVCo4Q&vBXveT#SFWHP;gS_T zhou>Kx!7Rnf>|tCU2_0m|4OAv0+P^J|1zc;1>BjEEy`RKCeg59_fINq2juN!6921v z0BoH9ya%8yFKzmPuy7aU-C zJhwPo5$M<`lv_*FOHrSKcba(2_G3kz!ueUAYvroK8HzGfGu~Cz18VaPC z_Qo8LrDuNS(okG3_Ffg@$sZHPSSn2;?2_Xa3dg0!S zcNc!+Z+AXFv>Vkar)Opimase6PF9$6QUnC=e?@me{|4X5!G)z9w<()VIt%`d z;fYvzv0Tc(%XMzCW-x!l&dIsW^I-TqFen?~h_;EV|Brm<7GDH&LE9g|`v97obDP-O zBB!=yxJ6pQowt}>m@N9gB|Nu;z)TphcJNs6B;hUQ18a11A9KyEeb5He4xER#@b$Oa zZEayi!7^_51N-oNwn#^K%MMsSHZJTIZjT*pdxU?3@Z1`0%XC;aa@$B?%-!}egeQ;% zK8NuX_>!AM=GN4=rV7M^&%#arr-UabY$iaGZmAa|${z?%jEQ&fnOndA|3P>HTWwio zYYR_m=V$Bqu3O;bTg2u+&TowiwmS(=OfK2FZ~T|>+$LWBM>~HR&%cZ(w8=PZJI+7r zy1$GkOvG;$i+>qU44t}7l>KEqF+3r*qy#+Xui_5;*bJ=gsuCuW|1zF1letq(*rF30 z+$t8q{fhrM}HCbueoOSfRgc-8L`u?>wIPm#y1k%(j@_e|Ra;dZpc9RHuPbvxf%`pedBb1na)oxg0| zU$*WqTle?*#I46Yz(a0-pHIYUV8^o`;92EgOw!-y6EXbc|KramvhMtQZGYLizii## z2Nc0`v47dRzige;-v<=`lMg6vvvs>Xp!naib-2(@mhOM>Y$6-{X5}_VxA{mUcm)&G z?C=ME9CEk6St^FMHpe{m2;K(X`un9|0@%kUugS52!-^;y@ah=+ULgFJR>5n{`tbAo znh^MFtl$kUlp}a~M$#O-iVa=?!)(EOV3@5We9>!idM4og986b9bMPD%h988VsRaN1 zdLJ1UW9#wD%`R|*?Z2UVu|JcA^9y?phXTu38l*3#lQ9_B{2&S^Dik zL%?fb_3^u)B0w%sYN;7}j!I zkD5FG_$xfs&9@$awzpNu&B?+8eqpr*3gI8R8Li%QT^ zhXsQ@PF7MsdU^$MLt1M!~!rGH+b z-B&t99vtIc9*+C@^QX(T@th8e@4jnw8>@1u{o#hu->B7e+~)RhUo)?8Tj)xZ*1C7+ z?%liR6(S-cW-X4wQXKOejg9n$ig`|Xd!4!Gt7x^*!BU^RcwX=w|njTNcwLb|A477>O%E=k+x&)6yOxld}=NqM4pRy_^Q&Un3vSxkz z@_a{(%A>vD>^p?0or1*$+89?hck7Al) %%n<_SBar z>{=7;##)lh)02~>);$tqZbg*XRnjN^@K==f6IwR9KC?FEE?i^jidJ)!|48Z9I5f~25ql&hoT(1xl5L6 zY-)Kup{0?;n9>`tz8UnGD-S;KrFOsd)i{wl#$fC{GVyA7U@C6s!pzJ}pYWZd?}ULe zgRf725jnB6w!V0ucEutJdSwcJ2Qkrp?X#axtFRo7B<#(O7$p-1V`e77E6Ey`EuWD z7uYlD6uUOPR2ZFJZc>taB+S43;D_5Qa$ zzPK1QhP#+U>eL@lxXjtA^~QN_w4p&U-Pd1u{jEu2oVdTA&SIVWIyOkLen2luckfWo-bmh*XCHufD&9+ao)Mh zuKHetpxkSWt1heE3&n{!Y2crsRcc8es`tYEIKqp2H#}fvyte6DYkiexGn8CqlRH%P zL$3Rk?uRF=`b|CWP+{6-B;hR^tgr^Xyd4y3LEL1JySJvQpDuaw8OgiK6a58A&hKSI zrYJo+I;!m1pK#9oz)+pIA8FD#8XYmFNLKv^8g9$8BhwU`+S+clyhAV1pJt6e`4FyB zx855Z99+Cl3#?tWG?U4=Kp9$2?($G=X!0EH{k|&khs5uz+!sGklu2nZhkcFjw|QeC zT6nxdGVm>(VYwgK^wA&=nsy}n^0HoKSZ5)VX3ooj*rV*uw=>2bkXZEEEkQZYpFb}l z8Ju&j6IwC5Zar1BK(p#`hFR++^D1x=R6Zd*EeXVXWS;X{A5lp!^-#16GttvFJ;Arb z{8NQNtFK$HyG%Sb8p?1cjbYKfyUdnn7(`Nfgf{YBkK5um&*s35^{EZ=Q)E*dOZxgm z*=4g8LqC(6d)znH+Voss6s{Him@cQ7(qU!tYLW*e&r>W{pV5fkJ zp;v2S2f8FG?1#!nDf@}9FVfdQyCUC#cEcDI;cp;Yt1r}mh zX$=CkgX6l7l}1_CMGKmEe?|h6jR#X`YG|D6IlByJQzs@?<>?hTe)>d>7ACvsiBuu# zN3DsUe2GqY%j%yC5}T_HJIeg_S?g4#m-9uVRDf?Rs)qG*E7_VUu0+7 zwP&I-3TG;Z>*F%=1|u3}`DN0&-I?=mtu3z`UCs1O*qD3GGtg1!WHVD*WLck_%rJ+S z{S;ly`@8^_y}P`;b>+MA%P?k}P78-+l6|nM4HWa~giF_3-PeeqjioyG^lmCj>w0OZ zh-09di^aFNUGZ&J=YyWR_w2(@dXhFG zu=76-3}&wEdoUTvXUA~5z^Jr{g_Ysr#qVw-g+FV3-IUo1BTK3#xQk1`P|0)|Np83E7=2=7Y|bNJsd0~M#>Pp`Et4Ln zWr^8uR+m$_yAFRjKi3s?N`{OT3UQWtP= z0FN%~)(!EJXIJGh;_^}w4RksUMU5I$TkwG7J@fg3lD%fe^&$lJH z9hP;13&%zxXXpuVlk|lX3aYHm z6|bag@R+r+tOw>Y*xNW9n8)N$M%zSRkwsUcBl@qm|!QSv7 zM&RvPSTduL_sE)?KS5B`_7u%jUfh_^Ejf#s-KgnZ91@qs0W%W$rYrv#OC zJ4nXPBQI7P1AEp+`kiGTe1(m{XfOx$p(U~GAm#p8 z!x%Svb8~aow1Y>_fRybw&dJF+`oeAX2buedH1SRW-Gg`#%S3lu~St`)*vD zECWlK%Th!r1wrCdWIiSmwqIt1WPNSE-6z768(F*2DCd!uR{yW?%jzPCpzf){{JK@e_#wc)dE-oFU5#ZG`E#!Q*MlGr8 zqlK}F(at8-je2kPUyT+BCkbvHW1^#Toc=^4$Mfxgon`X*t_(L7@n?OCT@4MVdM;!n zw^q=`RJZEAzNVq6DZDnTq=LX0F+S3Pek~(oiuv5JLr}puTX7rk*)9+IV}TTDv$lhe}U5BNdV>!wc_s~Qtzl#t*_WLOiLynQx`f1Zms$ee~Aq z02zcS-LGf4CHHOiI)LOSe0C;g`9BV{q-(o;IL;kuRd#muWZgy)$TrA(1~YPw>q|pW z;g8bOvOM47gJk<+g!RjgrQ?iti2#dz`^rV-{7C0YqRGQq?O^1J6A`nPW0zg)+OW8A zU$lLTx}mmRvx-u}RmsVWAQiKqyoP2pbmO6Y~x0HAZ^ z;`0}Zvg&x4Uzdbg(cOhMAU9?;%8~$YopI~IocLe<`s~FHkUwuInq+ZqeKt`T_RMb0 z6PT;a_76TqM+>^x3%f0*YZs`dYlY9GfyLI(qJ{28mobdkw{_3*+mF8pyPoWD5i2+| zKAtBMegDeySz*IHT3Mb6aG>wG%{UZkQ|qk_Ed%(p{hDPA%Dy@UaWcE{*X7rp210;k z3Ja5xj6XjJdhzlo;YHQLmB|9bD7!PQvMnRAz_n(wTd(uG%$pUj!mCI^1x}Fw z--xRTz-?f6f0>>>zp{2@>`|T(59vBU3`^kOC37vBodS6_(bkppWMpKkY4l17Km5-D zP%XWGl)`ORAw3FYs4W}mbn&U+?ZfRHh`Y{ zqFnU-0Ok^BANR&O^td^Ur=@A;TE92-DUDej)7g-E$)j6(pv92LjxP!%z|p0lm<>?x zwYId-q!s~0S8Zr%DU$}v?=}&oIlt4Gp$snF^-Xq83ha%Iahq3n{d5s5%~~|kC^vfr zJfhkTi-HL_stN0WgAnPsvXbJ}Me&D~WND;LXM;L>7|8Z7a-9%fwH!~2uJg4vb{&51 zc`eWvOsOC||0V81U16bs#goze9JBV)7y?4VPiJ0TV=VM2us9*p%rA=(=keZ=0=nGF zHuLeP+~`5-qh}a<>p;ex@;k@LpgIOhG8-}Ge7#S&-4B+Rm)kT!xiAT8>gt<6Uzd}w zwCVeQs!WuNxz!&a4$3u_w{-BBx6e!TAm#W*;)MZfcM9t`k5I3##`-cVqEX8*1cPeg z>Qk;qM@K&gT<`}|dg3elgykDQaH08$4t_qqZct2&r5B{9r!(g;HzQ?e8ohzzRZxdj zhqD{5uP%vr;tBS7U04*K&IQE9*>%^|#c5z)Cjjk>wB>%y9~# zRhOQXZcu7-g2j$q8?~&62*|9R&#q5Y^tb^+K#$q^NALTXwQ)n(2SD0dqWwA3*VosU zuje+)?=V@oJenfMI}jWMs!^Yd>0U8hvppN@3vg*ZnNM`xQMYt4)=~j8_4l_&V?iiQ zIK9rv$(c30ELD7+0Hf#I1`W3L4;KN39>$8d$Mb#*a>9P)?!lvefEWn7?kqXQ1hYAN z_O#0)PPT!y<|n=LOIgh+Y7`EgR;^k#GA=F!-I|9mP5!sO+Pyw}b9{I5Ay^Ut$E6T- z&gwqSisuzfDk@dm_F5mGo(Z$>_nuNa%P63m?IruoH%lP0+Ov!sLrD$SmglO%Ae}tx z7u((ND5-G*0^`JBrzs-QBYj9P5(lh2nvyZfbWi8g%zeV(&FNta6glH z`vUE$2gEp{Qeftn;dmyUh(25P~>eES7_q?cw< zjJH%cM08LOA=?i`#(qDfl2yMOKzZMn?JeEO{Z^Df8Q5=qDyzy<4+-eCg5M+W5dWRZ zzhjjzZ5TAlsmcWb{xu3UNU{pd-BD*3qT6E;Q9Fu}ZF!KLtSrrMLg z8``qI*z%d*qsXM;;f9QS+@K2sl*=C#xlFj8{&@Yz9zVW1BSpolM-$?X1n{|yC)E+) zli_E#bng4jEAX_kAjTyr_G)}9L-FtM;Im-x(40N)ftv4rbFMqS^Y_iEH7MK){mLk1 zPAaSu^UF=0F+BXcgS<7;F|v8JV(4I_uzszOyYzy&(TD`-;Aj)~C#i+>GT+qOoU*eW zvj8?2hj~{GIV(sc=e^FQSnWM1cYb zgabPQ0Z%I!H$844v7x?`5X8cxeMAQ-cZNf6tTK<=`1B`wE!&Y>MVc)*y}t$QJEWup zc`?0HXT8>e-BddD(RcWBxc3FW!7RVuLlkH+AeeHw+VIKh%Oe4(|~7-w!?_T3~~Db3?YlVmQO{ ziaBGvdP%NCi|-y(LGkCFFLWL%9se1BJhxl0YGs^laK_G8xw&4)&WBEmgXlLE*BAd- zH5qApiXMfK)y|He(a)czTN_;jo z`-jquUpk(o@U*6B(^i0B7G*q@{+E7Kvzta_Y*K3e&H#v)Fh;JfkJ|Q*v{3oXQ zO!nZTEbBo-#J(K~9*6m>sY#8GS18vTiO6^XH9`H)301-6p;i8!*7uqEn+n3+c8ysF zofwXduJ);RrF8p5L?-y(`aK+8 z`vf)f$e6@W=dzJyFX8?S9dOPS)Zfjw*{_7Et30KQ9bo!8mP8%(-OD0>wb(6a-j<+D z{9FKKB6_(ez1CSXs9W>#u3mRm*G?RVz}z7E_RNg1rjFn79uDj2iIi^AZ|U@tJ{-=E zUOF@^HgfC4G22uRUYzZ|yQ?Z>Vk|{TMMLCoOLZdr1f;~F78lf~#cdE3qho_=kvON;8X9K7o7Hy<+2|$Gh*}^(ZS}D7#!em$6{KyJUQy z*)EzYh-T%dY33!SyUvY$$n!mOwMZ_z=0|efzN16awCl7Kp_RS`-oxL&-$R$AFC_e7 zpe)`j&Cu2776UFaE~S04#v^?pOc;w?A8;R)DIQ zm{N_NVi=;#ugHh!JcF%7g~LgTI7cXwv}e6NUqnb1alhc+&D-a1Q@!aMz8!nK#bRJk z9*0O_-Du`%^WKT(g3zWnaqoxY#vTsS+BDyZj{nL~#7(!W)4RMgP!i4h9RJXo;sT{~ zhnbU>=z<-=CsAdRV&j?kw~f!g^G&6y9dgFSge7ZIvFen0<*{8rMFGG2nSE7YO;=iB zoAw8!Z4oM$yDzL#iU7YWhn~ytrH;}Ut7-1~a$AnYGGAx3>aOvdAbK;ElJA)j$~0Hg zkMBx)_Mmnr($uY{a!f*$i<1h^C}yVng-X?vI@;Xaxg6y3o?T245om8{d#V1=pf8EM z^355d`HVcTTkQ&nH?krVG_l{C#*l*S-3xMVY`V{Ak>WPTcbQAQ!Cl>fa|)BOOkF3w zs$4%!R1XC4voy?g3-n^>nvDS~uNpFykoQEDF?;`l`M6FK|w^Qt5%aTdx?Ts#*sw>M~DQR1}SY`eEJT4z^T$}V8Dv|Xsh!0}>L zyGshvuR5#*GG~LXO}P%oIfU0$1yVNOL@Cv+fH2-;HxFQfjol)M7h#0dbI9 zW8NBHuwRxmRjAM{Xf*b@f46Vfg&0!fFh8L=ukNQ$b`3Vx)$?f%-Q%OGx4tA@UC3+C z7pC$&oh%VD5aFe&_Oa}i)Y+uYmP$L0#leftXw0cklIZ@nfR2ohhO z#+4kP_VLQ$Dkma=E2G4;v_}O-pwPsW(9bB0Omioht!^pwGbWFCK^w8GQ$eG4F)d~s z+1#|PnS^I-$2-i0U-jr|Njrht{NT}ngGDHL!=|6H&*kX?zdIIPm!3sC!DV2R#1cic zGxtZHUNtGdx*?IxA`fS&IIEU%pjGnzBSjWj?$kSe&E`6(CM$(s&7~B|EshdV^!aJ`(_q1tu=479 zXLQEZ1!MoGlk8$tlxm@pFOZ5#`ZGA^=oM${*?%w1Mh|9IS85R8r&&y?ULP^>!*^c-d^GD4G?kjOQ8D!}*7LujpCL^R7&S!*+zul*M*%2v-TtRj3%`O5@izP!8J0 zuU`FAJvyD^NpN(4`RiA02K`1isfwbXfmCWC7Z+-|TaF61T>3*vDrHwd8H=iUtuRVB z>c&K$xS|}O`7|Ubc7WQNj%xYZpDu=P?+-Fo2SwNe0>#7?Mq9PSm*0O%p*~egD!9m5 zo&7^)JR4x3+7Rz$bPqURbQ}j?*HCf)h)_7AbaP0hWPN@_v~cGiO6M2;6$xHOx)+X? zFCjJGCCu069!djhu>ori)F(#b;iV6c{!k7_INtlpB(3V8rhCoL?Dgaj<#NayDkNEn zPJu(W3Get2##eW#;%y(FO)*jt&C1N~V3NK+$-o>-!sxQ9$dVVTteN)K$XtalpVtZ|radXM4s_aPO95MOkxhyZehW7gg zRs28nX<~QIAo-ySO~B+u67av5eZ|FJg{+1J~#@87bHkHt2+rDu<4-j4+KRJv!a zn3Wlcq&%Be^#@Pz{^6O`y)@4=Ff0B3kt!;`E>^kGt4;ZGb3b!Ig~ z2LV14dPFsPXgqRKNh)HJ{Z#11Dgu0St~tF&f9%KQ05DN(@X^!!H`F)k6IPDJ1I4z- zM-rsb1%3E7t48qQ528|^4!ve&dr3j5bN*Apia;-;bUI~$w#HOpT$rcXA2btuRRr`% zkNftv!yhm2BbZvCvdVKWBYn3FHIqMZRHrOgvNc1;t?u3XdPO$dq4<4_Nq${RD9F;( zj08X>NB+cAIU0nAWYbY>;WbB)dg){ zEcM;I%3!Htjo)R|P9hOK$xa%|AVo-IZX9dwtxLH}wOmfzM{w*7Wl&#ijg|0m)6dVr zvSPU-Fy(W?As|N8HLAdMpN99u0m;8F#aSQlG=pTxWPUS1R>$je!#sJyiY;H(UZl3{ zOd0V7MZ(|Ln1@c%Cdz?2@7n|e<0ILHyzb17Ab~DJDoT}5V56osiV4sr~sp zl9|?g!M;Z3oew2zUx`OezTq3Njj5p-h&MvgAN5>`lguMG7Wkbru-1;$fMvQDb)BAl z>F7$UmZ)&W4@7^>g8!^wNw-VEeSBgq7Kq_)I;Nfs4zAh1sN7#IfH>-bhlyL`x4s zxC(!0`nIAcU1s5|#8*=FP}Ah#pxEqY?MAk_6TIh)rFPNlOCC;8ED!LVn@1cIrk&i;y01L!!%>X1dVQo_L)o- znz0&YMqdx~4ezK+&IUkfO}Ys);B^+GfPjIXSi8RB#yzuX|eW<1AB){JMV^CyuwFNum*=27_C~98{M2=d^Ej0a6_IMQg9yX(7;I#;3YJ!+r^qpR>_=T_21Ia+)n;< z@=aN5aA9HRwL~u?yZZvSDFZYE=6AEmDcS;Ypq-IMDLnQrLrF&6EmP7YBhh>|XxF^2DT$lT$dxv|);-I9Ir=rqg}C*#zNPke3eUK+ejZDpK( zRE5yTi>qm*ZIIEQHh9C6w+a?Jezkj%AtOTOm9{Y}WS=Ukq|KQ$rd;Ws)cY7! z3`(#c{M}bue1fRnG~7grn$~FF|1^2dJikt&&o-udD5%mW#AV35)4;qrsE?V-YJvYxa)cC6khWdx>NCE8?11@wzo*xBdfD~IE&M&* zyh281`E?LKt|F*@jMYT5QlFsxyUptwBwTQQkL7WK#8ED!Xq9HZ6hqOY(h;Hkd$S#` zbLEb8S!VOL6B|$M62|xY4D3v4k%Cb}Jc9T=`9@Q&d5J<+#_FMCrXbOAhU72QrWokP z7ajUtH#y!)Jh2aZKO&n5T4Rg+D4AnB%yHu5yPa;nF)8{fl+kV8`Z-5pW#8|Dnw?HO zWC~VvG^k*gw2wW>cyuO5XvjoF&gT`1jzpnNn+v{a8)(M6?fw0ts@_4g&*o}71fyz# zu8=Uft58D_cN0A=GNp3BY{nziC(C~q=OMQWM5(sKdKsx`s(rgNY#Z}5ZRLtJ z&;Oz8tHYvR+P@z~K}A6%mJ}tVLplT`mXK~)x=T_zRYYP1q@_fb2I*W%5s(JyPU-HB zcL2|kbAIRjN4>nRo$t&&cgK8e_A$yiMpEnJI{Xh}mi!7=|MRD2FR95~(t@w^P(=kv z?+%0jW5-3W97anv*qP*5gWdQ^{1DVm{#6Q$rN>Iuc}~##socbT)`o>%taz~Mp7btF z3?b0x;fD10I!dM+4J48%{^OJPY%nB1V*3bpex3}Oo5a;sdj1?Eh~r>a#( zB0}WZe^K{tDnbv!yB*_<4@lV%%zjR#z9Fg3y0_mQJSX0WZ#STuEu+J;(k`ASx3U;u^4! zsfAm0&_(m@_e<8_pm$I~Irr72a&Cuwticaj7%+RM8GP&c^s6gewNN{sW1S7Vcw6<% zL0%>_FPHjWY-@S1EB~rv+2K&Z=|HB=%ho|Dc{CLya(eTd@oXQ_(;vD4zq^yys=%NO zaRrspjQL*`vOuj#oCx5D#PEETGAxtLWEn9|;O()CFY594?Nu&FDv!#)Q<_%&>R&ZC z7eB>HU@P?$`(7cO+Y~IyWNA#xWZ`PSwyZYWK~XR@x^@2T3~JUkPWPeW*xO|on*Yp@ z*&QIv1sg=W3Zj=}&DFxrtN(X*ZQtO3=_{U|GlqM<`x2}5W!NJ|hGtv$05TxZ17o~7 zS0U{9|EjhakJqIq3lqt)^$*}`_j8Y=YCo$G)GAgBAKlN_8(3_$gSuS#k6*p2Yb5P- znQe{ZY#I_Cw2R+v1Qt(7!THRu&Z4Z@p1b*f4;Fg`??O-U_9*Cq-<0zJ?MB0XA;4-f zrvLkopS;GQ%2em2S->!%oc`nj2>;T5p5IW+$7S`&X6ZHt4M>xrqWml4fBPesH+j1AlTUnyNy-mq6zpn2_ryPn9=Bg{6>;NW@pU&` zeV8Q@xAGEh@A+J2=}jaI2ss0j?u2y;UDLgB1d4>F=E&H|P6?RDXpP}q1wG6&A4l zWG_U2A_t$5=fm2}G3bP!ElaDi<095zr;_~?YdZm2V#e&cq``zr(LsJi#Sv())2eW? zY0vz(3>`DburHq_SLhTy^GoOXF?pJ?A=7Aw3%8&mKuO3X%YK8Rh1keXO5XQ8S_gYf z(QsPo&%22(8_hNkI-P=DW3}-9MWC%rE_){sxodc2O?@bT#x-K>dZ9R?vZ=uaFetiG zWxpQp48_STV{qm4xq>N$%C+z4jgq2d!Jw&rzXns_J(EVRPPH`WNcZUIXbsS+|EQ?T zocaNY40!~$$N=+WpOSN(C34G>SY9))+*FZ#x7f&YMtyhp;}OTP<8d#{Y^_R{<-sEC zM^gBpzYz+BZiF03P7ESVIuLt8MSc7e2c4eBi`t=+PBT%)rEnf%4AsCI2Bl-eg^|0z zK8um|+q%IoU1-J^Qec5X3T+A(iu>pTjk3qbQT zzhP6@6eUv1c3G-aQG$;NrarWMZyGd@RpHcwcDwJIVV+gI_421{g#$0nQqr|DL34l> zuf)&N$)J(vfLtN6**&edBhkB7vawwHW(D^E+b9!h%Efi9 z8RDyba14Af2r*1iVljlLT?)osv`UTJKTS=G1>HYxAP=gI1QRH%pjnY~B@wL>&aCJW(^tFsPQbrNF#hLm$`W;Q(D$`1hBIfj=-)=j)+VC>q^DkVqT*~ruP)fd z*R4d?LTetTSBP-a3=V#Ayd8EXb?oN7!$61u+deNe=>VfeTEJWd4wni*<9Yxww=I}g^EpoWm*8MzUHI%IajD#Yr)}!&;sdrp ze3@YzaVWO$1Vl&5!=q}j#{=~J@$XNC9`3FNRTk;ipk}9_Lm1zOa}fv5ccs2WC*gP7 zGta4v*w`|?`Y*%hBNt1d7NzY6H<6zGHX-c+Jp5vv&RJfPg;$7{m77U4KPMpL6F2Uq$QP)Y=SiF!!^Q=K0C7kY~y~MYIy- zGa9hiwjI#bm{!pi1I*S{m5-g!uW3pVQ3Q8;R_BLGMGHAPmK%O(=(p|?2B>6T-K%@#DC`kSElR+D}3D)~c1<@L+VCKCJN zJZ<;XnIJgXqW1|S!ju);l`pY4MMTwIBEU}+l_{r&cXi~wG(2e@c56V#O1|7yk{Wbj zra4MB6#lBOTDHqR9x=90K1-UWsAe`1oSmdkGpufYZf<}@B<${`{=V2M-%xrwPC6T` zEVHg>8I_NKGqTAj)*n}zisPE_;CyLc$t({We#0G{$9a+k(Zu z(LFb`3~`CLD=%^Ay4+!k(z|Nsb&>GV7~Z>p5bv&NusuSM8#j>~iSRDYGN;O}8cp+G zrfDT!Dy)DEK`GtT9ZwwkRg*-nn^@||!2qrXd@WKOG=Vezu!jWr&7}=ARZ6Z}-7zs1 zn`JI2SS1xmyWCniAIHUA0g61?bDyOzS_t5vg}}cS>Lmy?1=4YJIHH5r7ILDhAb z1?4~NqK~gDwg=jhSmc^{%DmC~-=*$?E_54z@$9MIgHkXrNxh8dq=Ah&ioc6jlcZRG76_<*Sv-5L;9v** z(`-S#6IzWVu z?c$HdYJpCJFpx5b4>=?^To{2CH7A4C}GfAI#R(yfRe7m}JxzASSU{VRf zXQ436{nBp2e6F9?bWQY>*Qx#EOwp6IL2_oN zaV}09qDPAitE_?SuhQqT0t}UoeKWa=h|$C0%(jH1J9qs|K7749oVbRj9>K+@wclzkCT*0 z?v{6qrJQ9hDDQVq_2Pk?j@nK1>)WF&Bd_3u?*-=&Qpg9fAOpnXU%dvsEWE@ju~&7M z9p3HC-B||#_E`b#s;c4lkoeyw!P6GwN6KMX*_biiS>cBaVb4B@@WKV$hu^~3w2m6aix*x!O!%=bd6&b0 z;4;(k!9!cK>^sv_MFzbbK+%fZxvYJs7^Hymh}TFh<517c6)F;LD}-ecf^N_3cK5RX zk%|T+^jUqZ?UtVdvOeW-c$He_xx&u54x3@?HMIr@Kw)e`yv$~Q&Q6kLrYK5;dV8`P z^Y7Mbm%FK=nH()TTrPJRZIx3$ttdt$4}yCgtewYG5|FS5vD3C0=lf@c`>C=qo&l#rkLS>8d@dyqBxqz< z6h&?1d4D08rrVeIHAKD^LMUW*32r?zqy#dz11^D59o*#YQj@E%b6F(P4ob{k)_<;2 z#=4}*sgk*hcoR!nIm})vsJamK%=5?T$_m-1C#ZmIiE{N+tbl^zdDz@xKoQjw%}z2c zi@L0{_vaE!wbNJ&fQE=$U8jI6QX_7g3wS~f`;4ZOsV3DS%W+J*6w=Avx~f)cdiUEW zRQP=_cJyo#t%k{pAjxY&_)o9AyRPD6qQ{{D+CQbmLhI6dTa4V>L|b(9XK9#rc|hT# zWC%n2X65EYE~pKcnn;39zzEHAG8dvP-Jh0KHDo?|G;`k#^8O!d)(O@@_w|Rb&TN=^ z-<+IU%O8k0hmUZ^uyizXh*Jf7zvz6jkRGY*c7}xr)8v;j3eb;+xCGriFOu;aU3?UuhVx`D& zp77|f8$TH^>9Q-d{dxNx<{x9Tb6rAZMf2HD`AlK#RQ7$4m(DAB*Ouq`JR4NduhHL( zC+9rZQ_1vCsI64Fh|f1BO&X&#VKt(qVS{;~^9VCH?H!|I%ilS=`6Vl5O4p zrQwVJaeC_ERw=0ci3!~Auq6b-`3i3IwmIj{JI^$EWI<#9B?aM5r6yM268x*(ccZ>3 z`me5sT}7fJCylc@v~b+$P;ljifGA#lXc(4S&5Y0gNBa+oCC&M6HS%QeAw{LpTLoiV3kI4P`V9sNa;rtlO*S!q>v3aH!#mfRVho_b$Bsw>FcoH|q)V`ET z44Mj*<{xjGZ`?oF{^VNTv-`W2RgOiUd3wQ;s~-$|?{VULeDhU}yv#roO~;8oU|ejD zK{cJHtcrQPVh&~Vc~ut34&rK$)JIMQWjE#1_bGw&t$Aj^@U-Ze4|p~Q;G;@(`)fmz z#xpbK%h$dZ<4&$zGsSbzn>}OM=PMdN#@FoHm0F%X%IeYZ$T07rdYOh2M+q>_@ab0* z7*yRCEE$CF^v`pCDOj3$=REw5C|jd~t)~vL5rB@Nx!V|dWV7@`V3V?ae-bat=)vcm zwr0G7QKDd!i)B{dQmTw_<9m4Za76|Q{gJ&&(TJ$7`ceAf?Q&=)Hocv!->*95iJ{p- z5vW~XTtS>;t3zDnHil%Ha28EZ4>dR|Z3k~W#`zskzFy^xUWyJ>Da`-;kdf5;kCt-v z8A-g8cQ3|V%l&qI;|y-(nmAEwR$tchHqEV^&5WLs`_W99p>LZw;DfyFZd#QkDuiSYACF1}mTT+zH^;+r<5StTRKkUvXO?I!K^m(2{fm~I)Rc?L& zbVMTzp5{W=h1K*ma=jiaCc9!jR50SV-rk_um?Y`&*b zF)?6zC0XE)Vw~5#Dfy)}p|!f6)H|Sw9hTs`(VKC^hy|+DtT8Gycx4jU@{Gx2qx-Wv zI$|$|JL*sLyFl4`cCQis`uh2PkFT7%B1aL+DQ1J=k;ogD4x*W3Lb~kB}S~{ufnoWD4UBG+rd;C zgQ`W*bY5Cun|kM3hYwT%s85lt(JJCI_hTG!?BCVLj$cLIw-WUB4>w?s;|-c9ihk&& zFO=YJx9-n7WWx&3qHlt3^|yL*36Ts}DT*0JDlW$vGwKIO@jbZU*fAo!g|fPkQ<6C0 zOFHH;(9jM^AH6sDkagj`Yw?<7gv)UPrb9(&G4NC6ytNzB(oCl_Gntsg8d+OyF@#&A zO|jR`_lEM6>wCsfQKecXr7`Mdld5JX5(XQ2KR=uWP$jeyp zdpZnoo7WQs%Y-~dw1pK|-p@kW2iF8#sMhwJh7>VLzOK_Mi$oS#?e0}RwwdHPShiOk z%UzQ_b#|rJ4iVTIo&0O(emz+utx7NSR$h%}aT8zgnwLiGfmby_I*+@tp?RdYN)ZiZ zl%D(k%^qgDPehw>S$*!E`FRW`i;rYN3LREsF~_RN@ilKV4)UJ6H$2~*excTs80|XQ z{$5To_8`0(kETSVEO(Os#8j(1_yX0ePHspax0K>c)~vN;u#SBJr1q)x)j{3gRO z#ZMZ9vZClB$y1pNEA^TRXZ~QZ{4xBLO7L94+~%+${2!`p(y-9-eU;u}b{mB(WKI|7 zSnTN@u4Hy=yvx`|--;UF^8NZLA2nt$5XrASSKzH7*b|DdZx7@%+r)LL|8PNGCCDCs zZuMXfHcnepv)IG)K{wPOt~>Z%4;6qB^kp(#;DIzbHeB(S{|p~t_iXZXT2HtRIv`Wu ztH^lpZ??Z`PWBD$A%v})ZrROoO^-q1qx4~{#V?{qb*CuKCAufLUuQ45wQ2m}0P4BS zwdveVz4D_!*etYx(>_MzQ>~D)f!ax@RxW5%pG{so3M~xW`_>ioTpgxChgT0bLFE*A zO>s`$3}D(h((_pGJ!bkEcj+!D`~#0lli4ikW@tlZmtzQ~sluZ93|U!kekS=+$fB8E zpEv3w>7VplkA11HTGLQGr4c?3M}rQ=TjvU~K5a6I{0E#(_dw?4?W!sHvGq@M z@q{}Oe8W(1YBLEbF@48YVUWQ%cmFsZG;02$-K0`u`E6wb*J~vl-{Yj4CHN4Qr>wqs zx%CTU=T*Ub_?*#VRqn8T6??@-LH~gKX_Duf+qO*IjdRv!lbqwg_sGmg?;*L+p64@< z@5PT3EV~>d*k~gCb{6CJe|N&)f$4isFcq60Z)k`ple^Ev(|>d?L7O5xKT<30()}!Q zztiKDoyFnbvxoY=iYmd2!3S@?v~}8hxgCTy?0WmsTNF&&V2!L&2jXUs5ys+FN^K2@ z9p_oP%$dZWS1?S9s~wY)JJwhBw6}~>KQUZHOO5t6O+vPuoo^+k=wMxOB2}wlE(@i; z*(FOmqJ^P11_|W1A5YtuyI21D*>`1*8tY$p^@i6>6 z^U+&n$Lyd@>(?#UzGz1qNY+ktcZem~!7zoVW&o|s$jTR=z?~}>UJ=!>0Fea`=5G%z z6h0X?Jtdd4K(T@A{z;_D%Ho3~h~!61d8x(Pzd!O#QRK2L@tg8QNnRuKQxH+=E&Z$^@dfGaYF6a{Ah z2C$2->XPMjH)d*Z3&uL_7WI(lBqpJ+dRU7mN1Kra~RMVjJt40owaU?2j0CJZ5Q7)4SP zGPbUg6sL4?oaCGr%z#n7`(q)y^ots|+ThrQu`o#Bn&wf%1+1Po&)Xw0RikEI5%RIH z#^VEgt_n2NC0dD}yV)rR&!a94(YCVlR18{1t%hnIyOmR)s{t8MxqsXcA}$N)rEED@^Q+>>FWeQHn~rIC3p*)1ty@FhnaM+|C0M?Q}SpGSDDEEkbwm$;andg4Z=Wgo_s?FcQ5cbJf5 zGJGQ<^nvd1vqU4=0e;D=)*}sM$L%dFKsFiK&lRJiKPTt5A8fjvb`5v1yHk07*M9f1 zONd+AqX2LgH~ubbs3Ee7V~|(dxqo8s^670Rc*Uwt``#^?&$rIsD5bs`Zm~2Wi-TTi zRs0Ciq#vgIA4Y~-$l>pfQf-EhQ!mhIr#bj5y4S$q@peC69NT*Q^-gw1OPY3Qeo>9Kvhjd$xQOq{gC@fsqRoa%6{e0MqS} zzWC0Q8$YaBGHi6exYv%q_HRL1wCm7e#&5H!(8{5eap9y`w08RLLr8{ZPQGs+TpW0t zHY0fBbjig^Fk8>&U45G}_?(Ihit6f5op3LuCvSA)ZltM=Y!H8ab()ogQ&pvfM?$GlWo=QO z?>fOta6)Z1br>a1owTbWud_a+TI61pZ*Lqh)V&-~wjd`jm{!3SHkyEas9+1;S~z=G z20YI)$Kb`&^<1q{)2^m+*HX&GZGWpv<%;r9iItSHx4?#R%iSK@kE~)!nYNoo@>wH_ zss3i{k&*RGrhW*1Fve-nM5|8f^t6ynQ8&Q^dYB@O-9xKgAgYvU(m^Jlg|b>nm7!d2 z9F*2O8zjJ((K9+6n}3UGb~T%b->iBs(n#Rwkte;!_xoW#BE5SX<5wIgyDK&4;RRrx( z9~lhmG{DbnT{|TW@01U2wtjlu?R8_$%Bb^4l`dj8C8fsKb2OTSTvx8TyV{iD zxPr6Nbh1e*pY;a$5?VydVTseTpsU)er1B`E$h14HyF5QnAF3<`g#seD?@zz=UW{JC zY>gyUqfUOWI=f4bp+NIH6Vq>{Qkf217BaOBoW zF_GP3@Jy|acqBa8Z7G&Zz7SDX=?X2{00|;+E$ez!-`?r&R`IVFr^}vL93+<1V+0#! zst%c(6Cdub`DEoiuEGHeiUAWH zc9zL33Z}1_vim`Lno0*6sju0x`+=!^RxhAaQSYc-XqrkxoXdf%RyM=SY9{`pnz0l6 zUnocGtLCd3))P5hgrx#I}!pd)IQjYQ_4m3YO?`yVZz zaaE0R#vGfoy6W(|S_^J<&s`~AKRuBxWYl(DfbSSc+*4L_UM@n+s8yZ70DaGBMH|B@ ztCVTiMl7GD8N^SUVc6CV>4ELn3;3JymC2PC>kePtSy9Pe4@Ojy=bAb29sc zw|aZ0O?xFsdBFN`8G##BY=OPfR4eB5reyS9PljgEGbQuIwTbVVYMOCry{QoG*!-qW zy6n5k^J!^!vrOtgKE8?GKGu4#t-t;>XY~NtrnlQJpS>?{a~N)U{04PrzL}J+JWODe zb(Ygp{JCm#wk85EyL=i?9^H#kwTV_2ye(N_3|IoFYXS6&wC(D!*o05wMtd7`Qzf#W(AT?<%XUe?5(<$Q=|r zd$bGz)0X=904BeGv=7+wiy8t}{zwL=$E>{>y9Z`}yp4bfO_8B|JYWpI#JoL4S@FapkB>F-OBB+y zwWXEFD4R($dhfWHr(bw8-28{=`meSY@uOgqt(M#agV7CC`OUo@|L8t1<&786qN7Rk z*L%Ak^PgcV)Os=?aMvpf|H*MKGczW78|18Ue6co@LdntJuq6pjNvU8EfxD?%q4e+! zB}N#9*?j?GUo#s{?4Eo)T%a~fM2hy%{LQwll`=FcxD3NR+v$~Fy4sqWc||jKWu4fq z#?aK6^=#$RW#&Kt3wh!e?}C5d3v_PtH2SU?CCs%48yH5WI~s1z!ZWV9^oLsv<9{`H zuBSgsX~bog!e(T|yra#3lDD&1D>1ceN!t*!FVSRgV|cG9m$N)yPFyM3WnQ7f|4A#j z$5viPAU~P|)lsA;v%hFoJz1PvD>%#CziKZjS*m5wt7faNU^udNl*KYIkU_Hc_I?2s zI#2+lvy2fYk7;Zl%gzq&`Q#V@)0pulQdXzSs5>RY?z!dsw#4P8qaivZWiMDvvoIMU zR7y8grd&v-Zj)!Ns`8vV!*H-lvy{#pSBpbqdp;Ebvz?W_!PsQI+Bu#$q_byJ!-^*K z;yYCbt4u*Z%4YwYkr!j9?W4}G8l{Ji$5Oy!<`1W&A6_|`q72}S8-H{28n)0&@dqD% zUQHoj=J@gI&PSOCucU82yxUcMhq)iWZ?ST}t?y8a$%EB>a9f$SK6fd;(*wo4TjLut*9C?Z!8~ARYGBr z#80F)8AD_2E&;9oL&mJP`SQPpk8USal!xF)RzUVCw|g66W#Iu@nlq+z1Vwb;bys$R z7NQ>%J&N9+oQVIQzcI@;Zr)+?)NE3Ymn+NgYAIZ#Yw_QGl>Y%AR(YLn0+-h zV4GmQZGYI3rik{V`f|$`TPvCm@n3?F9rFCY$n_P+SjnKAn4JDit$S7yg09 zq%y~>+*IWr22@h%fo<;>AJO3xn(|goFV8*L&4CAs+7cnq*eM!7b>A(%4wru z{8zm9%BfSWQh18I!3!;kK-6umod&5oXyaO3Pn?1coOmg~swrnH%IxmG#AiEd*b=e5 zad7_tBizJ2;9%iMa6%&X4;7_e81K2#B|^d~1=DjSU{!+KZ#RZI4+l23%Q0V03y>Zd zEX22SlTHN|wt2t9WU`OUnrELw{F?6gXxVS&u3F{^U$9x-oirxXFIdnI_&iQOHe_u6 zf>i0-zXuATl5B*l$=Mma{@#)j!qx zVLAEZXetRopcMO8-inbM@1yArhT(mso2 zjZCGtsuevzJ!&O6wixm9_gTuuZ{+Be-4dz$M$;CY;tAW7nCeIpkCe7Dd`5DdA5*l2 znh{M>WFdnvgSKvvJ2f#Rem2WWRL)Ab3d^d?Ae73>K*h#~EhRlEu4zr$f_{+aWeF`c z5zOLs*lXit-E)+e$0;B&e1K33>}-g8RY>s)#X@sSO)pZi5C9>qKn85hrCIx0)+`~ig(hN;khcwElgDKi}wm)iBXlOmbz zX>jqQ4-p#<`$(thojO-e35oCxwW_qop9{H`twM+p`e>Le!&3P1Dwx`G9+Wg%vEoSv z&vJrE^YxiKH;Uzr)O#K`xtf=Ys@Ztihp}?*2Cyt}9Hynn_Y>(4$z-7<65sa_sXa+~ zrOnyIL^5qjyjRz~7`E%MXUH?Y;b=Fsd`zCTtCPVxQS4)61@WDeQEz){k`$z9(?H_x z;GwgT_;8a#=KVLtq&dXBGU|J}lDdUHYjwf(rqy4BgpRHo3WnV$tR!s3IyCB6Ji~VowPgg` ztD$e_oj-}(Z6qbO=Vs^J1x+Pi6Q>|{D<&atTwgekBa~NCf{};^Y`x87gS{-toU`J8OTIjXkVjWT zG7rMwvoVJ8QENbM%m=D4tCR_ZQ0+r6-F-K-*{z43-47d;O>Lp2#>|6P2aHH<#D5am z&Z_qmYH%}H1r_U6`_S56Hy0XA5f8=rLS^bs79f+0o5}BwYQKv*A8gU_IwDL(&elkK zQnEZq*1ht9P9jt5GKiJCn8v~D|J^EkDC@HDl3QY9 z>0W&a-_OYbP4W`H24Q-OutT@CJLPF343!at$619=r8)9bpKEuz4`BXVEp5S%e!6lr z9-GvSik@6gjYDpu5+@1EX9~>73+ITXWrYwyNp>2)+F$!ib>Pe`EL?@OHaB}&ab!0v zJIspRRZcbNzoO$|llP&PY*` zkuZDwYs0O{%Oek$+NyXFi)8sxe&+1Z@s<*{(mm7fq}ga&N5RN6q9 zeQ>JHR|u=5+9`X3HvO2t3PZ&alJlMP`si<_a*rpO@EIl&J{2M?}8?l_@VX zu}Z-8-@&oVggeYQJjJ3{$1?SJUsPX-4B+jg#poUipAp;dG^@SOy3wpy;6GH3HjL%E zg*M%sn2?Ke`=&wSJ05Bsgvb_Lh=yAtdCVvZo=Mr^%oTnE6A?%wLm|L^QicM(#kvVKwgH{Y)WwclXM& zw1k8!{;Qp`)wP}oRzJ&{`PmB;x<6-IAv}8Lq^Xe^qLOPW$x+cfa_Y-Vou-KZI zs~ArGOnDxztg0HM84ez7Plw+^M^7-3V;4_T2v5gcew@LQMyBH--?CODp`3{JKX-LE zz4En5%hQ+I*Hm5*+6!#9u9_Ti#GGWbBobn(rZ+tZEhTHXGXpUTu(@d}j733;=iPmB z`~czY)uB&Nm3UN9G1F~;lrK!C*%700E67z!>=nk-CUB8nc02@TcnAY6zh9<*^c z7k}(iTt=&iVNsbOwfXatoiZxa8YdGz?&L$pwbKKYlYENRjlV#arvCZH0)c(7mNF{) z$1+-SMp{qB&>|d^OqMr#G;;`rsm~}7fQX`8P9PG7CwRg-07ZROaF9cauLSsVj8!~BWZy35iTf3p0x!~Cf?6f$h_s7-L)Z~&JaMNNNMbqin zJ_`GfvrpyEgzS>HKC_aP9K{9&&@vCPUv0-5$z^w($%6uFk<`w&gsTMe)SKvwJStKR zWM#ZMkNR=38u(L{xK4Uk|%;;*ftxHpHb#pO2ok6vtJO-5kC%b{!`j(|T{E z;p|&`KO};|jz>^uj7?iqI1|X^WTz**qUYrW`N6fQr$sjCzkWH#q4a(zzLplP+*$ z>whz|W_Ri^zGj^-tH_FD+Pf?#DI7e5Bx-vmm*l+iPP<;No7Lspfm?IKN{Jc#>5G>( z@82{Dr&nPsS=K85(7d|ao7toT=>&h&cV#v`tprSYmzlcrN#yIDIb}m@6@`p^E*4hS z`s}7|%M>L;UFDpmBwDWpG6$m<^;!B-rrw0xxam!7o;IOpJn~36Y6cmjWN4=P+M*+~ z>A{s`S~%UCVI6*hw37$oME(!fobl?>v%Y;1c3=$Y)>+7SUrN0m@R%N2Kua!Zs$y8Hz?PZs z1@kvj7Nw`mrX{~C`8wyerjkE#&^ThPC(WFpzrCC+E%HqlaZBHO2<id!2fjl62vzHuR8JJq%V>hy7r{r-{6z6bJfM5dDHhnE9TRY2JE0(oRg`8 zYH>^n7C+@V){YS((9>=xW$AiGkD_N1!A;lvyHoFj9}9dQFDEBT#%t?e4mhK4RmxP& z=yhl`JJnLCT++T5WXFKoY?T1$yvo4BUn3<@w-=eK)eI-HqAbaa_ zIN8^C|Jzo(8EB%wmLMGSVin(~cxFdIps;c0$Fa`aG4{Mc=Tor_sPoT5SB_hm0=K@F zs=C>#kI#IJvDF@gXjY+LCd5#;exszKvZb zP>wZ(cBjmnN~-8iXWK9h!ryqZE+lnF&mnITVBXKd&hF45N>7sot9G(+SMFd80B9qr$BTm- z&9Kq4pYnsIVwc*f3>o~lmrus^4o$f+JJNirlVd;1`>@!zq=J0(IAN^?(wS}RHykCg zQmc*}m}BR_Bnb>&iRo^YR^OnU;XTpy-5Qj?GFmBEb|`Gq=sKxM>f6lmJ`LyN!ZB&k zR#)JAQXlm}g}bnFy0v%%&-?5U{V4?*?z?Ny8!8*UYoExVkno?- zU|mZR{-8)q?F|1#ZMz16&LViSac4X$HkJrYxnLDtP~qU~H>3MiPjo6r__lXHsXc{m zFUdG-Q*v_DFCGtd42dKZmYFZaa|rt_K7y+r3F+i8kZKfu$Ne?z;kh;7tiC!m20Kz!-i78BaogHg={_SV zOCSFevC31R%KemwJQctp9Uv_8EVw(QixGS59!eZa;Adqf`UvfaYw!bvfOp(t&yZp@ z8PlEkQTtfdJ{ynIT^3WLH}IW1j?{zGq|UFPOC?a3;+&9*Pc#)Js822(=Dm?*^>f() z`mqPdv=Je8Y(Yq}?+C+k)`ydDiYF&(a|0yRn6}ebF#1UyLfuOpkNo9sYD_TFtipJag}fu#L!* ziPsoc%E_hW1n0D=WB2S|z3fgOeCCozBTvQ{n>4sVEH=2Z_%Mimz>7R_)|bS>HWBn} zCJ`pQ$Gc3sIj>0(lJ{Xe#PCr0HA2&;H&zO8U-?EL_1=LhqG(QAIkpqsz5n_|&ZNPE z3Q^cX1{&6gG1Cp-nvqWLnvTp(O%g?yMX%~hIRjsQYMJh8@${NZE`l48!8>SiQwwfB zRt8TgUu(qWK9(U)Dn)P2SI*U?$!S~_K{!cgf*UNl5hvHb%%>=42$B@N*EDz zF(dh>-EGsB|2jn!SKj}OgYj;wy!K7mhM+6h4Rc+DsMgVKkpKEdtRFV|8jfk|*Pti) z2Iv^^gd&0_Um|nbN{Xb0%+@Z&vhj09gqLF{ECGZ4` zVYg6IETk_2{myGhY9~zziAI1#;Rw*-7&S`Q&JY&Bq#i&y$p`QRFTe<+SrK97Na9ln zbnEE_H(A#Dk(Bxs4)@$3E31!~g}k%6we9kDkQOQ5tsCFQ{yW!{UY=H`r=QJ+=}_`A&~EH8nNS-1&$JqoSe!&ImgFHo#gS4D0!J_a_vK zPcAmiD_Fq?#YXQA23e64&r=b{ooO!cq`&$!eCLH#iS+5g?~5wb*aD~@LjbRU!Y(^C z)p))$8H^F;ync9ThiH?vghoN#L_4c_)34#ZrRrf(dIeCt4kP%(%-Rc<>a^{Uqc#|y zRJ^8%(}7aw$AeE}-@+(!TW?1kj+!YbqIdxRuD4b3XlGlL2h3VwYu*i8zu+{#mmL6x z<`m$GxC4BThIjzk=(Tb8*rH_Qk<2Q#o4yyL|J>^fosfb@#qrqcj1ONhUBhB<=oK~g zL;UQd;Pi(PrtA;wKmVH%O8)^&fcI7fQ2hIY42Rm++3?gvaCFrHl# z5710ro%Z$gw0`0{1czsH=@Prd*qi>?1s~Zut3S9J#}6l%z)N^}=qR-$4) zuDKxmqXy{L&A3%JfI*^s5(|JBg2nFNL+qc>w%WtZ)m;Tt6pGdgam`Q_H;9 z{~;bzsR>k0^@o@iERy36<5{0!(Hs)h^GtN~1IRHzbO7j|R3TTw0j^RQk=T`w&PVI@ z@h%JL|Jt6YjUI6qbOG`PtVZjoXs1F=-QCwfX2!QS$J&+mj`-Y2n!eH40+6IAr*7cF zVK5j#494wtwOLgi~*3lw9^LPdtIpoM&>!<`<R1{c%LNC3+M& z1`-e&P3_6`-&be13dzdOt^x=H(TBF&~pKJo8(scm;E7Q{E01U6VghYBqhMr>RrDbevSjg$cXn|t^ zOfrKU56)jkvpPHC_qO^|LxE@gmqSGl50%j6?t~=^%RlT)NTa?LfqIt9kPid+KI@*E%FjbMtFOX0*Y0Qx?E zUd1DO*=-jgy%1ep#aO93Ucsnu$NAN-_?jTggIll;UC7(pR3)$p^nOrZNZLi!p#$y0 zlHub}71Zb?C1<xUB+u9c{9`KhEe2_D zQHm&se~aWrhk%U}Om@R>E!H%ad zN81yC^JoANV&GyBwhSO&rp8zDs$aTZ8k#>F5TswfkLNoGU6_S?^?ik{`^+((3K8xLZ6({cVeN@w(dmOR- ziGxM#g?TU$lwmt0y-n>x!2qpXn3c6`u6;;IJqLixjt&-c=okU!Z%G?4Z)ccB#>S}~ zG5lBm_*=Q;GQ;0wlJ-%;A0SMR>)ud|L1=jn65HEc`9yODYyW6vAZ$R}1K7RBcz^g+ zpftP%R?8L{|EZI8#bQfe2aEF_yH5pEsxvEO`n(Zx4{);|9kj3$!hUz5jif)o0XFN& z>f?H1Z2bEn^u18wgr8^7NdV>CCCC97vV}0&>;iC3MtU{(#NNlWM-QK0S zgA52XX8A1$H*GtBxBCmYab+9EU*K;xNL-il1P(fa-j8l!UAo%qHj}q&$VqM>9R{bw zA_m=peWq)4ZSA$@7MOWrB)MIO2Au$?&+BSf5bxnp z02P}U$g=A^f*3Ojw4q9lNEAG?9i_ZFu(l&$%h~g4tyQ#a{bg&ow3wpSq^^W$VJcA-AFdr zVPr?E_}gqr&y`AoCsQ~_?79H%CdSE4G8cfXscln-tE{XnYGwE2kIg-cUs1>zY*oB` zQ~KfG5pK`!mGexAb{r7W;IQz7)BKM5WZIp!s&#%gujL~D`P)RY<((nm0q}XKF^fzu z9DSZQx4#fnI~R>Gn?IL^rWwZb?u?fjcZBYC=ox1namI!c0_!}a9?<6XJCZY<)UdO* zUMpzlR|oDejO3s9ez62>zW9~zO5XH7_-ZdBP{7$Bgu1i@5JKQ{r~|0PFOMJppWBe~ zO!9iT+r|(0887EaRq7mpzgC>%nhxPT?Z14r$~gr58^;+k833<5QAGPEyMP;LV-vgN zJ~pe)GMt$+4x+eg?7x-DHRK$Kvep2uUQtO&-`p}09;wGwBOV|^Jw>5tv-34?2B3rI|=>?^#DIf87J{S%&m_=V}$Whl)UlR%8^s2 z8Z={>&(O-L8C#IeW| z5AF~^zW!T9tc+H=0?1B)R`|YSh?DW3_t=GJYV}Q!2dC&@c`emsF&tW0e5?1{+->T= zJo0SWQ~(@KXlFNC;hd?O;{ikezeL;;A^f{KqCBMt+>gVS^|ZRTe{>!wcLAieCt3V) zJiNT|fEWOD`4}LCX(4S5RaAQMneG!3x`4NRz;j76L{=3xBW3o#uvq^n?X#=e5`isS zqGF9$G;hCk525yFzylnU{<<^A0ENLd4 zs=-<7z+>$HtglZm-VShuVyfbRfX>b^^15jQIh3DAm@xCf3TUx-c2Gnh5kA|=%Km6- z9*F6I+E@lUo`XL3F3+FKA5RhU*}u#LNHJ6hw1mZH(m+P#+Yvu!o-1hWZV+eE`tcxv z@&UqQDr;}(tC#X>OuBJvAVjd>`^y@<291v6)e8-ow91|@TB4vWpV{iOj^6f$Qc~!b zjjXKl54J&TJoO4&ljT^@H(uNQczf!7B>&&5?i*Cp@RjK-4r7I^y-~P%cA96T>ad=g zNMQZ^Fl=VfpuBn`A~zPaWSk1Obh?6k2{f3brK9tWD>vz(eh&BZ^D86puA2lM_s)8i z=CcVMt&6<)HV9iy{tf3b&pzx{rtam{;(1Sh)e5HdbVQzbLfo!Xz}PDCFY}JLbqwr8 zOrl_%lamuZi;fHEo&D-tBVKLfrQveN&AGv-nJib(sCT&yoa#mjS3$>erm~Dt3y1f& z2WiNVF*#ooOQg)?yq3QRqH833y^_W$3&fC^m|d6t=x(z$BE*snJ^`=)2HMh6z!;+5 z15MEGg`X{;(DFs-EC710GyWuDaRSCi019K}z@OZ()O_^ltc$8PXs~(K6SzAqjk>F& zQ|Of6p1M3yS5+0CZ!_BL+E8UTxcc?)P0p8Xx-4n4)rSw3QSVDs6Jy30wClqlu8HGzQ74 z<+k45Ub&}(RgP?B8BNpQqHafm`v*s&tWJ=vqE+MOy4(hQo6is1y;t$p-GG-!IyNFf1fq~HYQSRRntGoFVuF^2lNQ&-hpEy3on1zF@q6E zEtxP2XoAezA-*BBKZA39ps`VG!$@4b34GKE=w%oT!~KKC68#lvutVmqDSsjM%l2h5wt#KfG1=0I=lT^YQYkMOOntr&|6 zlf#{rnQi7sjQVXaA`8%@aS52iiWlFe&ZylI$HNJ=i=O8lEN*ziJcR7i;{B`P--Z?& zPjA9n(vW}P?Jh{Bh(I@IE}keZwexd%bniD_8bc7i;efHO1<&7Z<%|{Oi_$@%^pj;p z9i1@)-s@|lg+^FM?x3A;3%RGKClDqQ#@Rjt*3+?a)U8YIUb4U76Lld5la7UQci%n* z83NbpZIKtVT&w`PM95tE5e5W;@U}>~NLJRVbAeoAH8gj~SDU!Nr++{yEB*@4Pap~0 z+F)_ef|ne0|Ga&M9N)tsY=>AbJlp(l+ms4U*>Qt{baU+Cks4feW|%VZr(cg9{_}-g z-HK`gKC}p0iA;n&B)5YPE&Zf}`lsD8IC6t8)CMp+>9<^Z+cwl}^VE*L1LEsL63UKs z@}?P~!b$a89f?=3*^Lw?|YL=>Q7pycr*LIzIAq~AV>Bhp4eqg4`4=TFXx-G?G>E4!+eVdD%4&xhKS%`_%l0=kbbPIscjh=DkUB)S9t*vPX6c3^dWOt${ydt+M%>NkH9I5bTOH$ zEO1v!31J3aKM{(e zEgVFLV&ZUWL71s_84Q8QV2ulLh=VgUApL%O9G%!LToK7Lz^AANqx5^AbQVZ{64J=^ z@=oN>g|R)XNVt9$1G`;D>-b1Aeg~UCfT3huLBIQFa2Om0X*W&Ev#~{ObD73jkVQ3f z8KK20>uF+^??QeK0tC3PZQHA~S>PYf;XJNxH9A2N_A>?xb`o2J#G$7w$Yiw}z2O3Q zuXLLbCWHDDtbk8hbD8Hm@qN6N7W~+hAaJRA;V??AEflhqGc?F6h4jN|J_8de6nIqk zdaN#pvO6Sd-GJmbkoz0kOh5eNW`7%^YHnk`H6CodYL{n3g2p!BQCdgDD-uphHn;`{MmX`I`K_@H;`$C(T&AX(G>Ef&YnLLH4ae0-#m zo02Vq1l37Rh)22RX&l|z{@V6N)*<*jXVL0qh#nbZ5qmSaX=da@J44Caf2Jl&pgsO= zUx9AE-Ymz@B*^Cp)Xh63}kmg5-jgJ9V_{n!>@qmVc@Dn{~Ah|y7y@M1mQ=L)M!=f3+E z-BFud5J*){<%D?%1Lv>Eo)AWS^{0M<_?XgDF#-Y3ZI`!gx(z-VDYMY~o|SQr7|KIT zf|+u^3fVNByr=!6slFh1UGjFgj|Ta5$r(h<$IR_$`Uy2ane3dc!u{*hYL9v#5d?f9 zBAGkKGxUlPm6G9xPNWpm7k}Gqwj>EOk<~a?9{Jpo7nKbc#kYp`$x7iwwSO{nU}aU< zy!>xGc=DpA&B3&i$yH=;+%VAZ%gbg<5X- zCjb;J$t;!kf)BXBy5E3LCEjJQrNMUXxBWIc3gL9*lk|pql9hJBsm}tvQhF{@>?FU{ zBHED|eGBzAhylvK0f|rZovCX(6@Mh$NSMjBJpQ!R)30cFl(9S(Bm$Y!;n|(!zZHR< z0+ep_LT9>fJVcAN(}QotBwYTp@Y4a|BOBqMfE;}voL&2iE=HB`r4HIZzgE)5y-u0m z)_32Qy~pYkPUEY^sh{alV1NQ>y2oC6D#xqVXd|9Qg7lqPs{9v(oe}Pz>DMw#8E>jP zKq{Pv&05zmCKjII{#j2?u`}<&tz0woxZ@Ulg8033Q*YJFxFYx+x&wbAbQE?Ee_8`|W z3H|f)0OCxBWinzvPDftI_ex1MCe(cYs{Egt|Gj)(PgF9;s`NT>ggC~vY2zeX*)5P# zJ;OHl$nhZej1mJzp0d0c0l5UUyGNeqOSY32Hokr_PFlCD%xyxLvx3;%Sw#NKLfs4j zYB!4sG3UWNO*GDo)GjI>LTqzf{4Mp@%vqm_chYtV!ce=d)utbkFya1E-t-5$k0)w~ zzfSQM_xKmptDipVamu*KFp8hwbPL8ydif>{oBAiP`d;HueA##Mi4ju}zTdBtQ;Dp+ z##psGc8Q)exKE*7!atUyDeYvnsoY(rfB@NB>w$R$NGjH}Bbe@@ou5%KzlQM){xPL< zk^r~y)(+u?Us^015;a1{Vv&K>C8&B;bCLz;F0 zN|RNSukMw`J!kG?b<~D)>>pG0nwb|HRm6f7ee){eIX(IOhN{(wDh}nuvEG6@3@+Ul&1o17)lp;LHYm@3_mvyhJy1MV*uaPXQrI|jq9vi~-S*bt zK`yy|_)JF7m^KXaWOFura||)a$)}DFA3B_C+n+g_wbVHoA##ssb1!)B_8Yuh(YBrP z34H{ZHh81Eb=j}m5(2mP(r<i*;9S4DekoOm2l;n~lm&Wpgb zJ_tX0mUbK6E%|BUZv-#<<`Z8oPTf|n#+>V8QE>_!A4IG)rw%gtqPm^K%c|v924cAd zbU)$jnLSdt^@!i){BHidvLp0jEib6$LvljPZcs-Qo(b-8dlLql>iDd-ox>Ii)(G*) zP6*uWujz!}DEagWBo+vnikiQg*xt1Lg(~56iWW?EA9c{Z*^XFwDGr4*V&!2a_$)0_-uC=f9yQ^wA69Ne?uCE6`&RE zzA~FF*IB-(yFv;qN(Z8tlj_`jqc89tyS5l7jk?9av)LSf9gDU%UVN^;<uBH z6^Pe2ua-+|Z_lVpzmK=vU%&2{v+r0KI%_n~GG-fM8ScYu?y_aSp)Wv4xix^obqj{_ zmyY9z$zKJd0*80+%}cl$yWa0<@?VB_t(4Mn71bdNy<-{++SdMxB^NIzYCG0MFi2X} zdO9I6gQ}2_H4aZ!?BJm@3xUTxIj@uKwfD_ydZ@XCDZgT%J)kD~V?CNs-B3iBkhg=M zB!LXiMC#0wb{%KI1sP>*g;S(&kqb}lw#3WbIh;HnUO88zL>&*%z({0qHcM%{QQ98^ ztQ=hXwsNGe%xuq^zgGxkr;LngcQ{&2`y2c+M(~|#mZ2h+$6rq;(2IjeqWL=!?X2e) ziI?e1@|zuTaf~-T!h@85#A+U~{wQ6^dyF+ol%6cU0wIXIdT#l7wO_iwE58*Ee;}Zj05`LJ{!6mTWRzZ@RDf=W zyx~DomdU0%%CQkmD^UR&sRpl<=(tlm*MyH4?@Uk+yy$eqY)duz#&5v#fFMq1qUwJ9 zV^Exhz{qpk5&cpAW2-=CJ(~)2xgs=7+`VuXmnYl2)>F;viCn3ByK9Tm<{jup?4X4B zlJUA{X~3AO(jQohfcD6`cSY$32jOe1nczU|E1e&FA2g0|6?!K&F-}SZL83__0gxc+ z{8TO0Sb#Y;-YruqVV_`ITEn&jv{gtERDwosSf7O|!q}Y9GKc z*egO%k)|%3^3eQos~!+=+;S|EceX8(=Pta-1zDF-VdQKY0=y9_c`JC{f7=l@D5VEf}mIl94YN&VWuNYf@Jqk>pX<>Wvi8xmjg-PzfYGS z{Fxc}4*Uq~h<1#y#&6nxQa1v#_+gLF7~9Cg6c2ZU-M^&Jkr$P=1VRJtpjjL96pCr* zV1NQ3w2}iU=h$FYVzjHbe;Ec)DFvh^P*UZK8O2Ve)q&`9EN!%NFBEUNb8cI;(3=Y` zeVkm42QY_h>E#dHIfs{il2|yXPV`kgj9)+W?ig7ZHJ0-3HM;^I`x3KvF1AYLAcXlT!7Jd1ft>?LtUZ|*H?R&jBpTz~w&%D4>TCE%FBs+9>vnFr{M55NM%hKO zu+Wt^+wmnP)t{2?3?F=f`jgy)J#c6$bzkd)yuOLB#uT6ey&#j*f;7LIy4N}KfT_1NsRPE%k!vZqeZCm z?}px5kXJ71l}0-xY}H7)eg1=~Q-;1mEcm|~G324#D1&`}?LznD(&%9RZdA0x$>_R5 zS~9L%4q-8M53hbVrK;ug$7Lc4yTZBwASrqN`)PJ29xKjl=spm~fP6Pk7P^C(qB$ANxpm*J5?90Y*bM@i))_4 zcryNSm-YFeouc8~W;j+;B4=as+KEcK^cIjXZWS#v=$IU4v?+`ae^vNuHe|+9Rqw3)#f4 zN7`J|kgA=|y1(l{Yx}JvIKlzKdF9F;h?bwLh7TX6hYP*p)C3OvIrB}+g8ce%?PEM1 z=`*BZ!VmL5+W@xoTPUa?EWF$K4(o}Lsb84dI%$~GSj&OC%(2K2A*;z}5QP`>IxO0j zHRk!xfa2dl|5uYRxoEm$2vSnBgUR6J(y`iMlRJ~9jqJ$^0p6&MdtLTI(2d5Jmk%#i zAe0B6b|wH(pCbBY1*{HSn>WrBXv?T}j2SCxhSq-}IrBmgi*;BQ-Sw{;GhzxLKkQ2k za^1b>cfu;-%2pWKStPb-KL@L%qmxu@hTlZBqL+b;a#YfAqETikV58=rLydetMSb%*BPZ zc`Y7axhp=x{-sE6?fGo_-TI&~iJ13=H`MAs%Im04yR0IH!DQDLd|HdH`FuhRg^MKK zWkXkWT(vvw5f} zy#O$m%I^SL$r~7+b7(o4xoeoIBA)5-H>Ls{&W)vizF7WR;gmIWGJNRQp_3Gw4 zG6tp0nm};3;cU1tULo%0mg|3S-WI-NVI##9is*cC3_PlDGdZm+jOrwON}C=V1g<>K z-d##kQFLXGN0+}Otgm4=W86_pR_kHcQaYiXvDX?# zky5L8E@is5vg{BL8%k1AbupKR-7%hJdB8aV@m7+v-}_6i3&s?8&_`RVIo7JPOT$?V zg#v8g$8rLx+ZVX2SZ`!&(Q9WddOr@<#KOAYl8YW5q-(_P$7mR%_* z(ofmfD(b@KTl>S=gZXWd#AC;dP_%d*nEaq)Q&G zN3_oVy)SA>_vjidwMEBC(k3K#Rp)}QE6+;R)uvZXjFz82a`W-nj;FqD1oKoeU5_2bMI|IW-RRajbNxIK+IHfc^ zxmZOm7L)(9YK|Oopb|Ac+4Z4i%uUmh&eE7AYb z3J1pol*nx(EKumgYNTj)G21RuN>;Td{^8;2H!QGJ!QGl!v;Hhw@1#x+$G6{E7Z=m9 z_}jplYWCjDk!#L(4jkSQH~%n(W-wyqJ*Jpfx5T8ff3V~vJu7sSH>UVBw%$+Omhe^~7FSn1P*-OGBj#kbm}iC&Ej zOPz4;)k3azfI%Kx!VeR}YfiN-=?2-h-*xAZU3rfC2?Kxpzy>O*J4|-!{AG+s3Ta_^ zhR2(_6FC2C(#`Xq6UI9#l&`;mscEO~s*mZcFq1=syEOF=_ve>B#D6GKw#;0{W7v9^ z<$CH0OKp6+9~;2{wi(~^`<9t21X_u1#~AZ*WI}Pud3#06LJX3ovT9?87<;>imhqDK zBb$*m8Qu@2iq}eC@M$Q90;O^#yr9yrnYF*!x&2P6{F~TlR!%C!GBen1U%DM47{$ls zX0(2ImN+=X*cPNcwjXu-IHBe_rkEJyB#pBB^9Mxm}=f>P?=EzaZ`Vp>O@9k16WKH6p*yt(q%DTXUL>B1&+cTQfVgn%?Pi zXei09g6CiB7}aN7!|7&mDW3Vz=B%dAsQJ!%J}%Q?6s6T|{mY7x#R*cC;r@>E$847) z5F=M|$A6YPwkRy+M1(T5$b>QuB)u+R%6w_Oa*RCB9pNhf49Dtrp3x!`MXvjhC$6>?DnmGO8M?R437A+0G))6Yzi3BA z0V9}_y1;~jnyneJpxZujy4B+(tygKx2nxq1pd`UfM%G%)1m#W52t2)Lol@6ej@_OP z<@Zv{96AY+ZKoMZ76iu@bI>qW0u?OwkSI#U458)(lHB?J7Ndy(Eb@b|P(G0E?B{u$ zOX2YP1|Ki~{&KbysZ++B(5_0=#{JovRe*3#S;aY4g|ozRS(iSo5)duyL!{MIYc{e+ zYTQ|-gVT*p4HTM>v3jeVY+K4xaFMrtW(qM@A`<_o=B~ z3LC%Icg?nPG#^cn)lMSCzH$_SH;EQc4oC$fzb9G^-d-Uto>O{~*c1B3y$@RAj)Q-r#a$-@-6zG^$2$vAJZ z=#Sf{l7_9(xG{eS(>^EJ4+zg+5vwsVJ5VrCrD3O8E3rkw?Fxv!UQYbJ! zXo(?#n|fsg>-3m~rAzelFG6 zAW4DUw=<5w|57gZ>_w;;#xV&rQ)c^P<-9~x$80ybm$=A3&dn*-kov%pux)sh`3!G4 z8-T)v>-$c}>fWhsT(fT_al|P>((hq`2%LMJ$7ykWtBdRGhOSO>WEaHfF6<)OLXV-E z?rMKLyY;)8qOIz>(0S5OcCL@u!R+JhbXD8gCiA3mPOG{MB=?-Q)#r`M2b%aZ*>_WN zAe|*u31$r0C7jD^ACJmt>=YJ`H;NC#3jr>?uVc7HU)b?fW+_6#GSjV*;ft&ITP4MG zSj^R^t%sY=dHy63jJtEZdee+PMj_sUy#t784JiWLQNe924s-~$ zHW^lkWcIkA6g7ppLn7iZjsx5i7z$#AP&*7+H_^%JN>{#o)T1(sMaL$z@-WZ_?!Z^W zO&P*mF%XZH{eg>W2t%*3@%{bvg4?N1N}!OSTQHLc@0#iohjpmQGH16`DOHJ#O7*7w zc&w#?7JX^=TFmBjYz?fxy11;y$zAPDYHo4&=+^x@Gx3zzYlril8y{n^nHP7(y$_4N zrkxnxTk~U<`s;=@IlB|D&zlyd6t=e0=CZvukA}?+i?~RK)&(AS9>yr!2tru&QL9G{ zQCW#7wSeCE3j^!NtgJj8K!qFR`iB=x(_DSV54k0qWk>Uu6*D4!`K^{wJ@ih?sc7qksZJ{|9snn^oO2k7wJ5m?c z8PDND(>!Z{rF>QWJJFU#v_65kY2snye;j+t&oOadeL0#vkyB$gZF~429j1{@A7Vqb z^g7q1-!b%WMf7f_@A>K}tV1jUkQ0`%c$*_uF?B5qOIxQ_GIonp%40s!2roXCu|{ht zyq!7`Cb@IT;Hs5h&&TT9FN|Ke@8Rw-?ukx?vi2p=K-n#Es(C?R$o!tA``i~DT9w((RnUL!i&c1eFr=M(Re24_ejW4!DhzA8u6 z&WASX@Yf(IQC3Od--u1Z%k>YC8x5*B?TwNGjI;6H9}TYuP@A6h*d% zqrCA{fwxxnSG^nC8C2??HOLOviJ#IB<;TXM2}`uOV`s%$ZuWxqIkmpx!`^sz=XM+T z#rsNbL|^&(txPb2eb}e231Q%yn+me~8d*U<_m8ntDFOnPHheWTQ%zxEhjP|=t5(e! z17IORt)A@7q_^_A6uNPx_Db%S=fgSo9UQpIj6tpU^B2a-D7gcgc*3~Txk$WmvhQ`% zIqHd)iR61zI4D)+O0xVxZFy2g zcDA&3Z+G`4h^xlaOPrjXfC+~OnwEy@H5PO4N>BG>Du*Zg8NC|-pIZe)qczNP?dHdyAtO(K51 zYhsuyP9I(w>osg)nrHW+L%^ftld-@0*?b`8myIO6@#a@hx1j`Z%3;{4Z`uW=q+W( zG_@6y1dEz7S`z(U?#pXu2>B=*$Tw$bZzZazr}saHtvk_r81zAuGFy7R?VqXOtFbpG zT{BrC9$qIF#&*$NB;PYkx?F8whrNs?-F5R;q?wp&& zy{v*};xn3*??fh^GCnc6>JcHZ@SL%mfWIa+Kp}$ai{exktB195vcFVFG_n>u2ZxI5 z%R03vd|~ZZbxG)(pxC+B^71$V?#jTea6(f%+N_KWc>+*t4vFsj{0a#zQDT#}qqEbS z+?o;GC=(My1ynP`*f>f4|XcwB+#-- z#tVQJD=EqbmXe2zg{*QR%}IF!G{_!LE^oZD>&ihTQvnP_?X?aUA&XB*tN98P7HtS! zrc)$V4D-HzI=cEnxG%}ngM6^37b34KjJ=$&NIn=gE^#bh?iph1w_WJ5B9z^Emzh>Ep)w-t;fmoI>(YZ`Veprf z*a{Tzim&XLgr%Y5fu!7{_-6dsJcE6r-4f3|C;&L?w zt;d?QHu5+lu1O0&P}~#~?iBhS{@U7VE_;=ie^O1Zx^!|c}ZOBxf=6u+p_`~Q`IA|rQta( zn5J~mDunsC?Ygw}iW3YX1QA|eA(LMe6A~x2R0#kRn20voCWpfV?wurw!s<7!NPg^n zmsG)@yK;jG;J@fdOnGR@jV8GX-kcFH6|*ZFgXI#xxBS)VD@HMC+W>#UXPO5gX}q5g z?uVpl+~>Fqi9Vzm(rMQX*LUV{*Q>pF39_hhi=Mqe!IbP3vIKS&YSrbA(Q%0yiJ*?2 zok%3C(6P^wxuWZ9!Pch-)?epjb=T6(Gxe*)L?*iF>k=53!Ra*Oiy>$wkKfB@6?QkFn&*0WPy+Bxh@L?G-gecY<HyM ztC?<>?jb)Pk{IfmWzSWC=e?hZOgZsu^HCv-g`v16tSG%TRFgi`kVPE!*j9;<15i0k z?P_e6h0@l`S474O6Ct{!hPm=c$Korw5O1U%EGxuOpO@9HR&Fy_fM>~iHtSe8yWk$? znWgK+S_#Q|28vyun{5hbvu<}T0s^@`TR7nmf zeX}0=<>UV3U5br9HnJuWn($M7qEnkNq_*KUP4HE-XV?XfWA+Hw?Y#|$pfQ_ly!rn9 z>hgQrrBYQ^Js@^`!m_6Ew%gKB2M|>%orYOeMn$p(LH?B?rwv|z)GA7+!5rE>XDKL zMBl^9&`!X^RFq)&zj>07%Pb`ps<$GmqpR0FS4SVm=QC??_1X)6lc0e#r2@){h<4$a z+`HN*kT9Cju@SBvwb`%`WHu*ryfFWk(xQRR>gwx}0X4ykjwDe96U(2! zex!FpK%vfG60_6wPfgMTmGTK7A-1%`Bi!3uL`df$&UbRtyEE||PC2QTDY{1k1su|? z2{q-*3*)$B{CtiU%buJAs{E!~^f6v1iJQ52!S|gWr_iLGPA-`cxKSKKb5r86?#0LM zi-T!=Q!bAo$_S=@0Zr{N-io|(U%sQqgkbl=`9w0dYn7m-frG1qduQ)BIv|cw z^tQCR0Xw@mQO&#dzS@R)Jml6t8Ii`PC^o*Ee*Ma_A9+}rU1X5%AKMt`B9uPRixHY zC1!fQz~VR8ftFtw;`Z%T`Cg9Wgbst0#`85pzh{-i-3$_L&CJ6?7GLIVZf3&ICw+er z8m!aEPJyes!Xm0KYb37yEwvw2QQllu8-!g+#z^w|`G2);Fwcgcg*r%b5WM ze2!0t@y{1g%0gj)kO`U3?^SlKQcA!-B^k)=3vNjeEyx$;4h(WkQY+e@FHZSIkuWJ1 zTfc*^E()&q!;9~6g%lwDOO!NIDS!peo(M)?CE9Z1Uo+?TQQ~Ig>}??*?$g`K`m%nV z?&-g#{``*Y6Nhxbr>jfh^*}7w|0-HY0qQ^A7_`vPJngF!NBY5>e;tTCrV!2)(IyigIVWg(oK3M7#(znJ8jr^66Sk|q$(KEF zhTJttVf=Xq=X*uwFSU9qWrgnj`rsBQ+29M*!Mpa5knZb0KU5*M{$mhOtzt#)XImA< zj~gWj|D|4_w1?8k>x`QQP`gwZ0~Va2e?F!^_5Fs@GW0{cSgs#4wSVu`f4){aHXKy= z2#FB;HR^qokh%D;b@Hvha>O>AlmY8R*b!pjKuqz^+nQgb8lY9$6SFWz&`UVS?o7Jm zfMw+z3EnSWj5KT-z;bUMfgogcQo-~vA?LvQ*JE?n)4QIZ-HA_Do1J)1p{R7e-1WC` z(XDRqegzEUsA7EoTWR#0QUu4R=Pe)a2u=3cM|w!K|4SJ$m!^>I%$~xQq2^3txh*gB z1Qc=pX(r0JePOEVqJV+z@h2b-mH=^6v44r-sL5&uPx6Q|P0k-CzxJ@6YOJVtoGmTABjD0#t$)m^xcSfNWpGDjfpG23 zz}~iht#=-sEHo2TWqGzs$Xxr!T4Q=nQC4UJ!j0xxb|-CWB+roO^TkWQMLJh+0;-$x z0WpwD6lEL4^n4XN?k|maghP}6eX)yg-jxBiE)Sm$?Vpo6YsLvUfxul|l_>sakQS7G zX~b3Eu(KM%AxI3(aeZ<9!ndCAHuO(Dc93JGpx_(1Z;xZ+2 zH|J0s6Wi(9a@f_nwvJH!1bsvA&cou|s)A0o!cG41D91U0Sw}BE)$26*(5jMFMU~W{ zZxb)^9+YVQ@TzjstIw%>-=n?k-ogiWY`U!O6bhxg#`kq$y$-1IeqK|DS=PxywhP z?)MdRtp>4Zvo12X5cBzlLGqL&CZRX9aBm`6O<_K0Aug0J$xrIZAXogY3Ft`v zzdlnub`cd9k74C*Vg_m%t;4@W_V7B5<9)bife#7O-O3%AOIqQ4|B^#Hi)=OYLCm%P zb4GD<_%ueNFQEn{+P?qC_p=vr?^_ofA^DM%^t^=rUz0YGwYnjP+v+gS)(wg5S$5(g z`@e>Z;r8{I*u)L*L6fE8jAx)@91Q&a^HJI}dB{Ij zwtXHdZf$_w-cj^z)DiW+zs*NvFnp|B-@Z^Bglem>wvnJMz*h?)Li?d-(c-c78?2m( z88gq{Yfn?7onIuy7ZtRVt*utui_Da}cnl1W{Kg=#?8sJfUI(w0+^6i=KUB~J(`v)1 zL4=T#L|z%`ueayfDM+R#-y(S9nzyVJIU;Vv!JPIa z1eezFNC^%a%bPH{D!=Gmi5tih2^~Vo#{w1$7dC&=YU{N$S~C!1C%?7d_R*lQ+m=kg zNRN!lina324lr@WW&Vn>l6*!Tv6-^^4l8V+VydU9zP<=#Q)OZ}rJ7Rr_Ff#9uT%mO z&$oW_3GVSHp(F8^DWs&b(&WfCbF|olsg&Ciq;-j>#d)SH`mQc(M@KSPrg~%Y9@cX^ zizo2E*R%J%cwtg>VRE^kooGSHiE@0o;KGIFht}5Cii)8_Ehd+j{8JWOik>u}OO3#s zOWh8z0PV+z(lL!{7ZA7u zDZm%y^Um^-W{E-A5CrwYYpkZ=gyCp0a;L>lZiW7rF68mxJ1A_3bj!-fQQ2#=)6m>c z?+Bxj|)?aV%|@+8g^ zdol48j`3Rp@d38Eciw9Y;TZe9ji1hY3uT)1iF?3nJ@e%v66*U8v0iUULnVVqg>>S$ zT@P%f7+F|yJ6TdvQ{nYX1G!kNGLLq&H);&xag}d!zAeviXkcmCgHHKADUeswH&Rn% zQ5W^ThTHjSV21+*VO)21x*YKhEAyG6;WDvc95;Q&-qM~5isK!A1dglTsZReJ2`(Fw z10S{f=&Y=)fZD}pEQ#xCX;tYrhkpM2z-}r_B|CtGS>>`g0bVekBYs*%lRL!{VnQ&! zoXvnj`vAv0YSz2(b+y$4#T=ukBWfqLhiYpTDr4K(XR`RQ1&BMQM9r7<9PcZPm$UzM zNEBE5r=_G6JsVV{F>(pZFr>4OvG{RDjQfOuf`dR-vLxJ)5kdw0v3{$23%3McWOmy^cl886pER(o$ z04K76aF^0<$v)U)m7JlqJYZf08H~Q4uS?%Pr zZSQs+GS9t_&L4a*i4a41fQu;O@~%U2+aI|E0ZbTM40^!6d#}yJ_R8P7Np?yfdl^@8 zsghPfM@P^WAz#q7v+@;-gukq)NV!>~K))HVs<1w7yh>&8wZ#0&l`CMyNEXZz1YH+5 zG1H#gDwK;Wp1v3l`tT9K>=#t=q_X@;^^@IMJZw9;+e=Z$Fy~(RLjT+E?#GsSzP0uS zXB+f=B!w=nfXkO<-l(g8nS4%;Nd)x4IQLv^b=+5d;$v!cpEX&!zVWM85XhI^RT|e; z(O!Wi=l=5MWpZz-ALq>^mF!ez*s@*1lkM`cKJ5RTSQ%VgF@$Lh71<6e=xX{jWEK8P zhifz8d-1FAx+B-<&?=j>{-8bf8v_s`q#tL}o=;fQbSwXz@i-x`U6eY={vT9?E=LeC zvns7#%sX6Iczs-?`t~in$z|e(8*fg&*_Fq=*7Iy*epdEV0hy;L@kv5h{76~=Pembv zzTlsQ2jJ;*Kjg~V#}CDy2?;0t78d=MwUp8mS8HKrg`rQ{nGJuTAzG5+tq@>0PWwZjaiHA34Nd=pyCxT)wiDOAb|N9{vLxcYusgF`?FaD%{^(Yt zrcK0D#K%@8&IkH`^nixa%9__qcURr%^Rm+;(6IL$h67{0X4$=qqtNNa@$pN?=L|Ca z_>Oqp3Pc!gHi<*Ht#UVv6|WcV2liywi4_fr##R@#DQNrny7=WEg)Ym$jc2AOVX|$o z3xn8BM-N|IF*?3eX&lTlo1LtH9iQqS2gfc5Vm_4YheL#-{{W~!{PZ3qzy99$+E>gj z{zq25kqJvDdLCk;1_Z=!X}_GhNb)XfA0=P!YY;Ge@376<*@^MT2*ru{g|Q|gzL6K2 zJqFy2=b-GHbpIP;<|enW?XD3?Xc{(*jmqv6tBO;+zE?+CYhdqKcjM>R$;P^%?1Sy< zrP(uZ_3!nUBiJ@DXd&m_*X&##=3=-~hgO*EqOMLzggUP7U>BaY^^nBKKIHdE4`S=H zT85dWZ_2Y@|63J%APqPT@diSf5alesmN0-;EDb`}2E^@lnAIoL-;(3cNb3X-paeChVcy@bLVGuFMD|7{KSYE z+`dizgD;`bujckcOz3}%z-P!Azy(q?gV^d9gl1SNe2%fpurDaHsBQ}wuCx}%sL#vF z@%>>;()Ffha=sa)w61(=RxHG@(EhKCD^Mv9Ahtfd3TsfsxFB=^zUEQG;F){GKyZG2 zeTHh}S6^~lp^?qXBGpUcy7CNJW4sM}bI;rJe|AV#<~NJ4P)Q5~^z|d_wwU~y9qA!D z-Nq9Q3MN6XX$24^1HO{TXQn;d>24z5ut(}-BuxAivolxDpmhC9Xzky6nZ|tUG6g~3 z6n=eG?C{|>k25Jzz z?)TwO4F~EL|565CK*pos1l?IQ7UMk*#z?vN>g>C&T~XL-i|F6WgnjWGSO<${GYB!r7;b2KRHHFod#p$}JjV7zw4VS*$4`GrPQNzijoEVeLb!Se zf`#?hLd8vkE|%Ob-iu-`pWp7$f0yNcVz&Y$_L4uXjB}3teT8OEXVUFU=WY89t?G@eyT!qXui~DdTdu~GxtiN=A0%}Un;1Y#q z8d#KZ-Szg5<$J~7!?uYgNqD@LH`oz)Vc}*Iht?E^X1iB0q-8seD6I9%P(oMfrdKKmOCO!dnTC9fVfeuz4U1IjXx39faN3s4IF)W!m=E1qE>s$f7IxH9Lr@PK@= zr87drX+z&!EiFDyADc;FGM-=dW>mmM2+?0kK~;uOSGeZ8*4i1uKi}me_x&4_?!1d^ zwcJ@IUf~O$l=8FNUS_1EDQ}U@f9q8GOrLpCyWop}RnAf8=W;^=9>)L7>jT#O;@iFJ zfD(43I^eh?0bf}rW?3R?Qbsda!Rr>dJO&X5NAbsu%VcETm9B-r(KQ=f9p>x_Hu+QLiODHn*Wczw+xGVZQq4WP!LgRlu%H*OIkoW zhaOTv`7N@3B9;A2-Lb)*&&!xbN$} zuJgQd_7}6MlhdZWFQTidFs|_d&FZ`$KuD?q*fuA@Dkq7#E~`eyZ#_ogR1EE8v-(B! zN8L#@sEvgLUL?`1{s=Pf7RI>#$6*Ey=zO^Xztz5D)2}nj3xWd!tfl_Z#gbgU|066+ zom8ZsXjVeVyL+P|;Of=_t#gkWrJ&=ydp&_y=@!c8;K$?MF`JT^ywUt-e1ki z(>k-DTh5dZs6Zn~;H(-CsO(ES*(nX^rqCpD(^JVZVBWit7 z8RY^uL|2Yy-+A*N8tV_q`t&=&K6b4sm~9O3wVkDmeo}rMMxsLQKO}5kw4WaK8`pjlq_FrFc=%PIvO$8nTn?O*H$CW2u{4 zdM?QLeJAa8Qa_V2`APg^7+LUZER(RFSo#z{J=JRw^pt6l9If>h#Cl>FDr^HTg0r4`ZRereG`GF=s+ zj;Qr1725K;bTxG((K3R04BgufG5k@wuq%1m(I|~qtr*9b>e^gI1KGW zqj@ynl26GePRgOZ3AIR>DVa#4(9m9B$A)Rz-|j0@1rbpnGOpRFK+jK z%hO>C-&l}?&B9BX$}W;62;0{z1fD{&3mCnfev!4t#<~4moaoMfa6s(f0Vb z&2MB>{J+sBe%aXLr4cc-Hq2@eqf-(^*f#}jaQ{@8aVkV@Cys`E){&m4=_xV&+kK|8 zi};0UFQW81fy?oQ7%nae-py$EF=fdQrPLD!xkQnyx$?~J6mDDGYC3~buxOE-QRRl` z)Ol=lx>X+)ku;UU%#@xIyT9G&?AUW|7d}WL!y<%{NkUII z{=MRpEYbrq=`Ww$M@wL7%ki^+=ZJJ2^@n7(bpC*Xdh4|di(x3SG-_##t@_Fu;YEeEfca}N3lMkT!HFL~aH~ppO zcH^{2!3hqL@fRsCsk1R50g5I4ZlLF)A^)(Ea@N}}d|y@KqnqNt+5LZ9kOKF&-oDuf zS3hCs4|8;aXM`R`+_fbcz9Uykzw(RuL`=n0agc@VPm=l<=X@TKU1R&%)Wp}LUlJp$ z%6MRlW)E5RgwZDsL%~-i7zGOSqV{BtO-Sm0(fbXgolxh+kCtJJNnHK=m7L0EFk+QEy*JAmZjv{mN8{o5j^DpZDm$<}`Y94?DPyo^D z%w&S#{d^-t^J`Y3(daG zY9YqV^;H7)Ram56ashImjXVl$0bSZ^vJ|F4B(|g@yK8;{Qqzj z#VU!p#vX>7K+6eH0(+>X$EVe;g&O@VFj)O&y;Hc6(b;Cd1j>5nA^q$(wSp0!_576I!fJxO#19x6nTm#F0T7iG_5ezg-od1nn|06mygihwCZ5a#0 z0JHjE#6qHMjtAimlFlZhlK%0fQ?n6-xRLRiS%ZE9*sk}g7_hQ!pZv*H|F`ePH#7j( z1f$6oec9NgS4QPq7%n&YN{mJ0ilNYY&tTIng$x(Mr|!f#yzA65OL%g#FaLf>@MxxN z(m3I7oE?7aL<&RAoGG=7^$R1hmaNx{`5)xCBB7Z z|9C?}n6`!XmSPn(z%;I^ru9aRfZJUsJ#Ln-^tNkvT0W#L^HFhk+vzLvGu6A6T`KAS z?Cw9mkM2T5pViM4zthFZBE>6u+_XYAoCr^d~&y=X8s|o;q>+z=M-bc|hoC zCi_x#>R;JtAnqI;NA1~VzMF1-I5O}pE5B{@I{D7}>*w@d1b@bH{u*x_qU7%Rtr%Xh zt_i{zXr*q;mIKQ~3sDbLXBJ#c6^P2&WmYJT3k^8=$8CU2{yd2GXKYq~DXl>-`Yblq zRg29*kz9+*^VLE8>r^V!Q_g`+`l_Pabl}UxBhha{58DFE5SwUO`}=3<{#7h%kR;=^l;dP4rNSAm4%rPpbM3j zq#~|UC3&pCM`nbCbh5Ytqq053w5`8Jv6vvN0W;L4Zs-XA6zHVWyb(5T&G*j>;cXW9 z&Gvm&Pz+Jq=(#?JzHlq^1B!@2KA8FG^Rh_2A)@K_Mz7Fuo;w7dA|bR4%`USx8HUN1 z&GIz*T|TV3jTQFoY^g&Dz^`yP^R#c_-Chd1n#uebo7BZsJmp8N=wNBQ{K15jS>UDbgFSK0U3|g~P7Yd1nWYgCKtdCl| zl6@MaGu2VvG3th4lLe_M49#1YCpkO zdb^+}rMa5t`<*e$qWs$#*u$0mrTMvp=mUFY(s|1BY9>awQFqWy9}n#W`$MlIYuAf$ zwGWTz+=-SK4t-=j+5E8#ta7KVNM`vN_bxNBn@n2dyvDM!U>PZSyUj|HZ{mVwM7A`) zR*OxtW79+4SI-lH0uU|y9zoaSx8t)(d!g4FYnj^s_WbSO2Dav-uEd&NDf zT&zRZV@!KJOI50p8Qtd-M(oJU(qmRqwPKc8KhP1*mADR)TcsR|PtF{^YI=>b(vT>6 z%>}=&?DqV5HS=cFcTyI!lhf^oJuydv%HW9}aegx^*}RTL+T-+goHMgNwq-v8sVAm% zC)_0AsMKeb&P33=7kvOO7#hT`)BgvRH@g+$nG$$ciK8v>P_~mM+@rqBX?%(D^<&pC zsVR(BEAWf1nk>GqRL?L+n20>(X-CoNMPv`$8B)FzG?lTmeo-=J)8cm;%4>7SZo}i} zhqlO>!qsu;3c+Rqmtre(U0&N^i`VGcY1-SQda9<%qk~u;;g+JULpRB|h@$!EM{DsV z;_z1Ku}>|)aGYP#o)0usBP)Kw5n%pFqiCDtI*9lzUMJBIu|w{_kB{HkLrkFwDv*qm zy70izT{j^h*Bx&>4Zp8DlJ^iT*f0})HDf)PeiR<6>PPI#I?uG@7{*lu+x6m0qb3D^ z(9oQl^PQdWyUV%^bNCXcXJ;L~YDwsbt!7S{SLoSzwLU$}wjuvuxYh&Pffg+dZH;;Q zRFa>+{p2K@{mT02B4-hdnlu33FkyF2KHFp7jXpRDBPO&`b7{QZ zr(v+f!WLIRvA3U`O}D@ihqLEZA2|L)9?Wq2=r4+ROmUCRV`=PHTdZOmdU%+Di;*te zp&R{2Bafnx(?DF*mRyh{-WH28*Yo_;qy}C>sThl?92?JBmLF*AsT|6;=MCW_rCTPu zs~#w^k+mZy_#(4UzQ+x;Cd5nA4ENDoG)|&IvN(g6j z!oUY|w#*TkHN1p)=7_NfLmkzDpo6aSJPIQj?s9bx5}S`1wu6?!K77 zL{W$2?DtfC;wMm*4{(8Z+zGiJ->8jWH^47~_jjUH zZUm^__%m+{>3}!<=~x8(H-lGrE}_Sod)dCnd9?7Zs8w~7uA-8i{f}D9Mj?ijXla-G z=h zbe^$JduNRW+fIdQJD|Fk<|NzS-}t7qC)w#VSI&>s>vIX2IDV(`6rul43i=DXTB#4V zk@2*hz;X-fG9>okD}ordI3VIS)V20oD5*6x8sGxB=k5vmf$lUR)p2}hfm%sQvdwnIw&9fYBhC3$E?f@59I8KDUeo>F;%ouDq(1Z`;tOw z%z(i(NK(S}d(czzCE=LO7C}O*p=>*1Fw-qldbz0RRx&TS$JW0Me#47?y4`}BW{%$q zexvtpdWEKAr(+zXYVeyV9x%%g1>k*hXg!zeJunpE5e#~L`|se|;o%99sJF&GD>P5< z48;sx5On>*Wtzyu-@$J?4<<}L}%9y|7 zBKKYzLv&YX2@#kEeGy z$lG4*DN;DGUrC7LTa?e~xx0Q>>W~!`T;gTtIQC+6ii~J~`8l8pxt>ghceBk-FA={K zL#!{vC$9V*cYcX{y|KO_@2giLs!zlCeyc>FgFv! zjAzYF<7Q(XMIZl=weN>FmepQQ-fhb3T74!MqaoT>TEy|&iT^vs_E+|UbLJY$>^Ra) zO24z*y&D9!%I{ondQmaE^ixs9#-53=VD6~?%+Mhb4poyb@5pyX4xJS`msl4c!GY?t z<%;S;MlIHZ?i2=8L~J8-IEBG!m`${Pz%w(0zq7t;qHV`fxx!o=h(_}ikMD%bWgmJk zc6lVw^|^|B#wM<@teS=SOMA~sGd^YQ%o38d2n$*oQkA*HEjW9OmJqhcYQhXw0G{Sj zWEPN{+1kE4A}gbWY{bwlif6^1>9v!v1eAP_?MVFLOnA`|S`$EtF|jb% z$Ea-5VG>q3LSo|R&=S3RRVpbRwZgL%7rTio^j;F_qj)5QgeoI7FG@{&dV6z_q>BFjQB|8*asI+ay*LzP;e>?b<*%bQVSz0h>Zb}cWaf?^ z0UbEBry5`SJ{S}z z>F#B0e?Ok(oK+@`UkO>?(r=7z6W3jhpb#`;oge`o$t{S4>`USOrB7?P%?;MGEh!ad z_yd36xd7xFXS8(+g^c^_Ozgc#(Il^Vp*6{=%6VDzx~IGyie$&q9SI* zM6GR_2$Qfd`2005V%(elkhE$rRSSXRx?#}l&ST8r(eOlRXrfYXHHlL|m6XSryA+}( z^)MBp=6t;KIx{DX*B*iB?jAA)Q>?o37$%$CDhFWZ+KtZEJ2?tIcIt_o%{ApYVTris zzS?S1;bAk9P&FyV&)%#fFr=Xuu_fZ#)zzy$N zBFy>*Ou>y0rM-w_5-QBtMof_xy@{NdTjsU*V}3P3=0#ixn~62|8{#vzVlr;I|4sE8 zud!9hdquKJ*n7osD9h7VYTucE|Ic5OOiz8UlyC`NwXKzMd(v^dCB4%~`sv;WMhYtp zRkh4Mqhju29xq~o2Rfr@W&z?dyccQhH5LQB=rrC}&LB&pyjsCJ#e}$~u%brkb9+SF zMp+5w0v5Cf#;=OGdCbtv&t6?!@6utJ_;~=`xT46CDiUfvT+~87b~O}{FRNAL=d7Se zJgG6>V)%6nDedh1-N_+qn}wNvOn{ANy|Fx^bN-PUo#tgzy8CSEnjdyDO`yQ}c6C

=Pmf(5k0YWE>x58{c$&Ph^;13{5Maw)h_LT;wpRtrT06BYS(a&|Noi zfBkC3nzYMnY~~3+l<1GU+EugW14Lc$=Ha{+gi^+rh_>!Ap0dmbEWLRxV$RkA&g+%& zpZag0nKcd3Q1Dng2heUw^O*8dtBza*=lU(}S!zWxLB^X_hPB9QM$GC^ zlrg;7qIf%ccK`-C;DL%Ia)L`J{!7Lls6GwlF|H~kAC9|NtMuLYEnwusd_PKxZ4qO) zZ(1U_p$cW!N8;c;QwnMAH<-aL_ZD9piMxH<`XQQm1a{MEc--9S6B=v0dYnjz2gs+_ z%T{XebAUR1hDiiU>X}KI^JxexMSf|ypvKW>xb5H)Z_H3n& z+?SkQ*(2@d6`>fwJP##ZnB=VX=qkL`JaZwu!O}S{Is-&@#WfyQ$^z z*zx4SQwTpT1H*8xQUlx~&6T_Hhi2|G}7i1e_VU) z)u~j-SK_L8q#6OcBBsQr-^ovqRW~blvQ00O^7CtawG1 zGh}y2cC<`v?R82wYsKQO`{LsyEZTt_jB|O`KevysQ-k2UdgQ6p&TJ-0^k+|K0*F5@ z#(n5bFmVq4cPi9heQaMMLGaf~#`*P&CQjeH^+t#Wmyeyog(m4#ZVk5D+mENZ^>Lr? z5xoY9R9B8j-8NLV@75b8X{&DN?}moHbd+gLjFJ#y;PN~@{Bd2|tyx?=i>aVe9ZI=( ztxxiwzz7J^RG+WVMeH-jOZ4VbZOx=i)gCs-?vZ9i0jTtdp3Oy4T7}(fyt?^Eb)@jI zXPe!L1y>4gy#C0)lllIFs5IRI(>)=Ld2IXd`vs-$9_N+*CFSrh!4JaY`+#yF z2sK>zxmyBFUl%y^^{QbX#%)h;PM1l6_~vT$Lde$IiTMW*oPOwe!e4TndcmP1*of}X z!hg8%DEZq=HLcw8 z@;>(MM|R3&w$87=5^e6oso(y?o)D$_=Yx)q@+7BsjqB*boSsQF8Z`)b9NY^)fgE$j zrMG|$A*G9oZh1kMpMh+SzlQ~GT6l^8r};(g0RE26G+ zuZz@HeT25}g@P#Z-vci=j>nvbUE9|OrqZ{Erz_t+mK@(0@aOyIdtn7V?MO))(PB>2 zs<#SetT(e09ggte+z&8PqokAm+RjZL;vtJVSoPYUo{$ki9P&JInqC@;=js; z{#C{ES0Va$F6%!Kjepe-{lkiW|MO@6@0I@Cx&L2wsQ@&_Jw2kj<>0(M(+m>H!35cO z{d=ch#2ei9O)7DXs6Z(fs38MIUm#JtL+P=ig2Km8Fy-~4i&#iM5L4Fyrn8FPS8e&9 zKLV+f5k2App2v2T#`ks}g9)4&ft2pcDK6ch06|7kG5Zrh(G#F5A@C+Hv5L&B>pULj zK44<(%g0`u)uWg{I|hTG>9fGIaD$?dUvY-s8J{kWS}%FPcdy4Qj0p$g4BR_6e6QGH zf%5K-~SeTkWic8?|%zEc+E08HkO!- zEV$R~5x6U^KWe?Sl^632Am^QM0Hoz7f~6{Z({UHa+}^$dvlxOdTUnp(l5_vSydQ=X z$Z*);OM?;tiN~NRTkH7Et5^QdU-uY6c?l5_NO-#TzVhe68)t?q&+*8WuUG|t{YA=e zzW`>b4z+llg8RnC#E>aM_4Us~B0he+QE7S4q;7hP*LtEzx3(AD`9oV<+aG?|KIqk4 z!U0e?^SIzqeo@icPd|K6>h#80tS$9xApX(T^eHGK@;RDznV+8rbvE6skUFCw*AVz~ z+nJ`kyu6WOUFnxEt;Wj!*oY3e3PxOSj#n-VZZ}Ej=6F}W9K=Z^$Q>Lne8XyBGA6Q`}%}elH%1ON8y=dKPi<=bIH>ASpf>(m>)98RICA*(5 zEU+7#H{}M43|nV5_H@WY9!suGFyLIgZ=pN18hNBFZTh# zp!xmHy{gxQegVDpP@2aM*jvCRm?MdRp zXC)UfL_=~>8{?IrcFiP?Df!ln>*3aPqkg0Fg1nrZyE{7isk{&+$knc;&q@bJ>g-n_ z)lhJXfYaJkedWg-B5py0(m389@8-Yf6XYtT$R+843CrGSP-fS1G~)qAumg$=regQ@ z^zbxdz3v2;{sqsVxz$!>KIptLmZ6l)@dm3MqdiRGSi>?*GJv3zmzSgP^|&1a?o!AZ zl{(humz3CmMaEQDE#kgdD7do*6-nyqgqsIfbH#wf6ONF`4xR+*hve!ThzeVW zROJrZ7#J8R;Up#|%F$&tyB}~iQi3epDuywjFDQ0rJ@ zzL=P_O7vt+QKUGf?^Yi6H8lyznNQZRP^4;y(1!qDiZ?i7^78UMthQq>uy>s|Cy-mE zl**)vOlFiTQ^~xQU`GlGa+TARf)6%WLEEYTBo{eI=%-=QEJ`PEwh-rlzgo%qU2qjW zy+x}%n+@u!`Y%cS^4^tsC=;qW%|(h{$OMD2#9WPzzXDr!X`4;s5{EyRKQ>(1dF$-P z5<-2;*w{F6RJ$8!65rUAl%5tikeW`qG|z!*OD!ikJ-sO~c*4fE& z8ju#Zi7Imxe(?HIPf}0G%XlCw;z~+){dWsXThL0_l-6^gOd1zxHaPnL^bClkX;0#C zu`VB4z~j&gD8JYnRzh7}U83ixK~O>KGWRVQA@3W@qHE&(ngx$yi$RWqgQJQ;DNQ6L zVhI%!ar_`r^VNeY8hi=?7;}}^=^-eP)l4g9gI0X3=_KPap=SV^LS9D36;zbT7@ne5 z{ao02uGP_#0Q2_NQG)DfsSZ?PwD+b6j#nx#$Rm+RP`b*|`0`f>WF88EaCjc?0O2`K zahZACDvql{rCFkv5FbBQW34NYIQUu^xTCUxg84ri;s5L3ym~TOYb&ZO4OYgc_52Wk z6t}Mdcx%CdcgV!+!J%I;2;-ae{Ok>eyZ2;V!upm^&$oR_y-NkMyW=@BK}{jwjO%wX11WUc#ojkinqXbOnm7|afWS%CNVUW2*eIkG^@4uKbw zg_7tyLY$p>e?g-xpAJhf|G%&1eLdTz#N%yMt1Bkd{$PqiDT6tTD__di%L#bfPxjIImIFQy_>h9 zs9EKztS4jbKSxK8X@SZ`jb$*7{;j>Xipox*;dzL+b5|_ORXT%;x3?E(90wKOPRkkH zBblzXrTW|#5`uG^6$kT3_c;I1M}bM}xl>@@%1f{6JD%(<+BWTe2g}}3SkkZ-OqUuI z5~7-#r(S7>gpYy)y9CChU!d247|N#R^bR+!pbysBTF>?XBsm7w--w~UzCI1K`<@2J zW863nx!r^skO$6YB=gf}&z8nb(~K`MhrxRv97m8sFu9q!o|}^+C3|NM4r)M+_Mn=Y z&;F47z2V~De@HShGB!A_ibL^*QJ*^IUkAIBJUtS;Jlki8BHWts*tSN(Z=U-A8)x$N zZJtlD$I+I;q)qdIX0$9XxRqCq-lw6uJcE~mgM+1Yb2v5WfmGhdIY~A~`Xx(!X{84D z&W~4+nd6tGO?xJr?cHV>nLh#X)IIpybn((25pOP{Xc%nLfCB_Lb`5 z#VUX?CJf!TJ7tDHNBOJH0WABqg^{%&Xazu|p}|VrjD?SKz_fq@WME!xjZpXw1}B!&8A*cS4$831A1-yhm$>8Xmu<~c(kglx^I0bJI) z7w2V9X&4zpp2;gINMnUXOH8rYupE#NZR#G= z(7ZvyB~h0|T6z!_mGo38!gjoeyx+cm7r8i|Co%No*#81xMEPBnEZCZz*u|I4o*bG~ev)Nw*Iuqm0Ia1#O zK-@;5i2|6#7>PM9u(kjzL{CriX3Zpuud(%b4)+koKxPX}3xM*42|qV$XLD`?v?$W^ zRI~GP=BcLlo`I9QpB@Tew%lOw6vcf4-e=DnSL6`WLWK`74@13iYYy&lxba&5=PUq| zvG8}00eRli0qGVv+;?>xoD#N0mmGli831q)Okk1c2bL2kIcd2p=2$q@y7$75s14yg zfd56(B4gXC)K-OI2??^ka>16PB^czof_m5uF zx16NDxV&f8t=Rw;wQ)W|u%u}Z(Lw3ay+V;*Y0~rL4XyFhhZrLcU|^9quIHB{OWJRTbe}SWfk@nQ3xE(Wr~t|E#Lw&J@7$-2q$Pa(+0eo(T^1 z#^I!`&o6%x^xgM6`P ziVf(sLTHfDn>Y2$yB$ihe%4{3BY)yL0f|M1%SPph+kq`qI>W~j$foz zTTj*i<9qoBHWli`vq$sYE8}=E^NAO^gZ%I2zG2LL1@MF+jKvhb`R=#ZVsb zJC5#?Bu+@uJb41F3(=vd8x6&?v8zvjFBUWwfyv;F0-n377d?=l|28;0D2su04H{pt ztK1_k6GEg+=%8n0VzLCBDCfDN=fS;=RBj7Vfc7y$fo~FR`2*k&IOhlY5a4g}+UukP zFfs&#xvFqnV-PIv1~VryGvdQ(r+m#?OZhln(1U!2uZ#8&7;-n^yfO zKG3!v;RCh+&{{lNyR<}0Rg?W}QE3jX0d!Z@?BS~tnc_cS3~E18R)$g|$n}T?T(+hR z&)2ku+HN#;J{=BTvGv}6Sn-Ti{w=E$5_B{VKIwCH1h{8FwZ&HszG91NT)G z$qD5~Wlc@uIOFk_m>)LeFw~#{h8KGz1rXpUf^GqRySk3c*0du^1X$&pddHlp3<+$3Efqi6eGyc@1a~=l zJXmD~#=%_|RM96dc#`aUvy?&N-be7kbz3#8$k#zZRnQAh;`_#F)kR^0bZUr2z3YqxiXuprsF(1Imaq@ zKOO56#VHOBPF^_N*n0fIF+1o3^Z?Pknv<0093J%6q0vk+oQb1CgjJ4eTTTSuSon0K zQcLgMO^sx)?9Vm=_j7K=!C?KPz`!36c=li5Q#D1Q11E$rn^Dq~x8vnwCF`lYP>^$SFV7kQ$#;`gSR?6YCD?i&en99+xi4VF^qSK4BcLr`PQQa;Ra-10IO%p2pg+mZ1h! z4GQ@5x6bzamR*y*PB&}w5retPriRlte2tA6_2svE-}#afQoI<4WEO3pGR&5`6EOV4 zx^~*R^JyH0Er6pEyVThU$&@M&@IN$5jJ}J961Bcu^MC=21qvktWx4x8qXU}`CoF-x zf-5DZ+G;$yNTCyF1@1^W3UK%-5Pv}2Y4kHp!ea~bF;05GYTZHDV2~TtrCz{eIbxO* z92}f6PjTwA(w|Y44*1**y;&7s&q1b$TagcK(zs-E(#Wvd%kdO<**gbhVixOW&A?x- zlkyxKmVzOm;FyOgG{f%tU}!TQ-_(nijI1Jn%b#t5kV3X(l^N3aBA0AjDZ>Q`+;RQLzz6xX_(D5ZJec=^ z(twEiN2jJL30_-g*xC7tKS(KdImpTn$di+?`8&X7OfFmZ5KP>Pvn^gp^wB*V@U70- zky*)A8cLR1?}9-XBqCTomKj13X;En2yx)w@ zuscPM0=CtvbMdlzlx*&S-;jDw2oV#vyYmQXfbSrS;bu{5sFZ_;9-p1DFwn7#zylX> z*75!Oz+1=U0%iIY5=ys)`_Yd$1-csOt+9p#$~&C$rUwfX1AcB=f03#dV=XYxe%P#% z@2$vD+ySso6pt4Me?YfsIqwayoYw~j089<3d?6}7dHv`Bz5)hiND&0*sV~D-rw=cI z>y933L6SWc%G$63yjH<`5|{m2tin^PXTF+dUO8D#Vo#?UUD!$+D!T-MKjH1qDvkae z$y0_dcy>RrOowrS6(;05pOLu_7+9A*@E>8&8t~yI$;!%Zyv+jkwMp(Kv;R!GrwG;N zwig=p`Y8X6k&!kU9GGrq4ne`BY?V{xdYDX72&bUs;!1Kvuj3H@*~<2Y8Ia8YZKmR? z>qDIVsWQPs{jvfiz&l;njo%e>MlLbQb$WlX-WLHRBO$u{iN*7aezSXfu<9F8$=-t(E9>mARUixY(Q=j+nD_W4*CHOuNA`e}E~u4rlaR82^$#)X)fF>)-aDJa?c3{of@+l&EAMm}W zRfW9%L2`^FF=w^Korf`g{&R3R{lJG{X8m!>l{7$BKipwK6~d|zYbQY_?(PCa8V>c( z2+f$_I0o*k*+7}xJ+IIyUOkJd;agy#Lwr=*dd^ZUzCcG9xKA#U+{KaM2mb$w+w6z6{b=rX(EC%%!0oA* z#!Hx3kCh$CQ`^UZj~$Y@%~c1B2?atQbLASH9hCa;tU;-sSdbXk7GDBulktYI!K7rG^qz_i~S${`uK0tgurFaFnmnCGgyg*zMpx@MGAIGU*cKzOB zMla|DAIV=U3iei4n2QkHN+-ByQ99JOv6`XipTfDWUj_P9CI)sVPG;bH~yHfIBvxR_p|Ubw#~* zw_dv9_BK+jYst<`WD8=5D=avCIX~H|bxPwDSQa3zxDL31vXdJ?R3P4+m!+mw&%`CR zC$RJL1vr92$=k~=Zc*kV+`ccb02N z=*?^!Kqx9ER~9r=#i5L(V_^wb`bg!m!C`^u-FKsI0$ zlE%5o>u&HR{1On6jZ|OGHd(cFAsPX*{`8;Mh5daeByS7s!%GRD$S%U=<5-6cKfP`4 zJ)kqj^XILkPN5zu6NV}69!tHS&Q#|!syctD-~776Fk}GZE7B#g<_A%2c(dQATG1Da z8*T`QvZUnAlm$TLF2KQ}MjCCJz90cf47nyVKUo_v)XI@Y)J&z39{RLX_S5>OtaPiA za%iuMi~cIA>%Yb8+YRZ0HPcG zAqnMQKY_iO?|0M`Sg*?rmzsw=4xqkV$Jg^Z7zWW}auWTrS0CJ^RMq6pK>Syn`2~o0 zr)&)68h{>{!=*mwmIhx182Y8M`Pa(12q(-&S5(kP$knrWjyDEA=T3)32Y|j#OiBvRMZxJ~bEU&Ulp)f9=8BN0MqQW99Q8NNVBe)|!_W6in6lP&e(c=7LqOA=?FGXM!+xCg-zD z;yO^x!&+xR!|>WeYV3!nJ&zxhjd~KS=6#n84u8m}x*~G?^Lmf4dk<;BhAZszX9AeU+tbJVC&XxBX|DJBe}WHKbMG>Ctu?uQmDRD)&;zf|TglbqXd9xc9 z7Dh{uN@!4>DsClcvKN;%P6ux}*>8Ba=p|9Y3Bu>#rP97b`XW;|-~RQtgnj zd-Kbx%O}CMx34wOH#X&O)yjz6qoA1XSJ6J96t?~@7LnykjWN~SS3Lx>fqf9xhk1WE z--GH`{Z>0orhTdVXUcEZDlbm@e8}rpK%jf|N`~7v7>QVbtpyInaAV$r&rqzmwDBue zx!1WqHch)yi`KUm-k@S0)HgBh1D|hhWdJqbY<{`dR{e6KS<|)V&<9m@AXh1s77)&W zU0?<30B$okHSK#4`G{XNAzrv7eO3M1hL|v3h>q)o@sECPHuq;qb912QU}~{WwVW`? zy54Msu{<#1Ai5~^WjSbtIjP3FrheeA)1-Cns;zUS>0$Ke&z&DSiBW!9Wxw36i|{Dk ze^q)_L)1_IQigNvUTETD0x}TaH6_uPZIggjno`$&e0{z52|OyLS@6V>8iXQ_+%Av> z2PCo}-{t9LL2l1#s>jwUG83CZ7_?8uSOpb+slnirv(Joyz2uG( z{e4xt`^|6L)snjSn?9h6cr^Og$Dv?H0wd(9lw&8biIA*ZS!zve2+%TIKs?HNAXfSQ zLm^l8YOJbdP9{qg`l1aufW7K9*i@eVt9N#WI6Ut87wHzbcvvpL6UE}kQ)Vk0%rZ%I z(sJrR8C2N1(6m|*_T5LX-W=_2GB{}V3PjMmP~fe=>6DURkx0QfQb1CHR9AL*t5&H& z74K>qkWG42pd=Y4-GN%56#p1Ue_}e->kvADm1w4a!x0G>r2e~hqN)_f&54*^VGP<; zyE;arP}S3~Rd4~KN8-|#An2rzQ3rC|kY&zStOtDb6AWeu$xvbjhN@Ba53~*{4}jjX zwwDX}E{VyGG6Y^R=j_^RDNNY4M{Y5-XP6 z?I+UPp?7Df4ivxdz8g)6Gndiws2wZnC&c`|s`WkcvIBp{7#&@5H68BqZz=>+tC3zS z?9*`qjVW>pEol<>D#sI2HZGP-Q67qcXxujyGw87J^u%cq}utKPQ?rz@!mQ%3K316&bU`naAvWpJa-{xvrzXOU{#po*XA28WQ?cD<{r zxxU^A{=?BN%Q3K2HHtQ%8p?nMV|8`))AN#qf?DKQO7y|PilOAZNDS3n2CG<_A^J6mzNh# zO`yqQ75^H2(CPRKL&x17r=TLjYE0p%l~NA8i(sI3(8gXv!87QuCLUW|UCpAwGM7{b z66$2&+A|!9p5`hVcM>hGtYk2=%#{E_eNopJ7TCGD+nSq8CPA4&L`3YuD%K$U_PrDB zOQC#YoPhyopKu&n4p*;hH#B}eJ^~z^83Pdc2-NBjk&!!Y8@DNUdICZGud#5q2zwiw zg|`D01ATo$){b8n7lD`CiN)D-qu;G6#y;H(urygvpw-`2a4LaFV|seFw%qGX$UENa z$D6ArAT`?2@v&?B()22^z1zDn`eXJd8w(4?tCb@9W4nTNd9HxX2=&iyL8b3#Xhx={ zn)5&etQ}tdVpR~mBHx)8@o2GvhVgV+(CPqyOk@M&=pJD+113bBo${{{4h{eE!t{W*UE4}RHRH~ z04uMuu(SKZyqQCh3UWE|q`|1Ds+s@|cI9xhvZDE``MSEQO_!7Qkg0mtgnwaHRPc6l zO9eHAhS2h*!NHK5ww|7ziVErJ+O+cb#Gsy*N03oPQyKb64QNl*5=VC_sfa1;)tB5{ zBEo=|4qLI^xQ20J|JJWBj2CAC=mYFUS;uFGK^>)82H8GcFT16Zy_aP6Xk8aO5>r2! zrhbX+-)dCi^w>$qfkwxgn>iOv#)zNG71eN21U7pWHim*L&~wr;kiszEsauxO>uPD$ zH}Te1iOp}6?Jz~A*ueQv>iZ?<`rF%2uZHTHGkCeU?E3TXPEAc69$MD%o&|F8^Xr3J zr@sEi#zt&LI%wkWjJW*ZJr`I0(Esfe*Z;oVH_9{YR6>_(pq;6+ixbeu77lc0Z)^p} z%1**e0=gk#5(TcKIITiUn)ok*C( zZH!!iqCgXSQy|FE-pN+g(*a1r0>>l^v@>_HAYo%=1HduK8`%O$m^2kwLHEi)XM0yC z6QDE6Yv5uYE)ps(pfG;_GGhLH5fOR~!Ndsk&Cmb(0flVm^4rUdfti7sgq0hV8s*DnJ(!9VSIlaS|p~poa?yla#HIIZ*h|tH_^MDH6TkPpG&WySygycXHvFekT}| zHe)NGiOZ|OpQYrHoujQ|PuPO$w`ouxryS6*^*q-?#T<#{kobl6;oGhdcPc$KkGgR= zOuWG+DK}fTmfp5#pv{z%IT!J0M%?aP4YVIJ)~|E!n&J72apUG}Jp|W~Qq`v`zvu0_ zs?$^KtnJJTtfl#*$)o6?ZCkpB8Mr33n8M2HDe?Zj8uyM3N%Ex`{ZXn^@ahyufi}#1 zr5){t>Ec!A?JdRxJCYRGqwF@HY@|plapPjaP-`D$rwR{M%Ku zvA<*sH#h9(8?Jri=S&cPY~-oX5 zVNsZdPhEiHo;QL=H!4y@a2%p4GN-s%+Sp#{rPOGKtk*xjVwc5c4>wa|oxuz#8Xv1b zmINJ|EI~I+V)9ZUx`?{?3cGf2Nrk=%GN1i;-n+8JEZZF#Gx+6k7&z7>K};ut)5=0v z=P|$;=CTG}v?AgOE0#%G4){EG_5mZz8%;6_1M!E~2xBTNMhGkhF$r2)G$hY95Zv`0 zO+F(fc%&F)BpQSmZa^Y#V)h$pqWEnG1#qK$gy;|2poKMI^3`L@8mvTf7RisQh?Mp{ zMC&C5bBL6ZVarm;BV+1{#4+$w1*sIr$$3~|CbpQE45#dyo=Nl)A)(SW7z#RCF|Cbt zndHZ@m)<&C!0v8)d3xWRv@w|Hfo7#T9=C3eTz)-^$q0x2d>ri9`|+#aUc+C{{&d6N z{pQ+ze;-?b5v80oMRUjNjT;;4rzwx?;|_O?+q<8`=RYf>G7dMUboYDPEHEurQ(dC3IFhx z#+>Zs{%HPTvE{`RVJG9q6HMqfu@F2t$NtRl!O|W>nZ3eoP{>~3;p%$}R`IGM*glGy zRy2Zin^YvTD@naJ=JyS@Y7m|JL&gKm?;zF}urT`o>MepJ%;x7YI_JVlkF}1$FZa#m35Js zQUf=oW4e_Xs#YstIr;FaGz|PN9~O^&tHQN@SvU=^^rV;0qdOvvPx2X%!zPkUTyp)g zCir~slpJS^3s=cRtPA=p$HM`R?jtH;+_^) zpp)u14n512{M4JE(``;rmpBG4f!!vR7|L6ocf;S*}2@Az)Ee|@c7DJnMS zP>RJ(xuJoaF0fNP@1|!*wg1n!O7LZay}Jd)PbfV zm`PA14!-Hgn+ikKFi1>x`a)}=5X1~3wjVUwLsppFW5=;{Ka7W|I1sf&mlEOfQo*S! zcCR%lOWP^RQH)UuWoW}gjfg6f7c?m`CB}AQ-<^<^+s^E1Vq(g~n7>}+VoDb^Q_6UX z%03vQ>e0~_jj+dIwnQ(|RwOSGYOD;Z>CnLE({Cfx(bkA>NT^O7v^t3oVn%(m#B%(O zi>cAi9J6SHNvEd!E-o%-)7^0}_=Y_y1`+Rs_|lnPyf#6VB5ErFX0^j4$oHwH5ho*S&v@&p$ZDKQRCe&s<8vcni%v$De`V17n*hHy%-BcCiOH}^j<(zLP3!-0d4C52QJudy4^(q@K-$kZijh+j zxj+`L7>YsTlXUA`Jxi>QJYS={!H?>RlgEUIsf_XvXqcRh9kR zEmCpIZXA>XRWY#zRV;d{p-7$7KZ};-);N1bP4~QJgf)tVc0*ok(74JSm-sCop^}|s`c&d{WGZ}8^ ztZjB`;7&G2BpGh(TlivLWy;xb)q^m&%7+PVHwa}J4#_tHsbX~1R1+h3f?)uPskrvPo? z$SHyI!?T|K`dmG|U!r+b%+Jy9Cbb&ZDHF67%p40tK2H?xfSJ#hM;G$US}m!YV%cIN ze`W6ynBBIzlF6}1ILvw?_L<4JLN0T%7Vn9jUm)SU8D6tGFc6M5f=5mw; zYu`BJIQxhgEo-a0(Xql=D`UFdQ`Zg)>YG=%i8?$V4)_6RMJ6;#yzGi`RoAaH3eHa9 zN7w)$@nV{u|exLScBgjf@R{vSy2OWYaWn#r22 za_T_(Y%=@CO9+_`w2yV)Ab2{@Bi|0i;G*fyYId5C^%in1J0`vbb#{!f>K(F*jxw6- zBsG`sbIU1ei6J+2jsg2VPA5*<>lGmH{w+LuML?{37&E-ffJ#L)-K|0`lA~du!Y>AsB-lKW5RFy1n$Kx-~#O*`z^| z&;_gRPh!`&pKM6IJahtBe!_H~X|;;bzy-4T6AAa;kT(<042GkC3x~r598(8(MGP+y zV9KJ0JR6^7Cqq)BWdx z{Rb4RPmHeKW-f)KL9kezSYL*pg$e295oE`LPTdVwZA-s=DYztBY=e9s@3r|(Z{5)t z%f$Naki!|0aR-Ltk0p9R=qyV1HpBj~fFowZQ2;%lr$wY8j_369^n%ZwWfu{DVh%qni0NS9;RJoZo`3oNY4aakIJwwKIJrQU+^>bq$;R@hJuCMs zl72n7I9W*m>|7)OE&vH9Guv-BPBzd}EX*W;-}c;Guh0I+gOi076dpSX2RrxgD<=yF z$mZ1rgmnS0IpF|s|Gwh}fI?$q1BJ-)r$0OBj)V0zZ?9pz#>w?(-Z?-B|2O3Qvl4$7 z@n4lF2VQOwxE;uF;BL_*KrMZO*2@C6M)x1{At0x018waSOY>dpE|50Usj0P~U zFoU9GUN9f^@#0k?){Ga+ROD*_4>+6H?@G$2jHD3&*n+~cyS}jfYi+JD6o7tG7xFJ7DWy&mW3=2;a*A%to2tW*nw(1j z%69os_eMiKxpCMh+**Obtl~;@bCvaZ|KEYo9O`lA>U@8eqxA3Z-ecIa;+t2HsB0F_3K*(lPNpY*d@B+05T`xd?ARNtlM`Fp6!_p`{Ol2z~QM@Sb+_6X8fHzrEkwyjrG za>2z#;h2)cX71KCa{`+@TYqlcAmzS(%oUGZEKe zgeqiNR?!I73NezgQk2lDrJ0ixxK*WL&&iR?Ytpc9Ml=?;iaO{pmAjBe&uh!C)9g$# zITaZjknj_Ay`@XlVDogOqAVBIVZ@HtxA7!jwXVm4YQKy}{Ckr@0@gPlDl`#s=)zqu z-g3a(6KfL=?_FO*ccch0C40Q~e%NXND@3=DBV~~&qnh-ha%I`K!hRmfNit5lb4q*(PXGDtt`z*+leJ~plWp-!gTdPc=(O5A9@ppN zBcq#o__!{%vE*)#b(#%T zL&$RM(TTt+j*o{&lS5R;=lB_6@QGuI6-jS2f`<-tF*p{2#*Mmn8O!Atuywr;GtBi_ zwbzIryR@A`8(Z?;c{UmNz8~iTt-NMwVSruNqTJ<= zR9*F&@wvI?rZ$^+I=J1Y$|S44mZ^YTsae!HX7~{>Dxq+7)YPNnzM6Jq@v&pjJSGeI zWK;fgcO^xk?5L?zew59UF?mFG8P_Cc1QAR--gec&WI_e){@TLGQ?Mp`1`~!vHI7uM zDlP=HZWs5P^)<6Np^QRSl?#d&xH#h)YCHw*BM%&|S|({inr^q{He2 zT-q-Of!7_!#CnZ(?!M3Y?UZq+AsjN;zwUr*&o+#3GQ*nJbWf~c5hFf!qux6#p=jBq*mH^5rE#x`sBV$E{*`ZQ zyF{_R>-NqxTBeDng4|P5gIs=^Lr<*F;TVSad1kzc7UE5v7-PymoXJd0=)8+yNSfj0lU9cx0TyZz}2_gsp(I72fyHTd*VRe-|YW-zmnZ3+$38Z27yG0foF0Iw% zwYE1rO`R5?;h{iW-E2B;bqqk&muWFyvf=%4NjN#0;j&rg<8Lax7kToCKq3fQuEY!@ zk*ID&qF6%;Vn;P)L~(JIha`JVJ0{p_5nQ`lJs;DWkAUC8fIK)*sQjCH(~rBMx+T$x zH0)yV!i0T@^NJWSh^#_d&!h1oF#YeeieyS!E&6zB=6HaLDL%Wg0WY!^;$vP*RlNnS zoovVld8-MlMqKr=UKsebP=E7AsZsX$VJ-IK&!P3GaDeiJ5>1El!;0Yre5XZOl@HL% zC@_*>ek~8Y=|%Evm^0w&kWk8Cc?GGNW3q2$8C9K=m!!Jn?UAmhg1BvO>qM%@mkXM; zl>2hOMB(3l*qo7LgS)EPg+HVMb8S#wl~XUY6C|teIT+G3%Cj+-#GNGGp$gtb-_m8I zl1vf-&7yyIGRWpuYDkk1Tapm=#WrAW5~_^zVdJltAyAs^H|X2c8yJd(TV)e(9b{rbvi-%4j;_FXZ!L6XENK>U` z%p1j>yBcwK{qWV4x{pk$tf-^~*ruKsLk;Tt);2-eGjY+RfpbVfG z6vqW}Y9(A^K<|Et9Mv&`Rko)LkmNzyz{{nAm`qqh9hq|%@5gb=ipH7xG5J8-zrwsn zs$P`LB36?5qv-rXkw^F!VGhl7A@ZoquG*DUTcTZxdwLb=i{O}Hf6ysL)29mY*{__X z33)wxsofS*Y2B@RL#|))^}mN+Z6k@&hU#NT%euXjrYuVX1Nf&1(DJVSbYUJ`TFae} zTFgI4QCqB3uSq_UGw$7+7VELEoxQCwC_R1p3`S)ltKENxXP`hI>s3?J1bJu{o)A73 zy7HrhG=lUE9JkD}8lHbpZL5!ge3%GHuIRk{lf-^NPM=|qMp~{N^+{=3_o^Fsg4;Ba zs1DZz(Vh2v?&m`nW;I7*K0PkpH-KnrWzq*{jKi3+TGTqlw$H}!nfNYas;oT1Dsmzs zs`QW#5Yb{G`8$dMmm;No;lFUXFrd>ItG{X(;;aq2-50>+t(G)m4DMM~d45>^wW38WsZW0EN3K!&5xkVMkfyLZ0Rg&k0 zoM8}Ne*>fDU=NfU`5yjLAt(L2^NjHdXM1>RxZ20>J@?=5b1*S|Iy}sMb*?jKb=;pa zFsn)-`aL@!eT&|%3RC;dp-9F!XBo4@dvc4{v@G-O{UnFI<`3?}VT%aR>}Rx7Bot}S zkDSUDNrRP;-C4}41|Vvibs3E}?csIqB}@!V6v?`mP|s&&@X>I@<4D4_Lgan`#AoNF z=Ov#`-R^o{CS9sTsc2BDQ>eAl@nLSLjp8d;s${7o@fe)=lwXDf7J<75l=kXmtNviT z!SEhW9Qz8)WwrmYu$F$9)2I;E@5b_G3jaoxl6sf&$HAWO_&&c5F#?j6X4k398pD|W zs;+~_*Q$?6FVz`9?39Ug|1HJU_qOBKQ6)WHmy;$pbERH4RdRB+96#6tP`KZ+jLP;P zMt|D@)*NQC>n|C0+Rd$ajLA~zaVkcxQPfnxHGr}0%ySA)c=DK&KzNQ9Wy-aejOl`l z*CU13<>ITAk_WF0$>FM*1CW%4`Y?pqjZFouzO9wD!+c!B`^lNOn~@&tlCmhphv1;X zi}!L3@Ub$~?+Eam<8tw%AZ*9a1%;HcL&PGhATpX>r6W0XK|R$7oa>eBeS$AOdmcJ_C6K%JaZ^8;k&FBPV`y8fn zaSflIyaxF96-7gRh{_@kpPWwp6}YSvZI1|I^$it%CBHzkJ1MKw?b0c>;O1GImo{n}C*_vNXbK zqgx2KU6Xu&V&8qki3THDIblZDIrmAOeEh^Ja3hT>zN7Af^K%7TMW(ih%miEk<97|O zWx^&Rct>Uzv`~IgQQ3#|2jtw1m!G_k3Q^Y zssebiQQV`ozlJq=6z8aB=5bn`w~p9^J8PSWToQ4oEGmMAlDaHiE0(5)nksq`9egcmcbU5!TU7w z7FFmq>ASpxnDHH}=?pU&>4P3y+0D?dNT90|Pl0wW=4Q64fJ# zJdt8I*7cGRmr%nc%hFk@Qk)>hTa&u1+n|;gjbrR?%@>)NvZ^)f!AeqLqxW`?w$&CV z^jPHcik4yPmbjKLiO?!9^}$J#ILLz%#5fqHDXfJchTkLObD1t|S3#%KItQ1ABmKxm8pc zcj_U~Rzj4yC_K%=Z6VYa8_pFnseRxdXp_CDCoV8YSo?1 z*>*^4Ne=dS4*kqtYMMBixXX~5NG>rlzAoxBXSJR)&uQfCy>uGWL`*n%ZiGO{oBy!nO!Z8Sob4h5o@+B)Kd&=!nae&QDjq zYccb-fV=zE)JwiDa=YSE2Y&QJT!*ogM3#@*@45<|E8cO{nj)DS3v7h8Sh!r$v zMKC|MD9x{7$jz}B2MV+x4fGAptTxj%krG*ObT@6^AtW5<^y-ZWNKqRbNx#Wi#sL#W zXa8j5V#PV5wj)jiR-E|RX#ig>0a00my(X(_AFrQ#`>2h|`=>2*S4_fFq#ZOb_W^8y zjy^?d$CcD)W`ir+j;f8{nTcz<;_HSnL(#yLg98>IA8o`ROB{% zIveG3>y(?lMrHp&Tefm~+NsG1O47_>%Buy*K)s7l0gOC0R-^RzFqc>tK2~no;eN_F zCTc%=G=x0(kosvEz+4{_|I1TUVL*w!*TbRQ=YgVQ2A~X`d&GAgTZ0A_+dMVCCO1qv zG>CnEo?4IFdX1$YE*|ps-1pd5A$`F#9Wey*GVx(UVV)Lz7c(9~VM8wl=Q9v5!C}AB zVE2U3#?A#sWWUQwOwXQpY2*g3_>$m{fRO&f`ycwbVf4;uS^a*~#0&6OXtFFa2}Y@h zYE_BUw4x^A?1>Yw)S3x0IOLJy>{3gU18B(dqvb;x)mU-6dqgWAJyeN5N%xGJoNGeY z`Nfk1FJ+rrQmye!&tzDWV@<&{WrQlbdK4zqb^Ye|J(xP$vs} zEo+qg3c4-n(8tpn2O{8XSpW~Q0vxfH)0B^R~AV4t;(E`I<>Ft{Em43NRC>1z#7CFE9}x1Fx7pH{oU9=oSHa4{+FhmVS{mh zzpJi}l4er;BcZ)>-r{WE!$Xa!+^)o2{j5acx61HM^M2j(cEW%{1BkLlRJI+)ZI1LtZQ+i9(6fn6ig{d)f3265TYivR#{QdV1O< zrlOe8?88C5&lnKju}3pABf(OZUOm+0$zd!}m@!{=bQ_%?OgcQ=U)h(lvj_aL6R$YL z(f)9o=K3}6^J81j*Oxdzq_b|J&LL+z;q6iEGuKb*!_gUOzYZ;gDi~X-@L!g9YbmWS zTe$P~HHi_(&p9ae!>(4twNavpckzwE6ortEcUcz->Ib55OHj@Y(fg>&>^3avj2Ole z>jR>DX?@JgyDKrLS~zK=;R zdwX+2HMwiz^C>VpR6AlOfdbzfwyhVpO=>tngiD;!bw+o8#cg?FeW3-S=innw^|PlAtE)R!2d8`_{KQ?o^n7VDo`Ma@wJ zxwM_{EcK4g$fRx`Qb4A8n;-i$PmJjqN5eh1hGKsdD}LGGQ%+xYB&Ffq7PS8LH+ub z-YSJX#p13>EAjU}v3BVTDes~!Q(sL8MZqsS%Q`XcyJYQy`rj>t_)>U`h+(qBAW;c+DCr#FD6P}lX|pN#c~lq+8W(vwxr?yc`R@%mGt#I4dj z`g>Pf5oq^lo+3l$o0y#N(jiPYlb86u0`H+BP=k6A$5`+*o4Vn%^qpHel{&8SdoRqL zQuj76&eJ+C9gF*$#J0K!zt60mN5SQ7se<-tz3d$0R&nGx@f4YdDiKyhdeMMFUgKBu zjD^{k<1dKs8zmpovjY=$!(Cz~c95Kc_Yzx*saO@bV|))y?GAXdWCvO;%oIr3+~PgI zQMqR0d?uwbJVg%UroF) zkw`@m-3h~2Vfk~)fw8m%l`()Rb8M3VE!pMr^D?nWcdsWeVArb%73w(TL`7AarP=pj z6BzF35of|OWN$3zk3dOHh}h7)ioLd_qr|H~ZP+0pX^o}G? zMvRA%D*AMxTKKN%>>VAHmRA0A>Iba1PwU76=6wokX8`yonr|f^{4e{FvF@xf*&nwa zsZX~z;aioUHr0ew;CUvzo_&)<&uS{c7p(zm*~()XxCX`J^}nrw}w;XX;j`(%I_ zz})Hf?$O^FAl+5LF~YW5g!Rw|!Q1zig7fzc3ur;_Pb2Q^r}ecNot@|DJ(+;{yY;=G zoF#lfhVmi#W`mi{oii{E!<7ShYTow_Y*=IE?i|vC2@emdlkOWKm_0lf)i(w(38>C@ z`9i(;;IYg3-Cw7Vp&$+XgWf`Q`gzd$KO=f{$?`l=3kdLW%piq9%;h~fNcLK*IVkp4 zf*(N=pw#uC=-2mlclmYKqTGC0z1J65Slt5ZUX3%C*&T`4qLoxxp;|>ddgmKI_D7evdueg8A85r~0XlYuPHq+;y4E&{+*~lRlpooc6qU zjCQ$k_A|gWBnHc)1S)sq4BB72H}qQNJ``XmlF|4p&ac~#whCF&|Eiw{%sTIAbP3L* zf|@VFnBNn-O3QZJnxJkK5_4He31+XX+)1G7Q(q69X4~f>4LD7spgr(0$YeN}MuB=8 zTz}!-Pr$lH6zL&u&j?~7>^jk2V|aSjeqll^7>_Eb?=8I_8JKIKBB3TQ&%!?lcx>H* ze_y`1%dS6*|FAOtO{kLwjaf#p;6nzu8BF>HGMx4|Fg96CX0cvVuq3J@A7yyig)R44 z$0v6!W}#k>Y)>DeXy?zGR(mEUM$r&I@o@)F(c|p{{ZW}8^y!EhzUk9_B{#&_z@;?Y zg`p$Hg(62Izk{Jei2Gm!9;)6KPm4qU&F)Lk<~q2ieXrg}5)#As4*~A0_kwS~a+4%( zT84o=liz(1^rpCq```!e1a)ZL3gilEylYs5Ic#e` zWCku(E-n}@)xnC$&im>ak)WSi@f(xV^Y8^anK|y4BAJBkB@doU9VCm$dQVgvlxR;> zlBnO%;|XB(ljp79Dv-A7kid#b(gqK}ND3lS!8kMt`Vb9liDiFS77iDZ8w4$7npU{e z`>vOiMEV0EaUFbJK~*zf9b?dla2-rIv&1C~n3>Ni0U{MxJ0?3AEq@7|A^{0@U@(C4 z>J7Ca6O1S+o%#C@+5s^6LfWlXgmN$bM&vK+@OKI=11}Kn5e^LhMgsUfG5&uu^~uh{ z`M0S~J(Y309cF~iV>*`~L~y*UK#mWnhO=F}GjB6*AZ#M!5p7IG9FguHBQx`%yy`gc z5JyqvG~yopAGxaAz2q7M&MJoAg%0fBC-NN!#Hpy_f9&5~TC3@NE;mq32|`}F94qxD zFkf_S_Zkn|=g_-e-(*I9lCZ3j2xcTVO!&oJ^VsAJ)9WX6(%6)x4Qc-!F^K;0tlq=N0+yx$2{AGoD zQ7hq#tG?>KY@er+9xqmQT6AAa$ZfBF9aM85ghM`#$@S)4p3=KyQp81hP0^ zBa#>O6U1FubOijA%=d(O$*I(Q>b{xALQ)Ts^Kbc%VhsUiw)-fnFTM-WHx zYPXRWP*c1o{hGGh4ec7U&{Sq!ToW)9ULu5ye^?MgOy>u4U%nd#K(mK@kNMt5`}{E9 zy?DQDQJO(mLDFlEM1kBBfe#;{|D;>3X?Su=STiUI6fj1h?MI_n#>B@`Su${iGxL z-|>>JCUTFw3LxH@pc15^$ES5cgM&*ua*7ML4d$%E zJUw$+G^-Cs(3Qcs!<>+rwOLDPIVUc8_rUq?tV#c9JR*)BP3!w5Deepg`-jO3`zFk9 ziP>OnNv1aOWC`CZOP){M7VS4@9K5fa)_royPde@cVzKK5hV?4zhkeT?ky=LCpPLYX z(=yjw?*>MYCEq>t57bGBKnrd5q3QMQB_~C>@$Z(@Aji_~6X^?Vz+#e=-AS0rTqD>#VpQ;w%Cd2%-rsDj(bGCVp`q=+6$B zK`sk#6?m9*#MOvUl|H(5>WE0XnitdzA%Y19$2XC=9vqK3XwIrfD(Vt@cl)}!?LS)9 z+Qk((b6LX;b!5@PAr=}#o(GOr;JcI8*n!^+K$D-!*jOP=2gO7Xh@bd3YH{P$I9e8?C)o8Rm&ZusI8%eH03xG#488;DU^h|jLR4aJLcC#Un|g7R@6qMH5FBO1Bh zJ{+AlhA6!;nD^xCPl)DQhC^cC8ofAu&Voy07H?=(7gQy7gNjXw7O z^soOHX3pQ#MgJs|sc9-HDgTX3_6HHk)X2`|l`Qm1>w0BRDZes}{-TwE#66}Uu9BsT zCxhg_^2I=0F%VnmpJXxiKUqirWDWf*SLk0^bHe{sia;B&sdzer*ho@#X7<0?a0*VQ zK&My28_iz{0kNHwf##MVVjhSEB?Qu10YNS*t_}`1K-*VZ)o*&yZ~D_eiAx~**6-Kj zKN)YJm|ipenqCkk>tB-1r17f8(&1!gCgJ7;vC%+`GQGd^y#AG}_uu7tvHwBL`k&Lh z*k4I*f2VorjVNM!Bk@Ys4(AHH#_dWw^*{3`wrOcN?qq(crk}FFY zk5^&@?NCw%s-?CC0$}7TGXY&z z`PXb(6~3&rQl)T6c$ikDbq&h<9m@=q!6q;`$T#Gw84{NUA zf>tXMmfaG9)c0ei_qDtvk%$wDC$|fTlI4)G-7elQv(QzPX5e5iwwbdRYlCPmg&i)0 z>uWZ278ig~Q_g#GXPtl|rXT6)nf1K$#npvB0N&kpb3-21-?xHyT?~Z!pn6^cUqFhq zqqN~oiKy`d<0cC0!?x#(%Zp2BaR^67ryxZmZ49s}x6$CojmZyM~8(L4p!_jh;izg)XhEE1z0eQvp_G(ELi`M9SH z!6D-xb!1W5ETb4wmQI&?@}RFR6c@2q_NA)&OT%DuRLuJ`ev`miLm%Tt#cn0rMjW7cQ)&?55owc*xIe-C`W0iba*$hW z#UTnK>D#avvVN)knLwU)no^v^o(wBXY6etBB3mSe#k2$Tfs0tHg2}o45x=NrIbPBq zNS5P=7+r5fh~?KBRSGLJYG$1Ujj6z(bl?0-WOV) zIJ16QEWhy!NgW#RxHfFl#qtl22>kSOWqTcA|3I7|6)f=AAYIEp5b?e-8|R0)bOwSLApvLs*?t_rIp zVcpcKK9=1FjS07kqcGC~$;q;$Dpm3QNwE!NxjJjpano+4HCo#u7gMHnQ_};7l~BUh zcDS8S3z>YPLWL0OQLOpa3v`nRha0E?id%PyjRvVD=C==kbTjRFVRpt{&Fn7@k2kGo z(1MW${~+TrYW?)8@B$95oaSS8)Ym?hQx`bZBb>Xj4W@@Wxh;K7LuNO zxJp>kbhWZNCN-ltbwk7MJ{pdSFQ2&mpUv^28vdyyF;@p3%wcMr^lsh;=D2+ z1yFD(U+`jnewF$(_1%<=+CYGi=#;+8`sI^Q_YB;z(M9=Q!sFUTBn30ug@NrYbmY4z z+EGR=xmAjl(eU}5JzMsJaX9-;6e_uI*tYYJDWbj#eAwZOWl$>=P^e9WmZ! zJ1IpSh~-Z!syb+OD+47+9s%_%2a5Q(LT_`^^7n(5>o&?cU534={ZNLSkMS0rqg|U+ z7TU$Dbj%j`kJXw~6m=~4jt8}e*c5KnrKzR#kH&bnzdGNru7v z4YDrS9#&_D%~gB}&Q@l4d9|-O69{AU{YR&m0^aj)(@#Cy_9vn55 z*RY(e#*dB|xMn1HksFnbs!((m7$i#&ibut{Sg(h?nLgr1V4c-OR zfKZj;eO9LQbf{S%j&=~1X1)6OgTKX957n80&V>Njz!QArzx8#0_Xz*5dJe4If9opf z#rIhakf4N~@(&WYBS%SjAqubPJ6W`_pFrRcsubV?jgdSbE3}$tz(O|fd$!znT|IC| zsN+uwCc;@iq#}l`;QEhBp+4KjS zgT1*J!djU}D6KvA;EU0T7HB1*v@|z5>mfoUH*D&*<<=+8niT7aLv(8 zxM|=ZhgDFOrT8qM4daS24NdaKpoh0?G{b&!aUJV+fv>!xU4$d0t79zrVoyS}fo0#t zm5DW87uGA5|EsY6*+W)RkXKTc{ac^?pB?l6o&bdXRk{G` zFaC=F1VqdKF9jg%uj3z}NB-71{%;6C{*qz*ze_*X8w&VOn#}(K3V4m? zuiXAuD1eil1q2KKP=2tGaB*|~UF&h8<>`YdmRoydppkcaeeCDSq=5d#`}c9qBOxU zY~ShqC?tu($4MzF!coSeWPbWv7`+T$!bbG5uK2$MjYk^zoc7Y)`Fw~!Bc9IUt)wx# zEtCr$$kYTF5~;Vkk17~>BBUmLe}6SHXWoU?9OH{JF5vCZm#q~%W!LWnpSr}yZfzNH zm)sa1-&(ARl*-4~VjBKr?i1}DiozKOQn14Oi`pyY!*YAcZ<;&9|9Qg(-)&kZ;*=mB>hw8YPbNEOD(jBN;b5nY_8#Cvm zy~#t}8?{SleSp28SZGcvz6P2G`mMgM48b^M8@tBH_5M&Dzf*f+&CUO*>@1_A`r0=B zM?kt8Y3ZB+28I%(K|+vDi6I1}hL9AAp&LZHkuF8LaU`TWr9qI6A%u7Ed7o!J>RRtw z@A)!wu5<7E-m~{P`@`&OeltQIiw+H_YHwqZ&uADXQk53}k*qu%r>w&oC$kdYQ&SgM z;aWK;3GyEFmRuIT;7FZ3113EPlCY~2rtWMJh|c=HblYR%yTwp40`S7PwHT|D;oE~k zJE)e_k)KwvPh}IF8)8<7DleB0%L{CHqzt8`A zZM&xT(OcC5E)R*?p@;l9UQe6km{j7loM3iDFUK;r2Hj&O{PIMZ-~;)|lk<(uHt77}%!V3pj2pyF7* zjau7LtyifopS(T^X=i=s1YnC_=m2@hH*5(g2|TgOH6g{-9u3LQ!`=qHABgYKk%$@#s<=mXjLruWSZZ3=ZJ?uEhRpw z9o8(jUvSp7EqkBnO>`N$IG%MXgphx20pGrdjZi$Bp~oX!QWXIM<2C?86$)k{KjVuueau; z-rhb6ud7JFE~1hpkNMuC zzpZRL9z0c7%esVONPbPb*xV`;*-CgM8VZG{P*K*Z5I?y;qcIZIeE{Nzw2Vl7dK{!k zYW87$Lg90;Rq8fcl@*-jYvBVb7|v?{s@+P*p;F}Y8MPBqX=kQ%_aTCxr0&9kxPGrL z-r%U3?Q;@v+g3$(Vq^pc~xTpPvoyej4g1yT0!Pz3F2V(VQV$}&@b4+-Ws~gJ>63DVkc9=u=GsO zX;E~iJzq&||MD~^J+R+@Zt`qy9mAD6$gGIZSk(9XXu%Kyemz!?(7iknJ-d`}wHoUn zeXocyl=*D)OLnF|N20RH3M;BZ%n9y{RXyaMNm!6=$SYeP-xO4gR8)-B9f=%k4A$6$ zpnJs1T!G4OmNuglRfuY?yfLr~$hdUND~c;-8&!{P!U)>@G6)*Zs3b0yA+zI=s}n1p zIMS==_A$GWY|nDNdjP@TJY^jHmK^nT>ZkTe*1Ay^cWq3gRO64Ex8b4E_UVz* zq(<*RPEP}9<;VX40&(wK!ln}6H)*-Cg2j19lw-U3W*yUQ+X6@{PVen3^q{PP$NdTT zyVT;EU*y9Sookk~sVIU%Qt~tkRk&kv6)%!kdSjxraurxagR;x{i^6T(R_Y4!fsfRU*tuRJo*Xd z1QM5~7!^WRhDj6VpwjI`y39mGxb2ks4|{7IM}1`N=18;{Q8DsImO_#8Y;k#R&S+Of zRL)d~Q~{CC<)!6g=C#BIqd%d#535yquff;hn;1ud7g!e(N6(&U>cX`@>)z)KOu{Pl zwpT7!a!`uezxi;Lw?}}zih~;5(HRtvh?i2TV$Kn76@N;{tsa{G3MUme54GuyXE=!s zv1s0>bXwvhr|Y8tHgfT~>ET#!=+^*&A4{6c7Y@O&q5+9VtCEI6Whg2K)Uy)x-`@;f1 zC{QePFO3tPs$l+<{?4+fwmizB{0hzj{aoOKC)vBSkp!hmYkz+=>2(f!uiKGc3U^OT7$WY^BK2lw-kqY;`GR@vq<9fDk50tC}i35$5+h9jK%On<^xbj|g{}^ux^W4Z* zm&8g1^Urb=0d9;6x4XnXpU>TGY5|a+0u~loacm&e`u3hY~#T)B!1}QXi)=Ntnks5MGi; zrw1J8VDSaUAB0TbH`A6p^8`x3hWKdBFEa-p6`fXFDGH7crgN%UjIF`D;nOQAeeC+o z=82R%pR1-7zL;;1Y{>y*x+|E;G`R9~KbJ)=D4g}^g>LtP4|h>4zd3o83WSCJsFB?= zSeBg&l7q@jmWgFm*6;r;h{?#h!8(muB(1ha=ZujHr!xyyzx|5 zqPZ;_3mK7rLF9d^Hh4dZH)fV{+jQSHbcS56d2u9;7b|e%oP54#M(yTx?ZS()Pn6Cj zsOL`Qb0%|Hmy6*T!a~^F#*>`$ZV`ACKe}duJR3-*Xq@Awe%w6o4d-SM(M^AVSyHI^ zWNcg<&$e-e&@H1F8#B>XvkYt`Q@6KgxkHpgxF2*L>R)ds{fq}l6D@>i8MC5+{*@20 z2H=rXj0!7cOtn&qn^O(`jC; zTDm|CUcEcIM_u3#MFU7Y=P6cYF~=K4y+pAors@*P*=cva!$Q|s^Bp?>_N<+@q1*VZ z;1npEqneu2uIz5q>KKnt*V&H;2nMf9L$)K|e*dn~xupdo{pwfyNs`xxt7pFKS45%3 z-n27gn%>+I$&`JDk&{}XpMR7Qs-n*iD@QA*u+;D-)mMF#3*}O`xc6=FjK$b_JGDen zMYd&&-#Kk_NFMqlga5XgzqQHZk?6HZei|RkWAl4^I$;|VUNy|)eNJZJol1xeNsyR+ zrj;Ffr_VCmu3!06F1abw!9KkP9UiS}@Voc-3_{s~otS$YvTM{e%32y@M%|(J+qQIb zjS7iKN%Wp#_dRCOP#ujw%_b5I!470J95AQq`v;^jtMd{cnA*E}X#ph7Wx-{?ZV+DUSN*t%A__$jYe_&8I!qNz0bn5G#AHxa#Ae?H`hxIk9_ zJEvU30XSdV1t`_XP9Yoc723b5oR!XO%a$_l5= z)oot3{6Js(k%jaami>d91Sd>wze)jU&BbNF&^H(rR^NpFM9NKXTC-LwN>tcbM!STZ zllZ4nK6}oT#%zZA8>WWF7&dXWdVvgjF4E(CLwwHsLBy1$TPY&T7U8%MGdpCH`(7@> zk4NqS>tdlQ*?w}7UC*rfWlFuCpX?k%#ds!}N7&rj*I8BvKHtYaO7jnIHA_d#L4;JZ zh0z90 z%_fTax1;>$uF%rvtjWI9J&%RA3$%OC@c>S6C&Bh(?iI-#H;M})zcnV`U`e02!V$sh zs-G@bShMYL%+(;L*4UR&D1a~n@uZHVh2mjJ2Pr|b6t|aF@i-LC)Do?5%4x75L}=O6 zDzp{k;qr7okjAH1op#BMy^_xd^HhumyH_h&?(MKyC%?vb<H7 zsR`q-1wZLXd@k?gbRZ=i6OF%_o;p6LFGak2*DD-R!F%L5yLD@sPhV&=uPU-xr-rS7CB|_^??WM<>L5hz= zh!mWe>)y7?r4{hxhcpvNh6EpAG#aRoo`+d{`K*Q|#W)k{>*&!M7tWEZwZ-Y?MUfmD z{2|(hlf19CdjC@f*>VajsL4sLXN`S>B|glKT3WZuAiellC^|g>9=45;IAf@6*Cwg? zr>57(hm(abJ}{q8D6)EUUHLFO8BsyXf@VJqwAb2O!gJu7&LpvIKi2cFt{ z`P}*H!(|llaPo<{ghC8;3g(j^$HG0?92h%Fel6bmRXKSN{YCi^Z@kD233`=VQqmD4 zlHL~@OOIBMi!gTP6r#7jyIC${mZgc)(q|CsSp57#l;bg&eacOB`Uz#vy?dF5;)sLf0^RG&q?3EtqOHoBr`nF^$UfY zrF`?h$aag@IPd_nq40c2p<1J@PT?kX`GQsbj@#12MOIunq*|z~Ed)ZZ&wj>=Gt6fY z#OBO7Jy;z0Pchtx%VgUoY3Q5qlv1%^0C%T{0zemASCJ@!VM0y6G1|99eKAgw zky*B@Ci=93BSo@1TQAzqG@R<{;JHw>qaESKecZg5Y#tw(Q2PBFt4;p`#%<NW7PbY)z*d?vAPJC?^f4wUGOHV=wakiK&MCLKK8>g9=Er}b5+djZ)aHCPPT z=gYDS>va>sjUL`x@O1~yZQVWI*86Rn2Pi_f&(V#*lBvdeD4PmhS8Wky`>qIM+-n@z zIk>rh`4i2ccGrx+;Vw5!;P4<##S_-mJ3%f+h2sd5A%T4YH}@{1!GyuhgUeu;1x>Vz zXH*vv4mDT2{zF)ZuBincjfsSKLdzv}vjH#NjT^t1_}Jd-t~$hyCAfPz=>_=9G6esW z#$@HdoanmmHR@HuOaRlg=_ceVA!iVW?_0j9O|UOb=4RE~E;_z(7sUF@z@Pji?hg(k znBI4=f+ngWx(2hFVh`JgjH^}~*z58zGv{8`Z00_1!Ww#UfD3b&cv%-E^YKB+L0-v= z>BgSthTfgzs6wAQ$I?tJ)Lo}PIJ3^3P}_4-2tP_15E=}MszYe;+kdON>GiXhfU#Y5 zh#GA)&YAt7&L}_apw0*CY3QSou@{G1>N#`_fy)eRtF>u#3=Ys>F8C0d-DtRO+4C4z zz3O|(6pyz_CAm~X;9&06s`1w>yn1CFc{Fsk!E)sXPF&pf!NmAD;OoS$K&%52{Nr+Q zA%A}HcC$t7dU~85nhh8B6#%Tu9>4a!j2YMb&YO2Md&_`8nGR?a*@sRj-X4C8xrcS9 zApSGwwc=NR9)g78URH%p<3}liypgIav6WphV3p=EOA?WJ+fFq6)x)oz_^Y2&?TsiS zqg-pNStFghZ1D3+%q&5X-<)8>Zx=?eUsZPeSnb$|?KeXJ_#W&9n#oAmr%;68JJ;s8 z!!SbX%>XbO{J{H{)7d(d;C4$?U!)Vu8dc@XWRAbMWdq}}PzJ=MUKJJz&4AQ18pHg2 zwI_#yS~*X8gBDxj_4k>!k~SbM@r++_B%USc56v-xs5Il;pqxhIK14+G$v*neBE zm;ByeCJxYjU4)5?|56w1>gr@>rwiVq5hT&V#};}9{_-d`T&nvmlfi5HDspfxtT(7t zN|k}e29?L4CE)C)E%wXp8QRSp>AC4W&MUsZ_y7LeYW#m~um}kMf_DE0qHP?n`m&3G zF!((E4Wu#D-U1M1sb4Z0Me$Q(jIKR^@uQYSP)4w$X-Jr6afIa~0!KP^>TrwO`77u3 z9`E$8SUP+1jt{wyLDb)@>mxI@-^`7aeX6{6EM%-=!po|I1Q(ybv~uW=h=JCZIj=Uc ziDfMZI1fgAvSEzA>GUv#lYCU0BJ_KHuk9$_jrP;G_=SE#6Bw3f9PGSBb#t}x$Jlh% ziYrdL+)V|0{5u|QGXNomKgwh>1a7z~dVldotQ0*eJ>!Co$*w?ov_>WPqiTjoQa8p5 zD!*-N8H%}LnNps{!(zVbCGaTNQi)qb{f7GI4x=k)85v|lD}FI{k-^S9Dpg@roN27( z-oEx)<#1bd{--Zw&cQS6G2P;+d!wFjbH%ZbF;mIFhB$q!!?60t$@y{bHP-TK2-dYy zo;_2G5&=QRL>`=`&9e<@IX(WQ`mN+Zbz$js!{?OAEP9hgv6nt4PX1|T_~0b}wv5O| z<-exxzjtnB9NmxxL?ng%iLK?Go$_vTj)CD9ZSL;%mqi=-`pb*Rz`*}&f0Y3V$@^EF z;GH`Z8HcoAszNMn&He+17X%3~2q2w!$P2$4$L}1Bzg&{?NL<{%vLXG2{QM06s&M!I z4j%U_?cE;x|I-@&Z*2Ro5=gu}(%1LP=E;DKyuvSx+(`XbX5^Uw5?%kl9YEH(E2Zi9 zi#vbk68(jPe~M(ryWU+vjv#U?5!=%RVucN05EQ`%{O4d00)dcetQh`R69OV>f4{CW zy!@92e $@ - -killall -HUP mupdf - -watch: - while [ 1 ]; do inotifywait -e modify *; make; done - -.PRECIOUS: *.svg.pdf *.tk.pdf - -clean: - rm -f $(GENFIG) $(GENPDF) diff --git a/garlic/doc/slides/overview.svg b/garlic/doc/slides/overview.svg deleted file mode 100644 index e36e047..0000000 --- a/garlic/doc/slides/overview.svg +++ /dev/null @@ -1,479 +0,0 @@ - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - Garlic overview - bscpkgs - - Isolation - Apps - Postprocess - Replicable - - - - - - - - - - - - - - - - - - Filesystem isolation ensures the appsalways use the same software libraries - Zero human intervention, all datapostprocessing scripts are included - Different team, same results onthe same system (ACM) - Repository with BSC and custompackages - nix - Backwards tracking - Every result is hashed by the softwareused to generate it (datasets, plots ...) - Reproducible builds from source usingthe nix package manager - Several benchmark programs withmultiple programming models - nixpkgs - Large repository of common packages,maintaned by the community - Reproducible - Different team, same results ona different system (ACM). - Is a complete benchmark framework(green = not mature yet) - - diff --git a/garlic/doc/slides/sat.png b/garlic/doc/slides/sat.png deleted file mode 100644 index bca6bed66d6e30a8ea0b5814d4d1a85e5df78e05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 79137 zcmb^ZbyQVd)HaN78flOY>5?vyMkEBJTS}B}r9@J?5h(#_=@O8TMx?tN1f)x84)v`A z_w&5tH-2M$f4t*6V>oU-=j?UXo^j1<&UNiOR8?6H8-pAJf*@@9XVU5rgcJb(gN_K^ zab>r(1^>C{_)O0Uf=KY;e-L_@4kaLn29lS4qWSL2&b-wZ&G97Ezw2`^>Wd2Ni%zjE zGZ3EOg@#cmI3r`zF{uS-sAXr7-KP{#%ZlQWWwihNI9Uy?CPElh(BMgAs2`GSw%QZq z4?(E!7ZYa^JqA5%=VjcqDqk2TmU>@PA7rbf)?H1o5~R(oHb^Plql3TbS!78hrNOI> zq>Tk1yvTw~;zGa+tBqR${(g`NDI@%gZQKzA;Nut*QrS@OaThmMHvD5#(jYbX$Glko zU2a|*Fzn}tNE^`p>U=qz>x5_550ev5>Wk})Re6l@4Qx^>L_=A~?5_=F#wWJ22Hy5& zw=>;l??yZ+bTu6v3ug|nXxpUYz3e(m{yN7cDJ`u?PKGN(&wuTli8jMoGO{Fyl;-Vt z`1oqZZEO*UXP1YAH-CdANDXCRn5h0nWrBnlWQOS95?`?sE4ss9Ke|nM3%S7-$&E}j zqp51ywxI=IX$sOIZQQBot*EF#{LrGMV`%2g;mnep zk`l>jQsyac%ug;*7luGt9SBTeQX4pYaB$$U-$>H3czseMakZ&}c5xEVrZbQxCL$pr zK~Mks^6cNFv5%L9#rlGIn%Bv+jEu~ql#;JsP50)0u<4XiI+{PGY}ziZ&s9#{a(w*O z$;o2i%lS^_U`M}Rm391ezLJ)f)-t|*C>lJ6>;~?k(>{bG;ea_4nmpJ9^YIZGOMJ0NlMZ)GV=a;k9p4Pbl%R+ zu3RY5&@~Bfc{R}(_3y9yD(JMlyu3$=;^%+vU->(`xU^FV5n^m|mFQKC_&xg)_gHP)m2infbp^;f~@(x-+@4JF(xM`^(iyl(TcWjJym+CZk*!j{ z6h!gx;X__NK7_}q17Trd_wF-oaCu$+>x5kn3=Gur`n6_Uk8F(>9mmYfyzu&3U+*#V z-3pc-Ee_i-C8~V0BJ1UCXSYl1=~Yrv0z2C-YZm`)n5US?IlsTWf0|SnNgw^uHN4I{icfv zaJ#SC!`Mn|rEbt>cU|3_oqw$vyUn}6&UU6N6S<6=Oxl80dlLuK#Pq2f){<>r@nsKh z{0hB){0{|_xZ!L$e0!oqzs}jp#Do?q8V_6Gewfp+krfVodAA(;2F2%O+5(3EicDES z$aQB5+&LMa`Hq_&_|+m73}4`bt4P6xsqa>!ozy-J@BY0KOEYpA*YaXccbU+G*ok{N zYUV2J=j%B1tBVcmPu5Z#CMx$)NDMyKekzB#1QW}GI93mDe>>L2}80uch2nqTW}7_XlO7P{r*%pfx@8yzh$cY zb=#8T;=f+bdaRbuJg;BT&(F`lmk+hDIA(mv#kDW;`4>0FZ+~3z4G|&Mc7~u~e;BX; zS9!zff`LbZE!+eb0~RyvW+2Hcur-%ZI;d+~Edz^=TncB4fPp4lenT7mVTZ}0vs zKh=$$S{FMW(^%=K0Y3ITKLByoDZ913O~UcwyX9~e3Sd-u?L+FzkM~ z)~=+iz!Pe6l+o0ryC!lv=d{99R9I+y?7bYy{O3vCZuKn05_YiQvEPo0R`xBJ+M85T zSy_3GgRG8k;4xB5v0f$4i3X?xRJBXTkVZuND|g^sBAJ4Mf*-31`X?gB=$Yw$VTXt5 zH7@Qmic%`c1hIr_G$ij`_C4n3&9T zZvFCCGPZxSqs0pg3J@}ke@P(2yb!6q&z1lJ%?uCYQ&0=q&t0A$t`DYDQ?RnKx}Pn9 zJ)W6dsXIL9CpWCJ9?!fwLHcSOvv2b;Oz3Z!!{RDySk~f-$F~I#Ya;s%_gf6z=5x?m zTRC@nxtft3dsy>K&CD1loVE(`5YINUqj8oh;kQRqIgCQO7I0r#7nt%XuGmo*eYC;5 zr9kGicgx~Ti3tf?>*>psZ+_v!sE=&6-#3k=vnYIxc35mOyirjag+0hDi-?E-3EQ6{ zcuFO@-*9O3elushNCyB!?YuqIo5+Rc_$Nm0fya&i)cJMOt)$JI$VSCg)S5HNX)eI2Vnpxtx@N((U8WBa+<)W^BrK7aBd5eg+zQ{tNWhU+`TXXlegtzh`S@g^-a>JwMB| zm6MYL9XTZ_DJglO-H)Uc;^mBhms;0)F;c9i;|YBdW6$!!LI5xXzSK8oofoi=7jdz% zmEKpd#j8EHLGcS)L&JFhTOfR>xeV+0LD>cs<8!JbXpSsFIrmu&jB|u8)taYR0@(J;fJsz5n``mX`MP^nfzSWbfnO%Jy91 zt&Wb)vbnw&ivJThSt4g>JK}8|*mT}&iFQOWgQX6^t_l77qiS6BH1*M@!7E`z^ZXw0HIGJgDhaRaNOe*XJWS ziavk+ssH@>gN*nA&PU*A41zz;4g~vJUs91y*+5_4<)(@c47j+&xVZ^bD1iGC7t@1H zN(gn&7%FNK3Ea}izrMn`+*YgusP}56BQhx|iA}rsS;H@hcY&KexIV{4#=95x53Uky z=;*Rqxr^Iae1)9W;6Vv@8i5;AMHvkCzIieI@pZr8;)c%h?p2|J>ZLaP^O}Uw^*)h^>*g|)RfoAPz=!!IsG;^etQKH-e)P0 z1Y*~(w(W^yEnt5x0Vf7HON{^=<$317*3qavxw*KwC~>#|KWL8Ti^;VD<;;2cj6PZv z@#WFRu#k`t#H5sr{2O4T;!fY#@bEvtemd85*%Lth{}qL2F7adxqf98(e&%L%?e&=o_!5;f|En~M}z zE56@+bN!)N>__`AkUojhz+KnlW!@;ilWj_;XKRFj^{3rdd*aLID`3fd7E!&cuo|C> znx~I>@$rWdUXjW?dD2RK{TB_Q_Bt8{5K~!W1!#_&i;K%*%zB!Ka{3MR%|B{bm&>G~ zMY(QWeZ527W^Pm72ES7MW^M}jcZqgUlM*f@=64BTbIix%xuRn4a(g&hF)cH*+~RZV zf6MJuR8+*_tANu#K2w=%Gvz(Mx&|q|n z$#oHd$il+%?r$ra#QC3CK~UBUi%Lh(n4f>f3zdF#_*#r&y)UT}@Fb;_M~w$PY~@O> zuTq{Uux1Zi?x?}SIqNLj!-&$oVViK^H+G$@p^Kg+*HAbI9QoIl{G6|bpvlhwkljgq z7JF`b_RUwWSE_m7`!;2b)mEdq$J>+e--V6q*UVDQ`twaN{jQ$1FDfYbckXEnhyy_U zRmaCi50oJg`y?V2X`etX?E>YosXlD$DhG5OX`Tn&2?+^7K|v%&?=tr_3_>p=yaD%m zW5%5G89{`b>THr^cED+OrYaP(XtDLjzLzYZ-T=99GxGCUN)w-C0mNCfCTuy{9KE?{ zzVT=a!uGkjSahgeLcMZtny$2{M7ocUkNq%c_<@M)PKcP-7gE{v{uH-9o=%!punx72 ze<>z1B9c87#!Jv{p-rV+gBELYFqScB7;H?p?H|~fNz35d-;JB}F(FNfrgV9u=hqke zo#v?it-x%;Z)Wi2D9hg;Z|NdO950OtI!qqV`&}PT8moBH+-up)OD{FH6#vMb9QO(p zD;t2P*9CW7&1~RBSWX|<`{;MjP))unXLxXawi2yks{*>yID>}gi?2Z@0y;anU_>S1 zeF?{F5C$f&qs{5*X#h~M%qrKr4n7y43U|1=jZE*JNL>Gmc4$1F_#)(lfr-gMtY~7g zG;NXIXgyvCO3##i&1tIW+do}!E7b!QnEO0w+3~Rf=OnNkycmG8!ISWK<YdwY9F*B1e5z)cI99$DF-rg2|D z_T53fk{A#>om0z}ZC79ML%}riIv)43Eg@poyUUz-)G}_`Um!;(OAS3*QApwv5?+){ zfN~E!EdjbexbJc8PQ3w8BQcxuJ_yX!6khxMS{!@?(uw~h{{oK)3$?~!(a6AnQZW*M zBL5tB1P%{{p#507?{%;7HLuMCdv-5qj_F|9?~=Sj&IfNURiO6vb`bYCx1xkpnWV5>js4dHK-1tx9= z2fYC$hLnd+P|&T_4+UV}8s7leN8E!m2t+D;B4P3WK63cKoLKaIL(gVpWNg0Lb5B1p{_C92MN^S0aImVTIhZuqf+H3pPyBXdBzYQzcVSW03W1~j|9n(Qt`Z! z-bot{z0bS=02(m&(0Tj8uCB)bJ0vc4_#lATQXy^&?iQd1Y8SbHO<s;P6*X)aBgTVo41-#YN)U>rH06NdO z9lVYfa|LN$YfuU%1`AGuV#&F?^_uYAe&=ETY54`TSlT)|&6fwgpbNeM*zI%HM%`hQ zEaBsAWVE26BIeRgq{D99boqT$Ib5a<0gN6LF^Snfi@Lwn(i-RKqy+V5*wPvF^c0V5 zYtPqG_CaF`*w7EBbyrtc#99E}Wx!A%*X*Y&%qq?M>i|20zk|dY4_F@y_UA~>7KkZ! z6BLR5zCL8UNZGRHYgk8z>{Skkk;aR?dZyTWaAX4@e%QlSX5iS0;vu35hCHsW)${h$ z0MNv}R{*pY6&2mA`&_SoQshK{H+>QZUG$*i9=1FJKy?EcZf3fx;kTDh03v|K^eyna zwzl?O$=LGp^4Ag+;MthWpp~Ry&}cl3c7p&iAWCX!U3DsQ{*YG5f|^bn_d&DAY~(3{ zyt=w-Uvp7fzxV!VSPuMbo>IyQz%3nZ?Na0Bh5gf;Kg<%L{SClUV2~w-Qx&ZapPr4h zIAUdkZF_hW6c-ooN7Kti%{6+Q*4#wu>FL=_6ekJ0Y?XN*7Xi+#sj11XSHVCYIWaMT z$il%9+0PCJ5Gnlv0s^+hSX>8Qg(IGiq0X;ZDo{O_H7Vi zxKjZG*qhLIaQR^n%cO)jG&F>VnF_1A&}CYXD**W>ZT0eT8rgiXMY`9clV2kMg==gk zOYbW`M)2)6ooR4)vYqLPXT#n4R@Qhj6M|07uaU{9MTzXyntGDMgRE`dQIxL^7Z5Q}JqMzFE?4I3-4;O@%2BzD0ce9f z*F_ffnmo_X{;dErR)apW`TDr5yk^R_+OXc$3l#OyT%`-ZOd3D|2L(X1^xp`ef3wrm z?hEegpw;X$<%Kf@K!y+3`q@}mYQeY$PDrs(RfJaK1|@Dh!E_8{5~YNqlLjZ3DS+96vMu0ovQx9@eR0}+dE zM8oR0t380E9S+M;E3L{g0Tc1!&3-t^2lF=i6?mbFwgNp3h!XQdgjBV($8VOw@4*=& z!}v2eKH9%ay8cZ_L_~G=2Q>Ka;^9b~gAg?QWO#VE9IU;yWnD9CcRNs(4(0;&9t`h% zuC|Tylv9)AvP-}i25=+c-S4BsoCcrckl?VF!??hAH}zc_J%Y!?|KE7`{}OATC!gl- zZ*A29P>3w0yT`9I%CCfO!kZLP%BbWh9~-5Z?5wV!rS;>h{4Yi&VNR=&oas+#@3#sh zV8hWTV$A(*mX9f~J3&RSU%XOQ3px?{{fq&)l}P5zV@lb{PfWF6o|?Q3Q? zUW_0o@^4CIe(YRX`ox3}{(i)$MaU>9I0-(I7UtMY;LvX{+V{@G-VKq38|W$d|2Jdv z|4Y06&(!_@_i|I;{2)6eG~sCyDW!{-+OH|^X7ot5rJeH*OUHw!5|zEtj1J4gxTwuc z^ml`OG8)Q2$mj-<%cri)%FuhR5~KgYnSVhcd)3{*AM5Y6`7;Qy=+TZqv@4q9;;iLA z$Yb< z@nt^qaEHBRBOkj}^32H5Yw#6(s)Z`9>6|cjQTWE#nv`@xLA6Mqd#r>|EmemrjeK!a{&WPdZ znLj)J@$EAVyR@iU&|T?`{87jLldqd%*>~p`L1f&~TR~3=P;T8AQ+_0Bt=yH}MjPsP zdkoU$sOMVChe}Evo4zJbcy~$1h}#?ML+rmJg=HtB>8#lQ*)MnBI}7odD0W@+_4CaS zcUP0Xktg&;?s=PKE_0Hau;3nd_clabZU3Nojk(@)VQS!nAirz!`f6;zM&Sh(~*ZsEQv3&?Amh2G(N5BC!*c@#nJU?l}q~5)Se!yF=3%* zuf+BBTKAy%c=c8qZ2iazH>&w$Moq$+^{HWS`7sXhe?k5ZF`qpMl3im{TJ$Qi?4P!a zQPwJy^K^*`N}EqRV!+5~2}DfHT*VdMggb}GH3Duu%92xexq$;oCQ;-r zu)D5DF*_{&Gga|tZ`UWrr)rq94Kb$#j5u%UyYnLXV(9sy&CG6Nv3YUdg;$PLJ#G_TfTTMo z#O`sEwm5dYW9#GI8R8^voi?(-{8^sW*vIVjBKGj(J;`|MZV4{#jE&aLTTtQO$a+XF zj6gGq5R47&jZAEC8RjTmUCZl4)g-(~^Cqu%g?!_NCymr4jBHW_@;3hnzNI&ZmKLt$ zM(|eN?v)e>xXp?R9e6)YU987!+Z2hktBRhRP^F&fqC9;qt;fZO@HP3 zXXUhl1vj1k;8Bb=;`;%a=`X*1qk@CT@b3fE;Uy&BkuPEt;n%cB^XXpr*B$tT1ajA2 zMY&hwZ&$PaRs{1Ic`92Lx}Qwv###jJ|9Fb($@KCxSnerQiFJwRA9&ZppvJ`I??O-> zwtZ*Xzo9W!tpRl3;V1g8V_!@4OPX#-;^HusVwoygs56;|r##hodB=0|kOyxw6+3Iq zK7bAn(d@@nM>B6t(IZ83XyuX&7E))A&G3`?i|Im>z`+>rQM0@znZQFcfjol$u1ioQ zR00A18KvHA@SWYaALF^nO>TRp9E%LFKRgKdSog{VIYX19#9BS~&I<~J+V9+ss%R(j zHK0Ett60q6grlNcAN!^BaP0`2Jz-K*lS6s&O&&VxMbu~bW!H!yN1nc+x0zt6>U_3@ zjVS4E-^_4c}IW8-4>Zmu^8kTO!LpjkUW`VoAGGNA2s{&HpX#X_WJwt zS*_Fq+!%~IM3-N~&2DsityXC@cqZ7C>MdYN&_k$`Q*BlW^5`(h$>ZccHq1-~HNHF` z_tsBotA#ewuF6NN>l)WW>L2t);Ps0!7TJ21hLN}A>Jjm7#5-41a)@#%zl(pG$KLNa znL^t8iLHd%uv4hfkyo6NVGX(O@z!=uo=AK;o!L9{xDXtDlH~u<0)S>H=|vrWh$OV? z?vW<)>7%iGjbayNcoAJTGW0yRNnR5-12H5jg@sC)#X~B&&fevyu|~#3tx!T<;jSDp zDm@Ncvxiu^q-4`0Woj#~dxc-n=k*Rd?Qha^HVpqa7Pi0f(84S#u;9Dl8zfVA~$W5bN_brH$<6xi2aUqfp2eZO9Z`APLQAFrw1ze|&0Nk9DW@1TswXcOX@|7;vwq-E{GT(Yw1 z_Ba&IStK9E%Y9Z!XT-obbv0E}wfwO~9@*mlLwp5h^{ z{_sG}R=?`eN68r4;;bV2)yzx~V`$j(`OVK%HeR$^Nou)2tKkd1Fe|41w)|Wx=!L#M z5hIoTvk=*A(BI1H#As`3t$t!60mSt`N~|;*boi+-d1}F0O`w*q%9ok^?@@#U&W^F(IZmUU5Of?fbFjmB~6G0Xn0v<&i!}G7wDff9J=|evj%>KmSxLB1nq1dC`G4 z2}~$YBvNpEo4$r+nOay-7%C?JUL5&~8QdnG9<7)xZ7}k6G|~CWcN~I>*~MK{dNvJh;gigHGNW61dR$fjP}mUaVH&5dY%gi z`Y|jJk14kMKSKWMhGXRu6XyTi)iLTYk#AOpD9OZvw<5v8_MUYndX11xL}hW!iYJ#= zd`Z8$+Q;}3)Ae6e+^Gqg=(H#9T*da5a{RzgblJ~BXK*0LnI+d9va*9oS2?dR)!e#{ zc4jrnM*>}zgiNt459LkDgUd|bfUE?WxS3p!A0~+&SAU8H8*6#AXb-*S>oHh6V~+hg zNnbCFz|x+>%@>3qdfPcUqf6oQyC6xL5Zfk*t$?((`3U0vac9@rsUq-~I|_ULMMcOE zbsKa&rsX(@bRh9nBV&A%DuF=rJ290Ck~S&-rn_uR)p3_YIDSmSL-(M)=Z|+iI+_nO zWlYC%b)y_3!hE9RUU*S`m6pwQT<>TWZn&${Q~@v?X-yWy94~bNQIn zI2HMS+Xw-eNDDBrYqzp|!`#?K6*DE!Dy)+CpW%zARM6D!v(dGKMS|PZ9Y(M^roju^ zxS3Oz`&B72h5Rz>^>@Sajz*d4Y8p}~BknKaRm=8YB>*DQ&i(lePNn?|G&i zXMF?u-FspiZ6?zbh?0MZTGC>Atb_6H%FNX*6#y&_fMF40Ra(9=evjI~n^3PQv)5w$ z*xE&1T+ggZo?_f>#M>EdXcf(Og!1$I(OAK)vV(EN{5}@yj6(~#`XhJR0QDC*fuS~K zwfyi7@FGio%mtFvMy>GR%Zq#KU!&l^#&4l5+_SQHKe8)<38L$aj5zzE;B{x|T}2I& zpK2_KW_x1(^?s4NG4`~-)CLq7X;r&BQ1c)55vzT4q-Q<8^ZIw4R`le0gbWsniG}_h z{}=DVeSq6fmJJlO=k!f=@{XL5@Y)%hOPV{74CZS#pX@=ZuX?$v+ug3?jG}k8kZHog0)@o$Mttv_ zgsQq2m5*BKx4a|fpVA=38Z<*e^&fU>{NpSeYTqpW-Hjf8Blen4ZyE1ZuGgu6K)7#hyVOd<3~C?8Ka$ zT|6;s;S!le2+5+;U3Pl7i*#2IB$VdSWoVFlE^OKB#%pGaPzhNy4C4Ped?dg={zyQU zM8_<6tu@1Z7bjv;<&iR#FDs9*-fP~t`cV1)i;7X!kOjt8ti(d04ParFkqUl4^IvZu zwk1N#XzTTe7$|c#7MN%abNkAZ%-$TDakI?6fI|GFOe*tgXZTxgJu{Zw7Q6>R07kwn z&ym?~guD;HlQWCFzu~j2p)jqADLJ)ES4Pu5o{M(76*_3hbg(1K9_>qG-eV+d%W#nJ z7#^XsJ7FH-whm9Z(Iw;JfODLG-Y$*|9aDwmt0F7FYzgXy>%|_9W5wF}(Pti7FtF<< zBZM?~PPu=$#=Em%E46Gr38XOp?Z}?}1&OT{Mh(*E9ZDPG+}avTY?qJZ?AmIopGRJd zV%{1M+l?UKE*Kf4u*(satf_Ns8M$OqV@6lh|-1k=Ve+{G;K1?BpLW*sd zdcu6-aUR^=rbe=~AsS?AMk7*b@l9lFHq-68k8<fG62*JxLzrhtKbZ1jZJ{jja{Js*6O& z7$F6vU?0i0`Diqh$}y=(3IzXs)p{18#r}!Oerh@b$M^9Xik6Lt5!*^W`=*JpkY}V) z_>4KM9Q^aQejNTbx)#WVOv{}$Xnq3Hi)i*8C^Le{NEmHFXc%oD z#U!RLTL<5}vju0e*lpr^>|aK>T=&(i#Z6-@uk}VfLcYY4VqHRkwcf>kn-q&*E+@p& z?U>0OZ6491iFkwy#nqW??iL@#yVXCedLAnV>hO_s+bFgqD`#Lg zGP&ukPvfc?R7Y{{nZAgb^+5d2G&M%B2XVIxO)$VLXq93kv7rDq%+lhswqcq(>wh1g ziIgNoWEtH3!NFfF^T@*uk<^6i;kORK|ct4sj~TY50q=? z_{@#*mo(w{IzE!WZ*7)n~GYFBI?7nXrVh zgf2S8p%UG?EQdlVtC|X`;QcsXU%v4_y_xCy6o_-g4NJ5PLc-Qpkv{RL_wT-*(s7rU z@5^f3WJN`gu%ReNP19r;C{gM*`GIj=UA`iMtL->$cYDzWoZtC2GtHro|EI~KzvgkD zp72vemttXD6)TZqz3VkuO2(?-gOHyKUQ$JuRqxjaSqI8N^ggAXp1EZl=8A6$kZ*%m zf!WUJ-BU<)!J~0L54HN-6)C@uJ$nC=HOIC%IPTjtXa>U+8L93z9hx83p$v67{T%j4 zIAq&*#U3%~Ax#`QzfZM0vhR-v@Re(bBJj2dk4SqlXOZ{if9fd}rC*bE}Q-)G;0P@)G{kJ7m z_DH;DjB?L%WB1Yuf6coxTWuOpj6b!3ZDc;|3LJM?Jo8fx4t|-Kk1!$W99Vn}hGgcr z;isWnNPBKTxz|W9t*KV4x>|0&@f`hj=1}FVUY4I zBi>7kdFWHZyrmPsJ1(I7Yxnj3Y4?CDl$B3vuGkZv^WhGQKN6y6_ai}!Lf<|^EfmK( z=Qfb0c|v{P)oJ-8|FkL^l!Ox#OX0)Li*pA>Vg&hCpr{siIU)Gokw)iaJ~lqNUxN-- z2}|6BGI4Qas+_xb5rQpd_W=1PUQ5Va8fSXFztVrCv8Q$*ofq#Tl^rDgB=g&g$o#~U zu)Do~&JPNLM-*)v1_W;we3*S>PO<=NyrdxCrp@Wzi^zPV&wERu18Q|U*w8&w-?>=0A z`)}>1+Pe&mMd{F!t$KLXyKK}=GTBe$@@ZR;{Tq&j)q zqE|qY{?)DZN(!6RwX#gk*P#4lSbfu#exlaHmg9fN*PcqIkSyRqamPj(i)%&yq^~gI zvcG_A2L;~K2u^wkb5q;Mn;(FdQg}{Ai+_tETzKg_g0Bv??dc=}={hO?;<<*FnFbPM zy56$%+@-|%kB_w8mG8F1oq63c6aQHR z=n2PVqysa&S2zPaBUfMy!FQCgqC&&qAr<-IbsknOm58e-5ZwZqKw%*`{N~jC_B(_= zYHHRreem>XRK@3fHLTDq5;Vh;n_Z`3c8Sg1ge z6jWt1`M{}yaKyi0=&^(0>{8%WpCzQET48l6pf!j=G_2~_@4`*u}< z)hw(B&Z-Evj@s@c`yLpuAVQDM3y%lVW4~mtF!fHD2ehdYg05EewJ9l;*n4O2u!Eah z&2{$Tepyiwg_!Vl840ANwY|L!p5d5tb3FLdotl~o6z~Q4@PmK#XW%RW^8NC>b_W_F z-khPbMCJxeyXnw^1+*UsA(v7nRZNr-I5x6Ixf0-Bmb^a_Xzp>%Tl3UDwkhZ!sQ-$ z=qNekOeK%7{&DHtEUL4bkzaCPgU4Aqrv>TTEZyDDL4?2+P~ceKfLiFVj}J|huP|M# zzi(&K_{j|qA;vM)<)2uUYTIdo3mz^mm2|$bwtVH$@3n)9i=W~?DJC0$(G!>vQ6)Na zGq}n7;}6eS#i)i?YH}0?Wla$mZ(_LYe`htqg{tzUb9)6-9{2!Z#`w*C zPZJrLY$B&246aL92f+xQo?!_%1DQbWu~QC&Z{R5cG5g%nO=LWDWEc+6!9#j_SC2?V zu8v0G8osIMu&f3kvT&{U$`4l4(gM=BEEa7euV(~a@_~3jGxpyk>uxGn*Q%?|sZ!lsrP+oj+H&O$SP1JtvZU-mm@ZHKAjA-M2y+_6r!6q9-q~3FE zIBVyc(ivpucI&F3wxug&KK86lwTiD_ zjYoB_I)9}wfXA@p<-6DVQ!a3TpiM|bL}afIeyDYxOP zXrTB2g0(LKHVMCt#X-h7tIZ-P{VdL4K!XtfPwTwAB`821)2{?cN9iB=6b1wlw{eb& zP(}gyy(bVPO&a@L0fAn|A|G7Q0hfxyG~|FB%^S#B;2J!jR3T|evPe6x0g^EwD)=Jk zkcRgP$Z>&0!)bj$pb&`c=oC0r@C>~nmhE|6&Hc+HlL{p;b4w`}7<)SUvOMJHa-5Dr zx9K0G=imYkHa75(K|2uD2q;kqSdW1R3xvI3&R63vxu<4k3MF!oTY$J5{3;OB{VCKc z;#W%Hw;BOb`v-1!96~Kw+D-(j-RbH^_N{xVs8LG=CMMEuBn=NBg*U80XRm2Z9oHv- zEER}d?OGBCgttNJ3m5G_1Hc7D{^42o)`T}S>r(*z@K77y zY;fvmGz9sOhNsO>e%B>f9S096l5cIz&@%226PhDP>VGYIG==s1**dBI%|QCdc8p;v z6t}EyvL;mJ+(P3trRPXzt_)v5!o&Qx(6;LlgIebA&|Y;N3Q$Z&sxE$GBS`9TLiy_Y zi;dWlX^i!AMpF)jxtZk;d>!jP#-}FF72Z$+ z{9#pMt>GaA$X)rd9rSIydA1BiNh&E>fYZLh_J78;tYvYim~9&-)(-A*@?ds6BL9>G zuyrQFbx6`}tftKv?WN~h*YoxT2Wv2?0yXUCE0c0kRW@v#py4*1MbY=*T1I1 zKvDUW(eD2eIm_+!urKkj zJTCN&STQS%ZeOfsw|1Bo!Y2whmQ<`M&jAZiWpVlp>c!)`t0Mn=VH#UQF8QPJTg%p- zpa5lp9S0=sjj%%o-+Q#l(O)9PEG85m13biUzdyIIt{{Yd3o|Zj*?VL;GDQvqdN@&q znWgYnVW|SCWpaVagsl8wg8V?hKziiy5Wj8-7}TxCtA~M`$%TEn-@f9NmfE~S_~pH{ z%7d%)UHiR`m(r;K(jPrQB ztAewe6hY{@L`&zB;C8^ksXGNu8XjtP(YO^z{GC3M8p7y`aW)lym!Q57m*1;TJ zuhs6E>c*@i`*K_x{&&!6cKPSO(X*!BPaUZ(*|1Pl@0t>m? zi;`qxp2SMKU^a)JcP@z#!STwKt1ul8+xj=n(ME?og)c z%{Ua2%Q(?PF{$(hda{IytnE0@gH!*xpB*7Q>QRyedKxu-HE~7dgz}Y+u zmwX-{AbVV!Jal`C|F@{s?s^{T0FA-_8Ehxz^#!2>4IeF4yf)w|HqR$_tv>m#M-N1M z3IE&RAiVU?Q;2_&+bz!NWZGPyemjK$wA)GD_#v5}34h5H2dObu9u-7h|E(y#e)&Fr zOlxu-UX&C*%OHwx@=FbyulHNjN)=)^5}%rCfME!_3}GUv`t7yzp9v#a!L}i1*bg?) zg_E%o%7mjqVHG9<^kABE;R>JT^wb=QoU$T$Pfz1W5)!q(63EczMQ^htCy=zl>2ab( zMl`Xt2}r#CVSMoNl--QLnU|OtZUsR^|isu!!d|u{P2j!UJ$_x< zjJA(ST42Xgs;G0rFC@hBv(@QO3S{331N0ATj&fdLj%T_~S~SvpXmL9xM^;dv{pBl{ z?cC0tJiu#5YFzBF9#n@j_MO5Ih{3Qa3U9kpmliAkhgD$K0AZBuja-Ay>i>+*ahbz23|>-;m_efmZw>T_teHWlEIIY=I={t0-;za1@y?H2+pQcUtjdcsy`Y-#B;6h zVk6yn7Pm$!JDQG1niR|{7~snlV5fv05LpTmdnB1Fz?B+`TWcxK;M$L+b%bH2_Uasih7n|HI+`8t)D^U#n-5=n4Z=zQyc_+d-@L8E3+5J25`A~XkKshcuKh|1r457 zxyaNM`5rQuLTZQBOD^i-Y6VUg6B6TkvXeg|l?~>qL5i38mW;1eo=wIvFG~VX*K+JC zF`vJIj=gM1f7Gxy@LJ_7UZ_~Vw5W0MfFI$TWmRS^ifY9xVMEf| z)sAM^=H2}K;I`a|?{^T>P2Pj-j5GzDbUXpsXqn6A$uu2|e*YkHe)V{Qsmt54mSMF%CXfj&424~kDuW&t4-7(W%Li zTQD$am7l41f0F5D>Ha&Vy6La%`!$2rSCxMf0bdF|$%6l2m~hXl_3*jB*`Oy{~led=5 zYG%shfddH|5fD=$K*tkrV_%XkrkpCC!F&#c&%r-^Leo%FH53JWb-}=uF z0=cjoM1-hU>2EO3U}$5tRchMtJ`NXGTJ?LMha~XVGZ7%E57e-G@(=ZvS;!b;?_m+S z|BXx`l8~=lzAx4F$qgtN@P)kpMRDaF4hv)pPFD6cGvTv3b!HHrtIcZYOYQGPP?B%= z3GHtouX86Yy-h!w>TZyo#JP_hkvW7?i694y`8YJ?e;_)g^aCdxPwLs;GXqo_r`2UD zu7!-Dn1LqtN1>z92L#LuIJF%fFMg4SuDY8b5LCPXLIVoHXV(~KrR zKe(oW64qbKl26gZs_`o1zJ8Q3fjiq(e71Y#e2qpy3|VKR|jdMVp$zJOl2Ok_stop5+*l6b;OU`oqBgQ6ijroiya756R30 zu;-YFw%Y>;?b_&XPXD~|JE4Z{blk<`tm7tHuM~}kDm5MIJEeK88mvdhh%fEA`Vzhr zqa@u}`i4`oMPti~z3@-c`=>aUM3)tSCq+yqBf}Ebf@ewuk^1){&i*PAy8+8`uDCjS z7H{fPQnt9Rb6t*XHJl|M-6tTx+gL8_!9x9Wp?o6zG0U<=S>Uu?2gpOt6Ln8e2qZT@ryDK}FR1%52)s9t-{m)b7A0Y_8eIC7k%naCkc1^~e@3SbpcK@5@)G?*WS9?G zCQw;2jjA_XZ70YG4cf0ZPjkO8(=`g%4&yZ5*A05(PjFLy>6+f^vduI`B>Cs9H!tM- zW3EqjeYndRggIaCebuyJIug>Sk3V!k1S7*9@Jvlt=M{cBwHTO2n@^bqHQ(oCl`-fe`HoL?nc8kQ`3`xSI!Ss|W zI!rV}J_U2uMTG1Jc25`>`B7B2KE#$PF$a7^VPpgfaa9Js2G*_Lm2Eb`87`9-Rz#0? zDq3;H&;C>U`6!=5QeT4Z-VqipZRVIq91#1YVAL);KD^jl zlJxrJsPlITt({H5H2ht^h4Mz-$Mz2(VEz%fjEsA7Ras=ReJ2 z{n#zt|31!LZ9VvclvfQpsPtJ5PI++F{FGP=%`tf4q$qyzaep=Y{Clr|Xg)udqeSJ| zPArhVLcn|Ys$E(msL0AHDgn_s)^ zmvz7(La@8J4>RRX1%kHT)cHhV<$$&K%V`dax`{B)olisCm|UmSWE+;3i`#Vvmdg@v zci;hNvBd1BR!Vugrczi zz}>0$>9HJ3nE$*bHXZGU@;;8dadBO9j~NA4cA=Z^Ucv040WC6mCFj#C&mxPksiN7+ znv0tbwP6uHHEJ(h*V0yu@hb>bc@@?^8wDpL)|NorcuzDSUPBttr9jAzmAw6o}_e-(fyPv$urJr^ZZ7wrtzL5PRo+#IcB0) z(^1cg5k&<9eDw#fSq{z3+oPF=NhL*Hj*Nhfqk^-{M}`LQ7*HZw?b<7(;@nNf23r@6 zO-e4!+g&DpV(hP`v#5|F&@e8^Et-7#>a&Y-+&_tc!~OV6he)%VVfQL7SG^sMQ9L_> zGtzw_wUNr@|A(o+4(n=(`v2igNQtzxAf3|PUD6%WrL;6i3W!K|gLJomC=Jq`0@97r z-M{&q`@Wy=!yo79b+NazC)TXBX3hK6cu7T**F;cY#Aodnh683xUYyoh=s3a1l=Td% zlfkNs#enBX;*8YgS5m8piW1=k&-L1=E=(wZdcp68%IS&83c7O9@yMa&00*|S9T>(8 zvEA1R5#IES`nO};DI)&BTw6F;FnUGdB@hsvyhv5Za!C)P)!H@r_Dn3KWQ=BBPdah+ z(T8TS`}>!(KdK2~ae6u6ExIz;;3VBX-M8!QGl>G91n}_&0?&lUB25SxLyLo!!xCxe zoy9%iTNg+uhdFd2KDI9DXBl-embxB_5~Th(TZtfYyr#9xwZI2+fG(q5H8wl-xj$c2 zA*AkHX-)~O1rvoOqZd-w3aR&tFN?sw!Ec8#2F%hubEgY}v=?lpl8D|X8h?zuo;VRje!dZOMXLu7R;)*YYg9z-dB_4?C6SLZ%X&o zWw#3qq3K)(r$blh4s9LjM)YkLaVJCBMC&F2?>AWP6S&;{g(-T0YvO$Jy`q>QKs>m7 z#fkOilgkS%uyhyg*i92F1X`#w!{Apk zI_#p}JQi%!r*n=L`XmgI(&TOAbC9TNK;dwtJH%4G(Bgy?3dLj?^%x!@6TKsQE{8wT zlg(%i-KZB)Zt%r*33{%y5ra~2?Pja9i|rNK1^ISoezXI!{fb~yS8g~fzak@n&Sb~_ zy#CK~vB)szi;@(6dmzn)3Taa@o~j9!DP+|}1|DMYIy?OwQMY91Nl?`EA9fqsRuLuk z_ZQyGvycJfz~4@&AWVNA05?vJasHp9!9k1ntAp&@nPpTYFuGO-st=yu=)5I%!=zu- zqbtre`BLLuRfXNiPzO54PWodX{M(F-;xAyXapL3Lr`7z7GCUT*1n&f4a%Gsj3d?2+z<(!IPmiA` zWY_zS(s`a3h*-uT2*dGXieXomkeMYVkS7Kop~ zYlQIJj^fS=FyEjXFc6SxkUZm&~(yTO+o3v z&z}%mS!B~6{-;sQ>Io5XUb1+qo08Tj_t?R?a&pCegU9uE!B`x&)t($Bmf|ybUHdZpp5ey4iwiM@K-BUW<+1oC+2tb(5i@ieGF@~&jsQ_+X#d>&HNUI}0bl5x z1l>+Qszg71>TJ2p6#iH4&z@+qNLWfa-JB}O&rv9-4OLsMQBWD`z+SwJn$A#U`XUJK z-IW|AWYSdM+1_)Yvi83bfG*czr6W!F6A<6Da(*tpSd|m~v_|+@EKV(p*O*oe><3&U z?1YjgDDp(Whg}3lvh#bZ+~!`E94-{Sq-MiNXe&qUS&bPGV+^e~s}%Qt1fiN91e=xP zyb%W!!MJbHzfxk-rs;!D#w;d9z&^u}REI3k1_%TQ)HkEHJgs%3P!)B}t6sFd**`#p z@=%wW|47LsAt}i8e~dj0P>XJO8HbYwPk-M2j^CSA0)<~3?|BYAd{>O@glh7WC(48b zQ#!VqVZ6Vhpkf|^1y}Rr`V$j!ocEA~+WVQ2;GvKw^}R7Pqoy)SaQZ=V{coRj9!hnn zF;2wB%WAXZ5e&)5T=K<8t7wmtlQGe=vU%Da@qeA&6sS#IAmJ}}nl9y>)gSChqKy~& z{WM^6vx}?Ev6=EIIq0uIb`s!_H26v3r2a%yXyIb=V=1s`{zmRdE@Lp(Z3=I~ZDvTs zVicmTQW->Lk((d!P&{IX^~NX3W4#Z}yK4v2n$Zgl-AK^K{qs|{XHUjbcm67cK;^Ys zsCgct&Lbaww1ZG6X?xJp$Ku^riv(t&TnsG{SIk^T!XS2P8qFL0jeu^-F~W9Mqi!eC zk1-`o2rtt*4TgWu2S3$WStofpwrE8?8;4``JG2oq&%7Ol5H96hSLn=OGQFN zE87j#%4zM_JLbMObL94Ls&)gTyb+lbD>;5-;UC$ziUfE1`^CU$#t^D*d{mRNMtP4K z`a>g=N30y5p-_kzD9&}bFLnzV{SIF;{o{k}n>n14yOJpR zdnp$)WKl+#^>Q^%i{AM~3sq_Wu{35R!T7_?RNG z&1QtSR`plWnIPxyH1qxZ#-?RhTH@NWQNf0?->JnN0~WfXR*#D56>dabnKPNf-^G`f z4)BZaP2ziTq!E%LR&SnWXWyJo>+AoxD>I({_k6zPCph)xO82qOHoREHwB>QT8+|-$ zK8v%`m@X3894ZQRDt<8m?0 z|JBlT{pG_gclsM9otZcOEmj&w!@LGSuz@@M3$Hz6^A6>3;mq=*$anj6D~-iSUV*(m zuJntCtKQ=*vf)CFgC<#PE@i_?>UhD!?~>SO2p5d<)=N)xEi#mvu(Y z+|*U}BmXzYY^%uu0|$13cRF56=M|3q<2zHEig>f&&@;_2u45&O{6V#@9~4daPb#G; zAqcvYJ`6i@rQdC58$17co~E?#|KM_HTV-$fUW3CiNO;MGZ-rA~^<)J@O=(`IvSG^$ zue!bZpu(zUwD3&YZFC=BnSprk>X%BcY}DI_4({dasE!ghI^ykxwgv6-2JYAHBBU@x1Um=?^sg$(5euyZgFf zKw-ELs6}L=g>a>7FMoK9ZrpfA$A$EV@X93O{)p&!s>gWN)se{1f#H>5?4~Nu@3FD7 ztMxRcR?bo8pL>kMg;Ca7yK}YDbtNb9*=JeqX;zb5p|x5be@q8PXYIT%FBY1pl?++* zng6yv`VO-k9~`t?E*8ygylE)Ecxp`8MxBB6ux^<=T^jdzn@PNQ*$LD~5WGsd}d{@gr z!TKL3FFZ5=uF8MZVhU*mwMjXlp_2Wt{JZJ&NJHfvkCF?$;K{Zqve5^nx{{k8 zfB)Fdyye+{+*Qf5w@*{DZN0Y-dVPEM>!Qk6$)#y&_swnQp`w1u$WmuT#o^sqQCbUl zK-1VG(@JAyS|=JPd8H}Q=)744Dh$JgT`I!d)hUfQxgo_7utay8W2QMUO%eZZ;beRpR z);LG{X3A=Q3L3C-ZqIp#Tpp4SoLFwT^PTZTva2xQ$hXC`FDY+M)7U>T{VGbq?|s?C zB%xvamhF?Vow!E(;9W)#T#S_M)Xu)!t7R?s=@WmS%ZP&myHz(#3@d|@y+M}28Cw+t zg9an*mZi*s*_DGu`3ehZYc8ur@A0<{1UN*4^GvBjGi4_G^L8!z-XCt=_jmbhg*7&- z=ME44kTqVXhmRl!P?(^Aoqr~ARV;eMLQE(ubJ5|oxw*e&&=$GtoSjw0>{@A>+u!H- zR!^hJ`Qz%7XVWWmmiw&VmDZk)j?VlQv$k$9!p~$es9N>C&CSeAQ+m8`esF?EuO?J8 z{`^)Vl18%y!%A)ccg=_5D=v&?gHs~6OOmAq-U&lbp{)IowMo(e8{4|`J(Q5z^U|Mt zrR7!A6LA&ET^6<0x5bBl80c^`l1VZ~>{z*eFK6$EL|C2X8;#maE66Z&^EYi1GHoQS zAid}+8{SeXN|`LurbKncq@bk`N@*f16|0^R^UL#lutd&9UGh!U&!ImyP!N;_zk# zSO}NV%;6kGtXi^xi-Sm4*S9=ge;mZCkWF~fWs$v$YWm{AR`8_yg2@=W@J*f7$e=(- zRav2xVG=oe)^n!Xb4Ht@XCDW_nnZcj1X11RHtvhG@d1WS>#7^ufrXxy} zA0Hg81dB%@t#Fey%JL)A8V({-U>OI;>}Pf{Og)a@7jN9NXD3_PUwb}8l1%=b(OhB{ z{R*dP=w;dP%<{sr#nEg!lGoSBO@u4WVOoyt?Ujet{zES41m#b*G>Rz~kgm)5bcl6k!JLJwb>=_Y z;bGn~{XE(^cQ0tGxf%R&T--6H<#U7dmnuSuEMm_oTu>&ySBZuope@;S&Yk0;4J20> zYi*k2cNewJfZ@GS@fTRgQ5f>dLFq`@t)*nZ4xN%X}JNRWWA9?Jv200a(mhGAICDo+A({tX>vpur-2UA)&Z;{IHyXh1jcC)$CYhMgN7QzX7waUr%0}mDZ z#l?Lywm&52c8je4YJr|RupHz{r`^|HulW9OGxdG-^tjL~9B4o?Z=VoUwzjsyN`8PK zPmVFCz&mgb!AUA>fW;W9ELK7=5{qjlkJ|Xi`)S<-T?YHGO1wg4;PGL zxpejEwrSo45h}-4^ukfwhp)4i3L}b=tYYe>W2>=$m1Rtx&To^24w%kRv54cDj8SC` zyeWN)mxGI?0>@mpapHC*H?tgs2oWQE-%hOwwKV!ui|S3?wOXxQ@sZr>$KdqA(5-v;Vf=fEXr4gsS&Cba%VNr_Dy`BxJ)ztIc0!(?-3g14DuVbh^N1^tFQ3&mO zICdf=7@Rn$G&r$t0tfEvanqjrFSF*+w0>0?_m(_o8sYs5);P4)cMXbF6@^j zi>s`i0ftx{oS1qCo0sbsMK}7G{x4u1!PTPiqxQUnOj4bVg%41}Y%_fRwb(>W4;>@ueKBOdLaG+|m#+^|aGvv#zfmMUEGoHjV@@z) zG&9J!86{B?ZtsrmRmR7Yx|c4NjvH(N66QvMrgl@2m#**iJ}J{()fpK{^o2=+kR})e zsHdEDYnZ>Bs*y|4S}?LjF&7Mue)6(sKwpv1plalZ)jtA08?G5`q+lA{`auZvCSuC>ADtF*tk+)hF%|GK{?fgbF7MV z8{Htu$gB*Kj7pRO5`1G%oXQzG)~nVQwIk_VqLMM!hOKZ{{w~QL~_`5h3V)`?9 z$*dRG?g^gNmZJDA6#Ns!he*{!+NL-%EK>4oQ1Acg0;p;&O+E>j`}Y<1j9L~X)WTDH zk@CnM4_^f2w2=E!>CqyfWtMi))8XatG8Z~#DGglpFMpj?dU&AH;Eo!~z1n&G{al?; znx1hD6<;1jAoFF!kbhl0QAzAKH~PY35847+{)AJxl(bYkx!CtsFIe=Rh>J0XbQCKW z;bvg?m~d++uQWPJ#Bh2|$HEow1(&^~t@y>2`bUkN1%6kFf?+4Ro%=Y+QY~PaU%UTQ zW!`0;nK9#3c-0Ey0MCAt_VM?L7_SBD$9A+`MB#oSkc*(S=I(SrLgGlJM-XvEV=+Kx zNcI)D~-=So~t#31@j!OnA(Gz}12u)FanpJ};`0pwdU-IT7>0Jw) zaVsG{0%1^sIu&-m9bPjPCvA%}WRPyvQ2SKPHur-n24|1hVytQRY?BleYuKAERH>&^^l;RH6KN5%4zw)y4#MYRb;Mw`kQ%wpifgOE0+J6v}tEL6B;3T z0^4zp!&+hbd6&H2ijd|(SC%`p6TY&X)w@-vwk?Yjb1wA4OQLg9@aPh0Cs>+~O~^<+ z5kqDH(o8`!TSm=7{T*ibt@~~ zLj4&DCvFTJ3i^7MbwDip)Wj3fC#zv@!=Ds6(#USxF7-5TtA|+iT?-Q^5!ZiDM|q+d z34Sy+QCn{LM)!x>Dqf;G ziH)&}5ocT{i;6PNQr$&V!}{hT8z`QUBDnXBjBK=cDfRs7^kT@w z$cb=!ii#npBpLIU37%Ju0sSS|tcKxsr1LkfMS^+Abrm5F$*d~U?h$t%XQ~-IYyDfQ zt1h1A1vC*3X0-Wy*>w*IEDmcNq+GrVcuCYd#~@`jdng0HoJHIP@}+ui!D5&^_*Nxj zNwb)48q=h{!phEU+|zCY(HHJ4+(TQdR*d(R zK|_|=S_bZJz~Q*Pxd9jsP_mz2cz=6|`Ro}G?n_WCVY`3{#jhWimNG6U2&97B4pF)E zgGsN_H3JmX5`dB>ayw|-u+`~Rp{LLT_+2H1@zS7F- z>T1z}a(1VvdqYpZd?x{940rQ5P?5d7>{z3#P^77{zq?CBOzg}@8lr*b2yRYD<*8N8 z{v8}mPr&2hOij$6xZ7IO5?p|_|V5`OmcxqN9wiTj5)Ki)IYZtwkEXoS;k@ve6> z6CRSHtg)SCf&R``&8jdP2nY!^JC2I<{JvefY%O7$o|>}YB8KS}$}0hp_5J;QaJ2vw zo5!aC%@5%k=tu;r!DBo<+7kzd;K#?5rzXrL=rV7V;93SF;OV?SL0=mnpjj9ulLr*p z{o4E|$bEpMIgNa2Sy{TKL$J%k#eyS(E`W^Kn`O3GZ~%=?WvD32yVTTdx6g^M+2ha% zBH>l8+@D`wUjF_2EgPY7ruy&dpFktJV%Vy#LciWdUBke@0NhvUT?dGe^r@-ezdJl-Ah$2tX&Csfx<;_4mr8dq~Jq;#%L?NPDSR;n?;N5w`8lMBejo zjPC5PGZ9A~%+(?5ht{4|R(R2a7r1!RgBi1YbaVtf^1Z!1aCwrT(SD(@bPwE{ z%6Ff+5RQoc_~j9RaC$rT;^L#{x8^a-x{#(G_aESLh}g@RZ681Z2>f@GoD0 zNOU2OG$nT1vS|GqRqxr#swxi^W>mo+wkpC2kq7hhp|!rZU+Zm0Z&YU{C)dxtZEbDo zxL*jkzBMuV3MQeC!s0AYftI6IH!>1A4`>6IJ~SD%3=9P8Q*I3$Tb@)%>M~N|6k4>!*Hgg2w0xJRhZZ$VsUkD^OL_hHYO1uTw6tfP_<1NC{TH0*-scr-hqcVT z>&No4!CdljYscTU$}HR&6TkGX*J`6~y-ULh>*i7d7r!E7T(XQxh$T`}3@PPjY^UaK+*KkDVa{ZoF z2*^L|>$QQ+*2DcsNLYo$4%4FJ>{pHE_xMkR-jI}LodFf`Q zXN6I@Sn)u?s?|Ttjw&A3#*)emIg!Q&4TQ`koAH;ZdiBz zx#3oC88XoNc#^HOABhqHNBU)ejl;|ai(0%$#W=#2C_U9W3e$3zc35&Q?05Zh>D0wr z!Fq<4)^(k|+(gfNtSIz|v(0kuJ}6SjJ9Y)e7CW`coWd42@X_vf<3Ht`ji3;xyw8Nw*6BU$B zwYcwd4C=z(Wsb>LRatqRo^5-nXYu+KY`M>7J*e{JIqtt(x|w@1Frk$8_XDN7y_Oa? z)zr7X`0+C%eH|U%^GNno#ogt54N(A2XlZRVa=}!I#(I`9e%>bxdAE%Z%Ia|Smj zgucU!J;;g>uP6duN1UbCM1VNJK)l%Yv7s!U+?`vO19f3vx*2*4*tS&-TM&8$7$=dfO`mK|~?Psb-9l3~$?$$zD zJ2mz=^_cOu6S!HJUR`W{pu5K-Gc=Zu_4kcQ?s+SR@C648=mwq)%VfuI{gZ6rnP0SnUq$)_tqqQp|d@I@VRNiLy^%VJ8Dk* zwx_#nq<~s)!+Awzr8b`BUXhurZ_$Bv-d{oPtFs$r-{*=yZ9fQQ0daS`zqU2HTps(m zxs;6y6-~CuiHQa3C3Zg>a_QsK24qBZ(03wHyxcW6azx89q)c&|Mj52m)D8ODWm}m` zWsw$TFfBFa>>B3w=V{~RU3~BT+9r30-04rRebW9K=;<*G)>>%*s?b?zZOLKZmEi~V z_^lhi%dV>f2O@$`4JS=g0;anV4wg)3`2DlSC4f7VBlf>t|<8 z*enc;!+&wyS7u`6-`8M3Zb=;BET^Bg=Iphj4XS4A?%)Y&N#)Vd(Cl~?@^f~+0cKW} z&aJ98&Qaz@?+Dcx``wX^&<#i02UENbLJMw}O-qT7^3l=u0l90u1|0#tm7{KBPsPolOCgtp zZOz-GNG4?kuBXed{QHJxsL^7yAS^^ew>R1p4CpCNlb4a(=3N^az0_qDkyqy=F|<)D zY@g6r9e$$s{NiKo;$If{E=V1^FEGfTr6E(~qHd5D`?>2aPAP46cy{K0X~8GrDc)&W zLnCJM`2fzMN{!P5B#4Xr=gs~+K~3G!ur}L1%yA<9*ORHU(&BC>7|{cfM~JhYNF-M` zpl?q}_8t{1LjeZsy%gi%;PcQIQS>f*h*>$FZDlR->i0(YPdV-6s$cPrP@hFE4{m!? zzF1X|d~|R^e}NPn3(1Vm2HQl)_Ig5n8R85Tidi4h<-6*6G6|(S&X_jHtNw;XSX^}XY@D;H70YC!&e6~a2A8wzgNR`1Q)m2b z32L?st?)V|aQmX!2Vx8z9eE}otv07B!AmGz zL&BRC{i9#Q9GgSjm~kp~2cddGyTCft(Ti+vyM~?+7cJacw2O^`4hfuhm;CnODgRW{ zFGVGfFRy%cC#d-SmMXjn?WrCXI%TCW0=aJF{wuzzE@aAT!YRvNKk5r~wIntX`qYY5 zFJ%m%4U#|7J+Tb%`FJX6!ep1(lAE%adDeLtuF z=EtX=r+q%F^{a8yd{xTCP3^IjJ}-Oo*}bVLdl6BXD$GMrrw){h8PAY-EXn*Nwh+LG zOsKE@@(_*4kLWp=ifzsB>fU>8??qhKi7n9wkE^vdT%!!t�%T5W#Kcz)9sn!JTGo zKaL}ZWI>M>^wHva^hAbnB(_);WI%yWOvtMOgr&2XkM{4^e9U7ZneW{EK>nB>P`o8|Ym`ZW1ER9~l*64yrpSKzaB}ZYhki746KX9`!V%&OB@p zLim?g;V;$Q%xl>|h7c4KGYe(*x3+4cwA*~Bhmatj4aAmdq#~xqRcz;fU_ryGqX|=) zZYe|I@JLQi&r+M?k>O+1O9AeWZz0rrg%;Pe3Hb!7O!tSqv+xqp|p;o;#h zdcjkc@9l?|*`LKgA7jNeuujBj(!}m+|1RYSUt<^HgFy9=m}8%%Q}ECjdffQd)>bH) z_f9`u_P@i!S)b1TXgT*c=e&!qliag!%sEoGXk|bT21gJkbjilSK}>duGUAv}Mwwye z3aQmo7cXa0atcmY#s0Sj1`D&T7xggO55(i}=L!bx0auv;1p(|E0O^>j2+$&8FROtM z+_>cZ7XbU_n!Qdzc^DJE5?!2R!Ci*jTY~D zP|Y71xjw}g!bH9+fyk`@(~*>zxCjG{7=5r;8{c43Lq`f)bf_vX|MUI*Izii&^vE?p zx?D_YsQ8|yFWx09)$ALC3gO;k zYz`OMw!mH3C31j6nI9W-0@ya?VW}dsh)kCee%M#0r>6sY*cG7442h?}8XiZW6Bk)S z1(go{!Wd68>I1TZZ1JgHr_MyA6m@QAvEvBH7@yZ74qf0?0RRH9kOpdn!dth3s@D zI+8j9q&-*arU(UC#9dy(nOuulacuJlNm$wQoJNR%02u%PS^%AL0ux5SiuOCpgLQus z010$G<8>O@<`9FekpB3lT3%;Du^jfQ_7ym2Wcona2_q=h>(m8|Z?9=j{|@Ye^=6{Z z!BUHX8Xg3K^AUjO+dcP&;fDZ06QJeJs(>;*>gRaaQu%=)rT<@LN+K0*_=&3M8vBo+ zEM|C4Wf}joJaV`z7`>{((f4UP(0HJO+HL?L>ERSyhQB+Fr~zOX&yhT8&@oBnAdQ-| z0|yETO>xknmj#}Vpupw0)2`p%Gnll9$at*DdNgZdVgg7a@ZTSSImJ#K31hwl0FMm- z0bcMqnM9fY_omXJ?%WmSaczf@DL&X`3ay=I%*)9OM=1P$oMc2V2nh*aT`|Z{0AL1; z8I=H^)XXu8Um6;kFB*X=@#2_YS`1Iixa!{$r`GA$2oZQ&Z zP{>N?dw)I)q71L?%wz|3yaE8StFCr(ko>O+h53vxA{yX|*$4!rNuWNQ!>5pm17_X4 zGYJ`miy7!*{_Tv(*;!D0;sJY^WAXC0cN#x-mmlw!0UiikG2iBqf{oR?dE9q#O=fs! z=QAf~IQ2iDet}mTJ$dR(LPFtU30?!^AI$Gx9Qs}UOiWCKu{l73259D^L)IDZd%z8K zw3WXA;`%349<~VBTS`<_YRTJe3?lnnp{uc%l%|k`DJc3U?Un5#O?}Uyucy}p?$QI= zVR?T32B2|V-yiyCS4J*?h}XHv55Nebv+C(!3*aC98~guXkPYF5Apd}4D|*WF!J&)F z0%N#PXaZb-&+^(zm49js@O6M7BmR}C1hyBfyV1{(Y2XOZf*P-)gs}0vVk4{wY7tg3P2XRH%r%g zZyg=|51O{EW|Lua@gv{%&#p+)Kb3WwY2lEVmKP9R`TcKbbqGwNp{6{#JQ!G1yYkI7 z;1aN;Dw5qU0~+f6fKIErj!vWhnvH}`V@@rhe2#}>B{GZ!hdoEyF&~=Vi5!LH5!&ojreu3O`B@pCy_5f|d zGUAnSYjb5K3oGmB`Y|<5*u%pEu&zLo;Nig!#-4%U;CLLD>}pgy-hf%Np}QYRn)KZc z%$g);XeAZ*3|s(H)`<$D+UVw+AxY6L*F6R}K#)~y0pc0t#$bN|oX>wCc{AYx>zfi4 z-#r+-3t-wNPKCk66^aeQ^8yLO{+9uX4U8=>;a38oe7uzQ9YkThW-E8zmfrOyARXh_ zd zz}5ua2y>A;4r0zX*pI729$1GGNrgSS9~6~#QmW^Tp8daVsLAEU3ko`{VKtat-_VtU zRd2%;*C*&9;zCOVVvMNbSYH&xbXsGjBmdj(se3P^(zJ?s#z8?gd^_&@C33{}6CbuQ zRPWk8P=5-0$waKwS7T!M>Ur%~`EALQ9%JdlL+iu#cEK5l-~u1`e*XCXsbRIWYSTGH zY`^vPqQvOYh25ok4ga5SA~*6cydnR;0@VrJ@WQy}HcaSrtLyF1d#B(Gy#IMg6Z`*j zUl8wqzc`bj8XO#Ta}Ca*{@;J)tFxQOckBS!LwfdW=qS*y+Z^TzaMvlygocLtUk@7l zZEHbMe3|#kkEEH>v+|YV@HP^=2SNbLyEom$ANr!XIYE zxqA;ApPFdja)8ObcLK;yNaV6<6z&R`2X8P%M@@ZT%X$qsh)Q5zvtg#;$s~df58!o7 z*{X$%EAyx}^3#s{-hTLSiwxEf&@TZKxYqX56=0!$uK!w1-$1bC%b zIT*zgm{Ge}8iO-XGjqSJr6*$h2Oo3As;pZ-f^{s1F~mo-|L| z=gj4o?sWtq4-O1O2{OXQ0Gs%KJSR}3EYkr=uLe3mPCrcQH~IJ^oGn0OK^~tN9SMfN zRX+nvB868c&s{<$8?LRd0uOpv8o+<>OK7k$&Z%aTx!r2Nuo$0ak zu3n$laS6s&1P<0&C!qNOm&E75C42JkIp876Xt)Ld*7PPiR!l( zV)X^TDfDybZGb(s@?Z<)oE^uGdtKM0eFIR3{7x(F*1!e~^k8MJtONVxORM+@jAzdP#1x=MLtHAagJIrFFTr!zZZAjrxq)^nq0g1cUJLSc z#H2?3C`Pw~Dxd!TKP!Lyi@?LfL*UEGpv8aU@uk7JjvUnD3&OF5CpHPgrtme) z6=<+u$WYz}=&kguSy(_tk*05EDCtbi31Ux7q!V(`G>5wF17@?B({=?=<0UsQA2#-Kr#|Ngm!@o_MKFo8@zkJohyG8k*U0PN9 zK9BVdX@&Fqy$oxdJlQ{l_U3CgVwe#>9f^n^2cYCbI?-#?fC>W`n+Y5h_hv-^{Mf^R zM;plJlaZ1#Gca6_g^GZZD{Jcy&RoRzz<~sli$z*Z8#wr0r&h_~g->ZgceIg4H2g{k zjzR?$8|HL7kupiL8e>?wFpm~spzc-aH4P1Yhq-4rHaB04+Vb-90Qz>i^_&9zVQ{?(HDV!#xYOU|+Xz1(;Sc&kXO#|EmRv2C>i5`=5)w*Yheu$HB}1 zk8&IhNP=*003s09VSb=BMy})1pzLNM{ttvrFU>>3+AT?ao3TgnAPa&=*$AU<9(M(zwNW!>I+V< zRKVpyMDg!Ooll=jzq7GPySW1qLS=Ka5V%HFTvcTZkY2E@rGlCPOakY)dKzyD^%VAo z9-~3NsY;AuMs%bk*;5LfxapxdwV;j_C%&|_v?$Rh3Dmn(8|8b9WKG08jJ?RB0M4DC1(GdRX+1l%!s6^syxj8pmT zgGo$eB&43v_D3*D-0rxFcbOCjU7Qf$-q6^-@?ZgLL}nhz$bi zT0hepQx1Mp%tLTM`0pR7@6Ne5IO(b78`*Lhc`n`Wh8O^tCyXI{EuG#mDA=4$?B;I)>ng$LSs~Zqz3EXFc0C^uEYbSk1 zBI3rgva(LF#Xf2=V(OO36RfPK}sigh-CV%eTLM<=rI}cX9ESZ;u z_VL>fz`T-sZD1Eodfo0f0s>v?Tvk%6F{7}E03#^|!jY%YI)M>@2CkD|oS!cOy+GRd zcR6%{T4TZVLZEU_{G!D#J*|aYjJW`Tr%^PluvpLwcY#VmXF)k0lqxw-?yi;a;zgM&I zKw#*q=Hha?Q?3F?yw9FJ1AoWP;|~VD&BjQUTpijpi+<^lw2;tHXefm8K)K`Egb^7N zvbsZ+2g?J%Wq`AAu#vv-T@3`l;tomreQ8DFCXWIx_* z?^VnL2vC~KN(Z5z@H=_AfUquGhs_vZ0(LicOwWMbrOMS%=9hjv2DSc#$(Tj}2-t-wD%69^(hRI9Yx7Z`tPt6# z67M1xOSVX2%VOh=^0CS-PAGos-Ll~9Lk9)%vWUOVcD88Sp*1CL`inZcCJC@F3`ClZINJ)JfI)Pewq?Gpc&lq1)2WJ<-3L?&+3VK({L<7l&Yop@*vo5s603)q5U zQU$as74;vHZ{(8Xu~6uC;G&ktyLLcAXA(Hxjkw0B=(esSHiow^H~aJQ;&#+$n_m?z z>hw9oK~`A^{gvO-a8jzfr~Bej$pBJPTkV%(p+qtg+r8p#&Pup?P@TIbnPL6rkNl(u z>BnVRnsNK0MdKYa@Ng3FY+{6@q#`f$>FSP_94O7)kx!qm3fDvN5P_bGnm1YI?ZAg~ zq50GUgVw)4VsHQv4x%rAQKKFCUGf|Ey!G*e-++}oR`I+Ei!H>5&&MI)*0Om;uO=79 zPD!l>1;Fo(pd)UwVZVAm@?8BL88J1bwo%8>7OZuKFD7Hvq}2dkJNjP9*5(L#?aK4? z3TPniJ{tbBlLG%xiVug5d+$P6@r=Hj)yZ2!)et?GpP`RmPF$-@$7tiTA#`0rgU|}B$2B}dXScyClCgdEel3DfO|}%-3lB1xa9R^wuaMF-bDq^>^l~J4KT@u(t*^Z^ z0Ts5G3n+~e)~&F%^>cd*smZQuoE{1#>>4!PQh8JE%bl}$UzZvdJz{VFg@b7IZEFQf zsR?U-jWqeZjVV%>>UR1_W+Jr0b+5_A7{=l<;%~oM_4#d7UW5JBgEx>H*lpcPuIfGp zosF6}!Eqap9QL-drKcj*lTA!YQ(s2gHjoK{9t|Z=VU9(K5?EX=YM7O~$HDRa4Jy_P>{wPKZ);zIB zT)Y7TBo?djy%dvquQ5RfjfHH7j&kcfcF_p*>zCefN0S@i6bPfbwSQhde@OqV-Gpx^ zu-|AdWZzonusq*3?^`>^%biEW6NDkqYB9d!6CpWz1T_D37B&hMmy#^BRF(0wwQPVdp==$;M79e(nkP zUkgj052j_qz&bGh#x!2&G)=3&WpJ-=jET+O82aWwV)lt@29`{CjMBIA(^u)171X&0 zL+zkn<+%<2Yg?!o&r~uIb*^+2QjVzD7m3_%#WjiP`n48#%Z6%!d zrUy0^HbOBYuEugv>^(`pNpu}Cv11p-jf`{t!qfYqLdAtiK6uOTN5I=*Kc2GM0=` z@*9RYaXWX0v}{hwFIi3%Sd^P-mX2l6aYNz7j<0+VaYgyhR$3OLw+8dPaoobA21`fo zjC#q*1g;3l(Q$x{Ho>81}1kd#1VtB3@|&YJUK70 z8m*LCcyzlz{lM>~sgg?*w>|NN)6e;cBCtDpOkpiUucExXxjGSHhzkr>rdX%C&bsrg zPb{h$%`wQJ-sBHK(B!;tOY}1&Txy zW#WfrH^xWzZ{^uu%qQ8r;=L77|IojEd7tpPymTz3wlu%G*12{KQMlcrD^`tgD>mbA z%izGUNN0Br#CGr7-gHnnV#7&Ln`UR#<#qNc?NWqffzftAsqXRi^wwWNs{4B4dp(1L zAm^TW0iQUgDuj}EL|bPf@T<Ld1!Ci&4O|BP-|Z)ZLY_{Ntvh_i`YZBoe^)P6*73^YwS=~!=|>g z&*It-apHB-6BctNXRI0=#Qh^a-akfIFh}N!xIu&_zsxTwWN<1IpnH@5UV1xDNEauE zUQ96o(A7m{-U^Pk6*Y0)nvvFZAuL2DpCkyMm}87LjQoF0eRW(^PxSU0h>0v9A*D-$ zNO!3$UDDm%-Ki1_UqZS;YH5&`2I=k=R=T_U9sIq&_w(`xuy^mBJ9FmDnVB=sc}}f` z!NMR=2zfQ+UGs>_TB=}7daLLGhdH0}yqr=v-Hm@P1zc`OYUm`eU%?^N4}E1j&0Q8g zi|~=*$}TYYSL!Z&hQz(^-Srd*1i9_{(S$(-Zhz?)#bevT(&b_Os9h6A`;SW(56BBN zm7D&J5n0Zl>XT`CzW;iAGi8XTV}2VdLKulTlijWBD9LPKK~FZMMdU`;4!6fBrT^1A zd12klVN1C;zsf&iq8F>7)d>n=^p)kTVJW#D~c*pLjjRNco!kc3L8Kw~u+}v!CCSnQFng)00>i z1F?l*1m_UnO_SoG)6K7SN*0z^HD*2pPgxk7ORz*$mMr!XG-mWK7sU^<_9}t3zaR$0 z!}4%BDQDyb2J#Z8C}mUqxG(fF{zIz9pP`FCcKJxsCx&DJx;y*m{AD>*iBaBeduiL| z-r7IY?-{WZ5%?|2>(gkjY8v4cK2_!rIo%4XB5t@Kh5Yyq6ZbH;K(PoW0+Yk6tcFF5 zGHXn=MLzNPtWcyjeKB4XL|STOZN?q9+;Zqwq`jq4 zZ~Y6ap=Nr$j>LV&vbVv%0~5{j9Rou-(%nbYPL>Bl+lu_T%@u+1_F2HYTZA+eIW!6< zAs%8~jh8-B4^g)Z<~0`lYsd`?v{N)pLWDDWe;J(?&--NEmvI?^{;Yx%!b0?$Iw|UD znKcO8LyAgBN?C;w<40sjB!$3QmPsLuH=L{YxJFa9o^o$2@3~Jm7&+A4Yn8wQ)kF|HhO9^HMyhMW}feE9Ft&~1kfcRkO;z-8`mKUDVqX?|| zqIPz{HdEX#y0+`R8dw@Ya9;Z52Mk|ZcI||_2J%JNR~F`ttI;gQw;!VSpZ3jN_(YS- z1#}1*9VV7LYkv&twsh)=ZHPG^FBDlBdSRI@)n5u}qUQXHw)nx|H}gbYR#E$WpkT|) zqtdibcxaHeH)%+)thr%1n=W!u7w=(gr=_RJcEw8t>jUG$?pS!BNuYa?;!hIk^rSv( zX3Z-rBOVXkoEFpb;R(^#{3+d>h0gDj)(r1K1UZ?XE+YHYmT|TdGw2DV=J2QZ2AIrK zEOGT+8)+escFd@j?c9``-aV-uENo*{IzkIJ_944>49Y1kQKc;bvQt+^>%I0Wt`uDd zi32vA^x79{D~VMRwAkU5_A{Ie5C{>WuPnX{1?6dc!z=Z4cc4HGt<4A`?VSW`te^4s zno5Qdr^6dI{`=p_gt&#yrPM~i{x%+zHcbSAz zU&(pcX}@S%-B*O~N*gG4$&Mv<=FC;PYFVNxT#Ol!2VtbK>3rHwN;bY#;4Bfky?_0L z?O4ubMou{cC8u?#?Lk4*`(T~w?DipbNJ=jqCCA^f;w(nRf0BKcl0O;os+A3#L3Rs_ z%O_j)6guz07kmYtL8Kr3rk0F)ZC))=Y8_NgUU3w&Hurin`i!XF>D8>AyoRQXTE0ZZdiOd6?;mfUZ6TFf{Pd^_Wbwz?1M2xLaz;3iFQaWOM zLNGy?5<6N^vi0&o)7d9{zb<`j`fT^uo!}}G$zw<3M|x&<6}1GQr*ZOq6E;MTNyJa- zG$>&0Dt=QO9V-khu2g&S`m#YC2!!G}fw1}gI1Ef-UAZT%G2Ddp5%$J)^m*hdFY8PG zur-2Z`CX6lzUUePJ*sVFV~~SX9q)eR9wNnw1eHlbntIm5TmO)(v{tM`Mjrg8#`%z3 z(M`t1A*3d!W+Y#3SX5Zq#Y70|BI&%y?m8T}@P%K_0PCTWD%%rMbjad;U7V*aWLyJ# z^eC)Z!l;(Q+@iPym3b{j463E${5Yl&=(jjyxcx)HbUT6z^OzHfs- zs0h6h!~cY#Gu`AYfD^3Bn7pA`PBk}`u<_JD5X{$wb{y3dB4F|Dv&d+6K>j5TrN%#@ z$8pqYG2$fSEHp-$yG^(B(G{a30RucGydiyn+ic@ z{^^S3jj#{x)7GbF81-o6OK5MClsPg)HN&&4 zH-fqkn#35BcypVTcGkr()8p5zF+Pqy#-oldG|bH=Q>Z(P=0}< zcJb4%%!lw$k;jaFj#&=|&{1o_$G<2#uoAp$oC8~1zbY{2Mp6+AkUZIZ@Sj{(s*Nz3 z&TF7JxrU*AyD>z-;IOEjRCK-v`o;p5fQ8!pHm4Hv+ZXg7;c#N$(?Bt9)^ki~Z{tu| zdrA1^^TmIuRFJva`z&>_B*qpKl*{aykazHb6v`A9%a!a`^avvT5HyY}i*kUVVL1^- zuS+qGwTtN7H!M1rUrv0f1G8PIs<^q>v^-^>ymF>LJ5nuL)|yI5m(0e$(%jY8wr)KA zazk}nadjDT^5pgzeQp*{q$fIyXDMBG@E(TKGXA7)a`0n(Dj%-Mn7X50 z$};Z6I6D#fL^!(DY~Xhn@0*98F{`SsS57KX9%ke2UW~WGWjfsk%q1$l^xO#^_6*c~ z$MR)!_EVJ%DYfVFgfiECtsy8=PS;_gL~q*uKZ%kgBr4?*VrSeB;)-m_pCGBf_<~7h zx#%2*yRxDUm`PF{t_DVxTcH<8GJ1hj8VGct$oyzIupTEwsPYa3a1 zQj5aJqcM0n%9oe-PUa-PV>=@%h5G#u#bq>?EaHO1qU`4;1mOl%EW6|wg)f0o@D~3a zrS$)A7>56C7?Lz|=Jg?zqqmhedHrhV-_AKg_mliWxwI26!w0tNH;+wvUOOND^03-j z5(D!IQ6d<`+|j3OBt(DXS()1NuKbG7Akt4mgMTZm{_p+)W+vrW|9UQ-i?`$$9 zH_{F(%Od8V*No0MCf9is7ZQpNUc;(`E1B(ryCXv~oyyBfEu!_@T7B8-K%9!RTvwTU z@AS*67RmHn=ZTjB$PI3DZh6^U)p29xZZXr_Ey=px=|hJ1C^Hk zd=1gx1n#VG6f;rbC%dhozY+{DEH@>Oswp~te{SLFk=90E6iQzYsM4Kz~KX2j;{0bVLly8}-gpND~Q3vH!Q^>fqdv|BRvi zYncV?a`?G64+i=h@`C^DyJ+AD)pneRHQ8&MUN6Td@_V7uZ|l=Hwz2DRAk`WX$%u6E z^gmnIMl+tM^=@i zIu8t^KMF#dyA;5o#)J;!Mg8b;24f#5e1?0sbi3pM7~@3_Z;<55)rp~Q`Nh4_J(Zre z+6R9Z>2`Uu4cMSrBc}CTu(+Zew@aSx@do3|SpJ`zCtz+Esh6PiU<#5@2^8So82fbF+hp2DG z^(0gED;1kJrCWrF`tH)FL?cSwT%-!H?Oc-UPCBU)F!%V3qeSw=(_4WN_)r4O;<||5 z0*7*ACGG;Un1g5|ik_DQn+fkxzN43W3y#kM@V)@ex$Pi4vp zPjHEMutAYW?e&dy6)JYT^gfexE8>R8W1|OshkvAM!B~oe!HFHGrhQ$vq)a!o&-?I( zG+@akzRMl8P^_ikRKwq&LmkAQYW;@rrnjN?X7F;ir^-`oak}cY0MiK}hD=J-R%kxW zHMD9er%F*Ktt&E=RECloi}KFfrKd|Ng63n(p;=4r!)-pMwu|+jPPf^C`Mf5W3q1yw zS{rBfFQ19a)R!;xGR+NAOf_=s`X}#KihN^``cN-Kok!k3?QmUK*eGtU3X6EyMaR9J z9)iFgdX~naRPVM;_CO`R!>rKVRy5R=Vl{Nc+go;O{o)%9ea1_EZL5xidthw_(r8G8U+*mMvqaAgItimi9`AD06rQHRqyvCNyK8$4{WRTK zj*WfK#GgHWFoSC6=6;?SGv|^~(dt6JDup`pM^aaZNjP104|vb*Q2Hc_+Jc-e8#qaIjM0^!5x6DrSbWifM85QBn$_`0^M@1tFBTxjqGP}- z<~2T}h1zi&LlCaS<l-sMyValfc!)G1bUXAIhts!jsvRCiCT$3f)9NM#3x1M;$g z#rmuxz?t7;m#}eW{^`pE7AK)RiJ1vMy1A)c=N!3_pylYfhyZjg7v*z;4F3Gp>eZSQ z*Y=-v7ymI7saSe6Iig5!@#m@rewyxg2v`*}^AG%5u0<(Ebp7c-6H~j6F2-*l=VL9u z?hs(Nup67;&~`7>9r+tdTi^=Jki)E1pu}KA=@b*HXjq^m zwN*~}Ns=-hE3$T6ZYaWJ!0vpwFx7i|)Lp4M(dl|aIdI3Sg+t?JY9K*VSoytO!#-fS z@(7$-RK1BGz-MoJNj&X2z~?VT@gFW*!)d+0lqcX4+e6*5TjcTFUgeF*a9D8f4GIeK zIKO76zZwikDl;Nrx2)YPlH^RfJj=-D7sR5;+>#=yyBtLA?p(Nkz_9HP?EEd+2VGsA zNzjfTYZn(=o_6+jZSlJ&0Oh*65hA_LxxvVwlo5A6HA;1RGore-!-zMhC0HQ+k6?J`n#?V-;gS!#wDGRKP=oTid0 zgzTsv^e1xMZsflZFwyCWISC-8gefW-su~5WVmQ;V0jj#lY&!E>K+|>LVEC-4x}C+j zNOvSO^_KV5V|@0)tKWF!`OR!XYV5V^>L_N9z={Hk^=y8_NgPXSls(GRU&^p2mP5lW zS66=EaKM@Gx?ETOng^v?Vick~(jah6Q|DZ)J5qOhPHM;bJaO)9?sN$bJGG@0GY7AW z|7%Nqq>@GJUMzksefHuZ!|f2)^HmAjj*lSg#JJLR5#p3a2%ZZ_Jow5aKAT#iVx8AMmw_nJvd#bMdt$DE{ z|4E$gNWtMiasNnk1^>~YPD)Y1x%W*6n|r11NG0htHTF{i_QLZG<*EIoTIvJFJmd)6 z;cr?1{%S^ocdXUtNlDbNtktJ#y!+xqen^&)YjUqC<` zpYLqfTSQI$MWWIlk1}2SF@y8J54G;bV?em?}Te9nWLaLsw_7dr2r)BCS%x zPdr(3-1O}we?@o_?EGg~(CBw+qgO*O zPcy$2b``~OUvUI$0!m@!z;C&+yY7GX7$6@b4jc`^&|Fb}wM?hH@+@O^fPbdZHbYnb zuwnvywDi7Av^(3_CX44wk=2bdX7*qdg{yXzbI5G%rP>OEs&=8Gr)X(Gmo}~Ro!_V> zI?lPC|6WU@N_o7CvYsrzDodqkVa%gG@3tR^k7Nj@yu3WD7w<6wGX8v1)H^X6#Snbb zaHa?J<2C!3rdw_l)t$6$+)SuXZJQ0c@r0M*uw9PzJVhXjtG~@2#mv6Qb)PDl|fk2MC9Hxeb4UDv_x8o`QYGdMpYN*px)zkzMJe@XACS=EF zAJg`*1s|(*1sgOPE_$?u@jW9`bnZ0mG|WU9+^Rf9N4E(Ab$)h%I_@gYogW_7ch3*- zd=y3bv=fCj5 zN@Jpz-4%~(98=nbTPWWS6pVZ&UtjK{eEri%UGqtZ%bLPCzP*Aws`s8Qg*5z&#agMQ zcCXoijJQDE(Y~gJ>?B9LI<@N>^IqP-5ktP6@`oB$>7*%j?M)T&kCJU#<%%X?Gr=hD} zTvX8tFmYKmoli2{XH{Q}31xWtWaahG{Pj{4Tk(68Q`%O+uc^hZ(>#xlL5M$P#;fkp z?)^y~#k86=QRWG@esguJ;n&6~@m=TsUba>*Wmf?pLEGt8f9Ydjpiy(CnOO|oMYA-_ z?f7|^VECn3Yt6Iqp8H|!XDLJ;vo;5rX?db^MFm{DTT&^mhtfaj@>R+xQ4AE?a&Ob8 z5Bo<-AK9yja@D8u|D-NcA+Oqfcwua8S$IP$HF89jVRX26t$toZ45L*o7Zs?FF%K>( z*f<(FyzWZ2#xqKfvrgNJjy~t_R8vc+QXa5B`-b?V?_hY|h})xF#L5f%05__daUp$^ z2SAWqxILa8!EvxD3)jui!X4fDbcxgxs|=_X}UPc?f+Unyd#FStjVB_K8m$NECuJl9$S(6#IIPL+*S zX*q`)W9O!Hjz{JdX&--`wW5NBn5SA@4;8!Lj&0>IaJZ6cAGG{_--9y|v8GYItR~!Y z-r1Vz9Gg64lG{gH+N2`h$^Y7;n*PuTsl66g=DKYCRB^l9X7LM;_t@oLCoSJZty{JJ zT-iMR1G`Dugv705smhwvlPk8|tiaCqYOkbRb=w$fMjp6{z7pM2?^#i~y1T(k$uBcr(b)Ce1snYOU z!Lh`1o#4my67ufH+0dQ&OKZ7~O<>Pk&> z?PUwCzbRx)LvPh4l#91SkLTu25*uIeMt=4$R?u%%s=@PcI#E^j5f&lVOiMg-5%fx6`;ri291l+pnrtaxqX(DBb?_*SS8-vwpL|O+EU9Bv}#DfjezBYDugk4DgNRN0W=0 znDCy@Cmjue@X4k}Tv|WHJ2aUJ<^F4f5zc0EiiZct#>KBpn`x#mhO7b91po+;Wf|fW z!K2PAtQaU z?;f&mNFf!glytuIcT;)$7h4_%Dv-k755Za#lH|soX%j6!x7Hm<_veh(J-M6BviEeN zr&DB6qw$#5+BG3BB)ov2Yg%Qu*iNyh%;D5Y;1j&nw5WbX@aA?JcUA^o`c4G91Yd?b zSsJFcY<4kUM<_6AhYk&?j%xcf0-){C@D{NhQ*Xp$$iYW^zjQ~Jg~4mpBFz1Xbn8&M zRxUgOmPoO;ij*N39nbve3cIXo<4!|6)3dYr{PJlh6pFevWUG-TdE$8mG>^h~q|60e?gKkB|qXBxk;;Hzx>fk95Y#m_>A1FvCAO_g!MQiQ%p#nZP=iQIw8+) zyN^{Y0n}AlaTrHR@mF3%o}JVLxOF>Yw`X?l`EW){%fT~=Y?U4uFR@t}$p-+SkTq{Ia7 z=*BV+It&qvqU>RDdokde&J2X}<_Jy8@jX>W!#d)O9(*o5pTg>vF#2tgDN9UF(d&=t z0e+w)W;`{MtX_KO!*F=yn?Ok$vCy|%!ofFKE^c4+Pddy|^vtqdodh)WLFp5l6zP=; z10BxZB4pRr5bo?&9W6*CqY@rDvn=(7x_xN*AlItj)_7cui$mMXxvJ|P`mRu!opXLD z*F*CdI%Z{jCh04HL$J2KS{sfhH25m^-T$_koKN*Bd0bp3=_(78=@QE5BUa~G zC(5Ivpx=QM9H|#q&0Zc2uBdMmtM01kL4{;-+?ktE$NVSX`UMQx3=RHD)|&8^@~yq6 z=R5REE3~CxLupO8whJ@>zs03H$C1&aV}TG!N#1;eW%}cZx_X1Y0=zjP_=ro;xqefN zV3atP{-=t4_!G!yN#KeDFNY;>qPaV$O zO;KU0ek*STSuV0H;IO>)p`Ser>(EyXpJ|34r0a`NE+XN+4CGnS^zNwJsSFh82 zA7rF9kf)6X*hI2)zh`(GE#aXE;fw$b;s|on=C5cOb2cU>IaO5?MrCYEy%OUn6zTIv zn(vi0`-9B@rB%3SCitnTaZ`|YcXwxwFDNK@*AuT{b$fFS*iMkh_CyxVNx(10wR(Jf zoPjj{tmrt9!Ut!yo~sM*+HqKSsHv!kMgUGQk&v3#|FvO|@KOiq$@d$$rG3uH&h9X) z=Q>a!=BRDGj!|spLe;x-KvQ|D<}Su*SV1U1V6O@^0g@k9&nA*2#{vlP;1WNs z!vjzdc*4EE}a-W>jsDbeyX&$Ahx5vIZx(OU5jFUYeFO1tK}^8U;^G0DiJi?&LLwrE+SxL@6;D2g zwa=`tL4bN;cRYzru%~lw~aD`-t zeJ$x{8nJ3l!$vBIXKUF+u(lHx5u!0FdVb4wI6xceiGp@K4!aH?W`5{$*Mj_a2X?}vnhWO((J zzC*u%57s>!0_fJ>U@|85RX47HrRn2`8t;{Nin&O_z}GGDkJ5*7Zv3mI=0VLdKr;A0 z+J}Gt>d;1a&;AJn%wr!$SOH-fZLnDal0m`BH7I>es(RLeJ@LiE`x)b5@1w=h?%&HE ziN6eq79|HUxn``1mhW&uj8nwWyWVh70~3?Zd7C9KeGr#Gffe{?Cm8Xy`A0f+HBVP3 z;A@`w&vap_}U#GtTKh0M!yleYq?X*c{2 z`7=k>KfKW?o3d*xIg;Yz;KP*VB}t9VA%|Vlw(GFNTVtwMRe#Y$1Tp&+;BJ0o8w3D! zej%A>#K&o$sb7b@4xQ)a>4LCtBz_9*4F@rL^~aCz4-SIY#|kgE?VL_U^R5W_*Kr zq9pQ9G7}L-z^61>0tpUQ)+kBJ7qz82j5&E5;hF>>Jm6iPsajP*ce9SC^Z-j~UG)=n zZLO9R5pAu2mhel7(O59f|ICVOYYmxrjM#AUva%X*X6MXBpL;a?Va#(jR8&;VP#Wz- zrd%%bmFBK`k*;f2>AGGZtyEGPa#zQN17atB79-%WW}QJB{taOH!a5EuH-Kda!TTJ+ z8_%K>gM?!^s^o`bokz7bxIJA(mR*t?>PW(ho^*A1mh_O5*2_ zfM(x~OPO@<&sN=}=eRxF-JyRfMCdCTatTj(FOd9B$}uwzqfrsNr-|ZO!j6XAd|693 zuVQ7uw!kwxuc)YhRufQu?bz}p11iYNYk$6kmiayKmoR5$8$_4;dzK8W_Hek$sj9}$ zT5(*lDJv-%m8rJOfmtfSKBe~U)2}~(>Kr6UQr6Qr%T%frdNrbcij>W4gTFTaIag~` zo2slHyAkO7b0IS`Tg@t7zX#M;)g6mW8gsyWN{zIu%>B_G?>qnn?YbqEffGkJr>^ee zt@=ytZwy%#?;OLuoTbV#Zi>)Sy=2GgKw1up%f)G`c;_KAo!==C+n_6M#Bqx;NVIz0A-uhe|!$a)fg2r*zvNi3qbF zYv^y?8R&~5OO@)0%{~q~NmJ9@J|h+nMIj*}_C&TnF@U}jRcdbjdUuloHndG+{c|X; z_QclF@ZumvMLEXW_0Z7}Ot0~h?Rzis^Yc6AD(qvGBQiaD00SFfp3`+21?{&;17n`A zuw!vPiO_0*f`S4N+yzs%Smu_N9=Oz&hA;PNzV~$PsG=KTrAK{=mqdEhRszPgDs#FZ zy)#v+!RV0v{0WZZE-@Mq|LauChbl^*iC#a7())$>TF;-L+61gP>K2K2Cx6-M+e)I- zibd%D{#L?P=i=&~7l5ctazZ_w@PLV%#!sL-JHp*~DLaC?{VtZBX(w<79)ET%$bhUzZJzxko{gPp2Tvs?OH(YYFi`)Fy{eEcF(9&_ScLCej z@o_cjrP^jOINl8}On;-Hb;BMg=Dp*6klNGWZhKzyX^hL(#haA$X~dhyFG8?clOp+Z zqrFxa;B~jpS(%-)=3VJI@3obSd!Ll00xnf`t7^lo>{sXEvJjZsR#JX;)*(CU>>{@@ zUjK0BVGm7_-!s)@a2S{$2KJk?KUXi^u;_Q<8#;QjH;c{$cNc-P8zA14m9_EZ^@>GK ze4~B+#LBkE@@J(60_G;8O{jz$&2C^3Yb8G)Bfa0<_E#qg99Me^oDU~~d?XjdVQ5;W zf#t4JMTghsucYc-QDGb%CpR`XA#L&S5mSB~5MdMgMd+SU%(`bC=3kYhZS~w+{m-EV zF5BN4H$5#zb&}k%(HZ_6tZO@5Yu*m&GjldR%1}W&%%Jn)oKb5-Tvwk4ihM1OO5oj0 zmPoKD?!WaGicg{x4avG1lizWbS>~#$ymq_k*}e3-tv;+cIP&~Hhs`g8q%Lz9DwLFGo-**!O<~l4wt3&ni$jY7?~HNA+x)B zz+>I$x+^oVU-(Xc2WMbs5_RL|a2npiR9|*<^mUB8a8#9!Bjq9y?3r#Jc*=;ApBM*R zoc(Be=y`Ht*L7HymFjLx5V5wuFW!HNce_h}yzBkrHWx3T(n@N_Iq<}BE(`U(@uoDW z>eVKb5y|U?yGkGd_+!pZ@?Dg7DeM!c1(P-HYt`KQxTUwX%GBEJVqC?FO3kv^(v9)zA6K z-hI`TgsNUPnZ~}Nvq}Yjnd3*ljHd%Uz{Y?P+lFbJUii%!Z=aIww-J5|tQmH}79f*$Tv!n3z?{6L}wlAJM zXp;0X7uGc@35-F#zjzAG8qOUbOq2UByXIkR?mZlcOcpiKU96gruzeS#>G_vyQX^{N zf^G#E^SAg1_jH(#jxkqzjr6tt>_1}6n9n4KKqT4u@t4+dpt8}a!A6AWKJgE9aX|1y z>^pf7=~Pf23@EF82wwiX_G))?{(m5HI5oHsYF=M}K9A!C{Nwxh&=PThmZo$F0AfHO zks>2t? z5AEN5j|!_2IjQJ?+sdcv13nsv!0JFpr%tAXrPVxo@ZbHWa733eCu?Ym7yJ zi1VcWe=mprdpQot+9HUt5q))cXYkwL`;ewqQA&|VxP}o5Yi0n132^V=;U=!#&9|`9 zDW;#)kYM0P09aMhml%7J3HaDoYtNu=f9^PvHlL~t(w zc)GvK!pLy>-@V-b?v>#sS_tHXme2|q-8}Tk`1hjy3|ihn5qR&WKF0HRPaz@!cWqqs zKRpf4i=bw@yVUXMUy0~4l>avb6NDl4-1AxgpGviKHl;hESM%NZZICbiy#}3^^F#de z|3xezvF|Q&s`5q>07zEdS5*M}fDoboyJ+fp2jNsv6;n-p*2!WFE}snP7Fj0{~mf0lpgZ@otZOE&1*=U*IifrZ8g2F&&(qiE2t|StbjC? z``$HziWZt};)_tBxa()^|G5MoDW4;O!2%+I(m^<7?-axa1|sh4Vjh0XX$R_Mv+%FL zGsAyJa0LZlC0FkHZUq0X?}(`SD?qR%>_5L~y}RdW{yp#E;)0M9)0508qH&;yFmC_r z(*4gk{Fra$`Zg1^&<2|&j^j*b{pTC;=vn0#}m zi*Xn3Vk3Z@^C0L`?0tyPV-Ci9kPlj52$oQ@yV9s{bw`8B67hvx{E7AgFwhU`R?Gmad>wl?%_XO z96Hnao-GfDE8YzaO5D4q*G2y`6AEhWcWNUEVk#gYzH1KE!WJS)dGGGg)_;;EgOl;% zlEIYLsr+{zL*i`KOdycIj8Y>Pmk5MwU9TUQT6LKJHIT8eF%~Bt)KCEz*qAt)J5l%` z5W_S^(^xx2pZ zbpZI#Zyy0Z)Ua2N9*CE*>`kyVmzqeT2ddZw=JifSDb-!v^@tQ(6AOYx;c(fW^CA5aGq`J7qndqKdxdrA#SRs#%NZdiN3%6qF~YF?|_(t3z`f_67C}JIV>h^0P)fie|1$=EP{XU%S(_OK+-AAk0nB6 zs=m=+*S)daJ>X!=8`Smh(th*3ZX*Pf(l*1L05V=3Bn-VEWbaMkQt~#w-vmyvxKAUD#{*kh%>P>h5jYEtbxUwNlTAMvDVM7FxuhqazT} zF_BYpcjI4}ZZcd}Q4w%|y{}nICnMFUbJ+*ZK)}X9ODi1a2r?0@?z<=`0ABVZNr{&)!~$!?WF~F_94yVY7Rt@d^s$B9&0XvnUfPr59mL=~ zE%8f!xsEO?&QsfdK9Yg(mm30r4QSbaXl`!Kg8g#-r8g&Fx(F*!DOngvy(u3!jgd&^ zzZePwwRc|X7#@~`y(S{U@m*)JYASHsELA+3Y?RxFuaqCd4ZPO5dSXJv7w_-uxUcMx zQWal8oH_jle_RioVy||UF$_LJf2Milkd?RbW_<3~Z(3sK#RhRzRv5++cKAl|vY_DN zS^o%H<*;qG>1(C+Gulwa1o12bRsM~Ur?|oyL)6fz?GrN6QE$L?Q9S@CLza`gm&izg zH!lQfI>39_3%HuR?`UPbF1I7)_cC<@f2pZT^(e(_yCfif<*8|TNo&1l3=%l}T7kqb+ zB%UQF!kMxAd zw(mBxt_O_b^Xy`|qpZc8IHlY%Gew-`92q(l{4ts0%?=TfRy`&dEe%bVr`OImkYb|e zMMg&dZepJi3`@xNwV$k7U9hG@VCj5nL5;BYhrQ^>>7wvprlI9wNu&2Y8sA4j&BS}8 zl)b+9)mcYHW?N$gGeOO*lY+MlRk9Whuar%h)7tdO9~ad4$M=bNdkr-!aqs-!Q_gmL z9{*x(zF;RFgH8vx%BNqc8n~5m>#39I97!0 z(;_d>;_mJ?dFi_#9ll#uXi3_StDe;FZxE|mq9u%`3BNXQwnHYKX);43(UT4vz=RKg zkHOh^_W(74AFr0noa<^uWhDtn0TV0>_pfT5}$IdU8CV??A>v(0>Vh|iQ2g~e0M;J zC?O%i01!I$$G|Gq1D+X%>N|$XYH+5o1W3fH!Ie8U7B=(oI7wuzwpVp__Ube3v#)J( z4rEJN5v^QI!jzE?%{v%lY4acgDH1B5mD)8~=Ie z^CbsBkOEFEF2eys%|>qa!;gT};E`I zem@KIFgV^A%U3Q=iK=-j=6(eY5?*S&={lOrUq-+EP}*C7o6?_CQP!3sm`UBcI1)+C zwY9M#h`KIeBy&A2P zW5g7!*J|)&%OGp{YfrA_6deA$;Nt7}Gl+a5I4<&n>1mOc@!ZX(O(tMA4OJ1CINB!+ zL?{Tk9)WF=U4O12l`|)G@-X*I+(gH>dxj&zcd^4r83IzZN5FKF8d~+yBq)=ceuOdjL`Ps)0AC_@GelA+($G%_v(rznyHhJ;zQeWrEl#~xX zGndwYfXnyuUaPs(_+**6%lPPc%3D)L1|ksl)76@rqw!b#g}=Td z5RsGKKRCDKmJ^FdUu*_H$AhTRX~_!P`8@n6+U_X<8FYI~VCm@MN+SdhlDGPd>(Pr9 z=_{+e(c>3nw*ZItTErApE6tY1<6|=r77`DzFseaJvOM^`J~6yY02HKrU}S13NaXHqfnO|5~4W58p!vW)JHpXItMUxmjrUEsnB5nzQyei@av(=e60`DT|F0 zfC$?UEZBHBq2RbJKC;HN;eY&kb$DshrEp;F`@J7*PjGQV{xgO`g+(4fY;fc5npD2? z33YSKGPG2*qs_NeyQn1{%qlgtm&|z~?!_xJ+o=3nVCe4IS3}}KpTPop8WXf2)M@E^ z>0kXc8E@QIS;pz_vIxEZR>3nJB{&53sG+t}itHX#p(HI~#EL84Iu=bp{5DnpX-LEL zH04B_9>0WGLj3cT0@~Vx7O6@)MIC2k;H`2s@!pp%KNUMWS^4&;$=X3BeoEJv@wQRz zeO=6?ZTEW)DSR-IQyzy=BoHZ z_Gg>S!*Dw|TYk)xlLXrz3ncG!>%uJ`~%_k77m$ws5IlQ-6} zx<~9io&=YsV3cz58*O?fN^*TnGlwLLtm=WR#x`(9A|S~esbQYyo`1z!w_Le%RmOhjs2j61NOA+AUgR{ zkGbsV`Pp1Ju)GXzUSHNGa0^cT+@t&`#Y!Rh7`Sv$fcrRY4?68>PvPvgd?J& z=@6nxj9d<{v@|Ii54q2;k(Z=K**meT@2RRa;p9K=dRa>3hw1Visyhq2jEEf~?7;=t@DjKs;eWpqwRraS=HzX4H>dBj5qV@>RqT#s^Fl zaoREw^ESUs>bk%9VIlg-kp%A<9sng23$L-NaJY5|3Vyzlu`wNu8$05#gxO^`E2%Ks z;HMn*BpwgCiUV!7LBoKQ8;Xi>KX)%5%$VrP@6OBrOi22}ap+f#vGLzo4p9NdV~?YS zw3R{u=CIXZ$)gU^r~!VxEQ+O5DD+On)dLL5$-;p?)qMXG9rt6ReWF|Z^VyRhY&tZW zQl~fm%2DWZ$$j-Cum!fUs91>zxZQ`DG6`*|*-|mpqufDf2%#Gdw4_zqpyV|H2c&7h z-=KUkFa(>~En-X@?vVbKw1=V*wH62Q@eNKvx|95U_3EJ3+N90zfTvjg&ENV(EyK>o zhRb%2V-p)OQr&o|kFU|i^rdk2T_V80y(9@N!ocrCu#IUKsZR@)xsvT=;jda13tS2F1RJu14e;9N=dnr$;l z^0_`Iq}&z1ll!MHJk4Fks04FWO*JqkRU}ByK@viYBOwV>48J{mM8m9Fm}$~$l(B{n zgSCGROT*)KRKjH#JlTCU!t&3mp_6A2tHU2|&P8Hj1u`ADlqU%)Xc{_54nCqOb@tWzc_-2YIUH`+A^rJX0?g`%i?+Le%lO^< z*486TXWlr3v4hP(KpC2B;?N03#e=M5Zr+cW5jA_A)LI&#d#I`;K3IBiiX`O=6eufu zxA0jRF$Y4!C1s=N6A?_hO*%AmiEUK)#CkJ7tY!<|T}27O8KvRm4%kZ@x)Z%~CL21S6o~PG>&1A>L{!XNe_D4>XX7AQq;!_ZJ4&TGv#;mQGCwCd zU3Y5h%3CKzXZ*>~k5y$1`>GaREq3WWxw_OY*G--yK_9pBSq#fnC(~|L+cQI5WgVsxq&ukrZvH`A zN6wgy5)ySAFKGn$JEAF4fz3O(6vMpMr!php>sowk?(1}gU(DMKMCu?_*$f4;#3s+c z{1h=sP_ZAD&d%>EH}26QH`3ZuY+^u+YZzNvG!aTX7D{AM(lkx`@h9)493$zNlf|nD z`7RP-F7Hqmlg7bphI7xl4PQXeAXUlfR2M#UAHqsXo-yb%$(e?3wkYUw1xI6ekHt`b zRq!$$=h2I_&d$FNe19;x@w;uzeJsrUuBRs^D;nWqYrFR>jnj*nSr%d<^F8=eORw>G z`#{e}OOL_*1sGU(o$_Ahf%H(0s}w?byhx{yvnOPx8(sOi1V0)ZbG zQR$8MKINXz$-^${B&koZ&AD&AM#HRy1-`4KWJ2uRl#G*I2AxJcvC;ALLGFL>*nc>^ zJu8>#VPQbqeT01&Y9mFq04EP==ypH}UL1Z354-=N_KN{df9U^V@2$eBY@>EjYz(?P zC8R+nC7_hl1Syf0?h=p&MI;3z1d)>NkPZRq?rsE;lI~t(e*d)&);iq#WFPE#eHTpp z=6vIM$1}z~?opBw_k_7Mw4dPEhiP>Qi}jMGez;EeLz)aHd^vuXW^;#AYxpBZ*2d5q z%=Or;)$*FCrC^yMY3vXApNNA4RGqDiad0wfww9NcI?tcC)ar^EIY(7PzKOXdNv>P{lD3w z9vr$JcY{OjmcMCjmQiUpKxu}O3Hc*Y)&~^&uS;QbqQeT@Av7j(Gl~%Y7T|&`i+1;| zu|F-NGq{`ZDhB@`ld9JZ<43vW9oJg1t7mL@ui}~ooNFq_dLmS!ZtaawOf}1~5A25R z;eBx$QzGx*4R46H0>D=M*yZINcSY(Z>XOtY>Q(``oI;a^r-HYX% zdw{laBSOgx*S!=Qrz7}&)s*Cj+PAhQxw*jVmEs41w{aBWLwXS!sgqgM0Jx-G0+zIFwl@O$Nw(gPuZ{5{M32~!Ydzk|E-N4H=@Kgo$~Q`_ioIeD)-@892-TK#6Vv^((U&FaDP3&lSEgfN8~$(3ju zc||m|Z)6JjjeU4zj7IzPmN#(*0`DcpG%1nQf5^)JvfM*c5*YB6oVvHeXwYEMH7$gA zu~X^rFHFA^7rxU2m;!ex5{^K(j-bnejtL9wj9st{-_?)%`Uba!6CIAM`=Z zRn=DZGxGbNIuA`nl4LZF?-nxIo?u=>(?7&(N_fHFYUjHjGfpp3eyh{Pv%hep3Wjj@_798IMXwcOH_8(N0IdgN8Jk@ zG&Hkk(o)H>^z`=-aWB6rzJK@fW#dnq=y+LNk=Gkh8QF1lj0+1(>at^V5C@pM78S0s z{x&$pEI1fYV$$pCtK?r~g>MR<&y?wq+gA-d(|euMIl~T8IEB;}s`|`T{3>>)_aB$h zkH@JspsJl^;c?fuC^1QcpP!eqNeo?OsluHb;jZHLkK@>@Etfg>p7r7BdKL`3kpOGcZn zwcP}O*2nH(KqRbO**o%4z{BB$!&xnu@J8D8#K&gpl#e{DEO{%_c|a4rMJ4O z(m{Cye1cvr*Fhv-GQ(m~EpQX}ijpimQ~ z!^jZ`jeeIwea$G0Fc0TvTiy;@=jZjkyL;cH4*#bc9X z6A*qbj8;%J)5VfgFgC5JC)Q+(j&Dp!t?GK{y(>DTbHp2!WPax;m#m&np|M3=lfQAV zNuV%)$%e5i9v?m4O^T6AhE@w-vW-&yeG;L}Sa~3xLqFOoY7i4CB=iQrF(+q0_)yp_MkO{Z6TShGNP@LBY$sBZC-x_iImy$#WDi z?P9=z)?FWQctK11>@|WcI4FZj$VU1)8t2X1dYnGgMZF!YodqBqTCFLD-B_2pCl<-o z8;rCmy&XZ+K@vW>QY>sH2@&!~B1}e!;%tNWmsOqIbXCN!U9B!<<_tM#;WuH(prQGR zao=nGA$b?DglUHkiN#_C+soW2Pr8YJYllf$c@l%BHV+6wvjQK+o@9(&XnEh=pCv63M6=IJ_g1vkC7f z3IkGGrN(2?~kTBFSXsSn2DJVE~-R8wh&-?y|rzH{YES$<9ubyMch(N6J`Q?D676(Nr_< zpqUYYTv+0gwb9bLDCrj=>OK!blmi5Wb68C;GUe;$&x&vXtDQlmCmiY;HOjEo<0yFa2NC0B*}XYeo{LG;|AQ zHfT#EH-}|bhV&{O@q>#?DqW%@awtT`ql_M>2?`IauI)eR>}4hVgE}+?96J4#1pWQd z)}u<-O`K7I*p_Be3Lh62( zgC}GOZV*axJz@`4S7+8^Q;uex|3a{QsQcd4e^ zdF2_Y9tFH0_VciVV1xWAwjcv-`A=`{l8Ozo&*%8kS1SU=pKJFkl;ZhFU#AN(XY3pi zu(i)rSs4gctjrj-!Z8*|$&dvff2K~7oYzl-RigP;cz_G_QyU^@p$?S_Zl=_BX~K^v zO-O+fCY80Z);@u;$>Si}KF^pj)woS>--q-T?#{)j0yraTMjrF81Q@n=etiF!O; zk5;Ig9I~Ye!Z$ZOG(&B*mHA0m?oPEY0n@i`iM8tqH5di`n-=(&8#Spp7l@}HCy$7G z$nK%v@)U`j@6N=zt+BY6-rin>EHgH?pvEp|E|w#DcCY>ed|N+;=UGvDny>Ys(S(*J zX}L_XW`1Cxew1pxy zd!5;n$3I8cMP1%4FDYgAPCK@yY1Urk+RXmi3ya-6{Zj5+aoXfe-!)CyLr1G&obF~< zUwulJIITt@{4$40ozTnV@}j$cpK0Flc6@KqZw&)Oq|)y~{R1^sCK0{yjP~AyTPnwa zPZgDL3LSQ<3$liXi@JCvccOjk4Rs#J(M3cwBA9gx3yZm}A}4J4+M0c>ch%Lh1^gvx zdj*C>HI0o`*JJL71;2l17WQ{_l~p9X>z=(rbL>PnOXtPI7Hy?U`W>+0lHt5Q0pVRp z_6jzESYA67%@0&gN~I6|>oL_1&MdpSnh-KLIHSJXyq3l>U3w~bRKLa0Dox7G$*ko6Cr3)w2@~cekICAt$2@T}pTYvGFjd@^AcU|wfxVVNmh)_jf%y)0xQ8@h*SycFF zwsPle?h;|8vZZ%!FrIWpUT!|;zO@09xQdW5^-#XftkQ_Ej^|3X>WVHxj^)47SCqRy zRr>FAy(#o8y0UVW4B_i3gH-4ZS{-UaI8pGJbk|=|l&_BOE{O{7$&Y6|3wy#1BNH1D z8!iVU(~`lBh!FRq8TJ@G%5w7!&!dq)e|)X0SKFx8F~YkIjAz{2+x8Hm=bVf24sWW3 zXD5dYjph3i$JjOA?B3}GHQ_t!IpUOCc5ZlCO1b{`yl+FmbzUZAoL3-QFTYRwvz}Fu z^yv9y-Av=Vh64ls$KIC(6D3W4d<%u%JwH$W79r!3ysB4fwi*%0%Z1=d5zj`1r{`YJ z>GKN80fQ>zv%?~e9ee_|geL-4D$Bo{+Pt@)jc54qT6!G=jzk>P9L)xMU!4^pYv$a# z>b-i38ZL&PzV0fY5x;%#!-G2}K(3iT{OKP?+yH*MBkK+AW1LQa`ofh5VB3eCF;kq zyPogVInPm)n@@>e+FP{*6(QBV_mwi`*zZo2t`*cg`S?*k&St8N$pYS4Nt!<6q$}5UKnYGG4*)&BN2B&y@=Hi7LQ5P<-W>z z#(FeciSrgb7T33L$MM_9@f;ShM14I2_f`g$_@+UTECsO?xSeejA=_T&9j_L8!%?=A z{cnpLzaD33vn*Wl*pVF1_;GcjxSOSe`G4|j&g zf!cV+8-K%|+9N;fweSSb3DHZO^40D06CpisVXL9-`t$U${0cwo)15+ZHth;_tD#EQ zy|mlhUA8t=53NF(zRFh!lI!a&urZsPsnBCk2>x!uYC;ecXjSa&v^60PYL^q`F1P)Q zknWI&Fn@JPRK6NCrn~<3U_?I-@#OGTOK*aHTtFpy ztR8b}cw;TM&}E~n2w4SDU+d94*b3*ru{^h`7_ElNVVX?2hwFI8U58}`3^KxU^AR0q z-YE9DHM_@;r*H{-L!ZU95I=MjRMN=LdSr-iSkclZ6>Lzc;h%CHA|MGPno7y5n`gow z{BHrtuB>|S>e zbvTt-R(x;7XM&a2&O++ogjDc1yQIY9eS&9)E~J9`ra^)_uLhR)s%C6tA`A=-1DWRE zD2-FJy%LYy)>d1pb`BY)kyaMZoGpF&y+MVCP)=OsNY>g~+>|S=z1>e?yu-#>oYArS zt^4s^@7^2iL}QO%DoOE#~1@RkN~`mFbgKGT`DNabt`#n6$&ISR+$DlT%A7-MHp;PCQo>y|9p#6J)%fMnlIm zuPz&;7NZjIsq}E=q0!;r^p;)5D>^NDHPxbK`onp{9^UbZ6>st4K**))P3Oz=J(T4!>pm>(|>ywh3Ww!uiHbma}1}ePo?tSm&*W zg?*X4rOfQVUJxEW4EZ-_a+Sqr%``;cv2k$E2b=1R{?iXe&z-UX>%fY#exF9--gdTemsxx`w^KC z6tZMP#KBH4Nn^@u*jyc7_cr>>LxSFDAOAKMjpvn=qYI1Wk78~#xa~R(^_*sBGI4h_KfKU` zi!tH0S2AD-q*vLFGMHg&3{dEO*w(s(N5G1h1=^XFUH9;$66F#~Z!l;Ns|N0Czn=Xwft^Qq^WpVL}7Rd!nY z)nG{)j?C}EFs@IqQZJtV?ObKyS5&Zjw7WW8-OJM3&aa5Hi|9z}vssg2jFI4uJ&@Db z*XH?ZpU6vMdMa?9^XPX<-Q+<0uiPu_I~e*AMh9+RjX(#KO^g4mlwO&T>k^gvHrVfE z2v-JBUzj`AcmIK|t53`qyZW$c8`?o0CHsF8G^v^8mO^K4=225J5gM!0k}E@V=ktc| zolPR=6eF4q8poDv-u1@M4`25ZUd`ickZ~&RZe`@ZRq{~|H>r$7LzpfMR@JjZ@|?6_^){7aecx{Rka-t zHxOdit)C`Cc%Vp8WB$@<`-~al&1NK~p}p zve}!aGv0~^{sSEjD+V)n6}aQt9e3Y25$m5G=;C33C_#F5+VtVut4D5J_- zk}Duq;q!Yoz4*#n!mdOgS+$8Fo{Sg2?8SX?J>rH$v%#W^ zBnfz!hcytE^Ayqh1s>)AA<=WMr1R0Mp`iOR*wWbINrI66{2j=GjihR4B1VH|8S{As z(pCip`O#r)91af4{S5^Ly9n~1qB_5eQ(~kB)y=n!1QWO#WK8x*_#VV6j4hfGkeJf# zKN27?#wTf!XE;ec`bb&(Dl9Pqx-1ndt7UJ@wA4~Y4@6S+&>-nXqj1$>&)@(paH!ZN z+8Utn*C`xfZV(+ElN%lVB0naMO*zKM{#crgrv}_%@ z*Xk>=DKj8*eKjE~UT| z5WB~%uCDITo0^g$ZyL=j(sp^9N1Bl2nQmD1v4cXh^#DE|GACzaE_Y)RM$NpuKcqN{ zHx^2s!c!veO^uCyQ$p!eKg*3^TAtWSBZCOlT&8Owad|l9Ux^15v+wYIoMP)L9x(_B z2*`@DhfGT~wYb`PE-tR+GbN&%Gz<(Gx{-a$x%v5D=sPHtN5|I>a;~+=OGKoxo_2$l zS@^Qp%+hx9B(Za`t+E4uj(3C{=UGk;>+#q9JMqUJrW;$Xp2NJzT&R23^ltJRKDaEO z6t6MAuvYVbOLt+tn>^#2%(}lgR(e@|AU?^LeG&Ft*Lt7O9Je)jsT zwD*in#nc9-ju-od^{jY}@oDbXToKj5Uy4puwGq9Mj1qi7zK!#V{h$==0mw4EK@)bq z>sAXj*T3z2MY?mKDZp>Le={|F`xr3kkOH$US=?VNE+{C79J6g8^|^RlD232pJN|q3 zo3sO6{13(*l^49#&>)#Cg+Y&Sw0o`|@hMDhnPO*JFns=ncY;D@)9>*T z#_k&vit+qV91YzBh$mH*l^5*z-#9pk$L;RzQD8pz^71lMv9T$DvVCUeyVim(+ponh zW@_D?$1kBg-no-#cb_MzFOx4jr?WfrlD@gw;n|Cxmkacj33+J}Ol%ASPkq|EO_Ys# zD3Okj?h@QIL8utf2L~GO1QN(kd!{^kKI+>&E0%OAwpGVUr?L{=8&UJu-BajI7-eA| zeeRcDa}DB$+=%?WlWaLz&eX>fl3lzM#54GHTDOEg)=vQGpRx&q{( z7iUNQ2pHxKZ|R4P6`Z;cHfSW07lW{Ms@#atr7Cm=9y3+=SM z5_M{WRLB8|`0I5|{#Zifp)jYtpKrRtR&Vj*!0*Wr_qd+GADp+8Vb5*vT}Y0MjKJhS zDBMp*o+f)9Z~rd3aN;M2THRzM{J`{2&B|gbmD#e@(D*7RvVXbu!AV%idDD@feDiY~ z4Pv9;#)v65E6dD`iG`ir#*9hG>m1b?fr$y-y0fFh9R2~?Gc2Mc4a2iqv8Z0NUB~|P zMY;RYX1`ttRJO4k})Nm*3l?^71T+VCsgiyfF}-*0#lbip};^})6n=rS;whE;Z|vU)JGb}>6SIq9ay_rY`Dt@c2wU_QyAVoGOg zeXMV_Fee9A|Jg|C<(gt>!C2Z>8|tIT1`)T~SvP-6yDPURlswDVakw%PnAEo_D+Cz| z%+?j{35+{%={lNmakI6Y!CZe~VG<34d_7xPK0ybsMoXvLtS>bprZ3S>`lQiGR}1T& ze;d{F9P_X9UHjMbeSXd=tW9Mr&Uq%xtyX=trta?9*yaH)4hM@;#@wTSbGfW~c^)4WJ!$#A zqnQf-sf5Bd@-g!MCB`1yu3vGr|J!w{PSQ17I&qEKG@fxiPlmkNv-q<5mi3Hy&8&K1^p-7M&E-!xoK98+-UrV- zctZ8G$1-M(D8cX}pMgUx z+4+)+g^h;R%PCrL_nv2R3R5vFPpeNR<*Q#Db^D)mOQl^;Z>6SK9|j(ZPlIGK!=I$SROLH{Di(wYkCY;2rVr|DX6^`dFQ7)dEJy?-c3#3z*1cb zv*uGvD{gq-a-{LeKEsl>?*5OA!glnjDtimf>SE)_;lld2D`;oi^$Ti?>4A}+@^&>d zDNL;`trq@RkFMO4VhH6_Y$22Adlr+`AOlp%^KZw+)X9(-j;Hn9iYuKc%S2jm$BJ>3 z-yH#Ki$-e@t`y%X9U&clgG1Et+uJx!BbONh7d7M8o8H;S!(Z$+POP5oVf*Xmu%4?< zl}w%2hyn$!I_8mEa1x!Krb%b*lXA0DuBxj;@L}wx=jR&jv`1ov!JjZm(9krQA)J2s zgPj?Yy5dVJ`E}JEdLQC=o(RZo*@Og{)J;BhWfoOZh!40c26;fe&j1b&X!$VRTNN(- zN$xafv)Ve;lKK8&93uRj54n?EHj8HxX}K~#!bF3>|mCV z$dQ#{HmJY0hDF9apV@ThK!`zMR&Xx6#IEI)@oz?n=BZ0#LAGCxXET>XS2>Vh+0qPI zi7h*kMoE3Bh%#|95$!|b`>7hU>z z0-cqlu{AWEKhW9G`PSW#$GTMT+Oj^#M6QkYunHo$8yVWK)#v)1d(q_`Nmip@~75KcnD#_on4k;@jKZm zpVowHJ~~85*msYtwK|E_zh~v``OZ1T(3)nCXFe@ayZ?hJC>g5|AiWuSgwhJOuP#ww zYN0Zp7h`A~lhMBFSfR@O(2{py!4cSaEZW@i-7C!APhvFk21~U*VG;rezVVFv9>qt; z4b=s0or{p@c-m0PTwUi6!FO@TQf3sa&qjZBF0qOGIF%kge7X_NG%$ZVSeg;^IGIY&j%Pm&1g0F_XBeBUy)241;V_f ziNX5qhE=4AscDiEMTRr6W)<{`>Hfx>>%29)x7^1k82#1&N-VC9+eFnYh%im^M8SdkgshzKp@rPSy3?t-%joFna=YJRs7sCJ(7S~3%FY{;PCE-NQf-3@ zP*=wX6K;*(yxAR0mkS9^Pk2@JW(iblWJCmf+QyNUElrqy2K6-hrD&i~6kbi#|JU!* zy_(gjkm2FsyLb%rjErOy(r3xGgMeI7@4_P@xb+)-)6(cuZ3@dm$^Y}W znE%^<<6N0T=8pJirpW+8bEKYMvm?E&XeL$pKAQjxuRR#mT4y8YHfl#rUcFCgEJWM23pH0K{o+_U_ALakM1KJ)_9lsnGBRdXR}1p; z@(Kx&X_Zw~Y8OB;%b!6xj*A{!FGELGYb_g_O{n&B1jMZdW0HALb&evQr^FPhal`%d z9Rhk#yB8*U`86&sEh%G;SmJr%^JRnGdVh#J}`N2GoF^vR8?3zNIlz@)Tn z`^3Zhe31%@5;+}L`nx#lJzxU1_i+;yG&36iA}1>wVAlSb25+61+eo_0Z@Qd0Xt&`G zl}oKItTfh}H*ZY3zXFKAjYa9vgu%xYw7c8Iv3U2OXDNhlHY+dhv0xB_gM(vhr;Vzw zrze3^U+m?}m-_m@@f4rf&;PoXx@h4}3Z&6;iD_RVkEw!{)xT2np_|WKV7fG1(QE0? z)-6ydcLjAnP=N#BKqiub03W|QUSMy&;e(rH8;PcpX=xdVlQA3=MxUd`QsWJLQP&kP zZ2h#<6*Gq#1kVB^jqwIhLzpRr-AW-5RS${mslPlZ1e88cuHtYq{{ahM(seaGW^iaI za3v$o#%;HQak(eX8A=|ZH@hoQBb1cFjC%`5{EddpB(F|l)HG%xr!@%)l??e4=m~NG zHz>N-%_HSMYVugb1&ULhMsn0pmmbBe60mX#14;o={OyV3bK74b3@h#V%4e(V=5`9} zZB?cjXtw*O1NGs?kM~gpT@>bKE!}38KVUxk{=TCy-oun&_UbI#xjHC=i<>dvHmB&y& zRtq!FVJip&;FnfV7zJKEC@@e)QIUg*i4&1ip1#xI&=N#oZ*Q-mp>Z09gM-6k+S}dR z`w~ChxXFqKf_3)xI{`R!`;Z(3n@UAlc|S6V!UA=%x!-dvl$4aViAOPrIbVZ+vb406 z99GKA%p4dPsHEn!_qRtRsSp$glxEsi*-cfhtgOJ8ZFPxKhvi;m9D;;GA&#+Y7oPs< z;f9v8K6sEgu{|aqd@Qh3)h?Drp&ZG1YrUXm6ACB!1mh7LbaaMrOodDyLBVeUKXq;O z^=S%7ggHVUBnG8e!TF~v?S1UrB67W<$=%tJ?JCn0CUj+U-kg5TAS(f06tuZnYzxc8 zM3mB!SQEf*=3hQAnK__hVPO%E8~`QplzJNmfj@E?>3tZZy<0PPfr(}GA=(rX=e-lG zH>{(qsab6?lGEJW?AF8-A48TfNcq2W0bm1G78ThY{+sy31sV}6Loun&uq`x}$5hv$ zhu4QXPrC{{q&VBUiIvL9z}?+9$RS)@+-P>4YLlLCjg5_<V>*|GmdypMacvmuGKK6n_(fb48<2N`)QosB_mMJ4q1UwVvgwe?i_n4Whn z)B?hYZBD^z#A{F>@bqDgNk~ZaIjqd)#4X*|70IAmkfcOeS(y{jT3fl7RCc1eu4{|I zk8vx+MH>u5{}L0!p;i9xcxTbv+?+^6_Z_FKtn94YdOn5F+x?BHqk-pC8Uy>3yIpOcNI)%aw6PTa+#OPc{+<2eu&IqcSx*!%ZB)#$b8tMD{ArS$`TYIU z*YG@cc6KBjyVxq898EjudF-_O>4?O`#r=d?RObHsBT>K#oE)@-UZUGtHq^vE$Cs?l zrhAYNB|^6@zzaOXJE>Rnk#hX>9m#hmPddjeOnOQ#eg%VBiT9Ap3Uuhz{*80bkP9 z7D`d$xT;J$*fm2X?B=*Sm|p94&}bZr{2UioW53vn>i9F5CUY7A??;A*!4`r76uxo$ z0Vm?;+oxB#kKg=hpHE_J9fvCd#p8Cotxun-4=v#e$1E$RKDe4fqq~(Hzul=zD;8$%-RbY|x7j2HwdA{W3;`(tb4;?H&?ruu#lWx&<>ujeD5F_nqcdaR3Syr?jb7`1gy1YX zGy;tphp-;BJwi=!sJ}e5%pLg_8*6U$I)7A>NT-=ttJclQE--~@eXOuz!YVsE`+xM9 zC71bN8WeVlf`our3maNZ6dT{Yd)ICCH7%TH2G;Cr0_sIUMTIKhM(wY#C%f(2znhQ@ zLs+8cFH}|coADH1u@V}(U!!=TtE&s@v{1Kp7BU6gkdcgvHZd`L&*BVACN`^<6QLb| zXqfQfxN(e5h8~aGN`JD@X5|gj&v9MD*(!Y2ll$X_VIfqavnD0hlcmgjd`D2-dUtjf z?SP(vVb*C(hstBSAyQF90O&DEC4|lN*QBJYv$-o>{1%{9-o~SJgr$DPlxfem1GoS- zy*kheUKgh`P)FMkY>Udikd!R?JhZ<$1h4wjwOqLM`#zszV^91=&#x&OUKd|Eh{d83 z@c^dD4GH{|ttlNtsSa4Z)ycATFjp{XBg0%1ESZX;Ft0%~I#_g3z*M?!tLXv`f30SJ z0xN53WQ5vqP;HJ7EO-ewV*caD53qI@5f9}gl)Bs7UA7zU94zF)6EN1N+C~8|wEpXM zJrJ#hE;6?}{@?$0qz0X2NJB5PHjvs@24P>=cGWCbdr4PDpK6a6f_o@ik$f}F#Ajf} zy)U)QzePty?L)Eih|Vtji(&!_l8soDWn^T8lJNXJSx$1UnMvl@f@idHTSiuvRM>4l zPow1e`=@q2-@cL79yCL`5nU{wEv2E=pZ0LkdKI+(p8trQ-hYCGnH^lA!Jua^;e+a6%ldy#;4cODDX_Au6=I;6Wmdl3Tgn_uKk9& zCMlwqJ7EF>0#bKeC<)oM`Rx}ZYGxE9D8o$8k9R~bj}1Z4MXt{GzzPDdAJ$glcCeZVrSnx7}KNT zw|h;EeY{z-^$BzN39DwQ8A!nC70S0jp@H!teXRJ+c?N8Mq!#F{43q*N1uP-vs0jt_Mwh97QY(59rM(54y+sKYa6dGx4yvwA%s z>-+b4xGcI9zBvybX6EeKZxQqtCw<-wDPT$Mdi$1o3akE-(mrBkbrekKA?HJTN`X3} z!X&EADQjZxy&r9qQ()5p;$_Om z4)$AsQ}qW#tsW%!+qZ838vsc4eB)hT2<8w09(QJh())Y|K$Fctilh&9b4v?&0sp{C zqxKuww}|N?{U)%0#u~bncKR=065>G1S?B)m&$fQf$BNx~O@937g zR=L##*!DrVYq<75OG{wLY{60jM6dT{wRoxN(ZY2dd-ht4Ck1F1Ea}{(8gSO!C;3feRXg{ST-= zx1c&Gzyu(?1T11NtndB^tPzjpXdW0kPk;dxGjFwKjvkn#=y{!_dxd1!EcYfz?0a7w zd8aU~$$9O?%>mB4=}nL7@{CfV0H9C9AiN-ff4=kcrg*+7XbtbPskv|4erRaRm^`U) zh0U$4yF6;@>KvM-ObEmwI689U22$~R`C1kCeC(iC&Zv$}nI;O@;0}!DYX!D1z1f(p ztpVT-Yhyh#V)JvS6;RVpFvLzoURM`KpnyP>Jx}*z`Rxs6G5|p14M4MnGMgDejp)F3 zD+J0qI;k{OFRuhLq40N5aBy7FvxtZY<#kYMZMw<66|rjP!J{s8koCb zXW7nFA3(p!6XDX)5ynPF=f{g-puJtdafAb+ zKrxZ7i#}LQ=uQ;lMKqiu8GIXqG;O&-@M1K*H8@S=e7mXsY-S5uclOmfKWmUPKZu=t zaZ310{t;R{2Yo6UTIQ-^3%>8Ep`nF^g@l9zDj=pqFYK@}l!=9-0~16fqT=GTY$-IN zL4wX&h`=S`2k!>R0Qp#MV-Py_nB2{7CLFF-sSwhG^SwUX@uK96jK2V(K81x{Z8u>N zv1v-sz!Nls4}(*pS+;hzRgX1Q21qC6UHu@1o=d;U8fT7#d`?2N zr%b5pW89doDk&~j#=a?Bk{8T{Flxs}4$Xp9ukO0=x&oXeEpckTsgNKD>JNPOJe`{I zO^Z3`cLRQCej!Pk-ja+&VLh~Ebliu<+3!TfufSmdL3~hP{^J#BfrGdthzJ6j1}fkM z2MOGHUk~4$yabl;{)-uzmkB9M;A!#WfMhy>)mX05R~Y2LJq8O%F{!GlDd>IaL0i2) zC>MA57p-3(#Vb+MPQpe8>P1GPV%9Bp!~#R-H*9;l6wo7Y0Oy_JW)8WYZB!89<9`CL zqtW;7X(QOvEeS{1L~G8?cuGLO*m|F7(N?c!#5G>7MMdFXr~&*7nA%W~lglkAm<2=* z`YI(Sr>++wru%b32F(F2!9+l-d4lpem|X^`U74*t1Ivf%q7o)@npITvLa)ws?=Ki| z>*-1eB=5joTCEYLEGK!3aC7g1pH>692MD!vqNc+9d>!C;#(x+F2M2?5kGfpGlZ$KL zKP*7zwVzjC4K?~qAAa6?Vs!M&SFf@P&!Hz4_5f%n9}e$@Kt^Iii6~%(fF@aA-giG( zbAqdy7yOp7S|&hZ7#kZ4AZ?T*6*55MKCDl>EQUyw!j;YAs;T*7Q7K3OR3%V67-cz0S$!?x0aaq79lOwHr5Sr$rei>Ejh}{PMH2 z?-r1M`}C%_{^D?o%%(E=c^KuuzyLIxIRPr^_uCqA?*Vv-5dcxZ(fItC(_%Oa@Tb`~ zLMsfUl)y5Wu+Q`|_${z6?R&w6PWbX=1cJ%IEfLYtknV_5ozg?4fZa+aeu6LRce-+P zHs`vWpk-U@1Qrp*3q%u~V7JSYc+1=9;9=&^rJw8 zPFe`Enkk%62Rnrzzd2=GXYAivoYO?vpDE>t(qr?O;h;exNr?!9E@PI5k7fppYxNo(f%&Un4Q^7O^;J!x}H>~rXK?M1m3(ik^Ed%^3e1vRSKAH0-r zH2mjdpnWDq|L^Iby&Hat>eC3XwA((=*AHIQyx9M7!f0sO?_Wbf8od4=97=|i%it!> zQx%n=+1X1#BhcaghWb7LA9Nq|`}b3;F2IF+iY;Mhw~e-e71IUWz!UGwC$_S{jQZTV z@95+NsuX@^Dkfc`_&oO`h3)ly%i{F(Ti%o7<4lf<;C!L`6v4cPW3CA6E&f%$SK-Ms zbGMu}CTA&jDTFS`k4|_3iS{p9%nLDD14}*Tg?2;Ni}ok}PRx=8L*(E>FE1$OfQf-@ zL;h(}(zCeUd(pI?~e8Dk_ns&PYLo z?W{a@Ge`Y-W8C=QV6p3-3AE0F&Aqm|Dj+13nUNuz(+s1prbkA~YHMr9*kA`SD8#!g zG~bmWxJb;(&7G?Eu802<8XB6Dvj*e`>{ZXbzew|~vAG8~QzJ6LQ9omZz${>LAQLyn ze=MekiNIKlNNF7C2nQ{hK;PhYSW_FBn83AFS60qfz$AuPL8moAL%*(PXPS>ce z;WHbjXPiG#8e%QH;PCbuKg}zs$qqa~Tb&qGE=^^0s+dpIok$H3viQ6ZZHl4W7I1z^sDqoYh^Z5^F3 zN?}36)(3%qIO@PsOq7^HUJA7@WKzJ1D4)1+f`5E`d<@SYd~aY40Y!jn1ya$~#pS2h zg(hd@v#gFweLaQOj5<$sbkJ#h(d!hej5%KYrZEyvxGCpxd0rJ069bwXxLqI}wYA9) zlq@ZCUvqFfk-WpEmDvS13KJ7k+#j>7sHoEJw@=z(kYw>pk`-2VfPT2x0s(F~S7E`_ zMRj>BTiRX?m7B0~UT$vJ?5qeITL_1a1VvC`{(Vnxk_$~g-`nw4?6Q{_6t#i$Ud1b` zuOa*bE1X(bSeTo;zOixfw})(2(fp;Pq2c0JF2lOxh5L86N_}DU1;l+ICqvZZxTvnK z?)&%eu&XL7T_ExXke`y05*+(|2Rtj)I}OzMq{PIpw6tmpjety2N)7mhK8<&q&R#v7BF9sis+w1%|$@@}c+uIt@ zFM9I!w%zSTTk?U65a)|6v1iX9jb6?D@ZU4^9ZW5`QHXgUAf5!y?X}MOLLij^YWiN) zg7F2t1P2or6EfEM^9hpS1Ip#}xB2s%y#X zaKiN3wQF!X`1#gC_HG+SvePMhxH;sP$+Y`yo~$cP#zWjgHu8K2E7O-qy^|R_L%j}&DjB(_X9K<8WTVTM$lK;FG|hs zL%0H>e~O|PsDJhKQCv3C16Njql(b1O?^dQDKVhjun z7rl?@>2K0F0#@`|rWUiYVUO&vGBcx=DbC4}1>%5af4<>HGFfI?8nrJNrz&msFVgi4 z_G;9=cJ@BKiNe9y#Xo+8|9T%zKQTL7b(Y(^q1W&FW2itbO`egFk$~XK`MJk=bdQM; zKR;(|H$*@q=oc3kam2!?LaDNCU?a=tXEYwAcitmuAeX>Zu zQ}UuS0~zPyjdgr#4G-2M?zbhDo}Qk}7rBT(;IkVU8)qWp4k3p12*%P3!sr?#l8uq^ zxsNdfU`)|z&g(t?c9YD8Sy@?icX!jjihcn*6pRuCVCIFb??RA;8ePn1c_LjdhSnEs z`S#T2rj59G6WGu?w*y&aNoa`v2@}_1mS8I{w6c;P@?4=EEAVhPZro5*RIHnbgs7bh z!c9%>3ruB5Nil#BIMB3!Be2DQrl3P#0DCFst-$oV=KU%}HG>J+Am}+XikjFI78rd0cr*@bY6No#2^X z>l4_E1>cJgUT~lnej+O2AD#;5CVqg2hi7YRn)A?d0kH9_wZ62U_xWx#I==;Ln{zpHF2xyF+o*rCMn+awSI5RqfRm|L z>(){c5)>p0IUDmZ$iV#wTg>Ca)d2S&6-A|pk`+*ufMDQfYBl{$pn}_bdXSKt_xyQV zPtOMA*Wxk@LLzZ``VuJRn+>AK;4z4dPS4EjBqd68UI2#+ED;1%Se}RS>KnNWrC3|pw!TLbJ3~Z!N z&ds$2k_E^UF}Vlm=%|=BczGWhUZ|_ryB*|IR2-cith=A)284uwjSUD5Ey&M@Ay#<> z1<=Mf;jy{$%Tez({Ui}j3mY3s0s=LFJ{A_T5k{a}Q&SC@N$Tf2vjzI|}8gv`szdg`4R2K+2QhY71g z*h`><;$mW+TZ75)yf}3-H8ll)?$JO)gFav!$YLh1D_@>HMInw_1fARDqzFMw*{mygawB4Hi1lq{0n0f6Npqh0t|o` zfJ;=&WG8cWc9w>Q20Y|@4<0<`<;A#r*R}rgSNh1qvoL!8%I@EGx^*5^&Re=qR4Isw z1BQ0e_W%gN<=fp0Zh@~zptI@z{Cau{qK_j6D+Mf@wT(^2;H^8(5QKb(eGdbp#kNqp z$_Ja2*8-oHUs+W(Ei=;}ThMuvoS9QzPfaZykOWv$(T6g{a4U1U{1(GZ78dX0;>g5S zAk?m3GQt1w@nd`%BO4oq4H+dRoM+e|9mi~SKf+00es)1(PIC%ZhjCIGfGXpmFQh4tQ6%Y%uZg@3A9rE=J+ZJ;?r@uNVF z$ic%pwT}UGOCJ>x9&}?K$7`SK_}GlRM)Y)7B^}#VcGgQ&jC3D*Gq7!VPTp6#6Wmk` z@t|!RzbggUm^U4;^MbsgZ!cIApg~;l7cR&%7-e>03&OJ;4r>@6AD^2O6%hdyn`kR1 zxRL)l^?&tt<>64SfB&3=QVCfj6{R?lin2r|YbZk~ItL+T&15Gug=8yPBI_Jc(n47a zO-B(bSxeT*k|i{QFt+i2=J(Hgz5l=OJAZU_)x|u|{oMEW``Pa2j@HQlz@Pq(5~vz( z>f|JIrQ2Mt_qDc*axFmS5h8#}+`9e1-8uR<-<*;fsH0?MX+r1=mBfOQEAm^SqqpD{ z$>1B9&qX;iLK-LHR$sySUK6I3DgV&^vO*3JN9) z%bjlS6u6O{qF^IXtc!^N?bjUk>GZTVH`YMdqVw>_ROEPETwU*#WgIvcQB>sR=H}+@ zt)#^dZ}*EGjH&>S`%T$*aJBwy!D*A6MMwY}kP*#9I^-^CI%-ud|aqMCysr zxao5{Zk3pDj^EZ4km8-+SNKy?A-HDOrkD?usPHvY{Mrwx)NN7+aS&jiZqH?I1?o;K zVW@s^`Xk|{AstmS=VFJK%U9xrNJYg~;7>ih`+ymWieA9aSeWdf$n?;<(({zAZq_FE z`%9LxnGasPdbL$$7&>&$noIy2#`SV;6;dC%x(kU!K&K^KCcD6UY;A5HotRig*5Jg6 z6Kfs{y@%HE&yyz;bb6DD=Qz?@P=G;t^B$abb9QDfEI{fb1;`$ifZHCR?^`G&B*d6= zNH(xnOa;cl6UE4ZBW))yA0VrgLv9Zb=qv|emQ<2gvBk)7|7MKi=g(xGMpOsJ zMg>Dr^DcD;Uv7SWLr?@dwvwV^n!0}hg|c1v?cTpiB1uJ-=)*XxvY7|paDzuoo4mG< z8j?<-5WI#Adg2YwR0TW-@Mm>@^QQM536%sKd6xoOu$+{X1m-m$*Mi`Vg{9>ZlL^bB z?$akC5^qG}Pt2C@D@G0uIF(msR)iwNC8N}$mA88>uaw;J(XknsM|{Uf3nW{SYuSWb z)hX+WDP$jzein-b#RKy^kC>^bto&|lo=&qgGrK4L&(*6};ZWeIq4x}QjZaK`#M>A; zIKZ?s5S_sJkFQ@7KF77n`96rgnI=ZQH_^OXq_MWgFg#LC{L5R%RWGgkHT=`&LRvNP zt=9V<j6v03-$>Y?DR*{$Ug5wemw@ymE)D>02(X#X}sDxB%z`YvTx`x&uWQ(*uD z)O(D5sB!g@A;NS`>BnTi6A=-&$!gkodnfv~axdMg?I*#y5a2o}B@d4yE zaSQ7k7iPyhP(Oc;j;e+%mNb9gv+Iv0K_EvS0Fd?Ty?lL*F^X*qj|%}muvw%Mx-=er zOH)&vQQaoOArhI+xaAd@Hzt;Xefej6ao0$CcEykub{1>g# zzYr6ts!AfxZ9{KSVcSQr!A;tTVf=o!0->$ubb^)qX^>IFgq`}a{Ap692KJ7QLu_d} zKE8i+6uz{bz5M}LYkDyvCsztls9(NhTm0?*c+LEV4I8G6GmjV=7+^nLQF-~uW0kZ+ zYNnfca-V!m(T`OcI!(<@4|^hiF5`eekl$`?cyDamnVEeWL9PB3gJ&HJu({qziTmz; zP*(k6H^|k<&`>jJ`*$&~X&zS2-2zt2{R*DTV*C%@FX|f&^zD@zFc{;^+41AHrrPd= zCAY_Fy1(C)i0*5qY9DCP*I5(^pWjHmbB!bxTvZ@=>e`W0df=Gn&wq~=mSa4&6gD+jH!l=xi)96y9H||Stx;w(x*Kt@TUFH>sckDTv+w2PfZlM?M+P^P zF^n<>2zAw{hL9`(O5kN0t0AJB!0tR%R_#RLUbF*5H`j{{V2j1Vjf!BD*|ehWIv*BaXXy z_4@NpcitO2_wA##b$w_*crheo1>PjafsHxvuKVF(-2Wun1hSJbB{;PZ&cR{1gKDSl zhK1?4y8g&i_t)jwNQQTSAkYt*;E%gxzk@|q^4VLyb-&v+JHK1WMR8?xAlfQs2%BDIrAhkSOt|u|(>M;pr22@&To*k$ z$6$OjwzRNlZ)r)aRoM&eixx=5muLraexc#$pCszorZ=6 z6~0(HS{tk*l!{O>-nI@0|O4Ofg5aJP_B}mJ)lL#cPS=5Rej0F zXfA$eoLcm;vtS^_AYll~>qTeh07wDVZEmk`_kL#`};fE>P*leWGQK#ol#DY44%FE?)CD}p_1hK(PP;M(;Xx( z2n(+})5$?AmQBJ4!bPM1VJ6Vtwucxd5(fkLb4SNKf+Emi!H>DKAzUGe2Y>Y=yJy(A zxVuA=I;f;1NH%^9L~sdoa%pbT8_c=9T(n|idgI*O+=b#!rq*7LiwMXSUuZlGH4-7k z=sLab?r!9?>l~h=4JRii4vvm)-nI?vi~_j!{rgr432hyn)Rj!--JbLhSC&nXR0qa@ z7>FQsRTKTq+wp-d@MAn$JZRL((NTl@a3~6_UwW~Y7lNy0wd-MFAX}uYoJ`_$!XSay zX$DT`!gt3}qd7S_uvq=(w!1CXfA~;YSqa{aFk}*(#f}cA3l}18+?W}*b8r|K8xs>1 zEk_d_8glG8VfxkG@95Frw4EWWgiQW6+`E8DgQhiQyU77gg&GKrc70b@1vzztL&d(~ zFBb8Ix|*pGl6PZXMfQu|E6bMIw+}7I5!w^NG5t&RG#ahGo}J-^$B75*@9mwKlJXit z37RPaM_0$aakGdViHL{*`Qstr?RhI5-DnYGL6kJmo_QK5I_vcWYe|LWzS z^J1RrYtUwOx94-e`jUNj>49X-7xip%o}|dJcCP400@L0?AzVAf5NnCLbt{WNX2E9= z<5}}lvqmQST0yjy5>Pvk2)jN*lMS8u1~>ps8pS6Mhk;)R`k>O^55DZTd&d|cx?v7h=!Vj z2{JJ;fkp#J07aS!P!6z*FvQcRH)CVVJbKe86q2JOnp{d!k|V;Y0j){mfA-d{Hc1b> z)Cy)4&fq63j9gv!*j{dEZoX|)6B9Af_bQz}9-ze!l(gqM$t4fDG0E^>v$GV9z)*9e zd*RO}?kK==-4YBeC1ho0XlaobkYr^05f8!pet+G@~OUxmV zX(qF=ovP;>Xo?KGE#l$~W#|*wE|6cqI;TaMm~ij0e3YF%1-{4nENz$o+eKAkVJOiA zkqeco)yc_8@T@m&kK8X_#AF?F)DtY9wDGqHh?6_K;wCrJpksK4yLGHm9sF^oeX+vJj@vv=Sk? z1z;z44N^LH{xZ)mEft<$*%^-J@3|v$?)0;hwI+g4k8;1PdEe9ohBtUcmwG>w^Y1a1 zFr<^+h5`h_8B0qQ6p}n45fKF920{cYO4`8qznruzE&h6ckE zC#=goGTD%$$SXLwiZf>Ldf_Qf(SyMlJ+ul4IOxg{fs+#wv_mXTpDqht@B+F(zKu_l z`7ez|`Rm$L$ZypTrCa#20fBDuzv;kcYz5U5zow?D&0jF0s~wvfC1PsdgsDRCMyN=V zEA*s7N(}i4r&cVSfC0QIB`qE_qVFiflkFG0n+rD%!~lchyjXqFVOK)Fp71Me059Eh8X*kZyScdZ^9N#I z(u=o$JMWK9c{^z1Vzxu$$Px6ZzBZl9f&BpoRaLEu)Fo{sct`N8QT(bBZP8_iM@Ej> z{Qk#b$<23iHLsf@d>#V<9#t_bs}BY{&lU)5TQh;<}DN z$?ZR?Wg{rMc`(Uv{p#wKh(DUgtgH`F4nZ^sFyF!5{RyhGt7`@D6T-7pueR1!xSfDN zawyz+c`7KHH8r{lDc!BDJ7i>1GBbPIru=+;QSSQ{D|o;^!O-9)O!CaCiE zEOkjdHbJ_xy&YAGZI(eK10cjBs;I0)tTO5~^|^D;7m-gN%sv%s|9mW-FhkhtnN-@5 zyUH5XA1rL@J9Vky+S9ptu@q^y^!iZO=}x&K7FkYwyE}@4y}dnjdN6yap!gGy04hIT z-x2(Gs|#sC{8WN84zC-VoPS_yYR2|CY6WVe`)Tq@h`lYppv-2p%7|R+T zQF$vw;KNhIw|=xMsj6BAtv{&psi_IELfWoM4tf+EUN%pzRX-!2T;e}MzahD+qKciv zVDPoIw45|ISHeEK`H={|-<K9ngs5_5nu)oAm LkEmOqZ5R4qELF2% diff --git a/garlic/doc/slides/scalability.svg b/garlic/doc/slides/scalability.svg deleted file mode 100644 index 19fb326..0000000 --- a/garlic/doc/slides/scalability.svg +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - Efficiency - Nodes: log(n) - - - Too small - OmpSs-2 scale bad - OpenMP scale bad - - - - We can explain this difference - - - diff --git a/garlic/doc/slides/scaling-region.svg b/garlic/doc/slides/scaling-region.svg deleted file mode 100644 index 3cb2857..0000000 --- a/garlic/doc/slides/scaling-region.svg +++ /dev/null @@ -1,806 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - Problemsize: log(N) - Nodes: log(n) - - Constant workper CPU - - - OpenMPsat. limit - OmpSs-2sat. limit - - Too small - - ENOMEM - - - - - - - - - - - - - - - - - - - - - - - - - - - - Strong - A - B - C - - - - - Weak - - - - - - - Saturation - OpenMP scale bad - OmpSs-2 scale bad - ENOTIME - - - Saturationzone - - - - - 1 task/CPUlimit - - - diff --git a/garlic/doc/ug.ms b/garlic/doc/ug.ms deleted file mode 100644 index 8b7c579..0000000 --- a/garlic/doc/ug.ms +++ /dev/null @@ -1,1691 +0,0 @@ -.\" Point size fails when rending html in the code blocks -.\".nr PS 11p -.nr GROWPS 3 -.nr PSINCR 2p -.fam P -.\" =================================================================== -.\" Some useful macros -.\" =================================================================== -.\" -.\" Code start (CS) and end (CE) blocks -.de CS -.DS L -\fC -.. -.de CE -\fP -.DE -.. -.\" Code inline: -.\" .CI "inline code" -.de CI -\fC\\$1\fP\\$2 -.. -.\" =================================================================== -.\" \& -.\" .sp 3c -.\" .LG -.\" .LG -.\" .LG -.\" .LG -.\" Garlic: User guide -.\" .br -.\" .NL -.\" Rodrigo Arias Mallo -.\" .br -.\" .I "Barcelona Supercomputing Center" -.\" .br -.\" \*[curdate] -.\" .sp 17c -.\" .DE -.\" .CI \*[gitcommit] -.TL -Garlic: User Guide -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document contains all the information to configure and use the -garlic benchmark. All stages from the development to the publication -are covered, as well as the introductory steps required to setup the -machines. -.DS L -.SM -\fC -Generated on \*[curdate] -Git commit: \*[gitcommit] -\fP -.DE -.AE -.\" =================================================================== -.NH 1 -Introduction -.LP -The garlic framework is designed to fulfill all the requirements of an -experimenter in all the steps up to publication. The experience gained -while using it suggests that we move along three stages despicted in the -following diagram: -.DS L -.SM -.PS 5 -linewid=1.4; -arcrad=1; -right -S: box "Source" "code" -line "Development" invis -P: box "Program" -line "Experimentation" invis -R:box "Results" -line "Data" "exploration" invis -F:box "Figures" -# Creates a "cycle" around two boxes -define cycle { - arc cw from 1/2 of the way between $1.n and $1.ne \ - to 1/2 of the way between $2.nw and $2.n ->; - arc cw from 1/2 of the way between $2.s and $2.sw \ - to 1/2 of the way between $1.se and $1.s ->; -} -cycle(S, P) -cycle(P, R) -cycle(R, F) -.PE -.DE -In the development phase the experimenter changes the source code in -order to introduce new features or fix bugs. Once the program is -considered functional, the next phase is the experimentation, where -several experiment configurations are tested to evaluate the program. It -is common that some problems are spotted during this phase, which lead -the experimenter to go back to the development phase and change the -source code. -.PP -Finally, when the experiment is considered completed, the -experimenter moves to the next phase, which envolves the exploration of -the data generated by the experiment. During this phase, it is common to -generate results in the form of plots or tables which provide a clear -insight in those quantities of interest. It is also common that after -looking at the figures, some changes in the experiment configuration -need to be introduced (or even in the source code of the program). -.PP -Therefore, the experimenter may move forward and backwards along three -phases several times. The garlic framework provides support for all the -three stages (with different degrees of madurity). -.\" =================================================================== -.NH 2 -Machines and clusters -.LP -Our current setup employs multiple machines to build and execute the -experiments. Each cluster and node has it's own name and will be -different in other clusters. Therefore, instead of using the names of -the machines we use machine classes to generalize our setup. Those -machine clases currently correspond to a physical machine each: -.IP \(bu 12p -.B Builder -(xeon07): runs the nix-daemon and performs the builds in /nix. Requires -root access to setup the -.I nix-daemon -with multiple users. -.IP \(bu -.B Target -(MareNostrum 4 compute nodes): the nodes where the experiments -are executed. It doesn't need to have /nix installed or root access. -.IP \(bu -.B Login -(MareNostrum 4 login nodes): used to allocate resources and run jobs. It -doesn't need to have /nix installed or root access. -.IP \(bu -.B Laptop -(where the keyboard is attached, can be anything): used to connect to the other machines. -No root access is required or /nix, but needs to be able to connect to -the builder. -.LP -The machines don't need to be different of each others, as one machine -can implement several classes. For example the laptop can act as the -builder too but is not recommended. Or the login machine can also -perform the builds, but is not possible yet in our setup. -.\" =================================================================== -.NH 2 -Reproducibility -.LP -An effort to facilitate the reproducibility of the experiments has been -done, with varying degrees of success. The names of the different levels -of reproducibility have not been yet standarized, so we define our own -to avoid any confusion. We define three levels of reproducibility based -on the people and the machine involved: -.IP \(bu 12p -R0: The \fIsame\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R1: \fIDifferent\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R2: \fIDifferent\fP people on a \fIdifferent\fP machine obtain the same result -.LP -The garlic framework distinguishes two types of results: the result of -\fIbuilding a derivation\fP (usually building a binary or a library from the -sources) and the results of the \fIexecution of an experiment\fP (typically -those are the measurements performed during the execution of the program -of study). -.PP -For those two types, the meaning of -.I "same result" -is different. In the case of building a binary, we define the same -result if it is bit-by-bit identical. In the packages provided by nixos -is usually the case except some rare cases. One example is that during the build process, -a directory is listed by the order of the inodes, giving a random order -which is different between builds. These problems are tracked by the -.URL https://r13y.com/ r13y -project. About 99% of the derivations of the minimal package set achieve -the R2 property. -.PP -On the other hand, the results of the experiments are always bit-by-bit -different. So we change the definition to state that they are the same -if the conclusions that can be obtained are the same. In particular, we -assume that the results are within the confidence interval. With this -definition, all experiments are currently R1. The reproducibility level -R2 is not posible yet as the software is compiled to support only the -target machine, with an specific interconnection. -.\" =================================================================== -.bp -.NH 1 -Preliminary steps -.LP -The peculiarities of our setup require that users perform some actions -to use the garlic framework. The content of this section is only -intended for the users of our machines, but can serve as reference in -other machines. -.PP -The names of the machine classes are used in the command line prompt -instead of the actual name of the machine, to indicate that the command -needs to be executed in the stated machine class, for example: -.CS -builder% echo hi -hi -.CE -When the machine class is not important, it is ignored and only the -.CI "%" -prompt appears. -.\" =================================================================== -.NH 2 -Configure your laptop -.LP -To easily connect to the builder (xeon07) in one step, configure the SSH -client to perform a jump over the Cobi login node. The -.I ProxyJump -directive is only available in version 7.3 and upwards. Add the -following lines in the -.CI \(ti/.ssh/config -file of your laptop: -.CS -Host cobi - HostName ssflogin.bsc.es - User your-username-here - -Host xeon07 - ProxyJump cobi - HostName xeon07 - User your-username-here -.CE -You should be able to connect to the builder typing: -.CS -laptop$ ssh xeon07 -.CE -To spot any problems try with the -.CI -v -option to enable verbose output. -.\" =================================================================== -.NH 2 -Configure the builder (xeon07) -.LP -In order to use nix you would need to be able to download the sources -from Internet. Usually the download requires the ports 22, 80 and 443 -to be open for outgoing traffic. -.PP -Check that you have network access in -xeon07 provided by the environment variables \fIhttp_proxy\fP and -\fIhttps_proxy\fP. Try to fetch a webpage with curl, to ensure the proxy -is working: -.CS -xeon07$ curl x.com -x -.CE -.\" =================================================================== -.NH 3 -Create a new SSH key -.LP -There is one DSA key in your current home called "cluster" that is no -longer supported in recent SSH versions and should not be used. Before -removing it, create a new one without password protection leaving the -passphrase empty (in case that you don't have one already created) by -running: -.CS -xeon07$ ssh-keygen -Generating public/private rsa key pair. -Enter file in which to save the key (\(ti/.ssh/id_rsa): -Enter passphrase (empty for no passphrase): -Enter same passphrase again: -Your identification has been saved in \(ti/.ssh/id_rsa. -Your public key has been saved in \(ti/.ssh/id_rsa.pub. -\&... -.CE -By default it will create the public key at \f(CW\(ti/.ssh/id_rsa.pub\fP. -Then add the newly created key to the authorized keys, so you can -connect to other nodes of the Cobi cluster: -.CS -xeon07$ cat \(ti/.ssh/id_rsa.pub >> \(ti/.ssh/authorized_keys -.CE -Finally, delete the old "cluster" key: -.CS -xeon07$ rm \(ti/.ssh/cluster \(ti/.ssh/cluster.pub -.CE -And remove the section in the configuration \f(CW\(ti/.ssh/config\fP -where the key was assigned to be used in all hosts along with the -\f(CWStrictHostKeyChecking=no\fP option. Remove the following lines (if -they exist): -.CS -Host * - IdentityFile \(ti/.ssh/cluster - StrictHostKeyChecking=no -.CE -By default, the SSH client already searchs for a keypair called -\f(CW\(ti/.ssh/id_rsa\fP and \f(CW\(ti/.ssh/id_rsa.pub\fP, so there is -no need to manually specify them. -.PP -You should be able to access the login node with your new key by using: -.CS -xeon07$ ssh ssfhead -.CE -.\" =================================================================== -.NH 3 -Authorize access to the repository -.LP -The sources of BSC packages are usually downloaded directly from the PM -git server, so you must be able to access all repositories without a -password prompt. -.PP -Most repositories are open to read for logged in users, but there are -some exceptions (for example the nanos6 repository) where you must have -explicitly granted read access. -.PP -Copy the contents of your public SSH key in \f(CW\(ti/.ssh/id_rsa.pub\fP -and paste it in GitLab at -.CS -https://pm.bsc.es/gitlab/profile/keys -.CE -Finally verify the SSH connection to the server works and you get a -greeting from the GitLab server with your username: -.CS -xeon07$ ssh git@bscpm03.bsc.es -PTY allocation request failed on channel 0 -Welcome to GitLab, @rarias! -Connection to bscpm03.bsc.es closed. -.CE -Verify that you can access the nanos6 repository (otherwise you -first need to ask to be granted read access), at: -.CS -https://pm.bsc.es/gitlab/nanos6/nanos6 -.CE -Finally, you should be able to download the nanos6 git -repository without any password interaction by running: -.CS -xeon07$ git clone git@bscpm03.bsc.es:nanos6/nanos6.git -.CE -Which will create the nanos6 directory. -.\" =================================================================== -.NH 3 -Authorize access to MareNostrum 4 -.LP -You will also need to access MareNostrum 4 from the xeon07 machine, in -order to run experiments. Add the following lines to the -\f(CW\(ti/.ssh/config\fP file and set your user name: -.CS -Host mn0 mn1 mn2 - User -.CE -Then copy your SSH key to MareNostrum 4 (it will ask you for your login -password): -.CS -xeon07$ ssh-copy-id -i \(ti/.ssh/id_rsa.pub mn1 -.CE -Finally, ensure that you can connect without a password: -.CS -xeon07$ ssh mn1 -\&... -login1$ -.CE -.\" =================================================================== -.NH 3 -Clone the bscpkgs repository -.LP -Once you have Internet and you have granted access to the PM GitLab -repositories you can begin building software with nix. First ensure -that the nix binaries are available from your shell in xeon07: -.CS -xeon07$ nix --version -nix (Nix) 2.3.6 -.CE -Now you are ready to build and install packages with nix. Clone the -bscpkgs repository: -.CS -xeon07$ git clone git@bscpm03.bsc.es:rarias/bscpkgs.git -.CE -Nix looks in the current folder for a file named \f(CWdefault.nix\fP for -packages, so go to the bscpkgs directory: -.CS -xeon07$ cd bscpkgs -.CE -Now you should be able to build nanos6 (which is probably already -compiled): -.CS -xeon07$ nix-build -A bsc.nanos6 -\&... -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -The installation is placed in the nix store (with the path stated in -the last line of the build process), with the \f(CWresult\fP symbolic -link pointing to the same location: -.CS -xeon07$ readlink result -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -.\" ################################################################### -.NH 3 -Configure garlic -.LP -In order to launch experiments in the -.I target -machine, it is required to configure nix to allow a directory to be -available during the build process, where the results will be stored -before being copied in the nix store. Create a new -.CI garlic -directory in your -personal cache directory and copy the full path: -.CS -xeon07$ mkdir -p \(ti/.cache/garlic -xeon07$ readlink -f \(ti/.cache/garlic -/home/Computational/rarias/.cache/garlic -.CE -Then create the nix configuration directory (if it has not already been -created): -.CS -xeon07$ mkdir -p \(ti/.config/nix -.CE -And add the following line in the -.CI \(ti/.config/nix/nix.conf -file, replacing it with the path you copied before: -.CS -.SM -extra-sandbox-paths = /garlic=/home/Computational/rarias/.cache/garlic -.CE -This option creates a virtual directory called -.CI /garlic -inside the build environment, whose contents are the ones you specify at -the right hand side of the equal sign (in this case the -.CI \(ti/.cache/garlic -directory). It will be used to allow the results of the experiments to -be passed to nix from the -.I target -machine. -.\" ################################################################### -.NH 3 -Run the garlic daemon (optional) -.LP -The garlic benchmark has a daemon which can be used to -automatically launch the experiments in the -.I target -machine on demand, when they are required to build other derivations, so -they can be launched without user interaction. The daemon creates some -FIFO pipes to communicate with the build environment, and must be -running to be able to run the experiments. To execute it, go to the -.CI bscpkgs/garlic -directory and run -.CS -xeon07$ nix-shell -nix-shell$ -.CE -to enter the nix shell (or specify the path to the -.CI garlic/shell.nix -file as argument). Then, run the daemon inside the nix shell: -.CS -nix-shell$ garlicd -garlicd: Waiting for experiments ... -.CE -Notice that the daemon stays running in the foreground, waiting for -experiments. At this moment, it can only process one experiment at a -time. -.\" =================================================================== -.NH 2 -Configure the login and target (MareNostrum 4) -.LP -In order to execute the programs in MareNostrum 4, you first need load -some utilities in the PATH. Add to the end of the file -\f(CW\(ti/.bashrc\fP in MareNostrum 4 the following line: -.CS -export PATH=/gpfs/projects/bsc15/nix/bin:$PATH -.CE -Then logout and login again (our source the \f(CW\(ti/.bashrc\fP file) -and check that now you have the \f(CWnix-develop\fP command available: -.CS -login1$ which nix-develop -/gpfs/projects/bsc15/nix/bin/nix-develop -.CE -The new utilities are available both in the login nodes and in the -compute (target) nodes, as they share the file system over the network. -.\" =================================================================== -.bp -.NH 1 -Development -.LP -During the development phase, a functional program is produced by -modifying its source code. This process is generally cyclic: the -developer needs to compile, debug and correct mistakes. We want to -minimize the delay times, so the programs can be executed as soon as -needed, but under a controlled environment so that the same behavior -occurs during the experimentation phase. -.PP -In particular, we want that several developers can reproduce the -same development environment so they can debug each other programs -when reporting bugs. Therefore, the environment must be carefully -controlled to avoid non-reproducible scenarios. -.PP -The current development environment provides an isolated shell with a -clean environment, which runs in a new mount namespace where access to -the filesystem is restricted. Only the project directory and the nix -store are available (with some other exceptions), to ensure that you -cannot accidentally link with the wrong library or modify the build -process with a forgotten environment variable in the \f(CW\(ti/.bashrc\fP -file. -.\" =================================================================== -.NH 2 -Getting the development tools -.LP -To create a development -environment, first copy or download the sources of your program (not the -dependencies) in a new directory placed in the target machine -(MareNostrum\~4). -.PP -The default environment contains packages commonly used to develop -programs, listed in the \fIgarlic/index.nix\fP file: -.\" FIXME: Unify garlic.unsafeDevelop in garlic.develop, so we can -.\" specify the packages directly -.CS -develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi - # Add more bsc packages here... - ]; - ... -.CE -If you need additional packages, add them to the list, so that they -become available in the environment. Those may include any dependency -required to build your program. -.PP -Then use the build machine (xeon07) to build the -.I garlic.develop -derivation: -.CS -build% nix-build -A garlic.develop -\&... -build% grep ln result -ln -fs /gpfs/projects/.../bin/stage1 .nix-develop -.CE -Copy the \fIln\fP command and run it in the target machine -(MareNostrum\~4), inside the new directory used for your program -development, to create the link \fI.nix-develop\fP (which is used to -remember your environment). Several environments can be stored in -different directories using this method, with different packages in each -environment. You will need -to rebuild the -.I garlic.develop -derivation and update the -.I .nix-develop -link after the package list is changed. Once the -environment link is created, there is no need to repeat these steps again. -.PP -Before entering the environment, you will need to access the required -resources for your program, which may include several compute nodes. -.\" =================================================================== -.NH 2 -Allocating resources for development -.LP -Our target machine (MareNostrum 4) provides an interactive shell, that -can be requested with the number of computational resources required for -development. To do so, connect to the login node and allocate an -interactive session: -.CS -% ssh mn1 -login% salloc ... -target% -.CE -This operation may take some minutes to complete depending on the load -of the cluster. But once the session is ready, any subsequent execution -of programs will be immediate. -.\" =================================================================== -.NH 2 -Accessing the developement environment -.PP -The utility program \fInix-develop\fP has been designed to access the -development environment of the current directory, by looking for the -\fI.nix-develop\fP file. It creates a namespace where the required -packages are installed and ready to be used. Now you can access the -newly created environment by running: -.CS -target% nix-develop -develop% -.CE -The spawned shell contains all the packages pre-defined in the -\fIgarlic.develop\fP derivation, and can now be accessed by typing the -name of the commands. -.CS -develop% which gcc -/nix/store/azayfhqyg9...s8aqfmy-gcc-wrapper-9.3.0/bin/gcc -develop% which gdb -/nix/store/1c833b2y8j...pnjn2nv9d46zv44dk-gdb-9.2/bin/gdb -.CE -If you need additional packages, you can add them in the -\fIgarlic/index.nix\fP file as mentioned previously. To keep the -same current resources, so you don't need to wait again for the -resources to be allocated, exit only from the development shell: -.CS -develop% exit -target% -.CE -Then update the -.I .nix-develop -link and enter into the new develop environment: -.CS -target% nix-develop -develop% -.CE -.\" =================================================================== -.NH 2 -Execution -.LP -The allocated shell can only execute tasks in the current node, which -may be enough for some tests. To do so, you can directly run your -program as: -.CS -develop$ ./program -.CE -If you need to run a multi-node program, typically using MPI -communications, then you can do so by using srun. Notice that you need -to allocate several nodes when calling salloc previously. The srun -command will execute the given program \fBoutside\fP the development -environment if executed as-is. So we re-enter the develop environment by -calling nix-develop as a wrapper of the program: -.\" FIXME: wrap srun to reenter the develop environment by its own -.CS -develop$ srun nix-develop ./program -.CE -.\" =================================================================== -.NH 2 -Debugging -.LP -The debugger can be used to directly execute the program if is executed -in only one node by using: -.CS -develop$ gdb ./program -.CE -Or it can be attached to an already running program by using its PID. -You will need to first connect to the node running it (say target2), and -run gdb inside the nix-develop environment. Use -.I squeue -to see the compute nodes running your program: -.CS -login$ ssh target2 -target2$ cd project-develop -target2$ nix-develop -develop$ gdb -p $pid -.CE -You can repeat this step to control the execution of programs running in -different nodes simultaneously. -.PP -In those cases where the program crashes before being able to attach the -debugger, enable the generation of core dumps: -.CS -develop$ ulimit -c unlimited -.CE -And rerun the program, which will generate a core file that can be -opened by gdb and contains the state of the memory when the crash -happened. Beware that the core dump file can be very large, depending on -the memory used by your program at the crash. -.\" =================================================================== -.NH 2 -Git branch name convention -.LP -The garlic benchmark imposes a set of requirements to be meet for each -application in order to coordinate the execution of the benchmark and -the gathering process of the results. -.PP -Each application must be available in a git repository so it can be -included into the garlic benchmark. The different combinations of -programming models and communication schemes should be each placed in -one git branch, which are referred to as \fIbenchmark branches\fP. At -least one benchmark branch should exist and they all must begin with the -prefix \f(CWgarlic/\fP (other branches will be ignored). -.PP -The branch name is formed by adding keywords separated by the "+" -character. The keywords must follow the given order and can only -appear zero or once each. At least one keyword must be included. The -following keywords are available: -.IP \f(CWmpi\fP 5m -A significant fraction of the communications uses only the standard MPI -(without extensions like TAMPI). -.IP \f(CWtampi\fP -A significant fraction of the communications uses TAMPI. -.IP \f(CWsend\fP -A significant part of the MPI communication uses the blocking family of -methods -.I MPI_Send , ( -.I MPI_Recv , -.I MPI_Gather "...)." -.IP \f(CWisend\fP -A significant part of the MPI communication uses the non-blocking family -of methods -.I MPI_Isend , ( -.I MPI_Irecv , -.I MPI_Igather "...)." -.IP \f(CWrma\fP -A significant part of the MPI communication uses remote memory access -(one-sided) methods -.I MPI_Get , ( -.I MPI_Put "...)." -.IP \f(CWseq\fP -The complete execution is sequential in each process (one thread per -process). -.IP \f(CWomp\fP -A significant fraction of the execution uses the OpenMP programming -model. -.IP \f(CWoss\fP -A significant fraction of the execution uses the OmpSs-2 programming -model. -.IP \f(CWtask\fP -A significant part of the execution involves the use of the tasking -model. -.IP \f(CWtaskfor\fP -A significant part of the execution uses the taskfor construct. -.IP \f(CWfork\fP -A significant part of the execution uses the fork-join model (including -hybrid programming techniques with parallel computations and sequential -communications). -.IP \f(CWsimd\fP -A significant part of the computation has been optimized to use SIMD -instructions. -.LP -In the -.URL #appendixA "Appendix A" -there is a flowchart to help the decision -process of the branch name. Additional user defined keywords may be -added at the end using the separator "+" as well. User keywords must -consist of capital alphanumeric characters only and be kept short. These -additional keywords must be different (case insensitive) to the already -defined above. Some examples: -.CS -garlic/mpi+send+seq -garlic/mpi+send+omp+fork -garlic/mpi+isend+oss+task -garlic/tampi+isend+oss+task -garlic/tampi+isend+oss+task+COLOR -garlic/tampi+isend+oss+task+COLOR+BTREE -.CE -.\" =================================================================== -.NH 2 -Initialization time -.LP -It is common for programs to have an initialization phase prior to the -execution of the main computation task which is the objective of the study. -The initialization phase is usually not considered when taking -measurements, but the time it takes to complete can limit seriously the -amount of information that can be extracted from the computation phase. -As an example, if the computation phase is in the order of seconds, but -the initialization phase takes several minutes, the number of runs would -need to be set low, as the units could exceed the time limits. Also, the -experimenter may be reluctant to modify the experiments to test other -parameters, as the waiting time for the results is unavoidably large. -.PP -To prevent this problem the programs must reduce the time of the -initialization phase to be no larger than the computation time. To do -so, the initialization phase can be optimized either with -parallelization, or it can be modified to store the result of the -initialization to the disk to be later at the computation phase. In the -garlic framework an experiment can have a dependency over the results of -another experiment (the results of the initialization). The -initialization results will be cached if the derivation is kept -invariant, when modifying the computation phase parameters. -.\" =================================================================== -.NH 2 -Measurement of the execution time -.LP -The programs must measure the wall time of the computation phase following a -set of rules. The way in which the wall time is measured is very important to -get accurate results. The measured time must be implemented by using a -monotonic clock which is able to correct the drift of the oscillator of -the internal clock due to changes in temperature. This clock must be -measured in C and C++ with: -.CS -clock_gettime(CLOCK_MONOTONIC, &ts); -.CE -A helper function can be used the approximate value of the clock in a -double precision float, in seconds: -.CS -double get_time() -{ - struct timespec tv; - if(clock_gettime(CLOCK_MONOTONIC, &tv) != 0) - { - perror("clock_gettime failed"); - exit(EXIT_FAILURE); - } - return (double)(ts.tv_sec) + - (double)ts.tv_nsec * 1.0e-9; -} -.CE -The start and end points must be measured after the synchronization of -all the processes and threads, so the complete computation work can be -bounded to fit inside the measured interval. An example for a MPI -program: -.CS -double start, end, delta_time; -MPI_Barrier(); -start = get_time(); -run_simulation(); -MPI_Barrier(); -end = get_time(); -delta_time = end - start; -.CE -.\" =================================================================== -.NH 2 -Format of the execution time -.LP -The measured execution time must be printed to the standard output -(stdout) in scientific notation with at least 7 significative digits. -The following the printf format (or the strict equivalent in other languages) -must be used: -.CS -printf("time %e\\n", delta_time); -.CE -The line must be printed alone and only once: for MPI programs, -only one process shall print the time: -.CS -if(rank == 0) printf("time %e\\n", delta_time); -.CE -Other lines can be printed in the stdout, but without the -.I time -prefix, so that the following pipe can be used to capture the line: -.CS -% ./app | grep "^time" -1.234567e-01 -.CE -Ensure that your program follows this convention by testing it with the -above -.I grep -filter; otherwise the results will fail to be parsed when building -the dataset with the execution time. -.\" =================================================================== -.bp -.NH 1 -Experimentation -.LP -During the experimentation, a program is studied by running it and -measuring some properties. The experimenter is in charge of the -experiment design, which is typically controlled by a single -.I nix -file placed in the -.CI garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 2 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CI /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CI /usr -or -.CI /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 2 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.\"##################################################################### -.NH 3 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CI /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.\"##################################################################### -.NH 3 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 3 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 3 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.DS L -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Where Safe Copies User Std -_ -trebuchet * no no yes yes -runexp login no no no yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate target no no no yes -control target yes no no yes -srun target yes no no yes -isolate target no yes no yes -_ -exec target yes yes no no -program target yes yes no no -_ -.TE -.DE -.QS -.SM -.B "Table 1" : -The stages of a complete execution pipeline. The -.I where -column determines where the stage is running, -.I safe -states if the stage begins the execution inside the sandbox, -.I user -if it can be executed directly by the user, -.I copies -if there are several instances running in parallel and -.I std -if is part of the standard execution pipeline. -.QE -.\" ################################################################### -.NH 2 -Writing the experiment -.LP -The experiments are generally written in the -.I nix -language as it provides very easy management for the packages an their -customization. An experiment file is formed by several parts, which -produce the execution pipeline when built. The experiment file describes -a function (which is typical in nix) and takes as argument an -attribute set with some common packages, tools and options: -.CS -{ stdenv, lib, bsc, stdexp, targetMachine, stages, garlicTools }: -.CE -The -.I bsc -attribute contains all the BSC and nixpkgs packages, as defined in the -overlay. The -.I stdexp -contains some useful tools and functions to build the experiments, like -the standard execution pipeline, so you don't need to redefine the -stages in every experiment. The configuration of the target machine is -specified in the -.I targetMachine -attribute which includes information like the number of CPUs per node or -the cache line length. It is used to define the experiments in such a -way that they are not tailored to an specific machine hardware -(sometimes this is not posible). All the execution stages are available -in the -.I stages -attribute which are used when some extra stage is required. And finally, -the -.I garlicTools -attribute provide some functions to aid common tasks when defining the -experiment configuration -.\" ################################################################### -.NH 3 -Experiment configuration -.LP -The next step is to define some variables in a -.CI let -\&... -.CI in -\&... -.CI ; -construct, to be used later. The first one, is the variable -configuration of the experiment called -.I varConf , -which include all -the factors that will be changed. All the attributes of this set -.I must -be arrays, even if they only contain one element: -.CS -varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; -}; -.CE -In this example, the variable -.I blocks -will be set to the values 1, 2 and 4; while -.I nodes -will remain set to 1 always. These variables are used later to build the -experiment configuration. The -.I varConf -is later converted to a list of attribute sets, where every attribute -contains only one value, covering all the combinations (the Cartesian -product is computed): -.CS -[ { blocks = 1; nodes = 1; } - { blocks = 2; nodes = 1; } - { blocks = 4; nodes = 1; } ] -.CE -These configurations are then passed to the -.I genConf -function one at a time, which is the central part of the description of -the experiment: -.CS -genConf = var: fix (self: targetMachine.config // { - expName = "example"; - unitName = self.expName + "-b" + toString self.blocks; - blocks = var.blocks; - cpusPerTask = 1; - tasksPerNode = self.hw.socketsPerNode; - nodes = var.nodes; -}); -.CE -It takes as input -.I one -configuration from the Cartesian product, for example: -.CS -{ blocks = 2; nodes = 1; } -.CE -And returns the complete configuration for that input, which usually -expand the input configuration with some derived variables along with -other constant parameters. The return value can be inspected by calling -the function in the interactive -.I "nix repl" -session: -.CS -nix-repl> genConf { blocks = 2; nodes = 1; } -{ - blocks = 2; - cpusPerTask = 1; - expName = "example"; - hw = { ... }; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - name = "mn4"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - nodes = 1; - sshHost = "mn1"; - tasksPerNode = 2; - unitName = "example-b2"; -} -.CE -Some configuration parameters were added by -.I targetMachine.config , -such as the -.I nixPrefix , -.I sshHost -or the -.I hw -attribute set, which are specific for the cluster they experiment is -going to run. Also, the -.I unitName -got assigned the proper name based on the number of blocks, but the -number of tasks per node were assigned based on the hardware description -of the target machine. -.PP -By following this rule, the experiments can easily be ported to machines -with other hardware characteristics, and we only need to define the -hardware details once. Then all the experiments will be updated based on -those details. -.\" ################################################################### -.NH 3 -Adding the stages -.LP -Once the configuration is ready, it will be passed to each stage of the -execution pipeline which will take the parameters it needs. The -connection between the parameters and how they are passed to each stage -is done either by convention or manually. There is a list of parameters that -are recognized by the standard pipeline stages. For example the -attribute -.I nodes , -it is recognized as the number of nodes in the standard -.I sbatch -stage when allocating resources: -.DS L -.TS -center; -lB lB cB cB lB -l l c c l. -_ -Stage Attribute Std Req Description -_ -* nixPrefix yes yes Path to the nix store in the target -unit expName yes yes Name of the experiment -unit unitName yes yes Name of the unit -control loops yes yes Number of runs of each unit -sbatch cpusPerTask yes yes Number of CPUs per task (process) -sbatch jobName yes yes Name of the job -sbatch nodes yes yes Number of nodes allocated -sbatch ntasksPerNode yes yes Number of tasks (processes) per node -sbatch qos yes no Name of the QoS queue -sbatch reservation yes no Name of the reservation -sbatch time yes no Maximum allocated time (string) -_ -exec argv no no Array of arguments to execve -exec env no no Environment variable settings -exec pre no no Code before the execution -exec post no no Code after the execution -_ -.TE -.DE -.QS -.SM -.B "Table 2" : -The attributes recognized by the stages in the execution pipeline. The -column -.I std -indicates if they are part of the standard execution pipeline. Some -attributes are required as indicated by the -.I req -column. -.QE -.LP -Other attribute names can be used to specify custom information used in -additional stages. The two most common stages required to complete the -pipeline are the -.I exec -and the -.I program . -Let see an example of -.I exec : -.CS -exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; -}; -.CE -The -.I exec -stage is defined as a function that uses the predefined -.I stages.exec -stage, which accepts the -.I argv -array, and sets the argv of the program. In our case, we fill the -.I argv -array by setting the -.I --blocks -parameter to the number of blocks, specified in the configuration in the -attribute -.I blocks . -The name of this attribute can be freely choosen, as long as the -.I exec -stage refers to it properly. The -.I nextStage -attribute is mandatory in all stages, and is automatically set when -building the pipeline. -.PP -The last step is to configure the actual program to be executed, -which can be specified as another stage: -.CS -program = {nextStage, conf, ...}: bsc.apps.example; -.CE -Notice that this function only returns the -.I bsc.apps.example -derivation, which will be translated to the path where the example -program is installed. If the program is located inside a directory -(typically -.I bin ), -it must define the attribute -.I programPath -in the -.I bsc.apps.example -derivation, which points to the executable program. An example: -.CS -stdenv.mkDerivation { -\& ... - programPath = "/bin/example"; -\& ... -}; -.CE -.\" ################################################################### -.NH 3 -Building the pipeline -.LP -With the -.I exec -and -.I program -stages defined and the ones provided by the standard pipeline, the -complete execution pipeline can be formed. To do so, the stages are -placed in an array, in the order they will be executed: -.CS -pipeline = stdexp.stdPipeline ++ [ exec program ]; -.CE -The attribute -.I stdexp.stdPipeline -contains the standard pipeline stages, and we only append our two -defined stages -.I exec -and -.I program . -The -.I pipeline -is an array of functions, and must be transformed in something that can -be executed in the target machine. For that purpose, the -.I stdexp -provides the -.I genExperiment -function, which takes the -.I pipeline -array and the list of configurations and builds the execution pipeline: -.CS -stdexp.genExperiment { inherit configs pipeline; } -.CE -The complete example experiment can be shown here: -.CS -{ stdenv, lib, stdexp, bsc, targetMachine, stages }: -with lib; -let - # Initial variable configuration - varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; - }; - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "example"; - unitName = "${expName}-b${toString blocks}"; - inherit (targetMachine.config) hw; - inherit (c) blocks nodes; - loops = 30; - ntasksPerNode = hw.socketPerNode; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; - }; - program = {nextStage, conf, ...}: bsc.garlic.apps.example; - pipeline = stdexp.stdPipeline ++ [ exec program ]; -in - stdexp.genExperiment { inherit configs pipeline; } -.CE -.\" ################################################################### -.NH 3 -Adding the experiment to the index -.LP -The experiment file must be located in a named directory inside the -.I garlic/exp -directory. The name is usually the program name. Once the experiment is -placed in a nix file, it must be added to the index of experiments, so -it can be build. The index is hyerarchically organized as attribute -sets, with -.I exp -containing all the experiments; -.I exp.example -the experiments of the -.I example -program; and -.I exp.example.test1 -referring to the -.I test1 -experiment of the -.I example -program. Additional attributes can be added, like -.I exp.example.test1.variantA -to handle more details. -.PP -For this example we are going to use the attribute path -.I exp.example.test -and add it to the index, in the -.I garlic/exp/index.nix -file. We append to the end of the attribute set, the following -definition: -.CS -\&... - example = { - test = callPackage ./example/test.nix { }; - }; -} -.CE -The experiment can now be built with: -.CS -builder% nix-build -A exp.example.test -.CE -.\" ################################################################### -.NH 2 -Recommendations -.PP -The complete results generally take a long time to be finished, so it is -advisable to design the experiments iteratively, in order to quickly -obtain some feedback. Some recommendations: -.BL -.LI -Start with one unit only. -.LI -Set the number of runs low (say 5) but more than one. -.LI -Use a small problem size, so the execution time is low. -.LI -Set the time limit low, so deadlocks are caught early. -.LE -.PP -As soon as the first runs are complete, examine the results and test -that everything looks good. You would likely want to check: -.BL -.LI -The resources where assigned as intended (nodes and CPU affinity). -.LI -No errors or warnings: look at stderr and stdout logs. -.LI -If a deadlock happens, it will run out of the time limit. -.LE -.PP -As you gain confidence over that the execution went as planned, begin -increasing the problem size, the number of runs, the time limit and -lastly the number of units. The rationale is that each unit that is -shared among experiments gets assigned the same hash. Therefore, you can -iteratively add more units to an experiment, and if they are already -executed (and the results were generated) is reused. -.\" ################################################################### -.bp -.NH 1 -Post-processing -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 2 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 2 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 2 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.NH 2 -Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. -.\" ################################################################### -.bp -.SH 1 -Appendix A: Branch name diagram -.LP -.TAG appendixA -.DS B -.SM -.PS 4.4/25.4 -copy "gitbranch.pic" -.PE -.DE diff --git a/garlic/ds/ctf-mode.py b/garlic/ds/ctf-mode.py deleted file mode 100644 index 669508f..0000000 --- a/garlic/ds/ctf-mode.py +++ /dev/null @@ -1,83 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - ctf_mode = {} - - with open(".garlic/time_mode_runtime.csv", "r") as f: - ctf_mode['runtime'] = float(f.readline()) - - with open(".garlic/time_mode_dead.csv", "r") as f: - ctf_mode['dead'] = float(f.readline()) - - with open(".garlic/time_mode_task.csv", "r") as f: - ctf_mode['task'] = float(f.readline()) - - tree['ctf_mode'] = ctf_mode - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/index.nix b/garlic/ds/index.nix deleted file mode 100644 index 95cb388..0000000 --- a/garlic/ds/index.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -rec { - - py = callPackage ./py.nix {}; - - std.timetable = py { script = ./std-timetable.py; compress = false; }; - osu.latency = py { script = ./osu-latency.py; }; - osu.bw = py { script = ./osu-bw.py; }; - perf.stat = py { script = ./perf-stat.py; }; - ctf.mode = py { script = ./ctf-mode.py; }; -} diff --git a/garlic/ds/osu-bw.py b/garlic/ds/osu-bw.py deleted file mode 100644 index 4b1df43..0000000 --- a/garlic/ds/osu-bw.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - bw = slices[1] - - tree['size'] = int(size) - tree['bw'] = float(bw) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/osu-latency.py b/garlic/ds/osu-latency.py deleted file mode 100644 index 2df08ee..0000000 --- a/garlic/ds/osu-latency.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - latency = slices[1] - - tree['size'] = int(size) - tree['latency'] = float(latency) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/perf-stat.py b/garlic/ds/perf-stat.py deleted file mode 100644 index a6d69bc..0000000 --- a/garlic/ds/perf-stat.py +++ /dev/null @@ -1,90 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open(".garlic/perf.csv", "r") as f: - lines = [line.strip() for line in f.readlines()] - - perf_data = {} - - for line in lines: - if len(line) == 0: continue - if line[0] == '#': continue - - slices = line.split(',') - if len(slices) != 7: - print("error: mismatched columns") - exit(1) - - name = slices[2].replace("-", "_") - value = float(slices[0]) - - perf_data[name] = value - - tree['perf'] = perf_data - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/py.nix b/garlic/ds/py.nix deleted file mode 100644 index 012a159..0000000 --- a/garlic/ds/py.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, python3 -, gzip -}: - -{ - script, - compress ? true -}: - -tree: - -stdenv.mkDerivation { - name = "dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = [ python3 gzip ]; - installPhase = '' - mkdir -p $out - ln -s ${tree} $out/tree - ln -s ${script} $out/script - - COMPRESS_DATASET=${toString compress} - - if [ $COMPRESS_DATASET ]; then - python $out/script $out/tree | gzip > $out/dataset.json.gz - ln -s dataset.json.gz $out/dataset - else - python $out/script $out/tree > $out/dataset.json - ln -s dataset.json $out/dataset - fi - ''; -} diff --git a/garlic/ds/std-timetable.py b/garlic/ds/std-timetable.py deleted file mode 100644 index c183097..0000000 --- a/garlic/ds/std-timetable.py +++ /dev/null @@ -1,77 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - with open(".garlic/total_time_start", "r") as f: - total_time_start = float(f.readline().strip()) - - with open(".garlic/total_time_end", "r") as f: - total_time_end = float(f.readline().strip()) - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - tree['total_time'] = total_time_end - total_time_start - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/exp/bigsort/genseq.nix b/garlic/exp/bigsort/genseq.nix deleted file mode 100644 index 99a59f1..0000000 --- a/garlic/exp/bigsort/genseq.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, n # must be a string -, dram # must be a string -, strace -}: - -with lib; - -# Ensure the arguments are strings, to avoid problems with large numbers -assert (isString n); -assert (isString dram); - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "genseq"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - #FIXME: We need a better mechanism to get the output paths - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/seq.dat"; - in - stages.exec { - inherit nextStage; - pre = '' - mkdir -p "${outDir}" - ''; - argv = [ n dram outFile ]; - post = '' - # Link the output here - ln -s "${outFile}" seq.dat - ''; - }; - - program = {...}: bsc.apps.bigsort.genseq; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/shuffle.nix b/garlic/exp/bigsort/shuffle.nix deleted file mode 100644 index a841720..0000000 --- a/garlic/exp/bigsort/shuffle.nix +++ /dev/null @@ -1,102 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, inputTre -, n -, dram -, garlicTools -, resultFromTrebuchet -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "shuffle"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - - # We need access to a fast shared filesystem to store the shuffled input - # dataset - extraMounts = [ fs.shared.fast ]; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - inputUnit = elemAt inputExp.units 0; - unitName = baseNameOf (toString inputUnit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/seq.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/shuffled.dat"; - - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the previous results are complete: - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - mkdir -p "${outDir}" - - # Copy the input as we are going to overwrite it - cp "${inFile}" "${outFile}" - ''; - argv = [ n dram outFile 16 64 ]; - post = '' - # Link the output here - ln -s "${outFile}" shuffled.dat - ''; - }; - - program = {...}: - bsc.apps.bigsort.shuffle; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/sort.nix b/garlic/exp/bigsort/sort.nix deleted file mode 100644 index 2ae73e9..0000000 --- a/garlic/exp/bigsort/sort.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, removeOutput ? true -, resultFromTrebuchet -, inputTre -}: - -with lib; - -let - varConf = { }; # Not used - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "bigsort"; - unitName = "${expName}.bs${toString bs}"; - inherit (targetMachine.config) hw; - - # bigsort options - n = 1024 * 1024 * 1024 / 8; # In longs (?) - bs = n; # In bytes - pageSize = bs / 2; # In bytes (?) - cc = bsc.icc; - mpi = bsc.impi; - gitBranch = "garlic/mpi+send+omp+task"; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - # All CPUs of the socket to each task - cpusPerTask = hw.cpusPerSocket; - jobName = "bigsort-${toString n}-${toString bs}-${gitBranch}"; - - # Load the dataset from the same fs where it was stored in the shuffle - # step. Also we use a local temp fs to store intermediate results. - extraMounts = [ fs.shared.fast fs.local.temp ]; - - rev = 1; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - unit = elemAt inputExp.units 0; - expName = baseNameOf (toString inputExp); - unitName = baseNameOf (toString unit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/shuffled.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/sorted.dat"; - tmpDir = fs.local.temp; - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the shuffled results are complete: nix needs to - # compute the hash of the execution log to write the path here. - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - # Create the output path - mkdir -p ${outDir} - - # Verbose args: - echo "INPUT = ${inFile}" - echo "OUTPUT = ${outFile}" - echo "TMPDIR = ${tmpDir}" - ''; - - argv = [ n bs inFile outFile tmpDir pageSize ]; - - # Optionally remove the potentially large output dataset - post = '' - # Link the output here - ln -s "${outFile}" sorted.dat - '' + optionalString (removeOutput) '' - # Remove the sorted output - stat "${outFile}" > "${outFile}.stat" - echo "file removed to save space" > "${outFile}" - ''; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.bigsort.sort.override { - inherit cc mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - #{ inherit configs pipeline; } - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/nbody.nix b/garlic/exp/cn6/nbody.nix deleted file mode 100644 index e48018f..0000000 --- a/garlic/exp/cn6/nbody.nix +++ /dev/null @@ -1,164 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 2 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "cn6-nbody"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - # Parameters for nbody - particles = 4 * 512 * hw.cpusPerSocket; - timesteps = 2; - blocksize = 512; - gitBranch = "garlic/tampi+isend+oss+task"; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:nanos6/forks/nanos6-fixes.git"; - ref = "distributed-instrumentation-fixes"; - rev = "80058512527961fbde9bd81474b0a29141d7982c"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - "--with-babeltrace2=${super.babeltrace2}" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - #dontStrip = true; - #NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/interoperability/tampi.git"; - ref = "master"; - rev = "f1e77e6f439a0e964e98b5e0a4738b2e95e4fd3d"; - }; - }); - }); - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = true - instrument.ctf.converter.fast = false - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - - # Add nanos6 and babeltrace2 binaries to the PATH - export PATH="$PATH:${bsc'.nanos6}/bin:${bsc'.babeltrace2}/bin" - - # Also add the babeltrace2 python module to python search path - export PYTHONPATH="$PYTHONPATH:${bsc'.babeltrace2}/lib/python3.8/site-packages" - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_nbody - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - nanos6-mergeprv "$tracedir" - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = with conf; [ - "-t" timesteps - "-p" particles - ]; - }; - - program = {nextStage, conf, ...}: bsc'.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/timediff.nix b/garlic/exp/cn6/timediff.nix deleted file mode 100644 index 7d2d42f..0000000 --- a/garlic/exp/cn6/timediff.nix +++ /dev/null @@ -1,171 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 1 2 4 8 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "timediff"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:rarias/nanos6.git"; - ref = "rodrigo"; - rev = "5cbeabb4e0446c2c293cc3005f76e6139465caee"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - dontStrip = true; - NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/tampi.git"; - ref = "instrument"; - rev = "6e4294299bf761a1cc31f4181d9479cefa1c7f3e"; - }; - }); - - # We use the latest commit in master as src for cn6 - cn6Git = ((super.cn6.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/cn6.git"; - ref = "master"; - rev = "1d23d01d60164b8641746d5a204128a9d31b9650"; - }; - })).override { enableTest = true; }); - - cn6 = self.cn6Git; - }); - - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_timediff_mpi - - # Convert CTF trace to PRV - ${bsc'.cn6}/bin/cn6 $tracedir/$rank - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - ${bsc'.cn6}/bin/merge-prv $tracedir - - # We need some tools the path - export PATH="$PATH:${bsc'.babeltrace2}/bin:${pkgs.ministat}/bin" - - ${bsc'.cn6}/bin/sync-err.sh $tracedir - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - program = "${bsc'.cn6}/bin/timediff_mpi"; - argv = [ conf.cpusPerTask ]; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity.nix b/garlic/exp/creams/granularity.nix deleted file mode 100644 index 3fab638..0000000 --- a/garlic/exp/creams/granularity.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - granul = range2 4 128; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+omp+task" - ] ++ optionals (enableExtended) [ - #"garlic/mpi+send+omp+fork" # Don't use fork for granularity - "garlic/mpi+send+seq" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+oss+task" - ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - nodes = [ 1 ] ++ optionals (enableExtended) (range2 2 16); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-gran"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) gitBranch nodes iterations; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity16.nix b/garlic/exp/creams/granularity16.nix deleted file mode 100644 index fa8dc6f..0000000 --- a/garlic/exp/creams/granularity16.nix +++ /dev/null @@ -1,132 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = [ 1 2 4 8 16 32 ]; - granul = [ 1 2 4 8 16 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - #"garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-granularity16"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique ( - filter (c: !(c.granul == 1 && c.sizeFactor >= 32))(stdexp.buildConfigs { - inherit varConf genConf; - })); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/size.nix b/garlic/exp/creams/size.nix deleted file mode 100644 index dea3c8d..0000000 --- a/garlic/exp/creams/size.nix +++ /dev/null @@ -1,131 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = range2 1 32; - baseGranul = [ 1 ] ++ optionals (enableExtended) [ 2 4 8 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ (optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-size"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor baseGranul; - granul = getGranul gitBranch (max 2 (baseGranul * sizeFactor)); - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/ss.nix b/garlic/exp/creams/ss.nix deleted file mode 100644 index 86723be..0000000 --- a/garlic/exp/creams/ss.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - nodes = range2 1 16; - granul = [ 16 ] ++ optionals (enableExtended) [ 8 32 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-ss"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/examples/granularity.nix b/garlic/exp/examples/granularity.nix deleted file mode 100644 index 1c01244..0000000 --- a/garlic/exp/examples/granularity.nix +++ /dev/null @@ -1,189 +0,0 @@ -# This file defines an experiment. It is designed as a function that takes -# several parameters and returns a derivation. This derivation, when built will -# create several scripts that can be executed and launch the experiment. - -# These are the inputs to this function: an attribute set which must contain the -# following keys: -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -# We import in the scope the content of the `lib` attribute, which -# contain useful functions like `toString`, which will be used later. This is -# handy to avoid writting `lib.tostring`. - -with lib; - -# We also have some functions specific to the garlic benchmark which we import -# as well. Take a look at the garlic/tools.nix file for more details. -with garlicTools; - -# The `let` keyword allows us to define some local variables which will be used -# later. It works as the local variable concept in the C language. -let - - # Initial variable configuration: every attribute in this set contains lists - # of options which will be used to compute the configuration of the units. The - # cartesian product of all the values will be computed. - varConf = { - # In this case we will vary the columns and rows of the blocksize. This - # configuration will create 3 x 2 = 6 units. - cbs = [ 256 1024 4096 ]; - rbs = [ 512 1024 ]; - }; - - # Generate the complete configuration for each unit: genConf is a function - # that accepts the argument `c` and returns a attribute set. The attribute set - # is formed by joining the configuration of the machine (which includes - # details like the number of nodes or the architecture) and the configuration - # that we define for our units. - # - # Notice the use of the `rec` keyword, which allows us to access the elements - # of the set while is being defined. - genConf = c: targetMachine.config // rec { - - # These attributes are user defined, and thus the user will need to handle - # them manually. They are not read by the standard pipeline: - - # Here we load the `hw` attribute from the machine configuration, so we can - # access it, for example, the number of CPUs per socket as hw.cpusPerSocket. - hw = targetMachine.config.hw; - - # These options will be used by the heat app, be we write them here so they - # are stored in the unit configuration. - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - - # The blocksize is set to the values passed in the `c` parameter, which will - # be set to one of all the configurations of the cartesian product. for - # example: cbs = 256 and rbs = 512. - # We can also write `inherit (c) cbs rbs`, as is a shorthand notation. - cbs = c.cbs; - rbs = c.rbs; - - # The git branch is specified here as well, as will be used when we specify - # the heat app - gitBranch = "garlic/tampi+isend+oss+task"; - - # ------------------------------------------------------------------------- - - # These attributes are part of the standard pipeline, and are required for - # each experiment. They are automatically recognized by the standard - # execution pipeline. - - # The experiment name: - expName = "example-granularity-heat"; - - # The experimental unit name. It will be used to create a symlink in the - # index (at /gpfs/projects/bsc15/garlic/$USER/index/) so you can easily find - # the unit. Notice that the symlink is overwritten each time you run a unit - # with the same same. - # - # We use the toString function to convert the numeric value of cbs and rbs - # to a string like: "example-granularity-heat.cbs-256.rbs-512" - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - # Repeat the execution of each unit a few times: this option is - # automatically taken by the experiment, which will repeat the execution of - # the program that many times. It is recommended to run the app at least 30 - # times, but we only used 10 here for demostration purposes (as it will be - # faster to run) - loops = 10; - - # Resources: here we configure the resources in the machine. The queue to be - # used is `debug` as is the fastest for small jobs. - qos = "debug"; - - # Then the number of MPI processes or tasks per node: - ntasksPerNode = 1; - - # And the number of nodes: - nodes = 1; - - # We use all the CPUs available in one socket to each MPI process or task. - # Notice that the number of CPUs per socket is not specified directly. but - # loaded from the configuration of the machine that will be used to run our - # experiment. The affinity mask is set accordingly. - cpusPerTask = hw.cpusPerSocket; - - # The time will limit the execution of the program in case of a deadlock - time = "02:00:00"; - - # The job name will appear in the `squeue` and helps to identify what is - # running. Currently is set to the name of the unit. - jobName = unitName; - }; - - # Using the `varConf` and our function `genConf` we compute a list of the - # complete configuration of every unit. - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Now that we have the list of configs, we need to write how that information - # is used to run our program. In our case we will use some params such as the - # number of rows and columns of the input problem or the blocksize as argv - # values. - - # The exec stage is used to run a program with some arguments. - exec = {nextStage, conf, ...}: stages.exec { - # All stages require the nextStage attribute, which is passed as parameter. - inherit nextStage; - - # Then, we fill the argv array with the elements that will be used when - # running our program. Notice that we load the attributes from the - # configuration which is passed as argument as well. - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # This program requires a file called `head.conf` in the current directory. - # To do it, we run this small script in the `pre` hook, which simple runs - # some commands before running the program. Notice that this command is - # executed in every MPI task. - pre = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - # The program stage is only used to specify which program we should run. - # We use this stage to specify build-time parameters such as the gitBranch, - # which will be used to fetch the source code. We use the `override` function - # of the `bsc.garlic.apps.heat` derivation to change the input paramenters. - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - # Other stages may be defined here, in case that we want to do something - # additional, like running the program under `perf stats` or set some - # envionment variables. - - # Once all the stages are defined, we build the pipeline array. The - # `stdexp.stdPipeline` contains the standard pipeline stages, so we don't need - # to specify them. We only specify how we run our program, and what program - # exactly, by adding our `exec` and `program` stages: - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -# Then, we use the `configs` and the `pipeline` just defined inside the `in` -# part, to build the complete experiment: -in - - # The `stdexp.genExperiment` function generates an experiment by calling every - # stage of the pipeline with the different configs, and thus creating - # different units. The result is the top level derivation which is the - # `trebuchet`, which is the script that, when executed, launches the complete - # experiment. - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix deleted file mode 100644 index 63e88de..0000000 --- a/garlic/exp/fwi/common.nix +++ /dev/null @@ -1,128 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -}: - -with lib; - -# Common definitions used by fwi experiments -rec { - - branchesWithoutBlocksize = [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+seq" - ]; - - # Returns true if the given config is in the forkJoinBranches list - needsBlocksize = c: ! any (e: c.gitBranch == e) branchesWithoutBlocksize; - - # Set the blocksize to null for the fork join branch - fixBlocksize = c: if (needsBlocksize c) then c - else (c // { blocksize = null; }); - - # Generate the configs by filtering the unneded blocksizes - getConfigs = {varConf, genConf}: - let - allConfigs = stdexp.buildConfigs { inherit varConf genConf; }; - in - # The unique function ensures that we only run one config for the fork - # join branch, even if we have multiple blocksizes. - unique (map fixBlocksize allConfigs); - - getResources = {gitBranch, hw}: - if (gitBranch == "garlic/mpi+send+seq") then { - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - } else { - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - }; - - exec = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - - ioFreq = if (conf.enableIO) then (conf.ioFreq or "-1") else "9999"; - - in stages.exec { - inherit nextStage; - - # FIXME: FWI should allow the I/O directory to be specified as a - # parameter - pre = '' - FWI_SRUNDIR=$(pwd) - FWI_EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - FWI_PARAMS="${fwiParams}/fwi_params.txt" - FWI_FREQ="${fwiParams}/fwi_frequencies.txt" - - # Run fwi in a directory with fast local storage - mkdir -p "$FWI_EXECDIR" - cd "$FWI_EXECDIR" - - # Only generate the input if we have the CPU 0 (once per node) - if grep -o 'Cpus_allowed_list:[[:space:]]0' \ - /proc/self/status > /dev/null; - then - FWI_CAPTAIN=1 - fi - - if [ $FWI_CAPTAIN ]; then - >&2 echo "generating the input dataset" - ${fwiParams}/bin/ModelGenerator -m "$FWI_PARAMS" "$FWI_FREQ" - fi - - echo >&2 "Current dir: $(pwd)" - echo >&2 "Using PARAMS=$FWI_PARAMS and FREQ=$FWI_FREQ" - '' + optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - ''; - - argv = [ - ''"$FWI_PARAMS"'' - ''"$FWI_FREQ"'' - ] ++ optional (needsBlocksize conf) conf.blocksize ++ [ - "-1" # Fordward steps - "-1" # Backward steps - ioFreq # Write/read frequency - ]; - - post = '' - # Go back to the garlic out directory - cd "$FWI_SRUNDIR" - - if [ $FWI_CAPTAIN ]; then - '' + optionalString (conf.enableCTF) '' - # FIXME: We should specify the path in the nanos6 config, so we - # can avoid the race condition while they are generating the - # traces - sleep 3 - - # Save the traces - mv "$FWI_EXECDIR"/trace_* . - '' + '' - rm -rf "$FWI_EXECDIR" - fi - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - in - apps.fwi.solver.override { - inherit (conf) gitBranch; - inherit fwiParams; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix deleted file mode 100644 index 15bc484..0000000 --- a/garlic/exp/fwi/granularity.nix +++ /dev/null @@ -1,71 +0,0 @@ -# Regular granularity test for FWI - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+isend+oss+task" ]; - blocksize = range2 1 256; - n = [ {nx=100; nz=100; ny=8000; ntpn=2; nodes=1;} ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-granularity"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz ntpn nodes; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = ntpn; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/io.nix b/garlic/exp/fwi/io.nix deleted file mode 100644 index 0d107d7..0000000 --- a/garlic/exp/fwi/io.nix +++ /dev/null @@ -1,75 +0,0 @@ -# Test FWI variants based on tasks with and without I/O. -# This experiment solves a computationally expensive input which brings the -# storage devices to saturation when I/O is enabled. The same input runs -# without I/O for comparison purposes. Also, a range of block sizes -# deemed as efficient according to the granularity experiment are -# explored. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -, enableExtended ? false -}: - -with lib; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+send+oss+task" ]; - blocksize = [ 1 2 4 8 ]; - n = [ {nx=500; nz=500; ny=16000;} ]; - nodes = if (enableExtended) then range2 1 16 else [ 4 ]; - enableIO = [ false true ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-io"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + (if (enableIO) then "-io1" else "-io0") - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize enableIO nodes; - inherit (c.n) nx ny nz; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - enableCTF = false; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/reuse.nix b/garlic/exp/fwi/reuse.nix deleted file mode 100644 index 9a725e2..0000000 --- a/garlic/exp/fwi/reuse.nix +++ /dev/null @@ -1,91 +0,0 @@ -# This test compares a FWI version using poor data locality (+NOREUSE) versus -# the optimized version (used for all other experiments). Follows a pseudocode -# snippet illustrating the fundamental difference between version. -# -# NOREUSE -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# for (y) for (x) for (z) -# computB(v[y][x][z]); -# for (y) for (x) for (z) -# computC(v[y][x][z]); -# -# Optimized version -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# computB(v[y][x][z]); -# computC(v[y][x][z]); - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -}: - -with lib; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/mpi+send+oss+task" - "garlic/mpi+send+oss+task+NOREUSE" - ]; - - blocksize = [ 1 2 4 8 ]; - - n = [ {nx=300; ny=2000; nz=300;} ]; # / half node - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-reuse"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - enableCTF = false; - enableIO = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix deleted file mode 100644 index c54370e..0000000 --- a/garlic/exp/fwi/ss.nix +++ /dev/null @@ -1,90 +0,0 @@ -# Strong scaling test for FWI variants based on tasks. This -# experiment explores a range of block sizes deemed as efficient -# according to the granularity experiment. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/tampi+isend+oss+task" - ] ++ optionals (enableExtended) [ - "garlic/tampi+send+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+send+omp+fork" - # FIXME: the mpi pure version has additional constraints with the - # number of planes in Y. By now is disabled. - #"garlic/mpi+send+seq" - ]; - - blocksize = if (enableExtended) - then range2 1 16 - else [ 2 ]; - - n = [ { nx=100; ny=8000; nz=100; } ]; - - nodes = range2 1 16; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: machineConfig // rec { - expName = "fwi-ss"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - nodes = c.nodes; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/heat/granularity.nix b/garlic/exp/heat/granularity.nix deleted file mode 100644 index 8ebe5db..0000000 --- a/garlic/exp/heat/granularity.nix +++ /dev/null @@ -1,175 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, writeText -, enablePerf ? false -, enableCTF ? false -, enableHWC ? false -, enableExtended ? false -}: - -# TODO: Finish HWC first -assert (enableHWC == false); - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { - cbs = range2 32 4096; - rbs = range2 32 4096; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "heat"; - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - inherit (machineConfig) hw; - - # heat options - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - inherit (c) cbs rbs; - gitBranch = "garlic/tampi+isend+oss+task"; - - # Repeat the execution of each unit 30 times - loops = 10; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - # Assign one socket to each task (only one process) - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - filterConfigs = c: let - # Too small sizes lead to huge overheads - goodSize = (c.cbs * c.rbs >= 1024); - # When the extended units are not enabled, we only select those in - # the diagonal. - extended = if (enableExtended) then true - else c.cbs == c.rbs; - in - goodSize && extended; - - # Compute the array of configurations - configs = filter (filterConfigs) (stdexp.buildConfigs { - inherit varConf genConf; - }); - - perf = {nextStage, conf, ...}: stages.perf { - inherit nextStage; - perfOptions = "stat -o .garlic/perf.csv -x , " + - "-e cycles,instructions,cache-references,cache-misses"; - }; - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - # FIXME: We should run a hook *after* srun has ended, so we can - # execute it in one process only (not in N ranks). This hack works - # with one process only. Or be able to compute the name of the trace - # directory so we can begin the conversion in parallel - post = assert (conf.nodes * conf.ntasksPerNode == 1); '' - tracedir=$(ls -d trace_* | head -1) - echo "using tracedir=$tracedir" - - offset=$(grep 'offset =' $tracedir/ctf/ust/uid/1000/64-bit/metadata | \ - grep -o '[0-9]*') - echo "offset = $offset" - - start_time=$(awk '/^start_time / {print $2}' stdout.log) - end_time=$(awk '/^end_time / {print $2}' stdout.log) - - begin=$(awk "BEGIN{print $start_time*1e9 - $offset}") - end=$(awk "BEGIN{print $end_time*1e9 - $offset}") - - echo "only events between $begin and $end" - - ${bsc.cn6}/bin/cn6 -s $tracedir - - ${bsc.cn6}/bin/cut $begin $end < $tracedir/prv/trace.prv |\ - ${bsc.cn6}/bin/hcut 1 ${toString conf.cpusPerTask} \ - > $tracedir/prv/trace-cut.prv - - ${bsc.cn6}/bin/dur 6400025 0 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_dead.csv & - - ${bsc.cn6}/bin/dur 6400025 1 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_runtime.csv & - - ${bsc.cn6}/bin/dur 6400025 3 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_task.csv & - - wait - - # Remove the traces at the end, as they are huge - rm -rf $tracedir - ''; - # TODO: To enable HWC we need to first add a taskwait before the - # first get_time() measurement, otherwise we get the HWC of the - # main task, which will be huge. - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # The next stage is the program - env = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ - (optional enablePerf perf) ++ - (optional enableCTF ctf) ++ - [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/common.nix b/garlic/exp/hpcg/common.nix deleted file mode 100644 index a7c19e3..0000000 --- a/garlic/exp/hpcg/common.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, callPackage -}: - -with lib; - -rec { - - checkInput = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = optionalString (! (conf.enableGen or false)) ( - let - gen = callPackage ./gen.nix { }; - inputTre = gen.getInputTre conf; - exp = inputTre.experiment; - unit = elemAt exp.units 0; - expName = baseNameOf (toString exp); - unitName = baseNameOf (toString unit); - inputPath = "$GARLIC_OUT/${expName}/${unitName}/1"; - in - '' - # Force the generation of the input resultTree as a dependency: - # ${toString inputTre.result} - - # Ensure the input dataset is still available - export HPCG_INPUT_PATH="${toString inputPath}" - - if [ ! -e "$HPCG_INPUT_PATH" ]; then - >&2 echo "Missing input dataset: $HPCG_INPUT_PATH" - exit 1 - fi - '' - ); - }; - - getSizePerTask = cpusPerTask: sizePerCpu: - mapAttrs (name: val: val * cpusPerTask) sizePerCpu; - - exec = {nextStage, conf, ...}: let - actionArg = if (conf.enableGen or false) - then "--store=." - else "--load=\"$HPCG_INPUT_PATH\""; - - in stages.exec { - inherit nextStage; - argv = [ - "--nx=${toString conf.sizePerTask.x}" - "--ny=${toString conf.sizePerTask.y}" - "--nz=${toString conf.sizePerTask.z}" - "--npx=${toString conf.nprocs.x}" - "--npy=${toString conf.nprocs.y}" - "--npz=${toString conf.nprocs.z}" - "--nblocks=${toString conf.nblocks}" - "--ncomms=${toString conf.ncomms}" - # The input symlink is generated by the input stage, which is generated by - # the genInput function. - actionArg - ] ++ optional (conf.disableAspectRatio or false) "--no-ar=1"; - }; - - program = {nextStage, conf, ...}: bsc.apps.hpcg.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ - checkInput - exec program ]; -} diff --git a/garlic/exp/hpcg/gen.nix b/garlic/exp/hpcg/gen.nix deleted file mode 100644 index d34eccc..0000000 --- a/garlic/exp/hpcg/gen.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -rec { - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-gen"; - unitName = expName - + "-nodes${toString nodes}" - + "-spt.z${toString sizePerTask.z}"; - - inherit (targetMachine.config) hw; - - # Inherit options from the current conf - inherit (c) sizePerTask nprocs disableAspectRatio gitBranch - cpusPerTask ntasksPerNode nodes; - - # nblocks and ncomms are ignored from c - ncomms = 1; - nblocks = 1; - - # We only need one run - loops = 1; - - # Generate the input - enableGen = true; - - # Resources - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - getInputTre = conf: stdexp.genExperiment { - configs = [ (genConf conf) ]; - pipeline = common.pipeline; - }; -} diff --git a/garlic/exp/hpcg/granularity.nix b/garlic/exp/hpcg/granularity.nix deleted file mode 100644 index 7c3d2a8..0000000 --- a/garlic/exp/hpcg/granularity.nix +++ /dev/null @@ -1,76 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - blocksPerCpu = range2 0.5 256; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-granularity"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = maxNodes / nodes; - - # hpcg options - inherit (c) blocksPerCpu gitBranch; - baseSizeZ = 16; - nodes = 1; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = floatTruncate (blocksPerCpu * cpusPerTask); - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 3; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/scaling.nix b/garlic/exp/hpcg/scaling.nix deleted file mode 100644 index afb1aee..0000000 --- a/garlic/exp/hpcg/scaling.nix +++ /dev/null @@ -1,78 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -, enableStrong ? true -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - nodes = range2 1 maxNodes; - baseSizeZ = if (enableExtended) then [ 8 16 ] else [ 16 ]; - blocksPerCpu = if (enableExtended) then range2 1 8 else [ 4 ]; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = if (enableStrong) then "hpcg-ss" else "hpcg-ws"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = if (enableStrong) then maxNodes / nodes else 1; - - # hpcg options - inherit (c) nodes blocksPerCpu gitBranch; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = c.baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/size.nix b/garlic/exp/hpcg/size.nix deleted file mode 100644 index 1124323..0000000 --- a/garlic/exp/hpcg/size.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - # Initial variable configuration - varConf = { - sizeFactor = [ 1 2 4 8 16 32 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-size"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-sf${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # hpcg options - inherit (c) sizeFactor; - gitBranch = "garlic/tampi+isend+oss+task"; - nodes = 16; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { x = 2; y = 2; z = 4 * sizeFactor; }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - blocksPerCpu = 4; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 5; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix deleted file mode 100644 index 2f2bcc4..0000000 --- a/garlic/exp/index.nix +++ /dev/null @@ -1,108 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = rec { - granularity = callPackage ./nbody/granularity.nix { }; - ss = callPackage ./nbody/ss.nix { }; - numa = callPackage ./nbody/numa.nix { }; - }; - - saiph = { - granularity = callPackage ./saiph/granularity.nix { }; - ss = callPackage ./saiph/ss.nix { }; - }; - - creams = rec { - ss = callPackage ./creams/ss.nix { }; - granularity = callPackage ./creams/granularity.nix { }; - size = callPackage ./creams/size.nix { }; - granularity16 = callPackage ./creams/granularity16.nix { }; - - # These experiments are the extended versions of the previous - # ones. We split them so we can keep a reasonable execution time - big.granularity = granularity.override { enableExtended = true; }; - big.ss = granularity.override { enableExtended = true; }; - }; - - hpcg = rec { - granularity = callPackage ./hpcg/granularity.nix { }; - ss = callPackage ./hpcg/scaling.nix { }; - ws = ss.override { enableStrong=false; }; - size = callPackage ./hpcg/size.nix { }; - - big.ss = ss.override { enableExtended = true; }; - }; - - heat = rec { - granularity = callPackage ./heat/granularity.nix { }; - cache = granularity.override { enablePerf = true; }; - ctf = granularity.override { enableCTF = true; }; - }; - - bigsort = rec { - genseq = callPackage ./bigsort/genseq.nix { - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - }; - - shuffle = callPackage ./bigsort/shuffle.nix { - inputTre = genseq; - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - inherit (bsc.garlic.pp) resultFromTrebuchet; - }; - - sort = callPackage ./bigsort/sort.nix { - inputTre = shuffle; - inherit (bsc.garlic.pp) resultFromTrebuchet; - removeOutput = false; - }; - }; - - slurm = { - cpu = callPackage ./slurm/cpu.nix { }; - sigsegv = callPackage ./slurm/sigsegv.nix { }; - exit1 = callPackage ./slurm/exit1.nix { }; - }; - - lulesh = { - test = callPackage ./lulesh/test.nix { }; - }; - - fwi = rec { - granularity = callPackage ./fwi/granularity.nix { }; - ss = callPackage ./fwi/ss.nix { }; - reuse = callPackage ./fwi/reuse.nix { }; - io = callPackage ./fwi/io.nix { }; - - # Extended experiments - big.io = io.override { enableExtended = true; }; - }; - - osu = rec { - latency = callPackage ./osu/latency.nix { }; - latencyShm = latency.override { interNode = false; }; - latencyMt = latency.override { enableMultithread = true; }; - latencyMtShm = latency.override { enableMultithread = true; interNode = true; }; - bw = callPackage ./osu/bw.nix { }; - impi = callPackage ./osu/impi.nix { }; - bwShm = bw.override { interNode = false; }; - mtu = callPackage ./osu/mtu.nix { }; - eager = callPackage ./osu/eager.nix { }; - }; - - examples = { - granularity = callPackage ./examples/granularity.nix { }; - }; - - cn6 = { - timediff = callPackage ./cn6/timediff.nix { }; - nbody = callPackage ./cn6/nbody.nix { }; - }; -} diff --git a/garlic/exp/lulesh/test.nix b/garlic/exp/lulesh/test.nix deleted file mode 100644 index eb9b078..0000000 --- a/garlic/exp/lulesh/test.nix +++ /dev/null @@ -1,85 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - - # Initial variable configuration - varConf = with bsc; { - gitBranch = [ - "garlic/mpi+isend+seq" - "garlic/tampi+isend+oss+taskloop" - "garlic/tampi+isend+oss+taskfor" - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+seq" - "garlic/mpi+isend+oss+task" - "garlic/mpi+isend+omp+fork" - "garlic/tampi+isend+oss+taskloopfor" - ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "lulesh"; - unitName = "${expName}-test"; - inherit (machineConfig) hw; - - # options - iterations = 10; - size = 30; - gitBranch = c.gitBranch; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - /* Lulesh options: - -q : quiet mode - suppress all stdout - -i : number of cycles to run - -s : length of cube mesh along side - -r : Number of distinct regions (def: 11) - -b : Load balance between regions of a domain (def: 1) - -c : Extra cost of more expensive regions (def: 1) - -f : Number of files to split viz dump into (def: (np+10)/9) - -p : Print out progress - -v : Output viz file (requires compiling with -DVIZ_MESH - -h : This message - */ - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-i" iterations "-s" size ]; - }; - - apps = bsc.garlic.apps; - - program = {nextStage, conf, ...}: apps.lulesh.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/common.nix b/garlic/exp/nbody/common.nix deleted file mode 100644 index 8570332..0000000 --- a/garlic/exp/nbody/common.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, numactl -, garlicTools -}: - -with lib; -with garlicTools; - -rec { - getConfigs = {varConf, genConf}: stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec - ( - { - inherit nextStage; - argv = with conf; [ "-t" timesteps "-p" particles ]; - } - # Use numactl to use the interleave policy if requested (default is - # false) - // optionalAttrs (conf.interleaveMem or false) { - program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}"; - } - ); - - program = {nextStage, conf, ...}: bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/nbody/granularity.nix b/garlic/exp/nbody/granularity.nix deleted file mode 100644 index 4b59bae..0000000 --- a/garlic/exp/nbody/granularity.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 64 2048; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - gitBranch = c.gitBranch; - - expName = "nbody-granularity"; - unitName = expName + - "-${toString gitBranch}" + - "-bs${toString blocksize}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/numa.nix b/garlic/exp/nbody/numa.nix deleted file mode 100644 index a860fdc..0000000 --- a/garlic/exp/nbody/numa.nix +++ /dev/null @@ -1,64 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, numactl -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 256 1024; - gitBranch = [ "garlic/tampi+send+oss+task" ]; - attachToSocket = [ true false ]; - interleaveMem = [ true false ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) attachToSocket interleaveMem gitBranch blocksize; - - expName = "nbody-numa"; - unitName = expName + - "-${toString gitBranch}" + - "-bs.${toString blocksize}" + - "-tpn.${toString ntasksPerNode}" + - "-interleave.${if (interleaveMem) then "yes" else "no"}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = if (attachToSocket) - then hw.cpusPerSocket - else hw.cpusPerNode; - ntasksPerNode = if (attachToSocket) - then hw.socketsPerNode - else 1; - nodes = 4; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/granularity-oss.nix b/garlic/exp/nbody/old/granularity-oss.nix deleted file mode 100644 index 2dbe81c..0000000 --- a/garlic/exp/nbody/old/granularity-oss.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Initial variable configuration - varConf = { - blocksize = [ 128 256 512 1024 2048 4096 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4096 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - - gitBranch = "garlic/oss+task"; - expName = "nbody-granularity"; - unitName = expName + "-bs${toString blocksize}"; - - loops = 30; - - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "-t" conf.timesteps "-p" conf.particles ]; - }; - - program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/nblocks.nix b/garlic/exp/nbody/old/nblocks.nix deleted file mode 100644 index 4e7dd94..0000000 --- a/garlic/exp/nbody/old/nblocks.nix +++ /dev/null @@ -1,104 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 7 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, gitBranch ? "garlic/tampi+send+oss+task" -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-nblocks"; - unitName = "${self.expName}${toString self.nblocks}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps gitBranch enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - nodes = 1; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/scaling.nix b/garlic/exp/nbody/old/scaling.nix deleted file mode 100644 index 47fd221..0000000 --- a/garlic/exp/nbody/old/scaling.nix +++ /dev/null @@ -1,110 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 6 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - nodes = [ 1 2 4 8 16 ]; - gitBranch = [ - "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-scaling"; - unitName = self.expName + - "-nb${toString self.nblocks}"+ - "-nodes${toString self.nodes}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit (conf) cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/strong-scaling-oss.nix b/garlic/exp/nbody/old/strong-scaling-oss.nix deleted file mode 100644 index 2a6f789..0000000 --- a/garlic/exp/nbody/old/strong-scaling-oss.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcsAndParticles = [ 1 2 4 8 16 32 48 ]; - input = [ - { numParticles=1 ; cpuMask="0x1"; } - { numParticles=2 ; cpuMask="0x3"; } - { numParticles=4 ; cpuMask="0xf"; } - { numParticles=8 ; cpuMask="0xff"; } - { numParticles=16; cpuMask="0xffff"; } - { numParticles=32; cpuMask="0xffffffff"; } - { numParticles=48; cpuMask="0xffffffffffff"; } - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - inherit (c.input) numParticles cpuMask; - particles = 1024 * numParticles * 2; - timesteps = 10; - blocksize = 1024; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${toString numParticles}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-mpi.nix b/garlic/exp/nbody/old/weak-scaling-mpi.nix deleted file mode 100644 index e830a42..0000000 --- a/garlic/exp/nbody/old/weak-scaling-mpi.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcs = [ 1 2 4 8 16 32 48 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) numProcs; - cc = icc; - mpi = impi; - gitBranch = "garlic/mpi+send"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = numProcs; - nodes = 1; - time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "nbody-bs-${toString numProcs}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-oss.nix b/garlic/exp/nbody/old/weak-scaling-oss.nix deleted file mode 100644 index 4c7e574..0000000 --- a/garlic/exp/nbody/old/weak-scaling-oss.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - cpuMask = [ "0x1" "0x3" "0xf" "0xff" "0xffff" "0xffffffff" "0xffffffffffff" ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) cpuMask; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${cpuMask}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/ss.nix b/garlic/exp/nbody/ss.nix deleted file mode 100644 index d9ebf07..0000000 --- a/garlic/exp/nbody/ss.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = [ 128 ]; - nodes = range2 1 16; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) blocksize nodes gitBranch; - - expName = "nbody-scaling"; - unitName = expName + - "-${toString gitBranch}" + - "-nodes${toString nodes}"; - - loops = 5; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/bw.nix b/garlic/exp/osu/bw.nix deleted file mode 100644 index e2c3f37..0000000 --- a/garlic/exp/osu/bw.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-bw-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/eager.nix b/garlic/exp/osu/eager.nix deleted file mode 100644 index 69ee939..0000000 --- a/garlic/exp/osu/eager.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 20 24 ]; - PSM2_MTU_KB = [ 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/impi.nix b/garlic/exp/osu/impi.nix deleted file mode 100644 index 97f6283..0000000 --- a/garlic/exp/osu/impi.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - threshold = [ 8000 16000 32000 64000 ]; - #threshold = [ 4096 8192 10240 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - mpi = impi; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 10; - expName = "osu-impi-rndv"; - unitName = expName + "-${toString threshold}"; - jobName = expName; - inherit (c) threshold; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export PSM2_MQ_RNDV_SHM_THRESH=${toString conf.threshold} - export PSM2_MQ_RNDV_HFI_THRESH=${toString conf.threshold} - export PSM2_MQ_EAGER_SDMA_SZ=${toString conf.threshold} - #export PSM2_MTU=${toString conf.threshold} - export PSM2_TRACEMASK=0x101 - ''; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/latency.nix b/garlic/exp/osu/latency.nix deleted file mode 100644 index f58eb60..0000000 --- a/garlic/exp/osu/latency.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -, enableMultithread ? false -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = if (enableMultithread) then hw.cpusPerSocket else 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-latency-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - inherit enableMultithread; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = if (enableMultithread) then - "${nextStage}/bin/osu_latency_mt" - else - "${nextStage}/bin/osu_latency"; - - argv = optionals (enableMultithread) [ - "-t" "${toString conf.cpusPerTask}:${toString conf.cpusPerTask}" - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/mtu.nix b/garlic/exp/osu/mtu.nix deleted file mode 100644 index ca99091..0000000 --- a/garlic/exp/osu/mtu.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 ]; - PSM2_MTU_KB = [ 8 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/granularity.nix b/garlic/exp/saiph/granularity.nix deleted file mode 100644 index 30b9b28..0000000 --- a/garlic/exp/saiph/granularity.nix +++ /dev/null @@ -1,132 +0,0 @@ -####################################################################### -# Saiph, granularity experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Granularity experiment configuration: -# Single-core run -# MPI binded to sockets: MPI procs = 2 -# Mesh distributed across third dimension to ensure contiguous -# communications -# --> nbgx = 1, nbgy = 1 -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -####################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Variable configurations - varConf = with targetMachine.config; { - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = range2 1 (hw.cpusPerNode * 8); - nblz = [ 8 ]; - sizex = [ 3 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-granularity"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - - jobName = unitName; - - # saiph options - totalTasks = ntasksPerNode * nodes; - nodes = 1; - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = totalTasks; # forcing distribution by last dim - - inherit (c) nblx nbly nblz gitBranch sizex; - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # fix a specific commit - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all - # factors - allConfigs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Filter to remove non-desired configurations: - # --> tasks/proc < 0.5 - # --> nblz > 50 - isGoodConfig = c: - let - maxNblz = c.cpusPerTask * 2; - in - ! (c.blocksPerCpu < 0.5 || c.nblz > maxNblz); - - configs = filter (isGoodConfig) allConfigs; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString conf.cpusPerTask} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/ss.nix b/garlic/exp/saiph/ss.nix deleted file mode 100644 index 5ddc47a..0000000 --- a/garlic/exp/saiph/ss.nix +++ /dev/null @@ -1,122 +0,0 @@ -###################################################################################### -# Saiph, scalability experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Scalability experiment configuration: -# From a single-core granularity experiment, use a suited local blocking set: -# --> nbly*nblz >= 48 (at least 3tasks/proc) -# MPI binded to sockets: MPI procs = 2*nodes -# Mesh distributed across third dimension to ensure contiguous communications -# --> nbgx = 1, nbgy = 1 -# Global distribution limited by global mesh size -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -###################################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - #*** Variable configurations *** - varConf = with targetMachine.config; { - # FIXME: None of those selected nbl* and problem size is able to give good - # efficiency when testing strong scaling. We should find better values. - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = [ 32 ]; - nblz = [ 8 ]; - sizex = [ 3 6 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - nodes = range2 1 8; - }; - - #*** Generate the complete configuration for each unit *** - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-ss"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - jobName = unitName; - - # saiph options - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = nodes * ntasksPerNode; # forcing distribution by last dim - - inherit (c) nblx nbly nblz nodes sizex; - - gitBranch = "garlic/tampi+isend+oss+task+simd"; - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; # fix a specific commit - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all factors - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString hw.cpusPerSocket} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/cpu.nix b/garlic/exp/slurm/cpu.nix deleted file mode 100644 index d2ee632..0000000 --- a/garlic/exp/slurm/cpu.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list of cpus per task by computing the divisors of the number of - # cpus per socket, example: divisors 24 = [ 1 2 3 4 6 8 12 24 ] - cpusPerTask = divisors hw.cpusPerSocket; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "cpu"; - unitName = "${expName}.${toString cpusPerTask}"; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - inherit (c) cpusPerTask; - # As cpusPerTask is a divisor of the cpusPerSocket and thus cpusPerNode, we - # know the remainder is zero: - ntasksPerNode = hw.cpusPerNode / cpusPerTask; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - }; - - program = {nextStage, conf, ...}: bsc.dummy; - - pipeline = stdexp.stdPipeline ++ [ program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/exit1.nix b/garlic/exp/slurm/exit1.nix deleted file mode 100644 index bf60136..0000000 --- a/garlic/exp/slurm/exit1.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - script = [ - "exit 1" - "exit 0" - "kill -SEGV $$" - "kill -TERM $$" - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "exit1"; - unitName = expName + "-" + - builtins.replaceStrings [" " "$"] ["-" "-"] script; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - inherit (c) script; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = 2; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - pre = "sleep 5"; - post = "echo dummy"; - }; - - prog = {conf,...}: stages.script { - inherit (conf) script; - }; - - pipeline = stdexp.stdPipeline ++ [ exec prog ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/sigsegv.nix b/garlic/exp/slurm/sigsegv.nix deleted file mode 100644 index b77f58b..0000000 --- a/garlic/exp/slurm/sigsegv.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - when = [ "before" "after" "never" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "sigsegv"; - unitName = expName + "-" + when; - - inherit (machineConfig) hw; - inherit (c) when; - - loops = 3; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - argv = [ conf.when ]; - }; - - program = {nextStage, conf, ...}: bsc.test.sigsegv; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/fig/creams/granularity.R b/garlic/fig/creams/granularity.R deleted file mode 100644 index 524c987..0000000 --- a/garlic/fig/creams/granularity.R +++ /dev/null @@ -1,83 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.iter = time / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodesFactor = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.iter = median(time.iter)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="granul", y="Normalized time", - title="Creams granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(x="granul", y="Time (s)", title="Creams granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/creams/granularity16.R b/garlic/fig/creams/granularity16.R deleted file mode 100644 index 24f3bf7..0000000 --- a/garlic/fig/creams/granularity16.R +++ /dev/null @@ -1,136 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(median.time.elem = median(time.elem)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(log.median.time.elem = log(median.time.elem)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() %>% - group_by(sizeFactor) %>% - mutate(optimal.granul = (median.time.elem == min(median.time.elem))) %>% - ungroup() - -dfopt = df %>% filter(optimal.granul == TRUE) - -dpi = 300 -h = 4 -w = 10 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + - facet_wrap(sizeFactor ~ ., labeller=label_both, nrow=1) + - labs(x="Granularity", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=median.time.elem, color=sizeFactor)) + - geom_line(aes(group=sizeFactor)) + - geom_point(data=dfopt, aes(x=granul, y=median.time.elem)) + - theme_bw() + - labs(x="Granularity", y="Time / k (s)", - color="Size factor k", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("median.time.png", plot=p, width=5, height=5, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=5, height=5, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=sizeFactor, fill=log.median.time.elem)) + - geom_raster() + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="Granularity", y="Size factor", fill="Time / k (s)", subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -k=1 -ggsave("heatmap.png", plot=p, width=4.8*k, height=5*k, dpi=300) -ggsave("heatmap.pdf", plot=p, width=4.8*k, height=5*k, dpi=300) diff --git a/garlic/fig/creams/size.R b/garlic/fig/creams/size.R deleted file mode 100644 index 21e3335..0000000 --- a/garlic/fig/creams/size.R +++ /dev/null @@ -1,96 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 3 -w = 6 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=sizeFactor, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="Size factor k (nz=k*3072)", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/creams/ss.R b/garlic/fig/creams/ss.R deleted file mode 100644 index b367781..0000000 --- a/garlic/fig/creams/ss.R +++ /dev/null @@ -1,128 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=granul, color=iterations)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title="Creams strong scaling: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", title="Creams strong scaling: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=median.time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Median time * nodes (s)", title="Creams strong scaling: median time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("median.time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_boxplot() + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", title="Creams strong scaling: time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.boxplot.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.boxplot.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=nodes, y=time.nodes.iter, color=branch)) + -# geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time.nodes.iter, group=interaction(granul,iterations))) + -# theme_bw() + -# #facet_wrap(branch ~ .) + -# labs(x="nodes", y="Time * nodes / iterations (s)", -# title="Creams strong scaling: time * nodes / iterations", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("time.nodes.iter.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("time.nodes.iter.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/dev/shell.nix b/garlic/fig/dev/shell.nix deleted file mode 100644 index 840c83e..0000000 --- a/garlic/fig/dev/shell.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ pkgs ? import ../../../default.nix }: - -with pkgs; - -let - rWrapper = pkgs.rWrapper.override { - packages = with pkgs.rPackages; [ tidyverse rjson jsonlite egg ]; - }; -in -stdenv.mkDerivation { - name = "R"; - - buildInputs = [ rWrapper ]; -} diff --git a/garlic/fig/examples/granularity.R b/garlic/fig/examples/granularity.R deleted file mode 100644 index 7577dca..0000000 --- a/garlic/fig/examples/granularity.R +++ /dev/null @@ -1,195 +0,0 @@ -# This R program takes as argument the dataset that contains the results of the -# execution of the heat example experiment and produces some plots. All the -# knowledge to understand how this script works is covered by this nice R book: -# -# Winston Chang, R Graphics Cookbook: Practical Recipes for Visualizing Data, -# O’Reilly Media (2020). 2nd edition -# -# Which can be freely read it online here: https://r-graphics.org/ -# -# Please, search in this book before copying some random (and probably oudated) -# reply on stack overflow. - -# We load some R packages to import the required functions. We mainly use the -# tidyverse packages, which are very good for ploting data, -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Here we simply load the arguments to find the input dataset. If nothing is -# specified we use the file named `input` in the current directory. -# We can run this script directly using: -# Rscript - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Here we build of dataframe from the input dataset by chaining operations using -# the magritte operator `%>%`, which is similar to a UNIX pipe. -# First we read the input file, which is expected to be NDJSON -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - # Then we flatten it, as it may contain dictionaries inside the columns - jsonlite::flatten() %>% - - # Now the dataframe contains all the configuration of the units inside the - # columns named `config.*`, for example `config.cbs`. We first select only - # the columns that we need: - select(config.cbs, config.rbs, unit, time) %>% - - # And then we rename those columns to something shorter: - rename(cbs=config.cbs, rbs=config.rbs) %>% - - # The columns contain the values that we specified in the experiment as - # integers. However, we need to tell R that those values are factors. So we - # apply to those columns the `as.factor()` function: - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - - # The same for the unit (which is the hash that nix has given to each unit) - mutate(unit = as.factor(unit)) %>% - - # Then, we can group our dataset by each unit. This will always work - # independently of the variables that vary from unit to unit. - group_by(unit) %>% - - # And compute some metrics which are applied to each group. For example we - # compute the median time within the runs of a unit: - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - # Then, we remove the grouping. This step is very important, otherwise the - # plotting functions get confused: - ungroup() - - -# These constants will be used when creating the plots. We use high quality -# images with 300 dots per inch and 5 x 5 inches of size by default. -dpi = 300 -h = 5 -w = 5 - - -# --------------------------------------------------------------------- - - -# We plot the median time (of each unit) as we vary the block size. As we vary -# both the cbs and rbs, we plot cbs while fixing rbs at a time. -p = ggplot(df, aes(x=cbs, y=median.time, color=rbs)) + - # We add a point to the median - geom_point() + - - # We also add the lines to connect the points. We need to specify which - # variable will do the grouping, otherwise we will have one line per point. - geom_line(aes(group=rbs)) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Median time (s)", title="Heat granularity: median time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# Another interesting plot is the normalized time, which shows the variance of -# the execution times, and can be used to find problems: -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - - # The boxplots are useful to identify outliers and problems with the - # distribution of time - geom_boxplot() + - - # We add a line to mark the 1% limit above and below the median - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - - # We split the plot into subplots, one for each value of the rbs column - facet_wrap(~ rbs) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Normalized time", title="Heat granularity: normalized time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We plot the time of each run as we vary the block size -p = ggplot(df, aes(x=cbs, y=time, color=rbs)) + - - # We add a points (scatter plot) using circles (shape=21) a bit larger - # than the default (size=3) - geom_point(shape=21, size=3) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Time (s)", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We can also plot both cbs and rbs in each dimension by mapping the time with a -# color. The `fill` argument instruct R to use the `median.time` as color -p = ggplot(df, aes(x=cbs, y=rbs, fill=median.time)) + - - # Then we use the geom_raster method to paint rectangles filled with color - geom_raster() + - - # The colors are set using the viridis package, using the plasma palete. Those - # colors are designed to be safe for color impaired people and also when - # converting the figures to grayscale. - scale_fill_viridis(option="plasma") + - - # We also force each tile to be an square - coord_fixed() + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="rbs", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R deleted file mode 100644 index da9f1a4..0000000 --- a/garlic/fig/fwi/granularity.R +++ /dev/null @@ -1,74 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(legend.position="bottom") + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/io.R b/garlic/fig/fwi/io.R deleted file mode 100644 index e10e3dc..0000000 --- a/garlic/fig/fwi/io.R +++ /dev/null @@ -1,120 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(forcats) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.blocksize, config.ioFreq, config.gitBranch, config.nodes, time, unit) %>% - rename( - blocksize=config.blocksize, - enableIO=config.enableIO, - gitBranch=config.gitBranch, - nodes=config.nodes - ) %>% - filter(blocksize == 1) %>% - group_by(unit) %>% - mutate(mtime = median(time)) %>% - mutate(nxmtime = mtime * nodes) %>% - mutate(nxtime = time * nodes) %>% - ungroup() - -df$gitBranch = as.factor(df$gitBranch) -df$enableIO = as.factor(df$enableIO) -df$blocksize = as.factor(df$blocksize) -df$nodes = as.factor(df$nodes) - -ppi=300 -h=5 -w=5 - -#################################################################### -### Line plot (time) -#################################################################### -png("time.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=time, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -#################################################################### -### Line plot (time x nodes) -#################################################################### -png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=nxtime, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -##################################################################### -#### Line plot (median time) -##################################################################### -#png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() -# -##################################################################### -#### Line plot (nodes x median time) -##################################################################### -#png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() diff --git a/garlic/fig/fwi/ss.R b/garlic/fig/fwi/ss.R deleted file mode 100644 index 428d5c1..0000000 --- a/garlic/fig/fwi/ss.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - config.nodes, - time) %>% - - rename(blocksize=config.blocksize, - nodes=config.nodes, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/cache.R b/garlic/fig/heat/cache.R deleted file mode 100644 index dca6baf..0000000 --- a/garlic/fig/heat/cache.R +++ /dev/null @@ -1,78 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, perf.cache_misses, perf.instructions, perf.cycles, time) %>% - rename(cbs=config.cbs, rbs=config.rbs) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df=group_by(df, cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.misses = median(perf.cache_misses)) %>% - mutate(log.median.misses = log(median.misses)) %>% - mutate(median.instr= median(perf.instructions)) %>% - mutate(log.median.instr= log(median.instr)) %>% - mutate(median.cycles = median(perf.cycles)) %>% - mutate(median.cpi = median.cycles / median.instr) %>% - mutate(median.ipc = median.instr / median.cycles) %>% - mutate(median.ips = median.instr / median.time) %>% - mutate(median.cps = median.cycles / median.time) %>% - ungroup()# %>% - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.misses", "cache misses") -heatmap_plot(df, "log.median.misses", "cache misses") -heatmap_plot(df, "median.instr", "instructions") -heatmap_plot(df, "log.median.instr", "instructions") -heatmap_plot(df, "median.cycles", "cycles") -heatmap_plot(df, "median.ipc", "IPC") -heatmap_plot(df, "median.cpi", "cycles/instruction") -heatmap_plot(df, "median.ips", "instructions/second") -heatmap_plot(df, "median.cps", "cycles/second") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df_filtered, "median.time", "execution time (seconds)") -heatmap_plot(df_filtered, "log.median.time", "execution time") diff --git a/garlic/fig/heat/granularity.R b/garlic/fig/heat/granularity.R deleted file mode 100644 index 19c3f37..0000000 --- a/garlic/fig/heat/granularity.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.cbs, - config.rbs, - time, - total_time) %>% - - rename(cbs=config.cbs, - rbs=config.rbs) %>% - - # Convert to factors - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(y="Normalized time", - title="Heat granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=time)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=0)) + - theme_bw() + - labs(y="Time (s)", title="Heat granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/mode.R b/garlic/fig/heat/mode.R deleted file mode 100644 index 5e409bc..0000000 --- a/garlic/fig/heat/mode.R +++ /dev/null @@ -1,122 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) -library(tidyr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, - ctf_mode.runtime, - ctf_mode.task, - ctf_mode.dead, - config.cpusPerTask, - time) %>% - rename( - cbs=config.cbs, - rbs=config.rbs, - runtime=ctf_mode.runtime, - task=ctf_mode.task, - dead=ctf_mode.dead, - cpusPerTask=config.cpusPerTask, - ) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df = df %>% - mutate(runtime = runtime * 1e-9 / cpusPerTask) %>% - mutate(dead = dead * 1e-9 / cpusPerTask) %>% - mutate(task = task * 1e-9 / cpusPerTask) %>% - group_by(cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.dead = median(dead)) %>% - mutate(median.runtime = median(runtime)) %>% - mutate(median.task = median(task)) %>% - ungroup() #%>% - -print(df) - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.runtime", "runtime") -heatmap_plot(df, "median.dead", "not used") -heatmap_plot(df, "median.task", "task") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df, "median.time", "execution time (seconds)") -heatmap_plot(df, "log.median.time", "execution time") - -df_square = filter(df, cbs == rbs) %>% - gather(key = time.from, value = acc.time, - c("median.dead", "median.runtime", "median.task")) - -# Colors similar to Paraver -colors <- c("median.dead" = "gray", - "median.runtime" = "blue", - "median.task" = "red") - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_area(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - geom_point(aes(y=median.time, color="black")) + - geom_line(aes(y=median.time, group=0, color="black")) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - scale_color_identity(breaks = c("black"), - labels = c("Total time"), guide = "legend") + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("area.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("area.time.pdf", plot=p, width=6, height=6, dpi=300) - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_col(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("col.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("col.time.pdf", plot=p, width=6, height=6, dpi=300) diff --git a/garlic/fig/hpcg/granularity.R b/garlic/fig/hpcg/granularity.R deleted file mode 100644 index 29126da..0000000 --- a/garlic/fig/hpcg/granularity.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.ncomms, - config.hw.cpusPerSocket, - config.blocksPerCpu, - unit, - time) %>% - - rename(nblocks=config.nblocks, - ncomms=config.ncomms, - blocksPerCpu=config.blocksPerCpu) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=blocksPerCpu, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(x="Blocks per CPU", y="Normalized time", title="HPCG granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=blocksPerCpu, y=time)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Blocks per CPU", y="Time (s)", title="HPCG granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/hpcg/size.R b/garlic/fig/hpcg/size.R deleted file mode 100644 index a7cc5f1..0000000 --- a/garlic/fig/hpcg/size.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.elem = time.nodes / sizePerCpu.z) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=sizePerCpu.z, y=time.nodes.elem)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Size per CPU in Z", y="Time * nodes / spcz (s)", - title="HPCG size: time * nodes / spcz", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ss.R b/garlic/fig/hpcg/ss.R deleted file mode 100644 index 3978618..0000000 --- a/garlic/fig/hpcg/ss.R +++ /dev/null @@ -1,82 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.sizeZ = time / sizePerCpu.z) %>% - mutate(time.nodes = time * nodes) %>% - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time.nodes)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", - title="HPCG strong scalability in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=nodes, y=time.sizeZ, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time / npcz (s)", title="HPCG strong scalability in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.size.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.size.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ws.R b/garlic/fig/hpcg/ws.R deleted file mode 100644 index d12f35b..0000000 --- a/garlic/fig/hpcg/ws.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - mutate(time.sizeZ = time / sizePerCpu.z) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - mutate(timePerNprocs = time * npz) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="HPCG weak scaling in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix deleted file mode 100644 index 0e7ce41..0000000 --- a/garlic/fig/index.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -let - rPlot = garlic.pp.rPlot; - exp = garlic.exp; - pp = garlic.pp; - ds = garlic.ds; - fig = garlic.fig; - - stdPlot = rScript: expList: rPlot { - script = rScript; - dataset = pp.mergeDatasets (map (e: ds.std.timetable e.result) expList); - }; - - customPlot = rScript: dataset: rPlot { - script = rScript; - dataset = dataset; - }; - - linkTree = name: tree: self.linkFarm name ( - self.lib.mapAttrsToList ( - name: value: { name=name; path=value; } - ) tree); -in -{ - nbody = with exp.nbody; { - granularity = stdPlot ./nbody/granularity.R [ granularity ]; - ss = stdPlot ./nbody/ss.R [ ss ]; - numa = stdPlot ./nbody/numa.R [ numa ]; - }; - - hpcg = with exp.hpcg; { - ss = stdPlot ./hpcg/ss.R [ ss ]; - ws = stdPlot ./hpcg/ws.R [ ws ]; - size = stdPlot ./hpcg/size.R [ size ]; - granularity = stdPlot ./hpcg/granularity.R [ granularity ]; - }; - - saiph = with exp.saiph; { - granularity = stdPlot ./saiph/granularity.R [ granularity ]; - ss = stdPlot ./saiph/ss.R [ ss ]; - }; - - heat = with exp.heat; { - granularity = stdPlot ./heat/granularity.R [ granularity ]; - cache = customPlot ./heat/cache.R (ds.perf.stat cache.result); - ctf = customPlot ./heat/mode.R (ds.ctf.mode ctf.result); - }; - - creams = with exp.creams; { - ss = stdPlot ./creams/ss.R [ ss ]; - granularity = stdPlot ./creams/granularity.R [ granularity ]; - size = stdPlot ./creams/size.R [ size ]; - granularity16 = stdPlot ./creams/granularity16.R [ granularity16 ]; - - # Extended version (we could use another R script for those plots - big.ss = stdPlot ./creams/ss.R [ big.ss ]; - big.granularity = stdPlot ./creams/granularity.R [ big.granularity ]; - }; - - fwi = with exp.fwi; { - granularity = stdPlot ./fwi/granularity.R [ granularity ]; - reuse = stdPlot ./fwi/granularity.R [ reuse ]; - ss = stdPlot ./fwi/ss.R [ ss ]; - io = stdPlot ./fwi/io.R [ io ]; - }; - - osu = with exp.osu; { - latency = customPlot ./osu/latency.R (ds.osu.latency latency.result); - latencyShm = customPlot ./osu/latency.R (ds.osu.latency latencyShm.result); - latencyMt = customPlot ./osu/latency.R (ds.osu.latency latencyMt.result); - latencyMtShm = customPlot ./osu/latency.R (ds.osu.latency latencyMtShm.result); - - bw = customPlot ./osu/bw.R (ds.osu.bw bw.result); - bwShm = customPlot ./osu/bw.R (ds.osu.bw bwShm.result); - impi = customPlot ./osu/impi.R (ds.osu.bw impi.result); - mtu = customPlot ./osu/mtu.R (ds.osu.bw mtu.result); - eager = customPlot ./osu/eager.R (ds.osu.bw eager.result); - }; - - # The figures used in the article contained in a directory per figure - article = with fig; linkTree "article-fig" { - "osu/latency" = osu.latency; - "osu/bw" = osu.bw; - "osu/mtu" = osu.mtu; - }; - - examples = with exp.examples; { - granularity = stdPlot ./examples/granularity.R [ granularity ]; - }; -} diff --git a/garlic/fig/nbody/granularity.R b/garlic/fig/nbody/granularity.R deleted file mode 100644 index 85ee0b4..0000000 --- a/garlic/fig/nbody/granularity.R +++ /dev/null @@ -1,62 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.particles, unit, time) %>% - rename(blocksize=config.blocksize, particles=config.particles, branch=config.gitBranch) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(particles = as.factor(particles)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 5 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Blocksize", y="Normalized Time", title="NBody Granularity: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time)) + - geom_boxplot() + - theme_bw() + - labs(x="Blocksize", y="Time (s)", title="NBody Granularity: Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/numa.R b/garlic/fig/nbody/numa.R deleted file mode 100644 index b080648..0000000 --- a/garlic/fig/nbody/numa.R +++ /dev/null @@ -1,86 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(unit, - config.blocksize, - config.gitBranch, - config.attachToSocket, - config.interleaveMem, - config.nodes, - unit, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch, - nodes=config.nodes, - attachToSocket=config.attachToSocket, - interleaveMem=config.interleaveMem) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(attachToSocket = as.factor(attachToSocket)) %>% - mutate(interleaveMem = as.factor(interleaveMem)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -branch = unique(df$branch) -nodes = unique(df$nodes) - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=interleaveMem)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Normalized time", - title=sprintf("NBody NUMA (%s | %d Nodes): Normalized time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=interleaveMem)) + - geom_boxplot() + - geom_line(aes(y=median.time)) + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Time (s)", - title=sprintf("NBody NUMA (%s | %d Nodes): Time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/old/baseline.R b/garlic/fig/nbody/old/baseline.R deleted file mode 100644 index 5509d8e..0000000 --- a/garlic/fig/nbody/old/baseline.R +++ /dev/null @@ -1,212 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(egg) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, - config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.blocksize, - config.particles, - config.gitBranch, - time) %>% - rename(nblocks=config.nblocks, - nodes=config.nodes, - blocksize=config.blocksize, - particles=config.particles, - gitBranch=config.gitBranch, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$nodesFactor = as.factor(df$nodes) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) -df$blocksizeFactor = as.factor(df$blocksize) -df$particlesFactor = as.factor(df$particles) -df$gitBranch = as.factor(df$gitBranch) - -# Normalize the time by the median -D=group_by(df, nblocks, nodesFactor, gitBranch) %>% - mutate(tmedian = median(time)) %>% - mutate(tn = tmedian * nodes) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) %>% - ungroup() %>% - group_by(nodesFactor, gitBranch) %>% - mutate(tmedian_min = min(tmedian)) %>% - ungroup() %>% - group_by(gitBranch) %>% - mutate(tmin_max = max(tmedian_min)) %>% - mutate(tideal = tmin_max / nodes) %>% - ungroup() - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=7.5 -w=7.5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot(aes(fill=nodesFactor)) + - scale_color_manual(values=c("black", "brown")) + - facet_grid(gitBranch ~ .) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - - #theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - theme_bw()+ - theme(plot.subtitle=element_text(size=8)) - - - - -# Render the plot -print(p) -dev.off() - - -p1 = ggplot(D, aes(x=blocksizeFactor, y=time)) + - - labs(x="Blocksize", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - #theme(legend.position = c(0.5, 0.8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocksize.png", width=w*ppi, height=h*ppi, res=ppi) -print(p1) -dev.off() - -p2 = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocks-per-cpu.png", width=w*ppi, height=h*ppi, res=ppi) -print(p2) -dev.off() - -#p = ggarrange(p1, p2, ncol=2) -#png("time-gra.png", width=2*w*ppi, height=h*ppi, res=ppi) -#print(p) -#dev.off() - - - -png("exp-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=df, aes(x=nodesFactor, y=particlesFactor)) + - labs(x="Nodes", y="Particles", title="Nbody: Experiment space") + - geom_line(aes(group=particles)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("gra-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=D, aes(x=nodesFactor, y=blocksPerCpuFactor)) + - labs(x="Nodes", y="Blocks/CPU", title="Nbody: Granularity space") + - geom_line(aes(group=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("performance.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tmedian, - linetype=blocksPerCpuFactor, - group=interaction(gitBranch, blocksPerCpuFactor))) + - geom_line(aes(y=tideal, group=gitBranch), color="red") + - geom_point(aes(y=tmedian, color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() - - -png("time-nodes.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time * nodes (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tn, group=gitBranch)) + - facet_grid(gitBranch ~ .) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() diff --git a/garlic/fig/nbody/old/freeCpu.R b/garlic/fig/nbody/old/freeCpu.R deleted file mode 100644 index cf97106..0000000 --- a/garlic/fig/nbody/old/freeCpu.R +++ /dev/null @@ -1,112 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.freeCpu, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - freeCpu=config.freeCpu, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$freeCpu = as.factor(df$freeCpu) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(freeCpu, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=freeCpu)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans(), -# breaks=bs_unique) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~freeCpu) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time, color=freeCpu)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/jemalloc.R b/garlic/fig/nbody/old/jemalloc.R deleted file mode 100644 index 37fc991..0000000 --- a/garlic/fig/nbody/old/jemalloc.R +++ /dev/null @@ -1,113 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.enableJemalloc, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - jemalloc=config.enableJemalloc, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$jemalloc = as.factor(df$jemalloc) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(jemalloc, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - # Add another column: blocksPerCpu (we assume one task per socket, using - # all CPUs) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=nblocks, y=tnorm)) + - - # Labels - labs(x="Num blocks", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=jemalloc)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans()) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~jemalloc) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpu, y=time, color=jemalloc)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/scaling.R b/garlic/fig/nbody/old/scaling.R deleted file mode 100644 index 1d50a37..0000000 --- a/garlic/fig/nbody/old/scaling.R +++ /dev/null @@ -1,111 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Normalize the time by the median -D=group_by(df, nblocks) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot() + - scale_color_manual(values=c("black", "brown")) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/ss.R b/garlic/fig/nbody/ss.R deleted file mode 100644 index 2ce9b2c..0000000 --- a/garlic/fig/nbody/ss.R +++ /dev/null @@ -1,66 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.nodes, unit, time) %>% - rename(nodes = config.nodes, blocksize=config.blocksize, branch=config.gitBranch) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Nodes", y="Normalized time (s)", title="NBody Scaling: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", title="NBody Scaling: Time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/osu/bw.R b/garlic/fig/osu/bw.R deleted file mode 100644 index 876519d..0000000 --- a/garlic/fig/osu/bw.R +++ /dev/null @@ -1,59 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, bw) %>% - rename(unitName=config.unitName) %>% - mutate(bw=bw / 1024.0) %>% - mutate(unitName=str_replace(unitName, "osu-bw-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=median.bw)) + - labs(x="Message size", y="Bandwidth (GB/s)", - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=1.5) + - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - geom_hline(yintercept=12.5, color="red") + - annotate("text", x=1, y=12.5 * .95, - label="Max: 12.5GB/s (100Gbps)", - hjust=0, vjust=1, size=3) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12, - ) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(1,0), legend.position = c(0.99, 0.01)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/eager.R b/garlic/fig/osu/eager.R deleted file mode 100644 index aa9e573..0000000 --- a/garlic/fig/osu/eager.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=150 -h=6 -w=8 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - labs(x="Message size (KB)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=3) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") + - geom_vline(xintercept = 10, color="red") + - annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MQ_EAGER_SDMA_SZ.f), nrow=3, labeller = "label_both") + - scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - theme_bw() - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/impi.R b/garlic/fig/osu/impi.R deleted file mode 100644 index c62a334..0000000 --- a/garlic/fig/osu/impi.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, config.threshold, size, bw) %>% - rename(unitName=config.unitName) %>% - rename(threshold=config.threshold) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$threshold = as.factor(df$threshold) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -p = ggplot(data=df, aes(x=size, y=bw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU bandwidth benchmark: nodes=%d tasksPerNode=%d cpusPerTask=%d", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_boxplot(aes(color=threshold, group=interaction(threshold, sizeFactor))) + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ppi=300 -h=4 -w=8 -ggsave("boxplot.pdf", plot=p, width=w, height=h, dpi=ppi) -ggsave("boxplot.png", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=medianBw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(color=threshold, linetype=threshold)) + - geom_point(aes(color=threshold, shape=threshold)) + - geom_hline(yintercept = 100e3 / 8, color="red") + - annotate("text", x = 8, y = (100e3 / 8) * 0.95, label = "12.5GB/s (100Gb/s)") + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/latency.R b/garlic/fig/osu/latency.R deleted file mode 100644 index e116a73..0000000 --- a/garlic/fig/osu/latency.R +++ /dev/null @@ -1,77 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, latency) %>% - rename(unitName=config.unitName) %>% - mutate(unitName=str_replace(unitName, "osu-latency-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianLatency = median(latency)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=medianLatency)) + - labs(x="Message size", y="Median latency (µs)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12)+ - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(0,1), legend.position = c(0.01, 0.99)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=latency)) + - labs(x="Size (bytes)", y="Latency (us)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(y=medianLatency, linetype=unitName, group=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - breaks=unique(df$size), - minor_breaks=NULL) + - theme_bw() + - theme(plot.subtitle = element_text(color="gray50")) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) + - theme(legend.position = c(0.2, 0.8)) - -ggsave("latency.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("latency.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/mtu.R b/garlic/fig/osu/mtu.R deleted file mode 100644 index 09e2f63..0000000 --- a/garlic/fig/osu/mtu.R +++ /dev/null @@ -1,69 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - config.PSM2_MTU, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ.val=config.PSM2_MQ_EAGER_SDMA_SZ, - PSM2_MTU.val=config.PSM2_MTU) %>% - mutate(bw = bw / 1000.0) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ.val) -df$PSM2_MTU = as.factor(df$PSM2_MTU.val) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ.val/1024), color="blue") + - geom_vline(aes(xintercept = PSM2_MTU.val/1024), color="red") + - labs(x="Message size (KiB)", y="Bandwidth (GB/s)", - #title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=2) + - #annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MTU), nrow=3, labeller = "label_both") + - #scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - scale_x_continuous(n.breaks = 12) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/saiph/granularity.R b/garlic/fig/saiph/granularity.R deleted file mode 100644 index 66bd7c6..0000000 --- a/garlic/fig/saiph/granularity.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nbly, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksPerCpu, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + - scale_x_continuous(trans=log2_trans()) + - labs(y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/saiph/ss.R b/garlic/fig/saiph/ss.R deleted file mode 100644 index 6498f30..0000000 --- a/garlic/fig/saiph/ss.R +++ /dev/null @@ -1,110 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=sizex)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/garlicd/default.nix b/garlic/garlicd/default.nix deleted file mode 100644 index b34efdf..0000000 --- a/garlic/garlicd/default.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, nix -, garlicTool -}: - -let - extraPath = "${garlicTool}:${nix}"; -in - stdenv.mkDerivation { - name = "garlicd"; - preferLocalBuild = true; - - phases = [ "unpackPhase" "installPhase" ]; - - src = ./garlicd; - - unpackPhase = '' - cp $src garlicd - ''; - - installPhase = '' - substituteInPlace garlicd \ - --replace @extraPath@ ${extraPath} - mkdir -p $out/bin - cp -a garlicd $out/bin - ''; - } diff --git a/garlic/garlicd/garlicd b/garlic/garlicd/garlicd deleted file mode 100755 index 17e3f9d..0000000 --- a/garlic/garlicd/garlicd +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -e - -emsg() { - >&2 echo "garlicd: $@" -} - -msg() { - emsg "$@" - if [ ! -z "$st" ]; then echo "garlicd: $@" >>"$st"; fi -} - -if [ ! -z "$1" ]; then - >&2 echo "usage: garlicd" - exit 1 -fi - -export PATH="@extraPath@:$PATH" - -garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - -if [ -z "$garlic_sandbox" ]; then - emsg "Missing extra-sandbox-paths /garlic mountpoint" - emsg "Check the ~/.config/nix/nix.conf file" - exit 1 -fi - -mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') -mountdir=$(readlink -f "$mountdir_rel") -run="$mountdir/run" -completed="$mountdir/completed" -wipe="$mountdir/wipe" -st="$mountdir/st" - -handle_bad_signal() { - msg "cleaning FIFO pipes" - rm -f "$run" "$completed" "$wipe" - exit 1 -} - -trap handle_bad_signal SIGINT - -for fifo in "$run" "$completed" "$wipe"; do - if [ ! -e "$fifo" ]; then - mkfifo "$fifo" - # FIXME: Use more resctrictive permissions - chmod 666 "$fifo" - fi -done - -while true; do - emsg "--- Waiting for experiments ---" - tre=$(head -1 "$run") - - # Truncate state file - printf "" > "$st" - - msg "Attempting to run: $tre" - msg "Copying files to MN4..." - # It fails if the user doesn't have nix-store, but is already copied - # with the post build hook - nix copy --to ssh://mn1 $tre || true - - set +e - - msg "Launching the experiment..." - garlic -R "$tre" 2>>"$st" - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "Failed to run the experiment :-(" - continue - fi - - msg "Fetching results..." - garlic -FKv "$tre" 2>>"$st" - - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "The experiment failed :-(" - continue - fi - - set -e - - msg "Signalling nix build..." - echo -n "$tre" >> "$completed" - - msg "Waiting for nix to finish the build..." - - tre2=$(head -1 "$wipe") - if [ "$tre" != "$tre2" ]; then - msg "error: trebuchet mismatch" - exit 1 - fi - - msg "Removing temporal files..." - garlic -D "$tre" 2>>"$st" - - echo -n "$tre" >> "$completed" - - msg "Execution completed :-)" -done diff --git a/garlic/index.nix b/garlic/index.nix deleted file mode 100644 index ed8e2d8..0000000 --- a/garlic/index.nix +++ /dev/null @@ -1,138 +0,0 @@ -{ - super -, self -, bsc -, callPackage -}: - -{ - develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl boost - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi vtk paraver - # Add more bsc packages here... - ]; - packages = commonPackages ++ bscPackages; - in - bsc.garlic.stages.exec rec { - nextStage = bsc.garlic.stages.isolate { - nextStage = bsc.garlic.unsafeDevelop.override { - extraInputs = packages; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - extraMounts = [ "/tmp:$TMPDIR" ]; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - # This hack uploads all dependencies to MN4 - pre = let - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - stageProgram = bsc.garlicTools.stageProgram; - in - '' - # Hack to upload this to MN4: @upload-to-mn@ - - # Create a link to the develop script - ln -fs ${nixPrefix}${stageProgram nextStage} .nix-develop - ''; - post = "\n"; - }; - - unsafeDevelop = callPackage ./develop/default.nix { }; - - # Configuration for the machines - machines = callPackage ./machines.nix { }; - - report = callPackage ./report.nix { - fig = bsc.garlic.fig; - }; - - sedReport = callPackage ./sedReport.nix { - fig = bsc.garlic.fig; - }; - - bundleReport = callPackage ./bundleReport.nix { - fig = bsc.garlic.fig; - }; - - reportTar = callPackage ./reportTar.nix { - fig = bsc.garlic.fig; - }; - - # Use the configuration for the following target machine - targetMachine = bsc.garlic.machines.mn4; - - # Load some helper functions to generate app variants - - stdexp = callPackage ./stdexp.nix { - inherit (bsc.garlic) targetMachine stages pp; - }; - - # Execution stages - stages = { - sbatch = callPackage ./stages/sbatch.nix { }; - srun = callPackage ./stages/srun.nix { }; - baywatch = callPackage ./stages/baywatch.nix { }; - control = callPackage ./stages/control.nix { }; - exec = callPackage ./stages/exec.nix { }; - script = callPackage ./stages/script.nix { }; - extrae = callPackage ./stages/extrae.nix { }; - valgrind = callPackage ./stages/valgrind.nix { }; - perf = callPackage ./stages/perf.nix { }; - isolate = callPackage ./stages/isolate { }; - runexp = callPackage ./stages/runexp { }; - trebuchet = callPackage ./stages/trebuchet.nix { }; - strace = callPackage ./stages/strace.nix { }; - unit = callPackage ./stages/unit.nix { }; - experiment = callPackage ./stages/experiment.nix { }; - }; - - # Tests (move to bsc ?) - mpptest = callPackage ./mpptest { }; - - ppong = callPackage ./ppong { - mpi = bsc.mpi; - }; - - hist = callPackage ./pp/hist { }; - - tool = callPackage ./sh/default.nix { - sshHost = "mn1"; - }; - - # Post processing - pp = { - store = callPackage ./pp/store.nix { }; - rPlot = callPackage ./pp/rplot.nix { }; - mergeDatasets = callPackage ./pp/merge.nix { }; - }; - - garlicd = callPackage ./garlicd/default.nix { - garlicTool = bsc.garlic.tool; - }; - - # Apps for Garlic - apps = callPackage ./apps/index.nix { - inherit self super bsc; - }; - - # Experiments - exp = callPackage ./exp/index.nix { - inherit self super bsc; - }; - - # Dataset generators from resultTree - ds = callPackage ./ds/index.nix { - inherit self super bsc; - }; - - # Figures generated from the datasets - fig = callPackage ./fig/index.nix { - inherit self super bsc; - }; - -} diff --git a/garlic/machines.nix b/garlic/machines.nix deleted file mode 100644 index e2bdc54..0000000 --- a/garlic/machines.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -}: - -{ - # MareNostrum 4 configuration - mn4 = { - config = { - clusterName = "mn4"; - sshHost = "mn1"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - hw = { - # The rev attribute attemps to capture the hardware - # configuration of the machine, and will rebuild all experiments - # if it changed. It only holds the timestamp at the current - # time, representing the HW configuration at that moment. - rev = 1614253003; - - # Node and socket details - cpusPerNode = 48; - cpusPerSocket = 24; - socketsPerNode = 2; - cachelineBytes = 64; - - # Cache sizes - cacheSizeKB = { - L1d = 32; - L1i = 32; - L2 = 1024; - L3 = 33792; - }; - }; - }; - - # Experimental naming convention for the FS - fs = rec { - topology = { - gpfs = { - projects = "/gpfs/projects/bsc15/garlic"; - scratch = "/gpfs/scratch/bsc15/bsc15557/garlic"; - }; - - ssd = { - # Beware to expand the temp dir at execution time - temp = "$TMPDIR"; - }; - }; - - shared = with topology; { - fast = gpfs.scratch; - reliable = gpfs.projects; - }; - - local = { - temp = topology.ssd.temp; - }; - }; - - # TODO: Add the specific details for SLURM and the interconection here - }; -} diff --git a/garlic/mpptest/default.nix b/garlic/mpptest/default.nix deleted file mode 100644 index c142667..0000000 --- a/garlic/mpptest/default.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "mpptest"; - - src = fetchurl { - url = "http://ftp.mcs.anl.gov/pub/mpi/tools/perftest.tar.gz"; - sha256 = "11i22lq3pch3pvmhnbsgxzd8ap4yvpvlhy2f7k8x3krdwjhl0jvl"; - }; - - buildInputs = [ mpi ]; -} diff --git a/garlic/pp/hist/default.nix b/garlic/pp/hist/default.nix deleted file mode 100644 index 4a3d464..0000000 --- a/garlic/pp/hist/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ - stdenv -, ministat -}: - -stdenv.mkDerivation { - name = "hist"; - preferLocalBuild = true; - src = ./.; - - dontBuild = true; - dontConfigure = true; - - inherit ministat; - - patchPhase = '' - substituteAllInPlace hist.sh - ''; - - installPhase = '' - mkdir -p $out/bin - cp hist.sh $out/bin/hist - chmod +x $out/bin/hist - ''; -} diff --git a/garlic/pp/hist/hist.sh b/garlic/pp/hist/hist.sh deleted file mode 100755 index f3363b4..0000000 --- a/garlic/pp/hist/hist.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Use it either reading from stdin or by specifing -# multiple files as arguments - -# xeon07$ hist stdout.log -# x -# +------------------------------------------------------------------------+ -# | x | -# | x | -# | x | -# | x | -# | x | -# | xxx | -# | xxx | -# | xxxxx | -# | xxxxxx | -# | xxxxxxx x| -# ||________M_A___________| | -# +------------------------------------------------------------------------+ -# N Min Max Median Avg Stddev -# x 30 3.585183 3.763913 3.591559 3.5973344 0.031719975 -# -# Other ministat options can be passed as well. The -S option splits the results -# in multiple plots. - - -usage() { echo "Usage: hist [-hSAns] [-c confidence] [-w width] files..." 1>&2; exit 1; } - -function stat_files() { - tmpfiles=() - sedcmd="" - - for file in ${files[@]}; do - tmp=$(mktemp) - awk '/^time /{print $2}' "$file" > "$tmp" - sedcmd+="s:$tmp:$file:g;" - tmpfiles+=($tmp) - done - - if [ $split == 1 ]; then - for f in "${tmpfiles[@]}"; do - ministat $ministat_opt $f | sed -e "$sedcmd" - done - else - ministat $ministat_opt ${tmpfiles[@]} | sed -e "$sedcmd" - fi - - rm ${tmpfiles[@]} -} - -split=0 -ministat_opt="-w72" - -while getopts "hSAnsc:w:" o; do - case "${o}" in - S) split=1 ;; - c) ministat_opt+=" -c $OPTARG" ;; - w) ministat_opt+=" -w $OPTARG" ;; - A) ministat_opt+=" -$o" ;; - n) ministat_opt+=" -$o" ;; - s) ministat_opt+=" -$o" ;; - *) usage ;; - esac -done - -shift $((OPTIND-1)) - -ministat=@ministat@/bin -#ministat=/nix/store/sh9b484bnhkajxnblpwix7fhbkid6365-ministat-20150715-1/bin - -export PATH="$PATH:$ministat" - -files=("$@") -if [[ -z "${files[@]}" ]]; then - awk '/^time /{print $2}' | ministat $ministat_opt -else - stat_files -fi diff --git a/garlic/pp/merge.nix b/garlic/pp/merge.nix deleted file mode 100644 index 03caaea..0000000 --- a/garlic/pp/merge.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, lib -}: - -datasets: - -with lib; - -stdenv.mkDerivation { - name = "merged-dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - inherit datasets; - installPhase = '' - mkdir -p $out - n=1 - for d in $datasets; do - ln -s $d $out/$n - let n=n+1 - cat $d/dataset >> $out/dataset - done - ''; -} diff --git a/garlic/pp/rplot.nix b/garlic/pp/rplot.nix deleted file mode 100644 index df154da..0000000 --- a/garlic/pp/rplot.nix +++ /dev/null @@ -1,165 +0,0 @@ -{ - stdenv -, lib -, rWrapper -, rPackages -, fontconfig -, dejavu_fonts -, liberation_ttf -, noto-fonts -, makeFontsConf -, makeFontsCache -, jq -, fetchFromGitHub -, writeText -, runCommand -, glibcLocales -}: - -{ -# The two results to be compared - dataset -, script -, extraRPackages ? [] -}: - -with lib; - -let - scalesPatched = with rPackages; buildRPackage { - name = "scales"; - src = fetchFromGitHub { - owner = "mikmart"; - repo = "scales"; - #ref = "label-bytes"; - rev = "fa7d91c765b6b5d2f682c7c22e0478d96c2ea76c"; - sha256 = "10dsyxp9pxzdmg04xpnrxqhc4qfhbkr3jhx8whfr7z27wgfrr1n3"; - }; - propagatedBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - nativeBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - }; - - customR = rWrapper.override { - packages = with rPackages; [ scalesPatched tidyverse viridis egg - Cairo extrafont ] ++ extraRPackages; - }; - - myFonts = [ - dejavu_fonts - #noto-fonts - #liberation_ttf - ]; - - cacheConf = - let - cache = makeFontsCache { fontDirectories = myFonts; }; - in - writeText "fc-00-nixos-cache.conf" '' - - - - - ${concatStringsSep "\n" (map (font: "

${font}") myFonts)} - ${optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' - - ${cache} - ''} - - ''; - - # default fonts configuration file - # priority 52 - defaultFontsConf = - let genDefault = fonts: name: - optionalString (fonts != []) '' - - ${name} - - ${concatStringsSep "" - (map (font: '' - ${font} - '') fonts)} - - - ''; - in - writeText "fc-52-nixos-default-fonts.conf" '' - - - - - ${genDefault [ "DejaVu Sans" ] "sans-serif"} - ${genDefault [ "DejaVu Serif" ] "serif"} - ${genDefault [ "DejaVu Sans Mono" ] "monospace"} - ${genDefault [ "Noto Color Emoji"] "emoji"} - - ''; - - fontConfPath = - let - fixedConf = runCommand "fonts-fixed.conf" { - preferLocalBuild = true; - } '' - head --lines=-2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - - cat >> $out << 'EOF' - - conf.d - EOF - - tail -2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - ''; - in - runCommand "fontconfig-conf" { - preferLocalBuild = true; - } '' - dst=$out/etc/fonts/conf.d - mkdir -p $dst - # fonts.conf - ln -s ${fixedConf} $dst/../fonts.conf - - # fontconfig default config files - ln -s ${fontconfig.out}/etc/fonts/conf.d/*.conf \ - $dst/ - - # 00-nixos-cache.conf - ln -s ${cacheConf} $dst/00-nixos-cache.conf - - # 52-nixos-default-fonts.conf - ln -s ${defaultFontsConf} $dst/52-nixos-default-fonts.conf - ''; - -in stdenv.mkDerivation { - name = "plot"; - buildInputs = [ customR jq fontconfig glibcLocales ]; - preferLocalBuild = true; - dontPatchShebangs = true; - phases = [ "installPhase" ]; - - installPhase = '' - export FONTCONFIG_PATH=${fontConfPath}/etc/fonts/ - export LANG=en_US.UTF-8 - - mkdir -p $out - cd $out - dataset=$(readlink -f ${dataset}/dataset) - - ln -s $dataset input - Rscript --vanilla ${script} "$dataset" "$out" - - # HACK: replace the \minus for a \hyphen to keep the file paths intact, so - # they can be copied to the terminal directly. The StandardEncoding is not - # working (AdobeStd.enc). - find "$out" -name '*.pdf' | xargs -l1 sed -i 's.45/minus.45/hyphen.g' - - if [ "''${dataset##*.}" == gz ]; then - gunzip --stdout $dataset - else - cat $dataset - fi | jq -c .total_time |\ - awk '{s+=$1} END {printf "%f\n", s/60}' > total_job_time_minutes - ''; -} diff --git a/garlic/pp/store.nix b/garlic/pp/store.nix deleted file mode 100644 index 6005efb..0000000 --- a/garlic/pp/store.nix +++ /dev/null @@ -1,89 +0,0 @@ -{ - stdenv -}: - -{ - trebuchet, - experiment -}: - -with builtins; - -let - experimentName = baseNameOf (experiment); - trebuchetName = baseNameOf (trebuchet); -in - stdenv.mkDerivation { - name = "resultTree"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - installPhase = '' - echo "resultTree: searching for garlicd daemon..." - if [ -e /garlic/run ]; then - echo "resultTree: asking the daemon to run and fetch the experiment" - - echo ${trebuchet} >> /garlic/run - echo "resultTree: waiting for experiment results..." - stdbuf -o 0 tail -f /garlic/st & - stpid=$! - res=$(cat /garlic/completed) - - kill -TERM $stpid - - if [ "$res" == "ERROR" ]; then - echo "resultTree: the experiment failed" - exit 1 - fi - - if [ "$res" != "${trebuchet}" ]; then - echo "resultTree: unknown trebuchet received" - exit 1 - fi - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: assuming results are already in /garlic" - fi - - echo "resultTree: attempting to copy the results from /garlic ..." - - exp=/garlic/cache/${experimentName} - - if [ ! -e "$exp" ]; then - echo "resultTree: $exp: not found" - echo "resultTree: run the experiment and fetch the results running" - echo "resultTree: the following command from the nix-shell" - echo - echo -e "\e[30;48;5;2mgarlic -RFv ${trebuchet}\e[0m" - echo - echo "resultTree: see garlic(1) for more details." - echo "resultTree: cannot continue building $out, aborting" - exit 1 - fi - - echo "resultTree: copying results from /garlic into the nix store..." - - mkdir -p $out - cp -aL $exp $out/ - ln -s ${trebuchet} $out/trebuchet - ln -s ${experiment} $out/experiment - - - if [ -e /garlic/run ]; then - echo "resultTree: removing temp files..." - echo ${trebuchet} >> /garlic/wipe - echo "resultTree: waiting confimation from daemon..." - cat /garlic/completed > /dev/null - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: ignoring temp files" - fi - - echo "resultTree: successfully copied into the nix store" - - echo " experiment: ${experiment}" - echo " trebuchet: ${trebuchet}" - echo " resultTree: $out" - ''; - } diff --git a/garlic/report.nix b/garlic/report.nix deleted file mode 100644 index 6d7dc46..0000000 --- a/garlic/report.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, sedReport -}: -let - sedCmd = (import sedReport) fig; -in - stdenv.mkDerivation { - name = "report.pdf"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${sedCmd} - make - ''; - installPhase = '' - cp report.pdf $out - ''; - } diff --git a/garlic/report/Makefile b/garlic/report/Makefile deleted file mode 100644 index 81ae94d..0000000 --- a/garlic/report/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -all: report.tex - pdflatex report.tex -o report.pdf - # Run again to fix figure references - pdflatex report.tex -o report.pdf diff --git a/garlic/report/report.tex b/garlic/report/report.tex deleted file mode 100644 index f56c1c9..0000000 --- a/garlic/report/report.tex +++ /dev/null @@ -1,31 +0,0 @@ -\documentclass{article} -\usepackage{graphicx} - -\begin{document} - -\title{Example of Nix + \LaTeX{}} -\author{Rodrigo Arias Mallo} - -\maketitle - -\section{Nbody} -The nbody program has been executed with varying block sizes while the execution -time $t$ is measured, as shown in the figure \ref{fig:nbody.test}. -% -\begin{figure}[h] - \centering - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/scatter.png} - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/box.png} - \caption{Nbody times with varying block size} - \label{fig:nbody.test} -\end{figure} -% -The normalized time $\hat t$ is computed with the median time $t_m$ using $ \hat -t = t / t_{m} - 1 $. It can be observed that the normalized times exceed the -maximum allowed interval in most cases, except with the largest block sizes. - -Once the experiment \texttt{exp.nbody.test} changes, the hash of the experiment -program will change, therefore the plot will be updated and, lastly, this -report. - -\end{document} diff --git a/garlic/reportTar.nix b/garlic/reportTar.nix deleted file mode 100644 index c2b274b..0000000 --- a/garlic/reportTar.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, bundleReport -}: -let - - genCmd = (import bundleReport) fig; -in - stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${genCmd} - ls -ltR - cat report.tex - make - ''; - installPhase = '' - cd .. - tar -czf $out report - ''; - } diff --git a/garlic/sedReport.nix b/garlic/sedReport.nix deleted file mode 100644 index b776681..0000000 --- a/garlic/sedReport.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - stdenv -, fig -}: -stdenv.mkDerivation { - name = "sedReport"; - src = ./report; - buildPhase = '' - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - sed 's:\(^.*\)$:sed -i "s;@\1@;''${\1};g" report.tex:g' list >> fun.nix - echo "'''" >> fun.nix - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/sh/default.nix b/garlic/sh/default.nix deleted file mode 100644 index 27d34d1..0000000 --- a/garlic/sh/default.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ - stdenv -, garlicTools -, sshHost -, rsync -, openssh -, nix -, jq -, ncurses -}: - -with garlicTools; - -let - garlicPrefix = "/mnt/garlic"; -in - stdenv.mkDerivation { - name = "garlic-tool"; - preferLocalBuild = true; - - buildInputs = [ rsync openssh nix jq ncurses ]; - phases = [ "unpackPhase" "installPhase" ]; - - src = ./.; - - inherit garlicPrefix sshHost; - - installPhase = '' - substituteAllInPlace garlic - substituteInPlace garlic \ - --replace @PATH@ $PATH - mkdir -p $out/bin - cp garlic $out/bin - chmod +x $out/bin/garlic - mkdir -p $out/share/man/man1 - cp garlic.1 $out/share/man/man1 - - patchShebangs garlic-* - cp garlic-* $out/bin - ''; - } diff --git a/garlic/sh/fix-figure-subtitle.sh b/garlic/sh/fix-figure-subtitle.sh deleted file mode 100755 index a98614b..0000000 --- a/garlic/sh/fix-figure-subtitle.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -if grep -q 'output = args[2]' "$1"; then exit 0; fi - -sed '/length(args)>0/aif (length(args)>1) { output = args[2] } else { output = "?" }' -i "$1" -sed '/jsonlite::flatten/,$s/input_file/output/g' -i "$1" diff --git a/garlic/sh/garlic b/garlic/sh/garlic deleted file mode 100755 index df3e33f..0000000 --- a/garlic/sh/garlic +++ /dev/null @@ -1,298 +0,0 @@ -#!/bin/bash -e - -garlicPrefix=@garlicPrefix@ -sshHost=@sshHost@ -PATH=@PATH@ - -usage() { echo "Usage: garlic [-RFwv] trebuchet" 1>&2; exit 1; } - -msg() { - >&2 echo "garlic: $@" -} - -findClosure() { - what=$1 - from=$2 - mexp=$(nix-store -qR "$from" | grep -E -- "$what") - n=$(echo "$mexp" | awk 'BEGIN { count=0 } NF { count++ } END { print count }') - if [ $n -eq 0 ]; then - >&2 echo "$exp: $what not found" - exit 1 - fi - if [ $n -gt 1 ]; then - >&2 echo "$exp: multiple $what found" - exit 1 - fi - echo "$mexp" -} - -findExperiment() { - grep -o -- "/nix/store/.*-experiment" "$1" -} - -findOutputDir() { - garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - - if [ -z "$garlic_sandbox" ]; then - >&2 echo "Missing extra-sandbox-paths /garlic mountpoint" - >&2 echo "Check the ~/.config/nix/nix.conf file" - exit 1 - fi - - mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') - mountdir=$(readlink -f "$mountdir_rel") - - echo "$mountdir/cache" -} - -drvFromOutput() { - nix-store -q --deriver $1 -} - -checkTrebuchet() { - if [ ! -e "$trebuchet" ]; then - >&2 echo "$trebuchet: not found" - exit 1 - fi - - if [ ! -f "$trebuchet" ]; then - >&2 echo "$trebuchet: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a trebuchet - if [ -z "$(grep "This trebuchet launches" $trebuchet)" ]; then - >&2 echo "$trebuchet: not a trebuchet" - exit 1 - fi - - return 0 -} - -checkExperiment() { - if [ ! -e "$experiment" ]; then - >&2 echo "$experiment: not found" - exit 1 - fi - - if [ ! -f "$experiment" ]; then - >&2 echo "$experiment: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a experiment - if [ -z "$(grep "This is an experiment" $experiment)" ]; then - >&2 echo "$experiment: not an experiment" - exit 1 - fi - - return 0 -} - -checkMountpoint() { - if [ ! -e "$garlicPrefix/garlic.control" ]; then - >&2 echo "error: missing $garlicPrefix/garlic.control" - >&2 echo "Is the mountpoint enabled?" - exit 1 - fi -} - -status_line() { - unithash="$1" - name="$2" - status="$3" - - red=$(tput -T ansi setaf 1) - green=$(tput -T ansi setaf 2) - yellow=$(tput -T ansi setaf 3) - color_reset=$(tput -T ansi sgr0) - - case $status in - ok) - color_st="$green" - ;; - run*) - color_st="$yellow" - ;; - exit*) - color_st="$red" - ;; - *) - color_st="" - ;; - esac - - if [ $verbose ]; then - >&2 printf "garlic: %s %s%-9s%s %s\n" \ - "$unithash" "$color_st" "$status" "$color_reset" "$name" - fi -} - -do_fetch() { - expName=$(basename $experiment) - user=$(ssh -G "$sshHost" | awk '/^user /{print $2}') - exp=$garlicPrefix/$user/out/$expName - - if [ ! -e "$exp" ]; then - echo "missing experiment: $exp" - exit 1 - fi - - unitNameList=$(grep '^/nix/store/.*-unit$' $experiment |\ - sort -u |\ - sed 's@^/nix/store/@@') - - cwd=$(pwd) - - repeat=1 - bad=0 - while [ 1 ]; do - repeat=0 - test $verbose && msg "Checking units $(date --rfc-3339=seconds)..." - - declare -A unit_names - - for unit in $unitNameList; do - - unit_hash="${unit%-unit}" - unit_status="?" - unit_name="?" - - if [ -e "$exp/$unit" ]; then - st_file="$exp/$unit/status" - conf_json="$exp/$unit/garlic_config.json" - done_file="$exp/$unit/done" - - if [ -z "${unit_names[$unit_hash]}" ]; then - if [ -e "$conf_json" ]; then - unit_names+=([$unit_hash]=$(jq -r .unitName "$conf_json")) - unit_name="${unit_names[$unit_hash]}" - fi - else - unit_name="${unit_names[$unit_hash]}" - fi - - if [ -e "$st_file" ]; then - unit_status=$(cat "$st_file") - fi - fi - - status_line "$unit_hash" "$unit_name" "$unit_status" - - if [ ! -e "$done_file" ]; then - repeat=1 - elif [ "$unit_status" != "ok" ]; then - bad=1 - fi - done - - if [ $repeat -eq 0 ]; then - break - else - test $verbose && msg "" - fi - - if [ $waitResults -eq 1 ]; then - #echo "waiting 3 seconds to try again" - sleep 3 - else - break - fi - done - - if [ $repeat -eq 1 ]; then - #>&2 echo "$exp: execution incomplete" - exit 1 - fi - - if [ $bad -eq 1 ]; then - msg "Some units failed, aborting" - exit 1 - fi - - cd "$cwd" - - test $verbose && msg "execution complete, fetching results" - - mkdir -p "$outputDir" - - rsync -vrt --copy-links \ - --include='*/*/garlic_config.json' \ - --include='*/*/std*.log' \ - --include='*/*/*/std*.log' \ - --include='*/*/*/.garlic' \ - --include='*/*/*/.garlic/**' \ - --exclude='*/*/*/*' \ - "$exp" "$outputDir" - - if [ ! $enableKeep ]; then - nix-build -E "(with import ./default.nix; \ - garlic.pp.store { \ - trebuchet = (import \"$trebuchetDrv\" ); \ - experiment = (import \"$experimentDrv\"); \ - })" - - rm -rf "$outputDir/$expName" - fi -} - -do_run() { - - >&2 $trebuchet -} - -do_delete() { - expName=$(basename $experiment) - rm -rf $outputDir/$expName -} - -waitResults=1 -verbose= -operation= -target= -enableRun= -enableFetch= -enableKeep= -enableDelete= - -while getopts "vwRFKD" o; do - case "${o}" in - R) enableRun=1 ;; - F) enableFetch=1 ;; - K) enableKeep=1 ;; - D) enableDelete=1 ;; - w) waitResults=0 ;; - v) verbose=1 ;; - *) usage ;; - esac -done -shift $((OPTIND-1)) -trebuchet="$1" - -if [ -z "$enableRun" -a -z "$enableFetch" -a -z "$enableDelete" ]; then - >&2 echo "missing operation" - usage -fi - -if [ -z "$trebuchet" ]; then - >&2 echo "missing experiment" - usage -fi - -checkMountpoint - -checkTrebuchet $trebuchet - -outputDir=$(findOutputDir) - -experiment=$(findExperiment "$trebuchet") -checkExperiment $experiment - -trebuchetDrv=$(drvFromOutput $trebuchet) -experimentDrv=$(drvFromOutput $experiment) - -if [ $enableRun ]; then do_run; fi -if [ $enableFetch ]; then do_fetch; fi -if [ $enableDelete ]; then do_delete; fi diff --git a/garlic/sh/garlic-add-copyright b/garlic/sh/garlic-add-copyright deleted file mode 100755 index 089aafe..0000000 --- a/garlic/sh/garlic-add-copyright +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -if [ -z "$1" ]; then - cat << 'EOF' -Usage: garlic-add-copyright files... - -This tool prepends the content of the copyright file to the list of given files. -The word 'copyright' is used to determine if a file already has a copyright -notice or not. - -Example: - - Given the following HEADER file: - - $ cat HEADER - /* - * This file is part of NBody and is licensed under the terms contained - * in the LICENSE file. - * - * Copyright (C) 2021 Barcelona Supercomputing Center (BSC) - */ - - It can be prepended to all files ending in .c or .h with: - - $ garlic-add-copyright HEADER $(find * -type f | grep -Ei '\.(c|h)$') - -EOF - exit 1 -fi - -header_file="$1" -shift - -if ! grep -qi copyright "$header_file"; then - >&2 echo "The header file '$header_file' doesn't contain the word 'copyright'" - exit 1 -fi - -for f in "${@}"; do - if grep -qi copyright "$f"; then - echo "$f: Contains copyright word, skipping" - continue - fi - - tmp_fn="$f.copyright-being-added" - - cat "$header_file" "$f" > "$tmp_fn" - mv "$tmp_fn" "$f" -done diff --git a/garlic/sh/garlic-git-table b/garlic/sh/garlic-git-table deleted file mode 100755 index 58f4049..0000000 --- a/garlic/sh/garlic-git-table +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -progname="$(basename $0)" - -if [ -z "$1" ]; then - cat >&2 < - -Finds all garlic/* branches and their current commit of the given -repository and builds the gitTable to be used in nix derivations. - -Example: - garlic-git-table ssh://git@bscpm03.bsc.es/garlic/apps/heat.git -EOF - exit 1 -fi - -echo '{' -echo " # Auto-generated with $progname on $(date --rfc-3339=date) for repo:" -echo " # $1" -echo -git ls-remote $1 'garlic/*' |\ - sed 's@refs/heads/@@' |\ - awk '{printf "\"%s\" = \"%s\";\n", $2, $1}' |\ - column -t -o ' ' |\ - sed 's/^/ /' -echo '}' diff --git a/garlic/sh/garlic-propagate-commit b/garlic/sh/garlic-propagate-commit deleted file mode 100755 index 04617e1..0000000 --- a/garlic/sh/garlic-propagate-commit +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -e - -function printHelp() -{ - >&2 echo "Usage: garlic-propagate-commit " - >&2 echo - cat >&2 </dev/null | sed 's/^..//'` == $1 ]]; then - return 0 - else - return 1 - fi -} - - -if [ $# -lt 1 ]; then - printHelp - exit 1 -fi - -# Obtain all the tracked branches -branches=(`git branch | sed 's/^..//'`) -currentBranch=`git rev-parse --abbrev-ref HEAD` - -# Make sure that the commit SHA exists -commit=$1 -if branchContainsCommit $currentBranch $commit; then - echo "Commit $commit exists in current branch, proceeding..." -else - echo "Error: Commit $commit does not exist in the current branch" - exit 1 -fi - -# Distribute the commit for all tracked branches -for branch in ${branches[@]}; -do - if [[ $branch != $currentBranch ]]; then - echo "Trying to add commit $commit to branch $branch" - if branchContainsCommit $branch $commit; then - echo "Branch $branch already contains commit $commit, skipping" - else - git checkout $branch - git cherry-pick $commit - git push - fi - fi -done - -# Return to the original branch -git checkout $currentBranch diff --git a/garlic/sh/garlic-query b/garlic/sh/garlic-query deleted file mode 100755 index 6b0c2de..0000000 --- a/garlic/sh/garlic-query +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -function find_garlic_conf() { - grep -o '^cp /nix/store/[^ ]*-garlic_config.json' "$1" | sed 's/^cp //g' -} - -function show_units() { - exp=$1 - - units=$(grep '^/nix/store/.*-unit$' $exp) - - nunits=$(echo "$units" | wc -l) - echo " Experiment: $exp" - echo " Units: $nunits" - echo - echo " Unit file Name" - - for unit in $units; do - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - printf " %s %s %s %s\n" "$unit" "$present" "$unitName" - done -} - -function query_tre() { - tre=$1 - exp=$(grep "^# */nix/store/[^ ]*-experiment$" "$tre" | sed 's/# *//g') - - echo - echo " Trebuchet: $tre" - show_units $exp - echo -} - -function query_exp() { - exp=$1 - - echo - show_units "$exp" - echo -} - -function query_unit() { - unit=$1 - - stages=$(grep '^# */nix/store/.*$' $unit | sed 's/^# */ /g') - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - echo - echo " Unit: $unit" - echo " Name: $unitName" - echo " Stages:" - echo - echo "$stages" - echo - echo " Config: $gconf" - echo - jq . "$gconf" -} - -function query_result() { - tree=$1 - - tre=$(readlink -f $tree/trebuchet) - exp=$(readlink -f $tree/experiment) - - echo - echo " Result tree: $tree" - echo " Trebuchet: $tre" - show_units $exp - echo -} - -element=$1 - -if [ "$1" == "--help" -o -z "$1" ]; then - cat < - -The path may be a trebuchet, experiment, unit or resultTree. -EOF - exit 1 -fi - -# Try prepending the nix store -if [ ! -e $element ]; then - element=/nix/store/$element* -fi - -element=$(readlink -f $element) - -case "$element" in - *-trebuchet) query_tre $element ;; - *-experiment) query_exp $element ;; - *-unit) query_unit $element ;; - *-resultTree) query_result $element ;; - *) echo "unknown: $element"; exit 1 ;; -esac diff --git a/garlic/sh/garlic.1 b/garlic/sh/garlic.1 deleted file mode 100644 index 86915d8..0000000 --- a/garlic/sh/garlic.1 +++ /dev/null @@ -1,61 +0,0 @@ -.\" The following commands are required for all man pages. -.Dd Nov 5, 2020 -.Dt garlic 1 -.Os Linux -.Sh NAME -.Nm garlic -.Nd Garlic benchmark tool -.Sh SYNOPSIS -.Nm garlic -.Op Fl RFvw -.Ar trebuchet -.Sh DESCRIPTION -.Nm -is a program to handle tasks related with the experiments of the Garlic -benchmark. The -.Ar trebuchet -argument is a path to a valid trebuchet stage script. The -specified operations are executed sequentially and at -least one must be selected from: -.Bl -tag -width Ds -compact -.Pp -.It Fl R -launches the -.Ar experiment using the trebuchet. -.Pp -.It Fl F -waits until the -.Ar experiment -is completed and then fetches the results -into the nix store. -.El -.Pp -Additionally, the following options are available: -.Bl -tag -width Ds -compact -.Pp -.It Fl v -be more verbose. -.Pp -.It Fl w -don't wait for the complete results when fetching; fails if they are not -ready. -.El -.\" This next command is for sections 1, 6, 7, and 8 only. -.\" .Sh ENVIRONMENT -.\" .Sh FILES -.Sh EXIT STATUS -.Ex -std -.\" .Sh EXAMPLES -.\" This next command is for sections 1, 4, 6, 7, 8, and 9 only -.\" (fprintf/stderr type diagnostics). -.\" .Sh DIAGNOSTICS -.\" .Sh COMPATIBILITY -.\" This next command is for sections 2, 3, 4, and 9 only -.\" (settings of the errno variable). -.\" .Sh ERRORS -.\" .Sh SEE ALSO -.\" .Sh STANDARDS -.\" .Sh HISTORY -.\" .Sh AUTHORS -.\" .Sh CAVEATS -.\" .Sh BUGS diff --git a/garlic/shell.nix b/garlic/shell.nix deleted file mode 100644 index 9a38076..0000000 --- a/garlic/shell.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ pkgs ? import ./. }: - -with pkgs; -with bsc; - -mkShell { - name = "garlic-shell"; - - buildInputs = - # Packages from garlic - (with garlic; [ tool garlicd ]) ++ - # Packages from bsc - [ groff paraver icc nix openssh git cn6 nix-diff clangOmpss2 gdb ]; - - # inputsFrom to get build dependencies - - shellHook = '' - alias l="ls -l --color=auto -v" - alias ll="ls -l --color=auto -v" - alias lh="ls -hAl --color=auto -v" - alias ls="ls --color=auto -v" - alias ..="cd .." - - export LANG=C - export SHELL=${bash}/bin/bash - - echo Welcome to the garlic shell - ''; -} diff --git a/garlic/stages/baywatch.nix b/garlic/stages/baywatch.nix deleted file mode 100644 index 2b72f4d..0000000 --- a/garlic/stages/baywatch.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - stdenv -, garlicTools -}: -{ - nextStage -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "baywatch"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${stageProgram nextStage} - echo $? >> .srun.rc.$SLURM_PROCID - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/control.nix b/garlic/stages/control.nix deleted file mode 100644 index 3e04485..0000000 --- a/garlic/stages/control.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, loops ? 30 -}: - -with garlicTools; - -stdenv.mkDerivation { - name = "control"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<"EOF" - #!/bin/sh -e - - function badexit() { - errcode=$? - if [ $errcode != 0 ]; then - printf "exit %d\n" $errcode > "$basedir/status" - echo "exiting with $errcode" - fi - - echo 1 > "$basedir/done" - exit $errcode - } - - trap badexit EXIT - - basedir=$(pwd) - loops=${toString loops} - for n in $(seq 1 $loops); do - export GARLIC_RUN="$n" - echo "run $n/$loops" > status - mkdir "$n" - cd "$n" - mkdir .garlic - date +%s > .garlic/total_time_start - ${stageProgram nextStage} - date +%s > .garlic/total_time_end - cd .. - done - echo "ok" > status - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/exec.nix b/garlic/stages/exec.nix deleted file mode 100644 index 8717041..0000000 --- a/garlic/stages/exec.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, env ? "" -, pre ? "" -, argv ? [] -, post ? "" -, nixPrefix ? "" -, program ? null -}: - -with builtins; -with garlicTools; - -let - argvString = concatStringsSep " " (map (e: toString e) argv); - execMethod = if (post == "") then "exec " else ""; - programPath = if (program != null) then program else (stageProgram nextStage); -in -stdenv.mkDerivation { - name = "exec"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - ${env} - - ${pre} - - ${execMethod}${nixPrefix}${programPath} ${argvString} - - ${post} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/experiment.nix b/garlic/stages/experiment.nix deleted file mode 100644 index 19c0e45..0000000 --- a/garlic/stages/experiment.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, garlicTools -}: - -{ - units -}: - -with lib; -with garlicTools; - -let - unitsString = builtins.concatStringsSep "\n" - (map (x: "${stageProgram x}") units); - - unitsLinks = builtins.concatStringsSep "\n" - (map (x: "ln -s ../${baseNameOf x} ${baseNameOf x}") units); -in -stdenv.mkDerivation { - name = "experiment"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - inherit units; - - isExperiment = true; - - installPhase = '' - cat > $out << EOF - #!/bin/sh - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "experiment: GARLIC_OUT not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - export GARLIC_EXPERIMENT=$(basename $out) - - if [ -e "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "experiment: skipping, directory exists: \$GARLIC_EXPERIMENT" - exit 0 - fi - - mkdir -p "\$GARLIC_EXPERIMENT" - - cd "\$GARLIC_EXPERIMENT" - ${unitsLinks} - - >&2 echo "experiment: running \$GARLIC_EXPERIMENT" - - # This is an experiment formed by the following units: - ${unitsString} - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/extrae.nix b/garlic/stages/extrae.nix deleted file mode 100644 index dabb31e..0000000 --- a/garlic/stages/extrae.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, configFile -, traceLib -, extrae -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "extrae"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <&2 echo Running isolate stage1 -#>&2 echo PATH=$PATH - -if [ -e /nix ]; then - >&2 echo "/nix found, aborting" - exit 1 -fi - -nixhome="@nixPrefix@/nix" -shell="@busybox@/bin/sh" -nixjoin="@nixPrefix@@nixtools@/bin/nix-join" - - -env=( - PATH="@nixPrefix@@busybox@/bin:@busybox@/bin:@extraPath@" - $(env | grep ^SLURM || true) - $(env | grep ^PMI || true) - $(env | grep ^GARLIC || true) - $(env | grep ^USER || true) - $(env | grep ^TMPDIR || true) - $(env | grep '^TERM=' || true) - HOME="/homeless-shelter" -) - -mounts=( - #-m @nixPrefix@ - #FIXME: Use only the strictly neccesary from /etc - -m /original-etc:/etc - # The /etc/hosts file is a symlink to this etc/ - -m /.statelite/tmpfs/etc - -m /sys - -m /dev - -m /proc - # nscd cache: doesn't exist (?) - #-m /var/run/nscd - # Needed for munge auth - -m /var/run/munge - # FIXME: We should only need nix and the output path - -m /gpfs/projects/bsc15 - -m /gpfs/scratch/bsc15 - -m /bin:@nixPrefix@@busybox@/bin - -m "$TMPDIR" - @extraMountOptions@ -) - -symlinks=( - -s /etc/hosts:/original-etc/hosts - -s /etc/passwd:/original-etc/passwd - -s /etc/resolv.conf:/original-etc/resolv.conf - -s /etc/host.conf:/original-etc/host.conf - -s /etc/slurm/slurm.conf:/original-etc/slurm/slurm.conf - -s /etc/services:/original-etc/services -) - -exec $nixjoin -i "${mounts[@]}" "${symlinks[@]}" $nixhome -- \ - env -i "${env[@]}" @out@/bin/stage2 "$@" diff --git a/garlic/stages/isolate/stage2 b/garlic/stages/isolate/stage2 deleted file mode 100644 index 1924890..0000000 --- a/garlic/stages/isolate/stage2 +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -#>&2 echo Running isolate stage2 -#>&2 echo PATH=$PATH - -if [ ! -e /nix ]; then - >&2 echo "/nix not found, aborting" - exit 1 -fi - -if [ -e /usr ]; then - >&2 echo "Environment not isolated, aborting" - exit 1 -fi - -exec @program@ "$@" diff --git a/garlic/stages/perf.nix b/garlic/stages/perf.nix deleted file mode 100644 index cbbe937..0000000 --- a/garlic/stages/perf.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - stdenv -, perf -, garlicTools -}: - -{ - nextStage -, perfOptions -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "perf"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out <&2 echo "Cannot use runexp inside nix environment!" - exit 1 -fi - -#>&2 echo Running runexp for MN4 -#>&2 echo PATH=$PATH - -user=$(id -un) -group=$(id -gn) - -export GARLIC_OUT="/gpfs/projects/bsc15/garlic/$user/out" -mkdir -p "$GARLIC_OUT" -export GARLIC_INDEX="/gpfs/projects/bsc15/garlic/$user/index" -mkdir -p "$GARLIC_INDEX" -export GARLIC_USER="$user" -cd "$GARLIC_OUT" - -exec @nixPrefix@@program@ diff --git a/garlic/stages/sbatch.nix b/garlic/stages/sbatch.nix deleted file mode 100644 index 34a3167..0000000 --- a/garlic/stages/sbatch.nix +++ /dev/null @@ -1,105 +0,0 @@ -{ - stdenv -, lib -, numactl -, slurm -, garlicTools -}: - -{ - nextStage -, jobName -, chdir ? "." -, nixPrefix ? "" -, binary ? "/bin/run" -, ntasks ? null -, ntasksPerNode ? null -, ntasksPerSocket ? null -, cpusPerTask ? null -, nodes ? null -, exclusive ? true # By default we run in exclusive mode -, qos ? null -, reservation ? null -, time ? null -, output ? "stdout.log" -, error ? "stderr.log" -, extra ? null -, acctgFreq ? null -}: - -with lib; -with garlicTools; - -# sbatch fails silently if we pass garbage, so we assert the types here to avoid -# sending `nodes = [ 1 2 ]` by mistake. -assert (jobName != null) -> isString jobName; -assert (chdir != null) -> isString chdir; -assert (nixPrefix != null) -> isString nixPrefix; -assert (ntasks != null) -> isInt ntasks; -assert (ntasksPerNode != null) -> isInt ntasksPerNode; -assert (ntasksPerSocket != null) -> isInt ntasksPerSocket; -assert (cpusPerTask != null) -> isInt cpusPerTask; -assert (nodes != null) -> isInt nodes; -assert (exclusive != null) -> isBool exclusive; -assert (qos != null) -> isString qos; -assert (reservation != null) -> isString reservation; -assert (time != null) -> isString time; -assert (output != null) -> isString output; -assert (error != null) -> isString error; -assert (extra != null) -> isString extra; - -let - - sbatchOpt = name: value: optionalString (value!=null) - "#SBATCH --${name}=${toString value}\n"; - sbatchEnable = name: value: optionalString (value!=null) - "#SBATCH --${name}\n"; - -in - -stdenv.mkDerivation rec { - name = "sbatch"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - #SBATCH --tasks-per-node=48 - #SBATCH --ntasks-per-socket=24 - #SBATCH --cpus-per-task=1 - dontBuild = true; - dontPatchShebangs = true; - programPath = "/run"; - - installPhase = '' - mkdir -p $out - cat > $out/job < $out/run < status - EOF - chmod +x $out/run - ''; -} diff --git a/garlic/stages/script.nix b/garlic/stages/script.nix deleted file mode 100644 index 292a209..0000000 --- a/garlic/stages/script.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -}: - -{ - script -, shell ? "/bin/sh" -, exitOnError ? true -}: - -let - setcmd = if exitOnError then "set -e" else ""; -in -stdenv.mkDerivation { - name = "script"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!${shell} - ${setcmd} - - ${script} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/srun.nix b/garlic/stages/srun.nix deleted file mode 100644 index a2481a5..0000000 --- a/garlic/stages/srun.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, slurm -, garlicTools -}: -{ - nextStage -, cpuBind -, nixPrefix -, preSrun ? "" -, postSrun ? "" -, srunOptions ? "" -, output ? "stdout.log" -, error ? "stderr.log" -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "srun"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${preSrun} - - ${slurm}/bin/srun \ - --mpi=pmi2 \ - --cpu-bind=${cpuBind} \ - --output=${output} \ - --error=${error} \ - ${srunOptions} \ - ${nixPrefix}${stageProgram nextStage} - - >&2 echo srun exit code: $? - - # Ensure that none failed, as srun fails to capture errors - # after MPI_Finalize - for i in $(seq 0 $(($SLURM_NTASKS - 1))); do - if [ ! -e .srun.rc.$i ]; then - >&2 echo "missing exit code for rank $i, aborting" - exit 1 - fi - if ! grep -q '^0$' .srun.rc.$i; then - >&2 echo "non-zero exit for rank $i, aborting" - exit 1 - fi - done - - ${postSrun} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/strace.nix b/garlic/stages/strace.nix deleted file mode 100644 index 22fb4a5..0000000 --- a/garlic/stages/strace.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, strace -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "strace"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out < $out << EOF - #!/bin/sh -e - - ${desc} - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "unit: GARLIC_OUT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "unit: GARLIC_EXPERIMENT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_INDEX" ]; then - >&2 echo "unit: GARLIC_INDEX not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - # Set the experiment unit in the environment - export GARLIC_UNIT=$(basename $out) - - # Create an index entry - rm -f "\$GARLIC_INDEX/${safeUnitName}" \ - "\$GARLIC_INDEX/${safeExpName}" - - ln -Tfs "../out/\$GARLIC_UNIT" \ - "\$GARLIC_INDEX/${safeUnitName}" - - ln -Tfs "../out/\$GARLIC_EXPERIMENT" \ - "\$GARLIC_INDEX/${safeExpName}" - - if [ -e "\$GARLIC_UNIT" ]; then - >&2 echo "unit: skipping, already exists: \$GARLIC_UNIT" - exit 0 - fi - - # And change the working directory - mkdir \$GARLIC_UNIT - cd \$GARLIC_UNIT - - # Copy the configuration for the unit to the output path - cp ${jsonConf} garlic_config.json - - # Finally, execute the first stage: - exec ${firstStage} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/valgrind.nix b/garlic/stages/valgrind.nix deleted file mode 100644 index 9e156c0..0000000 --- a/garlic/stages/valgrind.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, valgrind -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "valgrind"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out''); -} diff --git a/garlic/tools.nix b/garlic/tools.nix deleted file mode 100644 index c092b55..0000000 --- a/garlic/tools.nix +++ /dev/null @@ -1,140 +0,0 @@ -{ - stdenv -, lib -}: - -with lib; - -let - gen = rec { - # genAttrSets "a" ["hello" "world"] - # [ { a = "hello"; } { a = "world"; } ] - genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); - - # addAttrSets "a" [1 2] {e=4;} - # [ { a = 1; e = 4; } { a = 2; e = 4; } ] - addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); - - # attrToList {a=1;} - # [ { name = "a"; value = 1; } ] - attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); - - # mergeConfig [{e=1;}] {name="a"; value=[1 2] - # [ { a = 1; e = 1; } { a = 2; e = 1; } ] - mergeConfig = (arr: new: flatten ( map (x: addAttrSets new.name new.value x) arr)); - - # genConfigs {a=[1 2]; b=[3 4];} - # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] - genConfigs = (config: foldl mergeConfig [{}] (attrToList config)); - - # Generate multiple app versions by override with each config - genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs); - - # Generate app version from an array of apps - genApps = (apps: configs: - flatten (map (app: genApp app configs) apps)); - - /* Returns the path of the executable of a stage */ - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" - else "${stage}"; - - /* Given a trebuchet, returns the experiment */ - getExperimentStage = drv: - if (drv ? isExperiment) && drv.isExperiment then drv - else getExperimentStage drv.nextStage; - - # Computes the exponentiation operation - pow = x: n: fold (a: b: a*b) 1 (map (a: x) (range 1 n)); - - # Generates a list of exponents from a to b inclusive, and raises base to - # each element of the list. - expRange = base: a: b: (map (ex: pow base ex) (range a b)); - - # Generates a range from start to end (inclusive) by multiplying start by 2. - range2 = start: end: - let - _range2 = s: e: if (s > e) then [] else [ s ] ++ (_range2 (s * 2) e); - in - _range2 start end; - - # Generates a list of integers by halving number N until it reaches 1. Is - # sorted from the smallest to largest. - halfList = N: - let - _divList = n: if (n == 0) then [] else (_divList (n / 2)) ++ [ n ]; - in - _divList N; - - # A list of all divisors of n, sorted in increased order: - divisors = n: filter (x: (mod n x == 0)) (range 1 n); - - # Generates a set given a list of keys, where all values are null. - genNullAttr = l: genAttrs l (name: null); - - # From the keys in the lis l, generates a set with the values in the set a, - # if they don't exist, they are not taken. Values set to null are removed. - optionalInherit = l: a: filterAttrs (n: v: v!=null) - (overrideExisting (genNullAttr l) a); - - # Given a float f, truncates it and returns the resulting the integer - floatTruncate = f: let - strFloat = toString f; - slices = splitString "." strFloat; - front = elemAt slices 0; - in - toInt front; - - # Returns the given gitCommit if not null, or the one stored in the - # gitTable for the branch gitBranch. - findCommit = {gitCommit ? null, gitTable ? null, gitBranch}: - assert (gitCommit == null) -> (gitTable != null); - assert (gitTable == null) -> (gitCommit != null); - if (gitCommit != null) then gitCommit - else - assert (assertMsg (gitTable ? "${gitBranch}") - '' - The git branch "${gitBranch}" was not found in the gitTable. - Is the gitTable outdated? - ''); - gitTable."${gitBranch}"; - - # Custom wrapper around fetchGit to be able to quickly specify apps - # and change the repository source for all at once. Returns an - # attributte set with the `src` as well as the selected gitCommit, - # gitBranch and gitURL. - fetchGarlicApp = { - gitBranch, - appName ? null, - gitURL ? null, - gitCommit ? null, - gitTable ? null - }: - assert (appName == null) -> (gitURL != null); - assert (gitURL == null) -> (appName != null); - let - _gitURL = if (gitURL != null) then gitURL - else "ssh://git@bscpm03.bsc.es/garlic/apps/${appName}.git"; - _gitCommit = findCommit { - inherit gitCommit gitTable gitBranch; - }; - _gitBranch = gitBranch; - in - { - src = builtins.fetchGit { - url = _gitURL; - ref = _gitBranch; - rev = _gitCommit; - }; - gitBranch = _gitBranch; - gitCommit = _gitCommit; - - # The gitURL is not stored in the derivation, as we dont really - # care of where the source comes from, as long as is the same - # commit. - gitURL = _gitURL; - }; - }; -in - gen