From 4533c94b4f367580b978803418965b3a760572b8 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Tue, 3 Oct 2023 12:24:58 +0200 Subject: [PATCH] Remove garlic from bscpkgs --- garlic/apps/bigsort/default.nix | 54 - garlic/apps/bigsort/genseq.nix | 46 - garlic/apps/bigsort/shuffle.nix | 46 - garlic/apps/cpic/default.nix | 43 - garlic/apps/creams/default.nix | 57 - garlic/apps/creams/gen_grid.py | 49 - garlic/apps/creams/git-table.nix | 12 - garlic/apps/creams/input.nix | 48 - garlic/apps/fwi/default.nix | 77 - garlic/apps/fwi/git-table.nix | 15 - garlic/apps/fwi/params.nix | 65 - garlic/apps/heat/default.nix | 39 - garlic/apps/heat/git-table.nix | 9 - garlic/apps/heat/print-times.patch | 13 - garlic/apps/hpccg/default.nix | 55 - garlic/apps/hpccg/git-table.nix | 9 - garlic/apps/hpcg/default.nix | 44 - garlic/apps/hpcg/git-table.nix | 18 - garlic/apps/hpcg/tampi.patch | 13 - garlic/apps/ifsker/default.nix | 60 - garlic/apps/ifsker/git-table.nix | 10 - garlic/apps/index.nix | 55 - garlic/apps/lulesh/default.nix | 48 - garlic/apps/lulesh/git-table.nix | 12 - garlic/apps/miniamr/default.nix | 48 - garlic/apps/nbody/default.nix | 59 - garlic/apps/nbody/git-table.nix | 13 - garlic/apps/ppong/default.nix | 35 - garlic/apps/saiph/default.nix | 106 -- garlic/apps/saiph/git-table.nix | 10 - garlic/bundleReport.nix | 31 - garlic/default.nix | 1 - garlic/develop/default.nix | 58 - garlic/develop/inputrc | 37 - garlic/doc/.gitignore | 6 - garlic/doc/Makefile | 42 - garlic/doc/blackbox.ms | 40 - garlic/doc/branch.ms | 22 - garlic/doc/execution.ms | 254 --- garlic/doc/gitbranch.pic | 152 -- garlic/doc/pp.ms | 256 --- garlic/doc/ref.i | 9 - garlic/doc/s.css | 19 - garlic/doc/slides/2.mm | 1468 ---------------- garlic/doc/slides/3.mm | 181 -- garlic/doc/slides/3.pdf | Bin 122774 -> 0 bytes garlic/doc/slides/Makefile | 29 - garlic/doc/slides/overview.svg | 479 ------ garlic/doc/slides/sat.png | Bin 79137 -> 0 bytes garlic/doc/slides/scalability.svg | 265 --- garlic/doc/slides/scaling-region.svg | 806 --------- garlic/doc/ug.ms | 1691 ------------------- garlic/ds/ctf-mode.py | 83 - garlic/ds/index.nix | 18 - garlic/ds/osu-bw.py | 64 - garlic/ds/osu-latency.py | 64 - garlic/ds/perf-stat.py | 90 - garlic/ds/py.nix | 34 - garlic/ds/std-timetable.py | 77 - garlic/exp/bigsort/genseq.nix | 73 - garlic/exp/bigsort/shuffle.nix | 102 -- garlic/exp/bigsort/sort.nix | 126 -- garlic/exp/cn6/nbody.nix | 164 -- garlic/exp/cn6/timediff.nix | 171 -- garlic/exp/creams/granularity.nix | 126 -- garlic/exp/creams/granularity16.nix | 132 -- garlic/exp/creams/size.nix | 131 -- garlic/exp/creams/ss.nix | 126 -- garlic/exp/examples/granularity.nix | 189 --- garlic/exp/fwi/common.nix | 128 -- garlic/exp/fwi/granularity.nix | 71 - garlic/exp/fwi/io.nix | 75 - garlic/exp/fwi/reuse.nix | 91 - garlic/exp/fwi/ss.nix | 90 - garlic/exp/heat/granularity.nix | 175 -- garlic/exp/hpcg/common.nix | 73 - garlic/exp/hpcg/gen.nix | 52 - garlic/exp/hpcg/granularity.nix | 76 - garlic/exp/hpcg/scaling.nix | 78 - garlic/exp/hpcg/size.nix | 65 - garlic/exp/index.nix | 108 -- garlic/exp/lulesh/test.nix | 85 - garlic/exp/nbody/common.nix | 37 - garlic/exp/nbody/granularity.nix | 60 - garlic/exp/nbody/numa.nix | 64 - garlic/exp/nbody/old/granularity-oss.nix | 60 - garlic/exp/nbody/old/nblocks.nix | 104 -- garlic/exp/nbody/old/scaling.nix | 110 -- garlic/exp/nbody/old/strong-scaling-oss.nix | 72 - garlic/exp/nbody/old/weak-scaling-mpi.nix | 63 - garlic/exp/nbody/old/weak-scaling-oss.nix | 63 - garlic/exp/nbody/ss.nix | 59 - garlic/exp/osu/bw.nix | 60 - garlic/exp/osu/eager.nix | 84 - garlic/exp/osu/impi.nix | 69 - garlic/exp/osu/latency.nix | 69 - garlic/exp/osu/mtu.nix | 84 - garlic/exp/saiph/granularity.nix | 132 -- garlic/exp/saiph/ss.nix | 122 -- garlic/exp/slurm/cpu.nix | 63 - garlic/exp/slurm/exit1.nix | 69 - garlic/exp/slurm/sigsegv.nix | 60 - garlic/fig/creams/granularity.R | 83 - garlic/fig/creams/granularity16.R | 136 -- garlic/fig/creams/size.R | 96 -- garlic/fig/creams/ss.R | 128 -- garlic/fig/dev/shell.nix | 14 - garlic/fig/examples/granularity.R | 195 --- garlic/fig/fwi/granularity.R | 74 - garlic/fig/fwi/io.R | 120 -- garlic/fig/fwi/ss.R | 95 -- garlic/fig/heat/cache.R | 78 - garlic/fig/heat/granularity.R | 68 - garlic/fig/heat/mode.R | 122 -- garlic/fig/hpcg/granularity.R | 63 - garlic/fig/hpcg/size.R | 71 - garlic/fig/hpcg/ss.R | 82 - garlic/fig/hpcg/ws.R | 71 - garlic/fig/index.nix | 97 -- garlic/fig/nbody/granularity.R | 62 - garlic/fig/nbody/numa.R | 86 - garlic/fig/nbody/old/baseline.R | 212 --- garlic/fig/nbody/old/freeCpu.R | 112 -- garlic/fig/nbody/old/jemalloc.R | 113 -- garlic/fig/nbody/old/scaling.R | 111 -- garlic/fig/nbody/ss.R | 66 - garlic/fig/osu/bw.R | 59 - garlic/fig/osu/eager.R | 63 - garlic/fig/osu/impi.R | 68 - garlic/fig/osu/latency.R | 77 - garlic/fig/osu/mtu.R | 69 - garlic/fig/saiph/granularity.R | 95 -- garlic/fig/saiph/ss.R | 110 -- garlic/garlicd/default.nix | 28 - garlic/garlicd/garlicd | 105 -- garlic/index.nix | 138 -- garlic/machines.nix | 63 - garlic/mpptest/default.nix | 16 - garlic/pp/hist/default.nix | 25 - garlic/pp/hist/hist.sh | 79 - garlic/pp/merge.nix | 24 - garlic/pp/rplot.nix | 165 -- garlic/pp/store.nix | 89 - garlic/report.nix | 24 - garlic/report/Makefile | 4 - garlic/report/report.tex | 31 - garlic/reportTar.nix | 28 - garlic/sedReport.nix | 19 - garlic/sh/default.nix | 41 - garlic/sh/fix-figure-subtitle.sh | 6 - garlic/sh/garlic | 298 ---- garlic/sh/garlic-add-copyright | 49 - garlic/sh/garlic-git-table | 27 - garlic/sh/garlic-propagate-commit | 63 - garlic/sh/garlic-query | 102 -- garlic/sh/garlic.1 | 61 - garlic/shell.nix | 29 - garlic/stages/baywatch.nix | 26 - garlic/stages/control.nix | 52 - garlic/stages/exec.nix | 42 - garlic/stages/experiment.nix | 60 - garlic/stages/extrae.nix | 34 - garlic/stages/isolate/default.nix | 44 - garlic/stages/isolate/stage1 | 58 - garlic/stages/isolate/stage2 | 16 - garlic/stages/perf.nix | 30 - garlic/stages/pgdb.nix | 30 - garlic/stages/runexp/default.nix | 29 - garlic/stages/runexp/runexp | 21 - garlic/stages/sbatch.nix | 105 -- garlic/stages/script.nix | 28 - garlic/stages/srun.nix | 58 - garlic/stages/strace.nix | 29 - garlic/stages/trebuchet.nix | 43 - garlic/stages/unit.nix | 107 -- garlic/stages/valgrind.nix | 29 - garlic/stdexp.nix | 148 -- garlic/tools.nix | 140 -- 178 files changed, 16994 deletions(-) delete mode 100644 garlic/apps/bigsort/default.nix delete mode 100644 garlic/apps/bigsort/genseq.nix delete mode 100644 garlic/apps/bigsort/shuffle.nix delete mode 100644 garlic/apps/cpic/default.nix delete mode 100644 garlic/apps/creams/default.nix delete mode 100755 garlic/apps/creams/gen_grid.py delete mode 100644 garlic/apps/creams/git-table.nix delete mode 100644 garlic/apps/creams/input.nix delete mode 100644 garlic/apps/fwi/default.nix delete mode 100644 garlic/apps/fwi/git-table.nix delete mode 100644 garlic/apps/fwi/params.nix delete mode 100644 garlic/apps/heat/default.nix delete mode 100644 garlic/apps/heat/git-table.nix delete mode 100644 garlic/apps/heat/print-times.patch delete mode 100644 garlic/apps/hpccg/default.nix delete mode 100644 garlic/apps/hpccg/git-table.nix delete mode 100644 garlic/apps/hpcg/default.nix delete mode 100644 garlic/apps/hpcg/git-table.nix delete mode 100644 garlic/apps/hpcg/tampi.patch delete mode 100644 garlic/apps/ifsker/default.nix delete mode 100644 garlic/apps/ifsker/git-table.nix delete mode 100644 garlic/apps/index.nix delete mode 100644 garlic/apps/lulesh/default.nix delete mode 100644 garlic/apps/lulesh/git-table.nix delete mode 100644 garlic/apps/miniamr/default.nix delete mode 100644 garlic/apps/nbody/default.nix delete mode 100644 garlic/apps/nbody/git-table.nix delete mode 100644 garlic/apps/ppong/default.nix delete mode 100644 garlic/apps/saiph/default.nix delete mode 100644 garlic/apps/saiph/git-table.nix delete mode 100644 garlic/bundleReport.nix delete mode 100644 garlic/default.nix delete mode 100644 garlic/develop/default.nix delete mode 100644 garlic/develop/inputrc delete mode 100644 garlic/doc/.gitignore delete mode 100644 garlic/doc/Makefile delete mode 100644 garlic/doc/blackbox.ms delete mode 100644 garlic/doc/branch.ms delete mode 100644 garlic/doc/execution.ms delete mode 100644 garlic/doc/gitbranch.pic delete mode 100644 garlic/doc/pp.ms delete mode 100644 garlic/doc/ref.i delete mode 100644 garlic/doc/s.css delete mode 100644 garlic/doc/slides/2.mm delete mode 100644 garlic/doc/slides/3.mm delete mode 100644 garlic/doc/slides/3.pdf delete mode 100644 garlic/doc/slides/Makefile delete mode 100644 garlic/doc/slides/overview.svg delete mode 100644 garlic/doc/slides/sat.png delete mode 100644 garlic/doc/slides/scalability.svg delete mode 100644 garlic/doc/slides/scaling-region.svg delete mode 100644 garlic/doc/ug.ms delete mode 100644 garlic/ds/ctf-mode.py delete mode 100644 garlic/ds/index.nix delete mode 100644 garlic/ds/osu-bw.py delete mode 100644 garlic/ds/osu-latency.py delete mode 100644 garlic/ds/perf-stat.py delete mode 100644 garlic/ds/py.nix delete mode 100644 garlic/ds/std-timetable.py delete mode 100644 garlic/exp/bigsort/genseq.nix delete mode 100644 garlic/exp/bigsort/shuffle.nix delete mode 100644 garlic/exp/bigsort/sort.nix delete mode 100644 garlic/exp/cn6/nbody.nix delete mode 100644 garlic/exp/cn6/timediff.nix delete mode 100644 garlic/exp/creams/granularity.nix delete mode 100644 garlic/exp/creams/granularity16.nix delete mode 100644 garlic/exp/creams/size.nix delete mode 100644 garlic/exp/creams/ss.nix delete mode 100644 garlic/exp/examples/granularity.nix delete mode 100644 garlic/exp/fwi/common.nix delete mode 100644 garlic/exp/fwi/granularity.nix delete mode 100644 garlic/exp/fwi/io.nix delete mode 100644 garlic/exp/fwi/reuse.nix delete mode 100644 garlic/exp/fwi/ss.nix delete mode 100644 garlic/exp/heat/granularity.nix delete mode 100644 garlic/exp/hpcg/common.nix delete mode 100644 garlic/exp/hpcg/gen.nix delete mode 100644 garlic/exp/hpcg/granularity.nix delete mode 100644 garlic/exp/hpcg/scaling.nix delete mode 100644 garlic/exp/hpcg/size.nix delete mode 100644 garlic/exp/index.nix delete mode 100644 garlic/exp/lulesh/test.nix delete mode 100644 garlic/exp/nbody/common.nix delete mode 100644 garlic/exp/nbody/granularity.nix delete mode 100644 garlic/exp/nbody/numa.nix delete mode 100644 garlic/exp/nbody/old/granularity-oss.nix delete mode 100644 garlic/exp/nbody/old/nblocks.nix delete mode 100644 garlic/exp/nbody/old/scaling.nix delete mode 100644 garlic/exp/nbody/old/strong-scaling-oss.nix delete mode 100644 garlic/exp/nbody/old/weak-scaling-mpi.nix delete mode 100644 garlic/exp/nbody/old/weak-scaling-oss.nix delete mode 100644 garlic/exp/nbody/ss.nix delete mode 100644 garlic/exp/osu/bw.nix delete mode 100644 garlic/exp/osu/eager.nix delete mode 100644 garlic/exp/osu/impi.nix delete mode 100644 garlic/exp/osu/latency.nix delete mode 100644 garlic/exp/osu/mtu.nix delete mode 100644 garlic/exp/saiph/granularity.nix delete mode 100644 garlic/exp/saiph/ss.nix delete mode 100644 garlic/exp/slurm/cpu.nix delete mode 100644 garlic/exp/slurm/exit1.nix delete mode 100644 garlic/exp/slurm/sigsegv.nix delete mode 100644 garlic/fig/creams/granularity.R delete mode 100644 garlic/fig/creams/granularity16.R delete mode 100644 garlic/fig/creams/size.R delete mode 100644 garlic/fig/creams/ss.R delete mode 100644 garlic/fig/dev/shell.nix delete mode 100644 garlic/fig/examples/granularity.R delete mode 100644 garlic/fig/fwi/granularity.R delete mode 100644 garlic/fig/fwi/io.R delete mode 100644 garlic/fig/fwi/ss.R delete mode 100644 garlic/fig/heat/cache.R delete mode 100644 garlic/fig/heat/granularity.R delete mode 100644 garlic/fig/heat/mode.R delete mode 100644 garlic/fig/hpcg/granularity.R delete mode 100644 garlic/fig/hpcg/size.R delete mode 100644 garlic/fig/hpcg/ss.R delete mode 100644 garlic/fig/hpcg/ws.R delete mode 100644 garlic/fig/index.nix delete mode 100644 garlic/fig/nbody/granularity.R delete mode 100644 garlic/fig/nbody/numa.R delete mode 100644 garlic/fig/nbody/old/baseline.R delete mode 100644 garlic/fig/nbody/old/freeCpu.R delete mode 100644 garlic/fig/nbody/old/jemalloc.R delete mode 100644 garlic/fig/nbody/old/scaling.R delete mode 100644 garlic/fig/nbody/ss.R delete mode 100644 garlic/fig/osu/bw.R delete mode 100644 garlic/fig/osu/eager.R delete mode 100644 garlic/fig/osu/impi.R delete mode 100644 garlic/fig/osu/latency.R delete mode 100644 garlic/fig/osu/mtu.R delete mode 100644 garlic/fig/saiph/granularity.R delete mode 100644 garlic/fig/saiph/ss.R delete mode 100644 garlic/garlicd/default.nix delete mode 100755 garlic/garlicd/garlicd delete mode 100644 garlic/index.nix delete mode 100644 garlic/machines.nix delete mode 100644 garlic/mpptest/default.nix delete mode 100644 garlic/pp/hist/default.nix delete mode 100755 garlic/pp/hist/hist.sh delete mode 100644 garlic/pp/merge.nix delete mode 100644 garlic/pp/rplot.nix delete mode 100644 garlic/pp/store.nix delete mode 100644 garlic/report.nix delete mode 100644 garlic/report/Makefile delete mode 100644 garlic/report/report.tex delete mode 100644 garlic/reportTar.nix delete mode 100644 garlic/sedReport.nix delete mode 100644 garlic/sh/default.nix delete mode 100755 garlic/sh/fix-figure-subtitle.sh delete mode 100755 garlic/sh/garlic delete mode 100755 garlic/sh/garlic-add-copyright delete mode 100755 garlic/sh/garlic-git-table delete mode 100755 garlic/sh/garlic-propagate-commit delete mode 100755 garlic/sh/garlic-query delete mode 100644 garlic/sh/garlic.1 delete mode 100644 garlic/shell.nix delete mode 100644 garlic/stages/baywatch.nix delete mode 100644 garlic/stages/control.nix delete mode 100644 garlic/stages/exec.nix delete mode 100644 garlic/stages/experiment.nix delete mode 100644 garlic/stages/extrae.nix delete mode 100644 garlic/stages/isolate/default.nix delete mode 100644 garlic/stages/isolate/stage1 delete mode 100644 garlic/stages/isolate/stage2 delete mode 100644 garlic/stages/perf.nix delete mode 100644 garlic/stages/pgdb.nix delete mode 100644 garlic/stages/runexp/default.nix delete mode 100755 garlic/stages/runexp/runexp delete mode 100644 garlic/stages/sbatch.nix delete mode 100644 garlic/stages/script.nix delete mode 100644 garlic/stages/srun.nix delete mode 100644 garlic/stages/strace.nix delete mode 100644 garlic/stages/trebuchet.nix delete mode 100644 garlic/stages/unit.nix delete mode 100644 garlic/stages/valgrind.nix delete mode 100644 garlic/stdexp.nix delete mode 100644 garlic/tools.nix diff --git a/garlic/apps/bigsort/default.nix b/garlic/apps/bigsort/default.nix deleted file mode 100644 index 4e7dd3d..0000000 --- a/garlic/apps/bigsort/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - stdenv -, lib -, cc -, nanos6 ? null -, mcxx ? null -, mpi -, gitBranch -}: - -with lib; -stdenv.mkDerivation rec { - name = "bigsort"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "${gitBranch}"; - }; - - #sourceRoot = "./BigSort"; - - preBuild = '' - cd BigSort - export I_MPI_CXX=${cc.CXX} - ''; - - buildInputs = [ - cc - mpi - ] - ++ optional (mcxx != null) mcxx - ++ optional (nanos6 != null) nanos6; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - "CLUSTER=MareNostrum4" - "OPENMP=yes" - "Debug=no" - "OPENMP_FLAGS=-qopenmp" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bigsort $out/bin/BigSort - ''; - - programPath = "/bin/BigSort"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/genseq.nix b/garlic/apps/bigsort/genseq.nix deleted file mode 100644 index 6036467..0000000 --- a/garlic/apps/bigsort/genseq.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "genseq"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/GenSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp genseq $out/bin/genseq - ''; - - programPath = "/bin/genseq"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/shuffle.nix b/garlic/apps/bigsort/shuffle.nix deleted file mode 100644 index 977d911..0000000 --- a/garlic/apps/bigsort/shuffle.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "shuffle"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/ShuffleSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp shuffle $out/bin/shuffle - ''; - - programPath = "/bin/shuffle"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/cpic/default.nix b/garlic/apps/cpic/default.nix deleted file mode 100644 index 284c1e9..0000000 --- a/garlic/apps/cpic/default.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ - stdenv -, libconfig -, nanos6 -, mpi -, uthash -, fftw -, tampi -, hdf5 -}: - -stdenv.mkDerivation rec { - name = "cpic"; - - # Use my current cpic version, so I can test changes without commits - #src = /home/Computational/rarias/cpic; - - src = builtins.fetchGit { - url = "https://github.com/rodarima/cpic"; -# rev = "73bd70448587f0925b89e24c8f17e412ea3958e6"; - ref = "simd"; - }; - - enableParallelBuilding = true; - dontStrip = true; - - buildInputs = [ - libconfig - nanos6 - mpi - uthash - fftw - tampi - hdf5 - ]; - - installPhase = '' - mkdir -p $out/bin - cp cpic $out/bin/cpic - ''; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/creams/default.nix b/garlic/apps/creams/default.nix deleted file mode 100644 index c083886..0000000 --- a/garlic/apps/creams/default.nix +++ /dev/null @@ -1,57 +0,0 @@ -{ - stdenv -, nanos6 -, mpi -, openmpi -, impi -, tampi -, mcxx -, gnuDef -, intelDef -, cc -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -assert (mpi == impi || mpi == openmpi); - -let - # FIXME: We should find a better way to specify the MPI implementation - # and the compiler. - mpiName = if mpi == openmpi then "OpenMPI" else "IntelMPI"; - compName = if cc == intelDef then "Intel" else "GNU"; - - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "creams"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/creams.exe"; - - buildInputs = [ nanos6 mpi cc tampi mcxx ]; - - configurePhase = '' - export TAMPI_HOME=${tampi} - - . etc/bashrc - - export FORTRAN_COMPILER=${compName} - export MPI_LIB=${mpiName} - - CREAMS_UPDATE_ENVIRONMENT - ''; - - installPhase = '' - mkdir -p $out/bin - cp -a build/* $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/creams/gen_grid.py b/garlic/apps/creams/gen_grid.py deleted file mode 100755 index 723988b..0000000 --- a/garlic/apps/creams/gen_grid.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -parser = argparse.ArgumentParser(description="Generate a grid.dat input file for CREAMS") - -parser.add_argument('--npx', type=int, help='number of processes in X', default=1) -parser.add_argument('--npy', type=int, help='number of processes in Y', default=1) -parser.add_argument('--npz', type=int, help='number of processes in Z', default=32) -parser.add_argument('--grain', type=int, help='granularity', default=9) -parser.add_argument('--nx', type=int, help='number of points in X', default=20) -parser.add_argument('--ny', type=int, help='number of points in Y', default=20) -parser.add_argument('--nz', type=int, help='number of points in Z', default=7000) -parser.add_argument('--dx', type=float, help='grid spacing in X', default=0.0025062657) -parser.add_argument('--dy', type=float, help='grid spacing in Y', default=0.0025062657) -parser.add_argument('--dz', type=float, help='grid spacing in Z', default=0.0025062657) - -args = parser.parse_args() - -grain_str = "%d %d" % (args.grain, args.grain) -boundary = "extrapolation" - -# Print -print(' %-49d number of processes in x-direction (0 if automatic)' % args.npx) -print(' %-49d number of processes in y-direction (0 if automatic)' % args.npy) -print(' %-49d number of processes in z-direction (0 if automatic)' % args.npz) -print(' ') -print(' %-49s subdomain granularity' % grain_str) -print(' ') -print(' %-49s -x boundary' % boundary) -print(' %-49s +x boundary' % boundary) -print(' %-49s -y boundary' % boundary) -print(' %-49s +y boundary' % boundary) -print(' %-49s -z boundary' % boundary) -print(' %-49s +z boundary' % boundary) -print(' ') -print(' x-direction') -for i in range(args.nx): - print("%.9e" % (i * args.dx)) -print(' ') -print(' y-direction') -for i in range(args.ny): - print("%.9e" % (i * args.dy)) -print(' ') -print(' z-direction') -for i in range(args.nz): - print("%.9e" % (i * args.dz)) -print(' ') -print(' END') diff --git a/garlic/apps/creams/git-table.nix b/garlic/apps/creams/git-table.nix deleted file mode 100644 index 9600240..0000000 --- a/garlic/apps/creams/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-03-31 for repo: - # ssh://git@bscpm03.bsc.es/pmartin1/creams-simplified.git - - "garlic/mpi+isend+omp+task" = "e6aa540820ee12d3d45d0eef8e7eeb2f0f1daea2"; - "garlic/mpi+isend+oss+task" = "016f33b8bec996a4546e8f08b1b6b1709f00499b"; - "garlic/mpi+send+omp+fork" = "e56e059264ad1bfe5e0c96a8b9303d21dd7fa20a"; - "garlic/mpi+send+omp+task" = "919580213de34bc5b6ba60c768c5dde5e501a1f6"; - "garlic/mpi+send+oss+task" = "adab8b66f27317d51445648302e7b133edf4837d"; - "garlic/mpi+send+seq" = "956125f9334493d31ceee3fa7024efa65bee9ca5"; - "garlic/tampi+isend+oss+task" = "14a121627679a251909d4b8103d260e27eac1d29"; -} diff --git a/garlic/apps/creams/input.nix b/garlic/apps/creams/input.nix deleted file mode 100644 index 5a12f91..0000000 --- a/garlic/apps/creams/input.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, python3 -, granul ? 9 -, nprocx ? 1 -, nprocy ? 1 -, nprocz ? 1 -, nx ? 20 -, ny ? 20 -, nz ? 7000 -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; - - gen = ./gen_grid.py; -in - stdenv.mkDerivation rec { - name = "creams-input"; - - buildInputs = [ python3 ]; - - inherit (gitSource) src gitBranch gitCommit; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -a SodTubeBenchmark $out/ - - python3 ${gen} \ - --npx ${toString nprocx} \ - --npy ${toString nprocy} \ - --npz ${toString nprocz} \ - --grain ${toString granul} \ - --nx ${toString nx} \ - --ny ${toString ny} \ - --nz ${toString nz} \ - > $out/SodTubeBenchmark/grid.dat - ''; - } diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix deleted file mode 100644 index 681509a..0000000 --- a/garlic/apps/fwi/default.nix +++ /dev/null @@ -1,77 +0,0 @@ -{ - stdenv -, lib -, mpi ? null -, tampi ? null -, mcxx ? null -, cc -, gitBranch ? "garlic/tampi+send+oss+task" -, gitCommit ? null -, fwiParams -, garlicTools -}: - -with lib; - -assert !(tampi != null && mcxx == null); - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # FIXME: Correct this on the Makefile so we can just type "make fwi" - # FIXME: Allow multiple MPI implementations - postPatch = '' - sed -i 's/= OPENMPI$/= INTEL/g' Makefile - sed -i 's/USE_O_DIRECT ?= NO/USE_O_DIRECT ?= YES/g' Makefile || true - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - - make depend - - cp ${fwiParams}/generated_model_params.h src/ - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - makeFlags = [ - #"COMPILER=GNU" - #"CC=${cc.cc.CC}" - "fwi" - ]; - - installPhase = '' - mkdir -p $out/bin - cp fwi $out/bin - ''; - - programPath = "/bin/fwi"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/fwi/git-table.nix b/garlic/apps/fwi/git-table.nix deleted file mode 100644 index c20adff..0000000 --- a/garlic/apps/fwi/git-table.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git - - "garlic/mpi+send+omp+fork" = "ea1ed53f20858dc082f9cbbe0e7e8fb28a6fe58a"; - "garlic/mpi+send+omp+task" = "aa8881056fb3fa98832d203899beacfb8fa702f6"; - "garlic/mpi+send+oss+task" = "c184484af8498fd939761575b34bb46ba3be0dde"; - "garlic/mpi+send+oss+task+NOREUSE" = "0062093ef744c694d69673d1881719958bbed353"; - "garlic/mpi+send+seq" = "cc184ad77f143481e2506933d0cdc038c349f071"; - "garlic/omp+task" = "1fe23690d74ae89ace5383ab165a6ce6346c2afd"; - "garlic/oss+task" = "68e6e8f17ee03addb460745acea2a38241ca89ee"; - "garlic/seq" = "aa6b6c5857125796c65fbf23018d557e4693f1ae"; - "garlic/tampi+isend+oss+task" = "7c98194e13786c4e8ecfa8a144587e5a95e09205"; - "garlic/tampi+send+oss+task" = "e08d66f7453c4034a363bb2d22c5248fe86ed740"; -} diff --git a/garlic/apps/fwi/params.nix b/garlic/apps/fwi/params.nix deleted file mode 100644 index 7fad46b..0000000 --- a/garlic/apps/fwi/params.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, nz ? 200 -, nx ? 200 -, ny ? 500 -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -with lib; -with builtins; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi-params"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - # Set the input size with the weird order (nz,nx,ny). - postPatch = '' - sed -i 1c${toString nz} SetupParams/fwi_params.txt - sed -i 2c${toString nx} SetupParams/fwi_params.txt - sed -i 3c${toString ny} SetupParams/fwi_params.txt - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - # Also, we need to compile it with the builder platform as target, as is going - # to be executed during the build to generate the src/generated_model_params.h - # header. - makeFlags = [ "COMPILER=GNU" "params" ]; - - installPhase = '' - mkdir -p $out/ - cp src/generated_model_params.h $out/ - cp SetupParams/fwi_params.txt $out/ - cp SetupParams/fwi_frequencies.txt $out/ - - mkdir -p $out/bin - cp ModelGenerator $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/default.nix b/garlic/apps/heat/default.nix deleted file mode 100644 index 02e8b76..0000000 --- a/garlic/apps/heat/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ - stdenv -, mpi -, tampi -, mcxx -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "heat"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - - name = "heat"; - - inherit (gitSource) src gitBranch gitCommit; - - patches = [ ./print-times.patch ]; - - buildInputs = [ mpi mcxx tampi ]; - - programPath = "/bin/${name}"; - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp heat.conf $out/etc/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/git-table.nix b/garlic/apps/heat/git-table.nix deleted file mode 100644 index c7670ac..0000000 --- a/garlic/apps/heat/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-01 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/heat.git - - "garlic/mpi+send+oss+task" = "947c80070d4c53e441df54b8bfac8928b10c5fb2"; - "garlic/mpi+send+seq" = "f41e1433808d0cbecd88a869b451c927747e5d42"; - "garlic/tampi+isend+oss+task" = "b1273f9b4db32ba6e15e3d41343e67407ce2f54f"; - "garlic/tampi+send+oss+task" = "554bec249f9aa23dd92edcfa2ada1e03e05e121d"; -} diff --git a/garlic/apps/heat/print-times.patch b/garlic/apps/heat/print-times.patch deleted file mode 100644 index bf5d28a..0000000 --- a/garlic/apps/heat/print-times.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/mpi/main.c b/src/mpi/main.c -index 44f4a99..08a1f5c 100644 ---- a/src/mpi/main.c -+++ b/src/mpi/main.c -@@ -83,6 +83,8 @@ int main(int argc, char **argv) - conf.rows, conf.cols, conf.rows/nranks, totalElements, totalElements/nranks, - conf.rbs, conf.cbs, nranks, threads, conf.timesteps, end-start, throughput); - printf("time %e\n", end - start); -+ printf("start_time %.9f\n", start); -+ printf("end_time %.9f\n", end); - } - - if (conf.generateImage) { diff --git a/garlic/apps/hpccg/default.nix b/garlic/apps/hpccg/default.nix deleted file mode 100644 index bb827a7..0000000 --- a/garlic/apps/hpccg/default.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - stdenv -, lib -, icc -, mpi ? null -, tampi ? null -, mcxx ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpccg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpccg"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/test_HPCCG-mpi.exe"; - - buildInputs = [ - icc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # The hpccg app fails to compile in parallel. Makefile must be fixed before. - enableParallelBuilding = false; - - makeFlags = [ - "USE_MPI=-DUSING_MPI" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp test_HPCCG-mpi.exe $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpccg/git-table.nix b/garlic/apps/hpccg/git-table.nix deleted file mode 100644 index 507dab2..0000000 --- a/garlic/apps/hpccg/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpccg.git - - "garlic/mpi+isend+omp+fork" = "c84af0480d231961201f2904ee4e3fced9d5f9be"; - "garlic/mpi+isend+seq" = "d1b47cd459440700de1b68233ec4fe794343dbd4"; - "garlic/tampi+isend+oss+task" = "7238e9be2e4a7b028abc05d40b476462eaa3de6a"; - "garlic/tampi+isend+oss+taskfor" = "02ec60f43b8d68d74575ea0563a9029fd441f1f1"; -} diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix deleted file mode 100644 index 245012e..0000000 --- a/garlic/apps/hpcg/default.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ - stdenv -, cc -, nanos6 -, mcxx -, mpi -, tampi -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpcg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpcg"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ - cc nanos6 mcxx mpi tampi - ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bin/* $out/bin/ - ''; - - programPath = "/bin/xhpcg"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpcg/git-table.nix b/garlic/apps/hpcg/git-table.nix deleted file mode 100644 index 0aa1fc8..0000000 --- a/garlic/apps/hpcg/git-table.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpcg.git - - "garlic/mpi" = "8c94ccfd97518ed947bd6be3386260b72fdcdff2"; - "garlic/mpi+SAVEMAT" = "5dd2ad9eba13dba67086f46c6e8519804d837383"; - "garlic/mpi+omp" = "d24c372dd9fda584e711efb612f172e5c3602804"; - "garlic/mpi+oss" = "519a867bb3a3e07440df05e60a62abad764524e5"; - "garlic/mpi+send+omp+fork" = "a08d31aedbc108e1c0081cdc5021827ac9022688"; - "garlic/omp" = "dcc8a40831cda884b9240af47e883ac997150ed3"; - "garlic/omp+SAVEMAT" = "40dbac86c905e192ecc8146e0e65e4c3a3c6dbf8"; - "garlic/omp+fork" = "042752b3dbcd9b0f4db524b6cdc911278ee1a51b"; - "garlic/omp+initsplit" = "5370e7ee26fb72ef100a79624f73ed2baa6bcc79"; - "garlic/oss" = "7e6e2d969b7904572f2475bf471e637651337761"; - "garlic/oss+task" = "034940756ccab88876609c3cba4dea0a0f5c944b"; - "garlic/seq" = "dee225571ab2572d7aa51df9846b01237ee941a1"; - "garlic/tampi+isend+oss+task" = "449a3980a767f91ca65d429490080961dcfba498"; -} diff --git a/garlic/apps/hpcg/tampi.patch b/garlic/apps/hpcg/tampi.patch deleted file mode 100644 index c233dfc..0000000 --- a/garlic/apps/hpcg/tampi.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- a/setup/Make.MPI_ICPC_OSS 2020-07-13 16:02:33.272257865 +0200 -+++ b/setup/Make.MPI_ICPC_OSS 2020-07-13 16:04:34.344413390 +0200 -@@ -91,8 +91,8 @@ - # - HPCG includes / libraries / specifics ------------------------------- - # ---------------------------------------------------------------------- - # --HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I{TAMPI_HOME}/include --HPCG_LIBS = ${TAMPI_HOME}/lib/libtampi.a -+HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I$(TAMPI_HOME)/include -+HPCG_LIBS = -l:libtampi.a - # - # - Compile time options ----------------------------------------------- - # diff --git a/garlic/apps/ifsker/default.nix b/garlic/apps/ifsker/default.nix deleted file mode 100644 index b575364..0000000 --- a/garlic/apps/ifsker/default.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, mpi -, gfortran -, tampi -, nanos6 -, mcxx -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "ifsker"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "ifsker"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ tampi mpi nanos6 mcxx gfortran ]; - - # Mercurium seems to fail when building with fortran in parallel - enableParallelBuilding = false; - - # FIXME: Patch mcxx to use other directory than $HOME for the lock - # files. - preConfigure = '' - export TAMPI_HOME=${tampi} - - # $HOME is required for the lock files by mcxx to compile fortran. - # So we use the $TMPDIR to store them. - export HOME=$TMPDIR - ''; - - makeFlags = [ - "-f" "Makefile.gcc" - ]; - - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp -r data $out/etc/ - cp nanos6.toml $out/etc - ''; - - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/ifsker/git-table.nix b/garlic/apps/ifsker/git-table.nix deleted file mode 100644 index 0242708..0000000 --- a/garlic/apps/ifsker/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/ifsker.git - - "garlic/mpi+isend+oss+fork" = "a6a20303101cb140571ddc1166e66843fbe83017"; - "garlic/mpi+isend+oss+task" = "a7bdb6c9b57aafbc50cdc790eb42e5bdd850f213"; - "garlic/mpi+isend+seq" = "bc97cf30835bbf6a825209485bb96fc8314e5bcb"; - "garlic/tampi+isend+oss+task" = "33408215bc231b70b60733fddea3d1b7431bb0d1"; - "garlic/tampi+send+oss+task" = "19dae956b4ef69916c0e8ad15bb6ced0085275cd"; -} diff --git a/garlic/apps/index.nix b/garlic/apps/index.nix deleted file mode 100644 index c0e2d4d..0000000 --- a/garlic/apps/index.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = callPackage ./nbody/default.nix { }; - - saiph = callPackage ./saiph/default.nix { - cc = bsc.clangOmpss2; - L3SizeKB = garlic.targetMachine.config.hw.cacheSizeKB.L3; - cachelineBytes = garlic.targetMachine.config.hw.cachelineBytes; - }; - - creams = callPackage ./creams/default.nix { - gnuDef = self.gfortran10 ; # Default GNU compiler version - intelDef = bsc.icc ; # Default Intel compiler version - }; - - creamsInput = callPackage ./creams/input.nix { }; - - hpcg = callPackage ./hpcg/default.nix { }; - - bigsort = { - sort = callPackage ./bigsort/default.nix { - gitBranch = "garlic/mpi+send+omp+task"; - }; - - genseq = callPackage ./bigsort/genseq.nix { }; - - shuffle = callPackage ./bigsort/shuffle.nix { }; - }; - - heat = callPackage ./heat/default.nix { }; - - miniamr = callPackage ./miniamr/default.nix { - variant = "ompss-2"; - }; - - ifsker = callPackage ./ifsker/default.nix { }; - - lulesh = callPackage ./lulesh/default.nix { }; - - hpccg = callPackage ./hpccg/default.nix { }; - - fwi = rec { - params = callPackage ./fwi/params.nix { }; - solver = callPackage ./fwi/default.nix { - fwiParams = params; - }; - }; -} diff --git a/garlic/apps/lulesh/default.nix b/garlic/apps/lulesh/default.nix deleted file mode 100644 index 8f64682..0000000 --- a/garlic/apps/lulesh/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, impi -, mcxx -, icc -, tampi ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "lulesh"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "lulesh"; - - inherit (gitSource) src gitBranch gitCommit; - - dontConfigure = true; - - preBuild = optionalString (tampi != null) "export TAMPI_HOME=${tampi}"; - - #TODO: Allow multiple MPI implementations and compilers - buildInputs = [ - impi - icc - mcxx - ]; - - enableParallelBuilding = true; - - #TODO: Can we build an executable named "lulesh" in all branches? - installPhase = '' - mkdir -p $out/bin - find . -name 'lulesh*' -type f -executable -exec cp \{\} $out/bin/${name} \; - ''; - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/lulesh/git-table.nix b/garlic/apps/lulesh/git-table.nix deleted file mode 100644 index 49c2bba..0000000 --- a/garlic/apps/lulesh/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/lulesh.git - - "garlic/mpi+isend+omp+fork" = "6cc85c55cb4840d6cde12bb285f5ab1ae7878618"; - "garlic/mpi+isend+oss+task" = "0a9e2cd1d64ab4fcf1860ace02866278ad289637"; - "garlic/mpi+isend+seq" = "9df5475c7dd2b345559fae5bd07ceea38f2e7b91"; - "garlic/tampi+isend+oss+task" = "28ce0cd69f9b4e65eff8141ec455d5f60e9b98b3"; - "garlic/tampi+isend+oss+taskfor" = "928f315ea426585a32231d950da651399e48d762"; - "garlic/tampi+isend+oss+taskloop" = "7957c1a2c84ae80edddcec9eafe7efdeefa68d58"; - "garlic/tampi+isend+oss+taskloopfor" = "7efa0535130a6726f5a46669cf171412d21adc9b"; -} diff --git a/garlic/apps/miniamr/default.nix b/garlic/apps/miniamr/default.nix deleted file mode 100644 index a014a73..0000000 --- a/garlic/apps/miniamr/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, tampi -, clangOmpss2 -, mpi -, nanos6 -, mcxx -, variant -}: - -with lib; - -assert (assertOneOf "variant" variant [ "openmp" "openmp-tasks" "ompss-2" ]); - -let - cc=mcxx; -in -stdenv.mkDerivation rec { - name = "miniamr"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/ksala/miniamr.git"; - ref = "master"; - }; - - postUnpack = '' - sourceRoot=$sourceRoot/${variant} - ''; - - buildInputs = [ tampi clangOmpss2 mpi nanos6 mcxx ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp miniAMR.x $out/bin/ - ''; - - programPath = "/bin/miniAMR.x"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/nbody/default.nix b/garlic/apps/nbody/default.nix deleted file mode 100644 index d5de8ce..0000000 --- a/garlic/apps/nbody/default.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi ? null -, tampi ? null -, mcxx ? null -, cflags ? null -, gitBranch ? "garlic/seq" -, gitCommit ? null -, blocksize ? 2048 -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "nbody"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "nbody"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/nbody"; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - preBuild = (if cflags != null then '' - makeFlagsArray+=(CFLAGS="${cflags}") - '' else ""); - - makeFlags = [ - "CC=${cc.CC}" - "BS=${toString blocksize}" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp nbody* $out/bin/${name} - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/nbody/git-table.nix b/garlic/apps/nbody/git-table.nix deleted file mode 100644 index e824251..0000000 --- a/garlic/apps/nbody/git-table.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/nbody.git - - "garlic/mpi+send+oss+task" = "20aa856baa3268d99262588807911ad0b3318d09"; - "garlic/mpi+send+seq" = "3be64af0f949db5fd60fcd0334cf2cd8c9fa25c3"; - "garlic/omp+fork" = "9c869272df7c775f467a2220211a414e33321c00"; - "garlic/oss+task" = "13ab26fbad8662a1052cc94410386080bbf6a2ba"; - "garlic/seq" = "9dfea29189d14477bd75e6f741f0518e7e4e5e72"; - "garlic/seq+BLOCK" = "99408705628b374df4308dcf1cdbe2d21d1451c2"; - "garlic/tampi+isend+oss+task" = "653d26e4a0913d36ea18d4e72e65a04838bb138a"; - "garlic/tampi+send+oss+task" = "b1440ebc5f79165e5dfaa6a4ce7916eda410ec9a"; -} diff --git a/garlic/apps/ppong/default.nix b/garlic/apps/ppong/default.nix deleted file mode 100644 index 09a72a4..0000000 --- a/garlic/apps/ppong/default.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "ppong"; - - src = fetchurl { - url = "http://www.csl.mtu.edu/cs4331/common/PPong.c"; - sha256 = "0d1w72gq9627448cb7ykknhgp2wszwd117dlbalbrpf7d0la8yc0"; - }; - - unpackCmd = '' - mkdir src - cp $src src/ppong.c - ''; - - dontConfigure = true; - - buildPhase = '' - echo mpicc -include stdlib.h ppong.c -o ppong - mpicc -include stdlib.h ppong.c -o ppong - ''; - - installPhase = '' - mkdir -p $out/bin - cp ppong $out/bin/ppong - ln -s $out/bin/ppong $out/bin/run - ''; - - buildInputs = [ mpi ]; - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/saiph/default.nix b/garlic/apps/saiph/default.nix deleted file mode 100644 index bb81c89..0000000 --- a/garlic/apps/saiph/default.nix +++ /dev/null @@ -1,106 +0,0 @@ -{ - stdenv -, lib -, nanos6 -, mpi -, tampi -, cc -, vtk -, boost -, gitBranch ? "master" -, gitCommit ? null -, enableManualDist ? false -, nbgx ? null -, nbgy ? null -, nbgz ? null -, nblx ? null -, nbly ? null -, nblz ? null -, nsteps ? null -, numComm ? null -, enableVectFlags ? false -, enableDebugFlags ? false -, enableAsanFlags ? false -, cachelineBytes ? null -, L3SizeKB ? null -# Problem size: -, sizex ? 3 -, sizey ? 4 -, sizez ? 4 -, garlicTools -}: - -assert enableManualDist -> (nbgx != null); -assert enableManualDist -> (nbgy != null); -assert enableManualDist -> (nbgz != null); - -with lib; -with lib.versions; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "saiph"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "saiph"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/Heat3D_vect"; - - enableParallelBuilding = true; - dontStrip = true; - enableDebugging = true; - - buildInputs = [ - nanos6 - mpi - tampi - cc - vtk - boost - ]; - - preBuild = '' - cd saiphv2/cpp/src - export VTK_VERSION=${majorMinor (getVersion vtk.name)} - export VTK_HOME=${vtk} - make clean - - sed -i '/SIZEX =/s/3/${toString sizex}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEY =/s/4/${toString sizey}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEZ =/s/4/${toString sizez}/g' testApp/Heat3D_vect.cpp - ''; - - makeFlags = [ - "-f" "Makefile.${cc.CC}" - "apps" - "APP=Heat3D_vect" - ] ++ optional (cachelineBytes != null) "ROW_ALIGNMENT=${toString cachelineBytes}" - ++ optional (L3SizeKB != null) "L3_SIZE_K=${toString L3SizeKB}" - ++ optional (enableManualDist) "DIST_SET=1" - ++ optional (enableManualDist) "NBG_X=${toString nbgx}" - ++ optional (enableManualDist) "NBG_Y=${toString nbgy}" - ++ optional (enableManualDist) "NBG_Z=${toString nbgz}" - ++ optional (nblx != null) "NBL_X=${toString nblx}" - ++ optional (nbly != null) "NBL_Y=${toString nbly}" - ++ optional (nblz != null) "NBL_Z=${toString nblz}" - ++ optional (nsteps != null) "NSTEPS=${toString nsteps}" - ++ optional (numComm != null) "NUM_COMM=${toString numComm}" - ++ optional (enableVectFlags) "VECT_CHECKS=1" - ++ optional (enableDebugFlags) "DEBUG_CHECKS=1" - ++ optional (enableAsanFlags) "SANITIZE_CHECKS=1" - ; - - installPhase = '' - mkdir -p $out/lib - mkdir -p $out/bin - cp obj/libsaiphv2.so $out/lib/ - cp bin/Heat3D_vect $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/saiph/git-table.nix b/garlic/apps/saiph/git-table.nix deleted file mode 100644 index 13cc477..0000000 --- a/garlic/apps/saiph/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/saiph.git - - "garlic/mpi+isend+omp+fork+simd" = "96823846b327b6860f05d428f0cd5ed8ca537a0e"; - "garlic/mpi+isend+omp+task+simd" = "de0346a559120f561bff554aa86b34d01214b714"; - "garlic/mpi+isend+seq+simd" = "1411dad765231f5d3cec9f621526583974232d42"; - "garlic/tampi+isend+omp+task+simd" = "587a7651df8eb69cae4a79bdfc5cb7f50723f3ce"; - "garlic/tampi+isend+oss+task+simd" = "3731197d3e35df248fa6bdb7e4cb05c5dd4f2597"; -} diff --git a/garlic/bundleReport.nix b/garlic/bundleReport.nix deleted file mode 100644 index fd84775..0000000 --- a/garlic/bundleReport.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - stdenv -, fig -}: - -stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildPhase = '' - pwd - ls -l - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - for line in $(cat list); do - localPath=$(echo $line | tr '.' '/') - echo "mkdir -p $localPath" >> fun.nix - echo "cp -r \''${$line}/* $localPath" >> fun.nix - echo "sed -i 's;@$line@;$localPath;g' report.tex" >> fun.nix - done - echo "'''" >> fun.nix - - echo " ---------- this is the fun.nix -------------" - cat fun.nix - echo " --------------------------------------------" - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/default.nix b/garlic/default.nix deleted file mode 100644 index af0a194..0000000 --- a/garlic/default.nix +++ /dev/null @@ -1 +0,0 @@ -import ../default.nix diff --git a/garlic/develop/default.nix b/garlic/develop/default.nix deleted file mode 100644 index 6ea0930..0000000 --- a/garlic/develop/default.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, bash -, bashInteractive -, busybox -, extraInputs ? [] -}: - -let - inputrc = ./inputrc; -in -stdenv.mkDerivation { - name = "develop"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = extraInputs ++ [ busybox ]; - installPhase = '' - cat > $out < $@ - -%.html: %.ms Makefile - mkdir -p img - REFER=ref.i groff -ms -mwww $(HTML_OPT) $< > $@ - echo $(HTML_OPT) - sed -i '/<\/head>/i' $@ - sed -i 's/^<\/a>/\§<\/a>/g' $@ - #sed -i '/

/s/^ $@ - -killall -HUP mupdf - -%.utf8: %.ms - REFER=ref.i groff -ms -mwww $(PREPROC) $(TTYOPT) -Tutf8 $^ > $@ - -%.ascii: %.ms - REFER=ref.i groff -ms -mwww -c $(PREPROC) $(TTYOPT) -Tascii $^ > $@ - -doc.tar.gz: ug.pdf ug.html s.css - tar czf $@ $^ img s.css - -clean: - rm -rf img ug.pdf ug.html doc.tar.gz diff --git a/garlic/doc/blackbox.ms b/garlic/doc/blackbox.ms deleted file mode 100644 index 313a0f5..0000000 --- a/garlic/doc/blackbox.ms +++ /dev/null @@ -1,40 +0,0 @@ -.\" Use helvetica family -.fam H -.PS -moveht=0.1 -boxwid=1 -sht=boxht + 0.1 -hspace = boxwid + 0.2 -right -G: [ - boxwid=1 - extrawid=1.8 - right - A: box "nix"; arrow; - B1: box wid extrawid "App source code" "PM branch, defines..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B2: box wid extrawid "App run config" "Input size, algorithm..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B3: box wid extrawid "Build config" "MPI impl, O3, CC version..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B4: box wid extrawid "Run config" "Nodes, affinity"; - move to (A.sw.x, A.y - boxht - moveht) - A: box "MN4"; arrow; - B5: box wid extrawid "Hardware" "Cache size, intercomm..."; -] -movewid=1 -move -circlerad=0.4 -E: circle "Execution" -arrow -box "Result" -rspline = 0.5 -arrow from G.B1.e to E chop 0 chop circlerad -arrow from G.B2.e to E chop 0 chop circlerad -arrow from G.B3.e to E chop 0 chop circlerad -arrow from G.B4.e to E chop 0 chop circlerad -arrow from G.B5.e to E chop 0 chop circlerad -.PE diff --git a/garlic/doc/branch.ms b/garlic/doc/branch.ms deleted file mode 100644 index b4e6216..0000000 --- a/garlic/doc/branch.ms +++ /dev/null @@ -1,22 +0,0 @@ -.\".fam CMU -.\".TL -.\"Garlic: git branch name -.\".AU -.\"Rodrigo Arias Mallo -.\".AI -.\"Barcelona Supercomputing Center -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -\".NH 1 -\"Instructions -\".LP -\"To name a branch of a program, please follow the flowchart below. -.ps -2 -.PS 5/25.4 -copy "gitbranch.pic" -.PE diff --git a/garlic/doc/execution.ms b/garlic/doc/execution.ms deleted file mode 100644 index 81742d3..0000000 --- a/garlic/doc/execution.ms +++ /dev/null @@ -1,254 +0,0 @@ -.TL -Garlic: the execution pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the execution of experiments in the Garlic -benchmark, which are performed under strict conditions. The several -stages of the execution are documented so the experimenter can have a -global overview of how the benchmark runs under the hood. -The results of the experiment are stored in a known path to be used in -posterior processing steps. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -.NH 1 -Introduction -.LP -Every experiment in the Garlic -benchmark is controlled by a single -.I nix -file placed in the -.CW garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 1 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CW /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CW /usr -or -.CW /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 1 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.NH 2 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CW /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.NH 2 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 2 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 2 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.KF -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Target Safe Copies User Std -_ -trebuchet xeon no no yes yes -runexp login no no yes yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate comp no no no yes -control comp yes no no yes -srun comp yes no no yes -isolate comp no yes no yes -_ -exec comp yes yes no no -program comp yes yes no no -_ -.TE -.QS -.B "Table 1" : -The stages of a complete execution pipeline. The -.B target -column determines where the stage is running, -.B safe -states if the stage begins the execution inside the sandbox, -.B user -if it can be executed directly by the user, -.B copies -if there are several instances running in parallel and -.B std -if is part of the standard execution pipeline. -.QE -.KE diff --git a/garlic/doc/gitbranch.pic b/garlic/doc/gitbranch.pic deleted file mode 100644 index 939e926..0000000 --- a/garlic/doc/gitbranch.pic +++ /dev/null @@ -1,152 +0,0 @@ -#.ps -3 -#.fam CMU -#.PS 4.5/25.4 # Scale drawing to 20/25.4 in = -# = 20/25.4[in]/25.4[mm/in] = 20 mm -# FLOWCHART - Basic flow chart blocks. -scale=25.4 #Scale units from inches to mm -csize=2.0 #Cell size in mm -pstricks=0 -dx=0; dy=2; -define process -{[ - box $1; -]} -# decision(): rhomboid -> if block -define decision {[ - boxwid=boxwid*1.2 - boxht=boxht*1.2 - B: box invis $1; - line from B.n to B.e to B.s to B.w to B.n; -]} - -#--- END OF MACROS --- -boxwid=30 -fillval=1 -circlerad=10 -down -START: circle "Start" -arrow -D_MPIB: decision("MPI-based?") -arrow " Yes" ljust -D_TAMPI: decision("TAMPI?") -arrow " Yes" ljust -TAMPI: process("\fB+tampi\fP") -right -move to D_TAMPI.e -arrow "No" above -D_MPI: decision("MPI?") -down -move to D_MPI.s -arrow " Yes" ljust -MPI: process("\fB+mpi\fP") -move to TAMPI.s -A_TAMPI: arrow linewid -line from MPI.s to MPI.c - (0,boxht) \ - to A_TAMPI.c -circle at A_TAMPI.c rad 0.7 filled -move at A_TAMPI -D_ISEND: decision("MPI_Isend()?") -arrow " Yes" ljust -ISEND: process("\fB+isend\fP") -A_ISEND: arrow -right -move to D_ISEND.e -arrow "No" above -D_SEND: decision("MPI_Send()?") -down -move to D_SEND.s -arrow " Yes" ljust -SEND: process("\fB+send\fP") -right -move to D_SEND.e -arrow "No" above -D_RMA: decision("MPI_Get()?") -down -move to D_RMA.s -arrow " Yes" ljust -RMA: process("\fB+rma\fP") -line "No" above from D_MPIB.w to D_MPIB.w - (boxwid,0) -line to (D_MPIB.w.x-boxwid, A_ISEND.c.y) \ - to A_ISEND.c -line from SEND.s to SEND.c - (0,boxht) \ - to A_ISEND.c -line from RMA.s to RMA.c - (0,boxht) \ - to SEND.c - (0,boxht) -circle at A_ISEND.c rad 0.7 filled -move at A_ISEND -D_MT: decision("multithread?") -arrow " Yes" ljust -D_OMP: decision("OpenMP?") -arrow " Yes" ljust -OMP: process("\fB+omp\fP") -right -move to D_OMP.e -arrow "No" above -D_OSS: decision("OmpSs-2?") -down -move to D_OSS.s -arrow " Yes" ljust -OSS: process("\fB+oss\fP") -down -move to OMP.s -A_OMP: arrow -circle at A_OMP.c rad 0.7 filled -line from OSS.s to OSS.c - (0,boxht) \ - to A_OMP.c -move to A_OMP.s -D_FJ: decision("fork-join?") -arrow " Yes" ljust -FJ: process("\fB+fork\fP") -right -move to D_FJ.e -arrow "No" above -D_TASKFOR: decision("task for?") -arrow "No" above -down -move to D_TASKFOR.s -arrow " Yes" ljust -TASKFOR: process("\fB+taskfor\fP") -right -move to D_TASKFOR.e -arrow "No" above -D_TASK: decision("task model?") -down -move to D_TASK.s -arrow " Yes" ljust -TASK: process("\fB+task\fP") -move to FJ.s -A_FJ: arrow -circle at A_FJ.c rad 0.7 filled -line from TASKFOR.s to TASKFOR.c - (0,boxht) \ - to A_FJ.c -line from TASK.s to TASK.c - (0,boxht) \ - to TASKFOR.c - (0,boxht) -left -move to OMP.c - (boxwid,0) -SEQ: process("\fB+seq\fP") -line "No" above from D_MT.w to (SEQ.x, D_MT.w.y) -arrow to SEQ.n -line from SEQ.s to (SEQ.s.x, A_FJ.c.y) to A_FJ.c -down -move to A_FJ.s -D_SIMD: decision("SIMD opt.?") -move to D_SIMD.e -right -arrow "Yes" above -SIMD: process("\fB+simd\fP") -down -move to D_SIMD.s -arrow " No" ljust -END: circle "End" -circle radius circlerad*0.9 at END -arrow from SIMD.s to (SIMD.x, END.y) to END.e - -# Error lines -ERR: circle "Error" at (TASK.x+boxwid, END.y) -circle radius circlerad*0.9 at ERR -line "No" above from D_TASK.e to (ERR.n.x,D_TASK.e.y) -line "No" above from D_OSS.e to (ERR.n.x,D_OSS.e.y) -line "No" above from D_RMA.e to (ERR.n.x,D_RMA.e.y) -line "No" above from D_MPI.e to (ERR.n.x,D_MPI.e.y) -arrow to ERR.n -#.PE diff --git a/garlic/doc/pp.ms b/garlic/doc/pp.ms deleted file mode 100644 index 700057a..0000000 --- a/garlic/doc/pp.ms +++ /dev/null @@ -1,256 +0,0 @@ -.TL -Garlic: the postprocess pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the format used to store the results of the -execution of experiments and the postprocess steps used to generate a -set of figures from the results to present the data. The several stages -of the postprocess pipeline are documented to provide a general picture. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -.\".2C -.R1 -bracket-label " [" ] ", " -accumulate -move-punctuation -.R2 -.\"##################################################################### -.NH 1 -Introduction -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 1 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 1 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.ie t \{\ -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -.\} -.el \{\ -.nf - - Prog ---> EP ---> Result ---> PP ---> Plot - -.fi -.\} -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 1 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.SH 1 -Appendix A: Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. diff --git a/garlic/doc/ref.i b/garlic/doc/ref.i deleted file mode 100644 index 1bd3e70..0000000 --- a/garlic/doc/ref.i +++ /dev/null @@ -1,9 +0,0 @@ -%A Rodrigo Arias Mallo -%D 2020 -%T Garlic: the execution pipeline - -%A Winston Chang -%T R Graphics Cookbook: Practical Recipes for Visualizing Data -%D 2020 -%I O'Reilly Media -%O 2nd edition diff --git a/garlic/doc/s.css b/garlic/doc/s.css deleted file mode 100644 index 324f1ce..0000000 --- a/garlic/doc/s.css +++ /dev/null @@ -1,19 +0,0 @@ -html { - line-height: 1.6; - margin-bottom: 50px; - padding-bottom: 80px; -} - -body { - max-width: 700px; - text-align: justify; - margin:0 auto; -} - -pre { - overflow: auto; - display: block; - padding: 3px 3px; - line-height: 1.4; - background-color: #eeeeee; -} diff --git a/garlic/doc/slides/2.mm b/garlic/doc/slides/2.mm deleted file mode 100644 index 7cb81ce..0000000 --- a/garlic/doc/slides/2.mm +++ /dev/null @@ -1,1468 +0,0 @@ -.\"usage: NS title -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "November 24, 2020" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic update -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Changelog" -Important changes since the last meeting (2020-09-23) -.BL -.LI -Execution of experiments is now \fBisolated\fP: no $HOME or /usr at run time -.LI -Added a \fBpostprocess\fP pipeline -.LI -New \fBgarlic(1)\fP helper tool (manual included) -.LI -A plot has an experiment result as \fBdependency\fP -.LI -Experiments run on demand based on article \fBfigures\fP -.LI -Fast pkg overrides (MPI) -.LE 1 -.\"================================================================== -.NS "Overview" -Dependency graph of a complete experiment that produces a figure. Each box -is a derivation and arrows represent \fBbuild dependencies\fP. -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The \fBResult\fP is not covered by nix (yet). This is what it looks like -when executed: -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -.PE -.S P P -.DE -.P -Notice dependency order is not the same as execution order. -.\"================================================================== -.NS "Building the execution pipeline (EP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" fill -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Run nix-build with the experiment name: -.P -.VERBON -xeon07$ nix-build -A exp.nbody.baseline -\&... -/nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.VERBOFF -.P -Outputs the first stage (the trebuchet). All other stages -are built as dependencies, as they are required to build the trebuchet. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: circle at B + (0,-1.3) "trebu." -arrow -circle "runexp" -arrow -circle "isolate" -arrow -circle "exp." -arrow -circle "..." -arrow -circle "exec" -arrow -P: circle "program" -line from R.sw to T.nw dashed -line from R.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ -"Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.SP 1m -.P -The stages are launched sequentially. Let see what happens in each one. -.\"================================================================== -.NS "Execution pipeline" -.2C -List of stages required to run the program of the experiment: -.BL -.S -1 -.LI -The -.B target -column determines where the stage is running. -.LI -.B Safe -states if the stage begins the execution inside the isolated namespace -.LI -.B User -if it can be executed directly by the user -.LI -.B Copies -if there are several instances running in parallel and -.LI -.B Std -if is part of the standard execution pipeline. -.LE -.S P P -.P -Sorted by the \fBexecution order\fP. -.\" Go to the next column -.NCOL -.KF -.defcolor white rgb #FFFFFF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\m[white]\(rh\m[]\ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBtrebuchet\fP: connects via ssh to the target machine and executes the -next stage there. -.P -The target machine is set to MN4, which by default uses the host -\fBmn1\fP -.P -Literally: -.P -.VERBON -ssh mn1 /path/to/next/stage -.VERBOFF -.P -You need to define the ssh config to be able to connect to mn1. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBrunexp\fP: sets a few \fCGARLIC_*\fP environment variables used by the -benchmark and changes the current directory to the \fBout\fP directory. -.P -At build time, next stages don't know these values (cyclic dependency), -so they are populated at execution time. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes -\(rh \fBrunexp\fP login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: once on the target machine, we enter an isolated -namespace to load the nix store. -.P -Notice that this and the previous stages require the \fBsh\fP shell to be -available on the target machine -.P -They are not \fBsafe\fP as we run target machine code -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexperiment\fP: runs several units sequentially. -.P -Defines the \fCGARLIC_EXPERIMENT\fP environment variable. -.P -Creates a directory for the experiment and changes the current directory -there. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes -\(rh \fBexperiment\fP login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBunit\fP: creates an index entry for the unit and the experiment. -.P -Creates a directory for the unit and changes the current directory -there. -.P -Copies the unit configuration in the \fCgarlic_config.json\fP file -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes -\(rh \fBunit\fP login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsbatch\fP: allocates resources and executes the next stage in the -first node. -.P -The execve call is performed by a SLURM daemon, so is \fBout\fP of the -isolated environment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes -\(rh \fBsbatch\fP login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enters the isolated namespace again, with the nix store. -.P -Notice that we are now running in the compute node allocated by SLURM. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBcontrol\fP: runs the next stage several times -.P -Is controlled by the \fCloops\fP attribute, which specifies the number -of runs. -.P -Creates a directory with the number of the run and enters it. -.P -Generated results are placed in this directory. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes -\(rh \fBcontrol\fP comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsrun\fP: launches the tasks in the compute nodes and sets the -affinity. -.P -From here on, all stages are executed in parallel for each task. -.P -The srun program also forks from a SLURM daemon, exiting the -previous isolated namespace. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes -\(rh \fBsrun\fP comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enter the isolated namespace again. -.P -Now we are ready to execute the program of the experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexec\fP: sets the environment variables and argv of the program. -.P -Additional commands can be specified in the \fCpre\fP and \fCpost\fP -attributes. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ -\(rh \fBexec\fP comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBprogram\fP: the path to the program itself. -.P -This stage can be used to do some changes: -.BL -.LI -Set the mpi implementation of all dependencies. -.LI -Pass build options -.LI -Custom packages (nanos6 with jemalloc) -.LE -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no -\(rh \fBprogram\fP comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -The \fCstdexp.nix\fP file defines standard pipeline. The last two stages -are usually added to complete the pipeline: -.P -.VERBON -pipeline = stdPipeline ++ - [ exec program ]; -.VERBOFF -.P -Any stage can be modified to fit a custom experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes \fByes\fP -\(rh \fBrunexp\fP login no no yes \fByes\fP -\(rh \fBisolate\fP login no no no \fByes\fP -\(rh \fBexperiment\fP login yes no no \fByes\fP -\(rh \fBunit\fP login yes no no \fByes\fP -\(rh \fBsbatch\fP login yes no no \fByes\fP - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no \fByes\fP -\(rh \fBcontrol\fP comp yes no no \fByes\fP -\(rh \fBsrun\fP comp yes no no \fByes\fP -\(rh \fBisolate\fP comp no yes no \fByes\fP - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Isolated execution" -.2C -The filesystem is \fBnow\fP isolated to prevent irreproducible -scenarios. -.P -The nix store is mounted at /nix and only some other paths are -available like: -.BL -.S -1 1m -.LI -/var/run/munge (required for SLURM) -.LI -/dev, /sys, /proc for MPI comm -.LI -/etc for hosts (FIXME) -.LI -/gpfs/projects/bsc15 to store data -.LE -.S P P -.P -Additional mounts can be requested by using the \fCextraMounts\fP -attribute. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -We cannot access MN4 from nix, as it doesn't has the SSH keys nor -network access when building derivations. -.P -The garlic(1) tool is used to run experiments and fetch the results. See -the manual for details. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To launch the EP use \fBgarlic -R\fP and provide the trebuchet path: -.P -.VERBON -.S -2 -xeon07$ garlic -Rv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -Running experiment 1qcc...9w5-experiment -sbatch: error: spank: x11.so: Plugin file not found -Submitted batch job 12719522 -\&... -xeon07$ -.S P P -.VERBOFF -.P -Once the jobs are submited, you can leave the session: it will run -in MN4 automatically at some point. - -.\"================================================================== -.NS "Execution complete" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -When the EP is complete, the generated results are stored in MN4. -.P -As stated previously, nix cannot access MN4 (yet), so we need to manually -fetch the results. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" fill -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To fetch the results, use \fBgarlic -F\fP: -.P -.VERBON -.S -3.5 -xeon07$ garlic -Fv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -/mnt/garlic/bsc15557/out/1qc...9w5-experiment: checking units -3qnm6drx5y95kxrr43gnwqz8v4x641c7-unit: running 7 of 10 -awd3jzbcw0cwwvjrcrxzjvii3mgj663d-unit: completed -bqnnrwcbcixag0dfflk1zz34zidk97nf-unit: no status -\&... -/mn...w5-experiment: \f[CB]execution complete, fetching results\fP -these derivations will be built: - /nix/store/mqdr...q4z-resultTree.drv -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -Notice that if the experiments are still running, it waits for the -completion of all units first. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -.VERBON -.S -3.5 -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -When the fetch operation success, the \fBresultTree\fP derivation is -built, with the \fBlogs\fP of the execution. -.P -All other generated data is \fBignored by now\fP, as we don't want to -store large files in the nix store of xeon07. -.\"================================================================== -.NS "Running and fetching" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" fill -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -You can run an experiment and fetch the results with \fBgarlic -RF\fP in -one go: -.P -.VERBON -.S -2 -xeon07$ garlic -RF /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.S P P -.VERBOFF -.P -Remember that you can interrupt the fetching while is waiting, and come -later if the experiment takes too long. -.P -If nix tries to build \fBResultTree\fP and doesn't find the experiment -results, it will tell you to run this command to run and fetch the -experiment. Example: building the figure before running the experiment: -.P -.VERBON -.S -2 -xeon07$ nix-build -A fig.nbody.baseline -.S P P -.VERBOFF -.\"================================================================== -.NS "Postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" fill -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Once the \fBresultTree\fP derivation is built, multiple figures can be created -without re-running the experiment. -.P -The postprocess pipeline is formed of several stages as well, but is -considered \fBexperimental\fP; there is no standard yet. -.P -It only needs to be built, as nix can perform all tasks to create the -figures (no manual intervention) -.\"================================================================== -.NS "Building the postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" fill -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: box at R + (-0.02,-0.8) "timetable" -arrow -box "merge" -arrow -P: box "rPlot" -line from PP.sw to T.n dashed -line from PP.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ - "Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -To build the figure, only three stages are required: timetable, merge -and rPlot. -.\"================================================================== -.NS "PP stages: timetable" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" fill -arrow -box "merge" -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The timetable transforms the logs of the execution into a NDJSON file, -which contains all the unit configuration and the execution time in one -line in JSON: -.P -.VERBON -.S -2 -{ "unit":"...", "experiment":"...", "run":1, "config":{...}, "time":1.2345 } -{ "unit":"...", "experiment":"...", "run":2, "config":{...}, "time":1.2333 } -{ "unit":"...", "experiment":"...", "run":3, "config":{...}, "time":1.2323 } -.S P P -.VERBOFF -.P -This format allows R (and possibly other programs) to load \fBall\fP -information regarding the experiment configuration into a table. -.P -It requires the execution logs to contain a line with the time: -.P -.VERBON -.S -2 -time 1.2345 -.S P P -.VERBOFF -.\"================================================================== -.NS "PP stages: merge" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" fill -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The merge stage allows multiple results of several experiments to be -merged in one dataset. -.P -In this way, multiple results can be presented in one figure. -.P -It simple concatenates all the NDJSON files together. -.P -This stage can be build directly with: -.P -.VERBON -$ nix-build ds.nbody.baseline -.VERBOFF -.P -So you can inspect the dataset and play with it before generating the -plots (is automatically built by nix as a dependency). -.\"================================================================== -.NS "PP stages: rPlot" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" -arrow -P: box "rPlot" fill -.PE -.S P P -.DE -.P -Finally, the rPlot stage runs a R script that loads the NDJSON dataset -and generates some plots. -.\"================================================================== -.NS "Building the figures" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" -arrow -F: box "Figure" fill -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -The complete PP and the figures can be build by using: -.P -.VERBON -xeon07$ nix-build -A fig.nbody.baseline -.VERBOFF -.P -A interactive R shell can be used to play with the presentation of the -plots: -.P -.VERBON -xeon07$ nix-shell garlic/fig/dev/shell.nix -$ cp /nix/store/...-merge.json input.json -$ R -> source("garlic/fig/nbody/baseline.R") -.VERBOFF -.P -More about this later. -.\"================================================================== -.NS "Figure dependencies" -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" fill -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The figure contains as dependencies all the EP, results and PP. -.P -Any change in any of the stages (or dependencies) will lead to a new -figure, \fBautomatically\fP. -.P -Figures contain the hash of the dataset in the title, so they can -be tracked. -.\"================================================================== -.NS "Article with figures" -.P -An example LaTeX document uses the name of the figures in nix: -.P -.VERBON - \\includegraphics[]{@fig.nbody.small@/scatter.png} -.VERBOFF -.P -Then, nix will extract all figure references, build them (re-running the -experiment if required) and build the report: \fC$ nix-build -garlic.report\fP -.P -We also have \fBreportTar\fP that puts the figures, LaTeX sources and -a Makefile required to build the report into a self-contained tar.gz. -.P -It can be compiled with \fBmake\fP (no nix required) so it can be sent -to a journal for further changes in the LaTeX source. -.\"================================================================== -.NS "Other changes" -.DL -.LI -We can provide the complete benchmark and BSC packages as a simple -overlay. This allows others to load their own changes on top or below our -benchmark. -.LI -We now avoid reevaluation of nixpkgs when setting the MPI -implementation (allows faster evaluations: 2 s/unit \(-> 2 s total). -.LI -Dependencies between experiments results are posible (experimental): -allows generation of a dataset + computation with dependencies. -.LE -.\"================================================================== -.NS "Questions?" -.defcolor gray rgb #bbbbbb -\m[gray] -.P -Example questions: -.DL -.LI -What software was used to build this presentation? -.LI -I used groff. -.LI -And the diagrams? -.LI -Same :-D -.LI -How long takes to build? -.LI -0,39s user 0,02s system 129% cpu 0,316 total -.LE -\m[] diff --git a/garlic/doc/slides/3.mm b/garlic/doc/slides/3.mm deleted file mode 100644 index 39b07d7..0000000 --- a/garlic/doc/slides/3.mm +++ /dev/null @@ -1,181 +0,0 @@ -.\"usage: NS title -.EQ -delim $$ -.EN -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "January 14, 2021" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic experiments -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Approach 1" -This was the approach proposed for hybrids PM -.BL -.LI -Perform a granularity experiment with a \fIreasonable\fP problem size. -.LI -Take the best blocksize -.LI -Analyze strong and weak scaling with that blocksize. -.LI -Plot speedup and efficiency comparing multiple PM. -.LE 1 -The main problem is that it may lead to \fBbogus comparisons\fP. -Additionally, there is no guarantee that the best blocksize is the one -that performs better with more resources. -.\"================================================================== -.NS "Approach 2" -We want to measure scalability of the application \fBonly\fP, not mixed -with runtime overhead or lack of parallelism. -.P -We define \fBsaturation\fP as the state of an execution that allows a -program to potentially use all the resources (the name comes from the -transistor state, when current flows freely). -.P -Design a new experiment which tests multiple blocksizes and multiple -input sizes to find these states: \fBthe saturation experiment\fP. -.P -Begin with small problems and increase the size, so you get to the -answer quickly. -.\"================================================================== -.NS "Saturation experiment" -.2C -\X'pdf: pdfpic sat.png.tk.pdf -R 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -The objetive is to find the minimum input size that allows us to get -meaningful scalability results. -.LI -More precisely, a unit is in \fBsaturation state\fP if the median time -is below the \fBsaturation time limit\fP, currently set to 110% the minimum -median time (red dashed lines). -.LI -An input size is in \fBsaturation zone\fP if it allows at least K=3 -consecutive points in the saturation state. -.LI -With less than 512 particles/CPU (green line) we cannot be sure that the -performance is not impacted by the runtime overhead or lack of -parallelism. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space" -.2C -\X'pdf: pdfpic scaling-region.svg.tk.pdf -L 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -\fBSaturation limit\fP: small tasks cannot be solved without overhead -from the runtime, no matter the blocksize. -.LI -Different limits for OmpSs-2 and OpenMP. -.LI -Experiment A will show the scaling of the app while in the saturation -zone. -.LI -Experiment B will show that OpenMP scales bad in the last 2 points. -.LI -Experiment C will show that at some point both OpenMP and OmpSs-2 scale -bad. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space: experiment C" -.2C -\X'pdf: pdfpic scalability.svg.tk.pdf -L 7c' -.NCOL -.BL 1m -.LI -The experiment C will show a difference in performance when approached -to the saturation limit. -.LI -We could say that OmpSs-2 introduces less overhead, therefore allows -better scalability. -.LE -.1C -.\"================================================================== -.NS "Reproducibility" -How easy can we get the same results? Three properties R0 < R1 < R2 (no common nomenclature yet!): -.BL 1m -.LI -R0: \fBSame\fP humans on the \fBsame\fP machine obtain the same result -.LI -R1: \fBDifferent\fP humans on the \fBsame\fP machine obtain the same result -.LI -R2: \fBDifferent\fP humans on a \fBdifferent\fP machine obtain same result -.LE -.P -Garlic provides 2 types of properties: for software and for experimental -results: -.BL 1m -.LI -Software is R2: you can get the exact same software by any one, in any -machine -.LI -Experimental results are R1: you cannot change the machine MN4 (yet) -.LE -.P -Same experimental result means that the mean of your results is in the confidence -interval of our results \fBand the relative std is < 1%\fP. diff --git a/garlic/doc/slides/3.pdf b/garlic/doc/slides/3.pdf deleted file mode 100644 index 0bbb7dcf0c3e7bda53062758fcf21c0e2a67f93e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 122774 zcmY!laBAO(e3E`9f+ z{L%u5Y_Nj9OKMqWa%zyf6PJFlf__L*VqS4UVo_>dawV6Y9hZIph+U#!1k%l=A5vM6 zs-Pc`n4SufOwCKlPs-u~nGCbfEiBmH#Vxma;rF&T(Ak1@;Q6s z@8%_a44)kzt4U{BUSs9tbUZ3}Pw|46o6WTcXLG}+NA^EvSQeVji?LJ}-}3#|I%|2REALKdcZDNoB>`QtrQSXm3l+OM!+QG}sMIvBtBZ87Qh~6479ziHT`*6C;$EUKbdqv^j3EEIW5zo4Fjj|dsk{-#pc{4ylB;H5!Hzw%lpoM zpW9u@#B`(PO@34rOWv992M>1~-rW9h_ji_fp?s%8DFvbSf>uFJ7L}!qIeX9F*I;f^ zyg4IvQs3(Z@&0WlPCe}oj~Nns)-H5R+FjLF{eM^Ht*B+z3ajT&xN~OiJS$EXu4!{s zcE=pvT$*s9>3f@}S-S`CUBAXTea7{^YeU(jv)wctYYwi{?MT7Q)BuIIb2@a9IDM>*`>t4jq%Zu%AU64(}VLO{#Bovtoc-g^Lxb4jHL!4Vj^MAXV*`Cmyr9DMI>>$ zf8g?Ktgenv-q?K8;)!NT+^BHKb(_jF`Go4&laD?a1*I{ycAqJ^e(kDQ%2eOE^0Ss4 z;STu8VKvn??}Bq_sdeQQvAdGs7|%I)Kn-TQp&z7Q0umNvsl(=^|PT< z9nV@ zd1_vPGHR(mbat*!ta%l^GPL$V=-xHF!q0Dh*%6x|Zwu$z*CE-_E7tUitiAMgpYK$;rw@1vHwOG)o-b(P zCYbI%Hz#k!WqB1R#-|QDnz#6!Ty**8>dwY}8~>U%TE9Q{oBKYaB0FaOG{>GlEohfN zK}HjkKP?O`QS#^1FyH*!20VLzhkuyCv{vn8naulbiSh^XSef2jc)P$VYth+hvEK7$ z|809`-#_im6+clP=ZQ>0nv<$(`adZfEco&Kzj)}82i*VaT&Ad8a2K(2@BdII?HkAG z&w2WKfitJK@r|$-oBHq7JTf%kyUiYx*IhW5@AXTC`x7i8na>A1X?|tDIQds7fBEWi z-^ts7QdD_GUv_w^@JR_rbzi@OtiZBz@FEjB)r zXL7ohch|OnmFn_lot(bR<~JPIzhS(ky(3ZXaOm>2t5mN1OV~Cq_HOFmr<>GNyv1*C zvh~e9qpi|6Za# zf9swLy}PFhZp)56EMPP*_s+@UG~d7G#`8tLRqrziyK&U*uR(lg$kF9(rqW;A_@`s^I9(F9$>1 zU-ZaaExabYYo80d&GVFtUcu$Lio7+_3Cy|^Cb-5vJmwr7e(rc*&x0uwOBYOZl1|kT zI>Y?7=JK2)f406h{ldO-;fq^htK04MulO^!ylGcjqSJqWoyUi!oXK&!9Z9wKR0I9IoQ>uSwES+UJ4A6j&2^PQWz zt9frp{_N@_TO?VVO%`?A>^(hMP%L0>&clttmp>ezcx0N5_TSIPBp%izMlxB|TJs1_ z`oZ$z&EtnpPMWDa8VZQyh%>?$o4nMQBfwg3(ifgTGm4|(!h^usnD|botgHvB-MRyjd zXuW$jE4!}5eaV!mx+`6q-Xy)9YAa*6uQK0aeIw)1`2SJY7@Md6_%0Ek^gt%yhhhb9 zy!b&8)e4`8c?Wn^b^I2tFnpr8^`AOd^p(lFx*<7;@o` zz1Hgc6>f%}T1T2nS$8qtK2W`xy*oQAx8!Yj${dc3Yl7pO&6eM5n|`Ic=Sk(Wd*a7$ zl>S<)ZM#^-dy%NXHx1nrtFmgYX(&fd(|>eM_6y(J=1l?GnOiC?zFcLSq_NZG)12kn zsjBZwm%bI=vg)<=?VmE9UYks2pUiWYdG}vKI@AXq3B~xo}C!Jq0Z-MjA1zi%0 z<9QmceLWoT-&?i1o`<2*rD&Il$gSjar|zx`t-f*kT}S>@vsJsfPx3rlwIV{vZuyEh ziT4o_IWJ~Ax-8O&%x9Bdy>xqT;O6JqDU;)#d4E|{=s)Lp_MF}O9+(Iw{*K7yzG17S zYRzq5C+c`}hTqh`?bnV695;!5UAx>(`gPwPUFMCKrrV0j1^X3E%e#|yFa4ievdXEO z$G3Lmu1&JS^d_thn{4M^Pb~i=hsuSiJG=l zlJCrM!9~-Sl}wU1yZY60nwW2ueW%(nL6zu~iXz<{yUs6Z1;4CL=tMHUJgSoxRVKe* z(*D`YO@*tzt#~3gOH3=WPwtsTwvOvxa})OP2!GR6laGInz2LLzQmn6Dmc`i`iTRx` zvWgxRmv%io^q8r(Q1Ins_tl=yn^oHWoxh&PRxSF6JE^2o6r(gT!rrtuqFvJ-Y&0>Y ziJ65dN@=n;%(r;9fxzC++TSwHU+LQCuwwf%PwUAu{bq5tpDC#dFzEAS5L|p^Zp`oF z+TITu4(1kh+E1*k+Wq@W*e7Ly1K(cSUp7)GNWA}@(M@I>W835|jCrpfwK7#B`h^TxpQcHs``a^cJ#iT;LD0*O6;UAm9dZE2O?{P}+AwwZ^U zDvs#rZ$9M^yxL^;|CT37E}~o_-&WKF*ob~%sGN6TsxJ593CH+eu8%6wkhRc#_~h9n zE{n^2cZCx?Xed z*@QdnWgA-r%G@@+X+JS-!`ZV@_L3LoUB6^-GHSu&4=gf)f_=~Pug#1)l(pE~;5E;$ zeHE*Ewzj2D6+GngEzenyK=hF#{Qyh7Q8L!#+FvrNBdWLcXX6ZU$)>rCXb z#;bK}HRPSYvgw`vwJS0p;Hr?`>ovc=?Jeb9))D+?nbOB>>%=DyH2?d*mz%$ackSib zrLUblCyJDW+`P87cJeBv_3s?kmmPjtrchXO#$(mG#NFq*c|M)`{5I;+LP}AM&Fs-5Su# z$hGyH_GO`?LY|+Ni7Ee)W%*U3A>eEyl|OB*!?&w1j6S!@3Uo(=^xmMdH*Jtu?yS=iWUF_HV+wUy)as6YmeD~`l zMm96XK2l-~8qlE4Pzl&*VzQa3F>0kfHRNF5Yy*)!pGDtGEIF1fEnU!`yM1a^_m6PUfngq{0r;PPj>6927LR_JhaE|E>1oD zcI;&<=X+`e{doaOm3Nfy-foz-iha529h*C>LUtjxRgdnf3Ph?$&tG~(+>6D1_v6+n z3*X+iW4Uzm!gedw8dLUdfj#q7s*QI^u(Y`S&r;e_8R)QieN66}4vy(tbzjW9?Id_r z^EjW_Df7sEw|`EsNnFs&P>-j*LxwF16!PZ&oUw)hIp`+igmd}s$ zxH2noLfCR8=d*ro6IzaY-{3Y|rh8}Vas9K?7VqD-Zc}j9tkT^xlCHne^0|D#K={SJ zs=3SU?$58?(d1I3dwcK2o3qb7y47>jXi?a*RjYVkERxv9X~7vDl`maka&6WxFaC>N zZ)dArZLK@geP^G=N}2y>yTv+YpTD{DxF}zufwj#Kle~jTGNOz=F$%IPV^rtLE#vrm z`uRVJUElvP%;57=#z@g7pc(>}zL3WLKwVMVq-Yb6(J&kFjs95}qmBtZ4fD;u?Eq?q z`tY-IzIyvE>#HbJ>m?J_3r;e3C&`Ia%4$ahDXo9}>rQrkiF?$^iEp;fHJk3l85r@Y z%$pYk`MBD5dY>ljIK(bj*%Gu$UrESbd&BWh3$@fwo9sUOY-W_T!2RuJD^9kF z3*;X^Hht61V-_FpO3C>fv>&_5weGFxiIk0PCYlLPYubMt`lKZEF?KMn;0O_GsA5m;*BoY4$|^eOQnzuDT?3-t^Y8EwXzb9DXss zRy%`M0uL#M{n=?7R?~t9Vo@(6Nmd&wU>)y`i+cB@~hQ#~A zHG+#3jx0GQa<;!hfNw%c!gFf!U=?)`}O8$LhvC(el3(8Fr#^ zee~26_CLAN+e&xNoyV=bd&<7H*s>|Hzk|)@nu#7uEqqZgBoT z=d;O+9es8wNR)E&|I_Lc7k!!_x~-+V@}b>BEB`Ftl&D*8E2mv^TKUy`M@40L+WbwN zTkjOHdS%TTb%R{fEppcX*3I11c7LYa^^JXow@mNUBfJ4kw5EHd<-77|IP}U!gVP!=+4Py zM!7;CzRo_dcG2d5$?tP@S=Et{3?nP+po6cyT{{QB2wv(q6|$ z#-DaDsIhtEUNM(?tN)I}_S5^#5_)UBUz?{aeNcpv^9_;up8<*Ksl|{Xdj&4&n7wmi zNn%cZI;iai9X~cf9xZY%N=+=u%+GU4EJ;<+aIrEnFfueSG&D9cFt9W+)HX0wH!x7w zkeGcwXM(d5$i%}*g@L_kq~N@;Q`#EgOzOhfGK zxS&o%ZU#Fg7N@%9=anevd!*)+rIutSC+a%o=cI7yyXGb5r)1`(D}V~WAO$Xcx6GpA z66cJ>A_W63eUKiP)Z*l#%z~2qA_Y)!1ybXa2vuTaYRaV_T$)q@p4tFSa2RswhvbLm zWr8Lz6hMIlQV)_)02{`oAD)>~l2NP>ZE9hn0K#Tw<_czph6<*pW(uYT1_~g)p@D&c zf|0QWSP(P{KrA*gw@@%PHdZh(Hv#Jf8ERo+sbFqoqyUox$s1c(f-Q%MfoKp0sV4@5 z%`i3wn*s6!2!lLfX>JO(3uGS*gFFP1GdH&Y%fajc$%Di#jE%wKAbF5l5Dk`t1`LdC zX<`Bn9FQ82K9CzgW`SrB2Dux=2hkvTWEvz75(8n78W0A_gJ@9rfW(n8IDAaaz+njz z2dM?Q3q*slfq?=@jgbX}2AOXF4q>n?NT;a*I0g*O%oV`V36lWPU;zVgw1Z3s#V*Jt z24MR@W?{pi!~w#fZ~&=H@6Eik51v}Bg*hB$_LFzzJgbjn82a*8C87MxX7-T3i2B`sKP#QtQ zV26UljEo@Z1WLorgk>I3213K2xCMCv9V6QfVuL(^3xn*(CI>PTBo3luk%~r8K>};j zxMZfKrGn-|Q;QX%4HWbfi<2`mlQWBwOLL)|lFXcxR0aLunUa*Fpr2Qon*^#h((@GbQ}T0i5{ne{3sQ@cQ$b~WVs1fdQE_5k z3RFi?W_m`6f&PgoJP|&YR1%+;YUaEqANqN44eo01AYN~>MT7GGff__?NS*n75ab|^resO9U zNOvm4>by*lhUEMlP}~-$=4L{fIjP0P3i_#qrHMHT`sqcfiQswf!qU`YP(7-kpID-x z@2H^fq@eGtpzorf@2a5hrl9Yxpzooe@2Q~grJ(Pvpzoug@2jBir=ahzpdX;1AE=-o zqyV0VSI`et&<|744_DBSP|%N5(2r8kPbx}G&Q2|X1Vd6{ayHaoP;qFOK$2!!PHKgM zeraAxYEf}=eo-nkI3aoy74(x7^ph3zQxx<;0id9tuArZxpr5IrpQWImt)QQypr5Ot zpQoUoub^L`pkJtYCCE9cP{%;C z20TS0QhQQjajJrTdTC}(PHJvG)KhRSRC`isPEKlxf__>~eoQ1e9h=5|bgRHK`~SBoEEtMW8l_etvp>UMehOrzGV-LI&*dl%yP}$MRDZ z^mCxm2a|$?h`+0Xz7JFe?r2CRO35$JQ_zRl3`+PpVEd9wi$JAQrGkDXDAgtvH zGeMT7rYPv=CMJVRx|Gy(a1H`xKB%&i^8Dh`g4CkS{32)uLvRz5OG{D}^m9wW`8hGY zD6t>|mPtXrN-Y8zo}8MJnUj;Kpr45kp z)DTx>LtGJ>J)!D6q3S%5)p;V+xt3%o==(t}_lIiohidXi*5nUUgwUK@np2WlkW;Ck z?+;Brp->Y-p(cbPn-Gf783~mS$;dCtQ_xROEy_*IOG(NpR?r7U5-5cyLX{*U8<7Zg zL?XhdL`X0trYh(s!-FXm>W);XA*slQq#`tDLe*tL)j?}nNKuxFP?rh{raY+S`A|*y zP)+&BhUdc+AvC9CmSuu!>wIV+l|uEGLiLv->n}y_KBP6F4{1#3 z!`l*wrUY_J0^E?$2iL&R3K$}rl9Yo~{X%PBSmmn^uX`bi!FdZ&^Fmk?Uevp2!eu#dk zewco^euRFceiU*wst>P4VWJTKA<_U+6$%LzPywl*sGp>tte>Kvs-LExuAiZwsh_2v zt)HWxtDmQzuV0{Fs9&UCtY4yEs$ZsGu3w>Fsb2-Q09;M#CnXj^co54Vu??;Tz(p{q z9zd-a5j=4UO2w1$HO2eg8Nb07}PPleWQaB;Aw{9U0{8#GaZiehkmrVpxF zK-HN(tTqExX8Q2DOdnd6>BI7)KD;8+&qvf_;BW$mAvl&0wHTxl(?`@{&?-zHQiFk$ zF}MN)B~)0Y14*!O9ytH&=az!<0jO9k$N&X+W_}7Nhl8pveQ1FPO7L)9$n_S&dC*XT z*IFPgph^p3C9KYZr*v>QL24|0NQI@3RA1>ssw;hXZKV&ctn?jS;Z>DBq^8n`R8;y% z^^`umn$q`#qyk7KrH@oc>BFliebob>(Skp-)opf!^| zq+-&C)JytE)sjBER?>%5O8VeB38_kg)=2u03P~TSKGKI&NBZ#ENFQ7o=_jVbt0H|! zO{5R0i1d-_A$@o?qz|cu^dXgyK2jZ|53hpsQ%f@RAr+85r2f%|R6qJiwU0ie^3l(S z2Lq()(TCJL`jCo8A5!n=BUL;4@LES7Qt9Y}>m2<`xIw8oIhh5;nZ=r1`k;|5P{&Ev z$jE{VwCXm!Si#r`GV0~zlwServV!Cw0~nTOMxX`2pdm58#N1TOp)}8u#GK4zM{sY{ zfJ;BPBsDh-G>C>6{Q_AC8A?N{@<4-RuqqGV9DtE2>Ks69fOqy`1D<`q!(ZHx>$)v- zitV6*5u4?kgPZ0a$y@Ro z?v`H;ymMADrod(TgA*+by=_u^Pkm5xXNk*b4-?M2T%>R=?BiCJmQ_cVzcx6PJ*O!| zVyX0mtL{RKU%5LIUVM>_i;dhHd0M_oOFfz8&jj1~o2P{o%A1GhzLTA_RdD~#xmQ$c zji>54|EZt;{A7Cc^w8p-E$KURBMsK4$*z0OZXfOtaD0Bb}qz)`*YNh~E1HzyILl6xb zaD~aiXs~&pL1%NwpfgA<$Uh(&hK-F)6pYM3BZwy8fHeS{3^##@0Z*7opy&m;5M-7y z*esC!xUi*(3D_o(Ja|MAWFGt zg$;w92MRZINsw=mF-Q#7Zgj*GG!PFCla!3EP)j}qoZ{s4bs6CkB-)ju20klue}%@ ztplx&%P&bCU7t9*J`ofGNNYb75Q{$)K&wAS*C#4Kmxd@HuMJT^EDlistqxIuE)P+F zt`AXwE)Y>bULm3YT_Qs7(K=8k1J-&N9jzN3ts7mS2%U@4A6=lBnFC%wIl4eGu`G3T zfg&gnsJB27K3a!mfg-`tIxcMfu5~_59KL0WNL|`XQ+mB?|hU zxryniPEgtzN_&C^BqRKjvQm>vz*;?XL0g3t3>1PuYbrsfuEO^tgPZ}|YXH>(QUg5+ zv|^6-lT8Mu7mT-=1R5|nrIZ!vP2ZQkuCdA6E?hyJVRj$;PmG0zMzAqUOouuv#KLx7 z8zB!dIAxX;2c#A`=jRsW=Ydw6T5#z*=jY@X1%vna8G#xULGW;l1}#H{9sUaPDs6r_PPS_4?aFC&U8M5UX6luu5zx8G>`wZ!;d-vyiaoA325jdsV zJKITtb3(x6JKQZQAK0Ckrq9pJOOW1>%)u6%>tmjv;Lz|OuQ@m8PPO@iQ@2i6Rp;MY zwe3~yvp?rE(zZr^E6?=JU_X>M$4YElfG zHEY(FJZ_LHc=}!~n$xYn&qAoPL#(+wsPOyU^7$2yI{9tCOt{A@XvU_;#lXPOz^MFe z(VXk?^|4zdVy;Yh>wj~`q)C&C7j4QRN zinr&7_A)XsFdR^DDGkh1R-a#ERA2OV>-A+ZJbJU9zRl0iKR@r7%7c#;haMK}Uw0^h zmzjZqfsKDvx83(U#qu>D8gB*Lr=OeinEQFnwO-vtILiC%l97Bzf54D;~CX>+O2+<8iGkEMudl7lUAJnL*T0tK^Xq2qFFmEX{6l|`cc0y_jJQvK-`CrpE$Hg% zdbjs`+`Tqof18Qhzn#)vU-S3t^;*05Pm?z3oS$L%_l*>1$PRM&#nD-b9!vq z&C~k(bGAk$pXQObtMMqWhg(&9#aW-X{dT2@ z&CmG1ufjumHTCqwV)bX8yMFxoqOj+>&vXAh)vsT<@5$K*_LF>Pn=PIEXGh`Vwl5s{ zPY#%@S+XSM#)ib_bIb3U@BewW_?%_C{?C`o=O0P{g|}9c-YM3$Zq@p_TbavM=cl|r z=(^lz=A<=TcfNo6^y!g(BliLOe;@m&#C^Qr%>VQ6zaOpAQc`ulZ{JtbuRJbWey8g7 zT0`-tlYXwvukGGn^LFd?N#<{l+y67HfBo=oyA_!}H`Qq7m7{8Yb54Yxy*g`p@y@5`>pxANzaszZIoDXd>+9?PW`C_bFa7PJyL{nH|F}v2-{hJ_tNlL@N?{Xb z&FKF0JT*1-{-0;&KTrSv{{LV6p(G`7+m}lw|Jc8Lex23!gR#5IbmRBcTv-wL>f^5~ z!T!DW*1tX+=3o8vdVKxgM@PH4xw(Jt|Fbj7qVCU+8Zmo5dZ>+ZMzwO_#fx%{~7E5IcC1^#B<;ao&BjxA;kJ)c?LHn6{nz1H-*Xu8<`NMvo-tSMh zw!Z%M2%SIGf9`$Xw|v{$wQJWt|D)>JrE<{z3A_Cd#{Vs#z}Q%P_wV%$JBy#MI$gZ| z`RNb-_ckORo?)1L<^F-x>9NP8XI}en2nxZD&ho%>pU>N`m#e?NCQ^9+j1#Vwe=O!* zaFw&}D3%TKe)Z~ANc)pzQuQp)&%5`@tdvrVchj9?nA{ds`|HcgkMhy6v9;$cpRdS2 z`X%#dm*^kW>&?%eJ^RtB?l7SoA>3mn3c3iHyC;a(4DX!K>r`>xb4xX<6aajJC zSB&2DTT7l4Z_aYr!+X+m(hiOE<9)IpxhI>b-E=kn^(*Pr^gDZWqqnVix`^Ajs$u=> zHtD;jB zT(qZY{ITc18&6M9UwypxE0gQj z8@1o>mM_mad|{#URlBf%ciw+roju<)JM7b>J2s)|9oLuN&fh5Mdyyg#o`;*r5 zHlNSL?RfF(>gucWK2DFXv)s4mztpm0rI&rpm;OoF`nzFwo*TrqDU)KSEWXEcXl}vz zdA3=dCtT0l{eCk=J~}E&>b~SJP-$AJYZ&tJHz@x+a>@Aq?yjbIn7(T7_2oyq#rZq^ zlBFiQeLXjwTW`mL>l;f-N>)r)cz#y>vukzb=l9_ckR7;y+Y#26lt^theUd9{J)khJSu;%e{T2YvbuZLO(9g|2JiU z|DLTe8!HxCy$Dtj`uVPW|LOQ5;n&_qHNx{M9&vtMbWj3RxPEwe_-tD7pGmB&tTr=$ z%zpRt^XK#S@q3$>{f{w8T*9>Kn9A;qU-9zY!TVx&m#ON{S=n7>BlmTYU|z$?hhO3k zC)Ia;={cI@S?ml-{1YZk(L431?cE%!QZ3=#8He^77;0&2Kdybb=kvMl+7P2XbM~Kp zYrb#Mi#;o<4rw{g&)fM_Ebhnk{r{%g-``cbS}Wn}*RLvqPoF>E|M8gg_Ip*YHJ8s> zw5_+>>rG2UX6DMOYwucw{UnUj&K#GouX%An@$22G+Tq{sy8d+UjQIj85+Ql`^8a5F zTf6TS9+$28`MBSH--ZnaKkxp_zVW{PfAznjOG`W@{{(=YYpEpu^iJjTxvw%+^DLCa zZ9kn*4lavcAGbHk_TB2kRx7s~-K~DVSMzqkdE4(iFXw#H6yaK#Y`<(z+?2&-{c0~R z|M~g(>f~vA4FBl<`%}LEw`^tDp4|NXf4A+5eZTMbx+sgB8wUR-ZPzL*F6a3w_up<; zhNGk7FFh|%p7j0yI(dnnRqd~muZwqtERR|6UuxOCw)r+YBtb6AczMwEY})6S%jciF zzxUHA?f>7l@8=b~d1TA!G|_v^I@FFx$le82er>G*#|k2=-+B#obKfaJk>FAus#KL6>qb#lm)Md|Zv z%W`vbYu2)Ii>=5%+V{rP)b!QT+CLv;>wkBucrJGD4?DjvEIfR5>*r@@gI5>xcK?rf zcT7B{z|l)QHPBWqvXOTJHPJYhppm#o~9Z8)z#I99@xKHX}RxPi|XrFug+P$ z)|vRPSHe(f--PFj^RCzb`#k^0r)}HJzMgM1iJh{<@TA_Xi`U&BEmE88Y0G%-KcAPx zBU8ogby+h`KmB#CeBWX3ygeU}wM52G*NI$I=Qr!~Oi*@Qz+P1r=$FGKJ+Zs!N8ssMVrsXqVA^`wmvQPI)%f4>BWDu5$#$((Y{zw6h(*OTI5 z)8GBZXy1+U{lD+FX!1;YxxZ~?`XtN!f8XZ+ywB0v+FJen-CT>pMOAF`{_j7q$D4tH z!QsB;n(k}YuB}pc4c?`(eV@19&J+)!JE}>+^H=})`>$QDD#JtTSg-W-XEk58-z}S6 z#hbXPyQsd~*XW$C-qUSM?B#8@XMBA(HO0^75RF3uj;k#y1x6ru5I73_1`1$ z{WGjevvPKQV7ay~w)!xy`JBRIl79rw|KI!Zn6zti$|wJde?On=@A+^@*w5nOlF5FL z=G!M9<5|c4>aO!iy;BcwnWiH5o#r1ckxL!=f-O}rSU%1;pnpbu$GCj8R zYH08>AJ2DxpC9^s$y*b$NxKL{eA`Khm?v#pjOiHKH1fJZ_0MR-}Cw0 zWk2h;J0AD_zInb*?7wAYW##reMW-vD&lR71Y@u_z)m*<&{mY0zU@W`%FYm^`zrT0x z`keoNMsnZ7{c~Qg-#_oaR+afi9Z=!XbM~ls{F&0E8yga<%$KY$nE%A-#9hz#Z6G;@ zhAkReT0!Tlmd*)fhY0+0mJR!I#`t`Xi&E8gP%ttu@Nn_T+3ZL=JL}|R_0{J8I+i^E zwE-9y8s_|1xHQNwk`JuY!QL?c-k##~w%@m0_LH_M$@o-zA1py-;zWn>d7IBWpU;#2 z@%7)A<@TVIc`uqDY|ED?&t~V(v#l<(ul=zuc6Zpin3GZf&<=S9P4tj!o?XLN)P7A--r!&NCXx|X^b$R~3l3QCc zzuzf7ALuSy`ex(txXPzfD=RDa>{+=o^TY&2X|tRkSJ&4aoj<4g-Oj~sy+u}Yv(|$8 zzxjJUy1ikx|Mw$#pO1pTjz6DHx0h_nzP_&I>#=zKsL;@_-}nFj+hcsr;^mUbb>Da2 zKdS%dAiI1_&Bvqa^C}*>ipPFArM=!`(v{+1u+ts*CO%kn|L3{#b^Ct364egd^ZVWI z^tq*C*Zr=&T%z2AQ&v{?&sxpkWgq&bjML5>YUP%%{Svrt?qc`; zcRN>tgG!;@qu`?Pd7I0xzpm2XDN`hxlbahF`(={1UZ;!FtLWF)*FRVFo~DzSB%W&f z?c(v6ElZZP=-cPtv)O86Z~uSa?{}+Lt=bf!b6mdukNy9j=OdHyv6ilZa4>j?GpwYmEzFUy{c;Eqz8-U*M5tvIjX&W&!=nA z`Kh&KfqdfUZs+ZO3(D1^=dIuG5s$Ail;Jx+*LwQ?Em3PfZtb)C^&<1~GBkJb+y5ze zvv%J3?xM_^7Yo}XCUx)o{ciWnT`Rc0pWm~`CU3^8UE6YQ9-1#ADtc6GLxj$rmfd;$ zzE`dNz|NV#_;R<#_ER@<-^@18f2PVUrt{$Z{@-`s_eq&*g}pvs|F76hZ&%V$u6u32 ze*L-;{d`{ayB&p(&!m0ccwFxHjpY8Lry_0FPGT= zed&M4KECRu>c2mWy7hwcQ-6V~;6t41KC{hwxnG@_sN8S&>&1+2zZnJ(t?&Ol_wDU% zYb&dJ_5c6AUblPQx^;Tn^sc)4s$K3>pI32Gb^4i^#_9k5{B&;TvwhT|{KNg@_IZ`h zB>&a?`SHS%@ygr%slNaFUS#@Q)0`Uyr8iXl zoShHzn%`Lv_x9xd+UIl2lk}=SJUBT2@0)Z`8u<73_fPfDpB_DWbYoXi?HB9vcYA)l zTJ3N7bjoCZyPsX!>pJF}eYyYt-*-^g{M|0`xQfI>KNb}Pg1vsQ;nkw->+9ZDZrrwQ z+n01luYGYMAPI(sluIQSdwx9Rujha`kD8rpHxzzSjkf<}fha z^ivSHv8#0T(Imy6TduASmoL9lcs-`rx4L@wy=XSDanvGib{u|KaX_F)()iiG-|yvZ ztG3+ZHRS79Vc(Hu5-|1;vrYb7?F-fi;2x1Cn2QaB#Tr|w=NOAQx6t7@Eqxs zaq;2VR0JV=RQw{FCMvad8|}~#U@lZ*pEwCZCaV;l z66z7rjG1!KwNV1%2?#mr6sIIf|)V3o|flKBwWS`kO12eV{xc+up*x5VdWSHEW zuo*@{?bB^{PO<$t^PcWG0nCUlxVW!iL0|8jl`mdgTEB~Hvhq`vdm_G*oafd zhg(&yF79VuEnCbv(W>^%%JU+EvVvQ$-C@E=6?`#U^mTrE8J`g>Om}|_GW<)BS=O{| z4E2+u4`3v&#_n&exvEj}wR`%GDftQn{*$owe=nBKftf&dWW;9_Z}^>FUL5!@VvqSD z<=l*y9s+^?DvB55aCI-ENKDOoEV{0U-+lguAUxp`!p->T#RAC7NqZwbiX(; zvs1b6*_J-cEZW%pZQ>NAUuV)E{!Ty5b|DaJq<`4C^viDPw!601W$Um1xiWRq%^b~j z#cFpV|KC2^t^fDW{3>%xV{9(_eQ2k3zVxP_pOnqxYp=`rx-ZvBx0^4P`NzS3e~eYL z{z-Y8JD8{VWsz>c&{~-zo}SeS49c zl>VaNQ)0dI7W>qtjt>njU9GF_(m-T#;#YS(|K#DoBs{QIsNK2 z4SsH1`7{5>xsH4BUxGI8Jesrh$U51lR=4)$_&xaeapK(QFL~8z*b0!eF8d{4&&`fK z<$d$*tW60OZ+c$*J*t1z!Z&Nl$L&V4TYqwWp7ykAyKOSIv~BD(Ei!br(xRujw_dMQ zo%^^WZ`YrFwUfV=c=M7-;17rHPuNbmb$ZxI=+wh zzDd!?x2x~TJY1*wuFIsVcSZT7AD7MJ7wZ+Dd8Ym4UK7@!@SR`aA{}t$%FJs!Xa8T~ z_3TA&a+rCZUiJHz2aNmKw7jNztPRs!bjoy5w7JWkd6%!nuGL%l-7SR;TPgX=BtleB zH}92HK*^o9_P!f6KYv7;>({@X_cr_9v^(b>%+@cK-TM4_rP$eU*C{`14{G{XsQ20@ zzkT0#sPdu=iyoH?qzM5b54c`F^7eXO8r^)$ z)Zvs=t(um9Gw zA6A`b;(q>7{hzx0v{T)?J6}A0+?>CE-S=fU4BBG3C3@PQg?Ue_eXGOv6#qY#d3f*S zT`k5RZd6>~_x@9#VI=jp3e zTYcC2@Mz5a6npOFzm#?LpOQP3c0Cb(%98EJh|!W_*Q<(Hbz4+WH~XF1BuC9HCpY!) zesO&r*UdX}HUCz!d^Tz5FxrtJvO`0F5vc*hEdEwEOYPqpR5m|)(Xfd3THe!V0qo*||1v(!zRLLa z>h5V*HZMN-V)5&q7wW}V8|N*#rM1^w18btMK9CaZvgO2*o+5c=CGXogl`$^gOGS>l zgh=np{Iq1p<%N%}-meK~tKtG@OT(4>!rtee;=xj>1Pb0;Pst{Ye4+58`NiWUCO?jKSLKIc&A-cB($*+hEyzgC-X7g&<+N_= zv)DVX*|ld+z4mZ1GgaZsnE&bC($Ld0&Z}?V{&kz{F9s}8F>6v;dWpkDwrAba0U>Fh z#J{cI*->{hP+_y5^INq=kFUITwKcx?-M`i-ae=z4doeE-zg$ddN_;96_->KzUH)64 zal+ykw?2#QD@fm7JE=qK?w9#zw$3;!8#?u%_&l!*7f;+0t;&zV>K(P<^gx#_TV}dG zdUII#am`+V6nf=n(xO_vF2qa zzxiehH0p)U$GHoNUwpfbS#alRPbK#6$N9CTW_@Lx{Pc&1aOszuomKhqSlx0=8B$7@ zPRo1ByDzNNA<%on?LwPPUnWPquX(Y!eCGMnla#Ker3RIjT&mb>Ziuz|f1J5ZTPf@W%Y%1-_Dt{pV7?PYSwyPZ@O1^%6juKjjoVwQA<7r?KL;S>XFbj?_M-4>OCX; zLW}40#I@Vj@920}F0y=uFqh|3)vk_hX2->)K|Q18YuEPf&Zw^54emlOkk828{!Xv3 zF7S!Ys+S%5{}&m)pZuzH{)_2H;Iasyt$ogzu|sTjCbYVuh-)%9T zzNWG7YHs|RtlxWX<$47k&*K)Cf7IWxSoxu>vg@yRyWj8o=;5<{t#j9_k)s*V?+w1RcO8H(QddlbDoq6khzGwE&erN2aq7**ObE#`kaPlX1 zDb1iS7mYEyX0KQ`ojDaHBDgm1eOTPf+$T!A&TYCa`fb^Yt2sd~TNZ`zwNJipile`` zcfr|(cQ=V}l!{go-Aap6&u$ESCHksXS9AIAkcrW=r%qnos#h7D;|ILNd4TGElx zyfnIU-eME0yU+JMd1*IO$NTEkQ=S)h&6;O_2S?X=Y5&P(9T|zr+{b^qMju%A?c8>@ zwI@SUif1b=Na@R1D05{O z$xYa2@pMajOt$W+hDBfZU3;Rn>FAodB42-BifqNvskhQOXrOJRsdr1L++bSsTmb}@KTr{ z4n0?lL51TAWk1WiiSMmG=e=7bI(+e`JsKwy`SGKo__H@*c+C*>-?gS%eANa_`cq&y!q6p-c<|N z7>nwrGg`Mr1aF2l3DLI-&`_BWUa>J!w znvZM1Yi0Igr>@o_@!aAP(OpXSz-^V)do%Fxi5Y_XWm`+; zKA4hPIel+K_Y1$6SqG?}_?CHxh0hj@@!2c5;mCwx}zU7uBeH zTw=o!okt9BiU{iFq%7RQ=(@jXN`c_rtlwHX?n2rvO+k^JML}Eknoj=ur3FWM@QLl_ z!i@KOuHFs3@x**JbJsT3;GD%*0#?rvS-UF1Xl5qXcx@9@pSiIs%+*CZdUswV*U5)x zFTS+<9Q#!BRmqL5_bU6_H>kFCPC4kh_yF24=;4mZH$`Owwlw~Hl;)#V5LzU7H{+(X z$GmsOGSjrYM8$J67v**)Dq-m@AD4I}!n0Xv@}os(3+;KcPb~a)&ivQyNvlsRV{>)U zJ{_Sl>0j5SgAx;0VK&7L+bpiSvWw{ErcLyV?>KSko7}SBC01J)U1R+|9g|QF62)v| zvI(d!_m>X1^6;&ePOsnho%c4~o@{eBT}3H$nx@jND>vld8aga7#O$U%*=6CmsmN+d zcSi1=9?x?d+~>aAZ1uEc{#muBOEO$Fx!Zp)_S`9#X@fatP!u~S3go8dxpA7tKe>Nj zk8!cz{(Sk2cb6Yut9mZ`x76j_SC7Yy7`^k!mmfXh&)oa@ZDDxrwI8#lytF&JTPCqQ zi8XT4DlHMg+!e3)hdnJ>!igCyCo>CIWr3S%tBUapmPX208f?D~{Fd+tOm@5m@^wP{eA7(MYDwippo-|fCT_2kRT zUVO%P)7yP#zq|bQTuE%LV!M*94i*Qx&ViOw32(L5e4E*Fxwh?akn2jNMY_H-*R(gp zbehOwiM=ih)0?8;GV0vVDSHI}SDns#w`l3RT~D5>LaO5)S0y~!TP%Jyy$RS-eow0W zSMmJCGrC`xMaQ}LO<}qDG^G0Z%+r=1)}HP3xQsa*>-VjD@|g(N)k=$|u8p1EV)pCy z6-L*y2bah*Pj5R_w6y%*l?C3-o9!=s!IH23K1vB=7s(Ac*SCsg=>@gZc6y6g*PVLu z^yxHEUasWiZa0b&6ul{*`3p;A6cuuLpE6x^$94AOeV?1p8I|tx*IU{tx<~Hqy`Ow{ z>r^7;GTpFhTh+~{5tS44ZgF@f|E;U{em-kZEzDfU9V*(@VRq%^jb}RjCd!xroU++s z=bu{*i~7znx=wv*mwT#Y{%_f%Cr^9l>PEe<(U@_uE92xd4`Wu$w0f;;vdZR5X|68W zXJcEBzLUz`b>8Dq$^0|VC!gw?TKzotn$I<7|7&iyxUpEe*21(rgSm_CaFnO}iKTwu zBVT9x#Y9b8I?LwgwlP24*18xv;ndQ;Qy--K({_wQNSW#sHSuW!v2@}@-Q zdo0&z1-_3wt4(!!gOek@N%~75{HXw zsZ*-g9=X4YPa}&{P1f02apxYFmvi&8ukL;rn-aaGmJ!p^u+GWq3%;JaZ9DhuWG89e z>YW{OMWL$t`sPMgdYbd8Wlh(Xek_Scu5eOn*tJC+8KKeP4;x>U%%8pXw^NZt z;ZMHZW_On#pL?UnpLc!RJCFvtCjl9e&6pmC)wOhx6lclqNQ}}(vtGS z!YAi>{N}^-r;gzAgb2xN^LA%sKQA=Wao=`ppOEi&&Dl~@CNG+rsv~r9`_q#to=a4) zSSeJv3LM|AGk*myJrR4>wWK+ybibYqx4JcU}7#B{cn+UqIwKP%F0WwWPu(#UL%rB>O_oZ_*jBSt5dKw;%CZGWT2a zsgn8Mm@G4YRP9_EX{i|%cj(YUpX;UuFXS|_Oh1%}FIU^Gw5YXL=St+Xgvdu~yq}pMtf>+qv6!Y!NEIvD)u@D-Hfb5f3JMTrVN{lCig<0hT=?<|hEmbCv(_*62#D0=lR zYkl_Ss^iy$3j&)N(dMyAMAW;rtrlc77nOx9y|>)w`_5~dPKy<(>FE5f35wi!$vS^? z%C1F>YEOI6R$&NV-E+xb?>{I4f3*S0wYRf5{pYgG?_eZh(8u{n%lkaEiU zYEaUxlKH!>f_%PfnnE&gq+ccj7A=h;>Z_xzHDB>%s6L%nmAcEnxa*}|s(X9Ydz;TY zzVc}rKUl=Zj5gxWGsB`)`4op=jEnEw)kW?#cbs;1hgC@nPw#VC(iL>nb+&Jn`V>Pf zse{dDj#TbNM%UR>0^ddcDVe{GS;*Vw)M?+|u8y!NMVeQw7xpxEOY!VL9S}D*c=Uub zWAEp({;;6u*@C{`Pu|hXO{re){Jx@!8H<+~1=J(M^1>b|MO|Gs^~cf&Y^UYkEpmPL z>d4Zciu*VD6ka@5wRiJN)q;zR%a$!mdv#?cXmQxw@_Uw*pPp>mv`J3vTZPU0eZSwW z2wdF8FK<`->r2TI9`qn+u-IxTB4~T%1e4-gm$~nXyG!dM{N6N8kD4VSsH>m&uwa?? zDWM08m_1`-@819SZF}P3wzIR%-#_lRe>SP|wC?tfk}2}fj`hh_AD1oP^Z(y(`Pwgn z<<}=FyYGrwH!mDLTm6cj<0P0GQT=Yw+PQz>FSIck8cBd-nW!HakCVPetMR)#!QXudbizThQQ& zc5wZs9WU+ptxgMzKV|>@ip$mIb@o@krTzz8*(VhSKJu2U|MPL8viq`_iZ?ela&r66 zGI=?#`rX9pIdNiW0rX|HMQO#>=xH%^YoAQm?PYN{y&6(*%v`6jx}lp-{eQpR zp6mCP&tm=c>g%!Pdp{iFKI!eIG;wzRzMqHr?R)fNHY6PMHNSi1-o1N=wb057zyF<+ zRW?sjb9H&G8`Ww0B%u1abj@1Nw;!_ieh%|)dwtRh6#BKbfA5yx|NEese_h<(s<*ec z3aj}@Se0bBy1IUv^vC<*t(_Sc7yWSBdfo2fuS@lRU&l{=RPk(P`i^cZ8NTqo!@41V zQ(xT`c)Y+ZYirckypBSLB~L!oiYqN;1+CpVrTG5|*W~p4o%!{x=dTIIPCUKw+?S)1 z+RjGL{;sP3Mo;gH%%4-SB_8i>I1DcB`*PO&e$C6J)4zN>JFmk<$vExI4o$vy=NATO zRGyqQOX}adRiUdNac|GNdy1Qzi|Y|LXg#U-qaFW#z5coX?c29vptVd-IF^aMp6+^1 zziZA^-mZvJDRCoFmz4h4<>}jhX6@B}y(9@a5>H9>EYp>sP zD0WKyKI`Lw|37TGnFAVYzB%Gz zqN1ugGF+_{Ki%bPMdE*)RGEG0^8^mJtxm+GRasw?s+OcDzjEk3FeaxMVLde+T z3tx{-I`3a;L?r(!TK;yynO*lc=$sC!?Jb&_;j8-kW85F>?;hW$rR7t@GZx zFRgT!f3-$oK;g~2(7Z@3?P+`C_D)t{*0V~eu$dQ~zxV6S^!dHhJ?}k;&f9r3Ypa_6 z{*srMX6NtQxoVYHZKl!8f4{EpKR4Ta{qwEpl~7UPs?8coQ8|~`M83K*y(*c1^xdo@ zu7?#@?s2iyymf0$InNYOt~1TPcBkaBudtd=#FmVUkB)YO)^+~-blKlN^#0wA$;Uec zmBZG>tlU?!GbZ_X-`?-{s_lQj*_?WMTCa>{(Ulc}?SGe{WgNb>kT$#Q+~t<=VVk2R z^LxLZ^1QgJXxeZ0pA`=lG5gfj?Ynig?)ltuyPr=c|Nj2IfA7}B!))u8af|EqOtp~d zdwF?zF}JK;&5!K$d#8Q+lF5Wt@wErcQHh)x-`Snv`~1K|-@4pg=eH$XdTuTIs$tQY zx2FnhZ}P=lNqDlTdlz#~@T0TQPU5*6HvHXbdwpgeWMUaY3Umc~w}Tr$5@B&|Pd+?* zQ8NGMx+O2|lm&N!3iD-ddzlxo>tz`fU37gv`>flU6#ldC!)9D^-M=Da+7tc@+>iw} z5OT$llk!RJi?-`*&7N$yX??6{jEi~HH?~R3f9IHKDn$iqwI8lxcL22+o-FFtyR%p_ zZBL)t?hGFn_p{e_AKl4`yrO5SMd)@-rKo$0_V)OG&-|o*?u#yX0QB<2mr+GbJ60S# z;VS<0%-^8TXV|Tdb9xk9j5WQ*q^!7TFDp2iLDpilyUfV}58F9ZN={swd(B2~QMDD9 zu=rG!ovUJYmhamb_ulut{)0tq%I>jKR9|N7>2`AQL~g6S+>NM<=Lf!8`B-V!y-l~Z ze+r9DgUxlUl04kY2ny%Cia zH7@wkBD?h=VQc1yY?VHE+2w#Mubba)jqb}AHx{S z_FA=9Ep?awXFc~Db9Zdm={xt`WLD;?{F48m78YAUpxE-oT^;8x?lyH}L<-I2`j4)> z3vu}pu-IsU@Nw|iQq?OV@vpzPRzF`Wws-rE`=I7U@{SklUg?R;8c%2Cv6wZB50cIx z*9O^M)&R(I}V(Raln+Lvpif_=Y7g6Bo1-kWVLaKM$fjWNbVZIN>NDdjX5>(|c> zmixa?duj&>O$fPHY@MGnY=$%HsFek&!4iC?nrCRSMnn4_Mr4pD&E`q zV#UoEiZh2@@Gy58$vEbxUaA?SW_eO=cP*Jj6+ zdt7#w-@7yIqE?YO$hb_9akZOwA1j&lmo@l&Uy*F?hOBzOrZ|Q|#PZ6^+=X2p;B`4) z=U%USW_@~RN8YA)+^0Ghx1HiGJ>_}v71y-c?VsWp#7!l3zFt(A?lN`y$K<=`e{S-2 z*N&R%o#&vBlKfgNOhH4OIVlTYq%M3G(0*z*)>`J1fOg5bKSt*AX^b!A?NSrAgP zbF$9nQ=k#!sI84}yluALDw#iV$xAz5-cz0zPZf!EF8TbhjS*ISOx0MVGqt-Y?ac+N z*H^UMsJZ%>&%RFH*~L=f_lIA~yU<`n19V3U70O6LE$ zf8?c|a^lWSK7tq1E+^Vws{my?y!yyv!_g6B$`@OVQ_CQx45Z9^Kn;8z5QnGtp5*Ret32F(MS|O&3bgC zKOpP>+^ULCZhv0u?%vrwP1X0iCb%?P+|29sU=dq#z|Iw$B08p;W&gi_Tc*m~YBwZ# zLP$^dIVzFXW>>dlt5#cg?dZtc{M7ZyJN69dQ1F`T)u&6C^|B1uW_NFTA#v#H+gW|r zq75YWp=AG(BP&7uq!3ww-52jK1C3m$%3XPKP3-BCj7Uyy*01-B?lyEw^*Fv*XAOUkZJ-LBExUBq{6+3zX4?x$;mCVrF_m73mK z)GWGV#X*TPdZ#|JO|cf6dhm|ajH64xo25uXk{g7y?VfygZRw?z%Uy0Ymu)TIXIZ)H z{I_ix?`t;7e2~wZ%Lb~h*miaoISJ-Y;x09)5kPK#EN3t5^6)x7r|3&(+0Nt3e&2bo z;TI9~4P1sjGr5*o0Pc~UT2#EwG{RJNZC0?GG#k&OEBp$urPWme%iUI+ok_5$63{CO zT|Vz!@U*43GTztZPQ4dBSrpvgH)%V%G}bQa)|r~-@86E-K@uZ`bODv(&mD3uPJCm1 zNUwC)d8>NwLz6aLUG(kT=eZpS6Xcfr&#v$Nf7|c{`{8z^!2?fCKcir`!a(cCi}atK zfBQ4WW&aj&6~FJ2TwbZFr#vry3)^e18r=Zu=4CK7ZZPs(@oRfl@$m)wLIsgZa`mMa zTdfywKWmm%928z9ef6c?+(dt;(v=ItEH$G-4?dY=9~2DkHeXCpnR#`Vsq9mKwzmmC zAi)nJGviknZw|k%IrsfXmF{4TMLgN|_2R#;r#)S&{wP}f?OgruD!xmvB!=flCdr+D z7-rW2ago;}_QU%5l1rK&IwRJ~*CqxVN*hk@jlbaQKKba%YiH7G&$*pRNPe!V|J`cJ zOS|p2`8+uHetx(iByiP^4DZSJqNl}zgLw=48@Z?c;i(3H)Sk)Ji1(nFdDQW!>(Asn zOwT@LwsJmt@?q1>`Ptof&1akbh~E0ruDSEL{>du~Ix8)mvl{j89Ydp(9b3C!lZ*ELx z19!}AbryCM&0l;gK7SMMqTMq=WYGQAJiSyN+N%%i`z zK6y92$yF&Rb*9oH-`=YHU^7s4TyU}E!M~dsmrJ%3UEpcsM^rZ7SUtnu#d=)`Tb1o%z+jVQBbp) zTm6c_otsbIfD{d2vQ|CP{rH)QkI#2}I(}*vY}Lc}VCg>Dd^u^g%65_4uKQ1-vIE&f)_yfr51D>DYSHD|X&Wvmyr_Gv z3--oa*5(ux$Lr^8duw0#`+wec*#u52tHq!1N3Pl7c`MU?$ELrZvp>oG`Z}+A_to#v zxvbrSI;;OflH~gdsn>+eCF%}zgil9KKo3v-UEH3t@6QX}@Re4(_8ENK^Md(v#J;tS z-7iGr?;fB2q<`J1O*+>@G}T?RSNGm|>IP~9g7yKlhfDM9Kd~gnE6V^m+CR*$RSrG; z#Ol|zb^77s5?itseP)psY7`-bd#B953_4Dqh4G;O$uYdz;OTR<~ zsz29ke)FuY@=lxdx|lsCSC20G)XU8m`~BqfKZS3O*?(3&|MQX9+BAg)-_D&D+k8)Y zA~VF8_bsYncP%CAny*?+?{`Avy}fSJW?nviPv_3p-L9*JbxQm@A0G{Uc>DPl7uS;O zb2rZ|KU+3;?^izW70zoluU^$qcg@ziXg!|~90#h6@2zLOnbjAzWzzyhJ^5BdDOKHm zcQNlqq1oASwLJ4rJ`%CcXK<$nPi?&oSt=Zuz%`^Y~f+8tIayRi+656~YVx#Bg zfPLrlH*D&4y&E&7dqtJ@Wx-SXr++*yID6fmmy?}0vdYal|KymJtIKNbCwJ@x_$45I zSC(-IEb++v zRN1!2s_g$8J;A^Go&8;1O{(Hdemz;C8=kxM>Bbd(=b|rtxY+I*lq%Wn^5xz>8Bp`% z!J@xim#>>j>{Gt}uD=D58iXH+-O31dn{Z8Mi>KxJf*Tuun+W#C>6~)?a^q~hg_G=^ zeZlUj(}Um6P1j5A^|4ieq}cn?Vy|``o%COJt~0oFf$Xa?tZ{r=wq!=@C)=Gzr(XLo zKUPWV?OglJ+gsUZcmEIF_<_UMSO5RBXRG;TV%B~+H(TnG?X7zw z_o}9|Ay=u9B`#BW>R0-|hqz2Y)zRe|Pfwt`(0Ksps!6TmE!`mfx+l zKC|`rXZ$q%fB(?_E4r2TdkWUwyFF_%*tsPM53Dyn>pOSwL-ZDFz0J_wVI9Jj%{D=K z3pWO{zj_z7@xm5=1s3}%^$$NPZU^(r?*FJ2>N0(6{`Vhy4GSMx{d!v(et*LU|NoWM z8u63tzubESb9v|0wRg{kOlZyjJs@w59C*9RBSVmDju^P-k$coq6lhS z@SI;$4~l7S4`>gcy)U>nWV^QYt9ym7cUkX@t5WNY{{L$4>E-;^3A!nFk9uZhbL#$8 zxuCfyG}!I6tb-qW<(7hR3W>ih<34$sMP+a$(o*m=*@D-MRj< z`TaTne~5aoJ3GJX{-^8jB5mKw9e>guX>dc;&o2JO=GGX;tCJU%p6|2G`Yc*?d;%nR zL&SgBZ8&)As<6npQ>Wb({hlZU6|wbO@Jci*26=|P>(I}9WhH*9`~+iH;DYYEruC=l zD?Tn{z2ka%c~6*Mo!#g24X2lOOui}=^)d5H$qS38_r>SWOpY?%mNV;K;u95cj=nna zM#7J!XHPv)yvYX6bLSESmNOo^F)?fM3G>p~F7u8!g{DOQyA-$K*q^7*m%jcpc}w4q z1DE?FqN8H8#g2Dpu$j9ws(?dxmA>D6vy~f-Bv2x*V)gMc3s=FTKUbS*I7x|TiMinFX((R?9gC@xM!|I#7vctyS8TKDHrv)zy;#* z6_1uEAL}+r{N?wr<$b2|-+~L=9t(M=+1_#OzH6H0>EoVUe%9H=6I_^Pm?@l$eXR9a zJ<u(_xEMcV6q8}5|uGu`uP^NpK<|CiK% zsQK!5l;Z$s^2)^fg_%U%>P_5NWg(@fGHB0ZkH_DId6&zlnP^V${PueOPv}Z`-l*i( zhx*&H|9*;b301CrHeY8(?RNGjQeZEg)p{ehR=M;4-*?|$-+}HuOzsE{oA`R-uckHU zE|oUd{F;2?duxYI?&hb3f_L9*^og!glA0s3*XPSk_rDwSs$OKw2Msao$k=wUe96*_ zAK(3m|B|+Ro-U-Hbh5PYlv9!Ur54X8+EJJ1y6k&DrRvp=j&0R}S*R${KY7n`Tar1QS zmJ6{N0z3-0zgaN!hEDtyYyUvV=^8ZJzJYG4-_}G$Hv->kihNv~}XD z%jMHb=I^`gZ(f_Of1-SCOqTO{%9iu%qDFVUq%5t{y*enp|GFxVZqRGH0S=3VOwemALgVw>{aW$&Me-O9LJFOZgfVj-W#ttaN?Q>*qa)Rh($ z)YY9dmv5_&ieFK+x5|I*x#r5|ki@`u@cXe>>vpV(w2ZuUA5vMQEVOuP6)CzhYvIP1 z)>l2B9F8p5Gik<_!*vt7U+mk#=(;%H1KiAB_A6p0(k9xordE4y<;{FQIreYOZVhlT z;1||8+3>_959f(_lTV1x7SVj$@A;(KedqFd-7l;@=at`(j@R36QYbj}ly}6A*ENmE zra9IqzU^B1PY&9iQTIM_(&L3(rokQ4MRTnB7(D(Cd(bq%D{}xKxgT2QWGess=$#=%S z&3jTGEPH=uid+5cs?(EHZ@eu~-fE_;?t0oQLZh%n6P)QGOCx7JzH#T)y4_1UD_6ST zXol4D-II(DOnhb0S)r=?Soa4NyN}$mJ&jcpsF?U`%CMS`g*U_RU&Um$_A8_A77Z4|Ksngy`Rry30^ft zGUeEzHz8&3PbK`&WeI)^ssBLE>Gx2zEc|uO>a>H;4M&fq+XE#7u57w3`?5^65xf@M zHo$LhT6e2ibG#qqWAi!W7@ zNR#Hid1WeSpi$|U#P)`>e`0oC+ zE57Vx>07tC>hvT_iSNdfQ+Mt&yJa0;dE?;^WZxG)w07V3|9ab-?3ZU>a_rFHoSOUQ zp}z?C-u!K9i%um^`&9HqXIp{(>A(MLs+h!=DhVWIbKlr}dUxGY{&UIi^Mn7b@|FLW z9zHpCca|-;`o#CFiI$s=?wa47v2V4_zL;&1TdJS$op-pd1KD?0VcW}M9$KnWS2?t8>ZZ)s zOVq{Om+5ey{}=T0G4~g~r~1oUPszJqHs5r#T6uoW%h)})&GW^7KX-Z>ZuPtS@cgS^ z+Jk)V&kUV>;(MP{xvBU?-f6aVD_^(OTS`nh?LFn*&u!OA?2f=Q%IlJg2?i493irgV z*Yc|}S9x7faj0ENi9K8SXX(UkeLPyLCVu+=rYhfYT~cA&@pGYkPxaSLyMIIBzI=UL ziCXoa;Gg!^PkyIqo}d2Qv&!6TI!Eha<1#Ph^s>{zwKkgq*5zH^UAxmQG5N_q%Qq{R ze!3$1X+`ASPcr)({;ypkvTt9Un5=5dzJlD{o2nmLOa9Zydh&awhsKicVSx!BZ-;NK z-M6VUz4!K&x4pYw1q3>-*kdx~-a_HuK}9_xFG1(ZoYG$ZLbuuKazy0G*pkD0zvuZ> z)%^T&IgBy%_V#?A&{q3T>1PD4YRxcq4_RIGIW2JQ-QeY2S*FvcAJ@<9Tx56Oq$uT- zR=v>*I%&JCgFmLH+5b_i*QtB(viES?N2h<8i$vev3HtJ=+xp)}ztb=HZTbJ+wtCrn zsovhO{B6g@J!1JgSBTw@3;bN+wJ54OzBV+ZyIy%w)Sqgf$kzJzi?se+SBet+-`kn> zF@D(!vH!N+Ss(wi)-;}6|L(c_=lVUjt%};d9QfTQKL7r|SH;r*_HN01$9I2??7!*Z ztv_~87ISwymvqPD^{lsl>gQy{>YAFayq>bLV^z-o>gj2n zS)%pdX0Esvp1)6XRp|Wb>Z`Wa9%6a;|CiY3mj0)LD_u&S_(a-X{c~^0&X1w{1m2%o zRsK%o-Kl?{$N46vt`5JQoh|eG`-k?m8x8k~+52={yBZN7P+!rNe!H1}*}bW~5{g-I z#jdRd;ot5QFN&HPz3od<__xxM@M)h8nmUBMKAk>a^xdgdt6jIcgl_AVpTB3rtn|MJ zn-9g_l+N!63)xh&vmkt1$x2X&?NwbBy1jn?ifeuDa)MVkZP~v1^qzmeUhD7q&~(pe z#rE!L(~89Z{%GR1`}y;{u$=V!9nKNir>8x6de!}&w|-|A5-vf{g36U`u88E>aTri zwPtny!;AIr>+k=4wpO_EwCmZ&x3#n6{5?amkArOZnC}wOUEi;{`osI!kkn&a^JtGkk_X{VWl$vK(8wXywn zTB}meofp3v_4BxWSC;LrfZf{??=XFP7xwK{ROYAjhvH!^_alUBDzfH@z32IMF8*DA z=*@!PZx@Nq+?Ff+?$ijXky zu2>e#NzW>hndeYW@4*yzjv5 z{ksz$<%;|5dAn$)c)6_nYOy+{&?#DbKd(K@xwXvh$AW_O|Gr*d#P>8cJbvz`Q;XvN z1zoY(eo`gs;&3?Z}$t){@-ivvXyVor>+%YkDr}g6g9hzPxR`iZJftt1VSS} z?*Bh~&xTv>phL?RMaBEfP>2fr`RC`M*qdu27e(z>^Aow6b^f&Q)u__HUl&CMc}ORS z*M6BZt&Q#7B3I+IDQD)){IDW+XHZD@d&|kw!{ygjf9uHFTKjeCifM~?MTM-+dm9zF zcEM%IXq|(uKMxwFPW{Bn?V|nZcFrPE)0|r>tB%%shIGq+blp13{(nr!>C0DCR)zk( znZ770$l&V})jc+K8*PoJubH_s^X!?k(rN4TV`u&NyUpr`^XIAA$J{nbkbY;PTpkCGTfz9?iP?WrOzW_2TbN9jO=H_I-o) zYN5W$bMVt-0RQu1@vG>fcdCTVE~-{S55 zZT$B7i06~_&dIG-qSGg)%wpVeMr*exg`!UfycFWR5PCK-3 z?+V?b6kWKnAUrHWV^!*{eX%>Vvqj^~V!pmyz9_2bxj%>Ase<>H9xT3{@T2gsF}qXF zA@z&Kew%MFf4N4otWd_-5VGCEc1or zR_#NH@4ofFnx(K+ZPw3659c#(yVxZk)<>cV|#!9Z{{N`P! z&i=^vRQA8K$M)0zrSg7%jV;x8*QKcYtn542^IAU#3S+^chg z3cF?A^yZ&h|7B*Dz)#NwB^pt0WJ=Q~-ngmPol)ES>(w8}hn{l~q3Dy7{-auZvG|m| zjw?5KM(uO_+Vbi4rQbI{ecjDp{M26lSIVpT=Pl)*^ldrU+QzS6I(_X+(-(^#KfO33 z??!%Y_^$fySB3AtKT20h-ux=LcfkXl`?u4xbL2zX-s|M|%lybc*Wt1Hj<1w#*RGh0 zh0oP1O6MJzY+=acKXFmF*Na8Q^Zt3H|ImLw+27>9-?V>ojjqk@^}aV*|DABQ>#?-j zG?Oi1zfbs_eaM_w&wn&C#^%q<4^tnNFaEB#?7G+7Zw*_oU%O}=a9MTlTvhY;_rLxt z*|9?7l(_g5O_6EsGC^wf0sFev`pj$i^I)gZ&YDXL(o>3fPPQ*Jx)~ERzc}TU z4wsj?x4nNOKTELsWx|QQoAudawj8}~Yb5SBfw%Ni?xMGACs#+$JJKmw(IpZa;w0{> z@{_T}T{-S|A^Z! zwm)ZO|K{Fp?rMLtPU585HUBRzM-OP8o4Bk#>a*vQi%UGF`{t*5$^P9md+QgrRkLKb z=K3wlay|P=x#E=W;ssf27l*E0nLlOi(N$7D8&|OjOGLd{Vzce=*(G0GCj?p>P2Q2= zxYoMt$2oK3ALW68BAaDX6&6MIT{>+t8@)=%9o?#j|MIVmktA*Am1Kr^T$kTX`K z^TVNcFIX*-7S5l3;`xFbZhO>b+qgXNSg-$Q*NRKWT%)#33DcQixZC=Y-_t2`mtQu# zq_Wh+Vd6Hm3DZ86{oGmc^xM&Db5GBypCb3IFkG<3;%)vU=|#!x{6=?tB<6?l#|PA@ zYR1j+Sd!wkexm%QWP?kewgsj|Uo|{coxR&Fvaq>yin00kUZYoLzgH{^ydj-@+ixxN zVi~zQmm~j{HN=~r(wBXjC-dXzmDf|gm;Fe!dhn^dJmOJR{X5u*K;6v$%{ooyEKFny%V>ze>ghvc;>-1PELPVQyr(s z$?r>if31Akx5^O3aPP#{)J7h+_3c)B1si3$xR#U}Xibn6Y|QC1=`)RR=T-^*xTAK5 z`SRQMI=4Nu{WbGTTGWrjI_Irl{+s^&q-?t8VVz0;kC>kFP3;nvW>&v3J!x*1*5pfh z|Idgk8)sWLAN|U_R4n}ThP94A%=gMptvPya7rWcdsP`|#!hgRId$42K%NwU(8yt7`nfNY24ly?z@+SKD>5m40V?IWhNQ(_q}dd zaolp1e|SpH1M^u$OCRs+cxAuW`Bs$3z1Tw5P;)vx5(DKU*0Qy- zsfW+blYV0SWB1KV(wPOj#Pe(2V}3OG|o6(DCiI7arY=Ro-rU!dWMG z%I}&Fyt`N2)!jQy>hIbu6+6}kK3Q<&+m&BYU0=?eo2Yi7+U(@tjHCD5{;@8fyYnlX z%o69o)%MQIyd=+o2Y=TC_`R3)`iv?Cj=6yLMTX1|}yfir>Qsu zR)1lCdV2A~4I(_8DT~`el+PYC_v>mp_=8;_!M^e5hVv_$Tjz3h@n!VR;XBSJa`4|u zy{yd4lP9%zWbO$6w!GZs_LOUH1>^qg`Tw&2_q+QtEtcA5r}&#EPybuIoW-4Y&%)4b z3rmY@3p`>!9@^+>#?^Oihmyr2&p-#$6^l+kRdRjBzv=c{gZ)dExWrz$dzaUBYSiJ{ zZQHgv1=iN?-L+!jLdLGBEytqO+s%%w&}%v{)j9g%U74q1f{BG^)>WLgPI@zMXT^@! z)k|mV+>rdT%BUzKJ1q6dg=G;_L&L-SW2b)o`gLVD7dQ99i~v)Q6+u&1^1n4)Wwb&m zp)T;uhpnEWi`f}!qid3Oe|VJlzc`*HEJv!;{`1-Q8Gp=X|66rtu@ry!%2NuHcPO3U zeI}mOTgaK|ET|$c)_UdNUxh`j%@>m;_bvXjXV0@MDLVtVEY<$>Q5p z*8pz^!7CE4lvYG9;CLljIx#Mk&2+{tEoD8e`ryf1MVMC`H6>}jSQPYR0kgQTuW#sX zNtY>Nj(>Fz2l9M=_#kM%;uV{KB`b3aRGcQJ-R0YAB9i&;!-r6(iO=}>?<;E4N<`pcFRKoZf!BvT)$yU zQ$;mbzMSY}>`3ru$I{~um$1%@|yO`SVvOP=B==JAy zVsf2TYCWf9fyd^&c3a7=u)HtYWnZ$ozW7c1@}w;BCVQi(Jdf?>+829eUe4=P{>3uW zVJoM1=Dgj-#l;If-f9)(OzI8#|E5W~Nn~Yx+9A$aA}j4VzA!v{p2d5VZ~t1bvBAb7 znb+P1csoygwp03?SHP0WcWv><1FjfX9r#sq`oz2KS=|mf#f3dhr<_;r6AsBc5_&9_ z>HD?SrxL_3tv;1-{POBk4tu|>G7>P*xMVZE@!N~g)Pk^}Rhuk~jE#@4J9Q;*Sx9PP z==wKHw3=Tjm8InzUz_Zc*(&u_bmb|N`ohVeG@=$d^V!bXA*os6jkyMEsN=d>OW8X!EcHl`$?8qb zQC(@FZaS*#w3G$B8`ty1hIGmFKUnnP>WXaL8|QZ`t(e|)S?0|39&s%$jZ1$T`ImcZ zT*?V)mtyX^V!cqrGSXpUl4`ECpU z?8A6=_bm7kYf|W2&$fS@~K%wfgFExlgu!CDn<1noZXy#Lb@fCnoCG zK@W5Ni|ViMheXYPQh)zE-z591=Tm+1szKqg)ys3@DVDA)%jHt5ui7touv=Mrso1$2 zZ+k)EyKmW(!}nNSXPK(MnH;d>XS4XX6pgI+k~=bY?Csm~`0jSsnw-_|59E!~*AzuRcs65pQT#3O*n*;Vv#+aqWNsygq|RzLySu9A&8t^$ zYyW%>Br?iTynQ);{RKDi*bSSxrttEcdqQytxt znA3}<8QF$Lu337-kuUc~TuC73dvRB#KiTnH-e0f_|I+kn()TCXr?&6Y{{Q?Of6COR zdgE71uFQI|lT(tP*Oom$uj#ynMyJfNkP! zgI62%pPFBpd|S0X<5m87r3clP6Zu??e|aZai-^wB{dD!`vSXTizH3zM+Bp&4%2%>z zV-G#KZhH2UOJ+Z3w*KDMv3-y7y}t`ToDsjYuBh|QOwV^mqIceWVQR)N+c)vu>?>z| zqd%!Wv;LRZ@#R(^ydAdz7-@8wOjkc z{FHh*Q}N^}Ve6-sXuRrp8*t^C_}hRh&*vVh?r%~rOk$a*HP7l zx?abZoUp&NwR>s$rTNCff-ai79rtPXZ@JajbnoXrn}of0ZNGk>7wO>o^^IR;ysEj` z@z;NEU%c(IaLcajSM$xBJ724HhyQU(Sg$J|v*nQJ<2!ZrUv5@LzkhN+>Tje>r?Bz| z5zSpITJ*Yjf-f2t`PEx{*CoXkUUcv{l`E5ZC;IEei?6ey6oRtX&;7l*#;Ne zzWvN!D^#3coD#J$GHRpezO4D7_FvD>+<8Vo-1fn$?h=hC36b`EM*-K3TP=%9_a%DC z?ppZ((PIr%VGFnrDDGe{B7akASN&#z+lhR0{?<;GnJn3vpXH{;{>S3h1aV302f05| zRo*YRxS2no5+3s4kZT+M@^ECHtXR|@_Oi!Bk z!3h7GI`_Psbr;L0z9;@x;&bzo-FNh) zz>hwY`VB%ZCU*msJPxPt4G37Ve&YIb(ZW}MG9fpoe=V>GWLI9wcmGiehm8BJcdr%u zb#68*FMi%3n_ILv!+Gu!sf9ga$YJ-Q`H_ok&lD?}?#Y24q%s{i-fb3b=$N)MsZFGu z&u#s=-NF^xLfEWW|0qRak6YBMRqTRy)Aczz4&S}hdidh%&#|(>Rj->5B8?ez*lGA( zIq0$S@M;T5j_DJ>vCn<-s-`_Z#X+?(X72j+o_DTRV)exIM=3ggT(vr!s_G3pGTu4t zGzu=+z3B0|-E%tQ7_kM^zau9Lespg+cv-JpBkOc&d;9TM4aITJb9ENIU3+;06E<)E z>74wqKrC~K^F^70i&g%!je_62nQ_2nl%%!=3jX8$=6G&_ICUw08!@8&xiw=S{#pMP#D)G*Cm8QIyd=l}on z+*#?&Z1eZ~e!siDt2Eoy)wR65TtT28V>LiR^Tq|*AYNPm03@o8iCwH?t@W+rWR6}y<$t+@IKBuQN=x)`95 z^59W0ORUI6=A)ff5}KuF^!#{|zIk-Bt#)(?{?&ab>B}CqZ(rS{U2E^H-1}Vx8ko!1 z`l|kzkoN?MxK-isd*cK6*p+Ue*co4NhIo0Z60PM+wl`|`DA_pW7MiTVgh zX`RIv-`?K-|I2dw+BX}IpO|lcr=avg?ogp>)(#MI3U^Fiq_ zN1sJdx9pl*dgr1GmsHHVqNJAny!kT%o)9yts;n-aGtItsV7{xX>xciseijec?S5zV z^U36g{}WcZ<^SS%v8Y21QVt}?6()&&c8hwsX#$^XtW%lsYRm4uHY_*_H}Qr1 zzYpvJ=fdMERRt#Sw_jZyet5oh*_(#{*RNhZJJ0s_A#Qz>s>|+33Hx)G@a7pNmkv&- zPM?2gVMh0MYb$LDfvnX}SAX8i#&38S67Xs#T=##T`@ZJ;?)#zt=2(}%o9HgbxpSg@ z{g=h^b89}GM009TVUnb=o0eD7w8bJWBL72w)qbb&YiyR%TnX> zHk1F@OYq2*UI|Q}U%M@c%hllc3txFgq&U{_>q$Q5F?(<966cEtC(J%0DEs$f`t$ZX zZtG>k!6VVL6vY2N#x_WepV<`iW!vpM=KnQsHXe_PiaKZcT;@l8UY?$2 z)x%ct2j{)@_g+DDrH{Jbo8(;{->tJ9TqeI^RW!6(w(Y__E%Pq3E1qusn|z_wyYQ1m zXV0EJE?2$gQs?FTy3f*=Ixo+$EIu*k%a@YH=Cv#(fiCaP-(Trr$$v9W)r(pz2& zkM>1j`{$k!WM@BdwesX!qpG)ypgF7ab^Ml1n?6nQ*0X%MWU}6#4^9EWMxUny`&qu* z@%Uc-|Juddt3{AfkzSz@t6Gx>fe}R%!xrsuwFX5?t01B64(2 zqmjVHo_~`v+7}f_Wv`eVyJhdy7hAjAAuYnCPeJ!yz2dL?(A+C+t|fb`zkS-YX-`j2 zm#=&>G5h*DQv>91ZMB%m9qe-J)#g*OUAKy44!&Uaate3<@~TE71=o>Ytru%5C%c zxc$G6Pp8M%oz~rM^WgyV>1n#Vzun3f*Ndq*#Hl_ZZ?2#H-!I|uwW2E5UhrIS6^w-z z!L@}(vTxj@9<5Q8?P6=4@MpUQNAK579ox1_?9I9bwXU6g(Npn8HFMWgT~|a*eF%)*`wARwA%U^WeH^P-op( zKDkA&3)Z00>7MMFXn84WsdJ#>rG+arxTdjz#&SH9o@FhT!q$43cJ$;D1L>94Pi|hF z&aHRrMQ7him2Xk0f@@b#W9o^Us}8jVJo*i5DHv$`J!`$`@tIA_!R4~7YLm+R4KDM| zf4$jrH|@(RS@_r_s6qP$+L}x|dUDSO$;{pB+if(>u`}BQjdeiZ#mndKwT?!LUd_Ts%*&M4%4abP-c3DS+OGUel80mK z)lXM{#;v}%+Kdt6=1WI*X88G?+rIyAZt9Ek)$ew)r_~ud1qM#Mc-xi>QQ9SX&H2$V z*Q5Q=z1UjI2aCSWGc$6}$(|_fs{3`)%5#yB0w%L5X3N^OZ?ofnOKH3~Djxsm*X#A# zdR4F2Za+8A_V?-de?{r(%P-EhV?wmIHYoeuIpDYCC-YQ=MVWt(HmTef*RuMx`24wQ zTetPMjzD8i_{pNv({!2t*L*n0esX?SH&gcYb*=XAcfHm_Et3=7=j>>l=wbc0_Q^!m zXoH~D?M+U#y)~uxZsnFZ3AaH?Bi{+#|K9)q_ulZhjIe~Pqu1-28nJ%&S zr81&~*-qz+%;&6(+%xj0O1nzGK2r*xL{vNB+Hd#k1*k1}YW_w& zu0Ox7?`I1(MXkxjj&vGFbU>P@*QdP~e-b8JSkS=pU6XfbZ(q6*G$U{pT{J#t@%VN8 z|6QNb*K{|1lC6HT(f-%P{+@eZ1d*baweS(oVP#p4on^P4xKF!PGqL^Y;l!fSxEHSs ztY&*2f;S8DcRXbKa9%fh+XVaM*pkC-ydR#&|Nj+^8fbZ96eVS=^~xw=+%2JZ4$zDyLRnL?z4RMVsZb!>-+z)Mhc3z-nm!%{qEvEt5+Wm z^Xu#BnW6f-B=D7l^va{R8I^Wz);}&OHR*rZ?h{ulFIr}l7)*v%556(~7tJ=yWfEuQ zo@-le_NV&EME6PmWy)?OuHW;i3pE`&iZAVJoLI&BKQCHuN5c288>lTGI zcPzWneP~gQm(XsjCvF#iNok50Sxwc1_WJzJSA02`aN+ca4LecwE>?Q`?Y2kzBHOx1 z1DB_-{w#QzT?)ylB~>v~3NN_ATH@@nQ>@xRY3bxQCxMGe>1+R(&zUTx9RB=vbnm%l zXiqNlQA|le!H3P~?V`8m-Hnn(jdzF)IMaLyXw)sbA8p`2C#SPUnka>uvTee<9fwH@Eylf|O7^tgW{9#iH&5W}utN zrfP@(3uZ@6E{@_$&zZO*H6KrhKi36SXJ*oKSxljY3d_zHoa%EvTwPzcb^G?~FWyRv zqlDVr%)TurFMC>v$nuc5G+5CC+|9-mV->Gm!boELC zU-_gju~Xb|d>Xr6l&#A4rmnd40nw`FUEgl^^fju%3!*QVy!CCrUJ35Kx8q}z9=!ek zV$qeE=RmE%Cw#8IyPc9Egj^S{bj%aU-RLp#qW#WBcoFgL-rnkKYa-9veD1k;+guMd z9xsb!<{xmh650Ce?T6(Vucx|h*~WG4zzZG?XzN66o&WOX^XsHaO788cTphl?E{Yp7 z5Wl!c!DmN8p9C)sfByT-#}!9g5M^F!YU*+MdYiR&bFHkc=hy%Hc{_jq-&d>G$L%ai z4GX&_p)V&TRrU31c>cbh&!qGB6yDuc`dH8yQD}pr>WfR%+dMYGySb;ow$x72=+YJ2 z8MtL-J*{EWq3-MJj%Y zUv|PugO=_pn|YC&({y8Y6uio3Wq$)It3fwXUN=8S7hNc zYmxO>@?=d*0Hg9!KDi%PJTsQOly#jw`+Lo`S(mckOTTzsEGBN%{fYhc>z;crjx@2` zl~#0>yuG!x^F`dt!~FK3^OGFi#dRV+9AM^Ob1_LMKRO};bcPo4HdUDXp=pQu zRFB;m#l^*M&Rn^2#XtGRo=W2#D~><@I8$c&>D2B&E=Ec2R=VqB^g9aXI`s8NHK!zKd^}Ob~xV+-4Ec^PjsoSi| z>Vf~p&9QYUD;I`wwfEk;^RfX_j-}plJ*eoYKV2u%=$!i}`=3vQ1+E`E=JqMw-TnB} z>GA*m{eIuSR|L@-ThJY(E;#WSZ^#MRE-NpVK!I4lNSACa)#;aCVU}UFmFl z$!xu-i-=lvK>$~KYg^P!SL<6n_b$9_hWKvj(?w4TRVudMuwVY~O;B9iy;LiK_jfm^ z^Z%*8yv+BMQgd_j>uYP7xk34Ce*M47-FNFQ_!}6ukc_2ujW4_7Ye4rn&P)VoRRZ+6DYZtA#p-D}l6PD^M*a_Q4W z{q7oCT1xf*ie*48#~Bx0jeqmkSbnvca3*-ue-n|uzW6m|7pF&=AI|3Uo%jFt-*Pk` znqLYEx9!k)q2aOeMVagRbJpz3|D98VmRqTR>Mp)K*rQjKa*AofnVTnzlS@s`xpsEQ zO)}yC*Lw0rsF^A}ImmWRyw%(c8iZaS-gMa~dG4P#HPx#x&bGJEze@ z=CPi6v8eZ5S<;M+R#LAQ6)OGe{v+KJmm1sz86kVI=*mKPAEc7`(}JsuwtDA@tX(bE z?6}yxa2IsUPV=6A)g@1L$-SnNkM!tqne4QFRC2}ciGrS#`Xc+Y`|rJ&YO0NJUDU*D z{97A4-feoi>u}sfW0!U3y7}auB*Ml?UM!lF85>D#}S*#v**>(bDLbe}Itg{JPJx=L#Qu;VXaejh@!g7b}eBPAs2$QTr$} z-1fPeZc-x7o}dO_Q21VzE~jA7KyuQ=W9AJC)1eVl@_W~O0k`~L95OpJDkD7SeG?S7 z)!ueLFy#_-aCnzS=`xATovQ_1O4jS%*Jp8wUazxgYv_k%Zu!$DLJfJbXwuKvk}JWV zH+9c5t-c>wc=v(wWat>|1=p3vQ$2n+`xTY$JDc8W-W4}@`&Y@mzHHFON>Itg_chP0 z|8*vOS=OJBCEojb-^<@K!_Q1|{j%of4J$8ZNT(Upm{_7Ls%m(-kio^Id_jzm>&Bx> z-5JG?eGuL3k2h6Mp3#%K-=FcWbVIt_i#00KU!HXKQBRNUv$D!WMCjzcEf+7t`UFC6 zx4kHsZ#Vy_t+;@W_;@C@LGI)aekj& zqR+K+6wsGVno=ZM@8# zFJ4_;J-_mqFomYPQ`$Q{f>Tb^|X;t#dj~U@*w(&v_cIS6!jSl+pdm&7X?f zFG?Ro%+RGoiH91ubqhHqRqWr~_5w5u7SAEI_tYBbwB3tES0w9~d{}IEI*`jW>zl(& z2`;X$rk}3<%xh!YnFejOio}M@^3?O%P+?tm>q-2_li!@&FQk5nGUkPs-|2rQnpIkR zwl8W{+xBOePNTS&ixtC`tH@~+;5L~zwJVV@G;4i=5OvN zpKk8ja_g?n&N&xXf7V-marJp6ctS8yjt%L$vgpYrAs_ZRrHL26%%0DIi1^1nQ|^44 zaw%Sf>t}0Q)X!Hn>ANq!KBs}u9ULnnptAAIRfiIj%Y_Gok2St}1Rc;S34GO~e(7FA zbI81~{jo1{rc1lN&R!aD;zen?2*UXX^}I4ZN0z(ps=k$d`aSo9mpLMc_-u9q^)Bjm zMo;>CPU_-YrSRuR4|ZPNYs`hv|4~oN^_Pn0OMRvNAG7c1F6y{z>xyuIjLD^}a^ox0 zcIWP#x4=DT&16^bm~I;KKwqzbK=aS$sV?r^?5mrU;xZ>8wD-ua)PAEV+hta1fheIQCT^?P{glr&*OKr5f{c~ZW4CO* z`Qoc7BRtew^i~~`{kH78_+++AMd6Qq+_^s^?9!aqTJrr@1J8PN z&oMo|?BBcNJL`QniLy~X^UmpKZc#No9FSzoznZG$-HpSJQyY%dui%x;y$wl_q5-pp%U*sZQ_!q73 zyn6A5M`y>pNne+zmS(4`pZz(1$>+u4&vR?t*6*H)9PNb)9!@(~7p80pNL;ps4dJvE z1(ALyb_nQv`7&p9oW!+@iu2oDyj;HgzFEI>`8!vEi_Vjut@qp&FLv>n+v4y$De$UK zXxED!FF$;p^lsj_8=uzH?Y=nMS{7kyMC5<1AI~RhawnC3IdzGBZLRG~SyzQc2A>}u zGfz`LYpas^F7tN%?u^Tz?&H3hpQ`5|imDbpD?v8rJR`2pJd#N12%e(tp6&H zWXRf@*Pr~DDL%p8(IwHU>ii#W^Z1IB%eas3n|bf!ClzFWa|#JGu{MVqr)&v;CD^%K z(y#6JeLs7;|Ax8$wTn+y^xccy%lGf|(``em?p0G(Ya&9qIb1 zXX-c2sDn3NIAcp5bxewQpQ&s$>r%XlK#Anumz_-T1l@Gu&Wq^M@9O$7;;R?8tDUu1 z^xfB4Quawk;9}&-;(gm6Z~wl2e@(&Hp!?bSQhQG=Yk{|>F1V_D`!A^P*X1uM36J-i zeLd^THFXw*vme@8|5-h)u`jRw-!J9w=cfKO&gA>^yLU@>n)>17eJwmO;r);zJ?{Qd02 z*pmB^-S>C$xLny^Wm&oO;_4hPW&dnZn|7mP#`4?9HJF<7aO}vuY;HEiUx=VSjr+?BnC z|GuNkl$g2tb>B}$dKTxcpZPNV%t>hDU=e8BONgM#!c%u$N=)tsKH{qE@k{lB=b0sq z_fJ>9sVwf!i;VmJsIdCny-$Zqf6x0{eK&b~<&*AD-h26bVrJF9eY)EJ{LWYLZ{_dD z?mqWotC=d2duK6N{jEKfvG8u)?2D_7D+6E!hETk*2d^e-f7+qMin*Ln>UdhD#DII#bHDAg;rPxZUe7VkQF142xC5;Qa z1@LGtZhgiZ@}srkVzx;l+?h5Ui!Qv~u4%sIFpK>(-FtmsV{h~O&+~UFxp!RrhWwt> z{~y>sVGEqU>i)&WLFX(rvnO7horIhwO%6Kb?!T;N?x=E`o83uG8c}*2XRBj%)@AwH zpR{l%^V0`QpCp})tG0XT8zLC^pZVkTm&v7H+;`TSRyF%@BCSzUn|H6F`OfsMtM#{^ zO<#s6fJ6EZPS}0t>W=ImMK)LLm#(Wy&(Qd0dOGXc(XJJD#H#kcS>&Am_xAq>TwD9( zX4#y7an=YqlB{%3EN=bg?#|tM+3|)7B3LtI7HNEA`SqDyYiipCmFmR3%XJm9Y|6HL zIB-{C(cg<=7gtoW-K&gU=e^tNFaKTX<%`|lx{A-`L5hB>*oOLPJEf*vbh1sHe=)nH z02Z<#iYjfh_r_{|Q#$Ija8>5#NY=mU>ZheQvCBtUdAZ!LI&1!*!_DeR-{p&2{@=;| z5;jW+-e7SOpBwjJkHfr#yXpEbxHr^&bAc@XDGAgpdb;$~U6);xCcD>fekx+QI`?36 zcw3VH_WHB6r-DB#Ec!I(hKKv>j1SG*d+vn=Bbv1@7G3tMzc4eUzfUaceWtK$+3sp5 zu%UA$-z`#@vtX{5^9#9XIggd*Z{(jpI>>*oCs$BZDofR|qrXHW?aVLscV<=HzQ_?J z)fmt5gg0bDvHQh@(f|(0y>7Q4`Rd%6l0Zix>zciieL_c_F8}JDy*E~Puej}xTitLj^j&{LU>0?NY-lbT&>eZZj)UT*_ac=th0iU}#|nR9t(f5|W80+)@W z^WoofQmGyy`vvCwst$toBP_(0Z*2Ej)S$t2R_v$QMX_#?&jPB)CrYs*We6kZHyI22 z6olc4cKV|yJiJqPOHH+#Rd%zed+ydN?1Eyyb3a|JEC`%$US>Caq7*lh{!J}LQ7_Kz zlL-i39L|2U(@sLOv_eSztm5RXJGyp?4*w4~Id1gZ7aa1YSANEf)8ByFt>C4A?G_UiCkOrh-d}%WiKfz)?WcMd^_DJL_362C&@!ZEK;XBA z&Hg)gPn^?lB+Jet11lKL==r6bSYo?vPbc3@6V;e=FRoT@KUP}5+5MCPQnos!m23BU z-NSeD*3|7*eBA(PcRmwc{%^tN8a#E%?cz78B({6->t zVHaNp7Tkd(>b}k+C-qrg+vfg$!`jvAmwIA}=cBKa_*`Av-#wqD5wsb((SAnQJZgop ziAk<}FT~x;1eWhmoLu(x>4EueJ%6-KOi%SLI=k3Pr1!AM#97=(+1`{hZ2w%FyA0_Y z%i_;}sez`m4LW|W6emx5+xm6hw#fnn0N$(x&&TNugxe__Rf(3;RA3Yb9 zYI30EOM@&kB$aPa^PApdt>AGnX}6O-Te@_Y_Uk)8p*9{m#tDRsf6VhBh|I=>6fGiQ4gIG?V2k+JaQ-=kM%H-oDKzq!X}-PV%|qo*j!xwnfWmP07J6KKF#xjLAptDoJw`5S9$U(}pc?X&hzD*GqzEx+L3%MY*p<&0L|-=H{I ztw*D4iSxy4%RcJ2poa9*rh^l#x7PnZk+E>fuU>F(*2+~+BeJaM+S85vESX)Gy>p|} zRL^P_UAyor=IaE{oy%ZZ@kJsx)7yJEtD=vDsxc>0!mwiIV zW;ggS?s^e&Bl-ONsV3?FUtIIEE&Nq`@)IZleMvufPmh1*6OHbdi&rxA=@(62r73Vb z>WN$6PUnlAvO8Zax`Lb&-o4=bGMODT61(N+8sXSG!rydPSAXo7!n%3#vwsH9U$Oq& z{@?2RG5zK9*!e!ZZko--zPk6GU2(IXmFT_Kw^)BCPQUN=uY@I5zU+Nu(AV?bT<>@8 zdh$?x);+zlYg+pIR=#@q{&|dLr$1+hWA}FEy0aUNQ!b?hSc%A5RkSZEL`%GtDH?A= z-MDp2ulD~x$Q51iAe_})ZeGnd+fx05C&LBR_Y2%f@@+HzJyDbUfxTP){7y&lxm@$V zEo%mi6tW$?kUVXBpO+>3KO2?EcS$ANY)=0F>HF}JQ8@FSA6=W+6VvBi_`TbCo>sYV zv$LA-i~^4p>nFNy*F_uW7OqHz|}aIP~JJu{^Wl_qKO?%R_9-BQKuXFB|c>|Jiphse;=O|7CBUD^#3ew5f=BX8n;+$qO*7w=ck|7>62B%QW7%VdXlFHFnTy6E zd#&s(PM#7#&Ag|q|2n1zEm5Ae(VbgV_2ZK@r}cFr<*b80H+>SXwJezSe~ZQH}R-qNIUp8lKb)$?CW@HD&D?la3^k%o!s38h=NY+inM5MRm%tKhgw zOwF6S#caMsEX-OJwLdOj)7{s1;WZ_Lt?zCv-5ye1wCP^ZH?F&<_nE%mw|e^&{WSZY z;){R%ZI|x!I9YuDW$lFHSyz8A_|+@@b=s@D`{VOhuIu61yL8sJ{Oi%DZZ5eU5cctX z{cBy(Z`t$966?MR_pgcHo^$tf`GGI*xW7NK{ihMha?iIia^`AAzx${5E|?!)SND@W zeBQR)hlRVvYJPwI{H&(r`7k)f@_5Xz%(~XS2gz*Pmx~*{{ zEMN17+m~rg?*oekf86^j*0|U8eRG2RSL;K{FmJSujcj9BToB& z9^JXTrtaUHT4m+s|4rWSIz0LL>1$I=`fW;>-RIm(XN|Ec*X>$#Pid+Cz7^Z6-hC_E zyHv{lUqrrUcHF#sLD%lYfcQUM@--uP_bxpZSLIneas9hEhR#~cw|jVwHy+o?KeW!m z+R{qp@5Gbbb^E?9@;!h5_wKXPg|F5>+8Vt$^k)74)q9HXOz@mPrEbs1oz=$kDrSU* zrfF+mj$ZX`@As0`M!Vl>i2j~m7qwgKvQ=egt*yeMdw=FUa+kll?vz>1h2IPIO%nPv zYmaHRQTW%!9Sd3%@sT3sriy0-4K@7+~rUS5gz+O8G7-A}otqr2=ye#m2Y`Iof; z{*I0Jg6{qbo$0z?t9a|SSGu9!KAo=e?mWEx!h&w5x2C2K7D?BAn%ryuukg;6@2^&u zPWSQ)+dt3e_y3RldzU8kUy5G!v|InnG_QZZ?|+?Ev~{bdes}Rj-TgmWpD9H@KY#tV zf_v`Td+e!q3fu1mZ8-c|H}vPP*Hzw6?f-1J7nD5bT6tXA%zHtO+rM4jWFYTl{^!Z@ z^Yg2Z)rS9*_u<$7wSRlPV*T9@d8bYvaC&&>?vuAU-@ncYuMz+A|Hk=qvp(NhuUGYB z`g6OVr-Sd?39E2DyHj}he8O3M%MC?w7uUWz-}Lrs^j^bNtzWF)U5Mt|C%AX%gX$|W zEc?p$x+dgb{Jmhx{{ON0nm3HUPP?>5k+VSL0JqrvU&V_&<8LYcuRd|l(u?`!n?Yy;HLh@I>5!L=O zZB_06+PzDUP53!ys`l4uudE9k@>jkUk6BRnm;3o#Mf#DxpoYkH4@8y$T=?Db3rd`OjAh39JGT7Gof>OB`X@}9e!qVrz( zl77wBCu`4Cho|U7Pb+#pcV+oq*3)$#PP|Re`^UT|?dP}Zb*pRs9N$?VTrZZh{@yD` zuPSNQo3C_P?|0k_`e2)-$8>-9J*5rxFKPpFgbQUhq`%TNJ;twL!S#UE{rx=oo$e)l zmtstNXssyB8EIDOu&+dohoV zZS{7MHEs8TT>WjA*8P?JzfY=Z*WO?AH@=jeX7m5WmEwM%IETE0s+T=)pR-N-{zw1) zJbv$`Yxku8&h~%7JM&6>=+fw^xp!A>uhO+H2RSE_v&S!AGe3U*y`Y_|!!q@l_H#Xs z)7)ZqI{f);sbecnCC%=5%#f?R{$$L)7|RbQuWZYoe=mqPdfSWIX^YqS<*%IdeBR|~ zttVfz^pxTsU5s4f^54ep!i5FfL;f!Ft@7TguKrrr)NOa<*XaD0x>xUCS@L`lYroy9 zog6dz@3H@UQ?{31YH_DT-LIEnsn%sLbgyp7oqaFp+ovK>sazMCuetwUZON|Co5KDp z;$?Ehb@i`DXZ6URh@CmJ%DZ&Q(wDldY^$TEzKlP6=l_?>Ro;oaGj#Uu-TPpXT>tU( zq8L5#oySkU5KWvFx4pLC^!w#=Av3(*n8wYYe&+v*;5_ladCwM2HWClzH<>t7C1{2H z$@y~YpIDTQ=);;$`u309mS1xZ6vbM7M`DV43eAS9)_TNMId~$fY|J}N{ z-~Wx=P2V}La$9fo`oKGJ>6brUy~D%ye_DNKf4%9UEkbWv^V_PZTDI0;^Q!fQ zVVCYWPkQboSQ)wg@U)9N>x(bXo4a;qX{~Qc+W&^fYyKQvu(&+bCjU#>20oW170)tf z*8kgoC;Ra%=lyf4?j3d27TL^HXjC;dGCk90iT~AorEw4Ew!6)mGWF$b$>}yxpO1M; zPIvof^Sa~7&lLOn*QQ0?3BLM#gXCUT&c*xhmnQ7bu5aADFz;1y0oN^WfLwsO@7$o%o73lM z`+dTb%KNAGgzRj-@**^CuCDD#mDSz+YZ_kcoBj28{kEEbg!OB-MtQ9@o&PoM&&nLp zH>;PY{eLO&v+8eT`_eP!PbYW^yQ|1{nfYvTzG(5mvg*}`FBi;}KZt0?EuMbCHQit@ zBm2wCY2oqjuFW$knznz-mwksPJ(r(kV!lqW-{k&T%{^aNXFivoTlFqpTh6fRYvl9v zo!5`stTg$)WR2_jZF}ZtYp>pB_U8S%KReV*17H6AwDtFt&HWyc4smH;v^Jkh_I*9o zBY$PxG3^tz?h}H8mhYX-GxLkj=jsjJa#MA_KeMtj4GzTck6CT1kXNi zb?nNcx7#YLkIi`17L|QRch}_r@13g)dtNqdUUwsMwPme-_C%R0`z)8I#!s{l@LaZL zAG39|k-7PM#q+9fH;eAM`|;A|N6B3YmI9cr%qS$GuV4&*Ut+l zSW{`8WBa#0y~P*Ht@m4~Ow`+TeIEbn zwdSffxf6FEo8j3K=cLpo|+vCT!zHt~K%YEgobk>$C}?Yiaj_cmli`yDy!u+b~_>nGEyOKUSe&2iWH|Mt`O zlV_K&o40J1g&2n%4sm@f#k3vQ-F6Bkvn;v{V z_?@(++S5CSC!e2ZTBCmQX)1pRPbB!Dt*BWk;<;H1ejh{Y*sC0xJ@B(l6rE@-) zT()*Smo@$P+BuVA63@T)o~C{6%hW5Mm;dNY4E*4l*TXXZ4ZqdNsqgsW^EYol{pjtA z?^{B5&OV^-_$tA-i2c-NnWYaVdwyd|4>!jDHtkGR#yvB0PeubBE7UEOx$lhLMv;W05+1i_Y=M_3da(*5>_TOb*)U4Gd zuT#Fn9k6;GWj1kv&OE;#VqvCxzO6muJ*9=;f8PELPu@MLtL!*7d6S;9^CjuMr#M|o zZlnsXT{~%y+^+X%V`;8RF(nDETc)3$!rIy#dhzt`YkQ{M`*tJv($mz{Q{v`5N||IM zD6BqtL9s$FH=(@q^)uLz5(lj>RG|B78%E)L?dNjXKFDf>c*VWnCxv;zCaN?%GMu$IZ|9#+F z;*tAIZ#wt0>sh@0e5Ph1kvd|ff%*CRB3rj@Th(o4ZM|^C>eb${S1w&*a;=R@W->D~ zbDH?fa?!G7Y+ZL&1-?v-KH+w7qI5V*v4>zvQP8tguiY=EKIMP=er?^}ip%@I=l@ud znV;p}_-D(e#U;H@dlv^O^&7dq(+TB%*mi90deQuv-LHa6Fa9o_?|iX(>+P$qAF6(= z>DQ@VJbkCE^7NhMq08TD_};r8d-KW5oX?R5RjQf!stW@4?TQT!3yQq;zNeq*Tj-`h zyQ>RUeCVCLC_qCboaHy?nf6*Y**6a!D2N=Kw@k0;FK^eZt!$fGY%5a_M!8z}pV`5& zDEMxZ^>^!{H^o!)JRYuSG~aUj?JM(^WZtehQOb>zdOxjl^J`I_C9;w~xBs|v*Om8f zd~d7PHfAIq{lb^K`YM;}+qZ9DDb3(?{lHXuQKH>!4TFf|kB!Es@Ai6Cxi6o`ZEvzf zqOfWG)-|jCTzXTzNO`r|+1E}G%FGI#dE=tk zDy@9Kd(p9h3+=WXUvX+dSHsVM*cJ92A*lf;SHyl<5E{AsTEX5ART@pHS6zcwp1RUr zmJ=!#ZgOlPZ{};+TOVW{1#9A8Dg8)W$gwZARPYUdz?H?O_pIKWS5;M=#dv5RyZ6C5 z3G;Z6#&1ZjFkH3iq%7!|6!1YN&AuJ8*GEpBxOd8$dlI2Dmo4oHoT_>v=+P>prlO3+ z-Ch%CvBa2|nugXk91fo;<`~;}cp=y4;-aD}`5gDfMc!VQnj)E1s|s$EBOmn{Ph5xM-(gmEImDa^>xM? zp{}m3t}Ct^>YW02UR$s?;J?GfXNf^!VQj9mt|lZVDvD&vaf6Rf`Ztkds_0FPtnHF3!JZlZ&*0^$`RQ@rw{E$AoNwX2g#T}k z9zFV@ZPxOgckaY!1ZDHc%uZQ+rPNwRaEZrTtpLMGN?+LWl( z^Y@4Bf7$dXwO#dWxQocYzZ#C&Y13YX zrn0iKdWDL)r=_LEgsSa|b@EMJu*jtPyPihS{M}Pm`Mga7ox0@o*KvtfvzOc3MN71r zS9$IgS$WDNbsqTS9ncv}J(E}YtTo)ATNNs)vLGb2&~<~TfxJ;jy}%M)*9m)eaqLsp zXnOA*%W_{wB=eNb8~%VLnHEd5F54VDxX-*SblJ6#rCPz;S4>!?V{01=wn^DL)@7BE zh1^xw4d()uXjT{Oah~|B-!y1yrj!oI!i5uhgR-UHOn9|Q$4ajDi21P%B4=0i=q!z# zD(HQ2%_+Vt%~_$68%55>Ya9uBpzSO^we7a+!xKUC7d(s!6Lm1Ae*|glr;@d)x z&HGwwnY&D`GKDdBnSA%ZF@K@QW=**)OAV#;hZf?kpo5pTnuKJeEWYAy_~v%iZSJk| zXRh1KKcwz7ahcVjUk($OMLC#qxX${melzZD{FIbx)B41Nj7+YtqNKD;HU(xWHnMmK z?qS=*aqjMYkNqBf=PrMn&dfjQ-XYKRE+5|a7ybWLbo-fK{QKp(D^vCJl#3S-q|wVjU(%Sv=fr{o>my{|g)DAE@!IZ`cbud`y3X zzthCDzs%J|xv_7)%L$>erH>7<$!dSnb^UNY_6|1 zPZG&AV!g6o^nmZCIj21){<`}>S*UC zy>#!YiIpi~OwVkp0$$JIt>*%)ugf7vF^r(sZh*4!o| z)U4Bac3$YR?R%eGxR&#LzE|k7Jhi5?A^Gbs>CNg`m9uD?QRgb3u8ktlbC>4K`kP)P zXaArpEGh2b{F~vQ>K=>Vj=p@W=Av+Rgh<_tVpF>eW50R1KOOe%H2k{VYfs^2!IRwo z^xsEKxRjq3_ei`c_`l1$#wm5N1(!d*6#uik_I2mW`i|3|FHETnpK4#?dvtNT%K6`m z^wVF-A6H1?|I6q4_`q-1m4`2%?L{2k`Eb*TiYcFByI+c%RWDu}trGNfib}^J*~8}_ zH`qz;J*9R)o#ot#-On~W`DqXv{qx77OH=Qk+VR9&^SRvgT_4haUcVpnxBvgyokD9e z%Ra@#9n};p`SrVQLiN?@L3O`FE?eiuy_Q}(Uspf=cf{HHVs>pK*`?`XcUUIwyR+@+ zi>p2yi_VL`4cIb24RfAkB8UFAyfCTh`)2L?(ERRh!n}=oQRY|596x%fyB=SvZ&Ee& ztmEwU&$qnM=JU7vePK!E1CR74am9`Y?-qZ`v(w8}tYmmTK_%~~W+6|@o$KLtjFoaT zH$8ZB==M6Tx)+NURi9X}dU?wK%;mko8^v8GmT26%WAW8oni;u?13ui;;K@AGsx?dV zm&E_MzRA@4^!)uh-@Lppa?i%PGVe{Z+4V4+tS?%#+x{*1yk4~5p(lZ#;r~h1jmP&klSX*?E-UahY>*fDCjk}D(4>$Mz zj|`~z{eAPgAFl6s6P8Qw`d*~tclnr);PQZg6|1MkZfUP;UsQNzipO$69G(VPp<>@T3 zLY`PtX}fN1*BkL-kMrc`E9HNw?cMrc;k#C$T5qGDcuC-^GaQ9QmW%s!cFqZ|-`s4Z ztTZ6(*wIpFOg?=1#9=%<)@a{me-%Za-T~PG*g#fD6|)}ablm<`x*O+ z_3lrU!g?ao{YNTh(J6wt8$BLmR9XgDlQ?v?cu99=9yy1s3qL4r00{ zzmSo67jVV&@BDMAOO7G08G^fNp~Xqru4^atZr#i~@w`}KVIeK2@Q57vSD{g36E=r@)R#c<46g z$TzV_7u~OuE^ak6pSXz`TTrS$T2i9%s!V#X)*hErLfBkx(-CZ;V|L|~(ygnBUsj#? zAfTCy&FKXSeo5ua?`TBDE4Mkht?%YxpSXz)i;aCRkDh$o88=rxAywE%Ub7mTSu>*N zxCG{Om1x|$l=5X&Pr^HonaJx}1#2OJe*DTKm2Y29DMdxAvi@;ff0|=i-xOcu3;{d& z!AHlhsC`kWPKkz8d#7@$D00?^oj~!qYw``Xat*DBSG+Rsf`vc3ozlS;8fv2Ig$D#{ zQ!-Ed;B$R!khw$$x(5fmA`_OZlD!JK+FO1}MY`;^?u`@W^^w%Pjn$)pbB?U*^JrhR zHOM08%dU)sV;+*&lGpK)M=DiqptET#GrsJ~NZ#df6RWb6N8lYwZx*dR>$bj2Pp$h3 z4_0fv=S<;qb+vbv?K+mY&cl}xTRe3h;rwHKYvGE$=U88z{T}x$H*CQ%Lri_YeO2kbJmacRwJq0p z+_U_!DZ3%69w-v^gwM5g8QbFU)T9m4D>IQR6IdQj44dQ9KDR_etJ2nO{plkKW{uNI zx{UM7tD60HUxd!I$#Ff!iY-O(bWR4XA-w52TlYi#;_GRj3`8{bu^OlBm$YB^*1{F% zv%l=};Ve^n%7!hQyB+c5VQbx$^>gA|rq@J+^K&_N+CpRX_NaXBT7 z>AWxhHv7` z^Jkm;Gb^9jKCNuC4iiw{xM%&@CVz&1<=YLDEh1U0TSfG=4!SlP3aGzRSaQcOcFvJx_ov;9 z*uQ(rZNGW&LGw$+t@33;>fH&;CmeEB-JxN?_UMRg%M|NpU6Uh%AD{M^uHQ9}>oKFN z=>gZB$?AohILf#*V?vs`nUWR!?l^kvOy1P}i(N#t_;9sDAa6n7yBnT!beiTmafwfT zu;_q9hj6(-(7PELen~GmucdnOe8ywYEQ_6Y7u7oi#uwh;*I31Z7cKUhoAG4PJyUI)?&mXOpKM@(#O&GUi`C=T@U+JlQ7uD8|=(^{+clGqsN>=4YMtX87qbN!&loR0Hr&1IEF(gBTWhzKOy5Lh_qs0^-QV8cE^k$mF=3zO z)vH(6zwq$z;Q06Z;p;`0W?HQ(3+&Jkha}a@PacH?$Nt{s|C)M&Vt3!T-ptOwE$3#@wKb89>i0Gz zGW+_t-CE$)asSgoRC@5qrZN3>d?@Kx_8N=B|I8hoO+wJZ`B~O-^#DCTau6W z#n=5@YNcvq^y%;S`{K-a`pK8RQg}w6mGGi3N?OyVce=Wt z)hmo(f;d=BMBQ;k?cH?GjW@UEgdsA~l}CE7zE(wV%dyP6V^R9(%F19*i|%gbExpR_ zeKqg*eqXg})ul_9!q&&lojqInL%J-Vdn?zho|q}qRNMIH$wsF>kJ;-p_r)C3sX}x2 zR{ij>d;LLFUi{awoyYmicb`b)oO^wP^Q4=s8~Hgj%SHA+|985i?e*RL*JqaoeEE51 zUFzmbC5pD`iHfR)M!OZh9rRP&x}!U2j;D3{xj8HSQp(>N8yS7MeEi6fEqQl$`OY@; zI==MBkBYxL-|l|D@ArhR=kx35{hM@W9*44**Q#ayil5iSnDncLoSE+(FZ# zJa1xWP1nv}+RkV7`-nxg`+S=jiDjLP3=dsiK76z3tn|y^DwX?hHZd@~@hK`Q3JRKZ z^He6s>+U5lYqlgEzN(X@*|?RrB1%-%Er*bf4qPGtyykLRj&@z zZSH1b*kCDE9{9=mlAo`?YFVs_y1K*ibCcEmt>nT_g5D$ z-F^4ohl88*%}*|`K37tHiszU6Kl@a-t%nN_)yJQlJo#t|hE0#)wS+wzzFZ{=Kk zR<9@`C^okCP`&VxH%&L##e!~moz?584UnsT@MHEo<;ro{}COq#xt z>suk#S?nUZ`rLny9}@f$`>1cO)vX8?>((oe&UXvVuleM8eO>HT&BDUM_j|w3Tk+aG zJ|<>Q%E?KG4mp)RS{1swM@}!4!(CjoU$jze*{tx2ExN3AN{C;a`c(0gB3gTexm1!T`Xa>;$2Sm_XWf2RpbC9!p1_p&8Rw&dJ2 z(q3KmXHipAQ+Rys*Wd5=U-$RSUJ|-GOi)m;E3|6wJvrN|J^%mx-g2r*M@c63%Jj*) z@!#IuJo(pm)18HN=S=)x>HM5{(em=#+9VH`z1G`LGC$yd^|t>th^X=={P;&8a_0#Nm6X)JdZm$mi zcc|pb*TXE9hU===cJN8RxVa%L?Ai*B@8yfn=oLYW2HCzxD!b-vW$v0bZAPrFabtJU zalc0?f^S5EUhmcjjEl1?c@ePb!ne1#`Q>bOY}pcWGHO%GN$vG}jFOM>sHWcDmdnY> z`F`(rxtH1V?P_0r=37=!61dYn=zqoizP0OK=ymn&O4_|MXQbTr^|4a=eIb&51WwGkH*od^6LBFBUo7w|F*Z-`v|NvVYYg zJKpelAX0{5Nw$w-Mld?R~QsP*5+(w%c# z5}VKHUt9s zynMmCgL|_#Pf9wgmj-JM@jZI7B8j=nYpQ74^-QJ0*$$Meo|^8tS9|)b*c%0`kQTwhF5#`Ukux`SKYKnaR&+Ik_|v|*M@}l; z&|FdwsA&@$ViT#=opImd{X5xpH`Y(N{n7Mf?advFKP`7x_diuqxkmj`U{C3N4M=(x zUS!dE^f*Sl>*32qB@YDJzD;*E@so4EvRC2hm$Sd-ZR+MKyqk4gR#o*vg**Ast690Hr&IPi(2%*8wdxUZ!=wY;@qJXiYfEX&5PglVzS6;x$-F` zA(!?ViC$rcv^`DY=D19pB^c>ayy?ukmaPql1ifR0$UHqQIo*<@rITGt9?!bP_%{8o z!tRWI{imxJ2MW$x+CBH8;HEyk-=5bly4ue|G@yFLmS1M>x^`toEN3<&T3B6~KL7Q< z#3nbFRB_n}8YlkOcv`)R{j*r$>QURY8RcJ%&Y$N-6hd>|x+WVfc=1tbb9RCyq81j> z(`sGbEp#t8Q%bOuSK1>w?dZL`e&4o+xhuCA9#%P+T2^^j2T)3pfkUpK2JZRu+}W-rDVrv(C=5I_yL3g_rX6|v&h*^c zUF6PvWW%a8oshz0dE%odYd)qfIsNv)$)L^Ld9mnWyo*icmupFfRp9$YL38`KY^_h; zP3wPRDEl=vwd?BDv~VS!28%*=#ixGEM?6JpEpHZaPng_l*xrnhLsqT)n0I@TC-;j* zEfGa$W^dg8&HJWi{IP#g%DQ%Kj{^(9A>({V_eaU&OVXW_BO(>$bZwNMKH&s4rH?c$ zI*6Jsm+47yD2rcxBz9!aoLj%YT1GCqoLP2oEjOqH%GCJr?QP-LH&0^sxy+Ram~&+E z(nAd_iANq>_RQN}QLw+>;bL;y^~0}F5`z_c?3BQ&Mc-a!xLN(}=U&3t2=W&%XW*AJ zvmal-IDe|iUW?Aue!PVX~5-Uw9P_)T!`n&yg_KWk9UNspbv7h!7p=%MpgagWc% z7gvVygB`WvMM;RVAERr?nl%a4yILliyh3WMA97tzSO!lnaEh_|SWx!TCH`-h_{><(vDYL`RUHiFD z@wfi#9V^O@hn>H4bl>5~Pw(c;?)}&K6{Sxk6cfVSb|f6!PmtnzbkQk1$2{?e!q&cB zX+9Iz_Z=&TS-! zd1keyyX({#KwSYTtFFFZU0kh=o$HRQ2xh+|yLp+B>{`)_@@gGFKWXb8dE?{B^ZVG^ znMmz`9V=MGdC%>%Rj~s{yvC1B{G82`k0iUN%J#l}*7ed!>D8(8Q=O|mrR0V$n>RN_ zP}lb1JR3iWnLUiI(kO{Sb;payK;F*xr_WEcd48e8AMD443e%>a&j0pC_O11lhhAN! zU*G7csP8=Gw{b$%_M8;GSFxK;*cPt&bN~c zEW~&A8;_X}eWUvW+oXClx?J5`|9xDyfe*_b&N=^3|fl#>b7Q8P}`C3bETah9d`Yx&~=YDyXD@+fA8+aU;WZstmby| z|72_5xf=6yOn28LHviGlh|GAep%_$l@z(6Gg?F=m9WxS~Y9sf3&dWEiR$NfOWvXxx z)Jj3Kh4;|wucC66Gy48Ro%GVux86;sPVZSgeYcSE%g5F= zC*B0LvFH8l4EQ-~dHKtv)RgM$i{v$rO5TrMqw)RBY~?R+p3F46D0)Bk+Qk(dpvn~` zGdFdcH5s=b*lDg ztpC#YYv&ob{j=Y<+(2Pr8B^DvRibIrEzskfmvuLPC%-taVB!sihqF_@wh0<6*%y1; zV@^%G(HXsbiv`y%&YOGZFx$>w>Kit`yF2;Vgax}^t?O`dk>*9NR#bOnba0f(3CZh8 zaV5RvyxFlWCq$i%PxEJJ(vs)LZ>EV``Yb4LEe~DSx66%x^8e14T3=p2_5V`7DE`;x zn@f3;QA(_Xt~(d-bk-N_p9_k{wx<01a+8!c{^$`8kZQeRo4Ka?jKSt^8MzJS+B+91 zOsw*m3PvPH!~Mp4ET_DQ)TxG z&e<|{3l^fqc#+GOlgx6PxR=aiU(!{y z*;z=d;9@|=gVKwIJM^l;YHqeqDRjxc|3mVsYhhrc9p}WH1?rWjZsfgt?=~CM075Ha z&67fx8ma87*y8Z1>MWnnHPWtpv}5IC1Y{H#@Ca^Wn^#N3SlPG)|sm z@S$XVW#xk9zOSbiy4U{Pd2jCRGoaoFTB;GLu-(`6XP4qf{;tr_h5wgm9&mMb`BBwi zP~>6z>tg=&-xt#t$?r8>q%r@gsOT>v{vGNk-_M%vS`wfn`BHNG%kx(PH*?FbL-V|u z>_dAMyBHIftmEQ$xAcXa+Frlj=3VXoiu4lk@T4i#R|MX7#h+OJw-qqE+EB2gtvnh&m zd$g9ds@RjCMwhb7<$t9tsSekjyuL#(Ql&?tWq<*8fS*C&xv=6gs8 zt+q&8@FwJKsH~Yv-46zFURf*2m9jjS|8Mj9%A2bAO*?$boJaG_F1|^MV!HTrikE*! zrRDO!m+k$eJXc1A6?|M`L)P2^swarG~vh1 zQxkq9&ENrLzp1mNeomdjX`A}_OZIZ@Z>wkh<2-j{MU;N;nj<5iZnH~)<6R=N$@(8Q&U4OPxahZU@9s0};v$&P62zRA+Y^2i zTom}I6M16J<;@DMi#%7~+xf!%IKQw=@v=(uHG=-reiiX^?@jgyXg#7a>%f~&X3BQ) zx$#SOIjZ~FpDNnZ@y6tV&%M&sU*6BU-ZeSO{ju`db?F%P+*!KD(pKf)#F&$h4R<>@ za&wFGvQAp`LvGy(zj;43y=K`hd9-oynYkwU+*{S-)UO<>Kn z@E5z5`zK9$8M!&glSjeNZhpY(#dFWDE5NYm&ySi%9~WH~_t3QgrAd*RO3P1ce=d>u zD(QY;Le=$kQ|3$kEn9NaHP7fyQ;nd-|1<24o{pc3HTM2XPdUUdY@(zk*(p4i{jHBLU;HYM*LsZ?4|RSz2BiC5)|G^GQjrGdSBMtuKKkd*t;63R zI$wL~&nwQ$djFg5C)-~?i?4JtI%`?2ikbMy<42nK&W9S!m*)ReSTW=J26jhhMog1R zH{1gys{nz1u}zBamd?7YEW*mOGW}wIQTcDp`!i#oRE6t4Snx%%wE2>GeloP{jx;ZE zC1^$4k?)ZS9yL2MJUSL#Fux{x^QLX)oJHxsCj8=)n_#W?_05ypdk?=6n!nVFk$tH+ ze=3W_lzo!m_BT>Dn^P~#QO6jR5M1`W4&PmM;{D`9BmP~bu5KroH6ytGFZlAQd4c51S?rEaKPzuue`7Tkvm>uKKT4M~ zNCkQJMTh9qLl^4Sy;$$z)q2F^spzy%M=ve>`*izNch?H@=x4p&{~TXFWp}*FRTWSt z5Ta$BdiHvYBStFNYO##3lVAL4^W~SGGxh96yyh?bD|?vDvYl7x*O3$Xsa}4UcpS2r zN}DhJ-o(!;GbL_I*<*0a6=inl%C9+=D*JBqfEs2yABx%7>z91(Ep{zSyYAXLWBr}` zeInfbGs2c!Zob4`A+SZ0ZE3y|xJi#Pqt(Wp1}e#(HtFlTzqszbPxJp<^L45A)7M5u z?yTudf9JEXTtu}>Y?;|>1&?REUiV!vGYPNqoxJ%*i;~NVK_>Ca#)mDt6a4S>za+a1&Jr`(%!t}q7D6+#TUaOz2wnLGw1OaVddQ?Z;A z51F4l)!A&flZ=yJ>a2et3y^xYH7FWvqH?Ye3?W zgrA`cdlg>n&z_~7oo)|e2nqnCt!zV)`XKB;{B{Jf~0 z33;KXc)tIb_^DuV;oZH`*Df7dvhvV%W~mRa6D>4qg~9zZ)D$IWauyW+0SS7qtfy$I zxN&-J+_!hPN?=HPh>rEsc~%pbebn(;JH5N@C)Q{Py#3(oq*krHAujHr8un__rd5RN z7yC+u8`mU!osl?CKK-}RrMDN~?z)-q{YC2%iNu6OMSE|(ESU04Mo(L7)m#ser-mP@ z%Zk4G8|-n(Xg?S360*}lu>Q!;;BZ0L`!(wEQ_$UyEk1b_!}n0 zcjZ0Lrj^qCmaeVlG5gc|uCDJ8ZatAPEx~u@<(oCWk)EElvo`%&cmI&n;z+5RQ!Hh2 zW=(GAm%q7tyMgUpC11(rOT5?L8y>!#n?K=e_8LQ{3BG#+eItESmrk^Z+^earZG7iS zHc$R8*696nW*KdJJl~?MCqG#2L*A)Xz1}NKQVRkfooHl!vgq==z2(1bzh~I(-;$YZ zasAWNUoUT_NAIt5D}F0wbtPln9oO!jXHowSvbzhf@wSZ?en%Q z2$qy|h3k2_4>xKUbkZ1x~HOxcQS3Cu;|6P(~&>d zFAw|s`Q%i8d&~C=*{^QCuzuxokzZen{3RaC{oSgqwJ7OK2Wx$G#LS{^wcm3q{@o0o zYgbnCTgiOY?lo(Em%X0u_SIBN&h_Mqpw;%j&t&LDDZ3S4tm3iYUR%C6G9%kC>OvR4 zox!Z`Z9a>4W>lNa>i*Wmt8e4oSXA=A=pFf+v0R?B+}4`2v_5v?+&h!^Pn|E` z`{vE7>)Ell%b(d4)E`KdludfUkQ=h+=XtyAkUhe?%OuZk-IpD{c&E|6|MxQ1F;7;P zKD%{cwE5zlUS;o0W@Y~?J}-H8spT|HJ+I&UFY0*Tt2&+G$f_MKWtQsc=~nz!&g@La zx;d=eJ-B{* zGu?{U{`^yXIIQ-%p0c=a?W4|y&1Y&JFfL}DcuYb!oKo{M znU(GQ{heE}X#VXqlj}{-=bO*+-g$!wv@}I4=f$Fbr?)LJKRvIer(}wqx7_dJYkvRh zKf9@^Z)UM#-hebbr6cz=hN6rigT=;WGZ+rL0N}FFVZyrys zi9hzbMCz>QlRO>K!x?&C@5=Oc2Q`aT=sni{P|{<(X3ED}^Z5!AXGNdv+Pi$&F$0lf z<*z+-dY>89oZO)iw66PFe%P`NwcUSiZjIHLlD})^*Q?vB*^5j=9+!nom*!I_JhS%4h8m*b4AknsH*R6uTQuohhp856lXOEajPH_2d=JTO( z&vf;tuaf=rdCpSxCx+9xKGp7Qzj?y@-CgUJR{K+mvI6?m&u%UA@=V zzf1f0vighTwe=rsLsy%m)|h|2Fw5v~pM1_=HcnSzlT_Z={ddg29y3in_bN+ST=do6 zmGjq$++WsnZT-h(?=nv<-exAkn|bO+Uwl;1rH-$s|E`ct3T6|ZdZGMfjMJJ`A;(rs zad!`2Tk`%{?-J*C8xQ;Yzx`~<-8bjf&b>Ct`$LL<{dnqq|8ae5z@D31GsSH3Dl2}T z;(B?-bK?&SR^Kz{r?$_sFP^isLH@kc&&YtrMOK9-GnZE%QAl-Ie#?7tXW*x7hvH37 z1-B?!pY(s&)YjZ*23{z&yw41_ZtG$(@6`!E?5fbM7Sfa=~B6X2uHpU6}S>?8vLE&X&#ZJ}vu`cXrL0&9~CtW_U08 zZh7;Q^uunhrxWc@`<-;YynD{Q+}VE@zEpA70OIur0S>03Fef^EH9-`-)FZe-je^d7qp@ zXy~p*R@{H9X2!NxDR*t1ZoXQ}Q{-uRhM4H7!=-ZPR-J$9Yqo8Yjp5m%EeFDDeB_>Q z-sE&Ge0xHL{lqVJweMe6fB7qXB)DjH<*^xli(dT9+^4g8$zPSF1sB~g8)0AOxSFci zbxte)s}(s@NGUsCp8w6;+uLtP1}e%=@|!2p-BvZY5$O8b8=yLr0m*i+x1zn)xd3ed|x zb=~9T@;RoyUyPLBU-SJ}Q6}a;rQ-XWqOY$O)67b3m&<)(nBjCk?$iH&+ayvWqr(sN(bG<&amKfM@p zZ;F1n+^H4spPpUg^Rwv1%<1M&y_cQkd(G$eP47$Ry4r%08xGg+NQ!=bZV{Beb9s$+ zJm1oc-~UfN^zF+1=X8q!v)y^1zd~hwn04~54E_6=c7GqGG)o!o*yt@Zb)VCx(3`tX z&;I&W`^deX%Cz`}YJ2zG%(zj#rsbe#(S=33H*9B~^Ks9L(=XMmr-s@eTEE5+JctjP zi$duObN#89rc-iluY|^j=Vv~4#1!6b{u^`msqfPD6V~*-`IKgBz5IIYy(`~Dx#agm zn_WAmWf=8By?k1Ez4fK7f1YpJx>_q-@9UYlnqT^+?bhj)jr-k`Hz~GFEh7%I*T-ak z=*Pt6w#J61c4yA5J|^;fL*u!BA5EQq?Og8rDK)KPnqJuImCrt@tzS4{$%p$QlKX6} zv*YDAluv(BR(*Y)$Ue`oDRD1kpUyP))v@xr_wGI4HBS#euG)pq6~NOOh~a1XMYET&&x&NsDQ(@uFqR+BU6X+N*r{VL?Dx>4Ng6kg%{* z`aih8Bz1iHRCG${kN%+xJ2bR+x_^JfA1D$zE&Tf;7O%v}MR^P4w%^X3`)e1|UWukt zZjpI+yA57N#wUKcu#53VHP;vOpt`1w%lvcBb~u{;6K~#CRrm7E!}^ph*Icb%@?PFq z?SAx{$??9m%YSn{P5IgPq|>$as{5x?pNeFU_g}7>bS_3yL|)6;cR_hdK*;Px`+9F& z(fssFC(3e3R}{&x96t*-P3aKqDbSdNSVIH?BB%GWLK?SfBp4L)2(GS7nj`F zd@Y>g%Q^AIzH?XIeDRCz&Xvg8+4a}?;^JNBNQK9HzrE%7>h_D+-GX1Q?#_7p>h_=A z+X8EUzPmN$DaV(p#R^}SD(B?oz5D2}e!opNNc?PY{I)vgyjOc>8Eu<7ZOW9EiAyD} zitBlah>1<=UKg`7NZu_fc=ewbmTXt=U`s{nh4g z@9+CZw)4qeN`HH2=j2VFK7INUnVFe6rS#1WL(k`nvahXKsPk9NYq!nnDRumd&i1Fi zEHB9N)TsUY>+3Hg8QZEUo3`cMU3LD{$&(A$*j9ggvDwJTNaOW`gU!Fz+}&Ls9G17> zdvN~EO|6@{y1KqZzP`40vCdylrQGLhf_@!q35pQ&ef%#kWNkN}^oy>p6)RtVc{`&Ht9Ud11+|v@hI? z4zs6ToVM)zsq^QTubJBzx5_X2)57=1e%v>@?=LaqMc35Q~V6C=yxpgjTEV@6J z#$`R<+-t?A&fJute=YsuzjL#UKvu3edI2PLOR`pM^^|+lGMlfMr{-vi+v-i*{3ajm zJM?D5I*R=ROYRNUymlQ6-9uda%o_4zg9rdRIeiR?PJn!nhe;EULDKl764=|y|5uGn|VZ{dNyLuYQ7 zZdfd&dv3)($Ff|jve%nuOt`@EOD8P;SLEVfK_9+Go^j2bx^aEZ@#UwCUUNj;SMZgz z)?&>$HKTxSjlS>(h23VdyFsBNyFoMgoovc=zDrXAznVrm&N2#%|K$r!V?sfy-KMEq zR!d!-{_oZ1rZ?aOCUw>P#JaWlr*3evW}h;N^x43CZTh0KcYNj_Wzk6pK6Y*`)3xcG zoVn7!bgoTd&AECrvvk5QwwqgTirSd2(Og{XvR~oa6xN$tGwZLU8!|558!`VVSj*gO zmYZ8w&b^qv(JS!v63^wfQTwwuubr~eL%^Fe;{N6+uLH~8teGaXVc!{0+By@IF}3#Z z*Xw_OeKn3|^KFaLEeee4?=x6_$lky@cXgV$icTTl+wf||lAl7VftD(onpck{{HO8>inXRMjQKQ+UGaTUoz{}Wkpff_%ya^B`3K~ zlU=L1lkb{ym27#)=X6Tex^&A%&?ba;7rhGt9~`$$lX#tRxJ}|~?tQ0COUeR2Uh%xX z!SPk%+%tNco_?%)syLfX<$>$h6*1G>HqEa;r?RrNt0_LXWpB8`|0$? zb2HP*I$=wpkjK~#^glm-GsmMW(P58K(~8%-14E7L3w-{CN!(e!jXgM}<#+VX>5~OS z6U!zZX80~q`6l2!-_H2i5kYTaKX!M!lz8l)HPP8d)41mMw-d|}m3y6>mT0fq91S!agxxs~$o?#;ck{oCGYpCpz{OcS@1(a|GD9UwHx1$73-0f z93oAAi2M*d?RWB&NNBmG$-egV#8t)T&jqu8oBsap+q)awBn%^7Zhg}A?$yuC#+$Bu z(ltLV*Zr9H&oIh*_pjsO{L)Wk<{Y`VRV7|)u8y+rF2-{+3%y!b zQ)cyc>8tM-zrT1Qr@4Qjr_Krf+Ro~Gq25!f=C(U&msRywfwnK7*TiT&OX2=obP?hw=q_T%nErzA z_Q4D;*p4AAuIYh@LHB4Nb&moQAZx73v4kRYyAX624{{=ct!2g1(SYtB!V;p#AY!9l zg2WsoMvy0vxJZwsRmB(o{{Ak{QC#-^-rmg1%XFi+&4?BFeZXl~<>$1`+)45RLGKin zfW|VKClnp`Q}p%mY3ZIeZQ8%nVm7hupnVopd=Q?ffW%0W4+y9l&856n3*3sdb=z8H}RVX*X_MkKX&8oQaYCrz6s zCMnr@E<@(#rd00MFR$0{Hv_rh_O@J+OADRbXT=Kq-jIBpk1MmNXcH(ftjpgmnZPGy zasuQ(P^ZAdFpuB;+nbxtif`}l-+z;pkpVXl(5kGg6crUso6gj*`N7xZGED{xUMYWu zoX*1!B`k}cxXagG*;o5}p>zARsZ%9=BV%L#cJ;Niux!p2NZWDuomuX!HPPG8y=l79 zc<9ifgU#&qzg{jExs-Qzm*Og!m21}gdC<&%tVc4r%<<5?!|nXv-`)NF_xt_&?{~`| zKYna%WOPO^a7WUWimpW#`ST+Guwdmz*0L4!)Vx#u&Xl@9oXa&#m9@agJVA zx7p<~*DT@Xmv3%NcE5GwMTc?SpNcFk>yj4^#}_mR2X~)(QoN*#m0N5^Y-hot#~%ac zX!hLL>a1>U{X5ov%hz+WOuPFWMQ%))BC`2(s_?Jx@BKfvbhmlr<-L2oe!pMX|G(ew z`_DG(J@+emYZj|)E{j{QdgqG{LFK4zIhF+v9J1JVmA&=4ye5ACy*Hntq`DMcUv}&7 zn_*R|l{>BEPW{)b;rwzoCyFj^X-Z3bc&K$t@7r5jyK9&FPW+Z}>l>4wkNRvAL&=pl zm-2a^I(^#n@$GH7r`M?;c6Cs(&CSiu?R;s|wdZi$tN8Zr?(W##Wly}lysD~xO;Yu?k?X(7x?N+*0^R6s ze;&5WU+J7{Rr>1A&dn{$L`|&Y)=xJ(QTX`SO5x>xbG>HA6c%o*`!!M7ec$(c)uExG znVFfZ!`GiXd)9O}TjI%gcXvNrv{0Gj(3Z^0Wp8dQ^qXUt%ytx1dZbO)>sgq#FnW95 z#*G`l@W+^xwDHM$C9N;nzH;5VyxZGyLA$G-a(RF0Rp(4!9k$l3MpzllAPGPGR+o*Rw2(msw4`yOsZkMX-;L&+BVzZ*R$5ToTjO-)~>~ zYRb*89L5h8`5yWB__%e+iw_Um;GAr zuI(1r@9XHOcs{q>$k_O-UUadUAMvIennb(dN7yl?0dD}W4k_|nQ6Sc{Jo!MiC1b_;$h~X3;lM# zR-~Su#>vTfyifMBf5S3S6Vszdk8U}iRi(A@^&(44%U>`3=h?h0u(+c6_*n1a&d;Ae zpIzr=!6^Qed+YpiI={Zrn?7w?+eKq3u9zvx+F@%J2$y9! z87@kAu>`!~YSP_zDiQaW%Gp+3_%G4CqHsgCzerH&qkj8;9-Y6xy*1ul^00m1N0q(l z(`Q|syiqa!#fOx#O5H96j?~oDS=S@Z%rfn6nl^Xt*?o&LSunYj$|4S4_;E z>i2t>zlUxi6FPhHB4h&@tJ?wB%8iAO-HMhkbZ%cIe~7)et<;~R+eGW%r>1IO z?R+CrcX^p_+H}1?95*VqWnNDE@!{dCDOc1v>wV`sgMx!WS>vL+ye9v>HF0}?9p<;6V_$Fg+w1ZLwm(0g&-b5aQ+aDk<`@3X z*U!$*KD*9k2BY{>CqF;F){m>#@B8J)E|>d=DP{_Hx0voCgG-Tr&dfAUKRfH{+Gz7% zR@L8fHlG%q#*$-m`t)i0zh5qw->)rybfoi?chfZ8=(Ms#U0&IDnqG#6hFdiK=G)Ef zd-i2&-w&Z#VF^N8l=)??rs!;`{r#<(oj+{;TK&jPE!$2?8mDP!-rkn$?cq`Je(!gW z_}^b%HkTWiGVRb1H_N@1QsON6{n-u%@ux@6&9%O{+5d_6#fulCx8)e_c6rzp_&72! z(6HvmhkpBiKjxO-Q{3;w9UvdJE@tM888^1&MqACgoM>xnJ9p{gOjg{LFwrszO1HP? z+xxqBi|OX=|NCuvd|l;cmSNZwb*@ge7oPYlN@B8}o zb?dE`o|$2|ICOQG`MrwAN5$iP>gQLzTDfG&l2xm;{=V7&`|kTMj0@#!{(L;n&Lg3~ z(PWfbS!sDXbpO1Fvs*GRYkZgg|AYN|0k|D;gZbH_=kx3T`D(4(_bbcw@V=kVq*EVl zD7d&a`}(AUu&}UA>yP)#|Nnje|GJRp=KFv4e)_-ldfe=3)0WMgyHg`o^Wu(&ZPI!> z9x$1mw*7wRae>8+DFuO`0B8_M)bnav=4oI0>dN%^I?qV)*pk4qGBa-;Zt>oZj*6#K z!*Be5();~hwf^5vXU*@=nKY^B%?(4ZHS5;J?JRowxZmFIW&MxC@)3)eiUYa2tuDSS z@tb4uanotN#cS5Z?%w3GLjz=uLx;pp4aKfUwcqc$?_83oqpB10Wm!%b&1O$9o9sjqhuI%(QT}Jh|O(~pD@0DEkO}XopBp`lw zd%pbS^IctBoO)a!CkYrGjQ#TO_xt5{DnCElxSlb#*(qVnztfkbH>F**W;?+KAWAt@87T2(c5x7=Pf&Tb@z*3UtjC*|MRI^f8USg^Xrl{ zckbLdzwXz|yXE(5Z>CPa_%ZPj-7DT*3#Nl^3o}i-}cLd0BzkbgrF5k=B^Y62aMdtKz_wtaDxPp6)0y}W7Drm(e9 zfsc&O+blk_wc=yaw;4~j<=&3qn)yn1`yHoF&rhE{`}T5w`Lg5l zzsz2G80svQ2R{~Z?f>_6{nU0>)w$i9_Sq}D_l5jlaQWq*!}9+eUak&bpQd?+^Q6bs z$rD?H9$CNNb9kSY-l~ollT^L0T;9yR@JWkTmjXv&VIe0criX|@iu-?E%C^1OukUrI@VKo0z8{ZP1TLPkdXo^; zB`HncW0gKGzH?e{cMq@Z`m&44?tL4c+kY+gong?p$?oS9;a~N)x8;8MzqsE{YLCPj z*Yo!OYc_?I*%lw^5PZd1R8;iu+xGp2-*-QmyFJXh}TwvyosV>_GilWVe?@1i^E z^J_MFUXua4;zx0q_}*u;vNM`ucb9pxeSb06Z+Y6Z88akOH%*!(WY)GkO=|mX+tgE2 zb{0S1wEm2%sCv!gUh@~Pv)AuU3(=Z>+O+;(P1v?8S3(+${y?(!FQhiMAtO)xpB8#K9=jH5ZP>U)Dxq;!KUxE1Dz18Ng zQ_r^qUoP+5>lQ5O-03*&MBXO-_DySsYkgSXr7FXMc-`@P*+)v%NQ@2Q5> zeTplsYL5Bx^?LmB&gkuVYtLKN{ju2n!t2yso7F-f*C@Qeyb!J~*2Wq*#VE-Dbp{tvEc|zi)O;gqdWoBmP z2rzi5XNH`=q#$lv@PMJ(vv#&wu13t7{h_|TyqaI`mft@sH^uZ7wEAW0)Aus?uAH~? zsaWa}zrPXR0|Ntd*8G;v+woAp{^w~|@mQDt|6lsoTUC5mVAQM8_4fAm^w`Po_Iy6a zE?3dOdu8RmIV;KoKUrOiT$z1+-Ivl+Q#9AuT#qR}tK4r>bl&#+hNpFZJ|3U)`_pOt z?{97zC$OHZe!us7Tx!6w`~SXepB`7W(&)p!x+BZkc4|zVG^yz8)$q;odwx9XUcdKS zRG7iHSq;kWeKYp9feYR>-4}l>njTYh^6&Ti_jeYje|uNzcA&gLFM8XXt=HprKkl<$ zx98KT<8sw!UdS%koOV_zStV6f8dQm^`_Efb$6a&a`^ERSw!S{Zt)Fvo5$kKqtwm40 zezAT^39>VK@_b%(-n~7QUz9_fiXLp)VzStZtLekr?f3Wn{dPNK@oBx?Z@%5muYcGo z{_ES@)$+YwSv$JD{t5nj{OHlV%4aiY=kJ^8GTmlp=<2YL_)@$B44`x|iir(@LsRbm zeOLbc<&VGb>;M0F+|Msz;2_YlEV?8qNl7>R$A^bs=hU5;plI^X9^#+L~4RxYsTZS=&6jyr=H_;n+1p@9RAMrFql6w9e{H^Zi)0H+Y#(<-=C- zcY8kjWreQW|L@mvxoRCt%b!xOP&fZ-Ux%o=x6_21_&iu9|M@7T$)Out4G*7RdP>>7?@s;yzmJdin_GL#(bUs>cYnXV?CiW958JfY z@A>rq@BaVS7dp2;Jw1JPtX{#pLp-l;Wv}16lJAy<{*DJsYTlq~?C`?}n-uzwZ?5?G z=<4clZOyP^P*e5N!iAeQeUqs9^W)?G|9{_ih5mbHzCZBa@`9yHmzr8lO53@j{BczH z`giN*&5g0$|K-2!-@^XiZaVteiw;Eb_P=;L&+=3F>G>Ah@(n zy|8uzcu&v zw$JD7{~u(R|MIiuY*=AJ>I}!sTMo-FTNXY#0;=ve?J9eFD%H{Fz>37cplx|~t+KAH zxVyXj`e*T3w_KH#l;SF%ihi@NRF+k{!KkFF`t-)eYmvcK`?WdFK>vufr0=2oea8*n_rCSwu@8{HePFLsuHs2fmY$aE1|%&Y&|zudM;KG+wqw|iUm^}e7C z>4bG2YY$G*3_daU#g%z|QZZsymJ z9RJpYDc-&rkCkrz(R+1wkK&GwYp*hjCOvufIfP9%tN!)#&FAf2UtWGb{@*9@Re}Hi z{f&HgZm#vq*WCJhKG=&MZsSefS+eZTLT7fXcLB|wpY6U}a5mZa>t<)biz$XCJ32O< z_qP5yr-^mFrn5lHxw+P_^Ml#?6FWM#MEbfLJUVz!0&HwlTU>N z23~w|e|d0J+4b9x^t=QYuuYKt&BevFqIXeh?oFW|276LY3Z2^fa@p(^WxX9K#k(_n zd$kQW+39)7zxT4^PqqF1W^;HFOrJ94!u2UrL{>dm7rUG3-0qBv zN~iulo1K3n{8pXDHWjUf)3f5QJ`_7Gzq-HsQhMFVTNkEF>{(W_?DL~;eJy{f>@PYM zJkBZ6etTV}*vb1&>Z-8WC%9*mvRluBBdMpSZCsz3nOR~qY08u<3j%y*7(9H`t-oze z_obgAQ{PT>m*bq_Jde7vZgUw+y8lxMR}vhhebyp*r| zp}7937n{{B5hbmwJ2j$D2QT*vO*ifDz9cCS`s!+Ue5m}kFKeQ=zx(s~{PT)AwZGqP zUwP{Ca{u)ByLsljcQ4$zk>gxplZ1$@!_C0OYuD@$r7K&;9?ZdP@Dz)A37owR@VJ@1E4Y=i9Ap^ZPZ%bLVc& zxjAW#&q=Nqi%R6K@2ma&``zyMn@;N~$1Y6kowj-;&$}Bn*P`-M%Bm~p?f>gsl@06Rs|R(PdCl)Q>|3~GP3-QnJ39&&eqZw2MtSEv zC9nDE+v_E(J~Utda->r@_RpSwzg|n(T4(b(r#yJ{YnMjtvgNJ&4kS!f&3JxQrf+`5 zqfStF>r3$bfRxnKzrU{UPg~n!8e8)4P;1Dx)$4X?JqVl;C7LB^;%~r~na^yts<5!| zi@oK}XUE^~`RwOazBA@c_JOcn7r9ffEZ1BX>ZvY!-sUrp=f%C>@BRLGTwYvMG&5$( zIsIimGZ)R7s`2Oh{{Kr)3JDAUTqn}oXEw3Jt5nac_eN~_U03OCzgDa&yeMm379j26 z?vTayQvT~>nS~DN-&JO|^T}Ghp0Z_{X?T>=!xtA9mneDKpV*db^(a{8g~Dnd2lk9l zp||dN1Z2AE#O;YVxoDcPmR8rcd*1qcrrKBb%dVZ8Xn8@-ckR2y{dQi@BZbxd{=C_IUgPhp)$5l{Q@6B? z<*13(GG`8mC>j*e#yH0+8_1LW$SgxM?%e4Le6%HYMWG*U6W{Qx~v=jcFq27x8mv;^MX`c zIsUC$@pYALe~uqJb}6M~&MJS67vgf2PXs5)7+Ia(rIE{a#cJd6(iL|vub$QR;BD>! zx2=wPt1_2L?+sX4=WDq3xOjZcMAKSO`y=wbijlx2-M>3K+d@}|1#Wh*_rA^i{p+F^ z8E$TF!ksHZo-Ycz`|bZx@pz5MEb*mxy7hLwm=>Ma`N{eJg>XIFJ1_qu=4|i`gq~u&)e_Uoi2*rmIG?7 z?fdy`wx_4(o0KKa?R|9?iu(As+T(tzLo zJEwTe?lQ8@e)?ig-?@@2{C{4&f0Ge22Y_S!AXSBx&q^XR;C!P81yS3_gR8UE8# z|42FCHvTR6bY~N2>UN5dy!`r@^yKHM-|yG&4-$$J$o=p2-_dGo`2FAqawnG+2L=k% z%w#u_PjqP7tf==Yc9+Lz?xeuPj@glZetc_Bfw~{DxBoo1|L>{JafL-wJYrt`zn_a1 zDV?k@?cO(O@8ztj4+q)x_x~xfYF?n6;5@zJPU)xV!kMqmpI4o&7xC>yrT721|L0qp zSZB#w#$4VIz_N0s)ZTe>P20XM-{EvCyx>^6uN$+=6ubYkHQw4T`a7}fm|eiK^*V1) zPjPZr`kInse|Jk}aIKT$BQGw;GWXp(Q)8z@>?&D#Nqf$Y*XwqBybTHp`m&ho%JRKe z%~$aj1=ecc-O+k&vnHtZ>K(az+qaj~w`?)_CG}w@uPJB+Xj#LojIX&y`zG{F`W#bq zQgzE}v(^qTE%B*;wy!<;ZEADtqeY7}E;V-Q#NL#9F<&}wMaO=8?fd`gZgqO1F4F#4%5KhS!-XY*C#$}y zZ>{)RdHUbI1?SCwZZ@?qTJg)Hxl3@#vaEBj7X6aAHvh5crFVaf?M{C)J>zQaGokEB zX8n>sZx$P~xy46bJM4WqxBYJG^3L;8U0+{c4-UC`a`DmFkWc*L24!zQ`g#)WBmP@#_98(*O%ORSa6}vG08#QTE?4cO=n$* z&!ax;cMF6CvesWY|K{aF-`2TCuVc6P&NiFX*}Pk0F3ZLL*JI1)rs;h$zopN`{j@t! zN__9@b-Q0I@vOIPy!G>QNbQ%niifPfeq9On4_v(b#)7tX`MMb@jz3SaF@&7^xYKIi`UOx4~FOkcQ5i_f5)xAXF@L9*CS7b{dSx-nZG*RKKy<4mnVzrXk_H?`g$#T>T=Ju?-y~&^{eaZ_WHHRSHCgLI(6=I`6j+1^RWB(+FmUB z6}GF8d#!))89(zY?p&;9RdEM;|KHe{{AOO0(IL;^1vbi785b5jV76^-y5-7g^?A{H z5hc&3i@c}nU5%fRviICq^;F(gv&{-~6|QWso#OZLhg-wl#UbWNTmDMN?oB)%-*pKz zRunCsYdP1dbkW}C>%nUdWU*A93iNk%Wu5zETmJojo$B)xE+)NAcMyMjbMx}Fdk^$> zfx1>D%JvxnyEP(JPkK&xJz>d$oON^N$e2ZN6`oX`e&L^ltnAu-+=tKox~jZt*GsEr9*tJfFpP=ltWZsJ}n!0^_W` zke%{vzPl_lqz8%rH!jtN&a2ZOzxs=k2mfd^uZ8znZVmjo!AQ zb7f=I>$TfgeEk2gU4C2MU8xJMU-!y_azxX{)zclh7L*0v@?9|zG}yLVVnK<^q(9=< z%yMoBNK60z@wk7pV*HJpR@t9dGN6tGi#%(S&a?10W$|rRdRbr*A=$&Wb%sf%(9@vm z6RD!z$Gc-D&VBuI(L2fe#gQ7OFU4aD9B*&S{d!mK^QTY8dZm}|JmP-)Qe`PWdtB+& z&?{Lhi*GKLT4;a3etSje-`Fknek&GQWc!LT?aG)S(RQwBE^oob*q=7yvKuz3cuPH9 z(0p#|^|;ezmlPHz%$>m`lp^CA{Nl#O$E(-x%POhdzhXs4r~Gc|==tBa{nPW>wX_^m z&vbz5nPrp9XT82#e*bFtpU3_7W<`P9imu+R`60C7dX4bQ6~|mVd!@~TUfBJ3za4`7Hx*MRPZmyM{Pymyw{_rryV@pyx4fFCQ^U=UKA2x6p1PjtVov+-@9*cEmzk^qpzc zD#h+u-v1BO)RVn=^}y}pCbqdJ-|%$Lat9B5n=UhdQpg z&$ne-Dbt1WXQ(W{d_uW_T66187lv3 zPrH8=yg2*X){>S#Y8N(N3vpbZelS_W%5VFX)T+HMTmCC`=5sRb(%8Lc<-6@3b^@}p zcP|7){FJlJJ?7ZAq)8#B=%lI%W3|;%ySKTvu~X_A|9*;$-2yIl_N1Sm_xX>1@~h4N zex9$F+WF$y+1W1{7FHgaBJS9hpT#A6b@z%V^;f1jxu+)Rc_l8d&)f6S?P)H@ljJW` zg?8@I{`~j*{pFk66wL3h+*tc==kr+^dzkii3vJo)dj0-+DsyU=wteigez)W2v)P&H z>F0&|Wh{ec&ahp#;AIHY8%JB-uBeJzsV6R&=vS%De)MQj(A;@zqqch4iocGzzbVz* z^ZBCuJs;f;Jnwj+GkaU^ww;&wN^j?Gf0Xz=XQmU|k1tuYO~@cL$CvV>K8pSQgKv|H?=7xj#n1n<#5#a0q% zKIlOXOk z#c=L4-DtPdOMCZ+uaDbe_Hur=-Y$(V?XHrb-m3h#ooF*ul8HyEw))1u`DGQ zV|SOu79JJdGPmUIt*Pg1L9N|j#v95$Th3kxU-eIYevOm+|FXBYVnM}$mr3ArqXfOL z<{Gy&Gfug7i)m_Zxo*&?QXC-Eyjvr;$gAs^D`<68eE$BwWlOg+&DGu*?frlM-rby? zkNF;3yz{)cNTEr$i$lpvpKD*Ux~St?*N+!OV?Jn9X>M(I60G_-NsjB$!hJGF*(N>+ z%oGgD6_DESwN-0sRJ2rZl*z^;0m64)_Se2JxI5iy_gpWZee<0KHs72%_szd&Ki~g( z{`0n$sNvQZ@7}F*@_GFC=g&%MV=bqz5_>0a74?q&dygp~b@}hRQ_jVIi!!-0Z|>Zy zYLDXA*y(?hkd@E9yLZjDjT;lW8(w`ke4V;Hq+6?TEo;1n$gNwqHvPTs?jPE9YHH7m zD>oC;kGh$IkDc$Azo+w?U#@ubr`A?a(LAKI?bVAH8ag^5KQG?ByLac#m$huJ zTbjK8oBsUyQ&(5l(RJg`e}A5vzyGSP?Q(y|?`wPA8%?k6{xo&gk56aL8}d8|`15Md z>_4|JZnQp^Ul;z^({aH4Bm4!a|xTjw0QoV#oZ zS7KFFlxR~q%PaN#$Ie%0@xNevH`m45+L|lWerv`p>94{^i(X&T-TmXKnL_~2t=>f^ zvjXRGmpC*dEwtjj~2Q0tzEm;Fx>Fc$&*L~&keqKP*9=WSGZ_YF@ zytK@`tyb{vl`AHemOHm?TXt4gSC^NUcdyveaM=gX_6S{DPp@9T zF5Pwg{4M$XGR;#uuTB2sWMl+70eein`QJHrl@H80oH&^U4K+_w4N-rkR` z?UsggTU}gGY|Z=XM8@;Gw^_2Xv)8|Ap4;c~#r^N{pV2Rp7(pcN9Gj&K3{%$f3V!i6*<9G z6~RZ@PD@_=_Q_2B;E$8He+1ZCT5h})xcuw4Z{K$9vQk~rw0hB^L;kw6?zzd<2fle* zx?bw4R!ZoD#fuk9{Fsz+~;EUU_-zdJQKpos;nTC?WN$&-O|EsczX>O*TwOGD4ERNSu= zQoXL@nn8enby?Z8nKLCfx2?Q>^l0n1o%{FiU$x5X{QBuDZRO?V{VSRiS8ZJ({C1t0 znc1?!a!=3BbKZ%-mr8p~=F|E-pSkJ}&Owy?cE#XD~7_e2`!H!q&oK!7*Z|xTvr0 zf$9hGX|g42Q}a@aONvqxbGcIUQu33sxQt8`3>5N{vJ~`PQp+-vQ-j={;8NyLDH|IF zeJ7{<3WaC`1p@_Rb5k&m<e zX?~Qe{p$kP*3jNXtJj`$JN$HA`R;oj%GWQ%om88~w00_s{Z7#Zfsdm$pWk*}D=+-@ zC-1i2_2(M*y67HBDdId^u;Og6u=(|S+f;L|s&0td{>W)_%%!x=&FZP$I~!sg{G8TB z7i1<*eVn!W=AMnK&t51zX0CiS&v2W1+3L04k>8?wBQqN0kLf3<>KQw0uWY*<@d$<^@HDt>O(tp#(E z%)LXm{XX;W`~BN8eb>Zx|KmFUkF(xhFaP=$$zW%V)$L9v^lJF^_eUT4RrPo2+3Fv% z|D9YoHgDQ%{(3)y!>ez>o^o8KF2UaKC1*#8SItY-sNcD>+?fDXug>bm>EbNPD%AN4E&?o2nCf(>O(1IYs0Z zgr@hfSUpN*D%I3jpr^{z<;v0}#Ox~BI7xES90o7B36Bc=nGzT9^fW{nF4Evw5Z1S1 zbC%28tIDs|$OS&HaM|eLFjp>c3g?Qc%3Zx1mbi&XPjgTUIIV1D#Bt`KzJ9{F{H=c# zVkfb%&Is3CxO<0ac2bPK-)Zsd?8O8s^s5aS8u2G zcfXJBe){I$;r#IW*!=hR>dU`Oa`O<(tM*#n@mcZDqr+J!%fm8;y7_=u`@COPTf6^vJQKW4mqN zY+nD)?w);2T}?#&-rrw>i$x;1f3a(%EB*iZ>hiCv+y7VcJf8XH4@cW0c}H$#)8`kC zzqtNXcSXMc_vY58O)qcRW*E7xdBu59W!(-TnHf3L_|m?3#O^e(-JBcByeIBh(vfYN z%y+Mdi5@f$+wO3}ApK)+^hY*<(oM~}M zKKJA=XU~Vr$JgK96@T{l>Gs^ce+zznxRAxL#B-%Y#95g`FD`w&Wxjm={X6n^PT$In z*uHPW>-q;Ky25_`dctgn@9TWlm2J!Xd@}sS+{4|f>ZkS^vt|bRb z1*8jSb}d_8kRm;)jZOR8@{B79^Wxy3B@2rtxtUd~jpJ6G-OtjVfBhP?+Qs!!!E!N6x!TxM z7FA5iGN!9OzECpvx>V31y8C;E$i#$Z6S*h6T4Abs6eOb}>N{M$zpK3%M^-|yMR9W+8;oC1SZY)rf?S2svz?j_J zB_W-1f?-ucNIQ$DiO!;I$5!5%a>j2oqK;*WTAq5{r~M}BxSA<%q|5b5T=S--0r!DoO`jO$nB_T_u@>k z!e^4A;fD=+ujYtqhsJ33_s_jwUU;nDb&bU*CmxV5_L=!!fZ zLG3b~EtV>po??d=3z%A~Fo!!KssG9GRr#t}VN=j2RU zq7umijsY>&?u}8?Vv9F_o&{4_I%k~#pNjIT?kpR&{J_O3swb0P*h=-zI=W)hDZUBw z{KRH^OkjG!*zLMe!s?EHCYklnjcdtPj&*kY?Y-H|7gN)+5CC2cITJ63JD zJK1oROTJ3?#K3#qk!O=sCNGKL*?X3WJ$L1r6vMFMO`elPx)-jFT=3wS3Os1PCxva2 z*y&ttXdT&8cwyQpFR|pK8q%ts8zrjx-O6>`riPX^d2PZjMpT(u-AL(90k zbEB7MZ)i~A5}Rq&8#`C7;5@k~&G1l2h{?yYu7HPmr?teALtI_2gox>=9qWpo1`S%n z?@4oRhQ@qccfzRKQ0tywCx+%wt^pDM$_x}!K38Qj$ zO!0zk{D%zftX9-ie|wf#^Y+J&eex^i98OMu^{cXyPvF?!1wZmDiteb*%{+ek?ZfFM zPu>)-xw-vTQch0$_JH44|9$!V^z0*v>|d`|bxbSrG*FUkyO=v?TGQh>rd>*seeB#z zt%FoAc7(iW=UV;ijAfaKrLf-Sq6(K)m!oHLG*1kPX1O>`aLtigRblPEODa)o92co> zoha6|;HJ^NbW`^;wqBFdP8+z^Es(0#@YuE@C4IF=={COXDJ3~)r%c>n5iDyLVKCD- zRK#`BX3@}ytAT7sm7ecR4O`^C z@ln+syJ~zsX(;wQ zsrB`oSMox$AMV>yxT-jE=J{QfF?nLF52`wn!??T4lw~m>+(_$9>&$W_84r zh#kIRsXB9%^0$0u_TDYDc*6r`t34mP*pKx}3dLNE*psYuV3Ea*!b$8NQ;#HyZ7i6* zwUQ6NP z**eS1{vViDwwd)$&U6+9?Us<|XT{f*zrE-A|JNRK!*?8eK1A+t*5GO~u9tH<@J)G} z{Dq?(0!&UF9F4ElTlRJyzb-RxXD##Hz?xMD*I%1a_h0SF{h6!ExUU^I)BJJjP=2}m z`guRr`Auh=zsp9LUHRkM@b}OE9(n#tKx~syj=l7p%$_=0JN}qYkFZty5q2-nB6*GQ2o6UOX8E<^={^Y8MVtijh z8t*b5D3rUDu<_}~vwXYlV($I_!g{Y|()7$LwzJgN)$FO+n3*G>apa}Zqxl?#=6Q-c zeP-XO3p{ynmVHn6rN2s-SL~a6^|RdNa}M)W{zv`ekK)n;_1B=?69qAdU?UI_67*-mbm{-iD@0ypKpOTrEuAm>D zndg{SoCy;E^%;wbOPn(jixiBFK%K?B5|`BCc7K$3yL0? zZ?O(nK!+&Ijg8^s6^05Rj~RfBf#$1qAFTdO( zpkS!zz_74)PoUouQyUkimX0$Tj!#k#uvn-pFjz35p+mq>fvu&3r_4Yp>xtrmdxCq6 z9=Ni)tQRzvlbJi=&7IJ@osZ^NKHvR({pY*y!#1v7T^9Ow@Ac2$qeVGXTV5>bVfc8a zrlRWO(P^^l1s>gU;+Sny{4iPWQ$UiQVy8Lh!bK~79BhC7HOiu0z&vj9UjBcHo`HLk zlZ58IS611V`EAY1)^&3~n+rTzd2HXk%^Q`I+BZqAw?F!N%1wUzZ9inAeMI86Kl(b) z{>)suu6 zh-y8;e&&*{()~qR`l&)EGi-4!-b1$#%KL7sgq&a+=b8pp@ERNV-`r7K^^m%(+S7z*2 zz4u%)SKcl-@ZXtt+T!-VH|6p#e8Qn<5TvDOHc7JJgj(wjpMXTpEZ#1?)h@DG+qapS zr9NB}o^NJ4&pLVY^EA0r{kH^<7Mz=RWx9@au3hxDVuiEky80R?z7xE=eQ!wo{r4wU zt(eDIa3asTcFWrZAshB*s6Ag{<@RiA{6w(_HEKf6ZfuL5R9UYyXMdZWY+C)KHldua z(={+(YS&@TwdO|5`y2~Zm$>MKdR+}#8=`MIb%wXHm+sPuORoh*ckZ=XXL+tI`(UfZ zqQh1mXO+yfKdMT*o7G?5w|A=a_qVri_`H4O_BsCO_bY7E*Uz^q{rh^d`o5cLeqCxG z=UO-xXNr_w*tgJB?x5+l6MQkrOfRK7nyZSJgvq~-k(MmU?NcuMy&+=rnq&OFCOwl< z*O)GT`ReR3wbvh|XSQ=sIyGt4WQOpTV=rPvOs_PbPTcoIZ>8TA@tEqDGt=XYqHfFo z-Wyfawq^T#&C0FuwYH}%YXAPq%*pY2SaMOJ?%3YMs)uqhysG(&@9Q5lRyetS^4XqM zDJxpC4W6CnafzsGc>b9w?#%3+KFi%wZ`b6rH*b3U-gj=5zUyA=j&2e5ppMB|PhVxd ztC{(!%D7f}+dVmx^}Yp`GhcnIvH3q8w$D)3f@T^p5*`5_dQ#yUCBOvL*zR~fVqu#yPlL5wC-u;R(ntm>|cy4j*tt*u)xR;xAnqJu5f7z*4?y%kp zKGi37T_LxsF4TPcYN?)f=Ff}okJM%;CBSThm`< zEZ%)Iy>~YM+xvcp_N`V~x4=Ed2GtF)fqzH;JE zON7ZxeG&Jy5}s`#lFV!|do3C6-cIXxoVmy-_)~CKu-TH3$N$1kJ)QDRg!j5!QSRZ> zhuQZ2U2^#S(k+U|!(=R?{=aDYy8E?!?3TRp+P@2SsrJ8bHT4$%_apW8-<;pvxBISb z3ElA4dVclmquaQ8+S$`}T*I9upKXh86xMLo+9@X?keDWRIih^hDnnQK*JbKYUoKPG z{zT{Y|3EgUgHS*Y3Y}|H}QGPXheD ziLqbTEz1j(?Bhw_{$ibH?A^?HCiz}jJLB@S%%{8G;N_HD$?B%!q z-ipwQtS|FUW?wsV!(>I_b^rN(b>I12k1t;zVV#|y@P68PdkrfU-dp##zTQ#TC>ZCx zJ^gx)^_$PKXHU*7xxb>OKE>;4*W2Gb3J&}Ko2H3OlG|{?eQCHS^W%`ICjzeCvQn6H zJF2%O>nc;!zOV(N*)}qE8S%<3O^z!cgl&EE`WDZVCKHZFn!G;Lwo#DFeC5Pt5d|w>t~!22y6CD;a0A;70S*twy6wNLXDsnE z7QMg}#=;uJIDg@^S*Lv1ee^<#r(E~k>zB{>^<1-M?)MGOYfs-;xGgg1;JjtbhXm9w^H%-r{W z-|ubxKQAx5)9h`Zk^48{<-t1&yFWi!>aFe{v+K*+A3-&fXMBnJo|nJRH1!$Jt$wDd z8T%!Tvw3G<>y(e^%uyFy;;#^N&&e!WNrb6vrprsQ!xI#Dzf6>NYdgm)9d|a1>!v`k zm)g{smlOLxmL^*6eN!9Z`Lk>4>Q$??%yXx`jqhx{zP|h4^?gf^+wb?0o?T#fq-I|4 zjP(;$8&!_FNv4`bPW&vw_N1ZfWOO2DP`+xThZWyFsd;M5XD8kkJaMVq?XX1I>>i18 zZ_fVGI(*aMsbbiYDTc1gX1!VR@l$}6(?8j{8s``BpY(nb`o(kSq`ax+Gu9~ncQ}=B zxcR-v)=jJ2F5WR)K5_p3r_=UjcuwE9s#+i431vEJ@0b&)der6hF)DAwr-=qUi&%5r=Uw`rYy7=Yq{~TgeOZJI6 z{6i|KFrT2|csru&iJ8I(n+MN~Vxm@=9H%MydoLV2553u^}R9`zu_U&SIX zdsnO%+z_zyP!jhn>5@|+MpjP2zRu1eIxN4Kdt6%=JPv65?7aM7$3IC65w_`iYwv`| z&;Nd_X?y=>DQ3R19P+agOXl_SZ*3y)t zUl&>4|GT=v)yD7i%Nw8G|LFOASx!Lx$nr-`79S66>2K#>ZvCRU#O$u2PK$5tg$Vl( zZ&&bdnbf*^;s>LR7umKtnJYRioV%v1dtpCg?+^Xe%JW-4h{%3@z^HC&(&TyKS;ss7 zxig*>Ur4%XwyR@WN9f{b`+q+BIagG)W_Lw;ZS>b!7o*Gn%oJU-ocVC&ZkF05?6;iM z>n;g+^cK%fGU2YYbX&LmhRxG=%A8;AS`JDWcQ!^G*2@c!b>6k0>)agY>74F9kFU;N zYt9>$wm0$g)~8Z?o=!Y-fqVIlda<4)~x*8+O&kJym9$~=s6*8^uCEypPqE; zPU<|P{5Mx_&v-d^i+}s!GDC)Q(%(W<)t{)odGWON^mDstc^t4<>iMyV#C-2hDic>ynv1S2Xw^i1y;@Zq;{P?n&)0s-EbOq+h(@SRg?pk&# zhRJOsU&n@d(xJl76hJFb49+>ht=8Q z`*w+mi0l9P+Rh#1m@Rt7@YdR_6q!sG=Mz@7ys0#OX7=NutS*al*k(rO%PPxOU(}hp^kYdWlXepS zs?Rr-_BE}2SRZ1(No!wZ?Zlg=>rS3w*>y=YXz4neSif+lXEy^9;+R)2F|C=qC+;TK z?5R-|o$_J#g6#t9n$muFW=(sv{G4&iwu?EvzYH$RtxW%yfBW0!`{vu{u9z;nmj9x5 zuKDe6hVi%4-FVyYEMfcH-e}0oyw0zOGwRwU^M#6nA5$YZGV@kHS`)s$?2wNG_v16q zJ%#4J`mZ{(d`iK7&9~2zxBm8KJ^rigT50vIZ>Ou?=9d1rmIcZz2TGDp;gAKW<@!BpW#}l_bj~XhU`{3?m~kSp*A}g z7oT4?B5H)Pg)v#gIhPULhHMZ64p=dD=a{liwNBm0LJ%gX{O|Jag>-Wf{Ux@D8N)-E(IxG#E%y-4Y1?=OSN+nM${ zO_`-=_{t+}#`zV#*SFfpYEH_Updx%dp-?(&*OXQ;I8%0lX)iQTaTfw<#1$WIB z>m{pVx5kw)_ynkDZoXo9%crSQRxz z>2Ox)!{^gV-sPzZW&TuB`R!`+MxCj0$J6O*jET)$;+^yvJ|dVJXpuh}7o0Vmjg1}?`BG7{-4Aa{C{K1y$?5NzXZrd7wqDsG zV&y=0&J^oSZ5852)Nbr>agsdut0_0wW$WY2&Jg3ItCClRlh!DxOEo@oQRp^nI3vhp zWZ<;!W6b4^$q)5e*`DrR8}j)6#a%4f9;PM}Oyg!9P}k`9?*H7nuFJwzDQJ#}{?3X+ z$ter>DAetHy|G1fiA|pCAFk_NEw(BBr`z%#nXlS>Z^?OwDr>mIp$ zT>H@bt**v@a|w5@uoSVGp_85p@j5VZ7F?}*m-YJUh3|*g_+%ygK4RYK#k9g?>I#kZ zYyIzgHfk)|*(vJHv_0v{;iI?sd}nj9b$-6NXVaJ3+r5`_&42v7HSM#_Ugf-{%M0v; zUrH97nmWmFag^rXMa8Q$E!L;-C3`Ghctjxotj&u)hjb&J^w$~77p&8rH#g&Ss{MAp zBL|OOxV24ttD;AtOxe(+3uJ@-KD342fVq(m=bCdasR z(xIXw(xDT1gPcv)&0h0d>agXbS3A|J-{rFH?Vb3iE04|E@&)I@=s1<$l!UosOqtmZ>YHkeH>Gczp_k(QFfdclsh&0V?5Q)+H#P^T9$j$N&{U#> zH}1nmw!-G0t|v{W{eFHm_j1_$=hL=X+_*o<&--Z66j`TTxm>l<+XVS{N^9o?t<0YK z?dg`F`lGqVD|NKao$XXroON9F%N38N*iUWtBJ+)n5@$?kFX=0>F|zbIepvOAzIvUG z5YK$>){W=A%`Y9)xBLI6dr{-E{Hk9seGeU3@>)05Tf4mTS#)mv=HT4!^sz*%pmQe0;Gx5BB_J|O_-^rE7X08?MfAUoB=HiMF`AuH? zPbPniVB1hPK{@ra&(`hJvZYc!cpFXWOJU69;QwHw^X2oE1}TTW%JnxN%n+7L>a@Jy z)Wf;V+qj}JW`W3x)~JPX8Bs1`mxQ)Sh%e>b%X;TgaZo7TM z!+odocWKoR+6N!pU9VU1TjlQrxy0SKm%YDHxTS6ComKb3O*Tt@lh~OO_gi#FUv`Ib zGm~DBU%2+8Pk-Mws^qVy}rrys4C&Z&L*GW^`Uy_4p}&7S1M z9>l%%a@{_^JSX!-@ysuOT3H^LFvs)K`X^^rv>b}?Wfq(%EC2BNXW1COS{IwI|9=0e z@zw}ey5&8&P)62rzA^tQ)v&YEwU1y%l^nLqy>qnXTZ4+%+Wi{t5-F$Dka&&gq(w~Q$4@aN7-geFR*5?}sFYKyjkx{w# z$Slb0+giM8_n>A%6o5ua#(E8Z+*(@(pjpoZV<|{}`Ejc);HKuKIfcLtGIzoN*v(~7aE@8R; zXhE^3rgmcThUYBy-rFA8iNBxks{dCayu4+d%JY+A6K)i2nmf;Zr{CoL6Q>&VO;uWv zzF8^eKc~!VoqhYKo|(>J?_a7EVfF3dtHO1X3zL<;Jb3P0ZR{4wDe0c|WWk$&8jJ3Q z8}ELJy`_?`a`j2_4*6#XU3YnX@~mF+Fr_wvIl$rC;~UYvl5jf4l^5*_osWb6w4(g2P{&$)Q~$d$93kf%b^?C!n5x9s-#by7{s=(Us>aHK$g=p zC%&1@>8dIAmL63OqLKZ|txu-@{;>by?AuH91D<^?k^L=g$=z&d)V)M{MK%vl^VyH z=f2rw)?T{)-t+aEI$I~psT{pDOVy10 zEqiNUrsTb_n(cFBM@E{zqH)=T$_HwtdqnRSPnLQ->-I^(IM-**{I>tBG$+ni5?(U* zM*HKB-A@+O@XyqbS6}|&Z}*MxgX<ypp;1DmvZM84%r*;%p^d}o0%TMyH=TTLc<=wpPmiX=G zGqgU2|K<>WRFG4VyGvg9SJoB&KEcj=sj>~c_qX14?vS0o@$rwq%^P!__8)w9WJZZt zQk3q2DcPzj-*^-Yf8N-qreY@?z4c?ggHzNyw#Ob;GkYr%Lxj&S3hj83a`A^YYh>ht zztbAT*#F$+w@*LlANs+Z`;W$hWexS85A%!t$QCvE{qVQ?`$zY=cLcKD4|NRYuDQ+h zKl*_A-v_U4cIZ7=t+S)7RA+w}Yxu{5tCJs|nlAari@RnfV?4)?m;+CBe#Go%P*;)70RcO`!<*nH3WA#Vf-rmOa0$)4xrTKpfJPu@s76cqv&Gd>zMqtB! zfw+wa_U*0Pw_f1ewOc>q?62HTotwC)D?1Kvz!+&5C&r5N?!ZTTR+Ibk_#oJLDl z&5~VQWgIvD?U|V`{$Y>gHe2!I+eHr*N3Z>})8a-}YkKJorG(;8misyjw3xI$EOtPyKUlW=8C|hR3Ef&!mfq@)PSF9|uU$g1pey)g}hmM7FMI<&qWje(&g^Od0 z{l*;0dZz=MQa5Z7@H%E$*(on4zQN@{61R@gVJBHJj*2y`&$b^}tZT5IwTtzP=!cF5 z`-+F(nV#MF^kdG(4-*=0o_^rCM?=WK+hJkDOr|uB8Ta_Pq8>0B`-mF29!O!FtoEuV zh}-AN!}5!Tf65~b91m2?sjQLfDLxdD^)xv-v4{DcwB*tE!V~hF>IDsc#c9jyK8Vx4 zqnsdePf|7E35T}4B&)KJ@(&Jep8keJrnazy6SO3B9^_qXzJHggGXFsAVg-eS2@Q?q zw|+XX-?3DfbU(A5@vria28U{uFZ~U5Ohv3;-adX%8@<3kU;F*zhI^K3_il4ps=b)X za#L_ZX?tfn=Z?cHGbPpLNis^Ry}0T8*mms&=jL~j6YiZof0=t>^sKq%pPgsP?ppS% z^7?LttNgKl)b6PY{ZaD}d>}kQhBI!;N~y2qs+I`{V-qtZ&RY zoqoJIDb9s_KYw@4TEJSh!BmR(q@Ztj|L^+W7HfA^`EIuVu<#-zCit^d>O#yV-2vt5W~N{G+|6llz?avx%2I&P`jO ze(=JUMT!*)A0I3zblu#TrfidCx$-LC1NMgxk9W#dx@mR+)M}Fy-3tQ#2>o|U{e9~?%l+R)-YFiWa@#TW@rO#Jf_0o0n_o%w0;ICZC zV`Xb58a4U58v9DFb2HgK9W3a0vc{N? zJM`g^$tn4hrfZ4ic^r{W3s05qNHzZE;~(Y#amPM z?%S|xirUAMEN68X*gQ9Vxe&fwM%9ry`uqm9s}GE=GJmDd$twL)yFRr-)$}X(dQphddDGLc{)h&|8%~)owpM8D$+_xq3f2LmEH~VaTkCdG0{)p_K*F7y~>gNAC`ze2` z=z~df7Il=Ft`RnkJ4eritj`t4=#U6Y*Wpa0!-Q$$ox1(>Z6F+S5+wA{^ zl4~8?`Ia2AkLKLj^l!_)-<#gHY&(7Q!sYKry%G;k(KTCWaO=+3+5P+FXUiBe%bkf- ztI1uv)y~a|?NGt2=8`My{#N?YZlQ95fs0G`L_2$wrY~G~jF-`=Wx`ga(ig8!6+~PN z>k5dGuijT%TK4>R=K8$;g~b-xT*r3L*5cw_l*IC}>EtTua^-b-Oy7>MDZls5$>KZT z+}kPR_O^aYq@~pQBBQvZng>GN$7(h@Op@8=CJ=4CmFw|>M=!J@UCS&kA3BjMd3f=} z1?%!9*<5)==0DC_b@AWl4&!roW#)3Xp5ivHyZFzNd+FQEoi_sdujSR>$6Yx3mT_#9ywoIXtn)<^~s^$-|SM-v6D=HQm5% z^Nuiw;6=0U#?9KLz4%U*@6MR_x+URmcbl^coL?@z)F^JdV0G)Ms|=6(`HJTp-&ZBJ z?Az7n&*T#vyQ(ou6wqoYSc&3VLC)SBT)7^xm<(JJz$|e>EG7$$2n-BDSxg36*pr%< zl9*QlTHOU&)&*KU6{HZ0yw1!sB{i=kv!qhj19`nzWkIR|@?tR)&~l(4*mfU~aiE$H zwpIvaEKx3U_H+rZEG|jS^~_7l2k%JoFG@)*0xfvcfH_1Hw8<$wv$&+FQbEHpB|j-u z6Xddjf}GS`(6TD<@*=c_OA5xI6?Q=&b@xqMHenrfhU1d3zZZ!*tq#V2?%OVO}IQ!iG^v~v;|uYb0;3xIy!C9f~CHKx}PL2 z`F>}KxV7%;zqhX}zt5@OSNrYW^WWcgJ)h*}a&b$KL)y=#pP64Wit3g2c6mJRS|z#p z`L}3;$tMJSs?*Ly#RLjhe@KnYpZa(5)XU4+gvvwKD+hTdon9@{{8)AMk5x$rXHMPx zT)*h{O3x2U2fsxYpIR=o(BZ@6rn5JL#X}`Sr|oPq=kQxS%Y1Lvg0IGrt7?t3JePCv zNr`4H->^C8xzp~#Zk_F0G`5DApSWJKX{-KB^=SWX-#SZ|?%wlq-50LrTdOACD4WKKY4D&puixH^rA6|n3wiuvy5mgkqZe~-*m;pq$Sb4)FNz3T3qIahMu zeEJpeQ=<00<2JW>0W1D-oX1O++{l}KEOgraTYZmn_gOD4%XuvK zcTz~#mJYj6^Ywp>2%r3d*0%jMI(I{k(4 zkxzYD3(sD9exj~>o3FLv8@|lbFGcn;ILrIp-V(fM^~`k_i$q>~KaD+9Z)@i3s9^G>JQo_dw<0n%*j+SM*Zw)(L zb<0ovgYTK6k-4oC6Zd5CEWc$f>L7X5>)qLj65g%bBDT2yn&eSdyD)Is)r+c&&sg~u z$#-i^GXDNj*d>rvcG|`ZNnOhtZ*i>=TQ>1-O4i0clNup?$u*{nW@YddWiDGYEx`8e z(j1nRVMbeDZ+)^lTWIH13;kKyTeto)&2pT0XBXGwO;={wxHv9i4(m2rkbOn_EYHh( zf{g(We@xmEKW$n1_h05S(xNUqndm)!eFz_|%~=65{qxo=c=_=o9Y)sFA8g0EEf zJQMkIs=r(^e|^Eq?UM_1gd zdRv}nKs57>O*ccLg*MzgvW%~yaiigj0BK1__IcA6Ja4|S;oeHKlH*nCbppqVUrSvr z?kU|Gd}Wv0n&`AEcCSOX1_wlE*t|X(b%^kasJU_ZmX$d0;`|Q%`-|UkE8CwB z`6>70>8+4?oiWnCmMR=)tdF1l$@tf;3&+!gYh^y%ykKn|_0?>mY;Rm$%GWQe)-Y6k zD_A`*?n}YqYV-B4>?H2)dGTuN%5{bxZcR9OBB7+yan}oPF@Tr`BXev zCrflDmdG`+Fmm?zF_uj6Dor`!9ZTt=iAbnB%+$dA~uGd{#BpxWxv{o z_~=jej7R=(ccr55e+Rb+^OLf`tvI|b2O~?Arb6t5gIPxu1lr!)9hE8P@Aj zl(F69EwhAd@IuMdB)-but0K2uU~IkrbN|EgC#4mVC)6jrmpRkHIY<5Uj<#OjLoJ)t zFP0_j^(+s~-~2)&rR{jF?+%_PO)sh^$nV^#cJJYroWm{Yj0~TSiQT-+vwZ#Q3Y*^O z9{U{%su?Xo2^XIhuD9CvlGQQnhtZ8Y-!u28ygC1Ir+dq~SKm5PnvR-gHLuBZyJ)^s zPA&+nI`>j`;oSA?2>Rqw#S8P_U+U0CIb?@#|Q$9U)ciC+Cyrfj$G>r2Jr~K+} zMX^=$;@K}2U&ah9Bcy#Lh&Fqmzh7XG4@#RIw%>!)4In0&=wYyUI3OPLxo-d$Qqb$@#SSR6GH<; zFTldm4BQ!m_X6HVR@R8R&RhQ~K79WBudnMW^~*oSop9YDB+S&Jo%@7srb0UlJFkF} zkrhiIAG6w_2@FjN6Be9sXyfc?yx75`#-Sm_$S!eL@J?cTWJtmV)1+gr$2&b!>;F#D zIJfcl^SW=JZ@#+ndfls6R(03EM~gafDoSapJ2Vx`%?d5jOnNTc;-b;QVf;*|PQ^p6 z;1&$P+UY`>hil)hJNTW_tvY5VGegL5KV&gC60;+}Th#w0tl^T(7e z6DI6f8o@Ws#%6m;$Dj0?p6WIMvwn^xGs3%%t`SSuo}6)8wa$A<{>nqASSBtppPRq0 zcgFMC=grewB00l~nx+QmOxoruaJj5E{PZ#Ncsre?f?oF&U1}!9Dp5^83a7wm#>$ zzlAql{MC;YyS{0cT7-IumBlo^PSWgD+F8c4azVDs;zN7gvbP*L*1?utyX8&kWW#+E zFQ(nT>XUvfypW^rbKB3Wy*$z1kIXQ9YB)P7D@kIm=ik2>N&F|@bDuvYV{joqp)o?u z^8`=(YvKBN|4*Hqy#C4mnn!c1)3$x)&xqIZy{nrh|KjfNQ#*^ptG=)E_F8m!rQ_a` z0;O}_3hD(#r&Z!T%J$|uTj}ld>Er6rQj%OcBiDJhapA%6+p`MAmSwH}nYA!#(IcHh zo7T8!pPpZ_h;eenkKEoTzpuRHzpgp$p;fI)+msugNBpmHrSg8>?-04Td!N|N7r6y% z`+Glfcf~6{W?i42qQmtqt$6a&voB-5g??s>GtWeH*ITPSG@k5 zy=nEiYWs)gKQ(7v`(`**y2Unoo4M4Xy~cg>U)}$IG_LlA!m5h~RqX$hFK=_!e^*jc z&XT*BzBs-WO?(z(wENb@8x7|+aqWx+0OO% z{V&!z;-^;CcP-vB;X>PEnZrjHwLLw(n$N`S+1^htIww2j{_by`ySq$oeeQY@?)W*; zUs(IBd;Fav#f124FYR*CV!tDpspo zlH2O8nB39xo!X!wcX9FTXForFy*1_I*HcrfmzghGdb^qH*!ADbEY8d3^JrycZBlZw zI*=M+8OC?2{?C(_n@-wI_g$)8ZoB!svJrPxz;0J1pNSy_?b(8Pm*>7XSJr>EZi#Qn z`GE9{&s)99cUE`bUM)DMdK8*Hcwj<9R7NBZ1&yk-mUY}uWIc*we{|ecPp=E$Nuj9JFn_>?CVWe6VJ}|Ofzc! zw(Mp`>dq-?D^0v_7!|($q|5I&W75F(b^SzYw2u@ z&l^~>QX_v%{=sM3`bzEICAaN_1{`2Ct!}R}lUE9`8<}f?7pOb~Er|?~xCtMyY!*I&61r@eoPe^7N|vvDUO% zFIhIE9alZOOj@#?`|;_=*^dr>^z2Ev>Gws%sZeQ7*?IoIB4y?#Jav^nXYyStO#6BI z?ecl|_f+2BcXsyt_QQwv*5}>}H|g$Mz!j&QR``GB>+9ze++_6+NYrzsPWM`O+xf2E z!3*l_k3>T!Y(3fNP&04YadsE`WB;};T7PK2*`?0ECnbJYAIq*%a8oNks?oOQM^(?8 z%x$KM39&~xW=x)ZnU^izGU0*C)Cran^A~SEwJ?4Q=cHFRjyP|OS|j)RReR&c{(Hww zU1S&WUToj-MS*is``0T~ul63gRQvK@+(X;P)ohoVGC4h0ibl1@JzP~#Evqfc;yJmb zpi!FVLjz}^0I#8d_OX>KUuN#i{JwPR(y7aHy;gf>dv?1tNByuBW6iMn`g_R{In9+f z7fSRr`7HmNQTCrvBf6sK(LTxS>*wyxy1nJO{{HzNuQeZ4PhT5jU%r3u|C#*t(KQ}t zYB#2z*9?BMk<)E``t^4fb7oZ7Snp1_^y5#8f6&!abJ-txdb~fu{vtXliDPw=;Oel> z>XTYWv;;h#lt;8`otNl1@T4L5^zRuK;=+|D*!LgbCAWT3E>q|U4Z++aKQ#`Q%|4U% zbIpXQ8^lVE3mj0~;kj?WS$hs&^2eSD@n2l_d-N0?U3sr-!v{BhlVvJqOXNbOZI3T{ z9JHw8k(6%b-h&e}`-FC%S^qL?IpZg8zEw{bhm&;L07=kksp;lI9EefwowoOb!3$O;ek^YenApI9)jUUQ09+#@yS-2N{-H`8{c zGR1YaWVTgi*3`~mW}n5({`j@+!gO}E36ooxguJwyy_)Vn@AdEz-1NJeoolJ6cKF(L zE7#rH<@@PhC;$Gsx%@l7N!tE=n0o6<&*Q6)D!82ASox-rWm_erK%@R`ryx7RJY68J;%isPP_$2**I=JYs;#LP4;2xydbyTV{-$itS> zetpV{-65rF7B+epy(S#dzm-2HbAO}Ay01*H?;d4*VsWLO-{`R4#64L{+C2(4-4B%% zZ)uycIB=PdPX8j0pVJWZfcKFuQ z?)|@RC<^Ys@o)l1;`4XUJ<;?dhC zx`sohGuP)xfyhH8L4lVAvluk?P1M}cv-$bOK5N01FJGNC>v2<`xVW^C<@a*QLdlPk zjXeo&PHz2oLafjAZD%gK;J2ar$?>PhpR<2z{>k~%_EWHTQ26oCW8rLC%~Ls7SLb-G z_009`egALE@5S%-82hR;E3vKq<1jy<5cg(5ep2f46d3RGqmvDxV z)#B}QWQC>{{Q1;_%pY)efb$%yT|s?Eo&ZKy4kHK=QaDn zKbP2#YlL_A?|rCSy(xbGt}9o}+{3!9)@|rsquSEk+0h&t7xc91*1>}s{$ega&z=zQ zsc&!I;x2yo$o(qK}gly6-z@zs%Y5t>sG@OR~T2tFs4pKP>Yxzh^4(`lX?TRC|QERZG=k znIosqym5KH`tF4E)pKWjPu$~?u2!G!X<^BDv@DxpQMZ&d`~H}!X(c6dbZa9eT2_4D zGx_`0{Ti92uA6@Z{^N~vPrrD6m!rN&xc0?$WsBDYoqQa-L;XlO>gBg^=(+xxPs<>Qb9uys52@i`?{mWB>oD|0l-J|LGaw#It?`W^58 zF72N?{~zbq-aYDF_t&gm9qlwOl)+b+6g308mg zJIB_m*MCdmW&Hchj+1aShXz3`|=eV-ppN( z1WMl;8MMYC#(CSjcPXz07Ye*@`QA~#Bikuh*HEUemeD*nqtfntCour9!qK!M9ba(z6_YLOk_pz)Ss`C zQfK~Rxyumx#qvYzjlEv7Ulw&LHr=1TZb9mn_qX@F;n8=Vc7NFc^R;^;7xZ>4`MP+5 z-yGw9-W_GE<=dLydPJp8()!A|ex+uUN5mBC>q;9}N!+?orvGtS@r0z;a`gx1zutfT z|D^e?GfLF+{rn2vxV-6od|9{V@VB?pKNPBdMXP_RJzanJ((U_qV zO*OXg<*Zd6J}!G-7!}Tbe%`I*?Odb(8;$nuF1NG#W4UCuzN%vq)4Qsa!c# zkZAFGeQJhqi*@9@v~1QxH4oL_)U6HBcD){d?o`CiFx`-i7Bg2cMRjNx9^!cS%;kyH zCa;jd%P~hf*!I5&Ulg-UP((N)fa{c_YM}3>NnaNTnzV5>D@LD5l03y=>VCd7Rbuf8 z4J*G5M<<@}{9G`3#_}6oYx^#|E4kU0>mz=FE#{}&#sd2@Lg|~GRtD&WbX`$e>k;oH zb~z|fQ%6uf@${0^PeMywB4240Pn8sPQe7i>ebd!fR<(;a`rb+1lXBGytvyJ5Pvc9g=Y zbG@BAcC&grlt$bL+8okrWghDsx=Kt-=&$DzjfK|(Z_V{@QrfwzU){!Uhw)r_ZIRDQ zqWQFzocnP-rF8X=`I$fR@Alt1pRs6#Yj>TltF+p^Wxk^Cr!CJudMoMIj2#hmF1Hod zU706Z-g*1U*;{J4`n8TVo_Cc02d%!CWGWzeYgu1ZN}Xwxf3wK6HQz2Z=ZOgOr2I{F z&zF3+(V(k08SI_eqpKjPrr2HzBX2 z#6V!~5ASWld&QMk9Tef;XIfqqaHmvvej#XI)t^Pl$MxQ}!5^_Ps85dgx}gNBFaI zdXj|WVUPdv^Ez{XA5Tv@dpA9ao!k2O-l?~u{vYm~@g)2HOPS8nKJLd8xFb~#fAf6t zej49Yx8|ylYbqr>1LM+fi&|~obMsEgzcs}j6&m(A;WLW4u6y$BdwxmKK+j;?`M&29 zzsf|)Ueg!jt6Xb*{5HQ*y579Q5-(#k1)iJSadk;ly7zcuE2E)@^=W}LVxv)D!abNt!uXa55pVxfSov+c~jN(W1Zfo{97;nr!0|QmT|||^7%GDE_Y6A&2REG;%tlP^`iVc*T`PQX;Q#!8dKYspIh=b?+H_^2Y>ED~u8mcS*RF7| zpUnECacz(5d4;F1x=X7)xhv(P!zLLAs0&&3r6lSqiqCY&yQKL<;M+v!S`&6<13~q! z_9MHGuzlQB(Z?eq-ue89Q=#D5rn@;(x4SLb?}_c{_|JIBL4UIA6OJ0L$bt|VMX4u7 zbDFJ+c-;%XRJY}*Y+WMq{Lx}=kC_2lcUn!C->}>yyPoyOKbfePzb$qvmx%6kn8z3> z;aql!d&fHCC)u00mT{!M*c|C(c<#T*w^F|S_x;2a5ASd=J=A(HqJ8u6jcM(5j~D3g z7T6Va;k4IDksxhT4?9ai_d=Cz)AzZ~UaGsKZ4bYFTk8k*T~R(F-#e2}9P{IJP`ueu zn_s?fVwdKkkQZ8|RpQe+TDSDoXo;QLyz%|^xuOqew72Xoa6S4&|?n=|=%=gS|q-xSs#2o>JMu<2w?jh3qZq~<01A(MM=1ZV8MgI(v1zi0~u zIGO!vcer@;-S2q2jM)hyE2eMgD7YsY{UN-r<9OnX2lYP}luio#P;RKssJmr`8ii)r z&W7uc2XYzymn-CZeS_tU?vC(?X6;`r({F6n78hLf?O@Neh4)lk4sYuFx=wS3{gDL` zecAC{lD|JBm~G6@7I?no{l8C}KJSlW(=958#b4*8#~17fT$Ym9~ctk?*~=kAAp@9$20U;HDM=MQ)LcZ)rz zA8wa97kc=n@y9nh?ZzKJ%=W2hKYa44n6)H#e!H`9`*w>xCm*u3n_KKT_t2s}S*EY3 zqW7>Mx0LpzNagmn@c3k2nK)aU$$NiE?TONp`&VjYZFBiu!{Kc|UZ+?bc{nx8;>5#L zzOOQIk3J*`{yEja{Co1vMx`6^&r^R-UU-gM|G|>@jMiG2dCxaYKP;zzrRc}gyym|~ z!e!DQx|LQOR5QK0q5JT;`MW24Ja9gC!{ePA+ZJ=oyPdb``J#l~oaeJX@U-V%&D+-N zyj6PDhsCS1ZtOam5M6ypjM;4aq06)5lKqQsY$;T`{Y5HAOWfRU>KU_hExp@*gc#*K z*zoUUU5;(L`tO5QG0%40PJAmhckS1MQcDaY0wY;OHCA?|MC2YkZ?PsSb7QDXSkwoW z_9DJinKymM!`CGA7e{PlnkMe}Qp8@{x9zxv&f^VN8dUWn)*M_cBbV9gy7ohXwoXo~ z5FfY2pDBl%IX?*VeO_y0R&Qi;An_j84tegQGS7-M9vQSB&f(k4=A5W~=XKLenRgEj znw<}p^onjdRKoj?uRp0g{g4S?aN(1cl?F%oC$BuI=6)#FZib0okME-i>ks!^`0Tly zB5Zjl^AHQ)37YE&bB!-#M}4^H!mz}XK&k)XwWXh*Ua|o#)2>jivualwFU~&>C8SS zt}U#$N!h9R*=%lV(GK7Co|3t#I}~Hv>uN) zX9X_qKpCZO1twL|gNictT>VV;?DFjfd~rDsp6;-Jb$X4BL3qP*4j#U|aEs#Ng2e1_ zix)B`!XbQIX2BLG>?U+M9GR{Z)gZq?%wkdd$(DT%z6vrK%(_u~8^8VAKjFmxZ!G_A zi+qn~()z8BIsXTmY9nTX#n>EtXg7RDC3cc;-eCoS*6(wgQg?OaBpdXmM6SBjEB-g| zvfP12#jPR9EpuD*Cx&rXF0>(x#cVCd^VMOkiMg5 zUTD+pi)*&8|C7H=af1@?{B5bNE5GNZ<(}^DFWIrC@bPYS>-i5G3(s|L&rt09cc?u6 zxbRlL>jHHNO#44gDwW;-VA9>w0&(q}6%*UP@AZBDc-Py@H|9So=04#1Z=J@ZnYucO z1_JE$jMt{69*=%*#dhzX^=j7MIdgx?DP=G9y0posb*t}O%c)1$A2Azc&RWBmIC}#lluEwu~=)3XzH_&N%tn)Q^-#0o#lSXM(OfiruU&w)?e7a zamn@*qJ^zDHTsX(%zAgJV)`N*uFo8^L@lDZPpMsK-{ClI$;1_bPZn?De;P6)IM}b( z(lqt5rS#X?cht{^6gY18eX#0-@7?-W#?+2w^1 zb;vWf&iSQzB?{(T`rer-pyiYb1`0t63gFY`Ks0!Xwt|8oXaQl6f`Sn!(t{KfjKNd0 z3JNBm)uBNOpyijosVSL>POyXFO-zjyj73J@DW zY-0sOkPCwpK$DmHq#frDJGLEcMTEbRp@I>3nWqAmKIpJ{1tW-uKs;kp1!JfKpl8Yl z<%7?i2dxqX9|s?hn4Ve;as+e+wW26Bjmtp6#FEPZ4iwBxO^rFvLtO%`wEx%rV?zXl`kMZm*Gn0ft))3``A-(CjrZFvD<*fuRZLP(su& zFf=tmx5L251QdCw>Wo0AC4z(u;9+WHXl`MNX0MTv0fs+}j0`N%)mfSwn_M#`j%u%=fq|s~nmUje==?`y^9&6PO+YJA(8SEl(cEWfU}*q4 zcM@5hfq{vM0b2SqFfcJSLJNCC14~m2Gc@-Z8kkw4t1~n(M^8hBh6a|PwaLi#8X6iJ zg4SN3i5Xj9m}hK);a@{zj5snhG%><(v!RIzdVClf8CaU4g|mTysUc<>GBv_XLuQ7^ z;a5_Wn3@LHYS53gBXdOFuZXDiu@$*x7Lvmn0UIfXf6! QBSUjTBQ8}{SARDy0KqwSod5s; diff --git a/garlic/doc/slides/Makefile b/garlic/doc/slides/Makefile deleted file mode 100644 index 5dfe6b1..0000000 --- a/garlic/doc/slides/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -GENFIG=scaling-region.svg.tk.pdf \ - sat.png.tk.pdf \ - scalability.svg.tk.pdf -GENPDF=2.pdf 3.pdf - -all: $(GENPDF) - -keep_figs: $(GENFIG) - -%.svg.pdf: %.svg Makefile - inkscape $< --export-pdf=$@ - -%.png.pdf: %.png Makefile - gm convert $< -density 30 $@ - -%.tk.pdf: %.pdf Makefile - pdftk $< output $@ - -%.pdf: %.mm $(GENFIG) - groff -Tpdf -e -t -p -P-p12c,16c -mm $< > $@ - -killall -HUP mupdf - -watch: - while [ 1 ]; do inotifywait -e modify *; make; done - -.PRECIOUS: *.svg.pdf *.tk.pdf - -clean: - rm -f $(GENFIG) $(GENPDF) diff --git a/garlic/doc/slides/overview.svg b/garlic/doc/slides/overview.svg deleted file mode 100644 index e36e047..0000000 --- a/garlic/doc/slides/overview.svg +++ /dev/null @@ -1,479 +0,0 @@ - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - Garlic overview - bscpkgs - - Isolation - Apps - Postprocess - Replicable - - - - - - - - - - - - - - - - - - Filesystem isolation ensures the appsalways use the same software libraries - Zero human intervention, all datapostprocessing scripts are included - Different team, same results onthe same system (ACM) - Repository with BSC and custompackages - nix - Backwards tracking - Every result is hashed by the softwareused to generate it (datasets, plots ...) - Reproducible builds from source usingthe nix package manager - Several benchmark programs withmultiple programming models - nixpkgs - Large repository of common packages,maintaned by the community - Reproducible - Different team, same results ona different system (ACM). - Is a complete benchmark framework(green = not mature yet) - - diff --git a/garlic/doc/slides/sat.png b/garlic/doc/slides/sat.png deleted file mode 100644 index bca6bed66d6e30a8ea0b5814d4d1a85e5df78e05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 79137 zcmeAS@N?(olHy`uVBq!ia0y~yU~U0n4kiW$hBBRu90mpk&H|6fVg?3rK@es>o=s01(vlt|uBGDEIk~ZRjYERqIa#DR z3=dB)(FM8XdI9I+T^T033LZLzhW6^I{Ee;ocvNqC_Qgf6ZsJTbX-5SF1q0&`Yxgky z`1ASv{}1i<60xqoxY?TH)|-oPNyc()n4Y8~=3NN_J5Qzg>u%u7pVfx)9kNsF@;m6W9RwS`OUeY5HGrcIle8V`hq$EKF(mT1hHJ-b&g zaea+lyor?Qt}aHIZLNZWf{8m?FMj&;srvn1@dNk&ecNvT_oYAk?{8DJ!_UpLwYImn zS5{8`^W)?7`1;!P^yl}|=GA_?>FVlgHhb-oB`NRj?9>ilCsUASA+zuGy4}nC=DsSh zC@L;aKR4&+o6YBsosHa@HTB;L-_X#|M}m2KzlJHc+q#@bXn2T_G}+&-^5vzajSdQh%cgwlRG(+@|IcT2|9Nv}&WwzUt2@N0 zey8^P-Q3*Vjh546%Vr8ZdOAIx?ZXjae-1?>Bcm_V?S4KH{@YwqT6$F0QiSWJ{@TdR zZt^QvufFWgP*PG7Q~7l2&!0cj=T)X{>iYTX_4@DM7%n%~)aK^q{<`hF@WL$9>^{3+ z8OtxPjQ=&?w)$KAzfaSdBY$SE+v)b^^7Z)oxw8(L<=*--dH$azixwTq+uko>nDp(< zP5pg8l$3P@1(!C-$bI_!nek5D@3*s-otMgbqv%kCT)(oPD+`mwl7b_xttx z<$gTskv3nqdbRetTa*3mDi8FBMMXsg2QT*D{jg11OH0eh*qG5`?zyh6E>2;!7y7k- zJ|6$|=iIq-JB*qRF8Fup)z#HsC*HahRl8*O?&@1xGJnnQxBK-XMRarFW4GG6+}yW{ zHMMKkuKo4n_4@sG4`P>lPuIKq{?YsIGgtLWn=f0iA@%gMqefNn#%X6Vva@f$U#_5} z^vkR*R$^*U-n~6H(>62yxY%DOVxF6yc;#sk?{l0ir8wC?sh zoAU1N(v9A>CU*C?XS4IKZGO)x9y4Lbq>aBy+r`}U_kIcLK3a6y*L-c~Ht;sCfLH4T*kX5G4L?DF-03je<4xB1|3|8Q7n=+x=cU+W+Jbb0mqeOwOvm)$A<#A|+s zVZoj~F_*VEM@*B>-y?Y7#>V8s4-4*pJgv8z=fDiZ2?fvRluKcklXq zZ@Wv>zF62U#Q$6W|0jKhIeyFk@&DX(}DCQCl)DF8cNB*N&6x_k7}F*nM|i#gip17b`zMJL>D_W!RbcWx>9E z5znl@-zk2qwtCg7_^)-3I@O;z?s~iJHiJgjr+d}!kFv3^-_R~!C*k1d=XXa}a+mQr z3+4lBA~*AW3ll;~7S3<#`uutRf0>5s zudm8CU6eQf`S1Jw`LkzBn;Yxv=il2?$zYayi$y_#=as+mOZSfAmrJJ~yL)*0#EA>% z8$@qRa?Sa$xZiHo%9V@@;*Z#~{HkS=ulwb2rmE0;6U_Y^i?WR-Zy$Mx&iZ){BF|7m{g@3(IKeGy)k`Skc$>r$(^>|YE!Iy(3+ec)ZpFij`&)2;0F zZ}07`et4)gzUE_VUG%+%N9PV4J(Bk>Co_|A!F>6^(R)S~W*K5(@@iiM)tNYRe5^ett~I-RlmD={LHaq$7=gu_uIeTs32zY_=WVg z*At!gUl(?$t^M0Dxz;W|Y3nb&*}ta#y1F{t+|-m)QA{UdL;tDor=J$xt^Iy?>(;IF z>;L_{wA8!)L$myoCr?anaJ!l0nUP z7R!UJ*WTaF*B^ftp)-wBSnWukK>F2Hq0Ha*+B~0Aye2GLL7?Jsulbkb z$M}CbLM3y>xi8Om9zKHNO3#-nymMHPVu=D6O|wLT`uQe#yn+* z-TVFj|26Kn-Fftg%#Z8ST~>#$|95qL9qWTx+3RlJ-fmgZ`toM^(^FG7-re+TmT`LD z>8HQ8*ZjSMexCMbFJU+`~7b3U#`&5-~S|b|NeZp{Qg_(dsnV(NjW)*dA*J1 zQ5CBd>-9CWva>-2X|J2V)ypO9auo_+TVo54in6k@%Jn}##I4_x&G7q@xBlM0-)_gP z|K4u@=b@s^%#My14!rNq1-AF@sC5%QI_qYF{_h?p4GoQo4+q)pe?FOPe!u4Ps@3Z* zv8(&c5a@JC`kZVl{rt?#%}GbQd}o{O{d7uuqijKppEdDyKcB{y-+fwO@kP#$;o910 zaqCUMYjtxY_xV&VJXn9kolb>D8L7pAH& zoxx~rAp7IG`0aDWZ*Ohw~dlc;I}|gS{`6 zCwAx<8=uy50Uy;;%0+Z{EB)bLPy+>i+-U=GXsz z*e-vpto+i=x8LmleeBQQ`}Nu!%VI8WZe20&#Kea;)93Hay1J@ezV1fe_V06Tik_SR zh0^ZFeb&o-Wh5iWgs43V>r#?@ssDz&(EiL}FeNUI0%|0t% z^Wos<^Y-<}B-49jt*_;6pFV9GDB?h!lCrnAN^fN@Kbm$cXWRb&f8Tpg)3N+^BYE?t zO)+}cIo0QUxZupcHhzEIvokY6EyudLeP`27pFVy0`TdH=y`Z4Y%gb|fYuk6VA@AOv z%Ktx(+n=-jekX9T+tHxMO|0B;wO_B+=ByP}kA&o$sch>F6puVQ+I@XpZ1q7_@hQ`$ zMXkNo)zx)#bGrTiKc7Ky&!79fPDyDIvwDh=TanA|bw;WLyr&$}|e;os@_e@5BYbQYghR8ms<6;?5`uH)zV|9|`$ zjvhU_e16@oi(3T*1qExP5ATwFA@J$v^ZD1WU$6i5a=E|VPnCw%>-SaVn@V2y=Z8F-qI~F7?**g7Ydt2e-W4#iFO6KO}2b)+|hp&HTe9pziWs2Mn z24TpEL^}Q<2Qo)2&TQaVtg5QI|L@!U;Q!U?ese&@_OgH9K@Q^;J*?E@H{Y)I$%%>0 zB_B#(U)%fd*K6i|U*^8AS-xmd(u6vFK|#SlDS;rb#n}?K*6sWC3RDVS_P00PC7`6F zv}j>^#2iW4028UZx4kENs7#&u=DM91Xdp%EQk$!UtYMPNn|T*Ih1Kts->;n>Q`C7k z@BE1q7C)a%w*URcn4z?^)TyKVUS;~NEtz4fOP`#W$Stn-=H0t%_d^2$J{(}?Kf?E8 z|Nr0nojL+EN*;Bp+kLx{JiqqaOo61kyGjf9Z+~yQI5af$`TY92z`($;uxl%q&%0Ir zDY@Ua?2O^@9Wm=x1~0!?eBO4}vbVcKK(5_lsBc0J6v_sb=3ZZVx1HkC!8p;rx$%WU4B zc`>8p+nbxxW;r{4KAZi1_xpY8)~(xF{JgBJjA_T;Z@1U)`}OK^pLO2NO{tsH&hGkp zEgB@ktU0x#qeFvtMz7uh2DQmA@0Q;`d*%#_K(E{S{r`R)JLdLxY2o8zdw;*%tu~nx zRI)ukGxPH)?e#GQ2U)AFLnWMo_2E z{?iHNwQJX2%=q%}_xt}x_3IYSFN@uf;Fy^BP8>#UNk;u;oM)&u*_#BlS63e-{anTJ6-%2G3@)k_x;rA)79f@>{*^ZeVVuZ_qyHh zs=i!wU%h(ua^KnCemw3E4+~otzyIF*@Bg2u+jpMb^W~CvXlSo3-`4&h(CFEv0|h*b zD=RCjt5>dBv*vxjvU}g1z182h|L={AjLg2aW~No?tMzqXR~sfD3keC)iQJ?j)qCyQ zH6{&R-QKf0T3SZ4&zhQ={?>&A8_)66Uz|Gj?hOeK7ytfvOXg*v<)tMhGH>+f8l|4{ z*551A>jsL{=ku!HU5m~y-5E1k&6nxIg$n}QbAN699iev5eI;lpFEhbnp5)#3^!c@B z-;P|We!q8n_Vsnbein=kCsS_isVvU9b!w_M|Emq__kO$8A*fvT_ExId?0(yCH`wKC z3XXIL-rklw+p2Wc-Mso|Gt-ld=T|^ar(Lb z`ah5F?k>MyeBRbww)9F9D>p;UbL;z&`)Ypv`1p9a@9b?gKR@mN_jUco?Kg6^#qKVv z{cw=|@-pAwpXdMIlXSG}xO~0Mx{sGO9+$g)^Je9}%ICUKTTY}Hfm#S#w{HFX?5wti z#*T+=($jPzg?QNfEuT&~Z~wpM<)x+E;(9Xnbv0e~7NB@mtVr83KlSvqsZ*xt#O?X9 z|KIEVbw3WvuZ!I5RzL6M#^Z9cEsLMUmftn4{`O}3{kq@Z_y4bbqYLV5*VX;Iv$Gh~ z;VpfA4dhU}`TFlRJbn5!fA7~YH@9P7ugCAtxVUJpb@{$uuU2nPJIhq?d~W%_kNx#A zyUX6DPLJJI^z@YV`#qm8y36}MbyrhTQZf>_-S}eZK(AA>5Y{!CwA-^$9$w9kxl z1Ls*3KDx8B7}P>kx#|HbAM+|^{(B%IDk}TqNr8n;#Z*v9yp*x+qT|Jj7dI9?J@x8U z*7n=E-~m;acKx=CazB6mtpEEue!1V=SyL9Tjotli)#`OtZ?{|)(~Frgg&9=rEmgf| z3>o!(c=-z`O<)Y~l3CXJ%=*6U{i|2Iem`@CnB z1bKyq?p)$|MAg?yFZ0sUo0H~My;^Di>tg>mTlKStGc08O9k>7Yargbdet~V@EFXBN z2=TCqx)xfv z#hrYACxj)-fV!PZO3XRyj^E9H`1SmE`^~5ORr4QRl zeP(Rkoy}Zl<$g4;T&1%WGT_hi;a{2u1Ji1=H>Uk%>;?C4XMA+HeBrDCIJuk*n#j*E zfr0JZ;=7D@whQ^jUNB(@aPd0rExGw3D=1y^x>xdb%+ujwSZ&5|z|#HepB3sCkL6|T zHP*?Tb?efXWKcaXd#_KY+0!xPw(+Jc=8&s@^rwDko09teyZ0i7W8zEq9ScbS&qNqG z1e)djwtH72D_DB|!S49kd+O3dyO$olKH25#=H>2F9_+g57HYMl;yfr`CK{f)`{V1* z4Ksf6u3Np$ywbPO^UJaKj!RzJE{4osv`HHDziM+9Vg*Oz#EoBUviwdgKCIZ;1fB*t zQ4m?S!0Z+uUjQDyKk%8RWO+$?bopzPsk$jdS0GR-Cy0@9&j!+5Vr8*v8lY z?-TF){bBk_eIr&_$Uk16pKX*|x$jGL;$-m#7Pn`Qj@=Yg445SPcK7|u&u{wwQT~4L zypz0STc`F@?e|Jp(eA6ZAvq``E>n8p^^6QOyuTIGXL-2f# zWJ28Y`deFNTxZ2j6qJ8bUTZc|nZmPyOj9 zZ)V<qfBJx+HoEfI40K3 zt-F2hpR+OXBCA49+yDFb?Bsd7tJ_}9`}O=wCV$`lZ?^*NvoxJ!+c$pY6N4u53Z@l0 zZ47SDw0-Z_uFg1nGk2n2=qmS`(ur;J?#}z7!nNQ)E7SCaPx++cmZ>W#DS2%z5IK97 zAz(r)54Vp~=B%AY=O-~M7iPFSruf%a@y?pkn*HpEZT$RckkANnpSyu0)9N$&0W7u5gg{Qtf4nGVCMkm^q>rd$k<@3Hx{pfHqQ3F3?^ znQvSyxLgn z_w~HeS((oy*~089piXx5y0VSA;cgR?xWd!9yXXJRy=cy$u+iZ4?&(wK2B&FXP@mm$ zspM63r4e%jS7a0Wt=q48v}+9@$!n?8ZvK4&Ee@4>3_8{S=iR^Z^-WHFl-Y1aqu{}P%(M2NK49!QQ7{ZH}B-TAeG|BeZY2)X~OUA^q@)uN}1 zK5S|?j;ry@SXKG$=CaA@ZSQwFv(>02d~bJmX4uU0N1(YC(uZN2IAz6K28QLldB^8{ zyl_P`^c+XSMaSr}D@)czuif`we99DV-^ps5HRKnnU%XKhXk>JC>2j9y;GD|4=viKi zk^occ&cDa@8*N{0=AFjy_>bhTj$L`YDu>kPq}&^8#XZ6`0tac@tbN?zp6#Co;2dU;f55|Ce*H{qCCq()N$3<4a9LCpaf$ z1eE-E$jjtVnYUl%=hl4p2)8v|sgsJgoX`(A*2li#`kIw9rX=@yUsvIRr0atVJg;nL zZn(|3y!L#B_ma(eH<%uH9GB2O1B%(}|1Xz(@_AUmB=BN;qYLW?}GIw$C$!e*sfR-)s$8 zKaFaqS-bK|zwF)ani#ms-qQudHOwZ@Z@A+W#u;9qkz|yDLntxb2&htNivaJQwYq{?s?PPt1M}zv7lVHmCj9_nuySacg$acf+6#a5~Vi+{OIj z6Emnpv{QVo{k|+Ti!b=tjx%#`L<+JPbqw8LGHy!$6ljV8dy%rj! z4wF~BW@vchlal&I^1f_F4sSWrj7KuJ*MENAr^PTqieYKdp45Y?h1R#fKe>43o{bbA z?>Bb?7H|z7C}dKmB-5(&D_2MA(xKV4){H%8G|p{*_GPaA=TBelHKq!mc^=o$bLL2L zSCiA}{a>F23PZBVMTaKAEfow4EZDEvE&Q0BU+UtP+f}+~vC=zTub&zYT2tL;_|I46 z;`&~e5*U!N^Oe-ZmjXYl=Nd0rntlY-cK4Fh47|+8P{z#_Q+;gqVcD+>#TeYJ+@oD; zUI^(ee|hEdRKv#sLHpic`@CqyjZd3jr)GO!(zcg@#8$2LTe~050!&YLR{l0>7#n+PM&Y5k+D7a<9OT`fJng=B`d6mf*~2TAWimuXM~kY;=6Q6u zEnd5Jty}oSu#nKFlhnkgo4kZJiG44(eV(*x){T2vfu=i*w>GFtefZLo&-W&WYx|7W zl~*j)3TvmdSsi^pIc@W-@Q#-7%gLV1jSeb8H{WkD;a^<1vR%iHss}olnV!fHQJbux<(as#eSKnK8fYbq#>-i|1-l$LrtJRC^Xtk1 zv-^1r$;PK*p8ZgtyyUv_ZYxHOqe~5p+Zb)YJ*N^5Cqa`^W`{(voN2a?<14pkPv8qa zW^w$)kM9{$Qm+{~mUBls|C@Fp1=3Kn%GfG9LCDOw>z57RcI^uSum7x6iCd7LH)SG+ z2W$R%Fma5Rn+~|n=jrFD5+XF*Pc<~!SuPZ3@(WqH$*&& zmpul_LwrqY7cX!!yz{YmTYYT*rx4eaE6XDK3VOQg`=>LJuW}{bYBRZbuThzepL`?d-4BHko%F%);CxZ7A0?q-FR%T>$YtQ z5)2Ebd|>)}0AlPneHqtaaR!kT#l_~uSwXiS+*Mmz5*?%bmecmd;l!KQm>iyn9+)@r zR93qHq?-8)ZVF_%CC=}4UCp$0-imm^Q-S9SJ8N&GCfl0uhOIuK?~%FInG52Ox)Sya zPuUwT$;_y_JlXu!`cGGxR-Dn0U+_8IefO72#g?EbzShp{^*c;JJ;2|KJg<~8IE1ln zsy1FerKowkSMxKI^f0SakM9fjo|vY_pme;q=f@ncT1ZI0^$r$46sW}zoc#Ypu7f8Lw{QAQ|`6dC0i_1T!JFnPkc!e#YYxAW=DkcANvZh?r@JSO9TfXVa z!ftShp1}~v%R850!fS`LeDU)uLiT@q#}JToT7TQo@_$p}`ro`_QRwPwwX|D)sR0s6 zYytQCRl*Qi^y~to4=|9GQXkzIQoX6$yBogb=|qX=WO8k1 zp7BN|TWr~m?HRnK6Qp#)0t04Tin`1V4nW>=p%uC9EJkZjlz!oHODUPWUdmzS-UTOS zZAc7GXRoni@bZ*oVZOTX@hw3?!HWggUpXr9Fiv^?t}-uk@^-5wt-6P7P8wSM=#5Cz}YzYV%Yz@`BDa)Pgfi{H-DAZY$=A0H8~GYb=6PhlY@9Oc$P~nJ3~sU zg7xO~4c4!2FD(CeM&_l*n?E(4?H%A<%5i-h-R00`7}`tK-@Q`p4R9&j6YjPJH zj~6#we%7kbqG+(_eA+kLx%Q&BD~_M|Va~$*;tj(B5z)g!`!5UEN<)HTk4vaY8B@dU zMxW^O3Ar0~YHpc+Zhp|IUsfKR)oev~7A#2ujjb$txOAUzAa5G0>fZ9Ywyh`9?)x!3 z{dPzxy5_n6&09LojvR_!r@xtGEK&j2UW*cshOGkD!K`2K9c`n}dpQ8!jas99+} zx-Yjvne;Z9z8qVXU6hJz)ABV>#u{5w13!1 zqiLBQgT|?*vzLr_9Nm6;JuCa0KbnUVx0o;syt7G3X<}oCL^1c*xf6u8*-3Yoxz}E8 ztYQ$BS!^`5<+tPPS4)?1t`=o&UAeZT0^+qV*B?5UGcb4+SFPLr{p}x1E9E-*>YF$9 z_Qo-2Xo+%tzNGNzk&vKZ;H=9kQdQ<+XHUJ5yd${ccJ}l086jroi~^Q<4?!vS)*dzDzCO zt+mx>vhBvt%fFNyzRc^Osn>dvMJfdxr&G461YG9hOo^I!yP9c>+Knqh?-&e@T3%E6 zm9FSJxtEurMVouwnjeQRDnLSi%hg-G4iZcn=ilzsHCf|oR$a~{k*Ke=;h6cX>At)j zZVNOP?NGV8l`kA(`n*L_Qo9)x?zi^bJLe_6_x52Uqk`xD`~Azx7zA8Z4Wb{#FFkk= z?75dKxAD$uiF7P{U+jJMVc)L+?S_-*Hij*&zjMW8?LNMS0t;8aKa#;xHV}i`Y#d8i z85Y>^cQG)8W=Ze8%_tIecj>ktZl9yPg_RODx->fP7f)dm1xMLZY2QWLWEnOYO}M*z z;|o~dEamx)ALcAfFEZE{h=?3a>aCy3CJa$N`xqo&R;}M&?q*}n%*)VM5Rx+MSNXpw zLD?%>r<^#^U~T^NC|^0mEn!W9TS~w=GmT*aqLZck&#?05wWa_LCk@r7wria<`rjj`rrT2T$;0{eaeQLE_IGw-m0S>>~-ey`yHRzwl-@lbWiLHvRq#61a>yb@Wu2X%pdfGMS#Xu= z_Qh`d6`6M#FW&yi6ns{948V zxf~W=jh%GiXRz5d=g>s6?n944u6uEUn+zElt8?4c5;rd}*q#1F`u@2FDI@O7BI&7j zQeJ-a@HYpQtCKa4E)9Z=iAf!@`ul-}ae)bY7em9j73tv{1P=)Ib@2rMuYF_cYj%xU zz;F7AQ+^f_k`3UnYt!;ww2zsg$Y|o!h5Z??ZEr7`VwrYbqxw&1#my@n87y5&US7d* zI=np))vMb4z%Al;^CoK5uT_5)-ePps)bGgKdL1J^aGdkn3a!X&XE9oH zqL{G((r7H1Wvp%Ne&l?9=Cp?fBCb=WNCxP=g`^N(+gv%8Ljngn?#$Dhzq{EN+Gsqr zzk)`W&MdQB>P_vy4nm~gb=i09FFvtp?f0v-# z(Amee`_g~IpB^sTOc({;#Y~(zv%tVA2HbVIcp>0AxQo7t!J&EE!QH-IFSqS*Q!Dv* zMe5ap%ncDPj)AANmo`J%Yt9a{#qS9)O!*qO;Qq7KA6@=(O<6qW<6QgHaP^uvP@_j= z+1$fY`{k6Bl&tQ!hMK&R?zVckg|Q)=H|P6xMv3PVg7<$0ho%NIGo(B}@X)7pZOH_% z4__2qztFmZZ8PUl%Z7VDFK_m3erA%|bG9e{y{7c*g{ur^PFWtD5s#|b#yssI0%m6es@*}1vD zFVFw?1+)<$zy9~_BPZYQ{T>$^dsnvnPT`k}?#vcnuLO(h$L-0wy6WfW=gW8Hfqd3E z%Pd1E%$RF4D3$u&%VJo%tqBx#YSOP5I(*ts6iNqZ%W5f~@40sE+O#QCUU-T)l$Cv3 z5x6)jJDYtD$gIG{ZZADW4zzFztNG2@QS{VHGqvlKs*=(okyl?13$rwQiqT+VF#Eac z>$&-p=PZ7#cKTXc%i#c#4@`dpl2asPZL3OxgM-VOsy7&9N%O-E*|9_1aPA@Ad z@zC77`M0gu^8YvP-HVHekdUr(|NlQ95D_a{ca-kM4?xgt9Kkl**s%E2fuu zsnBz>-(`~m&&gfhvr3w`FX4^#+U_675)&EN&n~saf0r4{mV|Ydr)5iz{I%Lxy_|z} z=~cD0W;`GD!~fq8(oB{5qs!ZV_}B59=E{dNDl31UnQ45#;&JbB`T7`n%g<+w*=>wh z&9^;!=FRnZ$Nz>RTwi7XJelmjE$61tZ$o{3|M|vO4zKXLyk*CE@B5RcO!A%;6LV?F z<4J0=(@P9g$~~I}FW$+V7|^{rM0Zu&q8nmsUh-6mXiWXKX~+GgC0jJ(_!+jYa9MWR ze70P@cWCI<|L;%jlazFut{3|Yv@YSf{r{hx>hm;Oetfy?FJJv;uYD{+y6gfeEy2NpZ(u2>(;GX zKEJN&@v+{*?d`kn=6#tb-+R$N{`s?K-(>eZ>eBx8@5ZG|pFW+|x1M_rGW{**_BF=u zz8qvoO`NNB$C8srReh~MuJw$$6vn@VQ&O!*+pSGB=+xKP+qq@4PfxK_EuYhH!#LC{ zx7_qiS>oo-iHjAE-0QbW;W;)*>S)&1w!;tq`WfpidKtKB$&&cX5r2;#nXRHD=I!Zu z^4Zzhzxnd+?Xk4Bwtja8G^xTH8z>4IPI&IrCdyF8CT*o8xA$}Ueig10=gNNk6<|7= z`f{R6yZYfn8L6pDZ|219tt!30cjmP{rLD;`1$!+rrE#0y$uZw{nfv9>sD#= zyg8=X(;mdzDJd-~3B1kdduYu96?O(DMA!MzD|1t+iD!KoT>7HF)URr36FaPT*;=OW zwEq5@o10Rv$JhVmRG+ipzVWWo*VjNx#eW66uMS^d_iSc5XboQRd0X)VGt=k&+`j+s z+jG|M=a}cu6G(b`YHDR=W%2WKrmN~e<%piYKx`d@#_PKgA@BdpOx)yK(9<=0jp-HV z%LyAh4n3L@?sYQd3A0b}^K-h|+R+=6j-HyTZR#s+nssIE_Ip;fze@gIO)$7|>((m! zr7Q=Y7TqmAZ|iUO^U12v)h52PO)@V%ogOb&4rxOK&T`4MX4vuO#7UNhT?yX7J7k$l z|GAWl?~!BhD%5QIk*t_f@cdflFJ&2kS+KWFuN&E|7ftKWN+m3`B%``A6@N~%ge zhkWkgHz$A3=S(Qqma?5;cc3i7#l^+h6EaHmbSHzr#EZWBvKUgoH7RBPFZKubofWjD z{lXtD%*Zgb=m}c%dVc$=hrRkAj(2}I{``O2!t&-8S0DY4*oyYPsyVNp)R zS%wc=%iTS=qgyH`fd(dmn|s_6jXD=iYEiy;;jr=T_PLk)85rs%y64@!z9mBF=~Yf8 zC8<}hZYiq15fe#cbckNE+Uy3;1CQqt(f`lQzw~-y>|BN^kAf1PS#B*kpiq1xvOp=# z4`hISb=m{V*JWQ+0vA5$JRKA+EhrdxY3uI?yo?T!dJHRqqVMnJb(p8R`%c`A{8h^q zZj@^%*fnA5+yWJ;NfK`S?$gh`IJx-X35Lp&yL>i385uZO{f#<>1TO}Ngi*m7S$&;m&N9`@+mr(vuGL&Wg^m00i^Qz_$|;^xIyx?SI*TuoI-@4?oWY^| z%J#$Y>w5m|>E{?4Orr(5y8c%f>o4Lo z=oAtRjGC}ly4xz}L@Cn&zWUhkE7}Un)XM(B$8#I`+?*v2zj&xO&Aock3QqrjM{aa= zbaYHwaI`srWl?$Z#Rjok&(<>~61! zWN`|7?4_ur~P^OiCsY**H|KUx~ZWzZw{ z?pp5g$*UP19GsM7j_$W_f7fs@6tsv`x>WXpijJB{DziiGhMlT+7$0bar^bLr9-0yu z8iEdSe|g!N_+^8b_o6PP_qN;L?zpJPASl`$@8G?C<5EFELD@xm_2(m}3RVWN9EwVr zyj{v+=iUtmUcECnm6?{S%ivPS_(l@s_Wr~#A4I(usj=3_2rzAUFynycW5&)6UGeJ= zyfe$VbVI7Mql2$+)xve4uEN7j%m=RDlTDP|s5Q;=T+6xnyukdtgP?tfE z+_GPDgpWS1oi}&?8<9CxF8*?V@6KM4gT>?v4vWGQrsRbizWz}zFm>Y{$4>~R@ubOzhZ?a4J#uAvHsXu9zfE>2<=)@@^Y}w+ zQBZkRroU?T!mQ?41wp}!j@_#=-mo;-s2i=FFhSZTRLJP|8=1M0xAIp>xfUxkOq{D6 z6X_=6t$CEsr+x0n${%0%Fwd=d7BYvE;lvv^XK@?HoNOo8y7O#@KtAle#kWH0jM|hZ z3g&$O|&UHtNJQ*w;qnzR*CW!t`OiBwinGMWbO z;4mgMFZY@nwd|ywUUAC)l|kT9n9Hi0ii6^{B%2aV*51&Si04@@%OJIT!|JKK9|;Ri zJUDBYK%bMK#(CC;SqZ_}ON0*uuJfAwV!Crmu&`1hhpWhRi6F4vyE8<3G+i9U7)pNa z)V328l++Y)?Ph1_+LUOjyvEh+cNu6j=hXHu*55Um>u%oUa}W_dm}I&j%!uP~hLn&) zCf})IMHZ&Soa~nAV2cGo^D=M5M0PMFEG~DC=#CIQ?c(mbzcOgiLb;%>9-oPmvt5>^ zNVHwtGWEGjs^S?n7t=QxPOFc z4Qfr#P5ym$a*){-CV{fuC$uGAcuaXF;dXhxfW*vpBlDI9{gzx`j+WkaogE#QX6d(- z$umrOc3{l{<5xFDb65;6T7sI))zVF%(alr5np1g>8(76W*&Q&Ee{xfS$nDt|PsV=i z?8qp1@x6_);Kqd4cP{@lKK@23_5J-2O`&PJXBk}b28xVTKo(Y?#jqF7}vwiZ0y$XxahG~8(#v1ZYN3p1i#8XPdN+H>QKM10R0 z9)^sU$Nb)PrM}z+Ukl*x;=-Pm&iyWy_5QzaTsKc$ zbT%)(f8`wCj~a$4lZ-gnCOz!F|98V;ju4SAZ@TWw9@ysguFK!W<@ffjlF?EHpS^#d zT)g$|k1bO2Ime!{G5k2SJGQxGhS4$0U*-RL+_pY*V>nTG_@TrD2a_Cw9zOS9>wYMI zUckYkeB;`c&HOTgf*0@oaAtVG`*7Lx2b?@M%TM=ToxQP4T0tOmqxFp&Qtwy{j#`Fw z?sm8z!5`XsbB%}?e~pfmnS@)r{&wc6t{Y|vhHhLese9K=9F$!*@@}=Q|Ik`+TATSv zVCdo3^B@1;_Vtg%w25c$&1G=A)%|(xU+em^@`IOo9Ym%dN|Cy<>E(yx23BiQBLX8? z+IfXmynmB<%gx1Q%8M6*)&Fb~M@G&u^>zoX^aUrI~;WbK1QU~rVHe8r;zQ0PY_wLiL*TgH*nHgR@ zH)&edx4AEy)!?S(HKjAP{mQeY1bPpp2)U~MKYwrT@#CS_el!2KNk8}Iw~x`xH`SN+ zwmRu0RAKsZ9B!537sr$(dNGA+~5CNZ8#}GU*70%CA%BDYv{6=AODUe@z=1G zFEq^mBmeLshrATa0WK~@KaZf3lUz?~ocwSj;AOL4!;%G>4FM+C83mmoIIkHA9`nt+Iwz(BfKVLLmY<_+7 znf+>ln#IQ#i2Mw^9USfS{#}oZ)ax^r?_aSzH<HqJv z9gIAW)ptHW7&p(S!7wa@=a|VqtH?tcDN|COZc0fz8XmIc!kQl~0-Ltn4Xw3h*!<|` zh5u9UvUjP8gd{OMS9A`JdA4j?|I8iS4xJknF39Ub{(d^8j z=ek&toii|9C*IkXT3=TG}%NWjf`Ew|C%RiUsNT?UT^?Txk3sjS<7PSaCiAlP;YIGeCND*a_XgOC$is9Eze zT1MtT*>n9rmtO7pG3$%4kdVNON2^Zpus!+h_swcTYqOi%NuG$_6}k;K8O{}^Fu1ch zgijE1I+3Kbnu8~?UfjxgLMR7MWton?(@b&C%r}|a^A3IY+u6bR;K9uGOwUZM{(WAY zy7+T{)MI8FA;yL#CAENkj+++ov(J=l?#Pml5o3_(Sg=&QYu4)-9EvS6=8=vG9KWqE z8nd{z`Z>&#J>4DnrY(-0;r+I>d6mCYBjQ?Kcj?8yk7z4w7W%*OPucu~rLWblh&nte zsxx9}*m>uS+w+SjN;D)@jh6`r%xhN?mvFXdJjNh&>0&h~r_K*DvU0w0zjdDC3YkR= z`-M+;2fS&s(DU8#NTh=)Z$Ho1I*pjWPv+(4%HNv5Q@xYCa&K9S-r|SB4&DV5t`@ z*)@5GQ;OrFiGfDde8yRSXGVUEZsy>5YNg=Bp?F+vo>R3wUrC|@6Q6{O$nK4cIDYcI z^(na$p?1phz+0=^(%SM>3m&CS%M8`4{=V8;DxUGc8@?N|>dC1mnB0{YKiM(k%L-kG zXnQ5jFFcBh3$Ij9VU|>EnIO}pcG0@$Va2nBp8WHf^|bq{o`mhTdiwuo{ruYkKavZ) zt2C}E-uk!e*YQ-1A0hWt!&ZO&{QKQQhuR|r;U6BdAJz6s>8?1N?)PHXgrkjaZYMtq zm9F@tc%PA>+0f*xLr{^duwu&u%|)Ij-xq04IO%e>YD#6`^Y-5oMu7~UHrGFDpDp)e zzq42y)?w>Y~^=WbQ)NA~)ikNRvpbv$3Je^Dx$cYpnxs_!i8cp_agQl^|? zWE0t$AaYUCDdy0^yeZEQr=2N{Xc2H?I49_*U| zwrcaath$7oy6L;Fb=AC?bm29({&#uF7a4!v>J>I;>7;@qJ%p zKk0P?hhj?RV;N^dnOBb9@oK_bX9h$!%xaQMU&LUyf17N>nTpMt|K#6`UYG9T;+gl+ zKj1f$c)7&7*PoUeUsQhfL}m?hB$I}uYVpY=m$~v{%ogs)8TxphfTA;Ul7+iQSM%Kr zUM0oGjc0V1U$u)!tG?x>sbl$nOKrPOXUhTC$hALjKKAHc%`4lly?+0ieV@aA_0(?< zj4FFJ?{U1~CXoOmuV*vj+GjECSj%7_;26wc6zk;0p{U+{N{}a=>y&ybLqT-ld-qg3 zowmonPTQ~9doXbGuM1P3Gp*>J>C35F`CWEq&6MJoKmP}A-^*FRci>^oPwwfp`@Wod z94Ea=1!THhvc~7Mw4?oM7bP5QO25Rl2srh%FG{;~;hca9YtO;lqnn(jR3=_IfAD^M z;@UrHM@z3A;(GsdzJ|X9qk`%~_xp*Hmejvx{d)E~BcFlAoMYDe%VSSI<=5Yx%NI9Q zhp~iN*fQ|a=11GVE2}QvDZVt$BE7CtNU>$Y;z^-j+AoTqyOMs5vmp9mamuNRb5pOm zd|x1w*0Xk{?d^)ko30$@XJ}BlUH415fHmO(kKgi-=iaBDG7>Dx$w-+p$Kj>-l%6UL zt*GA@OlR2DJ98+e`t}NmkC>V9a>jjP`wrDI`!fI8r8c; zc65EY6ELN7QslyvDRUy+1WW&&$(>z$ID^+|Y0VS?r@m+_UfT=32@+cEE1K_jPF2+? zax7=scUxNNtWkJR)4qFA^WqL^ttxqMFFteEmj^wP`9IZC7o~`4NUEyOIS?>${!Gz& zi@s$HK}GA5m0BjSYAdE{Rk9s^AX6pVbvGieh3R>xR>O65-ZJ%OhOpb&F4H6nr^Z)% z-u&fdCHj=%vCYrF*6MfS{ZDKJIgjUMpSqsdJpX{z8Lsf}EwdSZ@Z{}3I6F>b|F=n>&dis5oX}ACMDOnuld^hS#p6Ab z0*gNFSi3^A!Len{g980uZN;(ziY+<<7geHu6|7))He@i64tRESlhTu${y#cDI&b_o zW6}k~#Hnko_NvV7WO^{?U`}7m|D{j1&Ne>3tY*p?8P&y4mKaF~1T%Cq%Dj+rInHqR zk_(68e*PI_6JAw$@Y~pt=K#zvP7-tvDi81L3Xo# zJv8^v+L3Z_QOcAjJ;&R;;8Z)FDjDj1Zw=f6Xud=;+L?`U;)$2ynXWx{aT(z-F>u_(p-~ai?U;2d>`DjS0 z?o2UXxQ#*0*22Bo9@IvNT;!%*xoq{drP`ItE(d)6C^}{S*4m3=ou}OnZa5P)G47Bs zC&PwYM^>+I)ku26@>gZerduC=>d#$ca5Bl^se@;q1*9+HKfsl}bY7~H;hlobTfUmc1`0|f|5H*+ zyv)}P*}FX?%%W8&8D z@?-`>;?k`}e~RQ@wCOkt%-F9HzHy>!=cGWR{vA37uP$EV_H^`?=ZXb4iZ)p+NnvR- z6gX)kQu^WML)(d8C#PwxIx;(XE<@JZrDhk6cFl2Dt}$R~5EXG+5*(hQ7ii@5OnkFk zX=}!7XO+%%jz?6dR6M(wI7`4OO?{qIyhQWG65VnMmnXL$Zk_w#m#6Vo^-F7;cb??qQmfU?hK~s7rRl17=Jataz_$%15$irjBiPImq{%H{~%D$*l z)o%QCGQ*57A1ohOX?~rY_R2#^z=xM9B6qdev9sG3fBWX()yTPKg5QP7JKgVpNU=-e zQI(6}u`p_=;PPb*d2)Et;-9`8iYaAkOBbdwe|r1bgqL+f(9*vX!x)w>YqDw1F0t%C zKEcxQefvDVBPs7d-qMG(7$=CBum(IjCnBiYGNCfhqVIq}lJeH4YutXwsPC@37>TY>ucPuh%GSfsNiyE6YyPl#Mqx*ltWP^ zHeg>m2dC!dBUvr?IcIAwPn2zD%>3b@#E`Ia>!Ab3-uXX*_5dPG-mlA7?PgV7oU+fj zW5u_J9`1Mz9Sg%^&U<_AxVTMY^Kv`cH1T~yVE;CS zNkX%jG`=3lw5-y1;;?iS5nZUkba;aMrGV+ItbP*WSHiPHI?a#vF|05Xzg?0%ee0(L zd7(lEr@+E%=UUFRvTi*-LG^s1n=fNjv&YE{-b3$Q4hCvnnmE}#%uq!9jGNnciwUm& zVHf4)v^De%DsnzpF1Y^GYSYZy(w1uquC6k^9(DTIG_}b#H@F*~K0R!*KYx!Bx4i1& zltx48far#8buL0Pce8lyHPNoqTbizb8Q7A@)d zx8humKupcF^2aB*st&*4PuHsC;)%1aDeK@i;AOoq^Tli5R)#g3LOBJeF3ta5qim`Xr&a8|~Zq7bhR!kYJqgH6paBJAC1<8O3X-^z1YErTpOY zrUid=PQ=`l&t(kAId$H=XYQp33+v>%n+%^Vw|;T*q{B&#$O#ewPnrT<(m4t_+jj-d z+s>foADQy4^x^$^?BR-C7K^3)CeQI&#?t6$^8T&EZ-%%V-S-mj)CktjJv(F3gQ?Cp z?k(bPz3XPH|NY^lP>mwly~;CJ88*b{94wI0?%ud5An?f&zXZ>`y7jW}w>)@bRrBkw zyX18rj~BA@VoqL8j{fjG<(qr7aF$ofuBX>dU2L_P>J+cf@a&sWw`uvsiK;1{DvLMi zaC6rbxP^u?%qfnXRxEqk=8=ixmj!F*tE!rOTX5p&-s3jfcDa`xn@AjP{x1+JdetuP zQgXKPP{%O$zLr?`9V(9b4Yv>XJ^Zjj z!|RuQiOyW@?##)nR-Ll6e|1@`(rA{149kY6pXbm2_4#U4)UH3PIAJzplJ1z%lm_tf%h$k}Dh{I()_dDn8HYM;+Nwuf_brbd^M zi3!VtySMWmN6insm=#<8{qC%K-FsKJe*gE|basq(__dh&y=lQ8&YDO6|GnRIc9(gs z)}9wiXO~^O|KDy$?(DSSyS?UCuU?tV{`T+3@sIQ5&Mpgnx6?h{#4mQ8%4F;JJDdf- zFn`nXOuxq(G3oER2zCqe-+#69cl&-`Z~Ep<-QTwf>lfQ47QC3+qI)kw>`s{64K1zC zJ-pIdb{{#<#@+q@!~OaFf6H!$6n?#0nLcOX%_~3V|F_(dKR@mAspvdI@yzhbdtcY) zpS<|u%2OYa>;HcopA|dl;DdSc{{l9jjEjw}zwA59m)W{Z=HJhu$II!W}+O)mrmwj<9J^xnB zPgCmEkE8yJ-1{>A9W=fFv+v-_Qgx~4xA*@mJ*&qyJ^SqT{cCTEWaip_*gQYeZ0GfL zYyX`|H<`_}eoxVpm&;9Od)fVP{9pMz(@eJR)5*rz-6pfA)xPxJcFODQvb*K?=W?~0 z%nrL>{l9<(k^LpR^ z+Ov8e-@V)W{qFLcPhz&M`NkbDv0r75z`w_-)6cFbtYQ7> zxjlPaweRMYzjnR8lYK42?Ct-T{)=;SXZdFD`d*334Uc(f|G(z!vcvwirW=Zvrv?A@ zHNUxfwdw3{e;%-({8x8&+1KxPkC#|6oJ{!?5%kJ+UXRl>#xMJS>&>1uLw$1G`fnW0 zK7qfs=U>nHbnSZj>(Z12h3NIaRmM1PJ8^CG#@*J@`o_<1R<(YAFMsM&>a#M7 zFJ`@l{+dFb`zIW@=lArgYA$Qy0((Wvs_#3sor2>t4BFyz9D@&>{xheUae_WiOwcPn!Li@?I2W+y3SD$_6 zyL`swQs)tim6%J>!&(;=dP_zFW6B3`&+|w)rXb=;e9i z?U}cU|2XZZN^g=s^<@sT0L#J0Gi#1NdhqUUXJg~aSeCtZudj6<-@JE%-tX(~DpOBg zJH0*4cU#=rg9+Q>wC%pGTyy`HukYXYdzasQvgM{uWL2tdfQX1s?ZOlzNxt^0Z{~A( ze|!Jyfl2$O;`4LQF7y%DBzNlDyWWzL4718Rd5-JZ6U=72vsW$Ocw%zk#Vpnh-?ZZe zm&I&OJ9pZub^YpPX|dDf4d;n!Z~Cfb^E>D5McD&;Ybv+JnTE&5mVCQ8%lDX${RW0L zJAbCF`}!`6Pk!0V+I>0NsoJ_{ubtiva^}9TuQz|y+WjNReVUm7y;{EfZdMtG(t~|898vzi=g`Wap;6!PgEQ z%GrD8^z`)Crf0pT&Z(~}KloC{s$sW*jMg;?t>BP1S?yOlmas6~j{4B*UuPu8ll9N> zY`heMMDoo;m!mhwol4WxIde_>_HiMHe=C=#{aukR$;}c__j2jYZ+Y3%67w0f*X@rh z-ti(eN1}t#p;7ptLD?220k`v47XL6|S2CQDy6krCRpt1j>o0Zt-sDd`_v+P}_Dyft zmmV@ce%h~lqTThr$*y_r&yKKD)b*|X-WqZX^(EURRAwr9_p_Dz!}edJFS zjh?OGZ!=ZTK~%Ar`@L?-6yce=p~o%-$q>iXp{zO5`TL>A9!=t$n*<#ffX#yPR*n zT%P_qwC?57cd4gMW~;pod%SkXzh75&JpBA|S!MUu`Sm)_KZ&o~w{IV4$KQ~c0$&+*YfAhA-2|R9s@7Fu7^}qCa#hO=aLEpB|x@cFl z>LK^@#KqqvIgOt6-~O<_dcpNnuX8@m`VK*f7amT0?dO)jee1bg`>Dih%6g6>2^W;- zor+nNA+k_dz@JfP^OH}dKRmA8Z)RbTV&3t5=EnBaxI3FTt5gm@+~Bvoc=z4tZ;#9K zot`Ezu>bL^sd0XGlFzt?FJkk*UGX>WN!}9o;GI!=toZKoE}hT9)A@Nnt>a@@v!v&y z?D2osFCDI!rMx($Ozg56==fTti6`D}zi;>Vve?fP8j)EtX;(xSE#>#*Tw8mhe8C0I zV;|acCHj~Tyb{W>ljbuP;#3uySZejI?D3?l6>k;h-Qo&8x<3B+N>PIgE`gV&x|nB7 z=+x6bl$koI^JuDGiF-~opTi_ZhoFN7l_gz1r5Y1Xc0CeeVcf)J$$3nHAz^0G^8|qj zI~V?5`gcl?oNK}z(Vpw`8tbDN86Nvv#%z#Dzy9di{nTUcf43R4KK<)AL)^*n=gpUX zY|JYe8CK_9T|84mgmtei!-Ilz8p5h6egQ^aV$zFe%oMekalda6WaMR2@l}9fjrQUF zF&xJ(NF6!o7i*BSQ6i#}F-+{?oT?pbD;`v;q~_f0=1;M8_hm>~b#BQ`-yhf5C#L&9 z|E@XLue?t>e14z(`y-ooGb9+!ubFAB|DN4#+tgJ|4T1doV?`INNC=GN;cG5j!I1nw zCb0Ll#R(ScTgLw_x7$tb?3=#s$xZ8?^K+Zy_i#*gW_KzB~lr_Bn*srgR~8k~~j z8WVG_^nNcXjBN8$|FJnP&gn#_^GS`!mJ%kd&&Hb{@gFVNz`lxWFW-E7{@c+XcAFc{ zVOYTag<%7G&%--bHBUxx(@_omM-}aJXGCGblT|!n5UU%Yzd=ub-^gyMwqq0;MTNx5hY94W|b)NxBvVgz_4(G#7%u4^(RLbrcBZCdl~uBSINrB z;sR^I8qI@`*FU-&)~jCJH=X~{*|eIUNjW$B7aZHGn$mRSgsuAaoJ%KM+~;nW*?+%n ziqFr>7d4e5;#tFU##{CJUp_n$ zu%RZ~DF>VOomuqbh4+rHK@zSS zT8}uzJUOi9D)gMG-@H|F<-{$e8p}SGt1VTtobuUY#;ln+9uo`GWc4Ch9UZh<53cZA zE86MB*w~C_YB1ZC zaOp*}YfkNQQF=K$i=X}D1A7CJ=0giLxc42oZz5u9>@ww?jfkmnpXo38(9f32{Fna+ zRzAx!U@PH&_-^^f#;YGJCwOgiaS~xsRB@d2@mt~>33KC;)!(-GsW?BIxuq!9(r>jx zJqv?M=h4Sjjslm&JETf>d=mNdZS9$@OPN1sa?EFZ{9I?|MdwCe4TxrQiVSNO2s}#K@78!Gc}mZuqwBA9*TyoiC;xaNsqx)!-_m7T{PsUt-u>gg z#l~=EeTvE4|EWQ4eCHOfn#s7~VmsG)orf}?GES{%>i1Sv=UsT#H>V-)KmyPHdTCX; zDZ5J40;h=FEnw&Srl82!aJIukh=HfLe8bEsr=DC`V)L@1XWp$owM668fBG*Eh=eXr z_WOK8=k2KzAujVKTJpLk9-6JqB_Th>;N!x)N6Th^J?t6zz$WC~;dx~YFS;MxpLhJ( z=D_FfLKD8)$W}TXx@`U7JKKvER|^B1xS*+gZnqgiW^lzmI-#;D=IPy7h8b6n#Y8@D zQhlNV8cwNu`qJ~Don)u(CgU2fI&B8uM^&V40w&|>U6FHcA*hC^OUhH4zSibn|?u5>?hc-A`YvgLtHN%yv;Qp^>91D?&Bof5<=Z7e*2k%Q;)y!3{NhF31J z+Ot0FEIr%iV>CBk{=jSDu_5^zQxJ?f0L}Ss$_S(8C!8 zpBL+#zP&x4eM8<|tH~!#_GyZBzrD9t`am16wA$6hT&+&*{Bk0$AGc_x@-sI&czbsr zO^P&LYVl;Evb&YFb*$d?O`4`hX3d`c``51;yZF%1t4o&s{r#Pvmv`!vDN}q_8hpD~ z{r>3VkE>Q`-JfDL^ZlOBeTy_sPt%Q_ly>Q zzg(XE+vxaxTiG91SN}M%>w|T5--%W0Hf-2ne`<|kLEqolin=8QR_uIb$&(TcZrs10 zzcIpVd5HhpmoFuyq>3zLT7$Xr%SuXI76-1s{`&O!*;7Q!%*~fh`Z(#)qCGbGRSPvz zHl4BM-7V_5YVFFh>X==HnrhEq?Y_O8F=B7k*3u-~GcHOSv#;whyuQBvy1!1em5t4w z{PT9lZv4LYecwv$t5>e1m`M3sKAqy}8yg$@_U+rU-L+3oO`SX|DmvPBBf|$fhK(Xx zpFg~EobhGF$B#;%6LePcxW)J~?mgYZq+Zp=uwWZQ^|v>jZi_`lMeFw0Uy`r+ez#oJ zbJgnAvqSP)ivNAPozEPxHEU{a(%ii%Mmvk2`!Ot^SG6iJ>E($6i#_G<;~1pP{nov` zb^reT8#gixB>d(E-Po{u_wKc8&!%l&8?`knC^$Mgy1ab*@y9DyubzG7%8eU0?%#j^ z{kQhAkmug-Z>6TDx(2^$dUt21@k}4TTU-D7$-H7?cX15eVIXr{S)bA16$|t8oG*5T zW=7{V0=}Gka9~rZ4N(R%&>^zg1kJaeI6I{SO~Ln6z(t_3G6h&86bob8jzO z_3G2p({qKpx6Xp++cj^|4JEW&SpZM)tQ^l4&7x*`~?mSkuYudDF z4hj>dPY*XE&!-PeZw z$;bObv_hk!t+&qj9i6|Ib**!x*3?@Yli9cMrGBiixtlE(>BYyNnz=ZVtM9CmzuV=b zevd!?xUy>bs#V7xONq8KYIF(p9~WKw==~<0&0Aepugj|nUQ*BLV|4bH z+T@ctvu!7OoY=Hx)#URL{p;ReS$6U3(>cq%8704ZoUB;S%)TpkYuM_$8j-?HGU7_i z87~|XDqfvVTQ|X*-|7|XOwB;SrprcUpQL!I7&{wh^f2}Zt~enrz0k+Lz1`_==Rd9O z3pbT~+9#F8?>b3v{?%ZcrzxxbP@Sc z-j81#rkU$GDX4SC@i{Ub(xIa5)rDECYuHM76fANsJDw_E5;pBs`PNjUK(;@pTP-da z+>M^^qh@;lSI+zQZ|q*Z{5tp7b+_x`#rhug^MW;df8BQfYmy&{}QkH;<<8uG_o@~k4Gt>L{Q(vl`b-vkoW2>m^%)7s>w`%U) z_c;93qNx>s1^Re;zC2xHdV- zeGf`li)8m}a>@#a-84$=OO1SXFwQk0!AC<&uD&j1)_3!}Q{G1FO!dlq7uM;bG}X)Z zbExUxh0&WkGv{9LT)#f8^7EJHl|l2&>y|z%yFTwT4YNfK= z&`?w5D_c%XzW60BDm>i$Yp>x;M(^A|gIg4}4ayS~V%P@=}9iRBWv5^O`KB6_;OL(l#}9T6}RvZtdEz z?AqGBX`3Du?cB6BKQ3%K$5uZ$yg#GX$v)zjAf_Ox34uZUc1yyqm7>)V$ta|^w? za<4@4&1>Q7uP>jytS;|tT5)ad-e-b$9u{1=emy%)MXujn#PzK5j`PnqN3Av0essU` z+Khd5h3V23?bnap5i?coZfED|J-G4niRV`jXSqa1MsEAMcHO!#kHTmtTIhqCUH`t8M1}`t>X4b@s*xlk&HV7AbANy>@5z^r=&q2CdB5y={@kdf8EUx`q zxBv3XGi4f|f7Z-;9UU2|d3sY?aBl9~FYTLFzCZCJwmN9?vtM=J5C3=9XI7pX=!V#dd@QWHglTY6yLjB-T704uU}6vNQsW`K5F!L-CThs ze>d_mluk41*8AM{>}GCQfAsq6oBuFLsL6P8adG9C-CnZr#g<2plGca+4hV~hk?C|X zu@}91_3CB$Q)yeWFPAU4CU{rPkD07Z>t4R4rY$Z<af*3NgGwrJ-*bq()}Ij5#AO*w2*@2MkITU$HZEce#7+}Y0_Rxo<12u<{G5pg}s zEHm?7bN%si$`=g(e8{r-Wi+cgQRJXRifQrFRHr7-{{GyCMUTv*4>lZMcfRxjw_`HH zcbnoV&qIEH=+*Y~GU2|jTYuurH5IN2p$q~n5~VR=ikdt#3NC-|eV%;yN$_!%)oXiy ze3hP{*7v}mjY~goH_x%77CMJ5?_2G#R%w50b=q=Xu-o$Dpy*i>I}T?Etx}ucJ%28E zSdi6mM%Ao&XU-+vTEcJpB|t@J=ihI)m)GA)y2Hvj^F?ra()0HV3uN9Ln;ZUMZ&{9Y z&oKu19!LF8PSZDR<9U9?`S^#W_Zb?ZA0Jn|7!cj?zLv%Q&-^nwev=IzJevCB#*^;d z9%0khADx`fa3W=qnec&~OB~iWd-k_XIl0t@QBV3>_!EOmtM{C>umVS|*Q_rqPr>gO4}`dV+dXs8!EV$Ypa4CaJ(Z_b+Np~BX_*ln-D%X5)M zJuHmoe}(Hl?SCQBaKCY1L0X5&*;Gc0HsKxbbk;1WR6M!mn`^X+o;2%WqiEMO!RLFH zbI&=Pk#g_kWhsG4(NlURsV0_ko?CVJbvwg@ug(>J1#;XLpVm=RNmOXNSXu1CSYB(< zo~T!`Xv2*4j`o~y)b3PT{`)Pvr-nhR{rovzE7!oA0Z!%;t{bJz=k@P88JsL=apa&bor@F=abWtNZVeXDD6?!W?h`c+ zw?3S7)ndJ$)XoDF*VQN-Ut(CLqI8^vLwjvJk;j&n&8oTLhd=9NMOTYto}hju-Uh=SGACUYV!I zaOP!G?>aZ7&r%Q-OlAtaDHPl*K6~y> zUksZ&cwHOFQxg_>?J>N?x8PV%(&d?#xhoe)Z(?ywXmz#h?VFJ7dQwBNMPQSHsSDrg z2Vrw}9t)ZkcWAMoodlD@k8Xva(&SAB0Y+XNiY+=eb5HKu7s7EMt9mN~8$-<#mc3Wm zmTPZ+C|LAHLsAvAI$C0;_HU(jMiJ@HpTEr9Gnt)fLQq0U(>xsmV?Fj&t6r^(-5po) zu=Vcla(1T1dGq8p-+c4^cXxO9dAr|lxb=1%c>jHVhXbc#i@>C9`j;k!s7`ztc4L)M z_aj3)3x)WQ6zrLE^ExBxZ zJto;`<})=nCl1Ay9*HT3-6t_JXy;6IbL3~3vpRO}1DQ1L;>sc!c}3$ApOYEU(YLSf z`?eKy+Yf_T-kl$yOM4z4>n$xUt*oq!TW>zohpqXbgTjX7<9*xjRlS}X9(Qw3<>$BC z@7Fz_TVD6lJO0u|o^(*0%RGFr`>^kv;^`UsrfFK|KJQ})@-$Er5pCXh$Iat}kF2d^ z?Sn@4eLv6UPn$aRF6g3>xa!=%i%+xzdDxoo=B;0|=FX17$Kv~cxSso&y?*bvE5ZJ? zpJ(6y^ESW!_m9W@>*MzRI>O=)iU{5IYRcDU!U7QtnVz(8e!Q`2hpNk}A6gQWCnRt{IHC7t40(I_d7dM|-oVIArXV5)Ry1Kg3W;qtm z=M<;ink(SMq3C{fE)QD}Geg+TRi!=*4Jnhpe`ieiX_sd7+|0%)E!XUGkKNxdm$R?0 z+xvRm?o0gZcfZ@!U-xA(DEN0ipI3dD*Sz4|b0Y^(4mmk_=O*(a35SJ->km9&WH@*A zonHZ8^Tf)}#WM1W#}kj8d;!{E{d8*hyL)@114S6#@A-UgvcKKSuT`MyhdzD!YWy(}#;qFfX_KysAHP6#wGWK(;|FT$~@yAR5`d@4FYq<~1 z|Mx|mVgIkI>+Ak~o)5b0q-SHhd|k!4#VRcVPSe&XT3(v=f^ox{s6-(i28E>+`T7jc zJ|4%-H2nJYEA{j=S&M>%zGSdT ztIix*^uX_8Sl+oS79qQ>4~D+r)ww_Y_6H8jI%7FWSqTXV-5;8sm(I<#_P76Ave?jg zDRk_w<~-Lm!A?)E1hX-R}x&Lfr2=T6TH5?Q)(<;o2k3i6!6PFS`?OSWel;|c?DD_3cT6GHE~84~jN zH{U9fmQyTFI(G7fCISZ@;~-`@VbX)TyAmIof!o z&sjX?VMw1-cnoy;=7ZPq|9{agHxhl$u61W z?k8ks78*rGM(!+qeeJZ~?lnu6JelOJ_w(=j`uXd6Z(qLrxM*kH-`DZ|^R}vkGQb;+ zm|S=#n?aVd}o*0Tv`>ndS2zTnV#J7RWFxr+GOOoKzk*qE_ht3 zaZN5*$VzR^rqJcnKfLqP;3}~SbNHS%!8P?{d+2WUr>1X z_WJ+-9&_}AjETB6xp9I>TF=^z*O#e_g;wuU@V@Y}<e!sJ7)hf_kQ!kg#UpD(S$nVKhLW0?q z)-s&(t=oQ&ufbheLJGXQ>s7I|oTB=ni|Zw~M(BXUR`vTPwml- z-?wMoIz4^;^N&BO2zCDZ_s>CLL-F%-|Nhm1?w+y#ee=9o{=J-y0S9B8I24~myzP~l zwj)R6%hzc|3=d`se>szt>=)kGto-DN#Wm>}VQV5jzF6FUuloJo`ah55<0~Gv>PByS zb9;OKrD9Ontn{ya1-iWJ#iH&@pKS#}CQ5u=>J)oYp<(Jat-T6N4;mSkh|DUVTeKBLhtvFsp zHMFx>R#s76;A|kMTz?Yb0$RPzxMJ&+NETs+d*TjSoQ?urUJl{=Cc5^5t9zB>-&|&e zt}we8l&Z;eK)CRpJj;PkH3k=!rX)2GjumbA=Kfhe$CP*hsJL$Ja9 zy@ugWh7XN(N0LD;0n^PsZtd;u^*>L?*F7_y|MLL5-G}b@KTHQUpSPRsqxRgy3{+BF z%<3w2fi!BO7$yYmSuf7;xnkzcIgcN3Sazlvzx)e|pN;*sYgVt;{#)?#(^Ce!?>o;k z{&*e#zv|b^<>wP?K-v0{aOfl7jmivJI;v)KB^lnoV+v0+*kBXCZn5%{C9^)B$f&OV z&2InW;JS6|y!H2fdDt#57h88!G@OO0ky~%ag3X(Y{ggokpxA8NhiRgBRvBN9@M8RM zfuY~+z>I(prp9#(l)X>)*nhPDdD5SajqTlz$9?JZYrh?n&Xb+DQJ^r=&Q)>@!A5R`(noKEM8x*cS2eb|`CP_Qdu>r)F0r=pZ& zCts{wK2M5=4RrmrPwubv|9`D7u-H@f_SR009#EkZV)5f$f$;M?s<*s*Ss5z)#YI>b zX^4nCv$Aq3N;`J)#oY3HnOmdG^Y8t6(9F-)eDKSm6eG|@cQx;d=U;yPb^D#7(=qJe za=xPdF|4s;a7=*n^F#3DmR0P!#mXw_X>)q6UBCYS)Aao;1)tBF|E{rP;J5#?;bzXi zH|h4;r$7Duet&=6-(NfV!Ik@oM_Hc&w#YhcU3xynhq*;B1_bp8=o*Vgl@}{ z6|PV@)6O>0&Qbr|r|uaOTpw3_XWSdN{`%|ZDk7ku|2PrRsCm)2>JzBDZ=rWy*;X?c7j?ihO<*&L#)-7o486 z3&ExQ$D`uq<>mgLWxzcd0S4!yCA#;R6J~Bbq#{wFbIO^SVa9oWtMwin4UsB~x8E&` zE-x?l^gIc=`sUA{Kf;2-AUilx)WlfJyS5c=mS1&hw>6_d_tc%Xpd|bg)D+m3dwY)G z^WE?F&7OB1)S*(G6_ph0T%=;WQA2{CZGxAhKty8UI%nk+xtWDVtFFJkzBbx@Ca42q z)Y~%4N+g>-AXBrb%S^8Sb7LKY&VBY~#p@RZjrz}2%)5Bm&-(3$hlkabz+rSTqr>EM zjN$8^(rHEo=XP5kusoCY?AtRd6Q^g>CDj+l|9K=H9ToL%=ks~Og1lgNe_7^ad$Nu} zdu^!F<&XdDR3<0*Xy`B|gPopwZ23#`+*@D%zOVm(&iZ`~=+qs{=Q6X;ntipr|KphX zGbM1j^e}M8ErCq72gW=8y6I2ylzYQ`V9!qjDNrH)`CQNBi{I~6zgM^a$=R?rdVAj8 zU7?>XkI(BVD=Yi=u6%!XUY?wcjE<(}&kOB#CEsqQr&kx5Nd3LKzHV#wbv@U>u=R0! zD?g`A?_S2O)*@iEU#4f<(J3u`oU3!LCab^bb(`iMd_iCls73pI|Np-M8X>Wzc(<@XAA-#z~GmcPx%7M;_e`wmZ^KFt_Y`}Ha)!0LYAz8_a| z(baEx@ukWCKlay`l$F)Juf89>Jx{iw_?+eQZMXBP-|c*E_wQrBe7Xdve4F!N4s5qk zrk<15(dNAjGmf7=^98rU~$~> z;Sjf|Yi3r~tY3eg&#PXyXOGR?PNo&>*T?Ux`3b68)~aYKwg?nW+NxUR6g0zd+p)OQ z7FX8st z44krV-MY}#VZHG?vaYT=c<|u9efu8wfI32!b)KPC64}fFd7pNFEwi&|sNm`0bX65d zvnl6Q^ge#Z?{fE|isPU&;Ev0J5($W9qn~3Sv2O3TTOYld8V_*m@0l=t`u3WipFI8f zTAk*6_I&>CGCw;T+p+Tp{$E{RSNg|f#Y=53=Yis6Zzl2ef##!8=2Y{U#l23x;9yydsci- z)b(VkNH=fpB}@0&C3zS7?4SD`QF}YReb=N~hAdxIru3|G_LQ03iyjpvXYQ%}qB*y? zJ7IFsxqel~*@Z^GUi#Nt{hfb3uKMlP>v5NB&h<>XIB(uOPzUSzxw)sO>+k>hY<73I zsn)k;>BhasA3qj9mSo7l*Pi_6(ChX4>k0}Az9ec*-Sz+9?_9InPV$mdRWxV%yehFW zj%BiUzcf2UpvBX^=){_fYIBP>>8J~Vg75`PZcyuL-R5>5wcvdRW0UpM4?E9UZ7?$; z({g*n^AD}h7JgAn?TK95UE}LN>*yBGuT^_hJl$(=^-R9#E?1cn_A#US<56+>$|r)$ zFIWD4yM2D;vza&NP1_f*Wou`pCBw)3p~B{|cwE6j*2nVSzJ9%1|NpPrql z|L-IB_xJbTmsqLyf}4Ws!N@Jn_jXLKjT%Pt?XNEs++^@1XU~MoQf?^&du}K%ym^SYSkH68=Euj?F=0M zlwOqVJE)`+xD{^J^~WZB73_S7CAEP)wQht-^o52}{D%wNU{( zZ(U|(C}V$9a;U{4%m=#FIP|Khc3956J(ZVO7k3{vK4|_=prSDS$%S>j z{=W13?_K{k`!=8L6XOcd0NI4e`b)jolX4kO{j;zBE-B$&x}Nt>eOg*|_;s&osgc`M zzjH}Qa|n2Gm`F-^NEz~Zy@6Ubf>;DcaO5L}s4}U)2e*gP)CE?k!o7HzZF)%1By4(3}joNJ1 z;K|8m{l_a*d|aGDO`q}va4;lW{NgKqy>`3Y&&Rsk?-(T>We~jhGyl*29yQ}*RzZ8a z<9FUN=)ZUM=kxP#eou60I(7BAQ@+R&KZb^xIRSkoOPO7^TrfQDv#73Vo@MNlZ!&L} zGJFsN4RWoT$ZZ&uie_j{>x*UrvQig3&y_t7Z1Pium5%V{=aYAh2uGSvP;fo#r{@!dUEmN zNiTw~@6la;H~d5qBSYFe=9BAo%o3Wh;mFk!lR4$2l$62(kNPZgn$BkM*XQ@b2i-x< zo6ADvVz)lswPe~q*Z=q3RWIf)zZ?G|iIE{~TFWe}2{tY+z2BnLH-EU7r<7F2{#o>) z?N)}#4C0agfmL#If1j%UYIAGmyXbwdcTEs-7Gh=4SR=bJtapd=7SmhlAOF>zO7j$0 zaVtSDo54cN!pN$0@6#U|fgYWPN-BX24NR}j@vuBK5Y#O^CmNI~C#9kk7I*N>Gmz_I zWiL+*bPe3Kdw2g|-|Vap?2mp_eEy*!DERV!?3)}HAy$SNHAdlO$8UG8xE2+d_9-Kl zy`y;3%^B-|JYT>*!{WGz+wWh$va+-L|N2@!SC8WF`S#&K_u>DKTzWbxVtiGugzWq1 z&d89^wB?AkQPi~~OVZqJ9$SNzpmTwry=~$BepT z^Z1rBEzkQC@UV-OL8H$4@mj6xIZdhaa;IIp>^jRWZF4|a;EP=hT`Y%QOc4&h9CmO` zQM1ab*^jUE_ZywFOrCP}$#v!3qU*#KMKCldU7sxZ;yjPw#iZvkibBgmihl4_F~%=o zc>MUq?@QwQKD2`BK_M^3yVq{q+cHgc(bRRUJ3T|pk4NqL+beKO zDtOZ5XiWx(iNQNo@$0Pws`f(zCA<@5a5j#L#v8Vt&G}Z`=3Va+V(Gxc~Lv z?2fRVVmVfCwHLqCA{=H~#cA@E3%}bN6 zT$;VO`q_;?+;fX19d^C5Zol;iOmFPvB^E!kl2E;ZCe_?pcU412_M91^X z3H9Bo$7iZ)uhLoSa%JADO4U8ei%$xN$uc;6e0*g4alV2_ag(#Rt?Xg{WVl@%k&ZJcax& zF#E8yTwG*yJ#6=}y%&xw*)pwGbQQ^+Yc#QnqDe9di!3-yQ}VZsgco@|YQ1o+F=xdES4IY(QmyM{$D%t|oC=E4 zoUJLPs1z3ID#SLQafSxR;-hQMMQuH|`-)mr%DO#bH?Io4e=tXUmj*+_%Iy3_#w(Y( zs_ASwtM->=@8zR5_dw1&BjDCO$GvFY!|1rQZ$g3Dt6qP7kvCJjRga4yL91(}=ihzj zS-;)fcFW5QKa;TUBG0mYtkCVm&U-P^wBR*;(u`{mjh z79tE?SGWE6*T1#q`K{P^`z*Oy+gqQu`OdG{@IJFUA?kouMaG1Vh)P!D+3HJ!8fNk( zmK>WnW5=nxSKh2iKhpN(#pi|QY)?HFb~7ncv{9RSY*S1xbc$d7djLs|J__lhr{tb!jm0g)rP6{NS ziFfYttvkzLEPUw2nb53HYrgA!_R{m)qk8Y78;@{1lf%a!weN-PEx$b2d$<4D>BUt- zrQLm}SDpJ4cU)f1>W73pkC34D>IAXu##L-HItAR)O=f3U-rc_BT$HQJ-ItBi58a+5 zrN+nFkn~~Qr>{%X&aZR1{lE0@=W5H!5dTH0lb&DdGOjE7pxAh%|AN<}IgxAEtl6_+ zD)Si=j>X1v9S#MBKQ6jnGm$e;_2Zul(!Z9RVPLp?t9EZcZeWw(P%NcP$S&u?ch z+IpJq)x2eNJ?!?A8D|SliZJ*@TR&#+sBoKn*8R!F^KXB98pz(7mZfX{+b*d3UTK#EGB6zP zeV?0nM33*+zfHO8T$3a9jFqN_Ok_DG@qEtYw9wYmziQ+2R&O*2eDPMV5bIKw;!2rC$pRM{9<9a+;!`o*BoO%ah@vq8}~9UN~llr;8~$~$wE+Y z`RXdK{Oq+GCmjjtP2b$ACz;UZB*B`@cX8Xf-8m_bE=)gkJ2+|?$a^xI4{uj8OnT1V z{&|Jds=Bgy7S-t+p0xjdgJQ=gznE3l#D*UL=aE2*uASd*1>1I? zIkUuH=GL@Bsi6!8vKDiz^Y^GJ7btaczVdSqy7J=l+pB7R-AUhg^Us(Us(pC)-0pHl z()5_P6IqOD1}%XWyNot}=eb{0`{=^mkKc4J?rqy_!_43#TP-v<{q4UyJD-2w|H^yc zgKpVPX{N{gIxXrquYPz^*Eh^}!jBITr>tk6z90Di)ADaM`wxgRp9$euT;TJi)A{+0 zKQ31$OI8@}(Oss(9V^1nVA#4^_T{{^@V{37Z+TCDf7a+-Kx-@K?VZl|zJHruk>dHN zr^Di+?Y{$|pUeL3c&TP=EV?VJL6XbZ;NV6fkBBEZA7;hePX7Ja>(kN?+fvt6{*v`$ zU>vlC6(}lQrX?$EJy|!NP`y0)%Mep6b0meq`>hd;NjtN**0RZAvenX^Re<7d-k z(tk{QE!@6sc~u)#6+>PkslB{$pM3l5vSwrQunb;yOM3^R5X%#!nK>GWJ)CAM|hu5T(< z>`zZ{FI>CY_H5kA&=U1)ne|DRRfjq+K4f%tKX!P3(#uAZ)YwIP6TF!WHY@fV>YSauy0o*;d{On~sBcd% z^c`K-W|bYfu$!sjkvOOkvTt3-g z)w~lx1I9ty%&3uSJBknC-tJZW5pRn z_*+-EU;3MS^4-MBPTNVv+t#kS7Nz-Q=CRN0=f0o4_p4#5N>KZ8*R;>Gx0J^_cAd&C zJ3W2hw4c}3x<9+fo4efi+$M$t8Wv5gj~!AD8C~DAbltIsoz91Evu#@FB+klkRAJ$= zvZJ^E8UMJot|Hp&w81egUDf54;`1Ly$EB}5HQl8%qqBNl=mt5zMRU&P8^!*~OS7&? zTyZ$_=cVf&Qg-uBS*$JBoAV--k-^!*i0kpPb5T!4-p*dEE9rYr;->9+!|ZvvV#$^J zwHO$V2~0WscICW(XFmA86B163)RxXMHM>}(qaQx6V#=n&?SVJ{q`f>-98sxy^3&|| z1s1yTzoMSa-abi7c;dF=+ZOr^3^LL!ffD|zYTsw>6OCR}&HnVnz9R=aWa8h=$`{%& zlkvdDOC8^rutx>QDqr4X=@Pwn?XGiCr8U+Y`^n z9~kFk@4b7(a`Vj-uch0~SsFN-4Y-88lOAQ2Z?-FZF+rwQ@yju-^3CU4I-MEQR;-xx z!sBaMI+ND*4vz$fnY9GY;K`q9VX|lMNz3T{ zhdCM2I1ZnjoTV#T{HAWT!M*3lQ)jX#m`xqLo`$GnLmYM!k(%a`RJeRgBs z2AS;6a+jw^g09>6$Z!RG`EqjdtzcVE)3x?RS58j$n{@5E^DH+t;i;iMM}9LO_~9hc zdNNs9cX{>ncRkzXe7=^Zvn6E}ZQU&?+|DGjx@FcYkFRoG-?d{@m5xQU-ZVe{Oy9Ut z{X=kaVAt`Ej?kZyZrMyT_ypV%gE)^&vy$FBsW^R)?kbge{ChXvG2MSEC<+v#HoPY< zehcxBPb)}Su`Kf1DYoifUyEfYC;QD=mRY;k<;WemXGgZaWVpLK6ub~$bQ^R9}tWPS9Gww&( zT@R}^nyfxoqBbFL(=7=$G1eJj0&WW@eCc$qRl9!e#=e$WRfbziJ69J@+q&sODdT~_ zCw}u%M9-9NTV-ih@90SHB^Q;D>j?>g4KG;}MYz_V3fdJHv(0jw`ZAU6N58c{ zxsjCK+z>dQVcC*9CnwuXD(*WQpRB8YrDng)?577Am6Db|O*zUS%W#?JkcJPZ-^qRF zcIQOr)aR?M@R!+j>8{B?=S1cMo(6MlRvrus{$P2vqF%qScjnnOd-iOY%5Z?wL85h~ zQ_>@?^5E)K>E(NBJ^JkIzZ~=W-emOIrO%ziptW=<^Lh6BFM7W2P(3a*tA1B`=aC~# zAx0}FtIOKUalU5m4q7_Dz>PIQ%1NSiWoqBq_tRve7gZZaecO6M&qs#K_`>IfYvb(- zzI#7>`%Bl+#YHwGsp!7J>%HdJZc~qBoj-Ka zMu8Bgm`CjqOZTj$T3^ne8Jd7qMlM=MO998S4tRa4yqmY+1VQ*wb}Cm4qKN}{=;5>l6UVnem8&J#A{Re zY|OVjefr_sJ)a4}tHU%vq!!Rdy#lvISyy$^S+OBTKg>HXaAmf{>|RkUo%cim4Dt*llSh4 zr>fd_nc4eK1zAn~ZYn49V5W)Tu~mOs*ReAAarN~a3V9CeKPYASmhP3hH`CH)m!iW( zi4$$} z$xGd0OJcayhwqKgAT6I!&3|*6z9&m0MAHK;+2ps*Yl#*^!!{jQvf( zEwL)$$g|t8Z6+0G=fr!R-6L~r-wo64=c;PHpI5v3MV$Z7qj&4}pPsepKwP+975kG9 z`h0)iR?F5$yv_R;dHL+}3NgQczYNK*&+`aQyl*74)~V>t#RuN{yt%J^zU)*zekkCu z^=ZHOr+Js6vvr>8w``wxV0p^U8jZX6C%*DsDfxF|lmEODC8fl99m0aEKP5l&Z%}G5 z;JWO&bZ5%m@|(*tynQ_LU`?*$7O#Q>_~LgwHNKt9ZfE)W)A@7zR`uNZSP>)Qy8HOg z{sYDFnSYQ(2FhoRbvf zwX4m3z25Jm&Y%6y^mU5Qc589=m((O@9h;hxzUzI$uKZQL^Gfc7xW0a`s`TuuAv>EG zs|Fj#VuQ05F1M}TZ+g3t*>+~*x)OzDD&ae4_f@T4e`f0GZMB8*pB_Bces8lp#Z++V z<;=Y&m1eA4QT5yU)U|!-zo*-6c>OK%+NsNXz134Dri>4p8UU&Cv?{y_>(2(UfrTdwwPy9Ze$=mt)e)IahayR!r z+xq(7rZ>{FldJu{%UljSWhoz`^me7(-xJ!t8&6L5Yq)mbeU;auBL*8Z95y;gw7zst zdgS%JY15W;_cf-Aj>PliN570S_!e&t? z;>OL1eP`=G|C4WjQZe&n_5DZd<3h@I*_~UV(PcM3UqSlcnSHzeUHJX}^!8fM$G5gi z*&BY7=JTI7c8;hZj}bB?5k1l zg}M_?Wqrzi7`t}mx|;J(j_kkv@i(`Z|K2T+&KTaipn7w^`TMrHAGf(T>-#$Fb=6p< za_&~i-?eM^du2x|O=3BzZS{6hZ-?+j?uF)Id#m0mGe^ukp0&2R(4((X^?Bdr`uf0| zFMl;_>MrKDi_iP^ukK@ReBO?*$Fokfluv(tcgEzXU8?s!UU~5Pb~5{;tS-Z_OijzP zyMOrH@3P9i`o(vVGIO`l?&iv6*IWV^3XHn`oLC{#FMRR!_WSdyJ9jT$mUhwIF6qar z0PoZP*yRjzR@pqAa#K7u`M zKL2fg>9oY)+xGt@Yaa6+Ubc+C=D^*AnuTpHGO)D(D{_vXubU<%VUQE`^{9TAnD@KK za%Yo%-&`zv&ckQQccG~#rm=EvKOrf2^VI3-HJ>K`y1eZ;U&*rH`;7B{9B(hTt9dWI z|Mo*J{#nLluB*InUvJgEck_MuilEC~Pp96S)XpAP5&lzhvEAzDJD=~GS9xs1o?qSi z>mqV%4wT-P|1WWG`}qx-ACKGTtv9kezvJod`=OIg-Z{?d8TEJSdWn51wkBKt9=A8k zDY|;yJidBu;^hZ@vTN)Au1*a8`s?fFdwDfKbDp>H{*^Y*I%2;u<#O0}0m0DJHDSJQ zwx~Jzs4Wtjy!%?v#Yq|Gi+)~OJHP(zmFlpav&;Uz3h#WEci24U-_7&0V%@^^iuQfX zjyJC9y!klt-^}w8W=HvDZ`Gc+ZSz&_7Bk)VeQ%p@?fN}MAMXD*%UNap|Iez+{^mJW zA1*jMTbE0j>HfLUK4){9q*?Cn*Xw*N=KTJ#c)6Xkkmiy%zA}4$U%3DN-LGl~zQS{z zCl2|X6@32u_N@4sSC4slKD||5UwOEd(O%H<&p%iBzbpRnEmrgQD*v7~zhbLe?&Dck z+^&=zNuL`Le(l4%^69$KX*ae!-}igb+qZ4L-uXKYMnz@a*z$7r{VOS!-wRKsMOU8I zT`c$HAVaFrOplcB^W{T&-$j2qH{1OGtJS>r840JSeQTFjQwj}DjZEu_|L6AXpGs1>*X=xJT)r;xa^L*_HliF7W^4Z(5?5J$(9|``Z1>CP`$jdLTi4{g zTyB?oLnJr!E%H|@riAN#&f)iK#Ozx4b1dfE3zcHcJsSvLDj-uA@H$8P5t z*W7tiroaEC+Tmp>{^uKQ_Ew!d)Y|*LdVXT?|LpapcHd?sUS9M0{Q7g3r*5_Tav@>f zx-~!R{~b52-j;ZIoBUr5VJ=CtwLf2l-~2ddx5g)fRm(2Blss72F7bTvEU)Q0C&k?! z-@VH&S94*5r@`#cUmrBz-u}Eg^ysUrKM!Sm-+Z1euKw3o=3IE(*~s*#+>0-6+g7uFQ+Vi8?!}GQYt;Sr_xf_kHd6y`7^R}C0w^ctoC(F6GT4=7zmSdM|H17UC z7#m-oma+NtN%6PA&)h3dIn6cwD1VLrU;NK^-~TKBWps_Ktt|Flr2KtzTU6AiwOx~1 zxj(mFoF^Xtrt+QT;boQKaWnOMw%PyL@K8=KO6TXH6*B$LwbvJ&omG=)a_#2K_n~Rq zt>YzRZ|wSdpqbxp>$yFpbEkIXWWL(GjJ5a2!wP|*MS`je1-Bb!cP+oa^=)zcwf~!6 zpH%-`TIRiO|Gb_<8qaOwEBju(s!24lwV3ffwDe-eJ_D(H`|a@^xTN@Ld^Yq@Y6j6FTXZa3szEcmk=kBTt$!<%Wc0Ys!r|Lvq`E!x~`;z9hC)dre`eWV`yW!ln@XqFy)8AL$`Jm6>IO$Vqs6(3TdzOkUYuM1dv?)}S8tVP+Rh71oRV%-W8mQud2)-n z|C}5UMy6f$MODt{*5hvHqTc*Y_Lk> z=c(fJGbii!sQv!^d9T;$w9l;<1OBZ$bN$K`{hn|8ek6V0d|t}zYHZcYDe9omE~v`T z_+EH0<&?PFQx~Q8<}X);{ydcN=fT06^+r2mejdswxV^1r{i?X_dhrr(tA4$Fur0UE zx7+?>XHBBXd-KfH)O`k8&;MVW2X;+H#r=JIVk2W?_Zi&!ek*&fw3&q2)%cpieFnF7 zCOYqn+4t*oyqiSrCU-(UJ?bmuc4s1Ai{mPX0q4$1%{=E0p!}HdT z9#3bP{uEk%GVlENRgwAiyB|&r44x;~*l|WLeBY-}Z;M=>%rVuzzxq?@5too%8518i z+A3|DHuY(B=+|AP^NzV4R#QB`=5mwD%cp16q&B2Q$8z>oya>4_Q5;fz{|sZIs-H_^ z&WlaC@n^C=`Fq*#`SLL~CQ#Hhv1g|9vu}M*{Kd-tJ^GtiZ1m8$@^-Lo+v(Z)p-)cB zxP)vhTKce2K*`8X%)`ZHYx?}$3Jb;iRSHUz@76Kx*KEH(ALfPYS3=4Uvew(>D^JuC zSgPyda!c@eou&HJ5GAj`s01f}s~_7N)202b&jto3|7c5^7kWhEbojiZ&rgTXyZQL! z_f<8HT^)Ber$4WrT77O#&3diaT~o4W6}H~n^L*mp^m#K+{%or&%;OJy&UO7yeA)lx zCyt$+X$O~NWlhumbMKvb^Tgj-M|N>vJI&N@_G6w*&E0}eOBSy3xVLVN<@-0j z^D004Ezn*av03fDzG{4s;-Xuh&t2ZcRvh7Bxp%(Z293=TD%;<){7O&HUHebV_wpCz z#c7{E{Ehy(OIArMa*K!k-f7uA>(;#ba{2S$lXLUM(-zIhU-)L~)Sq@IqM~MP4?kc0 zqq*Zwa!KmX{bHxX*O}NTIk=>4zg;1(u*&kejHTG=+M1pjN;h*VzaO#5p4#!|>eb5a zU5iUokLAy(-x@xzDr?_kj-O$If=}O``cy8Ww%z*vYOm>fXTNKO|66xv>V9qQ=v`A* zuSyKw*Wu*y&-lC#zm)&wvn`Kz%XEC4!ojra}()aN6NG*$t`={T#neg@U!%!`b zrn`CSO1;_h^Zs63U;Y2;n-%)zmGbeQ4jx7+D%}(9VY0KhY%-CG{G@mQkA&~>vXQ$BUa|9N!7U;C5tVk_HXkzHMr z7iG@+cfaT2&B_ar9#P4;w$j?XxLH zlD>JKWna>o^e*08aV>17cjt?~TmAX{@;4;1EqA}V8*x5)_wwh7Rx-1nMg1u|`E7dR z_L$aZBDvy~*X_jWTh89qTyp_@mEvgIRvdzY5;6mGfViJ@eJQ zXS$ibrEfh;(spi(&5nPX6Zc@Avf)L=sO#bD-*|kzckkQm{k%4l1KQSyBp(UeC0foci&hS&U*7EyLJdgK(FDYC-RmC$S!{+D1IbSYrVsSjcW*N7^>v*i;+Mw>EO44_i zSa?lr51cXc*{!VDMUQ)5+ygHVP+poUvd(47kKNmAAAWqGEp^?b$~)(l(B0yzegFS{ zJjfm`lv{Rj)%_1F$r5K=+Wt@MH8{3s(&y8~b`{HtXNGT7uUr<(@n_%nkL~Pn9H$aD zC!ESE3XJD9v5OKqynRtV$G%SwXO;im@ltJe!o~fQjeN`^b>vr8UY9EWTy9mDp0>>7 z{Hi6OY5E(p)EWJcRD#xIrJ8~|pg>hUi@Y8^{ZOV9^*Zj)i31-R zOndfLZZ(nq$6s%{$vNrLO;zpa$(8Lotg#WbpO>}p|NQUm@AvCtb??ViRp04hbACQ~ z@H+hM4%1b9_d8XKTRTkF_0Fj>TT(DJ_}+_Co!WPPXjRWWWtw~D`}NZ&+#kp6|EiT= zIcrt##5Iy9Z$-RcUle+JbIJEIdncD8nd|mGbN=`Lme^f?mrXkE!W$+0t>*eEMLo^> z^>B~0{goni!QP!yjyheg08RNH_I#IF-@(D~$x)(pQEA1a3x6Zm34FZ1D8Enph^4rs z@nlcMP_WPZCh7sT@P7zDkgB}vW!(;tJY|OxF=&gUrhd0 zuxmxm+uy78D^&M5Uz-x5HrGd;)v+M!uyNb{TLpK|{;pdS&-wHHgVn4t>sIPd=@l$Y zx-#q4ugdH3_rojo1W#_76BOK}utW8@%xwFmsrFlLSD2hyw1kyeCEZh?<<*|@kIW8& z4ic?NYDS5z)dxS_uzTbqJ#Y0SZk3E9q343XJey&peBA4H(B1Tk&wLCfUwfB$x(1tG zN?E5dYgN(PvnHOi)@W*lhh7r#4c)P7(yAigYa&zoiXsYIj+xq8TB=6-NXi}aUVL%= zSFdxNw)Xi#Oa8bAdL|sYv6iyUEA4H+T;I+2Pjj{A={dc(AF&^+c&;?@ z2$GK;d(2`mMff9=!?bF z3p{QQ=b2Q^y3+jEaH(fdXx*L6PZFIz0y8xX1Cth=V43JLS)tRXz_U-$(yiy8P(@W^ z^0(vJ_jQc|Po9v})a_by{k7_%Z6W86m)E;1|IKHxT7D!Uj$yJvn7E2jpvJ4{;=U~x zET6Z&xi({3MafbAd)xn>);-zh|L&)qpe0N8G;O1B2}8d>m%5De^Z&fKH2r@3iz;=A z=TZ6Qf)kG(^7`EOf|KDWW1>jc{ml!{y14WRs?W~&_i0Y{d421xo~ur)TwbK{t7hMf zSvBXk>DwAw*y!JBJaRJ5c-8#V*ZAgNx#n^C@S}Tq-kWERlb1Y?%4veww4ui61BK*BoDSu0&o?aQXV}cYlAm z@GSb(`pEYYNrJ9{BGpC*?|-%X)_l%B)igEbfzR#?n@zSa^mJ`Mw=P^;sA_sY?t0}h z`+c`R{{CHU{w0m=t9`GJkIm|or@CLS@7bGjc8&NwSC`8bT|bi#9Cn`KC6w&6^`i?X zLx7Kf+rl4zbw9`m3aW2Ts4Dw!Y9M^^der-uUi<=qb4}m#7+#*Y=tCoepkY4e+&co_84PqqDkVZiT06?#-`ksT zkZJPCl9XVrsae_C%8OdL#a|ts*w^8mCf3FG3NkWQY~Ox8ZL{fJ8yh>(W66>wOLpz53JI>Rt~Q#vX3ZL%(=%i8{{Q=XGiTefWyfY4 z$KH5%cemy9uV23|UX~sZT6%t-t&2(iy*(38zAJfIvTN(ssU5Rs&vx(KCDnbDPtHc8 zD=GL~O4Y)E70Z_&&)RzK?93n|0S+D>9*eo>Zdz`>R#sl#emHUd`RC_n+H6?7Sb3sH zNJz+(>C^kOtO_1Ds7?O(`|rLvo~+SPQ6|2#ZL7D{`G4f_y&M%4)v;pZ#*3e>Ro*a~ zd8Uz>y<^47m6`5u7z70uuTiUC|G9sANbKI17tE$+Pn?u7^JaEB-%M?Pp1t2HK8Okn zr}{qisn+@NX@!*1L=P3AjZtewMMW3SYHMqI^l``GhZ<*RHdqUAq+FgUlg0P5!+-fxZV5 zHtgSjKX3c_lQSLHZQp)fQg+`B!=^8c0eeuap3-8^V za_IRlFSob1l1oxQR_xilIar|O*_@MpU)P;U+W6$3Q2(u|m3!~anlm%U+S>Z@$BN3z z%Dnahqu85wb{4z8z2#n(`Y4)xyHxUZzV^d+@7`_O{q4X32L*uw3mH)2oK;m_-QLz_ zBj=x+`*y{$W_JE#NgFq9+H`78rP0^Wz^JIGNyhn4=Gj(lSWyY~9-O)VAH6=?$-VmvjxO)6AKMPZDZ|~x|vzv6-nHZH$-R(7=>7yWEF#GJz{m)X2Bso}O^uph* zx0LH=XJR}&lWoz(7600H`aUeMkm0kQd+tvB=LZiGw5FcAd#5uaA;ICY$^QHHT0L2_ zmg$G&Rdy|U;k)y0iP>yk7N*75`t_!Iu`)3(zdZAI?^F*JrpAUWli9gCV%#iDlP>C< zndp&{k#XW-!IQdVUS3{Rq0T$+dODg8E?A_o)B5CzPuH$r@Bej7D)JsHx7d;`Jc6OK z_QcQry;1txp9d=sUO#_v+7x5|X6L)pEw4y(+uCl)^xsrGv-n=?H`Du^e+sWGe{tPw zlg@PI%-vaf#+6@9Gz|myw{0?ZIJhDzcV)&Wlb8M8K368EdcS{n@I^wc=ifP#F8*CG zr{l=ukFSd2duzXZk4Z96JH08)^w;l(;&u8V@3t>nwW=yOFMdt_tG3T}OD?~3P)OK( zvuOXuFTS?swgQ1Ra`UIJP*M7;bu7Wm<;b+1&m(tePU2w+isYi^l@UW^>+5_)2}COe6mipwz7D? z(~A%5&RU-Td+btqZuiBOD+>cOggSeE@ySKrE3t|V3oCnaE8@|d==8Nc%Y+lV)USSd z6uxfV_OSTCLchro;@5pJyZ89?NdBr_-c9+{U^XU2TKj8hTJ9*{r)MRWx`hdJs@;}8+zdhA=XREvJ)_p6bzuEj+ zW>*$I?RLqPDU;rRpY65Otu)ojcIlIUqSO5+wg=WOU$t)Cxx2Av(~92}TF*V3_WAeU zHrrW;AAb1#w@rG*?YG~4|81+?c>eim^U9#e$eDNZ%>S5%hlQnWjy!yC$*x_qis$WC zD)zx22EQ_4KC`j{kIqtq|ciQ^b=`*2eT;5OR z`j;QF*;TKvskt#i=jXKRZ_A|D9ayzWD>Ss&-1yjxeE~b`Ejt!jY}&JU))(Kq8`!U# zmmc2lJjZYO{rBd7-hHgFNzYt;_wKh9v!={gQ82aRec#rT?d}XyjV5^=KHR!XrP{}{ zZppJtn{sAeerXbO@6;(T*FcaT^omj{k7r-MHvM|R^=}n6t5!+<6RNeFuj1MEw=Obr z=HZ71Zv?cowNoSIw&qE0oSpXn`||JiwqCNz_pY)REL~E)zxV_H;<#)EiCixAhfc~{EH)7}Y*JsVOeKIpwo=r2Z33YXKHJfdEXWFXOtM6~h&X#)r zT2<-dTh06p7iaALR#=+)>(ji5@M-y#pG#9kMMY=&9Q(k#diCn#_qG;K51+K8=5+SP zh(C4v6U(lcmmbbpnE$HHcJ1-U59dhdomW*=P1{`AqjorV<)%%aDt2$0`1EY7UTx*^ z?b?@3bniRO^jTK3d)wugXSQc+cP0IZRKGB__v7m8=3h^GYA#*6v~0KS{dIG1Pks4i zO41HisiZc^of$klJbNC?zQ3`fFuBl&UCVP);-Th^rh6(Xt5>ymY@1VdpX+*!l2_); z>+I#FE>$*a$Gc9o{(4%jbzN+K&DD7+>Vh9X$7$xA4ZbfcyZ`s2fAepBudvCS$#yHJ);)jZvgbdKiBb8|hn2b( zj7PV4)qmSO|LEh7(Rn*l-z*pXIWdqyDeBFf)ytQDHoO1h*UJ4flRw{PIjy;_l{<2- zMwi{{{tlCMcQ-u$T_vRZZsxw@r z*M0t*E$X|s@_xVh@o)bM{`DT6>2f*KPDDj1tgq3PF>Tt*+C2gfBIdk}Uim*cpl?mZ z+vfeRYjdt7%&8V;*T3`WWqirs*#8%kjV$)FEskLQyjZ)=;&#;0j=wv# z`MxXf{8oBz>3zNZ|2B1P7kzD*CMWd>R{t;>@098we7<)kXw zij#Uw&ixb&oFce;@5jZhTBWx%4R`+E^5mQTmH5vN|0nD7<-R`461yn5G5g!m)Ytcd zEB!XAE%)4T;%33_&5FU+#(D)IDRvk9U7lzwu0D8HfFae%>j_bXPvTh&_U^EN!Zapq&>vsv5EnELNOyY9zi;U4pQ*S%*Q^HB?n z+oSxG*`aPnnn+qp>$$f1ruEZ=#b+q+&3BixlUgPC+$HSb8PDhkE2`k$yQ+A^)S z`lROW>;E{nUt;!=^S%7#_q3S2XGRj21sT$qS^{S{-U~0y`;q(q=Ys!p{Vtj*w6i}c zaNtwjdPb}K^5@m>|D9hY?|Z@L9YcZ#_m_!_CMa-(9((+a`G8MDwp(J_#?EQ?^?q>) zF1@t!+J%Kd-OG2eOkc?;!r;GDLk6^0&DE{=B*Ouo#=r7u8g6S&g(w*{&tzsyF#7UU z;Q5)f=rt1V7IJm6Y(vz4F z*dWOd788B{`r$0_5;>7wVhkEKErARSGi+?69)6S%+~{S0_`o8D1_LezhRN=iE9&J1 zC*D3Bv(71;nIV|zkOl*TkC5C8Wmh#v0jfl-vD8T+FR~aCx|~1;avrdJ z|5{OL(@nz-8Vo&*pwKxc&(HV4XO@KUA5+jupk*eY(2>Z$sUt6=pcEG6m*%~2HDiM& zFIcVQ``3!+PUfVgJ_W0_0;~16k$KcVwh0}imC@kGXL&fvP#8s z&7MscwlW;p(gN~?x|ZPbjFyfFRZu(&Lxfx|Z@DL@q|`n+F(Ht_A`p~73Y01g9Ezr~ zIIcK$U=c&XBd~!#Qo&PjYt}URJOjnuWeHH+$*7;(dM{5&sk?P*5=g%$Sbt4LMMscl zlFy2DnkBjn25-Sp`Y}MnPC!BF+1H6@<3O=`%NZ1_j}L(S(W4FaN0^P=eB7Y?%O5jcd8fmiskG=_o#c!mZjfD#&(2ibtj49`g` zFFa)c?JHwtU{J2L1ufs5Ar8uMEa0?w@khdh<-sm4p`fgxCI<5Qo~htsA|&bsP22|eEoAq#J{Kh zJfBzmdNcV{GJp5-HBD69&KL|Npam_wL`{Zs*7EtFaWETls7zXy?6o&W(bjqTx^Mc5dH3y~FN9Bfp7> ziBH+WR^v(TH}~#Od0)MMOLhCFH}w--7#}?Oc#!c_T8ClSTYXRfEEfGV>mskT*_nB^ z(f*5Ewj>;EdiLy@yL|1G;(5PQY6QI`^Ln;(9r6tD!pAV7Aece?5X(p?(S}OHa0CS zxBk-W*RMbSSP>Fh`}J!0<(DP3VDRp?nhS_h0CI z)$V^Bt5-kop0-g}dgoECJ%45AbXce>*$D*HeGLwaQmcJ)%qh&qwl_!Ua>nf$k2ZK6 zzk6hD(7oGdO?yFgm)YXF_#0lwWy474o}%JPw$(|VU>p11jY=I{6W z^`K*A>VChS9Q?$H zv+nhE7Y~M6y@Sqj9+R+cJ6O*pcF{*IxU{s?+xzrH{%u>fRM^bBV;{lXc;Ior{k^+) zYo9d#3cRY^F00G8XZ>_{Wv9nB_imp(b?RuaA83<~?SB)&jSCn2IX!=ZLdWwskBtVI zvv#}^dE0xPJJyK7hkefHeR4`lyPGR}mWwko%&2L#iL1C*`F!iPZE_X{Ha&si;n&a2 zwLWkCe$O2NUC&E*%kTf)`@YuyvdLe;&`|m6H-_u4Tc@9!tkp42tFEo{bF!`^!&<0pW3TCi$BhtzDnQbtk>zz$)8T#uXL|o=*`xU*}%_Iq2}DA z@FN@4COt5l%ksrvR`Hk*SAzX(|Gti|zrOq4QHkm=7u~(Rz4iC~NZR}LX8QcS|Ns5g z-w8Tla_5pIOPGsi6HXCNHRH>|8#;rF^{+F`g%gQM}X_I`WoYmm} zJ8^3GE6u*X&Lu!dlz&Ai!pyH0L+ zc?fi7$RY0jnv}1vuC5MWKX1w|F3;*ef?aM;_WwKi<=*P2?Ef7$omnF`@7#*5n~%IQh-i zpxX362FK#yz6ks4Up`8Ac7(WAeo0K*Safb>s(QJF#@tI*w`W{G-prVK*-a_w>eglc zlJZe(3<^s3?t8EDnsnrpHg_yHLjz;W#g!g2E^GY#ZtCLl<^RpypMPk)`d$Cv{rNL* zCgzx#H?~X=;5gA+q}im@t{}+URKE4s)vUGQ+0iRQvR_~9a$jqIan;tYh=bgY4h~O6 z90S%pwG(FA$zSwl>fO1IKmSfn;JoSEAYo(~ciz5Y!p}2#_vX)>>woX}%$eT{-tkXR z5P1IQiY>d3RcOT-x!iwud6J!28q5wpes)euNod!Sk^@5D87`_!+ClB z)KjKGp;9HC0kN*vI23oT4hwX25y@1Z9vu65?Z$;n4zHJHMimMQPSovsQ}IB6VM~yQ zTeNIkrBhwHnP6vdzS`u@j*jA6J7Zs8nfb?V?T!f@9g}a=FR*wM^mO+6E4;6Jc*Gf^ zwBrKZiW*oPJ&x6SGhC=rl*}!ysMK_I=?cyZ5UV*OG`S@({MG5!uA)aSE@yJ4t=k!P zchgJ>VSVYfvgeHrq8Kuk*4WiK>FdmDIkbr3!Yq*fmhNAkm-7Nax`kRFPS3QjIe+O= zXGif=59Ml;@X)sx7&9+zeKqau60K#o-x*n@Fl3}!T=(kg=!jgavv9Aiu_VJSNtMNG z`Si7>-2ZLv?IGe8vrI2m z%jT&1ho;9%w2rm%Fsx<1zGBOaj)*C97jDpC*dho@6w_w7T%DR|95{Dgf05Fq9iQ$> zDK1tox7>P1`OaG-g&2km(~3wFo~N#K zP517f5AxEHzGvQ)8p-prw8+gANmW;h3qD)iU$B0=s}gHNSab2(9UT_xtEV14DlN;) zaG=s5b9vbDPq!ss-Mjl}gJ61W@R>KNe?FeS)#)`!Wnr{cXn3aTQ5E(d8)N28&?tH3 z)3ep#0K)~ICkQT;J35L_ zx|FU77T@T`pQaT(;dP@#8?~5*iL5rPBPm3g-3BEw_xG?{^w6GbaFkq^I`+T1D%$C z%tt4NYiLhjy*kWflhpcklkV3xGcjI1yPrevidWZ;WfM=j%$=lHvxI@+EYI?nw->j# zh-EH*#PsfacB<-`$IG6q+p&;wL1?niseKbWrcBlD@R(y@St1%2tGfTQ)z81ne#@sV ze>eTPp8C#M7nKE|SoZaByR9xJ=yyn}fD_r)I!^B2jUp?cJ)&Cv?o^yAU;ZX8$G8 z$6t;_=vt;k)n=SM9R(cAqU-?W=ef`%>da*NQjWrn7 zhTg3S3Cy_R@oddR`ef`T>6c5M2%Ids>l1Q!Uu*aj&Y9wZ z&qV|e7O?U$Y)SQSx8kYZa$)iHldDgyS{)!Qzo7Kuj$P^L5AK?K*c*JU`SNJtv$gg{ z{m*-Rz8sgIa^5a!sp^yOf`XtV9^Ia@MR}si!suNS?o6Gr;QF!ar@R7wa4s-S(2Vu{ z{;w^33j4jeXZOk}JuwIfHbSrr^dSekmKJB+VHP zoNr6n^0>2C*r;mKRWFy4MNRUjRrlY2F@vF@rDxuXB=()WW7FE;lbX= zaj7RbQlA!;UcLTmn(ry``5b@QJ(JZ1l+(;MAMtZ@=Vka%tSI?*f{vJ?^5>M?KewXZ zz7V`}jNM?`v|@`7pFZm-9V$7`yk}2L-DA;rpD+IZ8};^zWkKn*IR5y^4;wGkPna@w z?V6uqk$+8J&+*|&wqV#&DRIJTLyOD8YX7v20?(r&JEzTe?@B81VUQK^_E8F*`2RDT zrcvbDB^hVZ7Qc{CQaaYuzfjz&G)zqY!ngOLmyN-t8oSD3*=QG**Y_q133VENs@O8e zIV3i<#ODH==BBa)`L#c<{W=@|>}ve#+2$Wsc`Z)PN{%Sm@^anplPQz#*U#4MD*60& z&$2eViuSC%m)5;Kb;!S!vH9p}H|^~G7x~@^>r1Nj&CMt|!N72qdFfm$8%KdjyFQ%I zh>KZp{h0c8Z-?9(3->vz?bn2d*B`dcOEc;eoahrcMR#_GVd8-VP3_r7)g@$;ZZF8Y zyGG~F6jv7mKNppfw`secGyeR$Jy-spd%@0@f6YGI@2cwm>gf=iSoWjge{X%%ydSeS z8ZtAOaI45TxExuuD55_u%2UuX`^3{L0n+m4T6Io;w!iT0t=KQV3F~CUUrc)RvC1{D zM|8%vaI%O-D`A4Pnc-T+-{2A}+XXx#X`BA64 zJ!;7t>8b0*KCYd8)KPqO`?uol(xpv|3+^5$U|Zg?=)%_76K1(BT5|XQ48CX)%3I6f5Q!n%6Gf>Y&mkn<93@e``1YS z)U(?ki9SA=#QEoU{FcPIw{+)*$W`P{+x4+V4<;#6umU+sg@JDK5c!J@^W!3U2VyizLJ-*^&N~@W_B$>*J=(2T|RPpeMX9`SnBjm>#q07UhR{~*?8cVmg5GS-$4fQSIcrlCk3CB zleMhxtH{`YVY&X#+C8sY_N^=N-DXj?XW~UB1`~Oe#TOcGSS)1EdhA^B$Awj2Pe_07 zpEdl8MAo;dMhqVSn^#Tc>kUi<_C0H+o15PJBM+lj6IN2GLhqc~`u>T*UW!=8Biwc29ZQ?RnWl zpm}YNM0ZC>c}}XJ;IiHCpFe$i>d#I#H*SUtJrjBk?M!5PJ?YS{rIS@AuF7itxo1T~ z$ki7c4oK)&&g|H-=g#K!t1@qWT)f8n6l?#=-sDctOHW=eU1_e#{cNRujecAWXLm2> zwxVN4JcCapfl7wWH+{a_>+r}XCTw9~ILmQTV-CAms(4k(B->dX6~0@iRk!Re{Iyj% ziF0nmbdj&i<qvbT?T>s|cR|5GeRddc8Qye<^lZX5G_yjP;Tj8lJVN8yu_@QdKnVjqzK%V}i)c=g&XhuvqAAWLUHD zWyF$?lb=);UpXs%-Vl`(lWv4ull98GXGw(Q29#J<3C3@lU!W>U3{!Fd%wEtwa zzWVF)OoRJ83~!ky1!f3UZxIkY+0DK4w(Hf%N(P6wlK5}c*wRMY5czmk#{cclmjVnO5Luc~wvv>X8mCc_Qp4Ky6 zB%A$PXR`XL_HT3R(ykuKW^Qg@H9cPN=X!AuK7r?_9Rlo&B1GQL7y$$8&pL|CGEHN(0O5L^^`9v*(`z=JGtgfKO88%zj?*l zsn5?UZtm=uA)|2aWXh$a$CXQZB_40`nG?Lk_x96(?Kx&vVa)~S_QtJU<8s>aNO6Bg zs;atiU|@>j#x>EqF6`F#_Dwh3sM2+0=dPT>yFOofPJK(ao9`L9UFzgoyQby(wV*gYZ=Xd!Smw;ky+7rB*vD5|k78s>mzorIm3{G0c)lSl z)H7_gscPs8>jvxWDg5^zt4-cjv(e(iqsjiCUvZf9R@>frs6TxPSwX81QYgoPd2 zlQ(USmylh;+bPRd&fcf~=+V<~uj}_cT%7JRPMH|_x~zLqMt8TelGyx&UH=yzkX~Q3 z-u>>@See3|7aijhR-L-5_pWHtZQmH{of8ZMKW#b((3h}wd$M_f$6y_{zf0o=UU}O|o!z5U^J^Hj|%dH=8r%tQysj(AF zoxX1FtrJ;myt_;cyE{ydf1Ad&_UK8*xJoDMx)AFv^TkCcoc4@SK6WB4dG6%aGu|aiJXO?t%cVzRovW7?_|?gm?>_PJN!|UI zPZu3ov}lIyten7u&nxFCf1dk&-s@E$k3IcdRm{+Ejpw9>iG5mklC4o?rs4XPYSV*P zf8{zEt9dG~B<4a+MWv=7r)~B8+R)82UEMx5ynmCut!~cb>!Y;xcHLgLPRc7_W&1~U$^rWtMA@=iVb z@$bL4E-JU&`jIQ|{m#qf)wcOdI~e>p$6F zj{R;rWqv06_tnz-mHiGKb&j$)^!c#m`n6B<)OXGi;h0n9rf%q|6fNR<+#p)y)iWlB z7zGcvg-frj?DgNeRc7nQ+a|}37z$5O<=)!-`-x?~xB2~pZ!V^#j&Dd$# zIDMX|zT~l8-$hg=uh4q6F01soSzL_Aw~TjXv-5T=Exr0JDD+*~#MN!P?;f!fEVQaL z)ac$6Jmv2iQ%6RI1(J%Atzzk*^j%r0x!&gB-Mo3#E-xlIT{LuSXMOHD$*26T`v32~ zafMB)r5DSrUmTbJa>|V>*(pf8tNXd>)l0nQ+4eLA)CI$l*ws?akg$?IU>YQ4)(5x={*rMb!{Qu&9W-Z+HR$Xb)xszQAT`och#{Vy$E?;%II9lW|uO|aTo0AdOv5l^r>DP{> zpNy5vdc0jXZ1t2{<(qF_eVe~xjmxEU8P&-bHg7MwdXAqdZpl@iSatQ?!~Y30>Wz7u8TjsdbLmao#Mo7wU)Jh{zsH@Q~J{`Ufki# z$FRWE*@$ax^GTH@=@a6l?Hg|W$Ym(Mc0zvd`CUt9ysx%6W6r+W_0*{}&dyVZ{I71+ zK3XU|+nyn0IXgp!_=Fy(Y$566N=g-hi5txJO}Ny1CT;QICW9-X@sk(Lc(-T!M9Z9y zmjZ3{!y|VGNf>UbT*vlo&h=}PR=36Y>78D_`tod_J7P!QTm*aXnSx~NwbsZrI*ZlU zYdjWx8WO!)*7lH9Y1&6lx8tnoK`Euj!wQdhh)G}n(e>y^_Ep(v)8l4^UqXHVm(Du2 zSFY$vX;gPa{fgx99QeZ}JM3T_WMHPXJtW0~h%($aBt?AZ} zwH;+T{Y-U<2NaZ)&TXyTeRb~b$hdWP-zsf(eRsw4#pU=NB_iFMHt(pIHffct-Hj~Mz((4Nk*NoQ< znx^w$s_64?ao{j?1SOI4Vya(GTt(L2I z-dt4TDfsJ~c-8l#XRkl^+aD70vqUd)?K1Yd>Y`&&p))o9bo;>80-Qd%M(`KcviLvq zKYa~e`|N2R z|Fx6Dr*wz=3b^-w?M_;ITemAI#;2#OVE(40m4EJ=FMYYa)7d@dg5MV5CJ!d%-c0pV zS1t1kmj}PN*zLb#*|i+ORLe!nyl!vyzPry=#yyFTjWiaY z@DOVSm4vdO#1#DM`mytt*KhskQaWEd{d&mkcl9l*Dp}R#n-*INUJXs@4VV<%V#j=@ z|BvIm?~R)h?7F*!tG?{|V4Hb+%b(;W8YxEF@_WxuvYP#V*Zk7AWygePEW5K{%`dmu z{j)5Vdq~^YIJ}DLizgWxK1=Im^rTQK_jRucveOcQ-a<^gny~QpaxM zX|6)QBQtNF(LVK8^=M4s4wa7`9-CYznl^PzJngHuUDtA^X+dyQvwUgb7S;Cse4$@| zXP-WO*vwN<@LJ0U)9lpgr{jZ#x1AIcPTthM{>J;&YeJ-cttfw`5~Y(k+ve!eJ$_rS z>{_&|j5m7A{MDa@b#5EC9%2mt<<)aoz63l zKVH8|X_13M!0V43O$k<}w)sy2%DAqZ_y-2wn9eatug=Z*@8%Zu=<#7Q z)|hn;&hlT?cvF*@nOV2~TexOdRN3ysW|6`|OFBL*zbkcn|5>XknT-z4>$7eJr1fi2J}-|yYvJAMC;r0<%+hs<*47%t8^J#B;U^|cX+{@&cTx4ilF z`f%UxX*+hjls0Rr_4M^S*Y)!9VY5mNJ!Do}`}yoI_w8~QlXj;5{PTpP=r;O7qyAFu!5cw2UU&Wf-y3{)->*8ozwYdYgMY6$B%I!M*O~3>#u}sP9o|PQCOSM(7wWK3UihBj z)9)>dEk$HEe?412{mo77(}`~Dq_3^x{cYZseQT2ECoMop*by_8WQQ`p{74*-HEB|7Y9%*2xJz$#1`={>j7*zI*d`=iUhRZ_T}B zpMLIM`F-BoCZAfxjn?hA%el;NoHpg(m*5-AUQg5AT>f5eTV`ch*$t87GQ!1=`R!Y3 zJ3qg;_|*K4z&6dNUE1rae!tz|dpvB-kC*G~d2gG1SlG^H_aotk+VSl5PisE9-dMKr z|4;u6ZG%#zxC(zJ}gdR@9z+cJKGL+*cFl z*50)LC%En9hd)29k*;QDMz9L6tkuKA{D;j%*F|st`0@B*v#)!f&-4EGp?!mI^}V{^5B~oX z-}bWWdfe->uU8Xio7sGeIC^ew;_PMXA}?PsKip;|dUNOJE2i0p%y#z4{QP!(Ki};u zZ(prmU-5S92H&0Wbvtvft=Zr^{m(=G+V?xvb1rWakF(fTQ+Z<&a$893T%Y6YqY|nmR)LXgS7;@RByT@@+^sulM`ZgVrnumNM z{xvwc^mGVwPI8%O_UMTJl-g`ySBglFAy)6z44h9>Gc{>Mx_sY$6PB&a~ z;zXO@jgxuh&gvy+%Bp{G)}Le66TfZ}7U*Vr)x~9z$6Tw)wzD{$3XDQ;?~I-IpHXmT z-u`F_8<+e0K@FL*?nNu|gOrr!_n)tvy(e#B@urstkH7k>b@hGz3!Nn?Te;GTZ>Sx8 zo%=$k?POZ^ix$7&Z|j$tmh4-(v45A2_SUV5QajXygnfmkr!LconYdGD@}dbc3a)|7 zjCFq`?r)g(=+TN*U8!G|C@O0_2>0p=ogUoLv*p_h)s%0~mLC12{$Dv-B{XEi5tVNb zlAnJHT)%vwx3WYs?_8_=H;eO(OiNy^o%r|mmi{W`-{ECDrKBoe&(;6-`>)<~Z;_)n zX3Svwo3iDiz%PJ?| ze&uQskS|JfX#Q>fWU<_pGtQC|GtA6O99@%te^38^b=~tlZAO0&q|do6{UgO}Q~m6} z>%X0sId!k*v+wrXf0wiQ9c115`SO=v#}CQK%FgOBbDFTEt#7hhM_9J={2y*6B^jpc z7AEcz^bwhTLGxganYn?PsZ&?S0=B-%ZYCxV%-S|ip4-*?G*4V#PsLR+-8A{gt~vY4 zrv0vQQAznC!p5&y``^tq^PE(6=~4l~#CKCX1Sh(Sn10f%NWy&4 zLi^s{>{I_lTRz|0-=(vnt5YyoMDO&H6C!{5mkCb%_uQmpiQw6^-W^^?R;{|>THDg- zyv6tR)GPK^zA2Tz7fJ#BPy z`EzC|8_Poyol2f+X_Izn7*6i=R5O~nQ^lyiy~9Fz)rx?9KV}?N_qy&o{qnAAmcS=b zTW&9XcVp3-n1e~?g3ph1R#e|y!oYCh&V(K(>!9Ukb|;PauN@Qjbvg34uJm#FsY|T~ zFT2GENIXAcxqi(Q`+45k;Zg6ZJBoYfp0{madpswkRist>)XUE=iyAbwrZXR``c!)P z?5~x}HH=OkQT-d=>8X}xR2Z0~)~OQwVM@|n-9sII=Twg$`zS1`V|K6PzO#;A+_#@Q zKwUzQw`wt0!aF-dudh#QI+slwm7F>JNA6*)?a5(k3S zyUuB!J7MAPTQKS5lJj}5Jk_VW7|s0Z9r-23ct+toQ_1vSmVZ`hJzTW!*2OsoR{s}Z z=S`fH^0jJ4N|2J$);rF7c6hwF>3!PXZPuj=rO6Bo3?6O^ZH;EOeBp_!Y??l`V1NIe zJf%yOb8{YC zVXCuzxyJmUXX&Oh$J8xYH@A8VO;(w-DxtxBN(#=D+-eXC>Tkrq1 z{o={u7fE+H{fuUxDEU-WD)#Ap>!Ly>CC^Fse~Fjx?iBQXcWU?2pQ@fArkhv0Sr>U& zZ_xj6UD8HC!HxHB3=acCLyE{P;K<>=X4|`tyt!! z%@;r6&l?MM*K7F_k3PNJ{7S^4_He)qoy~fIDwB7<<8@vrJm-0x(O;%X%q~Zg9#^*oKt_WyDhx2J%Ifb)DSMh1oj&PH5s=kF>PS5G?P znfbs^>X@IO&$TN8g8KT|U*7up6-<)ZV;}o))1v8am%=kYq*%>0Jm{JBS<;+mrr-6) z8KRQu%3riBMN>s2kNT+nT>QONZf`{Yyh%5;v^E8Wu6%p<=w7*L7bnY~RrFQ9Hbp3$ zf#HInf@JGA|I_xzI{coMPBhC*nl^3v>S@yg)=r*BR**M8O5O+G0i8uanTe17#ROR1}`Pwae6 zO4F)Ue#<9!Bqb-mo~kz~_2%2M?UVkwxw%C-aT$e&hZmn)yl9b6`ZQC!tQxa7Z*-Msa-?&(celX^QkR!lFnn(G%DI(zSaLqo&ozXQct z6xGz0#Z5l{eD&J3Q&mpN1zo%G`s*sjqdqQ^mgxLCQ*z(=(bUIBw2%L^nRn{(@f{~c z?wekf5p-O&O6SLnFYI%E)TFs$Wq`s#c;tCvGlw62lwZQC#JvsD?t|JH53nZv#E^*xXJyLsDhzvYXq z)BBry&v|$1uhsAGZ;=1`O!ibr=+*eE%QlNxnZNyW^Gnwvfs5znS~Dy5|M^t4m#_Ws z!-5vOGpm*@JNB^P&E4JRUwBWSJ}t|4e4%r@)9kLUE*ZY#7Z7o`Q_#-SDiYu{GZB^BgKvZXTu(;?3lY|iHnA= znx#Y2i8E&`Tv~W@6%BZelhLb zq}|V|Ypsj=<<_rJSv~cfOvbre_q)2fE-bUH{$?S=SKG3xyLessDzhgC%NrdOZrrc( z&D(zaVZobQTeZK4hlYk)%JlvH{k`36>eQ*0GJVSKeJy69Vq()Qwh0M#`>S-Tet3WV z%GYcUz6l!BuKePADlQ@_nt8nMRQ`Ya(&J(uH(hj@!?f>kJHNF|-=|NXE|}lho*)1C z@7Fe)%SJciEuRRM$@RPEbNzL_di5$_d-DB#wF`ZJEv)_V;o*kAr4}-km6Z#9D=RDc z+83`_p|QpDeST!*3~#$H|6Vnh+FF;tE3uGyH{rOV+1m?wDH^L+tjNgF(ALw-%h0f{ z`?KTlLk$fLj@(lxPiAIlSQk8ScnnHuytTXhH|&o(5nK2xU|aR~cNgN9@Tao#$#CRe zYTqcQn3K08t;adBqH2W>mlBd4d?3UXbJa5AB?WZ=~xey?BM}$}Q)^qDAFC)Ef z{F`U$%?V*H^f@2o#Q(N!%jcOFALxClu=#3t^I)>Wf3?`{p7MMa&llbfl$SL1<<7nI z-$SEIs4QWr)%~Q`myS<&P!PDagn#R=;Fn*ozWRIb)~#Dr@#f97U#j+gbzApfe)0C( zvF~4BKmT(5s#Obpe>q;=^6!;@-@5Bd-X8}k^*v};wfA0Cdv?M9DjPZD=S+fz%jRl? zzu&RlN^M=nYyU#g)RRKWCyxjTt&A==n%Oc{Tf@Fzuuv)A%iOBC?of6qn@(8dq&xo@ zexCQ>kb1d(pZlz9@wM06_qDuwb9TXl`#Jr`gWu0CzuRB6H~i)9eVo6qzxygAsIMKk z?7r5U`>S>0*VOEuvwT^p>b0HMq&F9CeNzI{LkH z^!EK<**$UXD{2OQM^qeF2 z^5@=l-hD58{j~S_%DeyVY1_0bcExt#=GryZQ(o>@`r(;z2$V3|WNtM&D1507(YSWO z`Q7dIO`A5YS+izq_Vsz!H@}Ql3=L&&Y*64>_`LSPnj_`^51FoQTdB?@uFPe7E$jO-PsjR>YsL8x&6PtQ|b2Z*`Ge2c>73A z*xTpP-d`WOe{pz=bFzOgTs@U()34|0y?wjhUgZC?Ykts=nFiC$tZTex$5!83@_yg% zyxiQkXJ#5p$FH4xzUNiET8>$Cw7*-e-F&--wzykN>7oy+LVc35!`=9H_Z&SYH1YiN z$*DiTs6CB|`?7J_`V$bc2*3`$881LvIbnB*~$S>HZ+R@rQWl#08e6``pK@9J0nxH7fi?av&?&jRgnt91T^ z+}?cq&u)9oc-@feHLKVCoVIjHlyYvF$J5QgqQl0s(cEawMC0b7FyLHR*hw<~SmEr&`?`#`@r9h^@6aZ%?SuN`}S zF}g{E>!t0MJu;>D^Ijgiebi8RORV*jJuwS+bw%ocirD%8oc1#wseG_g-7{fX_A&`s zLk0%WYTSi?lTT{8xUBN>)J)pX_=;az^TG1N>|d?b{O(!ZU%9#9XhiBmb(bTz|D{^# z9w_;J_n-LxRW)Y)M~`sY8Xx^QIc5HN(2TOgLUE;wTMkJ7f3h|1*MoT85kVe zQ$%jrr`bp5^~&EleYA1e@%c}7x=*RCzOu?|Q`Fg&caL`KU-a^xqHlLCT-<88ghcLa zLE+?GpTj@hDSuj?-hEhq@}~S%w^zvCT3_=?@cWLIEj%yVX7_dpGB8BhgTrXu-44T~ z9uwc&+oMn0>hSIEi8nfvzlozOr!ed^xBb)BpDr#v9s-e3<;}`@$}Rhtefa=i>E2=5^|Qvo%t_ zgj`=A%-y%Sp}%{QfUtIqk+bGbDm1A_y%gAtc(n}Wx6fAOmCq5I-i z)!6k5-t6q~P*du5l`)Rn^(xCLP@RFHVOnd7h?UqvfBWM3b$j+)see-Nz=d^lMDEMG zt#Y=WMv}5p3=9tLEh!>aO4n8z-%Mu8+pf{Y^llqyXCj|mH3tI&14Emm5!axiC-(3y zP?fPLXfRAG5wZ7H5cp8Fcag@ew`GfyGu6NNYH>J~lx(@Uaf|ScC1?3&Z@HN`*~~iU zQrh)-W|^+8tyi;bUuV6wk-Yl->0y5RoDv25>B};I)w3{8zInE}_?)GB&ZTF~zPH|N z@SH7cmbv+`T5j3foXKhF>GMlvmz7II?PTQ^J2T64b@}^ypP!#Mx0+U#^zcyY{~yQg ztG>PY`EvPuDOqppjVUK5rO&SgH3zuGbPTMfl|5R$e&4U#_y0|sHf>wMLnn|^r|HN4 z`*PX8{OzsMkB^SJ9lzvP^ZBf~zunK2qM}cimU{R0Eu9;&v*_uWnZ}bRO`4`3pI4%z zyTn%IE+{rGxG74qf*6Mt(C96ncfY*dd*#X%PGL0_b#-Z3kKYUPFP)of4Ps54G|4FO zP|K1fDvOyj?Z5cWnlU4zMB%M_QPHOsPT@z7ANTSt`TOG1($gT`qend;-kqJFTR4T2 zl9GD(mOOosR|T@wfXDALs1MtaIH8A$fq?;;c4{wsbHi}?<;)ThhV+H2R;^m)=FV6k z#pa*kwxe;S=*J41;%8?HEo4@$;*+uXFt7SuXKyd>?q%0s@BMl$`n9f%QOXI>&8ysU zsn$xTKg}(__w#}?e{A7VQMcolo{5TyottlOudcp)*DfnlQ&Vee>FBLHcf_o7m#cho z$y@*L+5EcX5}n-Wyu5er^0g&TP6&pE8XFsf_Bqxabkj^@M{U96pzzwceZJEeLBc4U$HlC zF>@w=QBhIanHdiYEZF$vY7X<7Yjl13bXxy0=hw4Od(H2yShL2&s>@7YU;q8b5vu2IwWHnI{ku!5FKR-CwT$O!XGAb~A%hrgUMX8C24>zB; ztNwUYT=?SD@HorlV?9q#PrqMuT6dwgxVX6RMbIV8a<)}lN*3hC`}&^U|L^O1W%s@Z zGQq_^A2joexK_SgI{nh+%bhQl&CaX(bW;8K+1ce^uZEYe2&xo6bZV;h_Po1UB_eMR zE}vg_N&<8nvvuh!7nk5=K0m*%umAh;xV-)6GsbT1*V0VvD)R2`D!sZY6y)?Nn!z0} ze!X75|NXw-&(6*^uln*rtbfbd8Rq$MRaL)E>+jFm8r7Nc>B&iu;zO<6pbcHJnHfjV zWMprw`T6O~ml9iH)kWQUyH@1i-*%5*tY;~6njs*n=uiy1*)kI}?DOvC8 z1D5x<=ik4u(E01bNgXVXGYk@&R;_aHlQA^gwr$;x9Y5}t-{1TBoOLhX5?c)o4G%xR zYsP^}U7ekt{{H$J8U|*owk`d5u=r?~XklStWMt&8enCM&j6zHAoAv8xxX$_fk&{2+0SSHd9S{=V9$PoFOA-=e9h z+1c3{7+U)I*;%{!>#M)NySuwQe`mq%9WP$Ii2wU4++|Dg^K-tl%_L?0W~-n6v?cTM zuWxUo%M&9bB?V`mO>1t-;tvV^``rHj&E4hiU$5K!?(g^e=2mme^p%y9KR-L`x7^su zYF5XdxctOK#dVUl`n^2!x)$Z!-&cEQM`2Sz)uh$+|Ns5{{XPGs#%;dMH*Lz_-APdM za=Ci-s;jH(g2BS(%@9oCpLU3%^%IoMi&e*1qrHf$*P z_U2}f;o{^>{$>92_az_in`xXLSA5pANnuUQ&Y;U%X6Ns_`C`Yd&QG6;d@ldbFDxk;S-$Y!;`{IUWB2daF{8Nde(iT&dApd)TefCjulw<^{r|7)`{&huyE%# zH*(DW9?!hIZ0hvs`qA6o{P_4dGc!}%F0R4A%ggIt)$6sg)@5Jb-u`~kUHB-3{rjbA?{DF+yZn9y#`?cFbg-FSS(24`ZQR~pZ@1s? z>+Jk_dH%naQ%&}lKRrEt`t<4JeX_etUIr;$I(=8(wyNairKNkTz8*^L^PMnz_U^*R z$D;Fgre0d&`D;?)<6~E^Tsbq-IQ{%QTV^&M1G7D44*yr16g@fdV7~gX8$Uii-kx{2 z>c@vfmBopyOpNQVXK%mVT$vsf6%`hCZBOOrPoF+jeS706o0j2fbW3Z7&$1^^p8Wm& z{d(+<`v3oC%$RXu=EY5$Hod*QonOXc!@KLswPB`)h8tNIv95UDE?-yi_v`iJJ(8av zH1l8EDvvTE1NI8~mn0aZ*)z#tl-|rM(kFU2)&bhfMm6MaR>Scmb)b_l)?ecXq z=7NB%ynOxhx#ghz)o=6Z#D|B6ebp}W+y4m&3HfqF*grx?EV-IdHXtr8uCTChecaxA z)$jLK`A$~%&znBo+;MjD=9}+!KA-ph&vW~J+iy2gr^hafH?DedVWIWA9n9$yW}m*Y zGT1!#)|VT}{k`(`_tG}!-rAyRZvK6uyBz1edkkMy)YR^kUXPWp`EYQuznx{phXmI^ zv%EVq=Fk8C`~Lqpk)w;<`^)Bl%82a5?YGm<&iZ<<`n_)azL?8f>i+(^TYTR3V0V6v zUA(KS>xK!Jzr4JB`SRuczh14rx;mV@X~N8zKilp9F#0DmIluTilV9Gh=EsMJRlbMY z`Sa72)waY2FY|eMYwPPnt=!XfqnBkyRaN~8_O~@nI>NE|ah82jQc_&i%cYjZ&vNeV zx%sv%JtYN{;hNd`&8*gi<=)zoxpCvh`}P0-PEz%D2|RP+M8$`L>|YXC>EjL+F#q-t0#mBc^j{{}FUE2RY&;Os&v-iiN?z3{o ze9iA3k@)@bxP1M;=*(HurcC*AGkyNm*iRpi%lpp-HOa1KZC$o(S)Ziwv!BoBm+!v2 zQ2+gvmBBCAJiWZ6Y^%PkiQLT8=umKW)-0*LUypPOCo0c0&zDQid2(W+{r+Hv3kx6I z{K9bIp~10Nb~kT6{wQNx_2tFI#iyt1*Z=)`UB2>(Aak;HytG+Pfz{l}>i*a2b#!&l zo;%04Z0E*}8(YEP_4W1hZL7aMI@+zj`^}~c=AK?&MzhcU`X3sqtE+o=N8#gRz0&`` z@Bjbz%VmGF*=tjOF)_#rr@1jMINaIUd2;>YO(5{;VY|HABKvOa1@>g%h(#corlPW7E{w|B`B zl`p>2^yB63NV&SY%Jo0r`~6;bPtTq0`S;`heVQ)zUYMao$HR@0K|@dP-Ph~!@)iXN z3Iadg?S9|G_uNJ&YD>ngEt$XmgZdVr*1(MoiP78h_I^Ajy}RV4lgq@36CWSzjozMj zS2R2(FeIczFI+Za?X~K6JD<-?pQpHJmpB6hLktJV{EiNZJ60@M@L=!vd&-N<@7EY>YOcIDYm&;}pXck<7O`@R6`VW&^z8bb&!m*J zbaYCr=AM-?XK;`Ondot5ny00u9Xoa`uJEX+ar(J06*dc3 zgAc-3=Iib4?d|7x?%cU~=J|1t_g5Wm<1O2LH*2e2{Jxm@NzxFJ?!g34|!e?=c3?~mV*a4>uQ-e;eGO7Xib z*Xa5u?|b=NYAr*j+_-Lq!R zx^m@;?`*TPvrJz<>eja_c;ImNs$Bh_kFVG5&bzqCwMpT~(WAQ2+urn9zXRnr=~SbCnYaW&(7Xn+v#5!8D@bpZi$88 za^uB)=jK`;KYDcUw_Dku_VTmY`6`+Z_Uccwk*6i!L(c7LpefswH_WbknY%?=6^Y?zerar$$=-0M` zJ|!0yxmKxpxzzuS7`elFn*r1?Isi&v?`Sa&`Ty<|Ts7J~kU-94os8;itYZdz6 zP+z~l!^p^}?%A=6Cc66i?{_|*H`_E@Oz>{)_q+C=c^Mg`+AsMrTzL56XV{BRTeGk4 z`+6r~d!@zMlVC{l<+O=huAlJU`F&`kKhid3SfMUbm~OBkkNAPnY+5 zKA+o=c=*fJUteAtXJ1=$JAZ%e%S%hQ-z|&2`<*9kQGDIcrFFf_@}8fY>++@k|M&Vu z8n3+TKF_`%5)#5vuL3&d8B~aBYHI%K4-E}<5&83R4~z5K^e-2LtJigPb$zPZ zTa{MX_a*rGugS(=rLUeZG~4JsUGMd^wbifJZtr-pa``+h4Gj*n)RdGzKOXl>o8?q| zI;q~yCktv6i^o+Y{`l~){BG&=I`J!uT)WwLrItKDX!ZBn(qr9A?EZW>y#M#z_gdQ8 z=l7O79`w%qI`dY*FR=qYS`2gaRTeWaEPv^LRHWrsa&lHymWzuEXhJMj#kTnE<>mZ> zu_YH>U8c;Q9i1;5IqmtmxxbH!$6txHD1UdyuJ)JDOrz9aUta1>U;XZkhM?gww_d5Q ze?Fhjzp-KAvSrUs>+kRB@BjbdFu%BNl#Al?7o8cuzPxNwSQEAN)$jNF`EPIaooyy6 zEieb<#+a9+pXQ@`g7MBh%q>5fZV74_Rh}Dn>Garu-I;1(`{&$ zazeoO^1b^1e|Me!Tlc?RzRtttSLQpvxmHV;E)^9KSrfZ^+q8-1&T-@*?&5HAOc!rT!9H ze!24Z+wI~aB0aL!Wrtcge;wNY|L=R3lJ|SRzbZc>9GI8)F2DYFw9>AZ%VrUcGMJy*-tmwb$=iv})D9a|@l@zudP@I?{2pTfF@Jz1jx{8o%Cq z-fMpE#l5}1-`(8}Iz^@5?$?S98v^3@RepX}`1n}vogIdTh9A$G-_Lm^w&?l%dOIVd zPn*x%?f!Z#y8P{}rQ(OzuU#9vr{d#{jmh2J-R^xdAm^I@{L8qY`4IzyY+K#=-3lCc z&x@>iJ#X5yu!x8i>;Ih6UN3NQP2}b#{rX3p>JrH=Q|8R6`FK?P_46;)Utg6zIMDdO zYSyxsTP-D%XU?42%+7yJfAOB5&t^BjzEknI*JVrP=VuRuLPOj6<^5dxZNEu8|I^HG zcj0kof4{t8lFR>~%cy9BiO=)Ll2ySb(w6y#+_s6c5mX-&y;V)ml%)Y*E z?e@B)BqhPzySu*1m_K+vzg|vUeEI%`N0U6gye=^-FIwo_E@xl&N9H~|g967%4Tchh zDkG^=SzGy}OkP}cmtVVbCFeSsSv^JP_W%8MTRLyYLic{Tix)3WkFWbF>~Hh1Y`1g$ zMfLeLlcr9cI%m$D2@?#muB@;uewHyyEQiZjN)N40v)CoQzPmIg zip2^n5?y;_(T&*aZ(hwVy}9+t?Vw3peq>I+(sfDP)n!uB=~Qi1t-BXfcV8&gJO93a z^Y5DU^kpws&Xn4H|NL*$`WeRO=X|y~UNY}|@wv)#pss}nFUw+3lrT7CDM*0Fni+za z4}$v`3=^a{z_G;8aK!<1P9p;Y!xFZpK(KNJQ-L1Pm@osw%Ekl{un-wWc}f{3u_!tO z2VcH-@85w&=DTE(HZ27~b+vKYjJe6`T5hHu+vT zId7U+xl2k*MMXqPN=ujf&;JJ+&#V6K_u=)I>`hCTvew_aaYLiTKPf4RqiMs|ty6y( zPRf06@zOkE;=;T4SxevEx+nWd*7~2!_n*d{;$Qx0XRhbd>D*HD!1`0)>kVISKD&8O zx|`oV=WqO)uIXPj)TH0~ILx2#IQ!hfy`CBm6<5`3zrFjb{ZWC1{ogN>H*em&aN)wH zrlxJ%w#}Uz>*3)M96WhTxv!O3{yiUoBl`RQ9GZRO(xpw6pVPd(PcL+CfA#XErM30) zWy`!)9XW6R|IXE`TlenWdtG<+s#R)!b1EJk;q(;!J8#A6)wlQ8@87$3Z(m>Es#ULk ze0&_fKCbrjGvCbj*YESNES|J{<;s=6)pK*-9y@ld`un@LH#RGQQ5mY znF$F6|Nqr4oXO+l`%PM1d$Dx|zx7k2?R%!w-0rfnKKaYa?_KS+mC0*X6l_`=xofw1 z#tjwq2R{QZ>#YB)Utf3Q*P3;^S9X3i`ycyq;;GNUa%(>6-T2#g+3-j8)cN!8U$|g! zqP1+pg^#yi?Eb%Ci&MzumRq|6v)Q%s9m3}=sE>6JFA|(CGogt6OTx_@v)O0m7MwX6 z5+9$Rm$$Fz>8aP(*PEN0_e-1i9ZgD2O=V?a@tI@s@!{e2+Ecz-y{{BRqJo2yv$CY5 zq@n@>Km}g;`+Kqb>uS%P9~K&FX=Qco$`w%K_m9){ zHIdE&N3O@$>;9OycJ14C`8tg{L6g?jR!IqoH+ObUKAEByxhduDuF}cs{vfxkiQGJA z&YZ05Y}L?!f&v3a0qyX0B7Zc@&8Hts@R?^b^Yl}%RXYx{A6)T7Rb4&&@5*i4wyji9s&|=x#+}z^l=e~abzC0)-< z<<_lRbLPyEv#Bts`tst(k00Ud<4jFWw{G3Kbm`KYH*a3OdbP5mVo%-QD1*t4D^C7P zjQ2}Wk9CNR&rcT?I`i@IIk)p|-LPRt!NaB{-@05k-JPGqzp^87qSGT$p^1j;*WLcR zYuilSC7*Z8SJ?)wS-tx7y?b`Ly1n|bF0x(g*K*g^Oa6^&Yhjy{qv!RHS9WiwO5LUx zFB4|Zo_&2yWquy5bK1q&7|T)1$@j*5qeT0=ua<>cgab#=cK3+IHeA5;*Ll8}gqj^17K^V9wM z|9e-gSg~=Vp_NsXn_Jrde}644EgwDKU-0nIjvYI8m%q2Nw&s?7=2Y*tsJFNG`nuTJ z28m5ST9exOWVLj3riA4E{qS^peBP}sCyyRox@C)rt*xz&&YK4Zo72zFTf26xZRx8k zps{hEnL$B8N)tD2%eiS(_vgp_`hSrh>T_RTTg$QV!-oR4$N4fB|Mr&ueedMxn0R$n zX!W-@jvOEPqP?$8(F``rxnWTF=m_Jf{p*EBa4d z@b>0bT(V?|N%6BYU%q_Fyu9q{wQHAV8W>oZ-qb$VTs`MZ$;I0f@}na!+AB>RZ=XKhTK~W9PiAJOrc=hbIhON3&M;2r`#0tE)4Fdr z(-|2VqoSf185w!ll21-jowRE2+N181q&KF;$ftVD!61D@!HT)utc!(^`##c(Coe+!p4=VoVb-nPxI{@cQ3E5sZ&Kw z`&;Wm*F9f{knU4y1sHtpNwVG{cYQ}Wo2YYNJ@6@5C56(oto$4y3(v9Zu+Zt9r24^DA#^| z*1LJL@xP}EEsq{Os<0Q0ZBII-sa)%SscP*%`%T+C|1ecsM%a|wn3{_2KYQ+6nt{aq zy5G8W-kY`j=GkcK>-P(M`uutF0Xc7}d19HVsj6Qt*Z%&d*b=rr&ez`gqjYFb)>8J( z0jHMv&aSBE|M4sL)D+E{*$=b%ERQ~x+nU6P2Y|k7U7_SxN2Qp zU74Ag{r&y--JC2aC|F@u6&gBq_3G7&79|xGeLB?2ef7!}0jKlxY?V14E$+8-s`t}) zxV!P?sj1rCV!D?uU%q?|bYoM0XQ$xa$>-Tz?B1Tr;?Pjh`({s{ zKCP>(TefW3sZ*z_s;btlTUY+>&coYkii(N9zPvo#&hO4K*S>z<(WJBUY;T`B<(09} z&(BX>FNTAc_wAcEB73W6T$m+eGDm%9$FY#V_Gaf!IV{=nnZ2{H&RVCx3$-I2$?%j{`=eeBv`>Oj_=#-R{JUctP{kg=-)vNPwZOQ!k>8X=94+jTm zPCF?{srg_)c=-ORudB9i|NiUi>*`;tJ9>M!ZrIS#*SBu%T2m8~HS5;#@$$|!$qWj+ zR)0KkzFlorUY?w&XlQ(VepZ&(q!cf&uEmR!gMx(e!}cV9`B5MvBU80XGBPrf9}H^B zUQSK$({5gYgVjqsGryG?d@%9YI-y0+fwi8 z=jPeYo<7|@_uAdNwSRw=rl+S%N=j;7Ja{ni?yl0JqN1dC5nj$_e|wE*&D&C(5u4MY z@W?~ZNlHqp9X$E5;lYX6^`#npeqvP@r@uVNdNoO`W5S8%)Xg_lRaI*)9|twZpPilE z)x~8MdN6JCbp80e+}yuUr^kCG#Iv{U-TQY#;^C;MC>>qh!-o$2NHUF#j0_D8wK3@J z%*)R-Tcz}2>!n8`UduJ#y?IkoS_&G=UmLwWB{g;StXY0@EDTFuU3q?f{^xgBI|P+$ zYHC)T7JBpT?QKm>O)oDmJ3G5tTc3*+k1wX5pU0b_(j_oy^5pK0j*8FEeErRnLCvOH zx5Pw5*2L_*bo}`81q%*byB4POedqEeOClm7B&4Li?Rt7?srTyDtGT(jj_8XhE?K;I zbM^OkuU@?ZP1u;5zrVY?{Lhb%SJu4y^XHG}WVO_^G`-keC9kioUAuPe=Vxa@b0!~` z-@A8DQBiTSnlBd*kBO-%FDGZ^{(lFSURC|wuVS&|%)^4XeKkAF-^ba6$Vy0bT>iN9 z^9;jeFE6hGuI{5;JUlKOx^a7E{Q7lqvAdvCR8*ArbiLfXyk(bI*~|H4EE)v3czKWJ zuaDW;Bp_v)C1Pq36cVyz`SNxFtMYd;ng%}cp`lBcfIy4As^_NFtGVy5w#d4ok(Jdr zd$zRGi`n`65;xzpP&F|TQB29p^Ly(V7%1o{U}|dmLwDcCiBe$`Dt!6w7}#!K-5nfr z;`^&x`&T#LllUvI{Pnv2=cfnOu6_4<-Um%>?aP-hOWRhJgolfJPBKnA6A%~}7#h0t z--`6}^J1f;y(c~C7S|VWvR-UkEX2Sdk`B5fp!^*VFYne38zhWUI+iYdx-xkAuc$|_ z&mGJDdUMbEC%bgqR_X4_^Zj4CNvl(T@}+P4LR&kOpSJp&)hJ)D`SEJPK6Blax~uVV zFI`KHzxeoib8t$WYQ387+uB>)f$ZIfH`!Ky`;gc6-*5A#O)SL%Yr5+HzK+kz%9=WL zYK6$7zcS0zJwN^W`uf$YS6BKE-nen&{(XBpyM6oi&CB<;db>7ZahLO)$?E=}o|@&| zv1kcMPgk$I9jtwKzf-2!(fjrPZT)r@Kkw%>xOwyDVz=Hyp^A@=&!0EXF6D$klft=q zw%VJ@e!oA|%I%{T?C9ti8XCH3@n;bmpZ@=QOI`-CH9OALS&>+!H$6KqZ{6a>&VS`z zy?(uQ>sEEYIT}?tbNPgYCyQ!_UAQUKzHqu;tk7l`uI{5o#>T-RAzwaya^kqVqpcoi`uV4TFY<7Nx&a}OI|GvAs`=gnxtZeb~b9cAr%bPwskz#axZFKuTneHb? zpBDMgu{c=G_wH(+thInsU|`@C)s>x{oO}@$a?6t^Prkjs z|NhaVpx|KVyYD~U&fl-tQd3*Ia-Laz{r?3E7PNAU2Suu?s6<3Z2iG!J9JGIxJO9GU zlZ-BtIu1Xq-*|J8TPn-vgpEIn1@iYi`}Fj*nVA_cFYm^U8+Vj%-m+!N>8Em5B`emg zTeof7w+9ECGcz-HoM*m&tLWF4m-+kuUORYDQ9+^MWXig@y}Q=0-Zg>cq|(hj>-jcy zckk$M_!9s2j^93?7yX}-mHM{$v%H>QC#1;tU4Q*MJH47Cs)F68=XbWSeH3t7v2UN8 zqDje%fO{@?syOuaDMyIToH>)7Po|@zL&COd%aL!H31{Zn?q0J-$H?f?)vKYdu1k09 zurM+zIyp%-Dk=)JP~_7S&aJ8QZ{7YUn6vbExW?RNJNE9CHGlT}+}y=%b8~WXKD2J< zFwehNQdpRopKo9KD#X63K6BHiO_wfQc=6`Vp9jtSK0ZEdD@0E5+y7axX3d-T@6VeT zzJ7XodSzwh>+9>6FAK}g$oTQ|=ihI)^D8tQWaiGDTj5r^UPMIX!?&bq9*IUX1qB6V zWo0*R+$bj}Ctv@^&=1sQE`5D%mTC41L+3x$+1c5<>;KzLKAB=5k(rr!^=duO*(FPs zeBjmd6c4^-{egeU%9SUp+a^z*Y;0sS%es8s{Q2^thjpX2bmVUekB*KG37K+%M^;w0 zok!BCp8ejw+S&8x+ZR3YxHHi+a!9CZ#b~xpKk>k=M0-u6K9FI)A6r;avxz!T?SWeCVrZut) zFKw4;(@mvwBq>Xk^zuK}XEy1Mu7-IJ4(Yw|t#+tb(g z?0FF2KVP}G^!2r?tHU>M-aL8Iq@dtn&}XN2=V9gpCA)!xK!{cAReCa#KqVU$1%+JrymlqWY z**-5da5T`WJbCiuWOe^{rJJu^yY~6{`TM)e*DqPJWX~R%{%r2Mo7P`G)7{-I=ppqL<0Z>trB@T|EY&9c&R%Wl z!q_X*b??GHC&PuXb|OnGow_wToD^)1^j)#^@3 z3+_>G>yx*SD`@`s`1p5`p13^~8<#Fk{q*D{vutP3XBS=`o;`c^uqn#@eblYLXz^lZ zW@gs+&mVJd``|U_k^S+52M;Rh=)AeRyL_))cu^5(Wy2Bwlpo@Usw+FIvznWlR_qJn z+IQv7mdwe^m!}5>T{=74JUJ=p(BZ>#=gbKT3i@!yd}VV&h*jO+ui@tA+uPdMVq;^$ z4RCj7XJ=ty;f34heD*uMoVlOl!vEHdH*@&fmG$-SU%qwvib-y5?O%3&xs=pYP@7}< z^5@TdQeugT#+gRXzwypmysC#&SXHPqZcjzw*H>5L;^G1W15efW z_G@ctt&81lmVAuop5MOh`S+Dt-1_Bgb8Z-fhKBBweqQLT^Ka+D=-V^P47&DgoqJTr z@5{t*ze)v5HSBH|z7TplyCnDc*L}SfPo6$K-YY$Q=FF3`E8gF;_4W0wc_kDmT5(I` zeuR4%7z6}ds21({_51h!y1&0(TwL6AJ>=N2W1XFyi`{y)w6#~SSRr9umNWm>?KnGdBINip68aD4_`Zbdwu<-J3K$!tE{oVtZ-zpk&)5yKH1INx8L7a zYdy8Y-Z{jT6!;5I zJ~_v()@o`*M8uCDACtYkPaivW?9-Qsf&u6n|o9Ex#TU~x@ zOJ0G~Hv;lk4&IxfvN1&(265yxhNL&6_q}=`&}~78ezLxWX#dEh;H#S@gu? z4-1RE{r<hMR9ztk9@YpFG2Qn@4fm)>kd;x`%crR=m{8s9Smc{5%h@ z_3m0tFRVoF=NOml=Gb~6;^wUam02s)KYseTHg2z#fq{X!x%hv5`|58x%F4;Nx8<@p zcC71)-Cg$f*4FN-(+>+~_$-@0KRzYpNyGaRc~Mc(yE}{1pPriPWc{O^UtUjN|NhOJ zo$0GQHtc;`wA0jd%d`_O+!vb6PLfFz73GY)$$Wpa#+1#QH_w_S^`0q6=*nd+zh<$1 zzqwXduV4TE<>lp7t5)sWwToln#^mF(X3mtgE<1DnJpcLF%$%H_o*olpW8r$=)%^B< z3Vwci`up44;*ydr+qaAF;+=i=+qZ9Nn{WPb)Md_nabe-+w6j)qe|{{VU)Ocq_4e)C z&(F?2p0shAZuGRtlOMl$At5LC@8|RR$}LZxK9!Y`adLK^`FF0FTH*WCb$|RH7FbkP zR`ReVpPr_>=luIahhIOEHcVp4yO?)>-`f{2JeDqBwrpL@&Z2*Re%2h+a5{16(xyd= zl+4Vg<+G%}y|s0^e!N}frzNvzM^{$vT)cSkv}s{w-{iA@oj84Z_wL=*m6fh8E>osX zy?Xt6dRkgrd;8~S2}wzhUcQtR7vG+Exb4Z4lsi5FH*(4x92lH3?(M0pearCd?$1-F zP9-1j`xqyflz(rJW%)Z9rxUks?W&*2owQ5;*|m^~>s{<-wkgc$-rMm|YSFfZ>iYWY zmn=y+HAOReTh7DZC+^?Bf8qpa4X<_C8v}Fm^5Wvj)2HvR`1q(>TwgC{N5kvl)6;Z0 zIXT6|#P;pmw`$cYPEJn$c{Y}npO*aKP8RI=arBS%YWCHycDdSCc{D1PAN*6DDp_dF zR#H;pBEOPj@3n<9elJt#zGl6!ez}5}`xVx%RYISNMQT}Fn#9XJ7yCSg*7s z58FGtBkBtq*L67>Cm-vvE`J9qpU=&)Z2J6Q@}x;t*4D>+rI#;WtejST+Wg_eXJ==Z zG+`Ts+&R^3(G9bzYO6+}xc0`t|GjUoX`=ZOn|7lpcM2d|XdY zPu*|Mf$WQ)SFK$eyD{nL+UV^`$;n6O%=fx~u$f&}M&`|%H+%N%5fKs5i`(<#)6>&C zc33nitXaFZx2I=Y-rZUA=jW%SOqo7?`jjatIXOBNm!?Q*-aY#Nyvn1Ujx`_FfBy9J zbck10mR4NV$qN@Icpd3(dvkB^?DwxL90hcBb+uaaGnX5upHpc0^7ZR{o64qOFK_Qc zbGy1Y*87+ISXo((jEuOrxC{+J*9WV3igk+$2{{S0Oq(VacckU;!v`GC?*IR1zI97* zVWFYjG*H{lqVUjx0}dY&q@|=jm`O=VrJtL#a^1gV$N6vG3Xq;F0abT&E36w_sbMl;q7l@%R?&qwXObUYd=5f z>^T**VQ1dIiu`+fI=AmEdfK&l^X6sCmTlYi?at0(dE2TZvz=>e|DKwv&CA1cs(w$B!G&1g*Hvsf=;$KX?vm2d(9qE3zO$!In&ea)5*D_t=4VlC zY;0xaPky@}2kzgWAJzD=mf_$Em5clf`{eEW1P)!-*3;9o@cHubq_bky#O2G??>T%Z1kD&#%wFi+?l;rO zH8@yUamkuBXKq)xy1PF^Vwzg7`^9(e&II` zo?E}~Prk&i*V@;PM&xZ)k^AxE`^A$l?Ly?Tr@TBV-PX6uSpS1-it+2*_xJy^XTDZ? zTzul}^M68Z?>f|jxjn?(Z&`ou4-N=8V13W3=10M*{qqkdyt%Ql`PL-AYV8{<%*@Ol z-3SZ~ExooTa@w?Mn>KCg>gw9@f5-KNE%mLFzRY^m*T7fyxsT6n7ks z&M@eje)w^0W{{GNt!+EM{JqQ0Rh17bOe`!SHd!mg%J4luGt*d1Ol;5OxHewtXV0Iz z$1*ZmDM(CRV%Hk5|IerH-rn9-t5(gMcdv<+`@^Cg4dGFNc*CgfStXZ-o zB`a%J-QQoUR%z`kx_s$UOl<7&K3Q#j{q@V2n`d8JbL5DN-t^m-E-9(2PoF#Y?&XhV zrKPT}uDY?iuH3m3gGwxgyJ9j$h+-9eDtEDD=wwEoOvi16s|NmaNJzjtQUTDF| z#(fo`P5A6o-1+nW|9-!(x|{iZ=dD|}K>g&S){#2Xwv<1A z->#s#YY%80#s~h(7cT||2OF2Yx$*sZ8$ZK8Ps9R^erAvo1_nAV)c`N?pw6lf8XEPp Y{^*Zd$=@Ld%|VuUy85}Sb4q9e04!CrVE_OC diff --git a/garlic/doc/slides/scalability.svg b/garlic/doc/slides/scalability.svg deleted file mode 100644 index 19fb326..0000000 --- a/garlic/doc/slides/scalability.svg +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - Efficiency - Nodes: log(n) - - - Too small - OmpSs-2 scale bad - OpenMP scale bad - - - - We can explain this difference - - - diff --git a/garlic/doc/slides/scaling-region.svg b/garlic/doc/slides/scaling-region.svg deleted file mode 100644 index 3cb2857..0000000 --- a/garlic/doc/slides/scaling-region.svg +++ /dev/null @@ -1,806 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - Problemsize: log(N) - Nodes: log(n) - - Constant workper CPU - - - OpenMPsat. limit - OmpSs-2sat. limit - - Too small - - ENOMEM - - - - - - - - - - - - - - - - - - - - - - - - - - - - Strong - A - B - C - - - - - Weak - - - - - - - Saturation - OpenMP scale bad - OmpSs-2 scale bad - ENOTIME - - - Saturationzone - - - - - 1 task/CPUlimit - - - diff --git a/garlic/doc/ug.ms b/garlic/doc/ug.ms deleted file mode 100644 index 8b7c579..0000000 --- a/garlic/doc/ug.ms +++ /dev/null @@ -1,1691 +0,0 @@ -.\" Point size fails when rending html in the code blocks -.\".nr PS 11p -.nr GROWPS 3 -.nr PSINCR 2p -.fam P -.\" =================================================================== -.\" Some useful macros -.\" =================================================================== -.\" -.\" Code start (CS) and end (CE) blocks -.de CS -.DS L -\fC -.. -.de CE -\fP -.DE -.. -.\" Code inline: -.\" .CI "inline code" -.de CI -\fC\\$1\fP\\$2 -.. -.\" =================================================================== -.\" \& -.\" .sp 3c -.\" .LG -.\" .LG -.\" .LG -.\" .LG -.\" Garlic: User guide -.\" .br -.\" .NL -.\" Rodrigo Arias Mallo -.\" .br -.\" .I "Barcelona Supercomputing Center" -.\" .br -.\" \*[curdate] -.\" .sp 17c -.\" .DE -.\" .CI \*[gitcommit] -.TL -Garlic: User Guide -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document contains all the information to configure and use the -garlic benchmark. All stages from the development to the publication -are covered, as well as the introductory steps required to setup the -machines. -.DS L -.SM -\fC -Generated on \*[curdate] -Git commit: \*[gitcommit] -\fP -.DE -.AE -.\" =================================================================== -.NH 1 -Introduction -.LP -The garlic framework is designed to fulfill all the requirements of an -experimenter in all the steps up to publication. The experience gained -while using it suggests that we move along three stages despicted in the -following diagram: -.DS L -.SM -.PS 5 -linewid=1.4; -arcrad=1; -right -S: box "Source" "code" -line "Development" invis -P: box "Program" -line "Experimentation" invis -R:box "Results" -line "Data" "exploration" invis -F:box "Figures" -# Creates a "cycle" around two boxes -define cycle { - arc cw from 1/2 of the way between $1.n and $1.ne \ - to 1/2 of the way between $2.nw and $2.n ->; - arc cw from 1/2 of the way between $2.s and $2.sw \ - to 1/2 of the way between $1.se and $1.s ->; -} -cycle(S, P) -cycle(P, R) -cycle(R, F) -.PE -.DE -In the development phase the experimenter changes the source code in -order to introduce new features or fix bugs. Once the program is -considered functional, the next phase is the experimentation, where -several experiment configurations are tested to evaluate the program. It -is common that some problems are spotted during this phase, which lead -the experimenter to go back to the development phase and change the -source code. -.PP -Finally, when the experiment is considered completed, the -experimenter moves to the next phase, which envolves the exploration of -the data generated by the experiment. During this phase, it is common to -generate results in the form of plots or tables which provide a clear -insight in those quantities of interest. It is also common that after -looking at the figures, some changes in the experiment configuration -need to be introduced (or even in the source code of the program). -.PP -Therefore, the experimenter may move forward and backwards along three -phases several times. The garlic framework provides support for all the -three stages (with different degrees of madurity). -.\" =================================================================== -.NH 2 -Machines and clusters -.LP -Our current setup employs multiple machines to build and execute the -experiments. Each cluster and node has it's own name and will be -different in other clusters. Therefore, instead of using the names of -the machines we use machine classes to generalize our setup. Those -machine clases currently correspond to a physical machine each: -.IP \(bu 12p -.B Builder -(xeon07): runs the nix-daemon and performs the builds in /nix. Requires -root access to setup the -.I nix-daemon -with multiple users. -.IP \(bu -.B Target -(MareNostrum 4 compute nodes): the nodes where the experiments -are executed. It doesn't need to have /nix installed or root access. -.IP \(bu -.B Login -(MareNostrum 4 login nodes): used to allocate resources and run jobs. It -doesn't need to have /nix installed or root access. -.IP \(bu -.B Laptop -(where the keyboard is attached, can be anything): used to connect to the other machines. -No root access is required or /nix, but needs to be able to connect to -the builder. -.LP -The machines don't need to be different of each others, as one machine -can implement several classes. For example the laptop can act as the -builder too but is not recommended. Or the login machine can also -perform the builds, but is not possible yet in our setup. -.\" =================================================================== -.NH 2 -Reproducibility -.LP -An effort to facilitate the reproducibility of the experiments has been -done, with varying degrees of success. The names of the different levels -of reproducibility have not been yet standarized, so we define our own -to avoid any confusion. We define three levels of reproducibility based -on the people and the machine involved: -.IP \(bu 12p -R0: The \fIsame\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R1: \fIDifferent\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R2: \fIDifferent\fP people on a \fIdifferent\fP machine obtain the same result -.LP -The garlic framework distinguishes two types of results: the result of -\fIbuilding a derivation\fP (usually building a binary or a library from the -sources) and the results of the \fIexecution of an experiment\fP (typically -those are the measurements performed during the execution of the program -of study). -.PP -For those two types, the meaning of -.I "same result" -is different. In the case of building a binary, we define the same -result if it is bit-by-bit identical. In the packages provided by nixos -is usually the case except some rare cases. One example is that during the build process, -a directory is listed by the order of the inodes, giving a random order -which is different between builds. These problems are tracked by the -.URL https://r13y.com/ r13y -project. About 99% of the derivations of the minimal package set achieve -the R2 property. -.PP -On the other hand, the results of the experiments are always bit-by-bit -different. So we change the definition to state that they are the same -if the conclusions that can be obtained are the same. In particular, we -assume that the results are within the confidence interval. With this -definition, all experiments are currently R1. The reproducibility level -R2 is not posible yet as the software is compiled to support only the -target machine, with an specific interconnection. -.\" =================================================================== -.bp -.NH 1 -Preliminary steps -.LP -The peculiarities of our setup require that users perform some actions -to use the garlic framework. The content of this section is only -intended for the users of our machines, but can serve as reference in -other machines. -.PP -The names of the machine classes are used in the command line prompt -instead of the actual name of the machine, to indicate that the command -needs to be executed in the stated machine class, for example: -.CS -builder% echo hi -hi -.CE -When the machine class is not important, it is ignored and only the -.CI "%" -prompt appears. -.\" =================================================================== -.NH 2 -Configure your laptop -.LP -To easily connect to the builder (xeon07) in one step, configure the SSH -client to perform a jump over the Cobi login node. The -.I ProxyJump -directive is only available in version 7.3 and upwards. Add the -following lines in the -.CI \(ti/.ssh/config -file of your laptop: -.CS -Host cobi - HostName ssflogin.bsc.es - User your-username-here - -Host xeon07 - ProxyJump cobi - HostName xeon07 - User your-username-here -.CE -You should be able to connect to the builder typing: -.CS -laptop$ ssh xeon07 -.CE -To spot any problems try with the -.CI -v -option to enable verbose output. -.\" =================================================================== -.NH 2 -Configure the builder (xeon07) -.LP -In order to use nix you would need to be able to download the sources -from Internet. Usually the download requires the ports 22, 80 and 443 -to be open for outgoing traffic. -.PP -Check that you have network access in -xeon07 provided by the environment variables \fIhttp_proxy\fP and -\fIhttps_proxy\fP. Try to fetch a webpage with curl, to ensure the proxy -is working: -.CS -xeon07$ curl x.com -x -.CE -.\" =================================================================== -.NH 3 -Create a new SSH key -.LP -There is one DSA key in your current home called "cluster" that is no -longer supported in recent SSH versions and should not be used. Before -removing it, create a new one without password protection leaving the -passphrase empty (in case that you don't have one already created) by -running: -.CS -xeon07$ ssh-keygen -Generating public/private rsa key pair. -Enter file in which to save the key (\(ti/.ssh/id_rsa): -Enter passphrase (empty for no passphrase): -Enter same passphrase again: -Your identification has been saved in \(ti/.ssh/id_rsa. -Your public key has been saved in \(ti/.ssh/id_rsa.pub. -\&... -.CE -By default it will create the public key at \f(CW\(ti/.ssh/id_rsa.pub\fP. -Then add the newly created key to the authorized keys, so you can -connect to other nodes of the Cobi cluster: -.CS -xeon07$ cat \(ti/.ssh/id_rsa.pub >> \(ti/.ssh/authorized_keys -.CE -Finally, delete the old "cluster" key: -.CS -xeon07$ rm \(ti/.ssh/cluster \(ti/.ssh/cluster.pub -.CE -And remove the section in the configuration \f(CW\(ti/.ssh/config\fP -where the key was assigned to be used in all hosts along with the -\f(CWStrictHostKeyChecking=no\fP option. Remove the following lines (if -they exist): -.CS -Host * - IdentityFile \(ti/.ssh/cluster - StrictHostKeyChecking=no -.CE -By default, the SSH client already searchs for a keypair called -\f(CW\(ti/.ssh/id_rsa\fP and \f(CW\(ti/.ssh/id_rsa.pub\fP, so there is -no need to manually specify them. -.PP -You should be able to access the login node with your new key by using: -.CS -xeon07$ ssh ssfhead -.CE -.\" =================================================================== -.NH 3 -Authorize access to the repository -.LP -The sources of BSC packages are usually downloaded directly from the PM -git server, so you must be able to access all repositories without a -password prompt. -.PP -Most repositories are open to read for logged in users, but there are -some exceptions (for example the nanos6 repository) where you must have -explicitly granted read access. -.PP -Copy the contents of your public SSH key in \f(CW\(ti/.ssh/id_rsa.pub\fP -and paste it in GitLab at -.CS -https://pm.bsc.es/gitlab/profile/keys -.CE -Finally verify the SSH connection to the server works and you get a -greeting from the GitLab server with your username: -.CS -xeon07$ ssh git@bscpm03.bsc.es -PTY allocation request failed on channel 0 -Welcome to GitLab, @rarias! -Connection to bscpm03.bsc.es closed. -.CE -Verify that you can access the nanos6 repository (otherwise you -first need to ask to be granted read access), at: -.CS -https://pm.bsc.es/gitlab/nanos6/nanos6 -.CE -Finally, you should be able to download the nanos6 git -repository without any password interaction by running: -.CS -xeon07$ git clone git@bscpm03.bsc.es:nanos6/nanos6.git -.CE -Which will create the nanos6 directory. -.\" =================================================================== -.NH 3 -Authorize access to MareNostrum 4 -.LP -You will also need to access MareNostrum 4 from the xeon07 machine, in -order to run experiments. Add the following lines to the -\f(CW\(ti/.ssh/config\fP file and set your user name: -.CS -Host mn0 mn1 mn2 - User -.CE -Then copy your SSH key to MareNostrum 4 (it will ask you for your login -password): -.CS -xeon07$ ssh-copy-id -i \(ti/.ssh/id_rsa.pub mn1 -.CE -Finally, ensure that you can connect without a password: -.CS -xeon07$ ssh mn1 -\&... -login1$ -.CE -.\" =================================================================== -.NH 3 -Clone the bscpkgs repository -.LP -Once you have Internet and you have granted access to the PM GitLab -repositories you can begin building software with nix. First ensure -that the nix binaries are available from your shell in xeon07: -.CS -xeon07$ nix --version -nix (Nix) 2.3.6 -.CE -Now you are ready to build and install packages with nix. Clone the -bscpkgs repository: -.CS -xeon07$ git clone git@bscpm03.bsc.es:rarias/bscpkgs.git -.CE -Nix looks in the current folder for a file named \f(CWdefault.nix\fP for -packages, so go to the bscpkgs directory: -.CS -xeon07$ cd bscpkgs -.CE -Now you should be able to build nanos6 (which is probably already -compiled): -.CS -xeon07$ nix-build -A bsc.nanos6 -\&... -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -The installation is placed in the nix store (with the path stated in -the last line of the build process), with the \f(CWresult\fP symbolic -link pointing to the same location: -.CS -xeon07$ readlink result -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -.\" ################################################################### -.NH 3 -Configure garlic -.LP -In order to launch experiments in the -.I target -machine, it is required to configure nix to allow a directory to be -available during the build process, where the results will be stored -before being copied in the nix store. Create a new -.CI garlic -directory in your -personal cache directory and copy the full path: -.CS -xeon07$ mkdir -p \(ti/.cache/garlic -xeon07$ readlink -f \(ti/.cache/garlic -/home/Computational/rarias/.cache/garlic -.CE -Then create the nix configuration directory (if it has not already been -created): -.CS -xeon07$ mkdir -p \(ti/.config/nix -.CE -And add the following line in the -.CI \(ti/.config/nix/nix.conf -file, replacing it with the path you copied before: -.CS -.SM -extra-sandbox-paths = /garlic=/home/Computational/rarias/.cache/garlic -.CE -This option creates a virtual directory called -.CI /garlic -inside the build environment, whose contents are the ones you specify at -the right hand side of the equal sign (in this case the -.CI \(ti/.cache/garlic -directory). It will be used to allow the results of the experiments to -be passed to nix from the -.I target -machine. -.\" ################################################################### -.NH 3 -Run the garlic daemon (optional) -.LP -The garlic benchmark has a daemon which can be used to -automatically launch the experiments in the -.I target -machine on demand, when they are required to build other derivations, so -they can be launched without user interaction. The daemon creates some -FIFO pipes to communicate with the build environment, and must be -running to be able to run the experiments. To execute it, go to the -.CI bscpkgs/garlic -directory and run -.CS -xeon07$ nix-shell -nix-shell$ -.CE -to enter the nix shell (or specify the path to the -.CI garlic/shell.nix -file as argument). Then, run the daemon inside the nix shell: -.CS -nix-shell$ garlicd -garlicd: Waiting for experiments ... -.CE -Notice that the daemon stays running in the foreground, waiting for -experiments. At this moment, it can only process one experiment at a -time. -.\" =================================================================== -.NH 2 -Configure the login and target (MareNostrum 4) -.LP -In order to execute the programs in MareNostrum 4, you first need load -some utilities in the PATH. Add to the end of the file -\f(CW\(ti/.bashrc\fP in MareNostrum 4 the following line: -.CS -export PATH=/gpfs/projects/bsc15/nix/bin:$PATH -.CE -Then logout and login again (our source the \f(CW\(ti/.bashrc\fP file) -and check that now you have the \f(CWnix-develop\fP command available: -.CS -login1$ which nix-develop -/gpfs/projects/bsc15/nix/bin/nix-develop -.CE -The new utilities are available both in the login nodes and in the -compute (target) nodes, as they share the file system over the network. -.\" =================================================================== -.bp -.NH 1 -Development -.LP -During the development phase, a functional program is produced by -modifying its source code. This process is generally cyclic: the -developer needs to compile, debug and correct mistakes. We want to -minimize the delay times, so the programs can be executed as soon as -needed, but under a controlled environment so that the same behavior -occurs during the experimentation phase. -.PP -In particular, we want that several developers can reproduce the -same development environment so they can debug each other programs -when reporting bugs. Therefore, the environment must be carefully -controlled to avoid non-reproducible scenarios. -.PP -The current development environment provides an isolated shell with a -clean environment, which runs in a new mount namespace where access to -the filesystem is restricted. Only the project directory and the nix -store are available (with some other exceptions), to ensure that you -cannot accidentally link with the wrong library or modify the build -process with a forgotten environment variable in the \f(CW\(ti/.bashrc\fP -file. -.\" =================================================================== -.NH 2 -Getting the development tools -.LP -To create a development -environment, first copy or download the sources of your program (not the -dependencies) in a new directory placed in the target machine -(MareNostrum\~4). -.PP -The default environment contains packages commonly used to develop -programs, listed in the \fIgarlic/index.nix\fP file: -.\" FIXME: Unify garlic.unsafeDevelop in garlic.develop, so we can -.\" specify the packages directly -.CS -develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi - # Add more bsc packages here... - ]; - ... -.CE -If you need additional packages, add them to the list, so that they -become available in the environment. Those may include any dependency -required to build your program. -.PP -Then use the build machine (xeon07) to build the -.I garlic.develop -derivation: -.CS -build% nix-build -A garlic.develop -\&... -build% grep ln result -ln -fs /gpfs/projects/.../bin/stage1 .nix-develop -.CE -Copy the \fIln\fP command and run it in the target machine -(MareNostrum\~4), inside the new directory used for your program -development, to create the link \fI.nix-develop\fP (which is used to -remember your environment). Several environments can be stored in -different directories using this method, with different packages in each -environment. You will need -to rebuild the -.I garlic.develop -derivation and update the -.I .nix-develop -link after the package list is changed. Once the -environment link is created, there is no need to repeat these steps again. -.PP -Before entering the environment, you will need to access the required -resources for your program, which may include several compute nodes. -.\" =================================================================== -.NH 2 -Allocating resources for development -.LP -Our target machine (MareNostrum 4) provides an interactive shell, that -can be requested with the number of computational resources required for -development. To do so, connect to the login node and allocate an -interactive session: -.CS -% ssh mn1 -login% salloc ... -target% -.CE -This operation may take some minutes to complete depending on the load -of the cluster. But once the session is ready, any subsequent execution -of programs will be immediate. -.\" =================================================================== -.NH 2 -Accessing the developement environment -.PP -The utility program \fInix-develop\fP has been designed to access the -development environment of the current directory, by looking for the -\fI.nix-develop\fP file. It creates a namespace where the required -packages are installed and ready to be used. Now you can access the -newly created environment by running: -.CS -target% nix-develop -develop% -.CE -The spawned shell contains all the packages pre-defined in the -\fIgarlic.develop\fP derivation, and can now be accessed by typing the -name of the commands. -.CS -develop% which gcc -/nix/store/azayfhqyg9...s8aqfmy-gcc-wrapper-9.3.0/bin/gcc -develop% which gdb -/nix/store/1c833b2y8j...pnjn2nv9d46zv44dk-gdb-9.2/bin/gdb -.CE -If you need additional packages, you can add them in the -\fIgarlic/index.nix\fP file as mentioned previously. To keep the -same current resources, so you don't need to wait again for the -resources to be allocated, exit only from the development shell: -.CS -develop% exit -target% -.CE -Then update the -.I .nix-develop -link and enter into the new develop environment: -.CS -target% nix-develop -develop% -.CE -.\" =================================================================== -.NH 2 -Execution -.LP -The allocated shell can only execute tasks in the current node, which -may be enough for some tests. To do so, you can directly run your -program as: -.CS -develop$ ./program -.CE -If you need to run a multi-node program, typically using MPI -communications, then you can do so by using srun. Notice that you need -to allocate several nodes when calling salloc previously. The srun -command will execute the given program \fBoutside\fP the development -environment if executed as-is. So we re-enter the develop environment by -calling nix-develop as a wrapper of the program: -.\" FIXME: wrap srun to reenter the develop environment by its own -.CS -develop$ srun nix-develop ./program -.CE -.\" =================================================================== -.NH 2 -Debugging -.LP -The debugger can be used to directly execute the program if is executed -in only one node by using: -.CS -develop$ gdb ./program -.CE -Or it can be attached to an already running program by using its PID. -You will need to first connect to the node running it (say target2), and -run gdb inside the nix-develop environment. Use -.I squeue -to see the compute nodes running your program: -.CS -login$ ssh target2 -target2$ cd project-develop -target2$ nix-develop -develop$ gdb -p $pid -.CE -You can repeat this step to control the execution of programs running in -different nodes simultaneously. -.PP -In those cases where the program crashes before being able to attach the -debugger, enable the generation of core dumps: -.CS -develop$ ulimit -c unlimited -.CE -And rerun the program, which will generate a core file that can be -opened by gdb and contains the state of the memory when the crash -happened. Beware that the core dump file can be very large, depending on -the memory used by your program at the crash. -.\" =================================================================== -.NH 2 -Git branch name convention -.LP -The garlic benchmark imposes a set of requirements to be meet for each -application in order to coordinate the execution of the benchmark and -the gathering process of the results. -.PP -Each application must be available in a git repository so it can be -included into the garlic benchmark. The different combinations of -programming models and communication schemes should be each placed in -one git branch, which are referred to as \fIbenchmark branches\fP. At -least one benchmark branch should exist and they all must begin with the -prefix \f(CWgarlic/\fP (other branches will be ignored). -.PP -The branch name is formed by adding keywords separated by the "+" -character. The keywords must follow the given order and can only -appear zero or once each. At least one keyword must be included. The -following keywords are available: -.IP \f(CWmpi\fP 5m -A significant fraction of the communications uses only the standard MPI -(without extensions like TAMPI). -.IP \f(CWtampi\fP -A significant fraction of the communications uses TAMPI. -.IP \f(CWsend\fP -A significant part of the MPI communication uses the blocking family of -methods -.I MPI_Send , ( -.I MPI_Recv , -.I MPI_Gather "...)." -.IP \f(CWisend\fP -A significant part of the MPI communication uses the non-blocking family -of methods -.I MPI_Isend , ( -.I MPI_Irecv , -.I MPI_Igather "...)." -.IP \f(CWrma\fP -A significant part of the MPI communication uses remote memory access -(one-sided) methods -.I MPI_Get , ( -.I MPI_Put "...)." -.IP \f(CWseq\fP -The complete execution is sequential in each process (one thread per -process). -.IP \f(CWomp\fP -A significant fraction of the execution uses the OpenMP programming -model. -.IP \f(CWoss\fP -A significant fraction of the execution uses the OmpSs-2 programming -model. -.IP \f(CWtask\fP -A significant part of the execution involves the use of the tasking -model. -.IP \f(CWtaskfor\fP -A significant part of the execution uses the taskfor construct. -.IP \f(CWfork\fP -A significant part of the execution uses the fork-join model (including -hybrid programming techniques with parallel computations and sequential -communications). -.IP \f(CWsimd\fP -A significant part of the computation has been optimized to use SIMD -instructions. -.LP -In the -.URL #appendixA "Appendix A" -there is a flowchart to help the decision -process of the branch name. Additional user defined keywords may be -added at the end using the separator "+" as well. User keywords must -consist of capital alphanumeric characters only and be kept short. These -additional keywords must be different (case insensitive) to the already -defined above. Some examples: -.CS -garlic/mpi+send+seq -garlic/mpi+send+omp+fork -garlic/mpi+isend+oss+task -garlic/tampi+isend+oss+task -garlic/tampi+isend+oss+task+COLOR -garlic/tampi+isend+oss+task+COLOR+BTREE -.CE -.\" =================================================================== -.NH 2 -Initialization time -.LP -It is common for programs to have an initialization phase prior to the -execution of the main computation task which is the objective of the study. -The initialization phase is usually not considered when taking -measurements, but the time it takes to complete can limit seriously the -amount of information that can be extracted from the computation phase. -As an example, if the computation phase is in the order of seconds, but -the initialization phase takes several minutes, the number of runs would -need to be set low, as the units could exceed the time limits. Also, the -experimenter may be reluctant to modify the experiments to test other -parameters, as the waiting time for the results is unavoidably large. -.PP -To prevent this problem the programs must reduce the time of the -initialization phase to be no larger than the computation time. To do -so, the initialization phase can be optimized either with -parallelization, or it can be modified to store the result of the -initialization to the disk to be later at the computation phase. In the -garlic framework an experiment can have a dependency over the results of -another experiment (the results of the initialization). The -initialization results will be cached if the derivation is kept -invariant, when modifying the computation phase parameters. -.\" =================================================================== -.NH 2 -Measurement of the execution time -.LP -The programs must measure the wall time of the computation phase following a -set of rules. The way in which the wall time is measured is very important to -get accurate results. The measured time must be implemented by using a -monotonic clock which is able to correct the drift of the oscillator of -the internal clock due to changes in temperature. This clock must be -measured in C and C++ with: -.CS -clock_gettime(CLOCK_MONOTONIC, &ts); -.CE -A helper function can be used the approximate value of the clock in a -double precision float, in seconds: -.CS -double get_time() -{ - struct timespec tv; - if(clock_gettime(CLOCK_MONOTONIC, &tv) != 0) - { - perror("clock_gettime failed"); - exit(EXIT_FAILURE); - } - return (double)(ts.tv_sec) + - (double)ts.tv_nsec * 1.0e-9; -} -.CE -The start and end points must be measured after the synchronization of -all the processes and threads, so the complete computation work can be -bounded to fit inside the measured interval. An example for a MPI -program: -.CS -double start, end, delta_time; -MPI_Barrier(); -start = get_time(); -run_simulation(); -MPI_Barrier(); -end = get_time(); -delta_time = end - start; -.CE -.\" =================================================================== -.NH 2 -Format of the execution time -.LP -The measured execution time must be printed to the standard output -(stdout) in scientific notation with at least 7 significative digits. -The following the printf format (or the strict equivalent in other languages) -must be used: -.CS -printf("time %e\\n", delta_time); -.CE -The line must be printed alone and only once: for MPI programs, -only one process shall print the time: -.CS -if(rank == 0) printf("time %e\\n", delta_time); -.CE -Other lines can be printed in the stdout, but without the -.I time -prefix, so that the following pipe can be used to capture the line: -.CS -% ./app | grep "^time" -1.234567e-01 -.CE -Ensure that your program follows this convention by testing it with the -above -.I grep -filter; otherwise the results will fail to be parsed when building -the dataset with the execution time. -.\" =================================================================== -.bp -.NH 1 -Experimentation -.LP -During the experimentation, a program is studied by running it and -measuring some properties. The experimenter is in charge of the -experiment design, which is typically controlled by a single -.I nix -file placed in the -.CI garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 2 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CI /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CI /usr -or -.CI /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 2 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.\"##################################################################### -.NH 3 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CI /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.\"##################################################################### -.NH 3 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 3 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 3 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.DS L -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Where Safe Copies User Std -_ -trebuchet * no no yes yes -runexp login no no no yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate target no no no yes -control target yes no no yes -srun target yes no no yes -isolate target no yes no yes -_ -exec target yes yes no no -program target yes yes no no -_ -.TE -.DE -.QS -.SM -.B "Table 1" : -The stages of a complete execution pipeline. The -.I where -column determines where the stage is running, -.I safe -states if the stage begins the execution inside the sandbox, -.I user -if it can be executed directly by the user, -.I copies -if there are several instances running in parallel and -.I std -if is part of the standard execution pipeline. -.QE -.\" ################################################################### -.NH 2 -Writing the experiment -.LP -The experiments are generally written in the -.I nix -language as it provides very easy management for the packages an their -customization. An experiment file is formed by several parts, which -produce the execution pipeline when built. The experiment file describes -a function (which is typical in nix) and takes as argument an -attribute set with some common packages, tools and options: -.CS -{ stdenv, lib, bsc, stdexp, targetMachine, stages, garlicTools }: -.CE -The -.I bsc -attribute contains all the BSC and nixpkgs packages, as defined in the -overlay. The -.I stdexp -contains some useful tools and functions to build the experiments, like -the standard execution pipeline, so you don't need to redefine the -stages in every experiment. The configuration of the target machine is -specified in the -.I targetMachine -attribute which includes information like the number of CPUs per node or -the cache line length. It is used to define the experiments in such a -way that they are not tailored to an specific machine hardware -(sometimes this is not posible). All the execution stages are available -in the -.I stages -attribute which are used when some extra stage is required. And finally, -the -.I garlicTools -attribute provide some functions to aid common tasks when defining the -experiment configuration -.\" ################################################################### -.NH 3 -Experiment configuration -.LP -The next step is to define some variables in a -.CI let -\&... -.CI in -\&... -.CI ; -construct, to be used later. The first one, is the variable -configuration of the experiment called -.I varConf , -which include all -the factors that will be changed. All the attributes of this set -.I must -be arrays, even if they only contain one element: -.CS -varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; -}; -.CE -In this example, the variable -.I blocks -will be set to the values 1, 2 and 4; while -.I nodes -will remain set to 1 always. These variables are used later to build the -experiment configuration. The -.I varConf -is later converted to a list of attribute sets, where every attribute -contains only one value, covering all the combinations (the Cartesian -product is computed): -.CS -[ { blocks = 1; nodes = 1; } - { blocks = 2; nodes = 1; } - { blocks = 4; nodes = 1; } ] -.CE -These configurations are then passed to the -.I genConf -function one at a time, which is the central part of the description of -the experiment: -.CS -genConf = var: fix (self: targetMachine.config // { - expName = "example"; - unitName = self.expName + "-b" + toString self.blocks; - blocks = var.blocks; - cpusPerTask = 1; - tasksPerNode = self.hw.socketsPerNode; - nodes = var.nodes; -}); -.CE -It takes as input -.I one -configuration from the Cartesian product, for example: -.CS -{ blocks = 2; nodes = 1; } -.CE -And returns the complete configuration for that input, which usually -expand the input configuration with some derived variables along with -other constant parameters. The return value can be inspected by calling -the function in the interactive -.I "nix repl" -session: -.CS -nix-repl> genConf { blocks = 2; nodes = 1; } -{ - blocks = 2; - cpusPerTask = 1; - expName = "example"; - hw = { ... }; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - name = "mn4"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - nodes = 1; - sshHost = "mn1"; - tasksPerNode = 2; - unitName = "example-b2"; -} -.CE -Some configuration parameters were added by -.I targetMachine.config , -such as the -.I nixPrefix , -.I sshHost -or the -.I hw -attribute set, which are specific for the cluster they experiment is -going to run. Also, the -.I unitName -got assigned the proper name based on the number of blocks, but the -number of tasks per node were assigned based on the hardware description -of the target machine. -.PP -By following this rule, the experiments can easily be ported to machines -with other hardware characteristics, and we only need to define the -hardware details once. Then all the experiments will be updated based on -those details. -.\" ################################################################### -.NH 3 -Adding the stages -.LP -Once the configuration is ready, it will be passed to each stage of the -execution pipeline which will take the parameters it needs. The -connection between the parameters and how they are passed to each stage -is done either by convention or manually. There is a list of parameters that -are recognized by the standard pipeline stages. For example the -attribute -.I nodes , -it is recognized as the number of nodes in the standard -.I sbatch -stage when allocating resources: -.DS L -.TS -center; -lB lB cB cB lB -l l c c l. -_ -Stage Attribute Std Req Description -_ -* nixPrefix yes yes Path to the nix store in the target -unit expName yes yes Name of the experiment -unit unitName yes yes Name of the unit -control loops yes yes Number of runs of each unit -sbatch cpusPerTask yes yes Number of CPUs per task (process) -sbatch jobName yes yes Name of the job -sbatch nodes yes yes Number of nodes allocated -sbatch ntasksPerNode yes yes Number of tasks (processes) per node -sbatch qos yes no Name of the QoS queue -sbatch reservation yes no Name of the reservation -sbatch time yes no Maximum allocated time (string) -_ -exec argv no no Array of arguments to execve -exec env no no Environment variable settings -exec pre no no Code before the execution -exec post no no Code after the execution -_ -.TE -.DE -.QS -.SM -.B "Table 2" : -The attributes recognized by the stages in the execution pipeline. The -column -.I std -indicates if they are part of the standard execution pipeline. Some -attributes are required as indicated by the -.I req -column. -.QE -.LP -Other attribute names can be used to specify custom information used in -additional stages. The two most common stages required to complete the -pipeline are the -.I exec -and the -.I program . -Let see an example of -.I exec : -.CS -exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; -}; -.CE -The -.I exec -stage is defined as a function that uses the predefined -.I stages.exec -stage, which accepts the -.I argv -array, and sets the argv of the program. In our case, we fill the -.I argv -array by setting the -.I --blocks -parameter to the number of blocks, specified in the configuration in the -attribute -.I blocks . -The name of this attribute can be freely choosen, as long as the -.I exec -stage refers to it properly. The -.I nextStage -attribute is mandatory in all stages, and is automatically set when -building the pipeline. -.PP -The last step is to configure the actual program to be executed, -which can be specified as another stage: -.CS -program = {nextStage, conf, ...}: bsc.apps.example; -.CE -Notice that this function only returns the -.I bsc.apps.example -derivation, which will be translated to the path where the example -program is installed. If the program is located inside a directory -(typically -.I bin ), -it must define the attribute -.I programPath -in the -.I bsc.apps.example -derivation, which points to the executable program. An example: -.CS -stdenv.mkDerivation { -\& ... - programPath = "/bin/example"; -\& ... -}; -.CE -.\" ################################################################### -.NH 3 -Building the pipeline -.LP -With the -.I exec -and -.I program -stages defined and the ones provided by the standard pipeline, the -complete execution pipeline can be formed. To do so, the stages are -placed in an array, in the order they will be executed: -.CS -pipeline = stdexp.stdPipeline ++ [ exec program ]; -.CE -The attribute -.I stdexp.stdPipeline -contains the standard pipeline stages, and we only append our two -defined stages -.I exec -and -.I program . -The -.I pipeline -is an array of functions, and must be transformed in something that can -be executed in the target machine. For that purpose, the -.I stdexp -provides the -.I genExperiment -function, which takes the -.I pipeline -array and the list of configurations and builds the execution pipeline: -.CS -stdexp.genExperiment { inherit configs pipeline; } -.CE -The complete example experiment can be shown here: -.CS -{ stdenv, lib, stdexp, bsc, targetMachine, stages }: -with lib; -let - # Initial variable configuration - varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; - }; - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "example"; - unitName = "${expName}-b${toString blocks}"; - inherit (targetMachine.config) hw; - inherit (c) blocks nodes; - loops = 30; - ntasksPerNode = hw.socketPerNode; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; - }; - program = {nextStage, conf, ...}: bsc.garlic.apps.example; - pipeline = stdexp.stdPipeline ++ [ exec program ]; -in - stdexp.genExperiment { inherit configs pipeline; } -.CE -.\" ################################################################### -.NH 3 -Adding the experiment to the index -.LP -The experiment file must be located in a named directory inside the -.I garlic/exp -directory. The name is usually the program name. Once the experiment is -placed in a nix file, it must be added to the index of experiments, so -it can be build. The index is hyerarchically organized as attribute -sets, with -.I exp -containing all the experiments; -.I exp.example -the experiments of the -.I example -program; and -.I exp.example.test1 -referring to the -.I test1 -experiment of the -.I example -program. Additional attributes can be added, like -.I exp.example.test1.variantA -to handle more details. -.PP -For this example we are going to use the attribute path -.I exp.example.test -and add it to the index, in the -.I garlic/exp/index.nix -file. We append to the end of the attribute set, the following -definition: -.CS -\&... - example = { - test = callPackage ./example/test.nix { }; - }; -} -.CE -The experiment can now be built with: -.CS -builder% nix-build -A exp.example.test -.CE -.\" ################################################################### -.NH 2 -Recommendations -.PP -The complete results generally take a long time to be finished, so it is -advisable to design the experiments iteratively, in order to quickly -obtain some feedback. Some recommendations: -.BL -.LI -Start with one unit only. -.LI -Set the number of runs low (say 5) but more than one. -.LI -Use a small problem size, so the execution time is low. -.LI -Set the time limit low, so deadlocks are caught early. -.LE -.PP -As soon as the first runs are complete, examine the results and test -that everything looks good. You would likely want to check: -.BL -.LI -The resources where assigned as intended (nodes and CPU affinity). -.LI -No errors or warnings: look at stderr and stdout logs. -.LI -If a deadlock happens, it will run out of the time limit. -.LE -.PP -As you gain confidence over that the execution went as planned, begin -increasing the problem size, the number of runs, the time limit and -lastly the number of units. The rationale is that each unit that is -shared among experiments gets assigned the same hash. Therefore, you can -iteratively add more units to an experiment, and if they are already -executed (and the results were generated) is reused. -.\" ################################################################### -.bp -.NH 1 -Post-processing -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 2 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 2 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 2 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.NH 2 -Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. -.\" ################################################################### -.bp -.SH 1 -Appendix A: Branch name diagram -.LP -.TAG appendixA -.DS B -.SM -.PS 4.4/25.4 -copy "gitbranch.pic" -.PE -.DE diff --git a/garlic/ds/ctf-mode.py b/garlic/ds/ctf-mode.py deleted file mode 100644 index 669508f..0000000 --- a/garlic/ds/ctf-mode.py +++ /dev/null @@ -1,83 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - ctf_mode = {} - - with open(".garlic/time_mode_runtime.csv", "r") as f: - ctf_mode['runtime'] = float(f.readline()) - - with open(".garlic/time_mode_dead.csv", "r") as f: - ctf_mode['dead'] = float(f.readline()) - - with open(".garlic/time_mode_task.csv", "r") as f: - ctf_mode['task'] = float(f.readline()) - - tree['ctf_mode'] = ctf_mode - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/index.nix b/garlic/ds/index.nix deleted file mode 100644 index 95cb388..0000000 --- a/garlic/ds/index.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -rec { - - py = callPackage ./py.nix {}; - - std.timetable = py { script = ./std-timetable.py; compress = false; }; - osu.latency = py { script = ./osu-latency.py; }; - osu.bw = py { script = ./osu-bw.py; }; - perf.stat = py { script = ./perf-stat.py; }; - ctf.mode = py { script = ./ctf-mode.py; }; -} diff --git a/garlic/ds/osu-bw.py b/garlic/ds/osu-bw.py deleted file mode 100644 index 4b1df43..0000000 --- a/garlic/ds/osu-bw.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - bw = slices[1] - - tree['size'] = int(size) - tree['bw'] = float(bw) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/osu-latency.py b/garlic/ds/osu-latency.py deleted file mode 100644 index 2df08ee..0000000 --- a/garlic/ds/osu-latency.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - latency = slices[1] - - tree['size'] = int(size) - tree['latency'] = float(latency) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/perf-stat.py b/garlic/ds/perf-stat.py deleted file mode 100644 index a6d69bc..0000000 --- a/garlic/ds/perf-stat.py +++ /dev/null @@ -1,90 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open(".garlic/perf.csv", "r") as f: - lines = [line.strip() for line in f.readlines()] - - perf_data = {} - - for line in lines: - if len(line) == 0: continue - if line[0] == '#': continue - - slices = line.split(',') - if len(slices) != 7: - print("error: mismatched columns") - exit(1) - - name = slices[2].replace("-", "_") - value = float(slices[0]) - - perf_data[name] = value - - tree['perf'] = perf_data - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/py.nix b/garlic/ds/py.nix deleted file mode 100644 index 012a159..0000000 --- a/garlic/ds/py.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, python3 -, gzip -}: - -{ - script, - compress ? true -}: - -tree: - -stdenv.mkDerivation { - name = "dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = [ python3 gzip ]; - installPhase = '' - mkdir -p $out - ln -s ${tree} $out/tree - ln -s ${script} $out/script - - COMPRESS_DATASET=${toString compress} - - if [ $COMPRESS_DATASET ]; then - python $out/script $out/tree | gzip > $out/dataset.json.gz - ln -s dataset.json.gz $out/dataset - else - python $out/script $out/tree > $out/dataset.json - ln -s dataset.json $out/dataset - fi - ''; -} diff --git a/garlic/ds/std-timetable.py b/garlic/ds/std-timetable.py deleted file mode 100644 index c183097..0000000 --- a/garlic/ds/std-timetable.py +++ /dev/null @@ -1,77 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - with open(".garlic/total_time_start", "r") as f: - total_time_start = float(f.readline().strip()) - - with open(".garlic/total_time_end", "r") as f: - total_time_end = float(f.readline().strip()) - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - tree['total_time'] = total_time_end - total_time_start - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/exp/bigsort/genseq.nix b/garlic/exp/bigsort/genseq.nix deleted file mode 100644 index 99a59f1..0000000 --- a/garlic/exp/bigsort/genseq.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, n # must be a string -, dram # must be a string -, strace -}: - -with lib; - -# Ensure the arguments are strings, to avoid problems with large numbers -assert (isString n); -assert (isString dram); - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "genseq"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - #FIXME: We need a better mechanism to get the output paths - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/seq.dat"; - in - stages.exec { - inherit nextStage; - pre = '' - mkdir -p "${outDir}" - ''; - argv = [ n dram outFile ]; - post = '' - # Link the output here - ln -s "${outFile}" seq.dat - ''; - }; - - program = {...}: bsc.apps.bigsort.genseq; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/shuffle.nix b/garlic/exp/bigsort/shuffle.nix deleted file mode 100644 index a841720..0000000 --- a/garlic/exp/bigsort/shuffle.nix +++ /dev/null @@ -1,102 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, inputTre -, n -, dram -, garlicTools -, resultFromTrebuchet -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "shuffle"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - - # We need access to a fast shared filesystem to store the shuffled input - # dataset - extraMounts = [ fs.shared.fast ]; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - inputUnit = elemAt inputExp.units 0; - unitName = baseNameOf (toString inputUnit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/seq.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/shuffled.dat"; - - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the previous results are complete: - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - mkdir -p "${outDir}" - - # Copy the input as we are going to overwrite it - cp "${inFile}" "${outFile}" - ''; - argv = [ n dram outFile 16 64 ]; - post = '' - # Link the output here - ln -s "${outFile}" shuffled.dat - ''; - }; - - program = {...}: - bsc.apps.bigsort.shuffle; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/sort.nix b/garlic/exp/bigsort/sort.nix deleted file mode 100644 index 2ae73e9..0000000 --- a/garlic/exp/bigsort/sort.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, removeOutput ? true -, resultFromTrebuchet -, inputTre -}: - -with lib; - -let - varConf = { }; # Not used - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "bigsort"; - unitName = "${expName}.bs${toString bs}"; - inherit (targetMachine.config) hw; - - # bigsort options - n = 1024 * 1024 * 1024 / 8; # In longs (?) - bs = n; # In bytes - pageSize = bs / 2; # In bytes (?) - cc = bsc.icc; - mpi = bsc.impi; - gitBranch = "garlic/mpi+send+omp+task"; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - # All CPUs of the socket to each task - cpusPerTask = hw.cpusPerSocket; - jobName = "bigsort-${toString n}-${toString bs}-${gitBranch}"; - - # Load the dataset from the same fs where it was stored in the shuffle - # step. Also we use a local temp fs to store intermediate results. - extraMounts = [ fs.shared.fast fs.local.temp ]; - - rev = 1; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - unit = elemAt inputExp.units 0; - expName = baseNameOf (toString inputExp); - unitName = baseNameOf (toString unit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/shuffled.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/sorted.dat"; - tmpDir = fs.local.temp; - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the shuffled results are complete: nix needs to - # compute the hash of the execution log to write the path here. - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - # Create the output path - mkdir -p ${outDir} - - # Verbose args: - echo "INPUT = ${inFile}" - echo "OUTPUT = ${outFile}" - echo "TMPDIR = ${tmpDir}" - ''; - - argv = [ n bs inFile outFile tmpDir pageSize ]; - - # Optionally remove the potentially large output dataset - post = '' - # Link the output here - ln -s "${outFile}" sorted.dat - '' + optionalString (removeOutput) '' - # Remove the sorted output - stat "${outFile}" > "${outFile}.stat" - echo "file removed to save space" > "${outFile}" - ''; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.bigsort.sort.override { - inherit cc mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - #{ inherit configs pipeline; } - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/nbody.nix b/garlic/exp/cn6/nbody.nix deleted file mode 100644 index e48018f..0000000 --- a/garlic/exp/cn6/nbody.nix +++ /dev/null @@ -1,164 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 2 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "cn6-nbody"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - # Parameters for nbody - particles = 4 * 512 * hw.cpusPerSocket; - timesteps = 2; - blocksize = 512; - gitBranch = "garlic/tampi+isend+oss+task"; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:nanos6/forks/nanos6-fixes.git"; - ref = "distributed-instrumentation-fixes"; - rev = "80058512527961fbde9bd81474b0a29141d7982c"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - "--with-babeltrace2=${super.babeltrace2}" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - #dontStrip = true; - #NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/interoperability/tampi.git"; - ref = "master"; - rev = "f1e77e6f439a0e964e98b5e0a4738b2e95e4fd3d"; - }; - }); - }); - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = true - instrument.ctf.converter.fast = false - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - - # Add nanos6 and babeltrace2 binaries to the PATH - export PATH="$PATH:${bsc'.nanos6}/bin:${bsc'.babeltrace2}/bin" - - # Also add the babeltrace2 python module to python search path - export PYTHONPATH="$PYTHONPATH:${bsc'.babeltrace2}/lib/python3.8/site-packages" - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_nbody - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - nanos6-mergeprv "$tracedir" - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = with conf; [ - "-t" timesteps - "-p" particles - ]; - }; - - program = {nextStage, conf, ...}: bsc'.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/timediff.nix b/garlic/exp/cn6/timediff.nix deleted file mode 100644 index 7d2d42f..0000000 --- a/garlic/exp/cn6/timediff.nix +++ /dev/null @@ -1,171 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 1 2 4 8 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "timediff"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:rarias/nanos6.git"; - ref = "rodrigo"; - rev = "5cbeabb4e0446c2c293cc3005f76e6139465caee"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - dontStrip = true; - NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/tampi.git"; - ref = "instrument"; - rev = "6e4294299bf761a1cc31f4181d9479cefa1c7f3e"; - }; - }); - - # We use the latest commit in master as src for cn6 - cn6Git = ((super.cn6.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/cn6.git"; - ref = "master"; - rev = "1d23d01d60164b8641746d5a204128a9d31b9650"; - }; - })).override { enableTest = true; }); - - cn6 = self.cn6Git; - }); - - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_timediff_mpi - - # Convert CTF trace to PRV - ${bsc'.cn6}/bin/cn6 $tracedir/$rank - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - ${bsc'.cn6}/bin/merge-prv $tracedir - - # We need some tools the path - export PATH="$PATH:${bsc'.babeltrace2}/bin:${pkgs.ministat}/bin" - - ${bsc'.cn6}/bin/sync-err.sh $tracedir - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - program = "${bsc'.cn6}/bin/timediff_mpi"; - argv = [ conf.cpusPerTask ]; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity.nix b/garlic/exp/creams/granularity.nix deleted file mode 100644 index 3fab638..0000000 --- a/garlic/exp/creams/granularity.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - granul = range2 4 128; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+omp+task" - ] ++ optionals (enableExtended) [ - #"garlic/mpi+send+omp+fork" # Don't use fork for granularity - "garlic/mpi+send+seq" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+oss+task" - ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - nodes = [ 1 ] ++ optionals (enableExtended) (range2 2 16); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-gran"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) gitBranch nodes iterations; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity16.nix b/garlic/exp/creams/granularity16.nix deleted file mode 100644 index fa8dc6f..0000000 --- a/garlic/exp/creams/granularity16.nix +++ /dev/null @@ -1,132 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = [ 1 2 4 8 16 32 ]; - granul = [ 1 2 4 8 16 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - #"garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-granularity16"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique ( - filter (c: !(c.granul == 1 && c.sizeFactor >= 32))(stdexp.buildConfigs { - inherit varConf genConf; - })); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/size.nix b/garlic/exp/creams/size.nix deleted file mode 100644 index dea3c8d..0000000 --- a/garlic/exp/creams/size.nix +++ /dev/null @@ -1,131 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = range2 1 32; - baseGranul = [ 1 ] ++ optionals (enableExtended) [ 2 4 8 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ (optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-size"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor baseGranul; - granul = getGranul gitBranch (max 2 (baseGranul * sizeFactor)); - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/ss.nix b/garlic/exp/creams/ss.nix deleted file mode 100644 index 86723be..0000000 --- a/garlic/exp/creams/ss.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - nodes = range2 1 16; - granul = [ 16 ] ++ optionals (enableExtended) [ 8 32 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-ss"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/examples/granularity.nix b/garlic/exp/examples/granularity.nix deleted file mode 100644 index 1c01244..0000000 --- a/garlic/exp/examples/granularity.nix +++ /dev/null @@ -1,189 +0,0 @@ -# This file defines an experiment. It is designed as a function that takes -# several parameters and returns a derivation. This derivation, when built will -# create several scripts that can be executed and launch the experiment. - -# These are the inputs to this function: an attribute set which must contain the -# following keys: -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -# We import in the scope the content of the `lib` attribute, which -# contain useful functions like `toString`, which will be used later. This is -# handy to avoid writting `lib.tostring`. - -with lib; - -# We also have some functions specific to the garlic benchmark which we import -# as well. Take a look at the garlic/tools.nix file for more details. -with garlicTools; - -# The `let` keyword allows us to define some local variables which will be used -# later. It works as the local variable concept in the C language. -let - - # Initial variable configuration: every attribute in this set contains lists - # of options which will be used to compute the configuration of the units. The - # cartesian product of all the values will be computed. - varConf = { - # In this case we will vary the columns and rows of the blocksize. This - # configuration will create 3 x 2 = 6 units. - cbs = [ 256 1024 4096 ]; - rbs = [ 512 1024 ]; - }; - - # Generate the complete configuration for each unit: genConf is a function - # that accepts the argument `c` and returns a attribute set. The attribute set - # is formed by joining the configuration of the machine (which includes - # details like the number of nodes or the architecture) and the configuration - # that we define for our units. - # - # Notice the use of the `rec` keyword, which allows us to access the elements - # of the set while is being defined. - genConf = c: targetMachine.config // rec { - - # These attributes are user defined, and thus the user will need to handle - # them manually. They are not read by the standard pipeline: - - # Here we load the `hw` attribute from the machine configuration, so we can - # access it, for example, the number of CPUs per socket as hw.cpusPerSocket. - hw = targetMachine.config.hw; - - # These options will be used by the heat app, be we write them here so they - # are stored in the unit configuration. - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - - # The blocksize is set to the values passed in the `c` parameter, which will - # be set to one of all the configurations of the cartesian product. for - # example: cbs = 256 and rbs = 512. - # We can also write `inherit (c) cbs rbs`, as is a shorthand notation. - cbs = c.cbs; - rbs = c.rbs; - - # The git branch is specified here as well, as will be used when we specify - # the heat app - gitBranch = "garlic/tampi+isend+oss+task"; - - # ------------------------------------------------------------------------- - - # These attributes are part of the standard pipeline, and are required for - # each experiment. They are automatically recognized by the standard - # execution pipeline. - - # The experiment name: - expName = "example-granularity-heat"; - - # The experimental unit name. It will be used to create a symlink in the - # index (at /gpfs/projects/bsc15/garlic/$USER/index/) so you can easily find - # the unit. Notice that the symlink is overwritten each time you run a unit - # with the same same. - # - # We use the toString function to convert the numeric value of cbs and rbs - # to a string like: "example-granularity-heat.cbs-256.rbs-512" - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - # Repeat the execution of each unit a few times: this option is - # automatically taken by the experiment, which will repeat the execution of - # the program that many times. It is recommended to run the app at least 30 - # times, but we only used 10 here for demostration purposes (as it will be - # faster to run) - loops = 10; - - # Resources: here we configure the resources in the machine. The queue to be - # used is `debug` as is the fastest for small jobs. - qos = "debug"; - - # Then the number of MPI processes or tasks per node: - ntasksPerNode = 1; - - # And the number of nodes: - nodes = 1; - - # We use all the CPUs available in one socket to each MPI process or task. - # Notice that the number of CPUs per socket is not specified directly. but - # loaded from the configuration of the machine that will be used to run our - # experiment. The affinity mask is set accordingly. - cpusPerTask = hw.cpusPerSocket; - - # The time will limit the execution of the program in case of a deadlock - time = "02:00:00"; - - # The job name will appear in the `squeue` and helps to identify what is - # running. Currently is set to the name of the unit. - jobName = unitName; - }; - - # Using the `varConf` and our function `genConf` we compute a list of the - # complete configuration of every unit. - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Now that we have the list of configs, we need to write how that information - # is used to run our program. In our case we will use some params such as the - # number of rows and columns of the input problem or the blocksize as argv - # values. - - # The exec stage is used to run a program with some arguments. - exec = {nextStage, conf, ...}: stages.exec { - # All stages require the nextStage attribute, which is passed as parameter. - inherit nextStage; - - # Then, we fill the argv array with the elements that will be used when - # running our program. Notice that we load the attributes from the - # configuration which is passed as argument as well. - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # This program requires a file called `head.conf` in the current directory. - # To do it, we run this small script in the `pre` hook, which simple runs - # some commands before running the program. Notice that this command is - # executed in every MPI task. - pre = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - # The program stage is only used to specify which program we should run. - # We use this stage to specify build-time parameters such as the gitBranch, - # which will be used to fetch the source code. We use the `override` function - # of the `bsc.garlic.apps.heat` derivation to change the input paramenters. - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - # Other stages may be defined here, in case that we want to do something - # additional, like running the program under `perf stats` or set some - # envionment variables. - - # Once all the stages are defined, we build the pipeline array. The - # `stdexp.stdPipeline` contains the standard pipeline stages, so we don't need - # to specify them. We only specify how we run our program, and what program - # exactly, by adding our `exec` and `program` stages: - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -# Then, we use the `configs` and the `pipeline` just defined inside the `in` -# part, to build the complete experiment: -in - - # The `stdexp.genExperiment` function generates an experiment by calling every - # stage of the pipeline with the different configs, and thus creating - # different units. The result is the top level derivation which is the - # `trebuchet`, which is the script that, when executed, launches the complete - # experiment. - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix deleted file mode 100644 index 63e88de..0000000 --- a/garlic/exp/fwi/common.nix +++ /dev/null @@ -1,128 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -}: - -with lib; - -# Common definitions used by fwi experiments -rec { - - branchesWithoutBlocksize = [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+seq" - ]; - - # Returns true if the given config is in the forkJoinBranches list - needsBlocksize = c: ! any (e: c.gitBranch == e) branchesWithoutBlocksize; - - # Set the blocksize to null for the fork join branch - fixBlocksize = c: if (needsBlocksize c) then c - else (c // { blocksize = null; }); - - # Generate the configs by filtering the unneded blocksizes - getConfigs = {varConf, genConf}: - let - allConfigs = stdexp.buildConfigs { inherit varConf genConf; }; - in - # The unique function ensures that we only run one config for the fork - # join branch, even if we have multiple blocksizes. - unique (map fixBlocksize allConfigs); - - getResources = {gitBranch, hw}: - if (gitBranch == "garlic/mpi+send+seq") then { - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - } else { - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - }; - - exec = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - - ioFreq = if (conf.enableIO) then (conf.ioFreq or "-1") else "9999"; - - in stages.exec { - inherit nextStage; - - # FIXME: FWI should allow the I/O directory to be specified as a - # parameter - pre = '' - FWI_SRUNDIR=$(pwd) - FWI_EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - FWI_PARAMS="${fwiParams}/fwi_params.txt" - FWI_FREQ="${fwiParams}/fwi_frequencies.txt" - - # Run fwi in a directory with fast local storage - mkdir -p "$FWI_EXECDIR" - cd "$FWI_EXECDIR" - - # Only generate the input if we have the CPU 0 (once per node) - if grep -o 'Cpus_allowed_list:[[:space:]]0' \ - /proc/self/status > /dev/null; - then - FWI_CAPTAIN=1 - fi - - if [ $FWI_CAPTAIN ]; then - >&2 echo "generating the input dataset" - ${fwiParams}/bin/ModelGenerator -m "$FWI_PARAMS" "$FWI_FREQ" - fi - - echo >&2 "Current dir: $(pwd)" - echo >&2 "Using PARAMS=$FWI_PARAMS and FREQ=$FWI_FREQ" - '' + optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - ''; - - argv = [ - ''"$FWI_PARAMS"'' - ''"$FWI_FREQ"'' - ] ++ optional (needsBlocksize conf) conf.blocksize ++ [ - "-1" # Fordward steps - "-1" # Backward steps - ioFreq # Write/read frequency - ]; - - post = '' - # Go back to the garlic out directory - cd "$FWI_SRUNDIR" - - if [ $FWI_CAPTAIN ]; then - '' + optionalString (conf.enableCTF) '' - # FIXME: We should specify the path in the nanos6 config, so we - # can avoid the race condition while they are generating the - # traces - sleep 3 - - # Save the traces - mv "$FWI_EXECDIR"/trace_* . - '' + '' - rm -rf "$FWI_EXECDIR" - fi - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - in - apps.fwi.solver.override { - inherit (conf) gitBranch; - inherit fwiParams; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix deleted file mode 100644 index 15bc484..0000000 --- a/garlic/exp/fwi/granularity.nix +++ /dev/null @@ -1,71 +0,0 @@ -# Regular granularity test for FWI - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+isend+oss+task" ]; - blocksize = range2 1 256; - n = [ {nx=100; nz=100; ny=8000; ntpn=2; nodes=1;} ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-granularity"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz ntpn nodes; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = ntpn; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/io.nix b/garlic/exp/fwi/io.nix deleted file mode 100644 index 0d107d7..0000000 --- a/garlic/exp/fwi/io.nix +++ /dev/null @@ -1,75 +0,0 @@ -# Test FWI variants based on tasks with and without I/O. -# This experiment solves a computationally expensive input which brings the -# storage devices to saturation when I/O is enabled. The same input runs -# without I/O for comparison purposes. Also, a range of block sizes -# deemed as efficient according to the granularity experiment are -# explored. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -, enableExtended ? false -}: - -with lib; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+send+oss+task" ]; - blocksize = [ 1 2 4 8 ]; - n = [ {nx=500; nz=500; ny=16000;} ]; - nodes = if (enableExtended) then range2 1 16 else [ 4 ]; - enableIO = [ false true ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-io"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + (if (enableIO) then "-io1" else "-io0") - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize enableIO nodes; - inherit (c.n) nx ny nz; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - enableCTF = false; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/reuse.nix b/garlic/exp/fwi/reuse.nix deleted file mode 100644 index 9a725e2..0000000 --- a/garlic/exp/fwi/reuse.nix +++ /dev/null @@ -1,91 +0,0 @@ -# This test compares a FWI version using poor data locality (+NOREUSE) versus -# the optimized version (used for all other experiments). Follows a pseudocode -# snippet illustrating the fundamental difference between version. -# -# NOREUSE -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# for (y) for (x) for (z) -# computB(v[y][x][z]); -# for (y) for (x) for (z) -# computC(v[y][x][z]); -# -# Optimized version -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# computB(v[y][x][z]); -# computC(v[y][x][z]); - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -}: - -with lib; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/mpi+send+oss+task" - "garlic/mpi+send+oss+task+NOREUSE" - ]; - - blocksize = [ 1 2 4 8 ]; - - n = [ {nx=300; ny=2000; nz=300;} ]; # / half node - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-reuse"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - enableCTF = false; - enableIO = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix deleted file mode 100644 index c54370e..0000000 --- a/garlic/exp/fwi/ss.nix +++ /dev/null @@ -1,90 +0,0 @@ -# Strong scaling test for FWI variants based on tasks. This -# experiment explores a range of block sizes deemed as efficient -# according to the granularity experiment. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/tampi+isend+oss+task" - ] ++ optionals (enableExtended) [ - "garlic/tampi+send+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+send+omp+fork" - # FIXME: the mpi pure version has additional constraints with the - # number of planes in Y. By now is disabled. - #"garlic/mpi+send+seq" - ]; - - blocksize = if (enableExtended) - then range2 1 16 - else [ 2 ]; - - n = [ { nx=100; ny=8000; nz=100; } ]; - - nodes = range2 1 16; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: machineConfig // rec { - expName = "fwi-ss"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - nodes = c.nodes; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/heat/granularity.nix b/garlic/exp/heat/granularity.nix deleted file mode 100644 index 8ebe5db..0000000 --- a/garlic/exp/heat/granularity.nix +++ /dev/null @@ -1,175 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, writeText -, enablePerf ? false -, enableCTF ? false -, enableHWC ? false -, enableExtended ? false -}: - -# TODO: Finish HWC first -assert (enableHWC == false); - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { - cbs = range2 32 4096; - rbs = range2 32 4096; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "heat"; - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - inherit (machineConfig) hw; - - # heat options - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - inherit (c) cbs rbs; - gitBranch = "garlic/tampi+isend+oss+task"; - - # Repeat the execution of each unit 30 times - loops = 10; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - # Assign one socket to each task (only one process) - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - filterConfigs = c: let - # Too small sizes lead to huge overheads - goodSize = (c.cbs * c.rbs >= 1024); - # When the extended units are not enabled, we only select those in - # the diagonal. - extended = if (enableExtended) then true - else c.cbs == c.rbs; - in - goodSize && extended; - - # Compute the array of configurations - configs = filter (filterConfigs) (stdexp.buildConfigs { - inherit varConf genConf; - }); - - perf = {nextStage, conf, ...}: stages.perf { - inherit nextStage; - perfOptions = "stat -o .garlic/perf.csv -x , " + - "-e cycles,instructions,cache-references,cache-misses"; - }; - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - # FIXME: We should run a hook *after* srun has ended, so we can - # execute it in one process only (not in N ranks). This hack works - # with one process only. Or be able to compute the name of the trace - # directory so we can begin the conversion in parallel - post = assert (conf.nodes * conf.ntasksPerNode == 1); '' - tracedir=$(ls -d trace_* | head -1) - echo "using tracedir=$tracedir" - - offset=$(grep 'offset =' $tracedir/ctf/ust/uid/1000/64-bit/metadata | \ - grep -o '[0-9]*') - echo "offset = $offset" - - start_time=$(awk '/^start_time / {print $2}' stdout.log) - end_time=$(awk '/^end_time / {print $2}' stdout.log) - - begin=$(awk "BEGIN{print $start_time*1e9 - $offset}") - end=$(awk "BEGIN{print $end_time*1e9 - $offset}") - - echo "only events between $begin and $end" - - ${bsc.cn6}/bin/cn6 -s $tracedir - - ${bsc.cn6}/bin/cut $begin $end < $tracedir/prv/trace.prv |\ - ${bsc.cn6}/bin/hcut 1 ${toString conf.cpusPerTask} \ - > $tracedir/prv/trace-cut.prv - - ${bsc.cn6}/bin/dur 6400025 0 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_dead.csv & - - ${bsc.cn6}/bin/dur 6400025 1 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_runtime.csv & - - ${bsc.cn6}/bin/dur 6400025 3 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_task.csv & - - wait - - # Remove the traces at the end, as they are huge - rm -rf $tracedir - ''; - # TODO: To enable HWC we need to first add a taskwait before the - # first get_time() measurement, otherwise we get the HWC of the - # main task, which will be huge. - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # The next stage is the program - env = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ - (optional enablePerf perf) ++ - (optional enableCTF ctf) ++ - [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/common.nix b/garlic/exp/hpcg/common.nix deleted file mode 100644 index a7c19e3..0000000 --- a/garlic/exp/hpcg/common.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, callPackage -}: - -with lib; - -rec { - - checkInput = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = optionalString (! (conf.enableGen or false)) ( - let - gen = callPackage ./gen.nix { }; - inputTre = gen.getInputTre conf; - exp = inputTre.experiment; - unit = elemAt exp.units 0; - expName = baseNameOf (toString exp); - unitName = baseNameOf (toString unit); - inputPath = "$GARLIC_OUT/${expName}/${unitName}/1"; - in - '' - # Force the generation of the input resultTree as a dependency: - # ${toString inputTre.result} - - # Ensure the input dataset is still available - export HPCG_INPUT_PATH="${toString inputPath}" - - if [ ! -e "$HPCG_INPUT_PATH" ]; then - >&2 echo "Missing input dataset: $HPCG_INPUT_PATH" - exit 1 - fi - '' - ); - }; - - getSizePerTask = cpusPerTask: sizePerCpu: - mapAttrs (name: val: val * cpusPerTask) sizePerCpu; - - exec = {nextStage, conf, ...}: let - actionArg = if (conf.enableGen or false) - then "--store=." - else "--load=\"$HPCG_INPUT_PATH\""; - - in stages.exec { - inherit nextStage; - argv = [ - "--nx=${toString conf.sizePerTask.x}" - "--ny=${toString conf.sizePerTask.y}" - "--nz=${toString conf.sizePerTask.z}" - "--npx=${toString conf.nprocs.x}" - "--npy=${toString conf.nprocs.y}" - "--npz=${toString conf.nprocs.z}" - "--nblocks=${toString conf.nblocks}" - "--ncomms=${toString conf.ncomms}" - # The input symlink is generated by the input stage, which is generated by - # the genInput function. - actionArg - ] ++ optional (conf.disableAspectRatio or false) "--no-ar=1"; - }; - - program = {nextStage, conf, ...}: bsc.apps.hpcg.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ - checkInput - exec program ]; -} diff --git a/garlic/exp/hpcg/gen.nix b/garlic/exp/hpcg/gen.nix deleted file mode 100644 index d34eccc..0000000 --- a/garlic/exp/hpcg/gen.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -rec { - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-gen"; - unitName = expName - + "-nodes${toString nodes}" - + "-spt.z${toString sizePerTask.z}"; - - inherit (targetMachine.config) hw; - - # Inherit options from the current conf - inherit (c) sizePerTask nprocs disableAspectRatio gitBranch - cpusPerTask ntasksPerNode nodes; - - # nblocks and ncomms are ignored from c - ncomms = 1; - nblocks = 1; - - # We only need one run - loops = 1; - - # Generate the input - enableGen = true; - - # Resources - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - getInputTre = conf: stdexp.genExperiment { - configs = [ (genConf conf) ]; - pipeline = common.pipeline; - }; -} diff --git a/garlic/exp/hpcg/granularity.nix b/garlic/exp/hpcg/granularity.nix deleted file mode 100644 index 7c3d2a8..0000000 --- a/garlic/exp/hpcg/granularity.nix +++ /dev/null @@ -1,76 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - blocksPerCpu = range2 0.5 256; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-granularity"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = maxNodes / nodes; - - # hpcg options - inherit (c) blocksPerCpu gitBranch; - baseSizeZ = 16; - nodes = 1; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = floatTruncate (blocksPerCpu * cpusPerTask); - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 3; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/scaling.nix b/garlic/exp/hpcg/scaling.nix deleted file mode 100644 index afb1aee..0000000 --- a/garlic/exp/hpcg/scaling.nix +++ /dev/null @@ -1,78 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -, enableStrong ? true -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - nodes = range2 1 maxNodes; - baseSizeZ = if (enableExtended) then [ 8 16 ] else [ 16 ]; - blocksPerCpu = if (enableExtended) then range2 1 8 else [ 4 ]; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = if (enableStrong) then "hpcg-ss" else "hpcg-ws"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = if (enableStrong) then maxNodes / nodes else 1; - - # hpcg options - inherit (c) nodes blocksPerCpu gitBranch; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = c.baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/size.nix b/garlic/exp/hpcg/size.nix deleted file mode 100644 index 1124323..0000000 --- a/garlic/exp/hpcg/size.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - # Initial variable configuration - varConf = { - sizeFactor = [ 1 2 4 8 16 32 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-size"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-sf${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # hpcg options - inherit (c) sizeFactor; - gitBranch = "garlic/tampi+isend+oss+task"; - nodes = 16; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { x = 2; y = 2; z = 4 * sizeFactor; }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - blocksPerCpu = 4; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 5; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix deleted file mode 100644 index 2f2bcc4..0000000 --- a/garlic/exp/index.nix +++ /dev/null @@ -1,108 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = rec { - granularity = callPackage ./nbody/granularity.nix { }; - ss = callPackage ./nbody/ss.nix { }; - numa = callPackage ./nbody/numa.nix { }; - }; - - saiph = { - granularity = callPackage ./saiph/granularity.nix { }; - ss = callPackage ./saiph/ss.nix { }; - }; - - creams = rec { - ss = callPackage ./creams/ss.nix { }; - granularity = callPackage ./creams/granularity.nix { }; - size = callPackage ./creams/size.nix { }; - granularity16 = callPackage ./creams/granularity16.nix { }; - - # These experiments are the extended versions of the previous - # ones. We split them so we can keep a reasonable execution time - big.granularity = granularity.override { enableExtended = true; }; - big.ss = granularity.override { enableExtended = true; }; - }; - - hpcg = rec { - granularity = callPackage ./hpcg/granularity.nix { }; - ss = callPackage ./hpcg/scaling.nix { }; - ws = ss.override { enableStrong=false; }; - size = callPackage ./hpcg/size.nix { }; - - big.ss = ss.override { enableExtended = true; }; - }; - - heat = rec { - granularity = callPackage ./heat/granularity.nix { }; - cache = granularity.override { enablePerf = true; }; - ctf = granularity.override { enableCTF = true; }; - }; - - bigsort = rec { - genseq = callPackage ./bigsort/genseq.nix { - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - }; - - shuffle = callPackage ./bigsort/shuffle.nix { - inputTre = genseq; - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - inherit (bsc.garlic.pp) resultFromTrebuchet; - }; - - sort = callPackage ./bigsort/sort.nix { - inputTre = shuffle; - inherit (bsc.garlic.pp) resultFromTrebuchet; - removeOutput = false; - }; - }; - - slurm = { - cpu = callPackage ./slurm/cpu.nix { }; - sigsegv = callPackage ./slurm/sigsegv.nix { }; - exit1 = callPackage ./slurm/exit1.nix { }; - }; - - lulesh = { - test = callPackage ./lulesh/test.nix { }; - }; - - fwi = rec { - granularity = callPackage ./fwi/granularity.nix { }; - ss = callPackage ./fwi/ss.nix { }; - reuse = callPackage ./fwi/reuse.nix { }; - io = callPackage ./fwi/io.nix { }; - - # Extended experiments - big.io = io.override { enableExtended = true; }; - }; - - osu = rec { - latency = callPackage ./osu/latency.nix { }; - latencyShm = latency.override { interNode = false; }; - latencyMt = latency.override { enableMultithread = true; }; - latencyMtShm = latency.override { enableMultithread = true; interNode = true; }; - bw = callPackage ./osu/bw.nix { }; - impi = callPackage ./osu/impi.nix { }; - bwShm = bw.override { interNode = false; }; - mtu = callPackage ./osu/mtu.nix { }; - eager = callPackage ./osu/eager.nix { }; - }; - - examples = { - granularity = callPackage ./examples/granularity.nix { }; - }; - - cn6 = { - timediff = callPackage ./cn6/timediff.nix { }; - nbody = callPackage ./cn6/nbody.nix { }; - }; -} diff --git a/garlic/exp/lulesh/test.nix b/garlic/exp/lulesh/test.nix deleted file mode 100644 index eb9b078..0000000 --- a/garlic/exp/lulesh/test.nix +++ /dev/null @@ -1,85 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - - # Initial variable configuration - varConf = with bsc; { - gitBranch = [ - "garlic/mpi+isend+seq" - "garlic/tampi+isend+oss+taskloop" - "garlic/tampi+isend+oss+taskfor" - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+seq" - "garlic/mpi+isend+oss+task" - "garlic/mpi+isend+omp+fork" - "garlic/tampi+isend+oss+taskloopfor" - ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "lulesh"; - unitName = "${expName}-test"; - inherit (machineConfig) hw; - - # options - iterations = 10; - size = 30; - gitBranch = c.gitBranch; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - /* Lulesh options: - -q : quiet mode - suppress all stdout - -i : number of cycles to run - -s : length of cube mesh along side - -r : Number of distinct regions (def: 11) - -b : Load balance between regions of a domain (def: 1) - -c : Extra cost of more expensive regions (def: 1) - -f : Number of files to split viz dump into (def: (np+10)/9) - -p : Print out progress - -v : Output viz file (requires compiling with -DVIZ_MESH - -h : This message - */ - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-i" iterations "-s" size ]; - }; - - apps = bsc.garlic.apps; - - program = {nextStage, conf, ...}: apps.lulesh.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/common.nix b/garlic/exp/nbody/common.nix deleted file mode 100644 index 8570332..0000000 --- a/garlic/exp/nbody/common.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, numactl -, garlicTools -}: - -with lib; -with garlicTools; - -rec { - getConfigs = {varConf, genConf}: stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec - ( - { - inherit nextStage; - argv = with conf; [ "-t" timesteps "-p" particles ]; - } - # Use numactl to use the interleave policy if requested (default is - # false) - // optionalAttrs (conf.interleaveMem or false) { - program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}"; - } - ); - - program = {nextStage, conf, ...}: bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/nbody/granularity.nix b/garlic/exp/nbody/granularity.nix deleted file mode 100644 index 4b59bae..0000000 --- a/garlic/exp/nbody/granularity.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 64 2048; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - gitBranch = c.gitBranch; - - expName = "nbody-granularity"; - unitName = expName + - "-${toString gitBranch}" + - "-bs${toString blocksize}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/numa.nix b/garlic/exp/nbody/numa.nix deleted file mode 100644 index a860fdc..0000000 --- a/garlic/exp/nbody/numa.nix +++ /dev/null @@ -1,64 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, numactl -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 256 1024; - gitBranch = [ "garlic/tampi+send+oss+task" ]; - attachToSocket = [ true false ]; - interleaveMem = [ true false ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) attachToSocket interleaveMem gitBranch blocksize; - - expName = "nbody-numa"; - unitName = expName + - "-${toString gitBranch}" + - "-bs.${toString blocksize}" + - "-tpn.${toString ntasksPerNode}" + - "-interleave.${if (interleaveMem) then "yes" else "no"}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = if (attachToSocket) - then hw.cpusPerSocket - else hw.cpusPerNode; - ntasksPerNode = if (attachToSocket) - then hw.socketsPerNode - else 1; - nodes = 4; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/granularity-oss.nix b/garlic/exp/nbody/old/granularity-oss.nix deleted file mode 100644 index 2dbe81c..0000000 --- a/garlic/exp/nbody/old/granularity-oss.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Initial variable configuration - varConf = { - blocksize = [ 128 256 512 1024 2048 4096 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4096 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - - gitBranch = "garlic/oss+task"; - expName = "nbody-granularity"; - unitName = expName + "-bs${toString blocksize}"; - - loops = 30; - - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "-t" conf.timesteps "-p" conf.particles ]; - }; - - program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/nblocks.nix b/garlic/exp/nbody/old/nblocks.nix deleted file mode 100644 index 4e7dd94..0000000 --- a/garlic/exp/nbody/old/nblocks.nix +++ /dev/null @@ -1,104 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 7 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, gitBranch ? "garlic/tampi+send+oss+task" -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-nblocks"; - unitName = "${self.expName}${toString self.nblocks}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps gitBranch enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - nodes = 1; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/scaling.nix b/garlic/exp/nbody/old/scaling.nix deleted file mode 100644 index 47fd221..0000000 --- a/garlic/exp/nbody/old/scaling.nix +++ /dev/null @@ -1,110 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 6 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - nodes = [ 1 2 4 8 16 ]; - gitBranch = [ - "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-scaling"; - unitName = self.expName + - "-nb${toString self.nblocks}"+ - "-nodes${toString self.nodes}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit (conf) cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/strong-scaling-oss.nix b/garlic/exp/nbody/old/strong-scaling-oss.nix deleted file mode 100644 index 2a6f789..0000000 --- a/garlic/exp/nbody/old/strong-scaling-oss.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcsAndParticles = [ 1 2 4 8 16 32 48 ]; - input = [ - { numParticles=1 ; cpuMask="0x1"; } - { numParticles=2 ; cpuMask="0x3"; } - { numParticles=4 ; cpuMask="0xf"; } - { numParticles=8 ; cpuMask="0xff"; } - { numParticles=16; cpuMask="0xffff"; } - { numParticles=32; cpuMask="0xffffffff"; } - { numParticles=48; cpuMask="0xffffffffffff"; } - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - inherit (c.input) numParticles cpuMask; - particles = 1024 * numParticles * 2; - timesteps = 10; - blocksize = 1024; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${toString numParticles}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-mpi.nix b/garlic/exp/nbody/old/weak-scaling-mpi.nix deleted file mode 100644 index e830a42..0000000 --- a/garlic/exp/nbody/old/weak-scaling-mpi.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcs = [ 1 2 4 8 16 32 48 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) numProcs; - cc = icc; - mpi = impi; - gitBranch = "garlic/mpi+send"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = numProcs; - nodes = 1; - time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "nbody-bs-${toString numProcs}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-oss.nix b/garlic/exp/nbody/old/weak-scaling-oss.nix deleted file mode 100644 index 4c7e574..0000000 --- a/garlic/exp/nbody/old/weak-scaling-oss.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - cpuMask = [ "0x1" "0x3" "0xf" "0xff" "0xffff" "0xffffffff" "0xffffffffffff" ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) cpuMask; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${cpuMask}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/ss.nix b/garlic/exp/nbody/ss.nix deleted file mode 100644 index d9ebf07..0000000 --- a/garlic/exp/nbody/ss.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = [ 128 ]; - nodes = range2 1 16; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) blocksize nodes gitBranch; - - expName = "nbody-scaling"; - unitName = expName + - "-${toString gitBranch}" + - "-nodes${toString nodes}"; - - loops = 5; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/bw.nix b/garlic/exp/osu/bw.nix deleted file mode 100644 index e2c3f37..0000000 --- a/garlic/exp/osu/bw.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-bw-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/eager.nix b/garlic/exp/osu/eager.nix deleted file mode 100644 index 69ee939..0000000 --- a/garlic/exp/osu/eager.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 20 24 ]; - PSM2_MTU_KB = [ 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/impi.nix b/garlic/exp/osu/impi.nix deleted file mode 100644 index 97f6283..0000000 --- a/garlic/exp/osu/impi.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - threshold = [ 8000 16000 32000 64000 ]; - #threshold = [ 4096 8192 10240 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - mpi = impi; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 10; - expName = "osu-impi-rndv"; - unitName = expName + "-${toString threshold}"; - jobName = expName; - inherit (c) threshold; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export PSM2_MQ_RNDV_SHM_THRESH=${toString conf.threshold} - export PSM2_MQ_RNDV_HFI_THRESH=${toString conf.threshold} - export PSM2_MQ_EAGER_SDMA_SZ=${toString conf.threshold} - #export PSM2_MTU=${toString conf.threshold} - export PSM2_TRACEMASK=0x101 - ''; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/latency.nix b/garlic/exp/osu/latency.nix deleted file mode 100644 index f58eb60..0000000 --- a/garlic/exp/osu/latency.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -, enableMultithread ? false -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = if (enableMultithread) then hw.cpusPerSocket else 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-latency-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - inherit enableMultithread; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = if (enableMultithread) then - "${nextStage}/bin/osu_latency_mt" - else - "${nextStage}/bin/osu_latency"; - - argv = optionals (enableMultithread) [ - "-t" "${toString conf.cpusPerTask}:${toString conf.cpusPerTask}" - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/mtu.nix b/garlic/exp/osu/mtu.nix deleted file mode 100644 index ca99091..0000000 --- a/garlic/exp/osu/mtu.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 ]; - PSM2_MTU_KB = [ 8 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/granularity.nix b/garlic/exp/saiph/granularity.nix deleted file mode 100644 index 30b9b28..0000000 --- a/garlic/exp/saiph/granularity.nix +++ /dev/null @@ -1,132 +0,0 @@ -####################################################################### -# Saiph, granularity experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Granularity experiment configuration: -# Single-core run -# MPI binded to sockets: MPI procs = 2 -# Mesh distributed across third dimension to ensure contiguous -# communications -# --> nbgx = 1, nbgy = 1 -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -####################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Variable configurations - varConf = with targetMachine.config; { - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = range2 1 (hw.cpusPerNode * 8); - nblz = [ 8 ]; - sizex = [ 3 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-granularity"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - - jobName = unitName; - - # saiph options - totalTasks = ntasksPerNode * nodes; - nodes = 1; - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = totalTasks; # forcing distribution by last dim - - inherit (c) nblx nbly nblz gitBranch sizex; - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # fix a specific commit - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all - # factors - allConfigs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Filter to remove non-desired configurations: - # --> tasks/proc < 0.5 - # --> nblz > 50 - isGoodConfig = c: - let - maxNblz = c.cpusPerTask * 2; - in - ! (c.blocksPerCpu < 0.5 || c.nblz > maxNblz); - - configs = filter (isGoodConfig) allConfigs; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString conf.cpusPerTask} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/ss.nix b/garlic/exp/saiph/ss.nix deleted file mode 100644 index 5ddc47a..0000000 --- a/garlic/exp/saiph/ss.nix +++ /dev/null @@ -1,122 +0,0 @@ -###################################################################################### -# Saiph, scalability experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Scalability experiment configuration: -# From a single-core granularity experiment, use a suited local blocking set: -# --> nbly*nblz >= 48 (at least 3tasks/proc) -# MPI binded to sockets: MPI procs = 2*nodes -# Mesh distributed across third dimension to ensure contiguous communications -# --> nbgx = 1, nbgy = 1 -# Global distribution limited by global mesh size -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -###################################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - #*** Variable configurations *** - varConf = with targetMachine.config; { - # FIXME: None of those selected nbl* and problem size is able to give good - # efficiency when testing strong scaling. We should find better values. - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = [ 32 ]; - nblz = [ 8 ]; - sizex = [ 3 6 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - nodes = range2 1 8; - }; - - #*** Generate the complete configuration for each unit *** - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-ss"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - jobName = unitName; - - # saiph options - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = nodes * ntasksPerNode; # forcing distribution by last dim - - inherit (c) nblx nbly nblz nodes sizex; - - gitBranch = "garlic/tampi+isend+oss+task+simd"; - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; # fix a specific commit - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all factors - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString hw.cpusPerSocket} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/cpu.nix b/garlic/exp/slurm/cpu.nix deleted file mode 100644 index d2ee632..0000000 --- a/garlic/exp/slurm/cpu.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list of cpus per task by computing the divisors of the number of - # cpus per socket, example: divisors 24 = [ 1 2 3 4 6 8 12 24 ] - cpusPerTask = divisors hw.cpusPerSocket; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "cpu"; - unitName = "${expName}.${toString cpusPerTask}"; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - inherit (c) cpusPerTask; - # As cpusPerTask is a divisor of the cpusPerSocket and thus cpusPerNode, we - # know the remainder is zero: - ntasksPerNode = hw.cpusPerNode / cpusPerTask; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - }; - - program = {nextStage, conf, ...}: bsc.dummy; - - pipeline = stdexp.stdPipeline ++ [ program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/exit1.nix b/garlic/exp/slurm/exit1.nix deleted file mode 100644 index bf60136..0000000 --- a/garlic/exp/slurm/exit1.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - script = [ - "exit 1" - "exit 0" - "kill -SEGV $$" - "kill -TERM $$" - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "exit1"; - unitName = expName + "-" + - builtins.replaceStrings [" " "$"] ["-" "-"] script; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - inherit (c) script; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = 2; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - pre = "sleep 5"; - post = "echo dummy"; - }; - - prog = {conf,...}: stages.script { - inherit (conf) script; - }; - - pipeline = stdexp.stdPipeline ++ [ exec prog ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/sigsegv.nix b/garlic/exp/slurm/sigsegv.nix deleted file mode 100644 index b77f58b..0000000 --- a/garlic/exp/slurm/sigsegv.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - when = [ "before" "after" "never" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "sigsegv"; - unitName = expName + "-" + when; - - inherit (machineConfig) hw; - inherit (c) when; - - loops = 3; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - argv = [ conf.when ]; - }; - - program = {nextStage, conf, ...}: bsc.test.sigsegv; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/fig/creams/granularity.R b/garlic/fig/creams/granularity.R deleted file mode 100644 index 524c987..0000000 --- a/garlic/fig/creams/granularity.R +++ /dev/null @@ -1,83 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.iter = time / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodesFactor = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.iter = median(time.iter)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="granul", y="Normalized time", - title="Creams granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(x="granul", y="Time (s)", title="Creams granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/creams/granularity16.R b/garlic/fig/creams/granularity16.R deleted file mode 100644 index 24f3bf7..0000000 --- a/garlic/fig/creams/granularity16.R +++ /dev/null @@ -1,136 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(median.time.elem = median(time.elem)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(log.median.time.elem = log(median.time.elem)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() %>% - group_by(sizeFactor) %>% - mutate(optimal.granul = (median.time.elem == min(median.time.elem))) %>% - ungroup() - -dfopt = df %>% filter(optimal.granul == TRUE) - -dpi = 300 -h = 4 -w = 10 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + - facet_wrap(sizeFactor ~ ., labeller=label_both, nrow=1) + - labs(x="Granularity", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=median.time.elem, color=sizeFactor)) + - geom_line(aes(group=sizeFactor)) + - geom_point(data=dfopt, aes(x=granul, y=median.time.elem)) + - theme_bw() + - labs(x="Granularity", y="Time / k (s)", - color="Size factor k", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("median.time.png", plot=p, width=5, height=5, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=5, height=5, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=sizeFactor, fill=log.median.time.elem)) + - geom_raster() + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="Granularity", y="Size factor", fill="Time / k (s)", subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -k=1 -ggsave("heatmap.png", plot=p, width=4.8*k, height=5*k, dpi=300) -ggsave("heatmap.pdf", plot=p, width=4.8*k, height=5*k, dpi=300) diff --git a/garlic/fig/creams/size.R b/garlic/fig/creams/size.R deleted file mode 100644 index 21e3335..0000000 --- a/garlic/fig/creams/size.R +++ /dev/null @@ -1,96 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 3 -w = 6 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=sizeFactor, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="Size factor k (nz=k*3072)", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/creams/ss.R b/garlic/fig/creams/ss.R deleted file mode 100644 index b367781..0000000 --- a/garlic/fig/creams/ss.R +++ /dev/null @@ -1,128 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=granul, color=iterations)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title="Creams strong scaling: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", title="Creams strong scaling: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=median.time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Median time * nodes (s)", title="Creams strong scaling: median time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("median.time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_boxplot() + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", title="Creams strong scaling: time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.boxplot.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.boxplot.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=nodes, y=time.nodes.iter, color=branch)) + -# geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time.nodes.iter, group=interaction(granul,iterations))) + -# theme_bw() + -# #facet_wrap(branch ~ .) + -# labs(x="nodes", y="Time * nodes / iterations (s)", -# title="Creams strong scaling: time * nodes / iterations", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("time.nodes.iter.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("time.nodes.iter.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/dev/shell.nix b/garlic/fig/dev/shell.nix deleted file mode 100644 index 840c83e..0000000 --- a/garlic/fig/dev/shell.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ pkgs ? import ../../../default.nix }: - -with pkgs; - -let - rWrapper = pkgs.rWrapper.override { - packages = with pkgs.rPackages; [ tidyverse rjson jsonlite egg ]; - }; -in -stdenv.mkDerivation { - name = "R"; - - buildInputs = [ rWrapper ]; -} diff --git a/garlic/fig/examples/granularity.R b/garlic/fig/examples/granularity.R deleted file mode 100644 index 7577dca..0000000 --- a/garlic/fig/examples/granularity.R +++ /dev/null @@ -1,195 +0,0 @@ -# This R program takes as argument the dataset that contains the results of the -# execution of the heat example experiment and produces some plots. All the -# knowledge to understand how this script works is covered by this nice R book: -# -# Winston Chang, R Graphics Cookbook: Practical Recipes for Visualizing Data, -# O’Reilly Media (2020). 2nd edition -# -# Which can be freely read it online here: https://r-graphics.org/ -# -# Please, search in this book before copying some random (and probably oudated) -# reply on stack overflow. - -# We load some R packages to import the required functions. We mainly use the -# tidyverse packages, which are very good for ploting data, -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Here we simply load the arguments to find the input dataset. If nothing is -# specified we use the file named `input` in the current directory. -# We can run this script directly using: -# Rscript - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Here we build of dataframe from the input dataset by chaining operations using -# the magritte operator `%>%`, which is similar to a UNIX pipe. -# First we read the input file, which is expected to be NDJSON -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - # Then we flatten it, as it may contain dictionaries inside the columns - jsonlite::flatten() %>% - - # Now the dataframe contains all the configuration of the units inside the - # columns named `config.*`, for example `config.cbs`. We first select only - # the columns that we need: - select(config.cbs, config.rbs, unit, time) %>% - - # And then we rename those columns to something shorter: - rename(cbs=config.cbs, rbs=config.rbs) %>% - - # The columns contain the values that we specified in the experiment as - # integers. However, we need to tell R that those values are factors. So we - # apply to those columns the `as.factor()` function: - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - - # The same for the unit (which is the hash that nix has given to each unit) - mutate(unit = as.factor(unit)) %>% - - # Then, we can group our dataset by each unit. This will always work - # independently of the variables that vary from unit to unit. - group_by(unit) %>% - - # And compute some metrics which are applied to each group. For example we - # compute the median time within the runs of a unit: - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - # Then, we remove the grouping. This step is very important, otherwise the - # plotting functions get confused: - ungroup() - - -# These constants will be used when creating the plots. We use high quality -# images with 300 dots per inch and 5 x 5 inches of size by default. -dpi = 300 -h = 5 -w = 5 - - -# --------------------------------------------------------------------- - - -# We plot the median time (of each unit) as we vary the block size. As we vary -# both the cbs and rbs, we plot cbs while fixing rbs at a time. -p = ggplot(df, aes(x=cbs, y=median.time, color=rbs)) + - # We add a point to the median - geom_point() + - - # We also add the lines to connect the points. We need to specify which - # variable will do the grouping, otherwise we will have one line per point. - geom_line(aes(group=rbs)) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Median time (s)", title="Heat granularity: median time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# Another interesting plot is the normalized time, which shows the variance of -# the execution times, and can be used to find problems: -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - - # The boxplots are useful to identify outliers and problems with the - # distribution of time - geom_boxplot() + - - # We add a line to mark the 1% limit above and below the median - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - - # We split the plot into subplots, one for each value of the rbs column - facet_wrap(~ rbs) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Normalized time", title="Heat granularity: normalized time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We plot the time of each run as we vary the block size -p = ggplot(df, aes(x=cbs, y=time, color=rbs)) + - - # We add a points (scatter plot) using circles (shape=21) a bit larger - # than the default (size=3) - geom_point(shape=21, size=3) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Time (s)", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We can also plot both cbs and rbs in each dimension by mapping the time with a -# color. The `fill` argument instruct R to use the `median.time` as color -p = ggplot(df, aes(x=cbs, y=rbs, fill=median.time)) + - - # Then we use the geom_raster method to paint rectangles filled with color - geom_raster() + - - # The colors are set using the viridis package, using the plasma palete. Those - # colors are designed to be safe for color impaired people and also when - # converting the figures to grayscale. - scale_fill_viridis(option="plasma") + - - # We also force each tile to be an square - coord_fixed() + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="rbs", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R deleted file mode 100644 index da9f1a4..0000000 --- a/garlic/fig/fwi/granularity.R +++ /dev/null @@ -1,74 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(legend.position="bottom") + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/io.R b/garlic/fig/fwi/io.R deleted file mode 100644 index e10e3dc..0000000 --- a/garlic/fig/fwi/io.R +++ /dev/null @@ -1,120 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(forcats) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.blocksize, config.ioFreq, config.gitBranch, config.nodes, time, unit) %>% - rename( - blocksize=config.blocksize, - enableIO=config.enableIO, - gitBranch=config.gitBranch, - nodes=config.nodes - ) %>% - filter(blocksize == 1) %>% - group_by(unit) %>% - mutate(mtime = median(time)) %>% - mutate(nxmtime = mtime * nodes) %>% - mutate(nxtime = time * nodes) %>% - ungroup() - -df$gitBranch = as.factor(df$gitBranch) -df$enableIO = as.factor(df$enableIO) -df$blocksize = as.factor(df$blocksize) -df$nodes = as.factor(df$nodes) - -ppi=300 -h=5 -w=5 - -#################################################################### -### Line plot (time) -#################################################################### -png("time.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=time, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -#################################################################### -### Line plot (time x nodes) -#################################################################### -png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=nxtime, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -##################################################################### -#### Line plot (median time) -##################################################################### -#png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() -# -##################################################################### -#### Line plot (nodes x median time) -##################################################################### -#png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() diff --git a/garlic/fig/fwi/ss.R b/garlic/fig/fwi/ss.R deleted file mode 100644 index 428d5c1..0000000 --- a/garlic/fig/fwi/ss.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - config.nodes, - time) %>% - - rename(blocksize=config.blocksize, - nodes=config.nodes, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/cache.R b/garlic/fig/heat/cache.R deleted file mode 100644 index dca6baf..0000000 --- a/garlic/fig/heat/cache.R +++ /dev/null @@ -1,78 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, perf.cache_misses, perf.instructions, perf.cycles, time) %>% - rename(cbs=config.cbs, rbs=config.rbs) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df=group_by(df, cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.misses = median(perf.cache_misses)) %>% - mutate(log.median.misses = log(median.misses)) %>% - mutate(median.instr= median(perf.instructions)) %>% - mutate(log.median.instr= log(median.instr)) %>% - mutate(median.cycles = median(perf.cycles)) %>% - mutate(median.cpi = median.cycles / median.instr) %>% - mutate(median.ipc = median.instr / median.cycles) %>% - mutate(median.ips = median.instr / median.time) %>% - mutate(median.cps = median.cycles / median.time) %>% - ungroup()# %>% - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.misses", "cache misses") -heatmap_plot(df, "log.median.misses", "cache misses") -heatmap_plot(df, "median.instr", "instructions") -heatmap_plot(df, "log.median.instr", "instructions") -heatmap_plot(df, "median.cycles", "cycles") -heatmap_plot(df, "median.ipc", "IPC") -heatmap_plot(df, "median.cpi", "cycles/instruction") -heatmap_plot(df, "median.ips", "instructions/second") -heatmap_plot(df, "median.cps", "cycles/second") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df_filtered, "median.time", "execution time (seconds)") -heatmap_plot(df_filtered, "log.median.time", "execution time") diff --git a/garlic/fig/heat/granularity.R b/garlic/fig/heat/granularity.R deleted file mode 100644 index 19c3f37..0000000 --- a/garlic/fig/heat/granularity.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.cbs, - config.rbs, - time, - total_time) %>% - - rename(cbs=config.cbs, - rbs=config.rbs) %>% - - # Convert to factors - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(y="Normalized time", - title="Heat granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=time)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=0)) + - theme_bw() + - labs(y="Time (s)", title="Heat granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/mode.R b/garlic/fig/heat/mode.R deleted file mode 100644 index 5e409bc..0000000 --- a/garlic/fig/heat/mode.R +++ /dev/null @@ -1,122 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) -library(tidyr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, - ctf_mode.runtime, - ctf_mode.task, - ctf_mode.dead, - config.cpusPerTask, - time) %>% - rename( - cbs=config.cbs, - rbs=config.rbs, - runtime=ctf_mode.runtime, - task=ctf_mode.task, - dead=ctf_mode.dead, - cpusPerTask=config.cpusPerTask, - ) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df = df %>% - mutate(runtime = runtime * 1e-9 / cpusPerTask) %>% - mutate(dead = dead * 1e-9 / cpusPerTask) %>% - mutate(task = task * 1e-9 / cpusPerTask) %>% - group_by(cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.dead = median(dead)) %>% - mutate(median.runtime = median(runtime)) %>% - mutate(median.task = median(task)) %>% - ungroup() #%>% - -print(df) - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.runtime", "runtime") -heatmap_plot(df, "median.dead", "not used") -heatmap_plot(df, "median.task", "task") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df, "median.time", "execution time (seconds)") -heatmap_plot(df, "log.median.time", "execution time") - -df_square = filter(df, cbs == rbs) %>% - gather(key = time.from, value = acc.time, - c("median.dead", "median.runtime", "median.task")) - -# Colors similar to Paraver -colors <- c("median.dead" = "gray", - "median.runtime" = "blue", - "median.task" = "red") - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_area(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - geom_point(aes(y=median.time, color="black")) + - geom_line(aes(y=median.time, group=0, color="black")) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - scale_color_identity(breaks = c("black"), - labels = c("Total time"), guide = "legend") + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("area.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("area.time.pdf", plot=p, width=6, height=6, dpi=300) - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_col(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("col.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("col.time.pdf", plot=p, width=6, height=6, dpi=300) diff --git a/garlic/fig/hpcg/granularity.R b/garlic/fig/hpcg/granularity.R deleted file mode 100644 index 29126da..0000000 --- a/garlic/fig/hpcg/granularity.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.ncomms, - config.hw.cpusPerSocket, - config.blocksPerCpu, - unit, - time) %>% - - rename(nblocks=config.nblocks, - ncomms=config.ncomms, - blocksPerCpu=config.blocksPerCpu) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=blocksPerCpu, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(x="Blocks per CPU", y="Normalized time", title="HPCG granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=blocksPerCpu, y=time)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Blocks per CPU", y="Time (s)", title="HPCG granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/hpcg/size.R b/garlic/fig/hpcg/size.R deleted file mode 100644 index a7cc5f1..0000000 --- a/garlic/fig/hpcg/size.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.elem = time.nodes / sizePerCpu.z) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=sizePerCpu.z, y=time.nodes.elem)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Size per CPU in Z", y="Time * nodes / spcz (s)", - title="HPCG size: time * nodes / spcz", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ss.R b/garlic/fig/hpcg/ss.R deleted file mode 100644 index 3978618..0000000 --- a/garlic/fig/hpcg/ss.R +++ /dev/null @@ -1,82 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.sizeZ = time / sizePerCpu.z) %>% - mutate(time.nodes = time * nodes) %>% - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time.nodes)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", - title="HPCG strong scalability in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=nodes, y=time.sizeZ, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time / npcz (s)", title="HPCG strong scalability in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.size.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.size.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ws.R b/garlic/fig/hpcg/ws.R deleted file mode 100644 index d12f35b..0000000 --- a/garlic/fig/hpcg/ws.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - mutate(time.sizeZ = time / sizePerCpu.z) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - mutate(timePerNprocs = time * npz) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="HPCG weak scaling in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix deleted file mode 100644 index 0e7ce41..0000000 --- a/garlic/fig/index.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -let - rPlot = garlic.pp.rPlot; - exp = garlic.exp; - pp = garlic.pp; - ds = garlic.ds; - fig = garlic.fig; - - stdPlot = rScript: expList: rPlot { - script = rScript; - dataset = pp.mergeDatasets (map (e: ds.std.timetable e.result) expList); - }; - - customPlot = rScript: dataset: rPlot { - script = rScript; - dataset = dataset; - }; - - linkTree = name: tree: self.linkFarm name ( - self.lib.mapAttrsToList ( - name: value: { name=name; path=value; } - ) tree); -in -{ - nbody = with exp.nbody; { - granularity = stdPlot ./nbody/granularity.R [ granularity ]; - ss = stdPlot ./nbody/ss.R [ ss ]; - numa = stdPlot ./nbody/numa.R [ numa ]; - }; - - hpcg = with exp.hpcg; { - ss = stdPlot ./hpcg/ss.R [ ss ]; - ws = stdPlot ./hpcg/ws.R [ ws ]; - size = stdPlot ./hpcg/size.R [ size ]; - granularity = stdPlot ./hpcg/granularity.R [ granularity ]; - }; - - saiph = with exp.saiph; { - granularity = stdPlot ./saiph/granularity.R [ granularity ]; - ss = stdPlot ./saiph/ss.R [ ss ]; - }; - - heat = with exp.heat; { - granularity = stdPlot ./heat/granularity.R [ granularity ]; - cache = customPlot ./heat/cache.R (ds.perf.stat cache.result); - ctf = customPlot ./heat/mode.R (ds.ctf.mode ctf.result); - }; - - creams = with exp.creams; { - ss = stdPlot ./creams/ss.R [ ss ]; - granularity = stdPlot ./creams/granularity.R [ granularity ]; - size = stdPlot ./creams/size.R [ size ]; - granularity16 = stdPlot ./creams/granularity16.R [ granularity16 ]; - - # Extended version (we could use another R script for those plots - big.ss = stdPlot ./creams/ss.R [ big.ss ]; - big.granularity = stdPlot ./creams/granularity.R [ big.granularity ]; - }; - - fwi = with exp.fwi; { - granularity = stdPlot ./fwi/granularity.R [ granularity ]; - reuse = stdPlot ./fwi/granularity.R [ reuse ]; - ss = stdPlot ./fwi/ss.R [ ss ]; - io = stdPlot ./fwi/io.R [ io ]; - }; - - osu = with exp.osu; { - latency = customPlot ./osu/latency.R (ds.osu.latency latency.result); - latencyShm = customPlot ./osu/latency.R (ds.osu.latency latencyShm.result); - latencyMt = customPlot ./osu/latency.R (ds.osu.latency latencyMt.result); - latencyMtShm = customPlot ./osu/latency.R (ds.osu.latency latencyMtShm.result); - - bw = customPlot ./osu/bw.R (ds.osu.bw bw.result); - bwShm = customPlot ./osu/bw.R (ds.osu.bw bwShm.result); - impi = customPlot ./osu/impi.R (ds.osu.bw impi.result); - mtu = customPlot ./osu/mtu.R (ds.osu.bw mtu.result); - eager = customPlot ./osu/eager.R (ds.osu.bw eager.result); - }; - - # The figures used in the article contained in a directory per figure - article = with fig; linkTree "article-fig" { - "osu/latency" = osu.latency; - "osu/bw" = osu.bw; - "osu/mtu" = osu.mtu; - }; - - examples = with exp.examples; { - granularity = stdPlot ./examples/granularity.R [ granularity ]; - }; -} diff --git a/garlic/fig/nbody/granularity.R b/garlic/fig/nbody/granularity.R deleted file mode 100644 index 85ee0b4..0000000 --- a/garlic/fig/nbody/granularity.R +++ /dev/null @@ -1,62 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.particles, unit, time) %>% - rename(blocksize=config.blocksize, particles=config.particles, branch=config.gitBranch) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(particles = as.factor(particles)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 5 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Blocksize", y="Normalized Time", title="NBody Granularity: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time)) + - geom_boxplot() + - theme_bw() + - labs(x="Blocksize", y="Time (s)", title="NBody Granularity: Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/numa.R b/garlic/fig/nbody/numa.R deleted file mode 100644 index b080648..0000000 --- a/garlic/fig/nbody/numa.R +++ /dev/null @@ -1,86 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(unit, - config.blocksize, - config.gitBranch, - config.attachToSocket, - config.interleaveMem, - config.nodes, - unit, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch, - nodes=config.nodes, - attachToSocket=config.attachToSocket, - interleaveMem=config.interleaveMem) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(attachToSocket = as.factor(attachToSocket)) %>% - mutate(interleaveMem = as.factor(interleaveMem)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -branch = unique(df$branch) -nodes = unique(df$nodes) - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=interleaveMem)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Normalized time", - title=sprintf("NBody NUMA (%s | %d Nodes): Normalized time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=interleaveMem)) + - geom_boxplot() + - geom_line(aes(y=median.time)) + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Time (s)", - title=sprintf("NBody NUMA (%s | %d Nodes): Time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/old/baseline.R b/garlic/fig/nbody/old/baseline.R deleted file mode 100644 index 5509d8e..0000000 --- a/garlic/fig/nbody/old/baseline.R +++ /dev/null @@ -1,212 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(egg) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, - config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.blocksize, - config.particles, - config.gitBranch, - time) %>% - rename(nblocks=config.nblocks, - nodes=config.nodes, - blocksize=config.blocksize, - particles=config.particles, - gitBranch=config.gitBranch, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$nodesFactor = as.factor(df$nodes) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) -df$blocksizeFactor = as.factor(df$blocksize) -df$particlesFactor = as.factor(df$particles) -df$gitBranch = as.factor(df$gitBranch) - -# Normalize the time by the median -D=group_by(df, nblocks, nodesFactor, gitBranch) %>% - mutate(tmedian = median(time)) %>% - mutate(tn = tmedian * nodes) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) %>% - ungroup() %>% - group_by(nodesFactor, gitBranch) %>% - mutate(tmedian_min = min(tmedian)) %>% - ungroup() %>% - group_by(gitBranch) %>% - mutate(tmin_max = max(tmedian_min)) %>% - mutate(tideal = tmin_max / nodes) %>% - ungroup() - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=7.5 -w=7.5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot(aes(fill=nodesFactor)) + - scale_color_manual(values=c("black", "brown")) + - facet_grid(gitBranch ~ .) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - - #theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - theme_bw()+ - theme(plot.subtitle=element_text(size=8)) - - - - -# Render the plot -print(p) -dev.off() - - -p1 = ggplot(D, aes(x=blocksizeFactor, y=time)) + - - labs(x="Blocksize", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - #theme(legend.position = c(0.5, 0.8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocksize.png", width=w*ppi, height=h*ppi, res=ppi) -print(p1) -dev.off() - -p2 = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocks-per-cpu.png", width=w*ppi, height=h*ppi, res=ppi) -print(p2) -dev.off() - -#p = ggarrange(p1, p2, ncol=2) -#png("time-gra.png", width=2*w*ppi, height=h*ppi, res=ppi) -#print(p) -#dev.off() - - - -png("exp-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=df, aes(x=nodesFactor, y=particlesFactor)) + - labs(x="Nodes", y="Particles", title="Nbody: Experiment space") + - geom_line(aes(group=particles)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("gra-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=D, aes(x=nodesFactor, y=blocksPerCpuFactor)) + - labs(x="Nodes", y="Blocks/CPU", title="Nbody: Granularity space") + - geom_line(aes(group=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("performance.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tmedian, - linetype=blocksPerCpuFactor, - group=interaction(gitBranch, blocksPerCpuFactor))) + - geom_line(aes(y=tideal, group=gitBranch), color="red") + - geom_point(aes(y=tmedian, color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() - - -png("time-nodes.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time * nodes (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tn, group=gitBranch)) + - facet_grid(gitBranch ~ .) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() diff --git a/garlic/fig/nbody/old/freeCpu.R b/garlic/fig/nbody/old/freeCpu.R deleted file mode 100644 index cf97106..0000000 --- a/garlic/fig/nbody/old/freeCpu.R +++ /dev/null @@ -1,112 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.freeCpu, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - freeCpu=config.freeCpu, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$freeCpu = as.factor(df$freeCpu) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(freeCpu, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=freeCpu)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans(), -# breaks=bs_unique) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~freeCpu) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time, color=freeCpu)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/jemalloc.R b/garlic/fig/nbody/old/jemalloc.R deleted file mode 100644 index 37fc991..0000000 --- a/garlic/fig/nbody/old/jemalloc.R +++ /dev/null @@ -1,113 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.enableJemalloc, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - jemalloc=config.enableJemalloc, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$jemalloc = as.factor(df$jemalloc) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(jemalloc, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - # Add another column: blocksPerCpu (we assume one task per socket, using - # all CPUs) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=nblocks, y=tnorm)) + - - # Labels - labs(x="Num blocks", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=jemalloc)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans()) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~jemalloc) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpu, y=time, color=jemalloc)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/scaling.R b/garlic/fig/nbody/old/scaling.R deleted file mode 100644 index 1d50a37..0000000 --- a/garlic/fig/nbody/old/scaling.R +++ /dev/null @@ -1,111 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Normalize the time by the median -D=group_by(df, nblocks) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot() + - scale_color_manual(values=c("black", "brown")) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/ss.R b/garlic/fig/nbody/ss.R deleted file mode 100644 index 2ce9b2c..0000000 --- a/garlic/fig/nbody/ss.R +++ /dev/null @@ -1,66 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.nodes, unit, time) %>% - rename(nodes = config.nodes, blocksize=config.blocksize, branch=config.gitBranch) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Nodes", y="Normalized time (s)", title="NBody Scaling: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", title="NBody Scaling: Time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/osu/bw.R b/garlic/fig/osu/bw.R deleted file mode 100644 index 876519d..0000000 --- a/garlic/fig/osu/bw.R +++ /dev/null @@ -1,59 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, bw) %>% - rename(unitName=config.unitName) %>% - mutate(bw=bw / 1024.0) %>% - mutate(unitName=str_replace(unitName, "osu-bw-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=median.bw)) + - labs(x="Message size", y="Bandwidth (GB/s)", - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=1.5) + - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - geom_hline(yintercept=12.5, color="red") + - annotate("text", x=1, y=12.5 * .95, - label="Max: 12.5GB/s (100Gbps)", - hjust=0, vjust=1, size=3) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12, - ) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(1,0), legend.position = c(0.99, 0.01)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/eager.R b/garlic/fig/osu/eager.R deleted file mode 100644 index aa9e573..0000000 --- a/garlic/fig/osu/eager.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=150 -h=6 -w=8 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - labs(x="Message size (KB)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=3) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") + - geom_vline(xintercept = 10, color="red") + - annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MQ_EAGER_SDMA_SZ.f), nrow=3, labeller = "label_both") + - scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - theme_bw() - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/impi.R b/garlic/fig/osu/impi.R deleted file mode 100644 index c62a334..0000000 --- a/garlic/fig/osu/impi.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, config.threshold, size, bw) %>% - rename(unitName=config.unitName) %>% - rename(threshold=config.threshold) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$threshold = as.factor(df$threshold) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -p = ggplot(data=df, aes(x=size, y=bw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU bandwidth benchmark: nodes=%d tasksPerNode=%d cpusPerTask=%d", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_boxplot(aes(color=threshold, group=interaction(threshold, sizeFactor))) + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ppi=300 -h=4 -w=8 -ggsave("boxplot.pdf", plot=p, width=w, height=h, dpi=ppi) -ggsave("boxplot.png", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=medianBw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(color=threshold, linetype=threshold)) + - geom_point(aes(color=threshold, shape=threshold)) + - geom_hline(yintercept = 100e3 / 8, color="red") + - annotate("text", x = 8, y = (100e3 / 8) * 0.95, label = "12.5GB/s (100Gb/s)") + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/latency.R b/garlic/fig/osu/latency.R deleted file mode 100644 index e116a73..0000000 --- a/garlic/fig/osu/latency.R +++ /dev/null @@ -1,77 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, latency) %>% - rename(unitName=config.unitName) %>% - mutate(unitName=str_replace(unitName, "osu-latency-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianLatency = median(latency)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=medianLatency)) + - labs(x="Message size", y="Median latency (µs)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12)+ - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(0,1), legend.position = c(0.01, 0.99)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=latency)) + - labs(x="Size (bytes)", y="Latency (us)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(y=medianLatency, linetype=unitName, group=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - breaks=unique(df$size), - minor_breaks=NULL) + - theme_bw() + - theme(plot.subtitle = element_text(color="gray50")) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) + - theme(legend.position = c(0.2, 0.8)) - -ggsave("latency.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("latency.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/mtu.R b/garlic/fig/osu/mtu.R deleted file mode 100644 index 09e2f63..0000000 --- a/garlic/fig/osu/mtu.R +++ /dev/null @@ -1,69 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - config.PSM2_MTU, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ.val=config.PSM2_MQ_EAGER_SDMA_SZ, - PSM2_MTU.val=config.PSM2_MTU) %>% - mutate(bw = bw / 1000.0) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ.val) -df$PSM2_MTU = as.factor(df$PSM2_MTU.val) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ.val/1024), color="blue") + - geom_vline(aes(xintercept = PSM2_MTU.val/1024), color="red") + - labs(x="Message size (KiB)", y="Bandwidth (GB/s)", - #title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=2) + - #annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MTU), nrow=3, labeller = "label_both") + - #scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - scale_x_continuous(n.breaks = 12) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/saiph/granularity.R b/garlic/fig/saiph/granularity.R deleted file mode 100644 index 66bd7c6..0000000 --- a/garlic/fig/saiph/granularity.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nbly, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksPerCpu, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + - scale_x_continuous(trans=log2_trans()) + - labs(y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/saiph/ss.R b/garlic/fig/saiph/ss.R deleted file mode 100644 index 6498f30..0000000 --- a/garlic/fig/saiph/ss.R +++ /dev/null @@ -1,110 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=sizex)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/garlicd/default.nix b/garlic/garlicd/default.nix deleted file mode 100644 index b34efdf..0000000 --- a/garlic/garlicd/default.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, nix -, garlicTool -}: - -let - extraPath = "${garlicTool}:${nix}"; -in - stdenv.mkDerivation { - name = "garlicd"; - preferLocalBuild = true; - - phases = [ "unpackPhase" "installPhase" ]; - - src = ./garlicd; - - unpackPhase = '' - cp $src garlicd - ''; - - installPhase = '' - substituteInPlace garlicd \ - --replace @extraPath@ ${extraPath} - mkdir -p $out/bin - cp -a garlicd $out/bin - ''; - } diff --git a/garlic/garlicd/garlicd b/garlic/garlicd/garlicd deleted file mode 100755 index 17e3f9d..0000000 --- a/garlic/garlicd/garlicd +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -e - -emsg() { - >&2 echo "garlicd: $@" -} - -msg() { - emsg "$@" - if [ ! -z "$st" ]; then echo "garlicd: $@" >>"$st"; fi -} - -if [ ! -z "$1" ]; then - >&2 echo "usage: garlicd" - exit 1 -fi - -export PATH="@extraPath@:$PATH" - -garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - -if [ -z "$garlic_sandbox" ]; then - emsg "Missing extra-sandbox-paths /garlic mountpoint" - emsg "Check the ~/.config/nix/nix.conf file" - exit 1 -fi - -mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') -mountdir=$(readlink -f "$mountdir_rel") -run="$mountdir/run" -completed="$mountdir/completed" -wipe="$mountdir/wipe" -st="$mountdir/st" - -handle_bad_signal() { - msg "cleaning FIFO pipes" - rm -f "$run" "$completed" "$wipe" - exit 1 -} - -trap handle_bad_signal SIGINT - -for fifo in "$run" "$completed" "$wipe"; do - if [ ! -e "$fifo" ]; then - mkfifo "$fifo" - # FIXME: Use more resctrictive permissions - chmod 666 "$fifo" - fi -done - -while true; do - emsg "--- Waiting for experiments ---" - tre=$(head -1 "$run") - - # Truncate state file - printf "" > "$st" - - msg "Attempting to run: $tre" - msg "Copying files to MN4..." - # It fails if the user doesn't have nix-store, but is already copied - # with the post build hook - nix copy --to ssh://mn1 $tre || true - - set +e - - msg "Launching the experiment..." - garlic -R "$tre" 2>>"$st" - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "Failed to run the experiment :-(" - continue - fi - - msg "Fetching results..." - garlic -FKv "$tre" 2>>"$st" - - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "The experiment failed :-(" - continue - fi - - set -e - - msg "Signalling nix build..." - echo -n "$tre" >> "$completed" - - msg "Waiting for nix to finish the build..." - - tre2=$(head -1 "$wipe") - if [ "$tre" != "$tre2" ]; then - msg "error: trebuchet mismatch" - exit 1 - fi - - msg "Removing temporal files..." - garlic -D "$tre" 2>>"$st" - - echo -n "$tre" >> "$completed" - - msg "Execution completed :-)" -done diff --git a/garlic/index.nix b/garlic/index.nix deleted file mode 100644 index ed8e2d8..0000000 --- a/garlic/index.nix +++ /dev/null @@ -1,138 +0,0 @@ -{ - super -, self -, bsc -, callPackage -}: - -{ - develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl boost - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi vtk paraver - # Add more bsc packages here... - ]; - packages = commonPackages ++ bscPackages; - in - bsc.garlic.stages.exec rec { - nextStage = bsc.garlic.stages.isolate { - nextStage = bsc.garlic.unsafeDevelop.override { - extraInputs = packages; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - extraMounts = [ "/tmp:$TMPDIR" ]; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - # This hack uploads all dependencies to MN4 - pre = let - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - stageProgram = bsc.garlicTools.stageProgram; - in - '' - # Hack to upload this to MN4: @upload-to-mn@ - - # Create a link to the develop script - ln -fs ${nixPrefix}${stageProgram nextStage} .nix-develop - ''; - post = "\n"; - }; - - unsafeDevelop = callPackage ./develop/default.nix { }; - - # Configuration for the machines - machines = callPackage ./machines.nix { }; - - report = callPackage ./report.nix { - fig = bsc.garlic.fig; - }; - - sedReport = callPackage ./sedReport.nix { - fig = bsc.garlic.fig; - }; - - bundleReport = callPackage ./bundleReport.nix { - fig = bsc.garlic.fig; - }; - - reportTar = callPackage ./reportTar.nix { - fig = bsc.garlic.fig; - }; - - # Use the configuration for the following target machine - targetMachine = bsc.garlic.machines.mn4; - - # Load some helper functions to generate app variants - - stdexp = callPackage ./stdexp.nix { - inherit (bsc.garlic) targetMachine stages pp; - }; - - # Execution stages - stages = { - sbatch = callPackage ./stages/sbatch.nix { }; - srun = callPackage ./stages/srun.nix { }; - baywatch = callPackage ./stages/baywatch.nix { }; - control = callPackage ./stages/control.nix { }; - exec = callPackage ./stages/exec.nix { }; - script = callPackage ./stages/script.nix { }; - extrae = callPackage ./stages/extrae.nix { }; - valgrind = callPackage ./stages/valgrind.nix { }; - perf = callPackage ./stages/perf.nix { }; - isolate = callPackage ./stages/isolate { }; - runexp = callPackage ./stages/runexp { }; - trebuchet = callPackage ./stages/trebuchet.nix { }; - strace = callPackage ./stages/strace.nix { }; - unit = callPackage ./stages/unit.nix { }; - experiment = callPackage ./stages/experiment.nix { }; - }; - - # Tests (move to bsc ?) - mpptest = callPackage ./mpptest { }; - - ppong = callPackage ./ppong { - mpi = bsc.mpi; - }; - - hist = callPackage ./pp/hist { }; - - tool = callPackage ./sh/default.nix { - sshHost = "mn1"; - }; - - # Post processing - pp = { - store = callPackage ./pp/store.nix { }; - rPlot = callPackage ./pp/rplot.nix { }; - mergeDatasets = callPackage ./pp/merge.nix { }; - }; - - garlicd = callPackage ./garlicd/default.nix { - garlicTool = bsc.garlic.tool; - }; - - # Apps for Garlic - apps = callPackage ./apps/index.nix { - inherit self super bsc; - }; - - # Experiments - exp = callPackage ./exp/index.nix { - inherit self super bsc; - }; - - # Dataset generators from resultTree - ds = callPackage ./ds/index.nix { - inherit self super bsc; - }; - - # Figures generated from the datasets - fig = callPackage ./fig/index.nix { - inherit self super bsc; - }; - -} diff --git a/garlic/machines.nix b/garlic/machines.nix deleted file mode 100644 index e2bdc54..0000000 --- a/garlic/machines.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -}: - -{ - # MareNostrum 4 configuration - mn4 = { - config = { - clusterName = "mn4"; - sshHost = "mn1"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - hw = { - # The rev attribute attemps to capture the hardware - # configuration of the machine, and will rebuild all experiments - # if it changed. It only holds the timestamp at the current - # time, representing the HW configuration at that moment. - rev = 1614253003; - - # Node and socket details - cpusPerNode = 48; - cpusPerSocket = 24; - socketsPerNode = 2; - cachelineBytes = 64; - - # Cache sizes - cacheSizeKB = { - L1d = 32; - L1i = 32; - L2 = 1024; - L3 = 33792; - }; - }; - }; - - # Experimental naming convention for the FS - fs = rec { - topology = { - gpfs = { - projects = "/gpfs/projects/bsc15/garlic"; - scratch = "/gpfs/scratch/bsc15/bsc15557/garlic"; - }; - - ssd = { - # Beware to expand the temp dir at execution time - temp = "$TMPDIR"; - }; - }; - - shared = with topology; { - fast = gpfs.scratch; - reliable = gpfs.projects; - }; - - local = { - temp = topology.ssd.temp; - }; - }; - - # TODO: Add the specific details for SLURM and the interconection here - }; -} diff --git a/garlic/mpptest/default.nix b/garlic/mpptest/default.nix deleted file mode 100644 index c142667..0000000 --- a/garlic/mpptest/default.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "mpptest"; - - src = fetchurl { - url = "http://ftp.mcs.anl.gov/pub/mpi/tools/perftest.tar.gz"; - sha256 = "11i22lq3pch3pvmhnbsgxzd8ap4yvpvlhy2f7k8x3krdwjhl0jvl"; - }; - - buildInputs = [ mpi ]; -} diff --git a/garlic/pp/hist/default.nix b/garlic/pp/hist/default.nix deleted file mode 100644 index 4a3d464..0000000 --- a/garlic/pp/hist/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ - stdenv -, ministat -}: - -stdenv.mkDerivation { - name = "hist"; - preferLocalBuild = true; - src = ./.; - - dontBuild = true; - dontConfigure = true; - - inherit ministat; - - patchPhase = '' - substituteAllInPlace hist.sh - ''; - - installPhase = '' - mkdir -p $out/bin - cp hist.sh $out/bin/hist - chmod +x $out/bin/hist - ''; -} diff --git a/garlic/pp/hist/hist.sh b/garlic/pp/hist/hist.sh deleted file mode 100755 index f3363b4..0000000 --- a/garlic/pp/hist/hist.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Use it either reading from stdin or by specifing -# multiple files as arguments - -# xeon07$ hist stdout.log -# x -# +------------------------------------------------------------------------+ -# | x | -# | x | -# | x | -# | x | -# | x | -# | xxx | -# | xxx | -# | xxxxx | -# | xxxxxx | -# | xxxxxxx x| -# ||________M_A___________| | -# +------------------------------------------------------------------------+ -# N Min Max Median Avg Stddev -# x 30 3.585183 3.763913 3.591559 3.5973344 0.031719975 -# -# Other ministat options can be passed as well. The -S option splits the results -# in multiple plots. - - -usage() { echo "Usage: hist [-hSAns] [-c confidence] [-w width] files..." 1>&2; exit 1; } - -function stat_files() { - tmpfiles=() - sedcmd="" - - for file in ${files[@]}; do - tmp=$(mktemp) - awk '/^time /{print $2}' "$file" > "$tmp" - sedcmd+="s:$tmp:$file:g;" - tmpfiles+=($tmp) - done - - if [ $split == 1 ]; then - for f in "${tmpfiles[@]}"; do - ministat $ministat_opt $f | sed -e "$sedcmd" - done - else - ministat $ministat_opt ${tmpfiles[@]} | sed -e "$sedcmd" - fi - - rm ${tmpfiles[@]} -} - -split=0 -ministat_opt="-w72" - -while getopts "hSAnsc:w:" o; do - case "${o}" in - S) split=1 ;; - c) ministat_opt+=" -c $OPTARG" ;; - w) ministat_opt+=" -w $OPTARG" ;; - A) ministat_opt+=" -$o" ;; - n) ministat_opt+=" -$o" ;; - s) ministat_opt+=" -$o" ;; - *) usage ;; - esac -done - -shift $((OPTIND-1)) - -ministat=@ministat@/bin -#ministat=/nix/store/sh9b484bnhkajxnblpwix7fhbkid6365-ministat-20150715-1/bin - -export PATH="$PATH:$ministat" - -files=("$@") -if [[ -z "${files[@]}" ]]; then - awk '/^time /{print $2}' | ministat $ministat_opt -else - stat_files -fi diff --git a/garlic/pp/merge.nix b/garlic/pp/merge.nix deleted file mode 100644 index 03caaea..0000000 --- a/garlic/pp/merge.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, lib -}: - -datasets: - -with lib; - -stdenv.mkDerivation { - name = "merged-dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - inherit datasets; - installPhase = '' - mkdir -p $out - n=1 - for d in $datasets; do - ln -s $d $out/$n - let n=n+1 - cat $d/dataset >> $out/dataset - done - ''; -} diff --git a/garlic/pp/rplot.nix b/garlic/pp/rplot.nix deleted file mode 100644 index df154da..0000000 --- a/garlic/pp/rplot.nix +++ /dev/null @@ -1,165 +0,0 @@ -{ - stdenv -, lib -, rWrapper -, rPackages -, fontconfig -, dejavu_fonts -, liberation_ttf -, noto-fonts -, makeFontsConf -, makeFontsCache -, jq -, fetchFromGitHub -, writeText -, runCommand -, glibcLocales -}: - -{ -# The two results to be compared - dataset -, script -, extraRPackages ? [] -}: - -with lib; - -let - scalesPatched = with rPackages; buildRPackage { - name = "scales"; - src = fetchFromGitHub { - owner = "mikmart"; - repo = "scales"; - #ref = "label-bytes"; - rev = "fa7d91c765b6b5d2f682c7c22e0478d96c2ea76c"; - sha256 = "10dsyxp9pxzdmg04xpnrxqhc4qfhbkr3jhx8whfr7z27wgfrr1n3"; - }; - propagatedBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - nativeBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - }; - - customR = rWrapper.override { - packages = with rPackages; [ scalesPatched tidyverse viridis egg - Cairo extrafont ] ++ extraRPackages; - }; - - myFonts = [ - dejavu_fonts - #noto-fonts - #liberation_ttf - ]; - - cacheConf = - let - cache = makeFontsCache { fontDirectories = myFonts; }; - in - writeText "fc-00-nixos-cache.conf" '' - - - - - ${concatStringsSep "\n" (map (font: "${font}") myFonts)} - ${optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' - - ${cache} - ''} - - ''; - - # default fonts configuration file - # priority 52 - defaultFontsConf = - let genDefault = fonts: name: - optionalString (fonts != []) '' - - ${name} - - ${concatStringsSep "" - (map (font: '' - ${font} - '') fonts)} - - - ''; - in - writeText "fc-52-nixos-default-fonts.conf" '' - - - - - ${genDefault [ "DejaVu Sans" ] "sans-serif"} - ${genDefault [ "DejaVu Serif" ] "serif"} - ${genDefault [ "DejaVu Sans Mono" ] "monospace"} - ${genDefault [ "Noto Color Emoji"] "emoji"} - - ''; - - fontConfPath = - let - fixedConf = runCommand "fonts-fixed.conf" { - preferLocalBuild = true; - } '' - head --lines=-2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - - cat >> $out << 'EOF' - - conf.d - EOF - - tail -2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - ''; - in - runCommand "fontconfig-conf" { - preferLocalBuild = true; - } '' - dst=$out/etc/fonts/conf.d - mkdir -p $dst - # fonts.conf - ln -s ${fixedConf} $dst/../fonts.conf - - # fontconfig default config files - ln -s ${fontconfig.out}/etc/fonts/conf.d/*.conf \ - $dst/ - - # 00-nixos-cache.conf - ln -s ${cacheConf} $dst/00-nixos-cache.conf - - # 52-nixos-default-fonts.conf - ln -s ${defaultFontsConf} $dst/52-nixos-default-fonts.conf - ''; - -in stdenv.mkDerivation { - name = "plot"; - buildInputs = [ customR jq fontconfig glibcLocales ]; - preferLocalBuild = true; - dontPatchShebangs = true; - phases = [ "installPhase" ]; - - installPhase = '' - export FONTCONFIG_PATH=${fontConfPath}/etc/fonts/ - export LANG=en_US.UTF-8 - - mkdir -p $out - cd $out - dataset=$(readlink -f ${dataset}/dataset) - - ln -s $dataset input - Rscript --vanilla ${script} "$dataset" "$out" - - # HACK: replace the \minus for a \hyphen to keep the file paths intact, so - # they can be copied to the terminal directly. The StandardEncoding is not - # working (AdobeStd.enc). - find "$out" -name '*.pdf' | xargs -l1 sed -i 's.45/minus.45/hyphen.g' - - if [ "''${dataset##*.}" == gz ]; then - gunzip --stdout $dataset - else - cat $dataset - fi | jq -c .total_time |\ - awk '{s+=$1} END {printf "%f\n", s/60}' > total_job_time_minutes - ''; -} diff --git a/garlic/pp/store.nix b/garlic/pp/store.nix deleted file mode 100644 index 6005efb..0000000 --- a/garlic/pp/store.nix +++ /dev/null @@ -1,89 +0,0 @@ -{ - stdenv -}: - -{ - trebuchet, - experiment -}: - -with builtins; - -let - experimentName = baseNameOf (experiment); - trebuchetName = baseNameOf (trebuchet); -in - stdenv.mkDerivation { - name = "resultTree"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - installPhase = '' - echo "resultTree: searching for garlicd daemon..." - if [ -e /garlic/run ]; then - echo "resultTree: asking the daemon to run and fetch the experiment" - - echo ${trebuchet} >> /garlic/run - echo "resultTree: waiting for experiment results..." - stdbuf -o 0 tail -f /garlic/st & - stpid=$! - res=$(cat /garlic/completed) - - kill -TERM $stpid - - if [ "$res" == "ERROR" ]; then - echo "resultTree: the experiment failed" - exit 1 - fi - - if [ "$res" != "${trebuchet}" ]; then - echo "resultTree: unknown trebuchet received" - exit 1 - fi - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: assuming results are already in /garlic" - fi - - echo "resultTree: attempting to copy the results from /garlic ..." - - exp=/garlic/cache/${experimentName} - - if [ ! -e "$exp" ]; then - echo "resultTree: $exp: not found" - echo "resultTree: run the experiment and fetch the results running" - echo "resultTree: the following command from the nix-shell" - echo - echo -e "\e[30;48;5;2mgarlic -RFv ${trebuchet}\e[0m" - echo - echo "resultTree: see garlic(1) for more details." - echo "resultTree: cannot continue building $out, aborting" - exit 1 - fi - - echo "resultTree: copying results from /garlic into the nix store..." - - mkdir -p $out - cp -aL $exp $out/ - ln -s ${trebuchet} $out/trebuchet - ln -s ${experiment} $out/experiment - - - if [ -e /garlic/run ]; then - echo "resultTree: removing temp files..." - echo ${trebuchet} >> /garlic/wipe - echo "resultTree: waiting confimation from daemon..." - cat /garlic/completed > /dev/null - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: ignoring temp files" - fi - - echo "resultTree: successfully copied into the nix store" - - echo " experiment: ${experiment}" - echo " trebuchet: ${trebuchet}" - echo " resultTree: $out" - ''; - } diff --git a/garlic/report.nix b/garlic/report.nix deleted file mode 100644 index 6d7dc46..0000000 --- a/garlic/report.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, sedReport -}: -let - sedCmd = (import sedReport) fig; -in - stdenv.mkDerivation { - name = "report.pdf"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${sedCmd} - make - ''; - installPhase = '' - cp report.pdf $out - ''; - } diff --git a/garlic/report/Makefile b/garlic/report/Makefile deleted file mode 100644 index 81ae94d..0000000 --- a/garlic/report/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -all: report.tex - pdflatex report.tex -o report.pdf - # Run again to fix figure references - pdflatex report.tex -o report.pdf diff --git a/garlic/report/report.tex b/garlic/report/report.tex deleted file mode 100644 index f56c1c9..0000000 --- a/garlic/report/report.tex +++ /dev/null @@ -1,31 +0,0 @@ -\documentclass{article} -\usepackage{graphicx} - -\begin{document} - -\title{Example of Nix + \LaTeX{}} -\author{Rodrigo Arias Mallo} - -\maketitle - -\section{Nbody} -The nbody program has been executed with varying block sizes while the execution -time $t$ is measured, as shown in the figure \ref{fig:nbody.test}. -% -\begin{figure}[h] - \centering - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/scatter.png} - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/box.png} - \caption{Nbody times with varying block size} - \label{fig:nbody.test} -\end{figure} -% -The normalized time $\hat t$ is computed with the median time $t_m$ using $ \hat -t = t / t_{m} - 1 $. It can be observed that the normalized times exceed the -maximum allowed interval in most cases, except with the largest block sizes. - -Once the experiment \texttt{exp.nbody.test} changes, the hash of the experiment -program will change, therefore the plot will be updated and, lastly, this -report. - -\end{document} diff --git a/garlic/reportTar.nix b/garlic/reportTar.nix deleted file mode 100644 index c2b274b..0000000 --- a/garlic/reportTar.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, bundleReport -}: -let - - genCmd = (import bundleReport) fig; -in - stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${genCmd} - ls -ltR - cat report.tex - make - ''; - installPhase = '' - cd .. - tar -czf $out report - ''; - } diff --git a/garlic/sedReport.nix b/garlic/sedReport.nix deleted file mode 100644 index b776681..0000000 --- a/garlic/sedReport.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - stdenv -, fig -}: -stdenv.mkDerivation { - name = "sedReport"; - src = ./report; - buildPhase = '' - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - sed 's:\(^.*\)$:sed -i "s;@\1@;''${\1};g" report.tex:g' list >> fun.nix - echo "'''" >> fun.nix - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/sh/default.nix b/garlic/sh/default.nix deleted file mode 100644 index 27d34d1..0000000 --- a/garlic/sh/default.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ - stdenv -, garlicTools -, sshHost -, rsync -, openssh -, nix -, jq -, ncurses -}: - -with garlicTools; - -let - garlicPrefix = "/mnt/garlic"; -in - stdenv.mkDerivation { - name = "garlic-tool"; - preferLocalBuild = true; - - buildInputs = [ rsync openssh nix jq ncurses ]; - phases = [ "unpackPhase" "installPhase" ]; - - src = ./.; - - inherit garlicPrefix sshHost; - - installPhase = '' - substituteAllInPlace garlic - substituteInPlace garlic \ - --replace @PATH@ $PATH - mkdir -p $out/bin - cp garlic $out/bin - chmod +x $out/bin/garlic - mkdir -p $out/share/man/man1 - cp garlic.1 $out/share/man/man1 - - patchShebangs garlic-* - cp garlic-* $out/bin - ''; - } diff --git a/garlic/sh/fix-figure-subtitle.sh b/garlic/sh/fix-figure-subtitle.sh deleted file mode 100755 index a98614b..0000000 --- a/garlic/sh/fix-figure-subtitle.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -if grep -q 'output = args[2]' "$1"; then exit 0; fi - -sed '/length(args)>0/aif (length(args)>1) { output = args[2] } else { output = "?" }' -i "$1" -sed '/jsonlite::flatten/,$s/input_file/output/g' -i "$1" diff --git a/garlic/sh/garlic b/garlic/sh/garlic deleted file mode 100755 index df3e33f..0000000 --- a/garlic/sh/garlic +++ /dev/null @@ -1,298 +0,0 @@ -#!/bin/bash -e - -garlicPrefix=@garlicPrefix@ -sshHost=@sshHost@ -PATH=@PATH@ - -usage() { echo "Usage: garlic [-RFwv] trebuchet" 1>&2; exit 1; } - -msg() { - >&2 echo "garlic: $@" -} - -findClosure() { - what=$1 - from=$2 - mexp=$(nix-store -qR "$from" | grep -E -- "$what") - n=$(echo "$mexp" | awk 'BEGIN { count=0 } NF { count++ } END { print count }') - if [ $n -eq 0 ]; then - >&2 echo "$exp: $what not found" - exit 1 - fi - if [ $n -gt 1 ]; then - >&2 echo "$exp: multiple $what found" - exit 1 - fi - echo "$mexp" -} - -findExperiment() { - grep -o -- "/nix/store/.*-experiment" "$1" -} - -findOutputDir() { - garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - - if [ -z "$garlic_sandbox" ]; then - >&2 echo "Missing extra-sandbox-paths /garlic mountpoint" - >&2 echo "Check the ~/.config/nix/nix.conf file" - exit 1 - fi - - mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') - mountdir=$(readlink -f "$mountdir_rel") - - echo "$mountdir/cache" -} - -drvFromOutput() { - nix-store -q --deriver $1 -} - -checkTrebuchet() { - if [ ! -e "$trebuchet" ]; then - >&2 echo "$trebuchet: not found" - exit 1 - fi - - if [ ! -f "$trebuchet" ]; then - >&2 echo "$trebuchet: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a trebuchet - if [ -z "$(grep "This trebuchet launches" $trebuchet)" ]; then - >&2 echo "$trebuchet: not a trebuchet" - exit 1 - fi - - return 0 -} - -checkExperiment() { - if [ ! -e "$experiment" ]; then - >&2 echo "$experiment: not found" - exit 1 - fi - - if [ ! -f "$experiment" ]; then - >&2 echo "$experiment: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a experiment - if [ -z "$(grep "This is an experiment" $experiment)" ]; then - >&2 echo "$experiment: not an experiment" - exit 1 - fi - - return 0 -} - -checkMountpoint() { - if [ ! -e "$garlicPrefix/garlic.control" ]; then - >&2 echo "error: missing $garlicPrefix/garlic.control" - >&2 echo "Is the mountpoint enabled?" - exit 1 - fi -} - -status_line() { - unithash="$1" - name="$2" - status="$3" - - red=$(tput -T ansi setaf 1) - green=$(tput -T ansi setaf 2) - yellow=$(tput -T ansi setaf 3) - color_reset=$(tput -T ansi sgr0) - - case $status in - ok) - color_st="$green" - ;; - run*) - color_st="$yellow" - ;; - exit*) - color_st="$red" - ;; - *) - color_st="" - ;; - esac - - if [ $verbose ]; then - >&2 printf "garlic: %s %s%-9s%s %s\n" \ - "$unithash" "$color_st" "$status" "$color_reset" "$name" - fi -} - -do_fetch() { - expName=$(basename $experiment) - user=$(ssh -G "$sshHost" | awk '/^user /{print $2}') - exp=$garlicPrefix/$user/out/$expName - - if [ ! -e "$exp" ]; then - echo "missing experiment: $exp" - exit 1 - fi - - unitNameList=$(grep '^/nix/store/.*-unit$' $experiment |\ - sort -u |\ - sed 's@^/nix/store/@@') - - cwd=$(pwd) - - repeat=1 - bad=0 - while [ 1 ]; do - repeat=0 - test $verbose && msg "Checking units $(date --rfc-3339=seconds)..." - - declare -A unit_names - - for unit in $unitNameList; do - - unit_hash="${unit%-unit}" - unit_status="?" - unit_name="?" - - if [ -e "$exp/$unit" ]; then - st_file="$exp/$unit/status" - conf_json="$exp/$unit/garlic_config.json" - done_file="$exp/$unit/done" - - if [ -z "${unit_names[$unit_hash]}" ]; then - if [ -e "$conf_json" ]; then - unit_names+=([$unit_hash]=$(jq -r .unitName "$conf_json")) - unit_name="${unit_names[$unit_hash]}" - fi - else - unit_name="${unit_names[$unit_hash]}" - fi - - if [ -e "$st_file" ]; then - unit_status=$(cat "$st_file") - fi - fi - - status_line "$unit_hash" "$unit_name" "$unit_status" - - if [ ! -e "$done_file" ]; then - repeat=1 - elif [ "$unit_status" != "ok" ]; then - bad=1 - fi - done - - if [ $repeat -eq 0 ]; then - break - else - test $verbose && msg "" - fi - - if [ $waitResults -eq 1 ]; then - #echo "waiting 3 seconds to try again" - sleep 3 - else - break - fi - done - - if [ $repeat -eq 1 ]; then - #>&2 echo "$exp: execution incomplete" - exit 1 - fi - - if [ $bad -eq 1 ]; then - msg "Some units failed, aborting" - exit 1 - fi - - cd "$cwd" - - test $verbose && msg "execution complete, fetching results" - - mkdir -p "$outputDir" - - rsync -vrt --copy-links \ - --include='*/*/garlic_config.json' \ - --include='*/*/std*.log' \ - --include='*/*/*/std*.log' \ - --include='*/*/*/.garlic' \ - --include='*/*/*/.garlic/**' \ - --exclude='*/*/*/*' \ - "$exp" "$outputDir" - - if [ ! $enableKeep ]; then - nix-build -E "(with import ./default.nix; \ - garlic.pp.store { \ - trebuchet = (import \"$trebuchetDrv\" ); \ - experiment = (import \"$experimentDrv\"); \ - })" - - rm -rf "$outputDir/$expName" - fi -} - -do_run() { - - >&2 $trebuchet -} - -do_delete() { - expName=$(basename $experiment) - rm -rf $outputDir/$expName -} - -waitResults=1 -verbose= -operation= -target= -enableRun= -enableFetch= -enableKeep= -enableDelete= - -while getopts "vwRFKD" o; do - case "${o}" in - R) enableRun=1 ;; - F) enableFetch=1 ;; - K) enableKeep=1 ;; - D) enableDelete=1 ;; - w) waitResults=0 ;; - v) verbose=1 ;; - *) usage ;; - esac -done -shift $((OPTIND-1)) -trebuchet="$1" - -if [ -z "$enableRun" -a -z "$enableFetch" -a -z "$enableDelete" ]; then - >&2 echo "missing operation" - usage -fi - -if [ -z "$trebuchet" ]; then - >&2 echo "missing experiment" - usage -fi - -checkMountpoint - -checkTrebuchet $trebuchet - -outputDir=$(findOutputDir) - -experiment=$(findExperiment "$trebuchet") -checkExperiment $experiment - -trebuchetDrv=$(drvFromOutput $trebuchet) -experimentDrv=$(drvFromOutput $experiment) - -if [ $enableRun ]; then do_run; fi -if [ $enableFetch ]; then do_fetch; fi -if [ $enableDelete ]; then do_delete; fi diff --git a/garlic/sh/garlic-add-copyright b/garlic/sh/garlic-add-copyright deleted file mode 100755 index 089aafe..0000000 --- a/garlic/sh/garlic-add-copyright +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -if [ -z "$1" ]; then - cat << 'EOF' -Usage: garlic-add-copyright files... - -This tool prepends the content of the copyright file to the list of given files. -The word 'copyright' is used to determine if a file already has a copyright -notice or not. - -Example: - - Given the following HEADER file: - - $ cat HEADER - /* - * This file is part of NBody and is licensed under the terms contained - * in the LICENSE file. - * - * Copyright (C) 2021 Barcelona Supercomputing Center (BSC) - */ - - It can be prepended to all files ending in .c or .h with: - - $ garlic-add-copyright HEADER $(find * -type f | grep -Ei '\.(c|h)$') - -EOF - exit 1 -fi - -header_file="$1" -shift - -if ! grep -qi copyright "$header_file"; then - >&2 echo "The header file '$header_file' doesn't contain the word 'copyright'" - exit 1 -fi - -for f in "${@}"; do - if grep -qi copyright "$f"; then - echo "$f: Contains copyright word, skipping" - continue - fi - - tmp_fn="$f.copyright-being-added" - - cat "$header_file" "$f" > "$tmp_fn" - mv "$tmp_fn" "$f" -done diff --git a/garlic/sh/garlic-git-table b/garlic/sh/garlic-git-table deleted file mode 100755 index 58f4049..0000000 --- a/garlic/sh/garlic-git-table +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -progname="$(basename $0)" - -if [ -z "$1" ]; then - cat >&2 < - -Finds all garlic/* branches and their current commit of the given -repository and builds the gitTable to be used in nix derivations. - -Example: - garlic-git-table ssh://git@bscpm03.bsc.es/garlic/apps/heat.git -EOF - exit 1 -fi - -echo '{' -echo " # Auto-generated with $progname on $(date --rfc-3339=date) for repo:" -echo " # $1" -echo -git ls-remote $1 'garlic/*' |\ - sed 's@refs/heads/@@' |\ - awk '{printf "\"%s\" = \"%s\";\n", $2, $1}' |\ - column -t -o ' ' |\ - sed 's/^/ /' -echo '}' diff --git a/garlic/sh/garlic-propagate-commit b/garlic/sh/garlic-propagate-commit deleted file mode 100755 index 04617e1..0000000 --- a/garlic/sh/garlic-propagate-commit +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -e - -function printHelp() -{ - >&2 echo "Usage: garlic-propagate-commit " - >&2 echo - cat >&2 </dev/null | sed 's/^..//'` == $1 ]]; then - return 0 - else - return 1 - fi -} - - -if [ $# -lt 1 ]; then - printHelp - exit 1 -fi - -# Obtain all the tracked branches -branches=(`git branch | sed 's/^..//'`) -currentBranch=`git rev-parse --abbrev-ref HEAD` - -# Make sure that the commit SHA exists -commit=$1 -if branchContainsCommit $currentBranch $commit; then - echo "Commit $commit exists in current branch, proceeding..." -else - echo "Error: Commit $commit does not exist in the current branch" - exit 1 -fi - -# Distribute the commit for all tracked branches -for branch in ${branches[@]}; -do - if [[ $branch != $currentBranch ]]; then - echo "Trying to add commit $commit to branch $branch" - if branchContainsCommit $branch $commit; then - echo "Branch $branch already contains commit $commit, skipping" - else - git checkout $branch - git cherry-pick $commit - git push - fi - fi -done - -# Return to the original branch -git checkout $currentBranch diff --git a/garlic/sh/garlic-query b/garlic/sh/garlic-query deleted file mode 100755 index 6b0c2de..0000000 --- a/garlic/sh/garlic-query +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -function find_garlic_conf() { - grep -o '^cp /nix/store/[^ ]*-garlic_config.json' "$1" | sed 's/^cp //g' -} - -function show_units() { - exp=$1 - - units=$(grep '^/nix/store/.*-unit$' $exp) - - nunits=$(echo "$units" | wc -l) - echo " Experiment: $exp" - echo " Units: $nunits" - echo - echo " Unit file Name" - - for unit in $units; do - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - printf " %s %s %s %s\n" "$unit" "$present" "$unitName" - done -} - -function query_tre() { - tre=$1 - exp=$(grep "^# */nix/store/[^ ]*-experiment$" "$tre" | sed 's/# *//g') - - echo - echo " Trebuchet: $tre" - show_units $exp - echo -} - -function query_exp() { - exp=$1 - - echo - show_units "$exp" - echo -} - -function query_unit() { - unit=$1 - - stages=$(grep '^# */nix/store/.*$' $unit | sed 's/^# */ /g') - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - echo - echo " Unit: $unit" - echo " Name: $unitName" - echo " Stages:" - echo - echo "$stages" - echo - echo " Config: $gconf" - echo - jq . "$gconf" -} - -function query_result() { - tree=$1 - - tre=$(readlink -f $tree/trebuchet) - exp=$(readlink -f $tree/experiment) - - echo - echo " Result tree: $tree" - echo " Trebuchet: $tre" - show_units $exp - echo -} - -element=$1 - -if [ "$1" == "--help" -o -z "$1" ]; then - cat < - -The path may be a trebuchet, experiment, unit or resultTree. -EOF - exit 1 -fi - -# Try prepending the nix store -if [ ! -e $element ]; then - element=/nix/store/$element* -fi - -element=$(readlink -f $element) - -case "$element" in - *-trebuchet) query_tre $element ;; - *-experiment) query_exp $element ;; - *-unit) query_unit $element ;; - *-resultTree) query_result $element ;; - *) echo "unknown: $element"; exit 1 ;; -esac diff --git a/garlic/sh/garlic.1 b/garlic/sh/garlic.1 deleted file mode 100644 index 86915d8..0000000 --- a/garlic/sh/garlic.1 +++ /dev/null @@ -1,61 +0,0 @@ -.\" The following commands are required for all man pages. -.Dd Nov 5, 2020 -.Dt garlic 1 -.Os Linux -.Sh NAME -.Nm garlic -.Nd Garlic benchmark tool -.Sh SYNOPSIS -.Nm garlic -.Op Fl RFvw -.Ar trebuchet -.Sh DESCRIPTION -.Nm -is a program to handle tasks related with the experiments of the Garlic -benchmark. The -.Ar trebuchet -argument is a path to a valid trebuchet stage script. The -specified operations are executed sequentially and at -least one must be selected from: -.Bl -tag -width Ds -compact -.Pp -.It Fl R -launches the -.Ar experiment using the trebuchet. -.Pp -.It Fl F -waits until the -.Ar experiment -is completed and then fetches the results -into the nix store. -.El -.Pp -Additionally, the following options are available: -.Bl -tag -width Ds -compact -.Pp -.It Fl v -be more verbose. -.Pp -.It Fl w -don't wait for the complete results when fetching; fails if they are not -ready. -.El -.\" This next command is for sections 1, 6, 7, and 8 only. -.\" .Sh ENVIRONMENT -.\" .Sh FILES -.Sh EXIT STATUS -.Ex -std -.\" .Sh EXAMPLES -.\" This next command is for sections 1, 4, 6, 7, 8, and 9 only -.\" (fprintf/stderr type diagnostics). -.\" .Sh DIAGNOSTICS -.\" .Sh COMPATIBILITY -.\" This next command is for sections 2, 3, 4, and 9 only -.\" (settings of the errno variable). -.\" .Sh ERRORS -.\" .Sh SEE ALSO -.\" .Sh STANDARDS -.\" .Sh HISTORY -.\" .Sh AUTHORS -.\" .Sh CAVEATS -.\" .Sh BUGS diff --git a/garlic/shell.nix b/garlic/shell.nix deleted file mode 100644 index 9a38076..0000000 --- a/garlic/shell.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ pkgs ? import ./. }: - -with pkgs; -with bsc; - -mkShell { - name = "garlic-shell"; - - buildInputs = - # Packages from garlic - (with garlic; [ tool garlicd ]) ++ - # Packages from bsc - [ groff paraver icc nix openssh git cn6 nix-diff clangOmpss2 gdb ]; - - # inputsFrom to get build dependencies - - shellHook = '' - alias l="ls -l --color=auto -v" - alias ll="ls -l --color=auto -v" - alias lh="ls -hAl --color=auto -v" - alias ls="ls --color=auto -v" - alias ..="cd .." - - export LANG=C - export SHELL=${bash}/bin/bash - - echo Welcome to the garlic shell - ''; -} diff --git a/garlic/stages/baywatch.nix b/garlic/stages/baywatch.nix deleted file mode 100644 index 2b72f4d..0000000 --- a/garlic/stages/baywatch.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - stdenv -, garlicTools -}: -{ - nextStage -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "baywatch"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${stageProgram nextStage} - echo $? >> .srun.rc.$SLURM_PROCID - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/control.nix b/garlic/stages/control.nix deleted file mode 100644 index 3e04485..0000000 --- a/garlic/stages/control.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, loops ? 30 -}: - -with garlicTools; - -stdenv.mkDerivation { - name = "control"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<"EOF" - #!/bin/sh -e - - function badexit() { - errcode=$? - if [ $errcode != 0 ]; then - printf "exit %d\n" $errcode > "$basedir/status" - echo "exiting with $errcode" - fi - - echo 1 > "$basedir/done" - exit $errcode - } - - trap badexit EXIT - - basedir=$(pwd) - loops=${toString loops} - for n in $(seq 1 $loops); do - export GARLIC_RUN="$n" - echo "run $n/$loops" > status - mkdir "$n" - cd "$n" - mkdir .garlic - date +%s > .garlic/total_time_start - ${stageProgram nextStage} - date +%s > .garlic/total_time_end - cd .. - done - echo "ok" > status - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/exec.nix b/garlic/stages/exec.nix deleted file mode 100644 index 8717041..0000000 --- a/garlic/stages/exec.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, env ? "" -, pre ? "" -, argv ? [] -, post ? "" -, nixPrefix ? "" -, program ? null -}: - -with builtins; -with garlicTools; - -let - argvString = concatStringsSep " " (map (e: toString e) argv); - execMethod = if (post == "") then "exec " else ""; - programPath = if (program != null) then program else (stageProgram nextStage); -in -stdenv.mkDerivation { - name = "exec"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - ${env} - - ${pre} - - ${execMethod}${nixPrefix}${programPath} ${argvString} - - ${post} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/experiment.nix b/garlic/stages/experiment.nix deleted file mode 100644 index 19c0e45..0000000 --- a/garlic/stages/experiment.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, garlicTools -}: - -{ - units -}: - -with lib; -with garlicTools; - -let - unitsString = builtins.concatStringsSep "\n" - (map (x: "${stageProgram x}") units); - - unitsLinks = builtins.concatStringsSep "\n" - (map (x: "ln -s ../${baseNameOf x} ${baseNameOf x}") units); -in -stdenv.mkDerivation { - name = "experiment"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - inherit units; - - isExperiment = true; - - installPhase = '' - cat > $out << EOF - #!/bin/sh - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "experiment: GARLIC_OUT not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - export GARLIC_EXPERIMENT=$(basename $out) - - if [ -e "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "experiment: skipping, directory exists: \$GARLIC_EXPERIMENT" - exit 0 - fi - - mkdir -p "\$GARLIC_EXPERIMENT" - - cd "\$GARLIC_EXPERIMENT" - ${unitsLinks} - - >&2 echo "experiment: running \$GARLIC_EXPERIMENT" - - # This is an experiment formed by the following units: - ${unitsString} - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/extrae.nix b/garlic/stages/extrae.nix deleted file mode 100644 index dabb31e..0000000 --- a/garlic/stages/extrae.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, configFile -, traceLib -, extrae -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "extrae"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <&2 echo Running isolate stage1 -#>&2 echo PATH=$PATH - -if [ -e /nix ]; then - >&2 echo "/nix found, aborting" - exit 1 -fi - -nixhome="@nixPrefix@/nix" -shell="@busybox@/bin/sh" -nixjoin="@nixPrefix@@nixtools@/bin/nix-join" - - -env=( - PATH="@nixPrefix@@busybox@/bin:@busybox@/bin:@extraPath@" - $(env | grep ^SLURM || true) - $(env | grep ^PMI || true) - $(env | grep ^GARLIC || true) - $(env | grep ^USER || true) - $(env | grep ^TMPDIR || true) - $(env | grep '^TERM=' || true) - HOME="/homeless-shelter" -) - -mounts=( - #-m @nixPrefix@ - #FIXME: Use only the strictly neccesary from /etc - -m /original-etc:/etc - # The /etc/hosts file is a symlink to this etc/ - -m /.statelite/tmpfs/etc - -m /sys - -m /dev - -m /proc - # nscd cache: doesn't exist (?) - #-m /var/run/nscd - # Needed for munge auth - -m /var/run/munge - # FIXME: We should only need nix and the output path - -m /gpfs/projects/bsc15 - -m /gpfs/scratch/bsc15 - -m /bin:@nixPrefix@@busybox@/bin - -m "$TMPDIR" - @extraMountOptions@ -) - -symlinks=( - -s /etc/hosts:/original-etc/hosts - -s /etc/passwd:/original-etc/passwd - -s /etc/resolv.conf:/original-etc/resolv.conf - -s /etc/host.conf:/original-etc/host.conf - -s /etc/slurm/slurm.conf:/original-etc/slurm/slurm.conf - -s /etc/services:/original-etc/services -) - -exec $nixjoin -i "${mounts[@]}" "${symlinks[@]}" $nixhome -- \ - env -i "${env[@]}" @out@/bin/stage2 "$@" diff --git a/garlic/stages/isolate/stage2 b/garlic/stages/isolate/stage2 deleted file mode 100644 index 1924890..0000000 --- a/garlic/stages/isolate/stage2 +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -#>&2 echo Running isolate stage2 -#>&2 echo PATH=$PATH - -if [ ! -e /nix ]; then - >&2 echo "/nix not found, aborting" - exit 1 -fi - -if [ -e /usr ]; then - >&2 echo "Environment not isolated, aborting" - exit 1 -fi - -exec @program@ "$@" diff --git a/garlic/stages/perf.nix b/garlic/stages/perf.nix deleted file mode 100644 index cbbe937..0000000 --- a/garlic/stages/perf.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - stdenv -, perf -, garlicTools -}: - -{ - nextStage -, perfOptions -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "perf"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out <&2 echo "Cannot use runexp inside nix environment!" - exit 1 -fi - -#>&2 echo Running runexp for MN4 -#>&2 echo PATH=$PATH - -user=$(id -un) -group=$(id -gn) - -export GARLIC_OUT="/gpfs/projects/bsc15/garlic/$user/out" -mkdir -p "$GARLIC_OUT" -export GARLIC_INDEX="/gpfs/projects/bsc15/garlic/$user/index" -mkdir -p "$GARLIC_INDEX" -export GARLIC_USER="$user" -cd "$GARLIC_OUT" - -exec @nixPrefix@@program@ diff --git a/garlic/stages/sbatch.nix b/garlic/stages/sbatch.nix deleted file mode 100644 index 34a3167..0000000 --- a/garlic/stages/sbatch.nix +++ /dev/null @@ -1,105 +0,0 @@ -{ - stdenv -, lib -, numactl -, slurm -, garlicTools -}: - -{ - nextStage -, jobName -, chdir ? "." -, nixPrefix ? "" -, binary ? "/bin/run" -, ntasks ? null -, ntasksPerNode ? null -, ntasksPerSocket ? null -, cpusPerTask ? null -, nodes ? null -, exclusive ? true # By default we run in exclusive mode -, qos ? null -, reservation ? null -, time ? null -, output ? "stdout.log" -, error ? "stderr.log" -, extra ? null -, acctgFreq ? null -}: - -with lib; -with garlicTools; - -# sbatch fails silently if we pass garbage, so we assert the types here to avoid -# sending `nodes = [ 1 2 ]` by mistake. -assert (jobName != null) -> isString jobName; -assert (chdir != null) -> isString chdir; -assert (nixPrefix != null) -> isString nixPrefix; -assert (ntasks != null) -> isInt ntasks; -assert (ntasksPerNode != null) -> isInt ntasksPerNode; -assert (ntasksPerSocket != null) -> isInt ntasksPerSocket; -assert (cpusPerTask != null) -> isInt cpusPerTask; -assert (nodes != null) -> isInt nodes; -assert (exclusive != null) -> isBool exclusive; -assert (qos != null) -> isString qos; -assert (reservation != null) -> isString reservation; -assert (time != null) -> isString time; -assert (output != null) -> isString output; -assert (error != null) -> isString error; -assert (extra != null) -> isString extra; - -let - - sbatchOpt = name: value: optionalString (value!=null) - "#SBATCH --${name}=${toString value}\n"; - sbatchEnable = name: value: optionalString (value!=null) - "#SBATCH --${name}\n"; - -in - -stdenv.mkDerivation rec { - name = "sbatch"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - #SBATCH --tasks-per-node=48 - #SBATCH --ntasks-per-socket=24 - #SBATCH --cpus-per-task=1 - dontBuild = true; - dontPatchShebangs = true; - programPath = "/run"; - - installPhase = '' - mkdir -p $out - cat > $out/job < $out/run < status - EOF - chmod +x $out/run - ''; -} diff --git a/garlic/stages/script.nix b/garlic/stages/script.nix deleted file mode 100644 index 292a209..0000000 --- a/garlic/stages/script.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -}: - -{ - script -, shell ? "/bin/sh" -, exitOnError ? true -}: - -let - setcmd = if exitOnError then "set -e" else ""; -in -stdenv.mkDerivation { - name = "script"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!${shell} - ${setcmd} - - ${script} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/srun.nix b/garlic/stages/srun.nix deleted file mode 100644 index a2481a5..0000000 --- a/garlic/stages/srun.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, slurm -, garlicTools -}: -{ - nextStage -, cpuBind -, nixPrefix -, preSrun ? "" -, postSrun ? "" -, srunOptions ? "" -, output ? "stdout.log" -, error ? "stderr.log" -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "srun"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${preSrun} - - ${slurm}/bin/srun \ - --mpi=pmi2 \ - --cpu-bind=${cpuBind} \ - --output=${output} \ - --error=${error} \ - ${srunOptions} \ - ${nixPrefix}${stageProgram nextStage} - - >&2 echo srun exit code: $? - - # Ensure that none failed, as srun fails to capture errors - # after MPI_Finalize - for i in $(seq 0 $(($SLURM_NTASKS - 1))); do - if [ ! -e .srun.rc.$i ]; then - >&2 echo "missing exit code for rank $i, aborting" - exit 1 - fi - if ! grep -q '^0$' .srun.rc.$i; then - >&2 echo "non-zero exit for rank $i, aborting" - exit 1 - fi - done - - ${postSrun} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/strace.nix b/garlic/stages/strace.nix deleted file mode 100644 index 22fb4a5..0000000 --- a/garlic/stages/strace.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, strace -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "strace"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out < $out << EOF - #!/bin/sh -e - - ${desc} - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "unit: GARLIC_OUT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "unit: GARLIC_EXPERIMENT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_INDEX" ]; then - >&2 echo "unit: GARLIC_INDEX not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - # Set the experiment unit in the environment - export GARLIC_UNIT=$(basename $out) - - # Create an index entry - rm -f "\$GARLIC_INDEX/${safeUnitName}" \ - "\$GARLIC_INDEX/${safeExpName}" - - ln -Tfs "../out/\$GARLIC_UNIT" \ - "\$GARLIC_INDEX/${safeUnitName}" - - ln -Tfs "../out/\$GARLIC_EXPERIMENT" \ - "\$GARLIC_INDEX/${safeExpName}" - - if [ -e "\$GARLIC_UNIT" ]; then - >&2 echo "unit: skipping, already exists: \$GARLIC_UNIT" - exit 0 - fi - - # And change the working directory - mkdir \$GARLIC_UNIT - cd \$GARLIC_UNIT - - # Copy the configuration for the unit to the output path - cp ${jsonConf} garlic_config.json - - # Finally, execute the first stage: - exec ${firstStage} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/valgrind.nix b/garlic/stages/valgrind.nix deleted file mode 100644 index 9e156c0..0000000 --- a/garlic/stages/valgrind.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, valgrind -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "valgrind"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out''); -} diff --git a/garlic/tools.nix b/garlic/tools.nix deleted file mode 100644 index c092b55..0000000 --- a/garlic/tools.nix +++ /dev/null @@ -1,140 +0,0 @@ -{ - stdenv -, lib -}: - -with lib; - -let - gen = rec { - # genAttrSets "a" ["hello" "world"] - # [ { a = "hello"; } { a = "world"; } ] - genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); - - # addAttrSets "a" [1 2] {e=4;} - # [ { a = 1; e = 4; } { a = 2; e = 4; } ] - addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); - - # attrToList {a=1;} - # [ { name = "a"; value = 1; } ] - attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); - - # mergeConfig [{e=1;}] {name="a"; value=[1 2] - # [ { a = 1; e = 1; } { a = 2; e = 1; } ] - mergeConfig = (arr: new: flatten ( map (x: addAttrSets new.name new.value x) arr)); - - # genConfigs {a=[1 2]; b=[3 4];} - # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] - genConfigs = (config: foldl mergeConfig [{}] (attrToList config)); - - # Generate multiple app versions by override with each config - genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs); - - # Generate app version from an array of apps - genApps = (apps: configs: - flatten (map (app: genApp app configs) apps)); - - /* Returns the path of the executable of a stage */ - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" - else "${stage}"; - - /* Given a trebuchet, returns the experiment */ - getExperimentStage = drv: - if (drv ? isExperiment) && drv.isExperiment then drv - else getExperimentStage drv.nextStage; - - # Computes the exponentiation operation - pow = x: n: fold (a: b: a*b) 1 (map (a: x) (range 1 n)); - - # Generates a list of exponents from a to b inclusive, and raises base to - # each element of the list. - expRange = base: a: b: (map (ex: pow base ex) (range a b)); - - # Generates a range from start to end (inclusive) by multiplying start by 2. - range2 = start: end: - let - _range2 = s: e: if (s > e) then [] else [ s ] ++ (_range2 (s * 2) e); - in - _range2 start end; - - # Generates a list of integers by halving number N until it reaches 1. Is - # sorted from the smallest to largest. - halfList = N: - let - _divList = n: if (n == 0) then [] else (_divList (n / 2)) ++ [ n ]; - in - _divList N; - - # A list of all divisors of n, sorted in increased order: - divisors = n: filter (x: (mod n x == 0)) (range 1 n); - - # Generates a set given a list of keys, where all values are null. - genNullAttr = l: genAttrs l (name: null); - - # From the keys in the lis l, generates a set with the values in the set a, - # if they don't exist, they are not taken. Values set to null are removed. - optionalInherit = l: a: filterAttrs (n: v: v!=null) - (overrideExisting (genNullAttr l) a); - - # Given a float f, truncates it and returns the resulting the integer - floatTruncate = f: let - strFloat = toString f; - slices = splitString "." strFloat; - front = elemAt slices 0; - in - toInt front; - - # Returns the given gitCommit if not null, or the one stored in the - # gitTable for the branch gitBranch. - findCommit = {gitCommit ? null, gitTable ? null, gitBranch}: - assert (gitCommit == null) -> (gitTable != null); - assert (gitTable == null) -> (gitCommit != null); - if (gitCommit != null) then gitCommit - else - assert (assertMsg (gitTable ? "${gitBranch}") - '' - The git branch "${gitBranch}" was not found in the gitTable. - Is the gitTable outdated? - ''); - gitTable."${gitBranch}"; - - # Custom wrapper around fetchGit to be able to quickly specify apps - # and change the repository source for all at once. Returns an - # attributte set with the `src` as well as the selected gitCommit, - # gitBranch and gitURL. - fetchGarlicApp = { - gitBranch, - appName ? null, - gitURL ? null, - gitCommit ? null, - gitTable ? null - }: - assert (appName == null) -> (gitURL != null); - assert (gitURL == null) -> (appName != null); - let - _gitURL = if (gitURL != null) then gitURL - else "ssh://git@bscpm03.bsc.es/garlic/apps/${appName}.git"; - _gitCommit = findCommit { - inherit gitCommit gitTable gitBranch; - }; - _gitBranch = gitBranch; - in - { - src = builtins.fetchGit { - url = _gitURL; - ref = _gitBranch; - rev = _gitCommit; - }; - gitBranch = _gitBranch; - gitCommit = _gitCommit; - - # The gitURL is not stored in the derivation, as we dont really - # care of where the source comes from, as long as is the same - # commit. - gitURL = _gitURL; - }; - }; -in - gen