diff --git a/garlic/apps/bigsort/default.nix b/garlic/apps/bigsort/default.nix deleted file mode 100644 index 4e7dd3d..0000000 --- a/garlic/apps/bigsort/default.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - stdenv -, lib -, cc -, nanos6 ? null -, mcxx ? null -, mpi -, gitBranch -}: - -with lib; -stdenv.mkDerivation rec { - name = "bigsort"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "${gitBranch}"; - }; - - #sourceRoot = "./BigSort"; - - preBuild = '' - cd BigSort - export I_MPI_CXX=${cc.CXX} - ''; - - buildInputs = [ - cc - mpi - ] - ++ optional (mcxx != null) mcxx - ++ optional (nanos6 != null) nanos6; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - "CLUSTER=MareNostrum4" - "OPENMP=yes" - "Debug=no" - "OPENMP_FLAGS=-qopenmp" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bigsort $out/bin/BigSort - ''; - - programPath = "/bin/BigSort"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/genseq.nix b/garlic/apps/bigsort/genseq.nix deleted file mode 100644 index 6036467..0000000 --- a/garlic/apps/bigsort/genseq.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "genseq"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/GenSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp genseq $out/bin/genseq - ''; - - programPath = "/bin/genseq"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/bigsort/shuffle.nix b/garlic/apps/bigsort/shuffle.nix deleted file mode 100644 index 977d911..0000000 --- a/garlic/apps/bigsort/shuffle.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi -}: - -with lib; -stdenv.mkDerivation rec { - name = "shuffle"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/dalvare1/bigsort.git"; - ref = "garlic/mpi+send+omp+task"; - }; - - postUnpack = "sourceRoot=$sourceRoot/ShuffleSeq"; - - # FIXME: Remove the ../commons/Makefile as is not useful here, we only need - # the CPP_SRC and OBJ variables. - postPatch = '' - sed -i '1cCPP_SRC = $(wildcard *.cpp)' Makefile - sed -i '2cOBJ = $(CPP_SRC:.cpp=.o)' Makefile - ''; - - buildInputs = [ - cc - mpi - ]; - - makeFlags = [ - "I_MPI_CXX=${cc.CXX}" - "CPP_BIN=mpicxx" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp shuffle $out/bin/shuffle - ''; - - programPath = "/bin/shuffle"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/cpic/default.nix b/garlic/apps/cpic/default.nix deleted file mode 100644 index 284c1e9..0000000 --- a/garlic/apps/cpic/default.nix +++ /dev/null @@ -1,43 +0,0 @@ -{ - stdenv -, libconfig -, nanos6 -, mpi -, uthash -, fftw -, tampi -, hdf5 -}: - -stdenv.mkDerivation rec { - name = "cpic"; - - # Use my current cpic version, so I can test changes without commits - #src = /home/Computational/rarias/cpic; - - src = builtins.fetchGit { - url = "https://github.com/rodarima/cpic"; -# rev = "73bd70448587f0925b89e24c8f17e412ea3958e6"; - ref = "simd"; - }; - - enableParallelBuilding = true; - dontStrip = true; - - buildInputs = [ - libconfig - nanos6 - mpi - uthash - fftw - tampi - hdf5 - ]; - - installPhase = '' - mkdir -p $out/bin - cp cpic $out/bin/cpic - ''; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/creams/default.nix b/garlic/apps/creams/default.nix deleted file mode 100644 index c083886..0000000 --- a/garlic/apps/creams/default.nix +++ /dev/null @@ -1,57 +0,0 @@ -{ - stdenv -, nanos6 -, mpi -, openmpi -, impi -, tampi -, mcxx -, gnuDef -, intelDef -, cc -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -assert (mpi == impi || mpi == openmpi); - -let - # FIXME: We should find a better way to specify the MPI implementation - # and the compiler. - mpiName = if mpi == openmpi then "OpenMPI" else "IntelMPI"; - compName = if cc == intelDef then "Intel" else "GNU"; - - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "creams"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/creams.exe"; - - buildInputs = [ nanos6 mpi cc tampi mcxx ]; - - configurePhase = '' - export TAMPI_HOME=${tampi} - - . etc/bashrc - - export FORTRAN_COMPILER=${compName} - export MPI_LIB=${mpiName} - - CREAMS_UPDATE_ENVIRONMENT - ''; - - installPhase = '' - mkdir -p $out/bin - cp -a build/* $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/creams/gen_grid.py b/garlic/apps/creams/gen_grid.py deleted file mode 100755 index 723988b..0000000 --- a/garlic/apps/creams/gen_grid.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 - -import argparse - -parser = argparse.ArgumentParser(description="Generate a grid.dat input file for CREAMS") - -parser.add_argument('--npx', type=int, help='number of processes in X', default=1) -parser.add_argument('--npy', type=int, help='number of processes in Y', default=1) -parser.add_argument('--npz', type=int, help='number of processes in Z', default=32) -parser.add_argument('--grain', type=int, help='granularity', default=9) -parser.add_argument('--nx', type=int, help='number of points in X', default=20) -parser.add_argument('--ny', type=int, help='number of points in Y', default=20) -parser.add_argument('--nz', type=int, help='number of points in Z', default=7000) -parser.add_argument('--dx', type=float, help='grid spacing in X', default=0.0025062657) -parser.add_argument('--dy', type=float, help='grid spacing in Y', default=0.0025062657) -parser.add_argument('--dz', type=float, help='grid spacing in Z', default=0.0025062657) - -args = parser.parse_args() - -grain_str = "%d %d" % (args.grain, args.grain) -boundary = "extrapolation" - -# Print -print(' %-49d number of processes in x-direction (0 if automatic)' % args.npx) -print(' %-49d number of processes in y-direction (0 if automatic)' % args.npy) -print(' %-49d number of processes in z-direction (0 if automatic)' % args.npz) -print(' ') -print(' %-49s subdomain granularity' % grain_str) -print(' ') -print(' %-49s -x boundary' % boundary) -print(' %-49s +x boundary' % boundary) -print(' %-49s -y boundary' % boundary) -print(' %-49s +y boundary' % boundary) -print(' %-49s -z boundary' % boundary) -print(' %-49s +z boundary' % boundary) -print(' ') -print(' x-direction') -for i in range(args.nx): - print("%.9e" % (i * args.dx)) -print(' ') -print(' y-direction') -for i in range(args.ny): - print("%.9e" % (i * args.dy)) -print(' ') -print(' z-direction') -for i in range(args.nz): - print("%.9e" % (i * args.dz)) -print(' ') -print(' END') diff --git a/garlic/apps/creams/git-table.nix b/garlic/apps/creams/git-table.nix deleted file mode 100644 index 9600240..0000000 --- a/garlic/apps/creams/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-03-31 for repo: - # ssh://git@bscpm03.bsc.es/pmartin1/creams-simplified.git - - "garlic/mpi+isend+omp+task" = "e6aa540820ee12d3d45d0eef8e7eeb2f0f1daea2"; - "garlic/mpi+isend+oss+task" = "016f33b8bec996a4546e8f08b1b6b1709f00499b"; - "garlic/mpi+send+omp+fork" = "e56e059264ad1bfe5e0c96a8b9303d21dd7fa20a"; - "garlic/mpi+send+omp+task" = "919580213de34bc5b6ba60c768c5dde5e501a1f6"; - "garlic/mpi+send+oss+task" = "adab8b66f27317d51445648302e7b133edf4837d"; - "garlic/mpi+send+seq" = "956125f9334493d31ceee3fa7024efa65bee9ca5"; - "garlic/tampi+isend+oss+task" = "14a121627679a251909d4b8103d260e27eac1d29"; -} diff --git a/garlic/apps/creams/input.nix b/garlic/apps/creams/input.nix deleted file mode 100644 index 5a12f91..0000000 --- a/garlic/apps/creams/input.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, python3 -, granul ? 9 -, nprocx ? 1 -, nprocy ? 1 -, nprocz ? 1 -, nx ? 20 -, ny ? 20 -, nz ? 7000 -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "creams"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; - - gen = ./gen_grid.py; -in - stdenv.mkDerivation rec { - name = "creams-input"; - - buildInputs = [ python3 ]; - - inherit (gitSource) src gitBranch gitCommit; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -a SodTubeBenchmark $out/ - - python3 ${gen} \ - --npx ${toString nprocx} \ - --npy ${toString nprocy} \ - --npz ${toString nprocz} \ - --grain ${toString granul} \ - --nx ${toString nx} \ - --ny ${toString ny} \ - --nz ${toString nz} \ - > $out/SodTubeBenchmark/grid.dat - ''; - } diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix deleted file mode 100644 index 681509a..0000000 --- a/garlic/apps/fwi/default.nix +++ /dev/null @@ -1,77 +0,0 @@ -{ - stdenv -, lib -, mpi ? null -, tampi ? null -, mcxx ? null -, cc -, gitBranch ? "garlic/tampi+send+oss+task" -, gitCommit ? null -, fwiParams -, garlicTools -}: - -with lib; - -assert !(tampi != null && mcxx == null); - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # FIXME: Correct this on the Makefile so we can just type "make fwi" - # FIXME: Allow multiple MPI implementations - postPatch = '' - sed -i 's/= OPENMPI$/= INTEL/g' Makefile - sed -i 's/USE_O_DIRECT ?= NO/USE_O_DIRECT ?= YES/g' Makefile || true - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - - make depend - - cp ${fwiParams}/generated_model_params.h src/ - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - makeFlags = [ - #"COMPILER=GNU" - #"CC=${cc.cc.CC}" - "fwi" - ]; - - installPhase = '' - mkdir -p $out/bin - cp fwi $out/bin - ''; - - programPath = "/bin/fwi"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/fwi/git-table.nix b/garlic/apps/fwi/git-table.nix deleted file mode 100644 index c20adff..0000000 --- a/garlic/apps/fwi/git-table.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git - - "garlic/mpi+send+omp+fork" = "ea1ed53f20858dc082f9cbbe0e7e8fb28a6fe58a"; - "garlic/mpi+send+omp+task" = "aa8881056fb3fa98832d203899beacfb8fa702f6"; - "garlic/mpi+send+oss+task" = "c184484af8498fd939761575b34bb46ba3be0dde"; - "garlic/mpi+send+oss+task+NOREUSE" = "0062093ef744c694d69673d1881719958bbed353"; - "garlic/mpi+send+seq" = "cc184ad77f143481e2506933d0cdc038c349f071"; - "garlic/omp+task" = "1fe23690d74ae89ace5383ab165a6ce6346c2afd"; - "garlic/oss+task" = "68e6e8f17ee03addb460745acea2a38241ca89ee"; - "garlic/seq" = "aa6b6c5857125796c65fbf23018d557e4693f1ae"; - "garlic/tampi+isend+oss+task" = "7c98194e13786c4e8ecfa8a144587e5a95e09205"; - "garlic/tampi+send+oss+task" = "e08d66f7453c4034a363bb2d22c5248fe86ed740"; -} diff --git a/garlic/apps/fwi/params.nix b/garlic/apps/fwi/params.nix deleted file mode 100644 index 7fad46b..0000000 --- a/garlic/apps/fwi/params.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, nz ? 200 -, nx ? 200 -, ny ? 500 -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -with lib; -with builtins; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "fwi"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "fwi-params"; - - inherit (gitSource) src gitBranch gitCommit; - - enableParallelBuilding = false; - - # Set the input size with the weird order (nz,nx,ny). - postPatch = '' - sed -i 1c${toString nz} SetupParams/fwi_params.txt - sed -i 2c${toString nx} SetupParams/fwi_params.txt - sed -i 3c${toString ny} SetupParams/fwi_params.txt - ''; - - # FIXME: This is an ugly hack. - # When using _GNU_SOURCE or any other definition used in features.h, we need - # to define them before mcc includes nanos6.h from the command line. So the - # only chance is by setting it at the command line with -D. Using the DEFINES - # below, reaches the command line of the preprocessing stage with gcc. - preConfigure = '' - export DEFINES=-D_GNU_SOURCE - ''; - - # We compile the ModelGenerator using gcc *only*, as otherwise it will - # be compiled with nanos6, which requires access to /sys to determine - # hardware capabilities. So it will fail in the nix-build environment, - # as there is no /sys mounted. - # Also, we need to compile it with the builder platform as target, as is going - # to be executed during the build to generate the src/generated_model_params.h - # header. - makeFlags = [ "COMPILER=GNU" "params" ]; - - installPhase = '' - mkdir -p $out/ - cp src/generated_model_params.h $out/ - cp SetupParams/fwi_params.txt $out/ - cp SetupParams/fwi_frequencies.txt $out/ - - mkdir -p $out/bin - cp ModelGenerator $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/default.nix b/garlic/apps/heat/default.nix deleted file mode 100644 index 02e8b76..0000000 --- a/garlic/apps/heat/default.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ - stdenv -, mpi -, tampi -, mcxx -, gitBranch ? "garlic/mpi+send+seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "heat"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - - name = "heat"; - - inherit (gitSource) src gitBranch gitCommit; - - patches = [ ./print-times.patch ]; - - buildInputs = [ mpi mcxx tampi ]; - - programPath = "/bin/${name}"; - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp heat.conf $out/etc/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/heat/git-table.nix b/garlic/apps/heat/git-table.nix deleted file mode 100644 index c7670ac..0000000 --- a/garlic/apps/heat/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-01 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/heat.git - - "garlic/mpi+send+oss+task" = "947c80070d4c53e441df54b8bfac8928b10c5fb2"; - "garlic/mpi+send+seq" = "f41e1433808d0cbecd88a869b451c927747e5d42"; - "garlic/tampi+isend+oss+task" = "b1273f9b4db32ba6e15e3d41343e67407ce2f54f"; - "garlic/tampi+send+oss+task" = "554bec249f9aa23dd92edcfa2ada1e03e05e121d"; -} diff --git a/garlic/apps/heat/print-times.patch b/garlic/apps/heat/print-times.patch deleted file mode 100644 index bf5d28a..0000000 --- a/garlic/apps/heat/print-times.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/src/mpi/main.c b/src/mpi/main.c -index 44f4a99..08a1f5c 100644 ---- a/src/mpi/main.c -+++ b/src/mpi/main.c -@@ -83,6 +83,8 @@ int main(int argc, char **argv) - conf.rows, conf.cols, conf.rows/nranks, totalElements, totalElements/nranks, - conf.rbs, conf.cbs, nranks, threads, conf.timesteps, end-start, throughput); - printf("time %e\n", end - start); -+ printf("start_time %.9f\n", start); -+ printf("end_time %.9f\n", end); - } - - if (conf.generateImage) { diff --git a/garlic/apps/hpccg/default.nix b/garlic/apps/hpccg/default.nix deleted file mode 100644 index bb827a7..0000000 --- a/garlic/apps/hpccg/default.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - stdenv -, lib -, icc -, mpi ? null -, tampi ? null -, mcxx ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpccg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpccg"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/test_HPCCG-mpi.exe"; - - buildInputs = [ - icc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - # The hpccg app fails to compile in parallel. Makefile must be fixed before. - enableParallelBuilding = false; - - makeFlags = [ - "USE_MPI=-DUSING_MPI" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp test_HPCCG-mpi.exe $out/bin - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpccg/git-table.nix b/garlic/apps/hpccg/git-table.nix deleted file mode 100644 index 507dab2..0000000 --- a/garlic/apps/hpccg/git-table.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpccg.git - - "garlic/mpi+isend+omp+fork" = "c84af0480d231961201f2904ee4e3fced9d5f9be"; - "garlic/mpi+isend+seq" = "d1b47cd459440700de1b68233ec4fe794343dbd4"; - "garlic/tampi+isend+oss+task" = "7238e9be2e4a7b028abc05d40b476462eaa3de6a"; - "garlic/tampi+isend+oss+taskfor" = "02ec60f43b8d68d74575ea0563a9029fd441f1f1"; -} diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix deleted file mode 100644 index 245012e..0000000 --- a/garlic/apps/hpcg/default.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ - stdenv -, cc -, nanos6 -, mcxx -, mpi -, tampi -, gitBranch ? "garlic/seq" -, gitCommit ? null -, garlicTools -}: - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "hpcg"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "hpcg"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ - cc nanos6 mcxx mpi tampi - ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp bin/* $out/bin/ - ''; - - programPath = "/bin/xhpcg"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/hpcg/git-table.nix b/garlic/apps/hpcg/git-table.nix deleted file mode 100644 index 0aa1fc8..0000000 --- a/garlic/apps/hpcg/git-table.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/hpcg.git - - "garlic/mpi" = "8c94ccfd97518ed947bd6be3386260b72fdcdff2"; - "garlic/mpi+SAVEMAT" = "5dd2ad9eba13dba67086f46c6e8519804d837383"; - "garlic/mpi+omp" = "d24c372dd9fda584e711efb612f172e5c3602804"; - "garlic/mpi+oss" = "519a867bb3a3e07440df05e60a62abad764524e5"; - "garlic/mpi+send+omp+fork" = "a08d31aedbc108e1c0081cdc5021827ac9022688"; - "garlic/omp" = "dcc8a40831cda884b9240af47e883ac997150ed3"; - "garlic/omp+SAVEMAT" = "40dbac86c905e192ecc8146e0e65e4c3a3c6dbf8"; - "garlic/omp+fork" = "042752b3dbcd9b0f4db524b6cdc911278ee1a51b"; - "garlic/omp+initsplit" = "5370e7ee26fb72ef100a79624f73ed2baa6bcc79"; - "garlic/oss" = "7e6e2d969b7904572f2475bf471e637651337761"; - "garlic/oss+task" = "034940756ccab88876609c3cba4dea0a0f5c944b"; - "garlic/seq" = "dee225571ab2572d7aa51df9846b01237ee941a1"; - "garlic/tampi+isend+oss+task" = "449a3980a767f91ca65d429490080961dcfba498"; -} diff --git a/garlic/apps/hpcg/tampi.patch b/garlic/apps/hpcg/tampi.patch deleted file mode 100644 index c233dfc..0000000 --- a/garlic/apps/hpcg/tampi.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- a/setup/Make.MPI_ICPC_OSS 2020-07-13 16:02:33.272257865 +0200 -+++ b/setup/Make.MPI_ICPC_OSS 2020-07-13 16:04:34.344413390 +0200 -@@ -91,8 +91,8 @@ - # - HPCG includes / libraries / specifics ------------------------------- - # ---------------------------------------------------------------------- - # --HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I{TAMPI_HOME}/include --HPCG_LIBS = ${TAMPI_HOME}/lib/libtampi.a -+HPCG_INCLUDES = -I$(INCdir) -I$(INCdir)/$(arch) $(MPinc) -I$(TAMPI_HOME)/include -+HPCG_LIBS = -l:libtampi.a - # - # - Compile time options ----------------------------------------------- - # diff --git a/garlic/apps/ifsker/default.nix b/garlic/apps/ifsker/default.nix deleted file mode 100644 index b575364..0000000 --- a/garlic/apps/ifsker/default.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, mpi -, gfortran -, tampi -, nanos6 -, mcxx -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "ifsker"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "ifsker"; - - inherit (gitSource) src gitBranch gitCommit; - - buildInputs = [ tampi mpi nanos6 mcxx gfortran ]; - - # Mercurium seems to fail when building with fortran in parallel - enableParallelBuilding = false; - - # FIXME: Patch mcxx to use other directory than $HOME for the lock - # files. - preConfigure = '' - export TAMPI_HOME=${tampi} - - # $HOME is required for the lock files by mcxx to compile fortran. - # So we use the $TMPDIR to store them. - export HOME=$TMPDIR - ''; - - makeFlags = [ - "-f" "Makefile.gcc" - ]; - - - installPhase = '' - mkdir -p $out/bin - cp ${name} $out/bin/ - - mkdir -p $out/etc - cp -r data $out/etc/ - cp nanos6.toml $out/etc - ''; - - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/ifsker/git-table.nix b/garlic/apps/ifsker/git-table.nix deleted file mode 100644 index 0242708..0000000 --- a/garlic/apps/ifsker/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/ifsker.git - - "garlic/mpi+isend+oss+fork" = "a6a20303101cb140571ddc1166e66843fbe83017"; - "garlic/mpi+isend+oss+task" = "a7bdb6c9b57aafbc50cdc790eb42e5bdd850f213"; - "garlic/mpi+isend+seq" = "bc97cf30835bbf6a825209485bb96fc8314e5bcb"; - "garlic/tampi+isend+oss+task" = "33408215bc231b70b60733fddea3d1b7431bb0d1"; - "garlic/tampi+send+oss+task" = "19dae956b4ef69916c0e8ad15bb6ced0085275cd"; -} diff --git a/garlic/apps/index.nix b/garlic/apps/index.nix deleted file mode 100644 index c0e2d4d..0000000 --- a/garlic/apps/index.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = callPackage ./nbody/default.nix { }; - - saiph = callPackage ./saiph/default.nix { - cc = bsc.clangOmpss2; - L3SizeKB = garlic.targetMachine.config.hw.cacheSizeKB.L3; - cachelineBytes = garlic.targetMachine.config.hw.cachelineBytes; - }; - - creams = callPackage ./creams/default.nix { - gnuDef = self.gfortran10 ; # Default GNU compiler version - intelDef = bsc.icc ; # Default Intel compiler version - }; - - creamsInput = callPackage ./creams/input.nix { }; - - hpcg = callPackage ./hpcg/default.nix { }; - - bigsort = { - sort = callPackage ./bigsort/default.nix { - gitBranch = "garlic/mpi+send+omp+task"; - }; - - genseq = callPackage ./bigsort/genseq.nix { }; - - shuffle = callPackage ./bigsort/shuffle.nix { }; - }; - - heat = callPackage ./heat/default.nix { }; - - miniamr = callPackage ./miniamr/default.nix { - variant = "ompss-2"; - }; - - ifsker = callPackage ./ifsker/default.nix { }; - - lulesh = callPackage ./lulesh/default.nix { }; - - hpccg = callPackage ./hpccg/default.nix { }; - - fwi = rec { - params = callPackage ./fwi/params.nix { }; - solver = callPackage ./fwi/default.nix { - fwiParams = params; - }; - }; -} diff --git a/garlic/apps/lulesh/default.nix b/garlic/apps/lulesh/default.nix deleted file mode 100644 index 8f64682..0000000 --- a/garlic/apps/lulesh/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, impi -, mcxx -, icc -, tampi ? null -, gitBranch ? "garlic/mpi+isend+seq" -, gitCommit ? null -, garlicTools -}: - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "lulesh"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "lulesh"; - - inherit (gitSource) src gitBranch gitCommit; - - dontConfigure = true; - - preBuild = optionalString (tampi != null) "export TAMPI_HOME=${tampi}"; - - #TODO: Allow multiple MPI implementations and compilers - buildInputs = [ - impi - icc - mcxx - ]; - - enableParallelBuilding = true; - - #TODO: Can we build an executable named "lulesh" in all branches? - installPhase = '' - mkdir -p $out/bin - find . -name 'lulesh*' -type f -executable -exec cp \{\} $out/bin/${name} \; - ''; - programPath = "/bin/${name}"; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/lulesh/git-table.nix b/garlic/apps/lulesh/git-table.nix deleted file mode 100644 index 49c2bba..0000000 --- a/garlic/apps/lulesh/git-table.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/lulesh.git - - "garlic/mpi+isend+omp+fork" = "6cc85c55cb4840d6cde12bb285f5ab1ae7878618"; - "garlic/mpi+isend+oss+task" = "0a9e2cd1d64ab4fcf1860ace02866278ad289637"; - "garlic/mpi+isend+seq" = "9df5475c7dd2b345559fae5bd07ceea38f2e7b91"; - "garlic/tampi+isend+oss+task" = "28ce0cd69f9b4e65eff8141ec455d5f60e9b98b3"; - "garlic/tampi+isend+oss+taskfor" = "928f315ea426585a32231d950da651399e48d762"; - "garlic/tampi+isend+oss+taskloop" = "7957c1a2c84ae80edddcec9eafe7efdeefa68d58"; - "garlic/tampi+isend+oss+taskloopfor" = "7efa0535130a6726f5a46669cf171412d21adc9b"; -} diff --git a/garlic/apps/miniamr/default.nix b/garlic/apps/miniamr/default.nix deleted file mode 100644 index a014a73..0000000 --- a/garlic/apps/miniamr/default.nix +++ /dev/null @@ -1,48 +0,0 @@ -{ - stdenv -, lib -, tampi -, clangOmpss2 -, mpi -, nanos6 -, mcxx -, variant -}: - -with lib; - -assert (assertOneOf "variant" variant [ "openmp" "openmp-tasks" "ompss-2" ]); - -let - cc=mcxx; -in -stdenv.mkDerivation rec { - name = "miniamr"; - - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/ksala/miniamr.git"; - ref = "master"; - }; - - postUnpack = '' - sourceRoot=$sourceRoot/${variant} - ''; - - buildInputs = [ tampi clangOmpss2 mpi nanos6 mcxx ]; - - makeFlags = [ - "CC=${cc.CC}" - "CXX=${cc.CXX}" - ]; - - enableParallelBuilding = true; - - installPhase = '' - mkdir -p $out/bin - cp miniAMR.x $out/bin/ - ''; - - programPath = "/bin/miniAMR.x"; - - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/nbody/default.nix b/garlic/apps/nbody/default.nix deleted file mode 100644 index d5de8ce..0000000 --- a/garlic/apps/nbody/default.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, cc -, mpi ? null -, tampi ? null -, mcxx ? null -, cflags ? null -, gitBranch ? "garlic/seq" -, gitCommit ? null -, blocksize ? 2048 -, garlicTools -}: - -assert !(tampi != null && mcxx == null); - -with lib; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "nbody"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "nbody"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/nbody"; - - buildInputs = [ - cc - ] - ++ optional (mpi != null) mpi - ++ optional (tampi != null) tampi - ++ optional (mcxx != null) mcxx; - - preBuild = (if cflags != null then '' - makeFlagsArray+=(CFLAGS="${cflags}") - '' else ""); - - makeFlags = [ - "CC=${cc.CC}" - "BS=${toString blocksize}" - ] - ++ optional (tampi != null) "TAMPI_HOME=${tampi}"; - - dontPatchShebangs = true; - - installPhase = '' - echo ${tampi} - mkdir -p $out/bin - cp nbody* $out/bin/${name} - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/nbody/git-table.nix b/garlic/apps/nbody/git-table.nix deleted file mode 100644 index e824251..0000000 --- a/garlic/apps/nbody/git-table.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-20 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/nbody.git - - "garlic/mpi+send+oss+task" = "20aa856baa3268d99262588807911ad0b3318d09"; - "garlic/mpi+send+seq" = "3be64af0f949db5fd60fcd0334cf2cd8c9fa25c3"; - "garlic/omp+fork" = "9c869272df7c775f467a2220211a414e33321c00"; - "garlic/oss+task" = "13ab26fbad8662a1052cc94410386080bbf6a2ba"; - "garlic/seq" = "9dfea29189d14477bd75e6f741f0518e7e4e5e72"; - "garlic/seq+BLOCK" = "99408705628b374df4308dcf1cdbe2d21d1451c2"; - "garlic/tampi+isend+oss+task" = "653d26e4a0913d36ea18d4e72e65a04838bb138a"; - "garlic/tampi+send+oss+task" = "b1440ebc5f79165e5dfaa6a4ce7916eda410ec9a"; -} diff --git a/garlic/apps/ppong/default.nix b/garlic/apps/ppong/default.nix deleted file mode 100644 index 09a72a4..0000000 --- a/garlic/apps/ppong/default.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "ppong"; - - src = fetchurl { - url = "http://www.csl.mtu.edu/cs4331/common/PPong.c"; - sha256 = "0d1w72gq9627448cb7ykknhgp2wszwd117dlbalbrpf7d0la8yc0"; - }; - - unpackCmd = '' - mkdir src - cp $src src/ppong.c - ''; - - dontConfigure = true; - - buildPhase = '' - echo mpicc -include stdlib.h ppong.c -o ppong - mpicc -include stdlib.h ppong.c -o ppong - ''; - - installPhase = '' - mkdir -p $out/bin - cp ppong $out/bin/ppong - ln -s $out/bin/ppong $out/bin/run - ''; - - buildInputs = [ mpi ]; - hardeningDisable = [ "all" ]; -} diff --git a/garlic/apps/saiph/default.nix b/garlic/apps/saiph/default.nix deleted file mode 100644 index bb81c89..0000000 --- a/garlic/apps/saiph/default.nix +++ /dev/null @@ -1,106 +0,0 @@ -{ - stdenv -, lib -, nanos6 -, mpi -, tampi -, cc -, vtk -, boost -, gitBranch ? "master" -, gitCommit ? null -, enableManualDist ? false -, nbgx ? null -, nbgy ? null -, nbgz ? null -, nblx ? null -, nbly ? null -, nblz ? null -, nsteps ? null -, numComm ? null -, enableVectFlags ? false -, enableDebugFlags ? false -, enableAsanFlags ? false -, cachelineBytes ? null -, L3SizeKB ? null -# Problem size: -, sizex ? 3 -, sizey ? 4 -, sizez ? 4 -, garlicTools -}: - -assert enableManualDist -> (nbgx != null); -assert enableManualDist -> (nbgy != null); -assert enableManualDist -> (nbgz != null); - -with lib; -with lib.versions; - -let - gitSource = garlicTools.fetchGarlicApp { - appName = "saiph"; - inherit gitCommit gitBranch; - gitTable = import ./git-table.nix; - }; -in - stdenv.mkDerivation rec { - name = "saiph"; - - inherit (gitSource) src gitBranch gitCommit; - - programPath = "/bin/Heat3D_vect"; - - enableParallelBuilding = true; - dontStrip = true; - enableDebugging = true; - - buildInputs = [ - nanos6 - mpi - tampi - cc - vtk - boost - ]; - - preBuild = '' - cd saiphv2/cpp/src - export VTK_VERSION=${majorMinor (getVersion vtk.name)} - export VTK_HOME=${vtk} - make clean - - sed -i '/SIZEX =/s/3/${toString sizex}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEY =/s/4/${toString sizey}/g' testApp/Heat3D_vect.cpp - sed -i '/SIZEZ =/s/4/${toString sizez}/g' testApp/Heat3D_vect.cpp - ''; - - makeFlags = [ - "-f" "Makefile.${cc.CC}" - "apps" - "APP=Heat3D_vect" - ] ++ optional (cachelineBytes != null) "ROW_ALIGNMENT=${toString cachelineBytes}" - ++ optional (L3SizeKB != null) "L3_SIZE_K=${toString L3SizeKB}" - ++ optional (enableManualDist) "DIST_SET=1" - ++ optional (enableManualDist) "NBG_X=${toString nbgx}" - ++ optional (enableManualDist) "NBG_Y=${toString nbgy}" - ++ optional (enableManualDist) "NBG_Z=${toString nbgz}" - ++ optional (nblx != null) "NBL_X=${toString nblx}" - ++ optional (nbly != null) "NBL_Y=${toString nbly}" - ++ optional (nblz != null) "NBL_Z=${toString nblz}" - ++ optional (nsteps != null) "NSTEPS=${toString nsteps}" - ++ optional (numComm != null) "NUM_COMM=${toString numComm}" - ++ optional (enableVectFlags) "VECT_CHECKS=1" - ++ optional (enableDebugFlags) "DEBUG_CHECKS=1" - ++ optional (enableAsanFlags) "SANITIZE_CHECKS=1" - ; - - installPhase = '' - mkdir -p $out/lib - mkdir -p $out/bin - cp obj/libsaiphv2.so $out/lib/ - cp bin/Heat3D_vect $out/bin/ - ''; - - hardeningDisable = [ "all" ]; - } diff --git a/garlic/apps/saiph/git-table.nix b/garlic/apps/saiph/git-table.nix deleted file mode 100644 index 13cc477..0000000 --- a/garlic/apps/saiph/git-table.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ - # Auto-generated with garlic-git-table on 2021-04-19 for repo: - # ssh://git@bscpm03.bsc.es/garlic/apps/saiph.git - - "garlic/mpi+isend+omp+fork+simd" = "96823846b327b6860f05d428f0cd5ed8ca537a0e"; - "garlic/mpi+isend+omp+task+simd" = "de0346a559120f561bff554aa86b34d01214b714"; - "garlic/mpi+isend+seq+simd" = "1411dad765231f5d3cec9f621526583974232d42"; - "garlic/tampi+isend+omp+task+simd" = "587a7651df8eb69cae4a79bdfc5cb7f50723f3ce"; - "garlic/tampi+isend+oss+task+simd" = "3731197d3e35df248fa6bdb7e4cb05c5dd4f2597"; -} diff --git a/garlic/bundleReport.nix b/garlic/bundleReport.nix deleted file mode 100644 index fd84775..0000000 --- a/garlic/bundleReport.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ - stdenv -, fig -}: - -stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildPhase = '' - pwd - ls -l - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - for line in $(cat list); do - localPath=$(echo $line | tr '.' '/') - echo "mkdir -p $localPath" >> fun.nix - echo "cp -r \''${$line}/* $localPath" >> fun.nix - echo "sed -i 's;@$line@;$localPath;g' report.tex" >> fun.nix - done - echo "'''" >> fun.nix - - echo " ---------- this is the fun.nix -------------" - cat fun.nix - echo " --------------------------------------------" - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/default.nix b/garlic/default.nix deleted file mode 100644 index af0a194..0000000 --- a/garlic/default.nix +++ /dev/null @@ -1 +0,0 @@ -import ../default.nix diff --git a/garlic/develop/default.nix b/garlic/develop/default.nix deleted file mode 100644 index 6ea0930..0000000 --- a/garlic/develop/default.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, bash -, bashInteractive -, busybox -, extraInputs ? [] -}: - -let - inputrc = ./inputrc; -in -stdenv.mkDerivation { - name = "develop"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = extraInputs ++ [ busybox ]; - installPhase = '' - cat > $out < $@ - -%.html: %.ms Makefile - mkdir -p img - REFER=ref.i groff -ms -mwww $(HTML_OPT) $< > $@ - echo $(HTML_OPT) - sed -i '/<\/head>/i' $@ - sed -i 's/^<\/a>/\§<\/a>/g' $@ - #sed -i '/

/s/^ $@ - -killall -HUP mupdf - -%.utf8: %.ms - REFER=ref.i groff -ms -mwww $(PREPROC) $(TTYOPT) -Tutf8 $^ > $@ - -%.ascii: %.ms - REFER=ref.i groff -ms -mwww -c $(PREPROC) $(TTYOPT) -Tascii $^ > $@ - -doc.tar.gz: ug.pdf ug.html s.css - tar czf $@ $^ img s.css - -clean: - rm -rf img ug.pdf ug.html doc.tar.gz diff --git a/garlic/doc/blackbox.ms b/garlic/doc/blackbox.ms deleted file mode 100644 index 313a0f5..0000000 --- a/garlic/doc/blackbox.ms +++ /dev/null @@ -1,40 +0,0 @@ -.\" Use helvetica family -.fam H -.PS -moveht=0.1 -boxwid=1 -sht=boxht + 0.1 -hspace = boxwid + 0.2 -right -G: [ - boxwid=1 - extrawid=1.8 - right - A: box "nix"; arrow; - B1: box wid extrawid "App source code" "PM branch, defines..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B2: box wid extrawid "App run config" "Input size, algorithm..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B3: box wid extrawid "Build config" "MPI impl, O3, CC version..."; - move to (A.sw.x, A.y - boxht - moveht) - A: box "garlic/nix"; arrow; - B4: box wid extrawid "Run config" "Nodes, affinity"; - move to (A.sw.x, A.y - boxht - moveht) - A: box "MN4"; arrow; - B5: box wid extrawid "Hardware" "Cache size, intercomm..."; -] -movewid=1 -move -circlerad=0.4 -E: circle "Execution" -arrow -box "Result" -rspline = 0.5 -arrow from G.B1.e to E chop 0 chop circlerad -arrow from G.B2.e to E chop 0 chop circlerad -arrow from G.B3.e to E chop 0 chop circlerad -arrow from G.B4.e to E chop 0 chop circlerad -arrow from G.B5.e to E chop 0 chop circlerad -.PE diff --git a/garlic/doc/branch.ms b/garlic/doc/branch.ms deleted file mode 100644 index b4e6216..0000000 --- a/garlic/doc/branch.ms +++ /dev/null @@ -1,22 +0,0 @@ -.\".fam CMU -.\".TL -.\"Garlic: git branch name -.\".AU -.\"Rodrigo Arias Mallo -.\".AI -.\"Barcelona Supercomputing Center -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -\".NH 1 -\"Instructions -\".LP -\"To name a branch of a program, please follow the flowchart below. -.ps -2 -.PS 5/25.4 -copy "gitbranch.pic" -.PE diff --git a/garlic/doc/execution.ms b/garlic/doc/execution.ms deleted file mode 100644 index 81742d3..0000000 --- a/garlic/doc/execution.ms +++ /dev/null @@ -1,254 +0,0 @@ -.TL -Garlic: the execution pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the execution of experiments in the Garlic -benchmark, which are performed under strict conditions. The several -stages of the execution are documented so the experimenter can have a -global overview of how the benchmark runs under the hood. -The results of the experiment are stored in a known path to be used in -posterior processing steps. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -\".2C -.\"##################################################################### -.NH 1 -Introduction -.LP -Every experiment in the Garlic -benchmark is controlled by a single -.I nix -file placed in the -.CW garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 1 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CW /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CW /usr -or -.CW /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 1 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.NH 2 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CW /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.NH 2 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 2 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 2 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.KF -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Target Safe Copies User Std -_ -trebuchet xeon no no yes yes -runexp login no no yes yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate comp no no no yes -control comp yes no no yes -srun comp yes no no yes -isolate comp no yes no yes -_ -exec comp yes yes no no -program comp yes yes no no -_ -.TE -.QS -.B "Table 1" : -The stages of a complete execution pipeline. The -.B target -column determines where the stage is running, -.B safe -states if the stage begins the execution inside the sandbox, -.B user -if it can be executed directly by the user, -.B copies -if there are several instances running in parallel and -.B std -if is part of the standard execution pipeline. -.QE -.KE diff --git a/garlic/doc/gitbranch.pic b/garlic/doc/gitbranch.pic deleted file mode 100644 index 939e926..0000000 --- a/garlic/doc/gitbranch.pic +++ /dev/null @@ -1,152 +0,0 @@ -#.ps -3 -#.fam CMU -#.PS 4.5/25.4 # Scale drawing to 20/25.4 in = -# = 20/25.4[in]/25.4[mm/in] = 20 mm -# FLOWCHART - Basic flow chart blocks. -scale=25.4 #Scale units from inches to mm -csize=2.0 #Cell size in mm -pstricks=0 -dx=0; dy=2; -define process -{[ - box $1; -]} -# decision(): rhomboid -> if block -define decision {[ - boxwid=boxwid*1.2 - boxht=boxht*1.2 - B: box invis $1; - line from B.n to B.e to B.s to B.w to B.n; -]} - -#--- END OF MACROS --- -boxwid=30 -fillval=1 -circlerad=10 -down -START: circle "Start" -arrow -D_MPIB: decision("MPI-based?") -arrow " Yes" ljust -D_TAMPI: decision("TAMPI?") -arrow " Yes" ljust -TAMPI: process("\fB+tampi\fP") -right -move to D_TAMPI.e -arrow "No" above -D_MPI: decision("MPI?") -down -move to D_MPI.s -arrow " Yes" ljust -MPI: process("\fB+mpi\fP") -move to TAMPI.s -A_TAMPI: arrow linewid -line from MPI.s to MPI.c - (0,boxht) \ - to A_TAMPI.c -circle at A_TAMPI.c rad 0.7 filled -move at A_TAMPI -D_ISEND: decision("MPI_Isend()?") -arrow " Yes" ljust -ISEND: process("\fB+isend\fP") -A_ISEND: arrow -right -move to D_ISEND.e -arrow "No" above -D_SEND: decision("MPI_Send()?") -down -move to D_SEND.s -arrow " Yes" ljust -SEND: process("\fB+send\fP") -right -move to D_SEND.e -arrow "No" above -D_RMA: decision("MPI_Get()?") -down -move to D_RMA.s -arrow " Yes" ljust -RMA: process("\fB+rma\fP") -line "No" above from D_MPIB.w to D_MPIB.w - (boxwid,0) -line to (D_MPIB.w.x-boxwid, A_ISEND.c.y) \ - to A_ISEND.c -line from SEND.s to SEND.c - (0,boxht) \ - to A_ISEND.c -line from RMA.s to RMA.c - (0,boxht) \ - to SEND.c - (0,boxht) -circle at A_ISEND.c rad 0.7 filled -move at A_ISEND -D_MT: decision("multithread?") -arrow " Yes" ljust -D_OMP: decision("OpenMP?") -arrow " Yes" ljust -OMP: process("\fB+omp\fP") -right -move to D_OMP.e -arrow "No" above -D_OSS: decision("OmpSs-2?") -down -move to D_OSS.s -arrow " Yes" ljust -OSS: process("\fB+oss\fP") -down -move to OMP.s -A_OMP: arrow -circle at A_OMP.c rad 0.7 filled -line from OSS.s to OSS.c - (0,boxht) \ - to A_OMP.c -move to A_OMP.s -D_FJ: decision("fork-join?") -arrow " Yes" ljust -FJ: process("\fB+fork\fP") -right -move to D_FJ.e -arrow "No" above -D_TASKFOR: decision("task for?") -arrow "No" above -down -move to D_TASKFOR.s -arrow " Yes" ljust -TASKFOR: process("\fB+taskfor\fP") -right -move to D_TASKFOR.e -arrow "No" above -D_TASK: decision("task model?") -down -move to D_TASK.s -arrow " Yes" ljust -TASK: process("\fB+task\fP") -move to FJ.s -A_FJ: arrow -circle at A_FJ.c rad 0.7 filled -line from TASKFOR.s to TASKFOR.c - (0,boxht) \ - to A_FJ.c -line from TASK.s to TASK.c - (0,boxht) \ - to TASKFOR.c - (0,boxht) -left -move to OMP.c - (boxwid,0) -SEQ: process("\fB+seq\fP") -line "No" above from D_MT.w to (SEQ.x, D_MT.w.y) -arrow to SEQ.n -line from SEQ.s to (SEQ.s.x, A_FJ.c.y) to A_FJ.c -down -move to A_FJ.s -D_SIMD: decision("SIMD opt.?") -move to D_SIMD.e -right -arrow "Yes" above -SIMD: process("\fB+simd\fP") -down -move to D_SIMD.s -arrow " No" ljust -END: circle "End" -circle radius circlerad*0.9 at END -arrow from SIMD.s to (SIMD.x, END.y) to END.e - -# Error lines -ERR: circle "Error" at (TASK.x+boxwid, END.y) -circle radius circlerad*0.9 at ERR -line "No" above from D_TASK.e to (ERR.n.x,D_TASK.e.y) -line "No" above from D_OSS.e to (ERR.n.x,D_OSS.e.y) -line "No" above from D_RMA.e to (ERR.n.x,D_RMA.e.y) -line "No" above from D_MPI.e to (ERR.n.x,D_MPI.e.y) -arrow to ERR.n -#.PE diff --git a/garlic/doc/pp.ms b/garlic/doc/pp.ms deleted file mode 100644 index 700057a..0000000 --- a/garlic/doc/pp.ms +++ /dev/null @@ -1,256 +0,0 @@ -.TL -Garlic: the postprocess pipeline -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document covers the format used to store the results of the -execution of experiments and the postprocess steps used to generate a -set of figures from the results to present the data. The several stages -of the postprocess pipeline are documented to provide a general picture. -.AE -.\"##################################################################### -.nr GROWPS 3 -.nr PSINCR 1.5p -.\".nr PD 0.5m -.nr PI 2m -.\".2C -.R1 -bracket-label " [" ] ", " -accumulate -move-punctuation -.R2 -.\"##################################################################### -.NH 1 -Introduction -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 1 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 1 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.ie t \{\ -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -.\} -.el \{\ -.nf - - Prog ---> EP ---> Result ---> PP ---> Plot - -.fi -.\} -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 1 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.SH 1 -Appendix A: Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. diff --git a/garlic/doc/ref.i b/garlic/doc/ref.i deleted file mode 100644 index 1bd3e70..0000000 --- a/garlic/doc/ref.i +++ /dev/null @@ -1,9 +0,0 @@ -%A Rodrigo Arias Mallo -%D 2020 -%T Garlic: the execution pipeline - -%A Winston Chang -%T R Graphics Cookbook: Practical Recipes for Visualizing Data -%D 2020 -%I O'Reilly Media -%O 2nd edition diff --git a/garlic/doc/s.css b/garlic/doc/s.css deleted file mode 100644 index 324f1ce..0000000 --- a/garlic/doc/s.css +++ /dev/null @@ -1,19 +0,0 @@ -html { - line-height: 1.6; - margin-bottom: 50px; - padding-bottom: 80px; -} - -body { - max-width: 700px; - text-align: justify; - margin:0 auto; -} - -pre { - overflow: auto; - display: block; - padding: 3px 3px; - line-height: 1.4; - background-color: #eeeeee; -} diff --git a/garlic/doc/slides/2.mm b/garlic/doc/slides/2.mm deleted file mode 100644 index 7cb81ce..0000000 --- a/garlic/doc/slides/2.mm +++ /dev/null @@ -1,1468 +0,0 @@ -.\"usage: NS title -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "November 24, 2020" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic update -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Changelog" -Important changes since the last meeting (2020-09-23) -.BL -.LI -Execution of experiments is now \fBisolated\fP: no $HOME or /usr at run time -.LI -Added a \fBpostprocess\fP pipeline -.LI -New \fBgarlic(1)\fP helper tool (manual included) -.LI -A plot has an experiment result as \fBdependency\fP -.LI -Experiments run on demand based on article \fBfigures\fP -.LI -Fast pkg overrides (MPI) -.LE 1 -.\"================================================================== -.NS "Overview" -Dependency graph of a complete experiment that produces a figure. Each box -is a derivation and arrows represent \fBbuild dependencies\fP. -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The \fBResult\fP is not covered by nix (yet). This is what it looks like -when executed: -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -.PE -.S P P -.DE -.P -Notice dependency order is not the same as execution order. -.\"================================================================== -.NS "Building the execution pipeline (EP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" fill -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Run nix-build with the experiment name: -.P -.VERBON -xeon07$ nix-build -A exp.nbody.baseline -\&... -/nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.VERBOFF -.P -Outputs the first stage (the trebuchet). All other stages -are built as dependencies, as they are required to build the trebuchet. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: circle at B + (0,-1.3) "trebu." -arrow -circle "runexp" -arrow -circle "isolate" -arrow -circle "exp." -arrow -circle "..." -arrow -circle "exec" -arrow -P: circle "program" -line from R.sw to T.nw dashed -line from R.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ -"Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.SP 1m -.P -The stages are launched sequentially. Let see what happens in each one. -.\"================================================================== -.NS "Execution pipeline" -.2C -List of stages required to run the program of the experiment: -.BL -.S -1 -.LI -The -.B target -column determines where the stage is running. -.LI -.B Safe -states if the stage begins the execution inside the isolated namespace -.LI -.B User -if it can be executed directly by the user -.LI -.B Copies -if there are several instances running in parallel and -.LI -.B Std -if is part of the standard execution pipeline. -.LE -.S P P -.P -Sorted by the \fBexecution order\fP. -.\" Go to the next column -.NCOL -.KF -.defcolor white rgb #FFFFFF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\m[white]\(rh\m[]\ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBtrebuchet\fP: connects via ssh to the target machine and executes the -next stage there. -.P -The target machine is set to MN4, which by default uses the host -\fBmn1\fP -.P -Literally: -.P -.VERBON -ssh mn1 /path/to/next/stage -.VERBOFF -.P -You need to define the ssh config to be able to connect to mn1. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBrunexp\fP: sets a few \fCGARLIC_*\fP environment variables used by the -benchmark and changes the current directory to the \fBout\fP directory. -.P -At build time, next stages don't know these values (cyclic dependency), -so they are populated at execution time. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes -\(rh \fBrunexp\fP login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: once on the target machine, we enter an isolated -namespace to load the nix store. -.P -Notice that this and the previous stages require the \fBsh\fP shell to be -available on the target machine -.P -They are not \fBsafe\fP as we run target machine code -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexperiment\fP: runs several units sequentially. -.P -Defines the \fCGARLIC_EXPERIMENT\fP environment variable. -.P -Creates a directory for the experiment and changes the current directory -there. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes -\(rh \fBexperiment\fP login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBunit\fP: creates an index entry for the unit and the experiment. -.P -Creates a directory for the unit and changes the current directory -there. -.P -Copies the unit configuration in the \fCgarlic_config.json\fP file -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes -\(rh \fBunit\fP login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsbatch\fP: allocates resources and executes the next stage in the -first node. -.P -The execve call is performed by a SLURM daemon, so is \fBout\fP of the -isolated environment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes -\(rh \fBsbatch\fP login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enters the isolated namespace again, with the nix store. -.P -Notice that we are now running in the compute node allocated by SLURM. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBcontrol\fP: runs the next stage several times -.P -Is controlled by the \fCloops\fP attribute, which specifies the number -of runs. -.P -Creates a directory with the number of the run and enters it. -.P -Generated results are placed in this directory. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes -\(rh \fBcontrol\fP comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBsrun\fP: launches the tasks in the compute nodes and sets the -affinity. -.P -From here on, all stages are executed in parallel for each task. -.P -The srun program also forks from a SLURM daemon, exiting the -previous isolated namespace. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes -\(rh \fBsrun\fP comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBisolate\fP: enter the isolated namespace again. -.P -Now we are ready to execute the program of the experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBexec\fP: sets the environment variables and argv of the program. -.P -Additional commands can be specified in the \fCpre\fP and \fCpost\fP -attributes. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ -\(rh \fBexec\fP comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -\fBprogram\fP: the path to the program itself. -.P -This stage can be used to do some changes: -.BL -.LI -Set the mpi implementation of all dependencies. -.LI -Pass build options -.LI -Custom packages (nanos6 with jemalloc) -.LE -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes - isolate login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ - isolate comp no no no yes - control comp yes no no yes - srun comp yes no no yes - isolate comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no -\(rh \fBprogram\fP comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Execution stages" -.2C -The \fCstdexp.nix\fP file defines standard pipeline. The last two stages -are usually added to complete the pipeline: -.P -.VERBON -pipeline = stdPipeline ++ - [ exec program ]; -.VERBOFF -.P -Any stage can be modified to fit a custom experiment. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ -\(rh \fBtrebuchet\fP xeon no no yes \fByes\fP -\(rh \fBrunexp\fP login no no yes \fByes\fP -\(rh \fBisolate\fP login no no no \fByes\fP -\(rh \fBexperiment\fP login yes no no \fByes\fP -\(rh \fBunit\fP login yes no no \fByes\fP -\(rh \fBsbatch\fP login yes no no \fByes\fP - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no \fByes\fP -\(rh \fBcontrol\fP comp yes no no \fByes\fP -\(rh \fBsrun\fP comp yes no no \fByes\fP -\(rh \fBisolate\fP comp no yes no \fByes\fP - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Isolated execution" -.2C -The filesystem is \fBnow\fP isolated to prevent irreproducible -scenarios. -.P -The nix store is mounted at /nix and only some other paths are -available like: -.BL -.S -1 1m -.LI -/var/run/munge (required for SLURM) -.LI -/dev, /sys, /proc for MPI comm -.LI -/etc for hosts (FIXME) -.LI -/gpfs/projects/bsc15 to store data -.LE -.S P P -.P -Additional mounts can be requested by using the \fCextraMounts\fP -attribute. -.\" Go to the next column -.NCOL -.KF -.S 8 14p -.\".S C +0.2v -.TS -center expand; -lB lB cB cB cB cB cB -lB lB cB cB cB cB cB -r lw(5.5m) c c c c c. - _ _ _ _ _ _ - Stage Target Safe Copies User Std - _ _ _ _ _ _ - trebuchet xeon no no yes yes - runexp login no no yes yes -\(rh \fBisolate\fP login no no no yes - experiment login yes no no yes - unit login yes no no yes - sbatch login yes no no yes - _ _ _ _ _ _ -\(rh \fBisolate\fP comp no no no yes - control comp yes no no yes - srun comp yes no no yes -\(rh \fBisolate\fP comp no yes no yes - _ _ _ _ _ _ - exec comp yes yes no no - program comp yes yes no no - _ _ _ _ _ _ -.TE -.S P P -.KE -.1C -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -We cannot access MN4 from nix, as it doesn't has the SSH keys nor -network access when building derivations. -.P -The garlic(1) tool is used to run experiments and fetch the results. See -the manual for details. -.\"================================================================== -.NS "Running the EP" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To launch the EP use \fBgarlic -R\fP and provide the trebuchet path: -.P -.VERBON -.S -2 -xeon07$ garlic -Rv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -Running experiment 1qcc...9w5-experiment -sbatch: error: spank: x11.so: Plugin file not found -Submitted batch job 12719522 -\&... -xeon07$ -.S P P -.VERBOFF -.P -Once the jobs are submited, you can leave the session: it will run -in MN4 automatically at some point. - -.\"================================================================== -.NS "Execution complete" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -When the EP is complete, the generated results are stored in MN4. -.P -As stated previously, nix cannot access MN4 (yet), so we need to manually -fetch the results. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" fill -arrow -box "ResultTree" -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -To fetch the results, use \fBgarlic -F\fP: -.P -.VERBON -.S -3.5 -xeon07$ garlic -Fv /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -/mnt/garlic/bsc15557/out/1qc...9w5-experiment: checking units -3qnm6drx5y95kxrr43gnwqz8v4x641c7-unit: running 7 of 10 -awd3jzbcw0cwwvjrcrxzjvii3mgj663d-unit: completed -bqnnrwcbcixag0dfflk1zz34zidk97nf-unit: no status -\&... -/mn...w5-experiment: \f[CB]execution complete, fetching results\fP -these derivations will be built: - /nix/store/mqdr...q4z-resultTree.drv -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -Notice that if the experiments are still running, it waits for the -completion of all units first. -.\"================================================================== -.NS "Fetching the results" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -.VERBON -.S -3.5 -\&... -\f[CB]/nix/store/jql41hms1dr49ipbjcw41i4dj4pq2cb0-resultTree\fP -.S P P -.VERBOFF -.P -When the fetch operation success, the \fBresultTree\fP derivation is -built, with the \fBlogs\fP of the execution. -.P -All other generated data is \fBignored by now\fP, as we don't want to -store large files in the nix store of xeon07. -.\"================================================================== -.NS "Running and fetching" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" fill -arrow -box "Result" "(MN4)" dashed fill -arrow -circle "Fetch" fill -arrow -box "ResultTree" fill -arrow -circle "Build PP" -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -You can run an experiment and fetch the results with \fBgarlic -RF\fP in -one go: -.P -.VERBON -.S -2 -xeon07$ garlic -RF /nix/store/5zhmdzi5mf0mfsran74cxngn07ba522m-trebuchet -.S P P -.VERBOFF -.P -Remember that you can interrupt the fetching while is waiting, and come -later if the experiment takes too long. -.P -If nix tries to build \fBResultTree\fP and doesn't find the experiment -results, it will tell you to run this command to run and fetch the -experiment. Example: building the figure before running the experiment: -.P -.VERBON -.S -2 -xeon07$ nix-build -A fig.nbody.baseline -.S P P -.VERBOFF -.\"================================================================== -.NS "Postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -R: circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -box "ResultTree" -arrow -circle "Build PP" fill -arrow -F: box "Figure" -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ -"Order or execution" above -.PE -.S P P -.DE -.P -Once the \fBresultTree\fP derivation is built, multiple figures can be created -without re-running the experiment. -.P -The postprocess pipeline is formed of several stages as well, but is -considered \fBexperimental\fP; there is no standard yet. -.P -It only needs to be built, as nix can perform all tasks to create the -figures (no manual intervention) -.\"================================================================== -.NS "Building the postprocess pipeline (PP)" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" fill -arrow -F: box "Figure" -circlerad=0.2; -linewid=0.3; -T: box at R + (-0.02,-0.8) "timetable" -arrow -box "merge" -arrow -P: box "rPlot" -line from PP.sw to T.n dashed -line from PP.se to P.n dashed -arrow <-> from T.w - (0, 0.35) to P.e - (0, 0.35) \ - "Execution pipeline stages" below -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -To build the figure, only three stages are required: timetable, merge -and rPlot. -.\"================================================================== -.NS "PP stages: timetable" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" fill -arrow -box "merge" -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The timetable transforms the logs of the execution into a NDJSON file, -which contains all the unit configuration and the execution time in one -line in JSON: -.P -.VERBON -.S -2 -{ "unit":"...", "experiment":"...", "run":1, "config":{...}, "time":1.2345 } -{ "unit":"...", "experiment":"...", "run":2, "config":{...}, "time":1.2333 } -{ "unit":"...", "experiment":"...", "run":3, "config":{...}, "time":1.2323 } -.S P P -.VERBOFF -.P -This format allows R (and possibly other programs) to load \fBall\fP -information regarding the experiment configuration into a table. -.P -It requires the execution logs to contain a line with the time: -.P -.VERBON -.S -2 -time 1.2345 -.S P P -.VERBOFF -.\"================================================================== -.NS "PP stages: merge" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" fill -arrow -P: box "rPlot" -.PE -.S P P -.DE -.P -The merge stage allows multiple results of several experiments to be -merged in one dataset. -.P -In this way, multiple results can be presented in one figure. -.P -It simple concatenates all the NDJSON files together. -.P -This stage can be build directly with: -.P -.VERBON -$ nix-build ds.nbody.baseline -.VERBOFF -.P -So you can inspect the dataset and play with it before generating the -plots (is automatically built by nix as a dependency). -.\"================================================================== -.NS "PP stages: rPlot" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -box "timetable" -arrow -box "merge" -arrow -P: box "rPlot" fill -.PE -.S P P -.DE -.P -Finally, the rPlot stage runs a R script that loads the NDJSON dataset -and generates some plots. -.\"================================================================== -.NS "Building the figures" -.DS CB -.S -3.5 -.PS -circlerad=0.25; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -B: circle "Build EP" -arrow -circle "Run EP" -arrow -box "Result" "(MN4)" dashed -arrow -circle "Fetch" -arrow -R: box "ResultTree" -arrow -PP: circle "Build PP" -arrow -F: box "Figure" fill -arrow from B.w + (0, 0.35) to F.e + (0, 0.35) \ - "Order or execution" above -.PE -.S P P -.DE -.P -The complete PP and the figures can be build by using: -.P -.VERBON -xeon07$ nix-build -A fig.nbody.baseline -.VERBOFF -.P -A interactive R shell can be used to play with the presentation of the -plots: -.P -.VERBON -xeon07$ nix-shell garlic/fig/dev/shell.nix -$ cp /nix/store/...-merge.json input.json -$ R -> source("garlic/fig/nbody/baseline.R") -.VERBOFF -.P -More about this later. -.\"================================================================== -.NS "Figure dependencies" -.DS CB -.S -3.5 -.PS -circlerad=0.3; -linewid=0.3; -boxwid=0.52; -boxht=0.35; -fillval=0.2; -right -P: box "Program" -arrow -box "..." -arrow -T: box "Trebuchet" -arrow -box "Result" "(MN4)" dashed -arrow -R: box "ResultTree" -arrow -box "..." -arrow -F: box "Figure" fill -arrow <-> from P.nw + (0, 0.2) to T.ne + (0, 0.2) \ -"Execution pipeline (EP)" above -arrow <-> from R.nw + (0, 0.2) to F.ne + (0, 0.2) \ -"Postprocess pipeline (PP)" above -.PE -.S P P -.DE -.P -The figure contains as dependencies all the EP, results and PP. -.P -Any change in any of the stages (or dependencies) will lead to a new -figure, \fBautomatically\fP. -.P -Figures contain the hash of the dataset in the title, so they can -be tracked. -.\"================================================================== -.NS "Article with figures" -.P -An example LaTeX document uses the name of the figures in nix: -.P -.VERBON - \\includegraphics[]{@fig.nbody.small@/scatter.png} -.VERBOFF -.P -Then, nix will extract all figure references, build them (re-running the -experiment if required) and build the report: \fC$ nix-build -garlic.report\fP -.P -We also have \fBreportTar\fP that puts the figures, LaTeX sources and -a Makefile required to build the report into a self-contained tar.gz. -.P -It can be compiled with \fBmake\fP (no nix required) so it can be sent -to a journal for further changes in the LaTeX source. -.\"================================================================== -.NS "Other changes" -.DL -.LI -We can provide the complete benchmark and BSC packages as a simple -overlay. This allows others to load their own changes on top or below our -benchmark. -.LI -We now avoid reevaluation of nixpkgs when setting the MPI -implementation (allows faster evaluations: 2 s/unit \(-> 2 s total). -.LI -Dependencies between experiments results are posible (experimental): -allows generation of a dataset + computation with dependencies. -.LE -.\"================================================================== -.NS "Questions?" -.defcolor gray rgb #bbbbbb -\m[gray] -.P -Example questions: -.DL -.LI -What software was used to build this presentation? -.LI -I used groff. -.LI -And the diagrams? -.LI -Same :-D -.LI -How long takes to build? -.LI -0,39s user 0,02s system 129% cpu 0,316 total -.LE -\m[] diff --git a/garlic/doc/slides/3.mm b/garlic/doc/slides/3.mm deleted file mode 100644 index 39b07d7..0000000 --- a/garlic/doc/slides/3.mm +++ /dev/null @@ -1,181 +0,0 @@ -.\"usage: NS title -.EQ -delim $$ -.EN -.de NS \" New Slide -.SK -.ev gp-top -.fam H -.vs 1.5m -.ll \\n[@ll]u -.lt \\n[@ll]u -.rs -.sp 2v -.ps +5 -\\$* -.ps -5 -.sp 1.5v -.br -.ev -.. -.\" Remove headers -.de TP -.. -.\" Bigger page number in footer -.de EOP -.fam H -.ps +2 -. ie o .tl \\*[pg*odd-footer] -. el .tl \\*[pg*even-footer] -. ds hd*format \\g[P] -. af P 0 -. ie (\\n[P]=1)&(\\n[N]=1) .tl \\*[pg*header] -. el .tl \\*[pg*footer] -. af P \\*[hd*format] -. tl ''\\*[Pg_type!\\n[@copy_type]]'' -.. -.\" Remove top and bottom margin -.VM 0 0 -.\" -.\" -.\" Set virtual page dimensions for a physical size of 16x12 cm -.PGFORM 14c 12c 1c 1 -.ND "January 14, 2021" -.\" .vs 1.5m -.S C 1.5m -.fam H -.\".PH "'cosas'''" -.COVER ms -.de cov@print-date -.DS C -.fam H -.B -\\*[cov*new-date] -.DE -.. -.TL -.ps 20 -.fam H -Garlic experiments -.AF "Barcelona Supercomputing Center" -.AU "Rodrigo Arias Mallo" -.COVEND -.PF "'''%'" -.\" Turn off justification -.SA 0 -.\".PF '''%' -.\"================================================================== -.NS "Approach 1" -This was the approach proposed for hybrids PM -.BL -.LI -Perform a granularity experiment with a \fIreasonable\fP problem size. -.LI -Take the best blocksize -.LI -Analyze strong and weak scaling with that blocksize. -.LI -Plot speedup and efficiency comparing multiple PM. -.LE 1 -The main problem is that it may lead to \fBbogus comparisons\fP. -Additionally, there is no guarantee that the best blocksize is the one -that performs better with more resources. -.\"================================================================== -.NS "Approach 2" -We want to measure scalability of the application \fBonly\fP, not mixed -with runtime overhead or lack of parallelism. -.P -We define \fBsaturation\fP as the state of an execution that allows a -program to potentially use all the resources (the name comes from the -transistor state, when current flows freely). -.P -Design a new experiment which tests multiple blocksizes and multiple -input sizes to find these states: \fBthe saturation experiment\fP. -.P -Begin with small problems and increase the size, so you get to the -answer quickly. -.\"================================================================== -.NS "Saturation experiment" -.2C -\X'pdf: pdfpic sat.png.tk.pdf -R 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -The objetive is to find the minimum input size that allows us to get -meaningful scalability results. -.LI -More precisely, a unit is in \fBsaturation state\fP if the median time -is below the \fBsaturation time limit\fP, currently set to 110% the minimum -median time (red dashed lines). -.LI -An input size is in \fBsaturation zone\fP if it allows at least K=3 -consecutive points in the saturation state. -.LI -With less than 512 particles/CPU (green line) we cannot be sure that the -performance is not impacted by the runtime overhead or lack of -parallelism. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space" -.2C -\X'pdf: pdfpic scaling-region.svg.tk.pdf -L 7c' -.NCOL -.S -1 -3 -.BL 1m -.LI -\fBSaturation limit\fP: small tasks cannot be solved without overhead -from the runtime, no matter the blocksize. -.LI -Different limits for OmpSs-2 and OpenMP. -.LI -Experiment A will show the scaling of the app while in the saturation -zone. -.LI -Experiment B will show that OpenMP scales bad in the last 2 points. -.LI -Experiment C will show that at some point both OpenMP and OmpSs-2 scale -bad. -.LE -.S P P -.1C -.\"================================================================== -.NS "Experiment space: experiment C" -.2C -\X'pdf: pdfpic scalability.svg.tk.pdf -L 7c' -.NCOL -.BL 1m -.LI -The experiment C will show a difference in performance when approached -to the saturation limit. -.LI -We could say that OmpSs-2 introduces less overhead, therefore allows -better scalability. -.LE -.1C -.\"================================================================== -.NS "Reproducibility" -How easy can we get the same results? Three properties R0 < R1 < R2 (no common nomenclature yet!): -.BL 1m -.LI -R0: \fBSame\fP humans on the \fBsame\fP machine obtain the same result -.LI -R1: \fBDifferent\fP humans on the \fBsame\fP machine obtain the same result -.LI -R2: \fBDifferent\fP humans on a \fBdifferent\fP machine obtain same result -.LE -.P -Garlic provides 2 types of properties: for software and for experimental -results: -.BL 1m -.LI -Software is R2: you can get the exact same software by any one, in any -machine -.LI -Experimental results are R1: you cannot change the machine MN4 (yet) -.LE -.P -Same experimental result means that the mean of your results is in the confidence -interval of our results \fBand the relative std is < 1%\fP. diff --git a/garlic/doc/slides/3.pdf b/garlic/doc/slides/3.pdf deleted file mode 100644 index 0bbb7dc..0000000 Binary files a/garlic/doc/slides/3.pdf and /dev/null differ diff --git a/garlic/doc/slides/Makefile b/garlic/doc/slides/Makefile deleted file mode 100644 index 5dfe6b1..0000000 --- a/garlic/doc/slides/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -GENFIG=scaling-region.svg.tk.pdf \ - sat.png.tk.pdf \ - scalability.svg.tk.pdf -GENPDF=2.pdf 3.pdf - -all: $(GENPDF) - -keep_figs: $(GENFIG) - -%.svg.pdf: %.svg Makefile - inkscape $< --export-pdf=$@ - -%.png.pdf: %.png Makefile - gm convert $< -density 30 $@ - -%.tk.pdf: %.pdf Makefile - pdftk $< output $@ - -%.pdf: %.mm $(GENFIG) - groff -Tpdf -e -t -p -P-p12c,16c -mm $< > $@ - -killall -HUP mupdf - -watch: - while [ 1 ]; do inotifywait -e modify *; make; done - -.PRECIOUS: *.svg.pdf *.tk.pdf - -clean: - rm -f $(GENFIG) $(GENPDF) diff --git a/garlic/doc/slides/overview.svg b/garlic/doc/slides/overview.svg deleted file mode 100644 index e36e047..0000000 --- a/garlic/doc/slides/overview.svg +++ /dev/null @@ -1,479 +0,0 @@ - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - Garlic overview - bscpkgs - - Isolation - Apps - Postprocess - Replicable - - - - - - - - - - - - - - - - - - Filesystem isolation ensures the appsalways use the same software libraries - Zero human intervention, all datapostprocessing scripts are included - Different team, same results onthe same system (ACM) - Repository with BSC and custompackages - nix - Backwards tracking - Every result is hashed by the softwareused to generate it (datasets, plots ...) - Reproducible builds from source usingthe nix package manager - Several benchmark programs withmultiple programming models - nixpkgs - Large repository of common packages,maintaned by the community - Reproducible - Different team, same results ona different system (ACM). - Is a complete benchmark framework(green = not mature yet) - - diff --git a/garlic/doc/slides/sat.png b/garlic/doc/slides/sat.png deleted file mode 100644 index bca6bed..0000000 Binary files a/garlic/doc/slides/sat.png and /dev/null differ diff --git a/garlic/doc/slides/scalability.svg b/garlic/doc/slides/scalability.svg deleted file mode 100644 index 19fb326..0000000 --- a/garlic/doc/slides/scalability.svg +++ /dev/null @@ -1,265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - Efficiency - Nodes: log(n) - - - Too small - OmpSs-2 scale bad - OpenMP scale bad - - - - We can explain this difference - - - diff --git a/garlic/doc/slides/scaling-region.svg b/garlic/doc/slides/scaling-region.svg deleted file mode 100644 index 3cb2857..0000000 --- a/garlic/doc/slides/scaling-region.svg +++ /dev/null @@ -1,806 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - Problemsize: log(N) - Nodes: log(n) - - Constant workper CPU - - - OpenMPsat. limit - OmpSs-2sat. limit - - Too small - - ENOMEM - - - - - - - - - - - - - - - - - - - - - - - - - - - - Strong - A - B - C - - - - - Weak - - - - - - - Saturation - OpenMP scale bad - OmpSs-2 scale bad - ENOTIME - - - Saturationzone - - - - - 1 task/CPUlimit - - - diff --git a/garlic/doc/ug.ms b/garlic/doc/ug.ms deleted file mode 100644 index 8b7c579..0000000 --- a/garlic/doc/ug.ms +++ /dev/null @@ -1,1691 +0,0 @@ -.\" Point size fails when rending html in the code blocks -.\".nr PS 11p -.nr GROWPS 3 -.nr PSINCR 2p -.fam P -.\" =================================================================== -.\" Some useful macros -.\" =================================================================== -.\" -.\" Code start (CS) and end (CE) blocks -.de CS -.DS L -\fC -.. -.de CE -\fP -.DE -.. -.\" Code inline: -.\" .CI "inline code" -.de CI -\fC\\$1\fP\\$2 -.. -.\" =================================================================== -.\" \& -.\" .sp 3c -.\" .LG -.\" .LG -.\" .LG -.\" .LG -.\" Garlic: User guide -.\" .br -.\" .NL -.\" Rodrigo Arias Mallo -.\" .br -.\" .I "Barcelona Supercomputing Center" -.\" .br -.\" \*[curdate] -.\" .sp 17c -.\" .DE -.\" .CI \*[gitcommit] -.TL -Garlic: User Guide -.AU -Rodrigo Arias Mallo -.AI -Barcelona Supercomputing Center -.AB -.LP -This document contains all the information to configure and use the -garlic benchmark. All stages from the development to the publication -are covered, as well as the introductory steps required to setup the -machines. -.DS L -.SM -\fC -Generated on \*[curdate] -Git commit: \*[gitcommit] -\fP -.DE -.AE -.\" =================================================================== -.NH 1 -Introduction -.LP -The garlic framework is designed to fulfill all the requirements of an -experimenter in all the steps up to publication. The experience gained -while using it suggests that we move along three stages despicted in the -following diagram: -.DS L -.SM -.PS 5 -linewid=1.4; -arcrad=1; -right -S: box "Source" "code" -line "Development" invis -P: box "Program" -line "Experimentation" invis -R:box "Results" -line "Data" "exploration" invis -F:box "Figures" -# Creates a "cycle" around two boxes -define cycle { - arc cw from 1/2 of the way between $1.n and $1.ne \ - to 1/2 of the way between $2.nw and $2.n ->; - arc cw from 1/2 of the way between $2.s and $2.sw \ - to 1/2 of the way between $1.se and $1.s ->; -} -cycle(S, P) -cycle(P, R) -cycle(R, F) -.PE -.DE -In the development phase the experimenter changes the source code in -order to introduce new features or fix bugs. Once the program is -considered functional, the next phase is the experimentation, where -several experiment configurations are tested to evaluate the program. It -is common that some problems are spotted during this phase, which lead -the experimenter to go back to the development phase and change the -source code. -.PP -Finally, when the experiment is considered completed, the -experimenter moves to the next phase, which envolves the exploration of -the data generated by the experiment. During this phase, it is common to -generate results in the form of plots or tables which provide a clear -insight in those quantities of interest. It is also common that after -looking at the figures, some changes in the experiment configuration -need to be introduced (or even in the source code of the program). -.PP -Therefore, the experimenter may move forward and backwards along three -phases several times. The garlic framework provides support for all the -three stages (with different degrees of madurity). -.\" =================================================================== -.NH 2 -Machines and clusters -.LP -Our current setup employs multiple machines to build and execute the -experiments. Each cluster and node has it's own name and will be -different in other clusters. Therefore, instead of using the names of -the machines we use machine classes to generalize our setup. Those -machine clases currently correspond to a physical machine each: -.IP \(bu 12p -.B Builder -(xeon07): runs the nix-daemon and performs the builds in /nix. Requires -root access to setup the -.I nix-daemon -with multiple users. -.IP \(bu -.B Target -(MareNostrum 4 compute nodes): the nodes where the experiments -are executed. It doesn't need to have /nix installed or root access. -.IP \(bu -.B Login -(MareNostrum 4 login nodes): used to allocate resources and run jobs. It -doesn't need to have /nix installed or root access. -.IP \(bu -.B Laptop -(where the keyboard is attached, can be anything): used to connect to the other machines. -No root access is required or /nix, but needs to be able to connect to -the builder. -.LP -The machines don't need to be different of each others, as one machine -can implement several classes. For example the laptop can act as the -builder too but is not recommended. Or the login machine can also -perform the builds, but is not possible yet in our setup. -.\" =================================================================== -.NH 2 -Reproducibility -.LP -An effort to facilitate the reproducibility of the experiments has been -done, with varying degrees of success. The names of the different levels -of reproducibility have not been yet standarized, so we define our own -to avoid any confusion. We define three levels of reproducibility based -on the people and the machine involved: -.IP \(bu 12p -R0: The \fIsame\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R1: \fIDifferent\fP people on the \fIsame\fP machine obtain the same result -.IP \(bu -R2: \fIDifferent\fP people on a \fIdifferent\fP machine obtain the same result -.LP -The garlic framework distinguishes two types of results: the result of -\fIbuilding a derivation\fP (usually building a binary or a library from the -sources) and the results of the \fIexecution of an experiment\fP (typically -those are the measurements performed during the execution of the program -of study). -.PP -For those two types, the meaning of -.I "same result" -is different. In the case of building a binary, we define the same -result if it is bit-by-bit identical. In the packages provided by nixos -is usually the case except some rare cases. One example is that during the build process, -a directory is listed by the order of the inodes, giving a random order -which is different between builds. These problems are tracked by the -.URL https://r13y.com/ r13y -project. About 99% of the derivations of the minimal package set achieve -the R2 property. -.PP -On the other hand, the results of the experiments are always bit-by-bit -different. So we change the definition to state that they are the same -if the conclusions that can be obtained are the same. In particular, we -assume that the results are within the confidence interval. With this -definition, all experiments are currently R1. The reproducibility level -R2 is not posible yet as the software is compiled to support only the -target machine, with an specific interconnection. -.\" =================================================================== -.bp -.NH 1 -Preliminary steps -.LP -The peculiarities of our setup require that users perform some actions -to use the garlic framework. The content of this section is only -intended for the users of our machines, but can serve as reference in -other machines. -.PP -The names of the machine classes are used in the command line prompt -instead of the actual name of the machine, to indicate that the command -needs to be executed in the stated machine class, for example: -.CS -builder% echo hi -hi -.CE -When the machine class is not important, it is ignored and only the -.CI "%" -prompt appears. -.\" =================================================================== -.NH 2 -Configure your laptop -.LP -To easily connect to the builder (xeon07) in one step, configure the SSH -client to perform a jump over the Cobi login node. The -.I ProxyJump -directive is only available in version 7.3 and upwards. Add the -following lines in the -.CI \(ti/.ssh/config -file of your laptop: -.CS -Host cobi - HostName ssflogin.bsc.es - User your-username-here - -Host xeon07 - ProxyJump cobi - HostName xeon07 - User your-username-here -.CE -You should be able to connect to the builder typing: -.CS -laptop$ ssh xeon07 -.CE -To spot any problems try with the -.CI -v -option to enable verbose output. -.\" =================================================================== -.NH 2 -Configure the builder (xeon07) -.LP -In order to use nix you would need to be able to download the sources -from Internet. Usually the download requires the ports 22, 80 and 443 -to be open for outgoing traffic. -.PP -Check that you have network access in -xeon07 provided by the environment variables \fIhttp_proxy\fP and -\fIhttps_proxy\fP. Try to fetch a webpage with curl, to ensure the proxy -is working: -.CS -xeon07$ curl x.com -x -.CE -.\" =================================================================== -.NH 3 -Create a new SSH key -.LP -There is one DSA key in your current home called "cluster" that is no -longer supported in recent SSH versions and should not be used. Before -removing it, create a new one without password protection leaving the -passphrase empty (in case that you don't have one already created) by -running: -.CS -xeon07$ ssh-keygen -Generating public/private rsa key pair. -Enter file in which to save the key (\(ti/.ssh/id_rsa): -Enter passphrase (empty for no passphrase): -Enter same passphrase again: -Your identification has been saved in \(ti/.ssh/id_rsa. -Your public key has been saved in \(ti/.ssh/id_rsa.pub. -\&... -.CE -By default it will create the public key at \f(CW\(ti/.ssh/id_rsa.pub\fP. -Then add the newly created key to the authorized keys, so you can -connect to other nodes of the Cobi cluster: -.CS -xeon07$ cat \(ti/.ssh/id_rsa.pub >> \(ti/.ssh/authorized_keys -.CE -Finally, delete the old "cluster" key: -.CS -xeon07$ rm \(ti/.ssh/cluster \(ti/.ssh/cluster.pub -.CE -And remove the section in the configuration \f(CW\(ti/.ssh/config\fP -where the key was assigned to be used in all hosts along with the -\f(CWStrictHostKeyChecking=no\fP option. Remove the following lines (if -they exist): -.CS -Host * - IdentityFile \(ti/.ssh/cluster - StrictHostKeyChecking=no -.CE -By default, the SSH client already searchs for a keypair called -\f(CW\(ti/.ssh/id_rsa\fP and \f(CW\(ti/.ssh/id_rsa.pub\fP, so there is -no need to manually specify them. -.PP -You should be able to access the login node with your new key by using: -.CS -xeon07$ ssh ssfhead -.CE -.\" =================================================================== -.NH 3 -Authorize access to the repository -.LP -The sources of BSC packages are usually downloaded directly from the PM -git server, so you must be able to access all repositories without a -password prompt. -.PP -Most repositories are open to read for logged in users, but there are -some exceptions (for example the nanos6 repository) where you must have -explicitly granted read access. -.PP -Copy the contents of your public SSH key in \f(CW\(ti/.ssh/id_rsa.pub\fP -and paste it in GitLab at -.CS -https://pm.bsc.es/gitlab/profile/keys -.CE -Finally verify the SSH connection to the server works and you get a -greeting from the GitLab server with your username: -.CS -xeon07$ ssh git@bscpm03.bsc.es -PTY allocation request failed on channel 0 -Welcome to GitLab, @rarias! -Connection to bscpm03.bsc.es closed. -.CE -Verify that you can access the nanos6 repository (otherwise you -first need to ask to be granted read access), at: -.CS -https://pm.bsc.es/gitlab/nanos6/nanos6 -.CE -Finally, you should be able to download the nanos6 git -repository without any password interaction by running: -.CS -xeon07$ git clone git@bscpm03.bsc.es:nanos6/nanos6.git -.CE -Which will create the nanos6 directory. -.\" =================================================================== -.NH 3 -Authorize access to MareNostrum 4 -.LP -You will also need to access MareNostrum 4 from the xeon07 machine, in -order to run experiments. Add the following lines to the -\f(CW\(ti/.ssh/config\fP file and set your user name: -.CS -Host mn0 mn1 mn2 - User -.CE -Then copy your SSH key to MareNostrum 4 (it will ask you for your login -password): -.CS -xeon07$ ssh-copy-id -i \(ti/.ssh/id_rsa.pub mn1 -.CE -Finally, ensure that you can connect without a password: -.CS -xeon07$ ssh mn1 -\&... -login1$ -.CE -.\" =================================================================== -.NH 3 -Clone the bscpkgs repository -.LP -Once you have Internet and you have granted access to the PM GitLab -repositories you can begin building software with nix. First ensure -that the nix binaries are available from your shell in xeon07: -.CS -xeon07$ nix --version -nix (Nix) 2.3.6 -.CE -Now you are ready to build and install packages with nix. Clone the -bscpkgs repository: -.CS -xeon07$ git clone git@bscpm03.bsc.es:rarias/bscpkgs.git -.CE -Nix looks in the current folder for a file named \f(CWdefault.nix\fP for -packages, so go to the bscpkgs directory: -.CS -xeon07$ cd bscpkgs -.CE -Now you should be able to build nanos6 (which is probably already -compiled): -.CS -xeon07$ nix-build -A bsc.nanos6 -\&... -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -The installation is placed in the nix store (with the path stated in -the last line of the build process), with the \f(CWresult\fP symbolic -link pointing to the same location: -.CS -xeon07$ readlink result -/nix/store/...2cm1ldx9smb552sf6r1-nanos6-2.4-6f10a32 -.CE -.\" ################################################################### -.NH 3 -Configure garlic -.LP -In order to launch experiments in the -.I target -machine, it is required to configure nix to allow a directory to be -available during the build process, where the results will be stored -before being copied in the nix store. Create a new -.CI garlic -directory in your -personal cache directory and copy the full path: -.CS -xeon07$ mkdir -p \(ti/.cache/garlic -xeon07$ readlink -f \(ti/.cache/garlic -/home/Computational/rarias/.cache/garlic -.CE -Then create the nix configuration directory (if it has not already been -created): -.CS -xeon07$ mkdir -p \(ti/.config/nix -.CE -And add the following line in the -.CI \(ti/.config/nix/nix.conf -file, replacing it with the path you copied before: -.CS -.SM -extra-sandbox-paths = /garlic=/home/Computational/rarias/.cache/garlic -.CE -This option creates a virtual directory called -.CI /garlic -inside the build environment, whose contents are the ones you specify at -the right hand side of the equal sign (in this case the -.CI \(ti/.cache/garlic -directory). It will be used to allow the results of the experiments to -be passed to nix from the -.I target -machine. -.\" ################################################################### -.NH 3 -Run the garlic daemon (optional) -.LP -The garlic benchmark has a daemon which can be used to -automatically launch the experiments in the -.I target -machine on demand, when they are required to build other derivations, so -they can be launched without user interaction. The daemon creates some -FIFO pipes to communicate with the build environment, and must be -running to be able to run the experiments. To execute it, go to the -.CI bscpkgs/garlic -directory and run -.CS -xeon07$ nix-shell -nix-shell$ -.CE -to enter the nix shell (or specify the path to the -.CI garlic/shell.nix -file as argument). Then, run the daemon inside the nix shell: -.CS -nix-shell$ garlicd -garlicd: Waiting for experiments ... -.CE -Notice that the daemon stays running in the foreground, waiting for -experiments. At this moment, it can only process one experiment at a -time. -.\" =================================================================== -.NH 2 -Configure the login and target (MareNostrum 4) -.LP -In order to execute the programs in MareNostrum 4, you first need load -some utilities in the PATH. Add to the end of the file -\f(CW\(ti/.bashrc\fP in MareNostrum 4 the following line: -.CS -export PATH=/gpfs/projects/bsc15/nix/bin:$PATH -.CE -Then logout and login again (our source the \f(CW\(ti/.bashrc\fP file) -and check that now you have the \f(CWnix-develop\fP command available: -.CS -login1$ which nix-develop -/gpfs/projects/bsc15/nix/bin/nix-develop -.CE -The new utilities are available both in the login nodes and in the -compute (target) nodes, as they share the file system over the network. -.\" =================================================================== -.bp -.NH 1 -Development -.LP -During the development phase, a functional program is produced by -modifying its source code. This process is generally cyclic: the -developer needs to compile, debug and correct mistakes. We want to -minimize the delay times, so the programs can be executed as soon as -needed, but under a controlled environment so that the same behavior -occurs during the experimentation phase. -.PP -In particular, we want that several developers can reproduce the -same development environment so they can debug each other programs -when reporting bugs. Therefore, the environment must be carefully -controlled to avoid non-reproducible scenarios. -.PP -The current development environment provides an isolated shell with a -clean environment, which runs in a new mount namespace where access to -the filesystem is restricted. Only the project directory and the nix -store are available (with some other exceptions), to ensure that you -cannot accidentally link with the wrong library or modify the build -process with a forgotten environment variable in the \f(CW\(ti/.bashrc\fP -file. -.\" =================================================================== -.NH 2 -Getting the development tools -.LP -To create a development -environment, first copy or download the sources of your program (not the -dependencies) in a new directory placed in the target machine -(MareNostrum\~4). -.PP -The default environment contains packages commonly used to develop -programs, listed in the \fIgarlic/index.nix\fP file: -.\" FIXME: Unify garlic.unsafeDevelop in garlic.develop, so we can -.\" specify the packages directly -.CS -develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi - # Add more bsc packages here... - ]; - ... -.CE -If you need additional packages, add them to the list, so that they -become available in the environment. Those may include any dependency -required to build your program. -.PP -Then use the build machine (xeon07) to build the -.I garlic.develop -derivation: -.CS -build% nix-build -A garlic.develop -\&... -build% grep ln result -ln -fs /gpfs/projects/.../bin/stage1 .nix-develop -.CE -Copy the \fIln\fP command and run it in the target machine -(MareNostrum\~4), inside the new directory used for your program -development, to create the link \fI.nix-develop\fP (which is used to -remember your environment). Several environments can be stored in -different directories using this method, with different packages in each -environment. You will need -to rebuild the -.I garlic.develop -derivation and update the -.I .nix-develop -link after the package list is changed. Once the -environment link is created, there is no need to repeat these steps again. -.PP -Before entering the environment, you will need to access the required -resources for your program, which may include several compute nodes. -.\" =================================================================== -.NH 2 -Allocating resources for development -.LP -Our target machine (MareNostrum 4) provides an interactive shell, that -can be requested with the number of computational resources required for -development. To do so, connect to the login node and allocate an -interactive session: -.CS -% ssh mn1 -login% salloc ... -target% -.CE -This operation may take some minutes to complete depending on the load -of the cluster. But once the session is ready, any subsequent execution -of programs will be immediate. -.\" =================================================================== -.NH 2 -Accessing the developement environment -.PP -The utility program \fInix-develop\fP has been designed to access the -development environment of the current directory, by looking for the -\fI.nix-develop\fP file. It creates a namespace where the required -packages are installed and ready to be used. Now you can access the -newly created environment by running: -.CS -target% nix-develop -develop% -.CE -The spawned shell contains all the packages pre-defined in the -\fIgarlic.develop\fP derivation, and can now be accessed by typing the -name of the commands. -.CS -develop% which gcc -/nix/store/azayfhqyg9...s8aqfmy-gcc-wrapper-9.3.0/bin/gcc -develop% which gdb -/nix/store/1c833b2y8j...pnjn2nv9d46zv44dk-gdb-9.2/bin/gdb -.CE -If you need additional packages, you can add them in the -\fIgarlic/index.nix\fP file as mentioned previously. To keep the -same current resources, so you don't need to wait again for the -resources to be allocated, exit only from the development shell: -.CS -develop% exit -target% -.CE -Then update the -.I .nix-develop -link and enter into the new develop environment: -.CS -target% nix-develop -develop% -.CE -.\" =================================================================== -.NH 2 -Execution -.LP -The allocated shell can only execute tasks in the current node, which -may be enough for some tests. To do so, you can directly run your -program as: -.CS -develop$ ./program -.CE -If you need to run a multi-node program, typically using MPI -communications, then you can do so by using srun. Notice that you need -to allocate several nodes when calling salloc previously. The srun -command will execute the given program \fBoutside\fP the development -environment if executed as-is. So we re-enter the develop environment by -calling nix-develop as a wrapper of the program: -.\" FIXME: wrap srun to reenter the develop environment by its own -.CS -develop$ srun nix-develop ./program -.CE -.\" =================================================================== -.NH 2 -Debugging -.LP -The debugger can be used to directly execute the program if is executed -in only one node by using: -.CS -develop$ gdb ./program -.CE -Or it can be attached to an already running program by using its PID. -You will need to first connect to the node running it (say target2), and -run gdb inside the nix-develop environment. Use -.I squeue -to see the compute nodes running your program: -.CS -login$ ssh target2 -target2$ cd project-develop -target2$ nix-develop -develop$ gdb -p $pid -.CE -You can repeat this step to control the execution of programs running in -different nodes simultaneously. -.PP -In those cases where the program crashes before being able to attach the -debugger, enable the generation of core dumps: -.CS -develop$ ulimit -c unlimited -.CE -And rerun the program, which will generate a core file that can be -opened by gdb and contains the state of the memory when the crash -happened. Beware that the core dump file can be very large, depending on -the memory used by your program at the crash. -.\" =================================================================== -.NH 2 -Git branch name convention -.LP -The garlic benchmark imposes a set of requirements to be meet for each -application in order to coordinate the execution of the benchmark and -the gathering process of the results. -.PP -Each application must be available in a git repository so it can be -included into the garlic benchmark. The different combinations of -programming models and communication schemes should be each placed in -one git branch, which are referred to as \fIbenchmark branches\fP. At -least one benchmark branch should exist and they all must begin with the -prefix \f(CWgarlic/\fP (other branches will be ignored). -.PP -The branch name is formed by adding keywords separated by the "+" -character. The keywords must follow the given order and can only -appear zero or once each. At least one keyword must be included. The -following keywords are available: -.IP \f(CWmpi\fP 5m -A significant fraction of the communications uses only the standard MPI -(without extensions like TAMPI). -.IP \f(CWtampi\fP -A significant fraction of the communications uses TAMPI. -.IP \f(CWsend\fP -A significant part of the MPI communication uses the blocking family of -methods -.I MPI_Send , ( -.I MPI_Recv , -.I MPI_Gather "...)." -.IP \f(CWisend\fP -A significant part of the MPI communication uses the non-blocking family -of methods -.I MPI_Isend , ( -.I MPI_Irecv , -.I MPI_Igather "...)." -.IP \f(CWrma\fP -A significant part of the MPI communication uses remote memory access -(one-sided) methods -.I MPI_Get , ( -.I MPI_Put "...)." -.IP \f(CWseq\fP -The complete execution is sequential in each process (one thread per -process). -.IP \f(CWomp\fP -A significant fraction of the execution uses the OpenMP programming -model. -.IP \f(CWoss\fP -A significant fraction of the execution uses the OmpSs-2 programming -model. -.IP \f(CWtask\fP -A significant part of the execution involves the use of the tasking -model. -.IP \f(CWtaskfor\fP -A significant part of the execution uses the taskfor construct. -.IP \f(CWfork\fP -A significant part of the execution uses the fork-join model (including -hybrid programming techniques with parallel computations and sequential -communications). -.IP \f(CWsimd\fP -A significant part of the computation has been optimized to use SIMD -instructions. -.LP -In the -.URL #appendixA "Appendix A" -there is a flowchart to help the decision -process of the branch name. Additional user defined keywords may be -added at the end using the separator "+" as well. User keywords must -consist of capital alphanumeric characters only and be kept short. These -additional keywords must be different (case insensitive) to the already -defined above. Some examples: -.CS -garlic/mpi+send+seq -garlic/mpi+send+omp+fork -garlic/mpi+isend+oss+task -garlic/tampi+isend+oss+task -garlic/tampi+isend+oss+task+COLOR -garlic/tampi+isend+oss+task+COLOR+BTREE -.CE -.\" =================================================================== -.NH 2 -Initialization time -.LP -It is common for programs to have an initialization phase prior to the -execution of the main computation task which is the objective of the study. -The initialization phase is usually not considered when taking -measurements, but the time it takes to complete can limit seriously the -amount of information that can be extracted from the computation phase. -As an example, if the computation phase is in the order of seconds, but -the initialization phase takes several minutes, the number of runs would -need to be set low, as the units could exceed the time limits. Also, the -experimenter may be reluctant to modify the experiments to test other -parameters, as the waiting time for the results is unavoidably large. -.PP -To prevent this problem the programs must reduce the time of the -initialization phase to be no larger than the computation time. To do -so, the initialization phase can be optimized either with -parallelization, or it can be modified to store the result of the -initialization to the disk to be later at the computation phase. In the -garlic framework an experiment can have a dependency over the results of -another experiment (the results of the initialization). The -initialization results will be cached if the derivation is kept -invariant, when modifying the computation phase parameters. -.\" =================================================================== -.NH 2 -Measurement of the execution time -.LP -The programs must measure the wall time of the computation phase following a -set of rules. The way in which the wall time is measured is very important to -get accurate results. The measured time must be implemented by using a -monotonic clock which is able to correct the drift of the oscillator of -the internal clock due to changes in temperature. This clock must be -measured in C and C++ with: -.CS -clock_gettime(CLOCK_MONOTONIC, &ts); -.CE -A helper function can be used the approximate value of the clock in a -double precision float, in seconds: -.CS -double get_time() -{ - struct timespec tv; - if(clock_gettime(CLOCK_MONOTONIC, &tv) != 0) - { - perror("clock_gettime failed"); - exit(EXIT_FAILURE); - } - return (double)(ts.tv_sec) + - (double)ts.tv_nsec * 1.0e-9; -} -.CE -The start and end points must be measured after the synchronization of -all the processes and threads, so the complete computation work can be -bounded to fit inside the measured interval. An example for a MPI -program: -.CS -double start, end, delta_time; -MPI_Barrier(); -start = get_time(); -run_simulation(); -MPI_Barrier(); -end = get_time(); -delta_time = end - start; -.CE -.\" =================================================================== -.NH 2 -Format of the execution time -.LP -The measured execution time must be printed to the standard output -(stdout) in scientific notation with at least 7 significative digits. -The following the printf format (or the strict equivalent in other languages) -must be used: -.CS -printf("time %e\\n", delta_time); -.CE -The line must be printed alone and only once: for MPI programs, -only one process shall print the time: -.CS -if(rank == 0) printf("time %e\\n", delta_time); -.CE -Other lines can be printed in the stdout, but without the -.I time -prefix, so that the following pipe can be used to capture the line: -.CS -% ./app | grep "^time" -1.234567e-01 -.CE -Ensure that your program follows this convention by testing it with the -above -.I grep -filter; otherwise the results will fail to be parsed when building -the dataset with the execution time. -.\" =================================================================== -.bp -.NH 1 -Experimentation -.LP -During the experimentation, a program is studied by running it and -measuring some properties. The experimenter is in charge of the -experiment design, which is typically controlled by a single -.I nix -file placed in the -.CI garlic/exp -subdirectory. -Experiments are formed by several -.I "experimental units" -or simply -.I units . -A unit is the result of each unique configuration of the experiment -(typically involves the cartesian product of all factors) and -consists of several shell scripts executed sequentially to setup the -.I "execution environment" , -which finally launch the actual program being analyzed. -The scripts that prepare the environment and the program itself are -called the -.I stages -of the execution and altogether form the -.I "execution pipeline" -or simply the -.I pipeline . -The experimenter must know with very good details all the stages -involved in the pipeline, as they have a large impact on the execution. -.PP -Additionally, the execution time is impacted by the target machine in -which the experiments run. The software used for the benchmark is -carefully configured and tuned for the hardware used in the execution; -in particular, the experiments are designed to run in MareNostrum 4 -cluster with the SLURM workload manager and the Omni-Path -interconnection network. In the future we plan to add -support for other clusters in order to execute the experiments in other -machines. -.\"##################################################################### -.NH 2 -Isolation -.LP -The benchmark is designed so that both the compilation of every software -package and the execution of the experiment is performed under strict -conditions. We can ensure that two executions of the same experiment are -actually running the same program in the same software environment. -.PP -All the software used by an experiment is included in the -.I "nix store" -which is, by convention, located at the -.CI /nix -directory. Unfortunately, it is common for libraries to try to load -software from other paths like -.CI /usr -or -.CI /lib . -It is also common that configuration files are loaded from -.CW /etc -and from the home directory of the user that runs the experiment. -Additionally, some environment variables are recognized by the libraries -used in the experiment, which change their behavior. As we cannot -control the software and configuration files in those directories, we -couldn't guarantee that the execution behaves as intended. -.PP -In order to avoid this problem, we create a -.I sandbox -where only the files in the nix store are available (with some other -exceptions). Therefore, even if the libraries try to access any path -outside the nix store, they will find that the files are not there -anymore. Additionally, the environment variables are cleared before -entering the environment (with some exceptions as well). -.\"##################################################################### -.NH 2 -Execution pipeline -.LP -Several predefined stages form the -.I standard -execution pipeline and are defined in the -.I stdPipeline -array. The standard pipeline prepares the resources and the environment -to run a program (usually in parallel) in the compute nodes. It is -divided in two main parts: -connecting to the target machine to submit a job and executing the job. -Finally, the complete execution pipeline ends by running the actual -program, which is not part of the standard pipeline, as should be -defined differently for each program. -.\"##################################################################### -.NH 3 -Job submission -.LP -Some stages are involved in the job submission: the -.I trebuchet -stage connects via -.I ssh -to the target machine and executes the next stage there. Once in the -target machine, the -.I runexp -stage computes the output path to store the experiment results, using -the user in the target machine and changes the working directory there. -In MareNostrum 4 the output path is at -.CI /gpfs/projects/bsc15/garlic/$user/out . -Then the -.I isolate -stage is executed to enter the sandbox and the -.I experiment -stage begins, which creates a directory to store the experiment output, -and launches several -.I unit -stages. -.PP -Each unit executes a -.I sbatch -stage which runs the -.I sbatch(1) -program with a job script that simply calls the next stage. The -sbatch program internally reads the -.CW /etc/slurm/slurm.conf -file from outside the sandbox, so we must explicitly allow this file to -be available, as well as the -.I munge -socket used for authentication by the SLURM daemon. Once the jobs are -submitted to SLURM, the experiment stage ends and the trebuchet finishes -the execution. The jobs will be queued for execution without any other -intervention from the user. -.PP -The rationale behind running sbatch from the sandbox is because the -options provided in environment variables override the options from the -job script. Therefore, we avoid this problem by running sbatch from the -sandbox, where the interfering environment variables are removed. The -sbatch program is also provided in the -.I "nix store" , -with a version compatible with the SLURM daemon running in the target -machine. -.\"##################################################################### -.NH 3 -Job execution -.LP -Once an unit job has been selected for execution, SLURM -allocates the resources (usually several nodes) and then selects one of -the nodes to run the job script: it is not executed in parallel yet. -The job script runs from a child process forked from on of the SLURM -daemon processes, which are outside the sandbox. Therefore, we first run the -.I isolate -stage -to enter the sandbox again. -.PP -The next stage is called -.I control -and determines if enough data has been generated by the experiment unit -or if it should continue repeating the execution. At the current time, -it is only implemented as a simple loop that runs the next stage a fixed -amount of times (by default, it is repeated 30 times). -.PP -The following stage is -.I srun -which launches several copies of the next stage to run in -parallel (when using more than one task). Runs one copy per task, -effectively creating one process per task. The CPUs affinity is -configured by the parameter -.I --cpu-bind -and is important to set it correctly (see more details in the -.I srun(1) -manual). Appending the -.I verbose -value to the cpu bind option causes srun to print the assigned affinity -of each task, which is very valuable when examining the execution log. -.PP -The mechanism by which srun executes multiple processes is the same used -by sbatch, it forks from a SLURM daemon running in the computing nodes. -Therefore, the execution begins outside the sandbox. The next stage is -.I isolate -which enters again the sandbox in every task. All remaining stages are -running now in parallel. -.\" ################################################################### -.NH 3 -The program -.LP -At this point in the execution, the standard pipeline has been -completely executed, and we are ready to run the actual program that is -the matter of the experiment. Usually, programs require some arguments -to be passed in the command line. The -.I exec -stage sets the arguments (and optionally some environment variables) and -executes the last stage, the -.I program . -.PP -The experimenters are required to define these last stages, as they -define the specific way in which the program must be executed. -Additional stages may be included before or after the program run, so -they can perform additional steps. -.\" ################################################################### -.NH 3 -Stage overview -.LP -The complete execution pipeline using the standard pipeline is shown in -the Table 1. Some properties are also reflected about the execution -stages. -.DS L -.TS -center; -lB cB cB cB cB cB -l c c c c c. -_ -Stage Where Safe Copies User Std -_ -trebuchet * no no yes yes -runexp login no no no yes -isolate login no no no yes -experiment login yes no no yes -unit login yes no no yes -sbatch login yes no no yes -_ -isolate target no no no yes -control target yes no no yes -srun target yes no no yes -isolate target no yes no yes -_ -exec target yes yes no no -program target yes yes no no -_ -.TE -.DE -.QS -.SM -.B "Table 1" : -The stages of a complete execution pipeline. The -.I where -column determines where the stage is running, -.I safe -states if the stage begins the execution inside the sandbox, -.I user -if it can be executed directly by the user, -.I copies -if there are several instances running in parallel and -.I std -if is part of the standard execution pipeline. -.QE -.\" ################################################################### -.NH 2 -Writing the experiment -.LP -The experiments are generally written in the -.I nix -language as it provides very easy management for the packages an their -customization. An experiment file is formed by several parts, which -produce the execution pipeline when built. The experiment file describes -a function (which is typical in nix) and takes as argument an -attribute set with some common packages, tools and options: -.CS -{ stdenv, lib, bsc, stdexp, targetMachine, stages, garlicTools }: -.CE -The -.I bsc -attribute contains all the BSC and nixpkgs packages, as defined in the -overlay. The -.I stdexp -contains some useful tools and functions to build the experiments, like -the standard execution pipeline, so you don't need to redefine the -stages in every experiment. The configuration of the target machine is -specified in the -.I targetMachine -attribute which includes information like the number of CPUs per node or -the cache line length. It is used to define the experiments in such a -way that they are not tailored to an specific machine hardware -(sometimes this is not posible). All the execution stages are available -in the -.I stages -attribute which are used when some extra stage is required. And finally, -the -.I garlicTools -attribute provide some functions to aid common tasks when defining the -experiment configuration -.\" ################################################################### -.NH 3 -Experiment configuration -.LP -The next step is to define some variables in a -.CI let -\&... -.CI in -\&... -.CI ; -construct, to be used later. The first one, is the variable -configuration of the experiment called -.I varConf , -which include all -the factors that will be changed. All the attributes of this set -.I must -be arrays, even if they only contain one element: -.CS -varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; -}; -.CE -In this example, the variable -.I blocks -will be set to the values 1, 2 and 4; while -.I nodes -will remain set to 1 always. These variables are used later to build the -experiment configuration. The -.I varConf -is later converted to a list of attribute sets, where every attribute -contains only one value, covering all the combinations (the Cartesian -product is computed): -.CS -[ { blocks = 1; nodes = 1; } - { blocks = 2; nodes = 1; } - { blocks = 4; nodes = 1; } ] -.CE -These configurations are then passed to the -.I genConf -function one at a time, which is the central part of the description of -the experiment: -.CS -genConf = var: fix (self: targetMachine.config // { - expName = "example"; - unitName = self.expName + "-b" + toString self.blocks; - blocks = var.blocks; - cpusPerTask = 1; - tasksPerNode = self.hw.socketsPerNode; - nodes = var.nodes; -}); -.CE -It takes as input -.I one -configuration from the Cartesian product, for example: -.CS -{ blocks = 2; nodes = 1; } -.CE -And returns the complete configuration for that input, which usually -expand the input configuration with some derived variables along with -other constant parameters. The return value can be inspected by calling -the function in the interactive -.I "nix repl" -session: -.CS -nix-repl> genConf { blocks = 2; nodes = 1; } -{ - blocks = 2; - cpusPerTask = 1; - expName = "example"; - hw = { ... }; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - name = "mn4"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - nodes = 1; - sshHost = "mn1"; - tasksPerNode = 2; - unitName = "example-b2"; -} -.CE -Some configuration parameters were added by -.I targetMachine.config , -such as the -.I nixPrefix , -.I sshHost -or the -.I hw -attribute set, which are specific for the cluster they experiment is -going to run. Also, the -.I unitName -got assigned the proper name based on the number of blocks, but the -number of tasks per node were assigned based on the hardware description -of the target machine. -.PP -By following this rule, the experiments can easily be ported to machines -with other hardware characteristics, and we only need to define the -hardware details once. Then all the experiments will be updated based on -those details. -.\" ################################################################### -.NH 3 -Adding the stages -.LP -Once the configuration is ready, it will be passed to each stage of the -execution pipeline which will take the parameters it needs. The -connection between the parameters and how they are passed to each stage -is done either by convention or manually. There is a list of parameters that -are recognized by the standard pipeline stages. For example the -attribute -.I nodes , -it is recognized as the number of nodes in the standard -.I sbatch -stage when allocating resources: -.DS L -.TS -center; -lB lB cB cB lB -l l c c l. -_ -Stage Attribute Std Req Description -_ -* nixPrefix yes yes Path to the nix store in the target -unit expName yes yes Name of the experiment -unit unitName yes yes Name of the unit -control loops yes yes Number of runs of each unit -sbatch cpusPerTask yes yes Number of CPUs per task (process) -sbatch jobName yes yes Name of the job -sbatch nodes yes yes Number of nodes allocated -sbatch ntasksPerNode yes yes Number of tasks (processes) per node -sbatch qos yes no Name of the QoS queue -sbatch reservation yes no Name of the reservation -sbatch time yes no Maximum allocated time (string) -_ -exec argv no no Array of arguments to execve -exec env no no Environment variable settings -exec pre no no Code before the execution -exec post no no Code after the execution -_ -.TE -.DE -.QS -.SM -.B "Table 2" : -The attributes recognized by the stages in the execution pipeline. The -column -.I std -indicates if they are part of the standard execution pipeline. Some -attributes are required as indicated by the -.I req -column. -.QE -.LP -Other attribute names can be used to specify custom information used in -additional stages. The two most common stages required to complete the -pipeline are the -.I exec -and the -.I program . -Let see an example of -.I exec : -.CS -exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; -}; -.CE -The -.I exec -stage is defined as a function that uses the predefined -.I stages.exec -stage, which accepts the -.I argv -array, and sets the argv of the program. In our case, we fill the -.I argv -array by setting the -.I --blocks -parameter to the number of blocks, specified in the configuration in the -attribute -.I blocks . -The name of this attribute can be freely choosen, as long as the -.I exec -stage refers to it properly. The -.I nextStage -attribute is mandatory in all stages, and is automatically set when -building the pipeline. -.PP -The last step is to configure the actual program to be executed, -which can be specified as another stage: -.CS -program = {nextStage, conf, ...}: bsc.apps.example; -.CE -Notice that this function only returns the -.I bsc.apps.example -derivation, which will be translated to the path where the example -program is installed. If the program is located inside a directory -(typically -.I bin ), -it must define the attribute -.I programPath -in the -.I bsc.apps.example -derivation, which points to the executable program. An example: -.CS -stdenv.mkDerivation { -\& ... - programPath = "/bin/example"; -\& ... -}; -.CE -.\" ################################################################### -.NH 3 -Building the pipeline -.LP -With the -.I exec -and -.I program -stages defined and the ones provided by the standard pipeline, the -complete execution pipeline can be formed. To do so, the stages are -placed in an array, in the order they will be executed: -.CS -pipeline = stdexp.stdPipeline ++ [ exec program ]; -.CE -The attribute -.I stdexp.stdPipeline -contains the standard pipeline stages, and we only append our two -defined stages -.I exec -and -.I program . -The -.I pipeline -is an array of functions, and must be transformed in something that can -be executed in the target machine. For that purpose, the -.I stdexp -provides the -.I genExperiment -function, which takes the -.I pipeline -array and the list of configurations and builds the execution pipeline: -.CS -stdexp.genExperiment { inherit configs pipeline; } -.CE -The complete example experiment can be shown here: -.CS -{ stdenv, lib, stdexp, bsc, targetMachine, stages }: -with lib; -let - # Initial variable configuration - varConf = { - blocks = [ 1 2 4 ]; - nodes = [ 1 ]; - }; - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "example"; - unitName = "${expName}-b${toString blocks}"; - inherit (targetMachine.config) hw; - inherit (c) blocks nodes; - loops = 30; - ntasksPerNode = hw.socketPerNode; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "--blocks" conf.blocks ]; - }; - program = {nextStage, conf, ...}: bsc.garlic.apps.example; - pipeline = stdexp.stdPipeline ++ [ exec program ]; -in - stdexp.genExperiment { inherit configs pipeline; } -.CE -.\" ################################################################### -.NH 3 -Adding the experiment to the index -.LP -The experiment file must be located in a named directory inside the -.I garlic/exp -directory. The name is usually the program name. Once the experiment is -placed in a nix file, it must be added to the index of experiments, so -it can be build. The index is hyerarchically organized as attribute -sets, with -.I exp -containing all the experiments; -.I exp.example -the experiments of the -.I example -program; and -.I exp.example.test1 -referring to the -.I test1 -experiment of the -.I example -program. Additional attributes can be added, like -.I exp.example.test1.variantA -to handle more details. -.PP -For this example we are going to use the attribute path -.I exp.example.test -and add it to the index, in the -.I garlic/exp/index.nix -file. We append to the end of the attribute set, the following -definition: -.CS -\&... - example = { - test = callPackage ./example/test.nix { }; - }; -} -.CE -The experiment can now be built with: -.CS -builder% nix-build -A exp.example.test -.CE -.\" ################################################################### -.NH 2 -Recommendations -.PP -The complete results generally take a long time to be finished, so it is -advisable to design the experiments iteratively, in order to quickly -obtain some feedback. Some recommendations: -.BL -.LI -Start with one unit only. -.LI -Set the number of runs low (say 5) but more than one. -.LI -Use a small problem size, so the execution time is low. -.LI -Set the time limit low, so deadlocks are caught early. -.LE -.PP -As soon as the first runs are complete, examine the results and test -that everything looks good. You would likely want to check: -.BL -.LI -The resources where assigned as intended (nodes and CPU affinity). -.LI -No errors or warnings: look at stderr and stdout logs. -.LI -If a deadlock happens, it will run out of the time limit. -.LE -.PP -As you gain confidence over that the execution went as planned, begin -increasing the problem size, the number of runs, the time limit and -lastly the number of units. The rationale is that each unit that is -shared among experiments gets assigned the same hash. Therefore, you can -iteratively add more units to an experiment, and if they are already -executed (and the results were generated) is reused. -.\" ################################################################### -.bp -.NH 1 -Post-processing -.LP -After the correct execution of an experiment the results are stored for -further investigation. Typically the time of the execution or other -quantities are measured and presented later in a figure (generally a -plot or a table). The -.I "postprocess pipeline" -consists of all the steps required to create a set of figures from the -results. Similarly to the execution pipeline where several stages run -sequentially, -.[ -garlic execution -.] -the postprocess pipeline is also formed by multiple stages executed -in order. -.PP -The rationale behind dividing execution and postprocess is -that usually the experiments are costly to run (they take a long time to -complete) while generating a figure require less time. Refining the -figures multiple times reusing the same experimental results doesn't -require the execution of the complete experiment, so the experimenter -can try multiple ways to present the data without waiting a large delay. -.NH 2 -Results -.LP -The results are generated in the same -.I "target" -machine where the experiment is executed and are stored in the garlic -\fCout\fP -directory, organized into a tree structure following the experiment -name, the unit name and the run number (governed by the -.I control -stage): -.DS L -\fC -|-- 6lp88vlj7m8hvvhpfz25p5mvvg7ycflb-experiment -| |-- 8lpmmfix52a8v7kfzkzih655awchl9f1-unit -| | |-- 1 -| | | |-- stderr.log -| | | |-- stdout.log -| | | |-- ... -| | |-- 2 -\&... -\fP -.DE -In order to provide an easier access to the results, an index is also -created by taking the -.I expName -and -.I unitName -attributes (defined in the experiment configuration) and linking them to -the appropriate experiment and unit directories. These links are -overwritten by the last experiment with the same names so they are only -valid for the last execution. The out and index directories are -placed into a per-user directory, as we cannot guarantee the complete -execution of each unit when multiple users share units. -.PP -The messages printed to -.I stdout -and -.I stderr -are stored in the log files with the same name inside each run -directory. Additional data is sometimes generated by the experiments, -and is found in each run directory. As the generated data can be very -large, is ignored by default when fetching the results. -.NH 2 -Fetching the results -.LP -Consider a program of interest for which an experiment has been designed to -measure some properties that the experimenter wants to present in a -visual plot. When the experiment is launched, the execution -pipeline (EP) is completely executed and it will generate some -results. In this escenario, the execution pipeline depends on the -program\[em]any changes in the program will cause nix to build the -pipeline again -using the updated program. The results will also depend on the -execution pipeline as well as the postprocess pipeline (PP) and the plot -on the results. This chain of dependencies can be shown in the -following dependency graph: -.PS -circlerad=0.22; -linewid=0.3; -right -circle "Prog" -arrow -circle "EP" -arrow -circle "Result" -arrow -circle "PP" -arrow -circle "Plot" -.PE -Ideally, the dependencies should be handled by nix, so it can detect any -change and rebuild the necessary parts automatically. Unfortunately, nix -is not able to build the result as a derivation directly, as it requires -access to the -.I "target" -machine with several user accounts. In order to let several users reuse -the same results from a shared cache, we would like to use the -.I "nix store" . -.PP -To generate the results from the -experiment, we add some extra steps that must be executed manually: -.PS -circle "Prog" -arrow -diag=linewid + circlerad; -far=circlerad*3 + linewid*4 -E: circle "EP" -R: circle "Result" at E + (far,0) -RUN: circle "Run" at E + (diag,-diag) dashed -FETCH: circle "Fetch" at R + (-diag,-diag) dashed -move to R.e -arrow -P: circle "PP" -arrow -circle "Plot" -arrow dashed from E to RUN chop -arrow dashed from RUN to FETCH chop -arrow dashed from FETCH to R chop -arrow from E to R chop -.PE -The run and fetch steps are provided by the helper tool -.I "garlic(1)" , -which launches the experiment using the user credentials at the -.I "target" -machine and then fetches the results, placing them in a directory known -by nix. When the result derivation needs to be built, nix will look in -this directory for the results of the execution. If the directory is not -found, a message is printed to suggest the user to launch the experiment -and the build process is stopped. When the result is successfully built -by any user, is stored in the -.I "nix store" -and it won't need to be rebuilt again until the experiment changes, as -the hash only depends on the experiment and not on the contents of the -results. -.PP -Notice that this mechanism violates the deterministic nature of the nix -store, as from a given input (the experiment) we can generate different -outputs (each result from different executions). We knowingly relaxed -this restriction by providing a guarantee that the results are -equivalent and there is no need to execute an experiment more than once. -.PP -To force the execution of an experiment you can use the -.I rev -attribute which is a number assigned to each experiment -and can be incremented to create copies that only differs on that -number. The experiment hash will change but the experiment will be the -same, as long as the revision number is ignored along the execution -stages. -.NH 2 -Postprocess stages -.LP -Once the results are completely generated in the -.I "target" -machine there are several stages required to build a set of figures: -.PP -.I fetch \[em] -waits until all the experiment units are completed and then executes the -next stage. This stage is performed by the -.I garlic(1) -tool using the -.I -F -option and also reports the current state of the execution. -.PP -.I store \[em] -copies from the -.I target -machine into the nix store all log files generated by the experiment, -keeping the same directory structure. It tracks the execution state of -each unit and only copies the results once the experiment is complete. -Other files are ignored as they are often very large and not required -for the subsequent stages. -.PP -.I timetable \[em] -converts the results of the experiment into a NDJSON file with one -line per run for each unit. Each line is a valid JSON object, containing -the -.I exp , -.I unit -and -.I run -keys and the unit configuration (as a JSON object) in the -.I config -key. The execution time is captured from the standard output and is -added in the -.I time -key. -.PP -.I merge \[em] -one or more timetable datasets are joined, by simply concatenating them. -This step allows building one dataset to compare multiple experiments in -the same figure. -.PP -.I rPlot \[em] -one ot more figures are generated by a single R script -.[ -r cookbook -.] -which takes as input the previously generated dataset. -The path of the dataset is recorded in the figure as well, which -contains enough information to determine all the stages in the execution -and postprocess pipelines. -.NH 2 -Current setup -.LP -As of this moment, the -.I build -machine which contains the nix store is -.I xeon07 -and the -.I "target" -machine used to run the experiments is Mare Nostrum 4 with the -.I output -directory placed at -.CW /gpfs/projects/bsc15/garlic . -By default, the experiment results are never deleted from the -.I target -so you may want to remove the ones already stored in the nix store to -free space. -.\" ################################################################### -.bp -.SH 1 -Appendix A: Branch name diagram -.LP -.TAG appendixA -.DS B -.SM -.PS 4.4/25.4 -copy "gitbranch.pic" -.PE -.DE diff --git a/garlic/ds/ctf-mode.py b/garlic/ds/ctf-mode.py deleted file mode 100644 index 669508f..0000000 --- a/garlic/ds/ctf-mode.py +++ /dev/null @@ -1,83 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - ctf_mode = {} - - with open(".garlic/time_mode_runtime.csv", "r") as f: - ctf_mode['runtime'] = float(f.readline()) - - with open(".garlic/time_mode_dead.csv", "r") as f: - ctf_mode['dead'] = float(f.readline()) - - with open(".garlic/time_mode_task.csv", "r") as f: - ctf_mode['task'] = float(f.readline()) - - tree['ctf_mode'] = ctf_mode - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/index.nix b/garlic/ds/index.nix deleted file mode 100644 index 95cb388..0000000 --- a/garlic/ds/index.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -rec { - - py = callPackage ./py.nix {}; - - std.timetable = py { script = ./std-timetable.py; compress = false; }; - osu.latency = py { script = ./osu-latency.py; }; - osu.bw = py { script = ./osu-bw.py; }; - perf.stat = py { script = ./perf-stat.py; }; - ctf.mode = py { script = ./ctf-mode.py; }; -} diff --git a/garlic/ds/osu-bw.py b/garlic/ds/osu-bw.py deleted file mode 100644 index 4b1df43..0000000 --- a/garlic/ds/osu-bw.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - bw = slices[1] - - tree['size'] = int(size) - tree['bw'] = float(bw) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/osu-latency.py b/garlic/ds/osu-latency.py deleted file mode 100644 index 2df08ee..0000000 --- a/garlic/ds/osu-latency.py +++ /dev/null @@ -1,64 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - for line in lines: - - if not re.match('^[0-9]+ *[0-9\.]+$', line): - continue - - slices = line.split() - size = slices[0] - latency = slices[1] - - tree['size'] = int(size) - tree['latency'] = float(latency) - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/perf-stat.py b/garlic/ds/perf-stat.py deleted file mode 100644 index a6d69bc..0000000 --- a/garlic/ds/perf-stat.py +++ /dev/null @@ -1,90 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - with open(".garlic/perf.csv", "r") as f: - lines = [line.strip() for line in f.readlines()] - - perf_data = {} - - for line in lines: - if len(line) == 0: continue - if line[0] == '#': continue - - slices = line.split(',') - if len(slices) != 7: - print("error: mismatched columns") - exit(1) - - name = slices[2].replace("-", "_") - value = float(slices[0]) - - perf_data[name] = value - - tree['perf'] = perf_data - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/ds/py.nix b/garlic/ds/py.nix deleted file mode 100644 index 012a159..0000000 --- a/garlic/ds/py.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, python3 -, gzip -}: - -{ - script, - compress ? true -}: - -tree: - -stdenv.mkDerivation { - name = "dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - buildInputs = [ python3 gzip ]; - installPhase = '' - mkdir -p $out - ln -s ${tree} $out/tree - ln -s ${script} $out/script - - COMPRESS_DATASET=${toString compress} - - if [ $COMPRESS_DATASET ]; then - python $out/script $out/tree | gzip > $out/dataset.json.gz - ln -s dataset.json.gz $out/dataset - else - python $out/script $out/tree > $out/dataset.json - ln -s dataset.json $out/dataset - fi - ''; -} diff --git a/garlic/ds/std-timetable.py b/garlic/ds/std-timetable.py deleted file mode 100644 index c183097..0000000 --- a/garlic/ds/std-timetable.py +++ /dev/null @@ -1,77 +0,0 @@ -import json, re, sys, os, glob -from os import path - -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, flush=True, **kwargs) - -def process_run(tree, runPath): - - with open(".garlic/total_time_start", "r") as f: - total_time_start = float(f.readline().strip()) - - with open(".garlic/total_time_end", "r") as f: - total_time_end = float(f.readline().strip()) - - with open("stdout.log", "r") as f: - lines = [line.strip() for line in f.readlines()] - - time_line = None - for line in lines: - - if re.match(r'^ ?time .*', line): - time_line = line - break - - if time_line is None: - eprint("missing time line, aborting") - eprint("stdout file = {}/stdout.log".format(runPath)) - exit(1) - - time_str = time_line.split()[1] - - tree['time'] = float(time_str) - tree['total_time'] = total_time_end - total_time_start - - print(json.dumps(tree)) - -def process_result_tree(resultTree): - - eprint("processing resultTree: " + resultTree) - - os.chdir(resultTree) - - experiments = glob.glob(resultTree + "/*-experiment") - - for exp in glob.glob("*-experiment"): - eprint("found experiment: " + exp) - expPath = path.join(resultTree, exp) - os.chdir(expPath) - - for unit in glob.glob("*-unit"): - eprint("found unit: " + unit) - unitPath = path.join(resultTree, exp, unit) - os.chdir(unitPath) - - with open('garlic_config.json') as json_file: - garlic_conf = json.load(json_file) - - tree = {"exp":exp, "unit":unit, "config":garlic_conf} - - for i in range(garlic_conf['loops']): - run = str(i + 1) - runPath = path.join(resultTree, exp, unit, run) - if path.isdir(runPath) == False: - eprint("missing run {}, aborting".format(run)) - exit(1) - - tree["run"] = run - os.chdir(runPath) - - process_run(tree, runPath) - - -if len(sys.argv) != 2: - eprint("usage: python {} ".format(argv[0])) - exit(1) - -process_result_tree(sys.argv[1]) diff --git a/garlic/exp/bigsort/genseq.nix b/garlic/exp/bigsort/genseq.nix deleted file mode 100644 index 99a59f1..0000000 --- a/garlic/exp/bigsort/genseq.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, n # must be a string -, dram # must be a string -, strace -}: - -with lib; - -# Ensure the arguments are strings, to avoid problems with large numbers -assert (isString n); -assert (isString dram); - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "genseq"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - #FIXME: We need a better mechanism to get the output paths - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/seq.dat"; - in - stages.exec { - inherit nextStage; - pre = '' - mkdir -p "${outDir}" - ''; - argv = [ n dram outFile ]; - post = '' - # Link the output here - ln -s "${outFile}" seq.dat - ''; - }; - - program = {...}: bsc.apps.bigsort.genseq; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/shuffle.nix b/garlic/exp/bigsort/shuffle.nix deleted file mode 100644 index a841720..0000000 --- a/garlic/exp/bigsort/shuffle.nix +++ /dev/null @@ -1,102 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, inputTre -, n -, dram -, garlicTools -, resultFromTrebuchet -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { }; - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "shuffle"; - unitName = "${expName}.n${n}.dram${dram}"; - inherit (targetMachine.config) hw; - inherit n dram; - - # Don't repeat - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - cpusPerTask = hw.cpusPerNode; - jobName = unitName; - - # We need access to a fast shared filesystem to store the shuffled input - # dataset - extraMounts = [ fs.shared.fast ]; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - inputUnit = elemAt inputExp.units 0; - unitName = baseNameOf (toString inputUnit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/seq.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/shuffled.dat"; - - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the previous results are complete: - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - mkdir -p "${outDir}" - - # Copy the input as we are going to overwrite it - cp "${inFile}" "${outFile}" - ''; - argv = [ n dram outFile 16 64 ]; - post = '' - # Link the output here - ln -s "${outFile}" shuffled.dat - ''; - }; - - program = {...}: - bsc.apps.bigsort.shuffle; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/bigsort/sort.nix b/garlic/exp/bigsort/sort.nix deleted file mode 100644 index 2ae73e9..0000000 --- a/garlic/exp/bigsort/sort.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, removeOutput ? true -, resultFromTrebuchet -, inputTre -}: - -with lib; - -let - varConf = { }; # Not used - - inherit (targetMachine) fs; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "bigsort"; - unitName = "${expName}.bs${toString bs}"; - inherit (targetMachine.config) hw; - - # bigsort options - n = 1024 * 1024 * 1024 / 8; # In longs (?) - bs = n; # In bytes - pageSize = bs / 2; # In bytes (?) - cc = bsc.icc; - mpi = bsc.impi; - gitBranch = "garlic/mpi+send+omp+task"; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "01:00:00"; - # All CPUs of the socket to each task - cpusPerTask = hw.cpusPerSocket; - jobName = "bigsort-${toString n}-${toString bs}-${gitBranch}"; - - # Load the dataset from the same fs where it was stored in the shuffle - # step. Also we use a local temp fs to store intermediate results. - extraMounts = [ fs.shared.fast fs.local.temp ]; - - rev = 1; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; - let - inputExp = inputTre.experiment; - unit = elemAt inputExp.units 0; - expName = baseNameOf (toString inputExp); - unitName = baseNameOf (toString unit); - - # We also need the result. This is only used to ensure that we have the - # results, so it has been executed. - inputRes = resultFromTrebuchet inputTre; - - #FIXME: We need a better mechanism to get the output paths - inFile = "${fs.shared.fast}/out/$GARLIC_USER/${unitName}/1/shuffled.dat"; - outDir = "${fs.shared.fast}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN"; - outFile = "${outDir}/sorted.dat"; - tmpDir = fs.local.temp; - in - stages.exec { - inherit nextStage; - pre = '' - # This line ensures that the shuffled results are complete: nix needs to - # compute the hash of the execution log to write the path here. - # ${inputRes} - - # Exit on error - set -e - - # Ensure the input file exists - if [ ! -f "${inFile}" ]; then - echo "input file not found: ${inFile}" - exit 1 - fi - - # Create the output path - mkdir -p ${outDir} - - # Verbose args: - echo "INPUT = ${inFile}" - echo "OUTPUT = ${outFile}" - echo "TMPDIR = ${tmpDir}" - ''; - - argv = [ n bs inFile outFile tmpDir pageSize ]; - - # Optionally remove the potentially large output dataset - post = '' - # Link the output here - ln -s "${outFile}" sorted.dat - '' + optionalString (removeOutput) '' - # Remove the sorted output - stat "${outFile}" > "${outFile}.stat" - echo "file removed to save space" > "${outFile}" - ''; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.bigsort.sort.override { - inherit cc mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - #{ inherit configs pipeline; } - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/nbody.nix b/garlic/exp/cn6/nbody.nix deleted file mode 100644 index e48018f..0000000 --- a/garlic/exp/cn6/nbody.nix +++ /dev/null @@ -1,164 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 2 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "cn6-nbody"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - # Parameters for nbody - particles = 4 * 512 * hw.cpusPerSocket; - timesteps = 2; - blocksize = 512; - gitBranch = "garlic/tampi+isend+oss+task"; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:nanos6/forks/nanos6-fixes.git"; - ref = "distributed-instrumentation-fixes"; - rev = "80058512527961fbde9bd81474b0a29141d7982c"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - "--with-babeltrace2=${super.babeltrace2}" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - #dontStrip = true; - #NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/interoperability/tampi.git"; - ref = "master"; - rev = "f1e77e6f439a0e964e98b5e0a4738b2e95e4fd3d"; - }; - }); - }); - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = true - instrument.ctf.converter.fast = false - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - - # Add nanos6 and babeltrace2 binaries to the PATH - export PATH="$PATH:${bsc'.nanos6}/bin:${bsc'.babeltrace2}/bin" - - # Also add the babeltrace2 python module to python search path - export PYTHONPATH="$PYTHONPATH:${bsc'.babeltrace2}/lib/python3.8/site-packages" - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_nbody - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - nanos6-mergeprv "$tracedir" - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = with conf; [ - "-t" timesteps - "-p" particles - ]; - }; - - program = {nextStage, conf, ...}: bsc'.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/cn6/timediff.nix b/garlic/exp/cn6/timediff.nix deleted file mode 100644 index 7d2d42f..0000000 --- a/garlic/exp/cn6/timediff.nix +++ /dev/null @@ -1,171 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, pkgs -, targetMachine -, stages -, garlicTools -, writeText -, enableHWC ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - nodes = [ 1 2 4 8 ]; - }; - - machineConfig = targetMachine.config; - - genConf = c: targetMachine.config // rec { - expName = "timediff"; - unitName = expName + "-nodes${toString nodes}"; - - inherit (machineConfig) hw; - - loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = c.nodes; - - qos = "debug"; - time = "02:00:00"; - - jobName = unitName; - }; - - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Custom BSC packages - bsc' = bsc.extend (self: super: { - - # For nanos6 we use my fork for distributed instrumentation at the - # latest commit - nanos6 = (super.nanos6Git.override { - enableJemalloc = true; - }).overrideAttrs (old: rec { - - src = builtins.fetchGit { - url = "git@bscpm03.bsc.es:rarias/nanos6.git"; - ref = "rodrigo"; - rev = "5cbeabb4e0446c2c293cc3005f76e6139465caee"; - }; - - dontStrip = false; - version = src.shortRev; - - # Disable all unused instrumentations for faster builds - configureFlags = old.configureFlags ++ [ - "--disable-extrae-instrumentation" - "--disable-lint-instrumentation" - "--disable-graph-instrumentation" - "--disable-stats-instrumentation" - ]; - }); - - # Use clang from master - clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git"; - ref = "master"; - rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c"; - }; - }); - - # Use mcxx from master - mcxx = super.mcxxGit; - - # We also need the instrumented version of TAMPI - tampi = super.tampiGit.overrideAttrs (old: rec { - version = src.shortRev; - dontStrip = true; - NIX_CFLAGS = "-O0 -g"; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/tampi.git"; - ref = "instrument"; - rev = "6e4294299bf761a1cc31f4181d9479cefa1c7f3e"; - }; - }); - - # We use the latest commit in master as src for cn6 - cn6Git = ((super.cn6.overrideAttrs (old: rec { - version = src.shortRev; - src = builtins.fetchGit { - url = "ssh://git@bscpm03.bsc.es/rarias/cn6.git"; - ref = "master"; - rev = "1d23d01d60164b8641746d5a204128a9d31b9650"; - }; - })).override { enableTest = true; }); - - cn6 = self.cn6Git; - }); - - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - post = '' - rank=$SLURM_PROCID - tracedir=trace_timediff_mpi - - # Convert CTF trace to PRV - ${bsc'.cn6}/bin/cn6 $tracedir/$rank - - # Merge on rank 0 only - if [ $rank != 0 ]; then - exit 0; - fi - - # Wait a bit for all ranks to finish the conversion - sleep 5 - - # Run the merger - ${bsc'.cn6}/bin/merge-prv $tracedir - - # We need some tools the path - export PATH="$PATH:${bsc'.babeltrace2}/bin:${pkgs.ministat}/bin" - - ${bsc'.cn6}/bin/sync-err.sh $tracedir - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - program = "${bsc'.cn6}/bin/timediff_mpi"; - argv = [ conf.cpusPerTask ]; - }; - - pipeline = stdexp.stdPipeline ++ [ ctf exec ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity.nix b/garlic/exp/creams/granularity.nix deleted file mode 100644 index 3fab638..0000000 --- a/garlic/exp/creams/granularity.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - granul = range2 4 128; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+omp+task" - ] ++ optionals (enableExtended) [ - #"garlic/mpi+send+omp+fork" # Don't use fork for granularity - "garlic/mpi+send+seq" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+oss+task" - ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - nodes = [ 1 ] ++ optionals (enableExtended) (range2 2 16); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-gran"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) gitBranch nodes iterations; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/granularity16.nix b/garlic/exp/creams/granularity16.nix deleted file mode 100644 index fa8dc6f..0000000 --- a/garlic/exp/creams/granularity16.nix +++ /dev/null @@ -1,132 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = [ 1 2 4 8 16 32 ]; - granul = [ 1 2 4 8 16 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - #"garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-granularity16"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique ( - filter (c: !(c.granul == 1 && c.sizeFactor >= 32))(stdexp.buildConfigs { - inherit varConf genConf; - })); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/size.nix b/garlic/exp/creams/size.nix deleted file mode 100644 index dea3c8d..0000000 --- a/garlic/exp/creams/size.nix +++ /dev/null @@ -1,131 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - #nodes = range2 1 16; - nodes = [ 16 ]; - sizeFactor = range2 1 32; - baseGranul = [ 1 ] ++ optionals (enableExtended) [ 2 4 8 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+fork" - #"garlic/mpi+send+omp+task" - #"garlic/mpi+send+seq" - ] ++ (optionals (enableExtended) [ - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]); - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-size"; - unitName = "${expName}" - + "-granul.${toString granul}" - + "-sf.${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes sizeFactor baseGranul; - granul = getGranul gitBranch (max 2 (baseGranul * sizeFactor)); - nprocz = ntasksPerNode * nodes; - baseSizePerCpu = 2; - baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; - - nz = baseSize * sizeFactor; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz nz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/creams/ss.nix b/garlic/exp/creams/ss.nix deleted file mode 100644 index 86723be..0000000 --- a/garlic/exp/creams/ss.nix +++ /dev/null @@ -1,126 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - - nodes = range2 1 16; - granul = [ 16 ] ++ optionals (enableExtended) [ 8 32 ]; - - # Max. number of iterations - iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; - - gitBranch = [ - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+seq" - ] ++ optionals (enableExtended) [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+oss+task" - "garlic/mpi+isend+omp+task" - "garlic/mpi+isend+oss+task" - ]; - }; - - # We use these auxiliary functions to assign different configurations - # depending on the git branch. - getGranul = branch: oldGranul: - if (branch == "garlic/mpi+send+seq") - then 999999 else oldGranul; - - getCpusPerTask = branch: hw: - if (branch == "garlic/mpi+send+seq") - then 1 else hw.cpusPerSocket; - - getNtasksPerNode = branch: hw: - if (branch == "garlic/mpi+send+seq") - then hw.cpusPerNode else hw.socketsPerNode; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - expName = "creams-ss"; - unitName = "${expName}"+ - "-nodes${toString nodes}"+ - "-granul${toString granul}"+ - "-${gitBranch}"; - - inherit (targetMachine.config) hw; - - # Options for creams - inherit (c) iterations gitBranch nodes; - granul = getGranul gitBranch c.granul; - nprocz = ntasksPerNode * nodes; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - ntasksPerNode = getNtasksPerNode gitBranch hw; - cpusPerTask = getCpusPerTask gitBranch hw; - jobName = unitName; - }; - - # Compute the array of configurations - configs = unique (stdexp.buildConfigs { - inherit varConf genConf; - }); - - # Custom srun stage to copy the creams input dataset - customSrun = {nextStage, conf, ...}: - let - input = bsc.garlic.apps.creamsInput.override { - inherit (conf) gitBranch granul nprocz; - }; - in - stdexp.stdStages.srun { - inherit nextStage conf; - # Now we add some commands to execute before calling srun. These will - # only run in one rank (the first in the list of allocated nodes) - preSrun = '' - cp -r ${input}/SodTubeBenchmark/* . - chmod +w -R . - sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat - rm -f nanos6.toml - ''; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" - ''; - - # Remove restarts as is not needed and is huge - post = '' - rm -rf restarts || true - ''; - }; - - # Creams program - creams = {nextStage, conf, ...}: bsc.apps.creams.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipelineOverride { - # Replace the stdandard srun stage with our own - overrides = { srun = customSrun; }; - } ++ [ exec creams ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/examples/granularity.nix b/garlic/exp/examples/granularity.nix deleted file mode 100644 index 1c01244..0000000 --- a/garlic/exp/examples/granularity.nix +++ /dev/null @@ -1,189 +0,0 @@ -# This file defines an experiment. It is designed as a function that takes -# several parameters and returns a derivation. This derivation, when built will -# create several scripts that can be executed and launch the experiment. - -# These are the inputs to this function: an attribute set which must contain the -# following keys: -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -# We import in the scope the content of the `lib` attribute, which -# contain useful functions like `toString`, which will be used later. This is -# handy to avoid writting `lib.tostring`. - -with lib; - -# We also have some functions specific to the garlic benchmark which we import -# as well. Take a look at the garlic/tools.nix file for more details. -with garlicTools; - -# The `let` keyword allows us to define some local variables which will be used -# later. It works as the local variable concept in the C language. -let - - # Initial variable configuration: every attribute in this set contains lists - # of options which will be used to compute the configuration of the units. The - # cartesian product of all the values will be computed. - varConf = { - # In this case we will vary the columns and rows of the blocksize. This - # configuration will create 3 x 2 = 6 units. - cbs = [ 256 1024 4096 ]; - rbs = [ 512 1024 ]; - }; - - # Generate the complete configuration for each unit: genConf is a function - # that accepts the argument `c` and returns a attribute set. The attribute set - # is formed by joining the configuration of the machine (which includes - # details like the number of nodes or the architecture) and the configuration - # that we define for our units. - # - # Notice the use of the `rec` keyword, which allows us to access the elements - # of the set while is being defined. - genConf = c: targetMachine.config // rec { - - # These attributes are user defined, and thus the user will need to handle - # them manually. They are not read by the standard pipeline: - - # Here we load the `hw` attribute from the machine configuration, so we can - # access it, for example, the number of CPUs per socket as hw.cpusPerSocket. - hw = targetMachine.config.hw; - - # These options will be used by the heat app, be we write them here so they - # are stored in the unit configuration. - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - - # The blocksize is set to the values passed in the `c` parameter, which will - # be set to one of all the configurations of the cartesian product. for - # example: cbs = 256 and rbs = 512. - # We can also write `inherit (c) cbs rbs`, as is a shorthand notation. - cbs = c.cbs; - rbs = c.rbs; - - # The git branch is specified here as well, as will be used when we specify - # the heat app - gitBranch = "garlic/tampi+isend+oss+task"; - - # ------------------------------------------------------------------------- - - # These attributes are part of the standard pipeline, and are required for - # each experiment. They are automatically recognized by the standard - # execution pipeline. - - # The experiment name: - expName = "example-granularity-heat"; - - # The experimental unit name. It will be used to create a symlink in the - # index (at /gpfs/projects/bsc15/garlic/$USER/index/) so you can easily find - # the unit. Notice that the symlink is overwritten each time you run a unit - # with the same same. - # - # We use the toString function to convert the numeric value of cbs and rbs - # to a string like: "example-granularity-heat.cbs-256.rbs-512" - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - # Repeat the execution of each unit a few times: this option is - # automatically taken by the experiment, which will repeat the execution of - # the program that many times. It is recommended to run the app at least 30 - # times, but we only used 10 here for demostration purposes (as it will be - # faster to run) - loops = 10; - - # Resources: here we configure the resources in the machine. The queue to be - # used is `debug` as is the fastest for small jobs. - qos = "debug"; - - # Then the number of MPI processes or tasks per node: - ntasksPerNode = 1; - - # And the number of nodes: - nodes = 1; - - # We use all the CPUs available in one socket to each MPI process or task. - # Notice that the number of CPUs per socket is not specified directly. but - # loaded from the configuration of the machine that will be used to run our - # experiment. The affinity mask is set accordingly. - cpusPerTask = hw.cpusPerSocket; - - # The time will limit the execution of the program in case of a deadlock - time = "02:00:00"; - - # The job name will appear in the `squeue` and helps to identify what is - # running. Currently is set to the name of the unit. - jobName = unitName; - }; - - # Using the `varConf` and our function `genConf` we compute a list of the - # complete configuration of every unit. - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Now that we have the list of configs, we need to write how that information - # is used to run our program. In our case we will use some params such as the - # number of rows and columns of the input problem or the blocksize as argv - # values. - - # The exec stage is used to run a program with some arguments. - exec = {nextStage, conf, ...}: stages.exec { - # All stages require the nextStage attribute, which is passed as parameter. - inherit nextStage; - - # Then, we fill the argv array with the elements that will be used when - # running our program. Notice that we load the attributes from the - # configuration which is passed as argument as well. - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # This program requires a file called `head.conf` in the current directory. - # To do it, we run this small script in the `pre` hook, which simple runs - # some commands before running the program. Notice that this command is - # executed in every MPI task. - pre = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - # The program stage is only used to specify which program we should run. - # We use this stage to specify build-time parameters such as the gitBranch, - # which will be used to fetch the source code. We use the `override` function - # of the `bsc.garlic.apps.heat` derivation to change the input paramenters. - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - # Other stages may be defined here, in case that we want to do something - # additional, like running the program under `perf stats` or set some - # envionment variables. - - # Once all the stages are defined, we build the pipeline array. The - # `stdexp.stdPipeline` contains the standard pipeline stages, so we don't need - # to specify them. We only specify how we run our program, and what program - # exactly, by adding our `exec` and `program` stages: - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -# Then, we use the `configs` and the `pipeline` just defined inside the `in` -# part, to build the complete experiment: -in - - # The `stdexp.genExperiment` function generates an experiment by calling every - # stage of the pipeline with the different configs, and thus creating - # different units. The result is the top level derivation which is the - # `trebuchet`, which is the script that, when executed, launches the complete - # experiment. - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix deleted file mode 100644 index 63e88de..0000000 --- a/garlic/exp/fwi/common.nix +++ /dev/null @@ -1,128 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -}: - -with lib; - -# Common definitions used by fwi experiments -rec { - - branchesWithoutBlocksize = [ - "garlic/mpi+send+omp+fork" - "garlic/mpi+send+seq" - ]; - - # Returns true if the given config is in the forkJoinBranches list - needsBlocksize = c: ! any (e: c.gitBranch == e) branchesWithoutBlocksize; - - # Set the blocksize to null for the fork join branch - fixBlocksize = c: if (needsBlocksize c) then c - else (c // { blocksize = null; }); - - # Generate the configs by filtering the unneded blocksizes - getConfigs = {varConf, genConf}: - let - allConfigs = stdexp.buildConfigs { inherit varConf genConf; }; - in - # The unique function ensures that we only run one config for the fork - # join branch, even if we have multiple blocksizes. - unique (map fixBlocksize allConfigs); - - getResources = {gitBranch, hw}: - if (gitBranch == "garlic/mpi+send+seq") then { - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - } else { - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - }; - - exec = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - - ioFreq = if (conf.enableIO) then (conf.ioFreq or "-1") else "9999"; - - in stages.exec { - inherit nextStage; - - # FIXME: FWI should allow the I/O directory to be specified as a - # parameter - pre = '' - FWI_SRUNDIR=$(pwd) - FWI_EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - FWI_PARAMS="${fwiParams}/fwi_params.txt" - FWI_FREQ="${fwiParams}/fwi_frequencies.txt" - - # Run fwi in a directory with fast local storage - mkdir -p "$FWI_EXECDIR" - cd "$FWI_EXECDIR" - - # Only generate the input if we have the CPU 0 (once per node) - if grep -o 'Cpus_allowed_list:[[:space:]]0' \ - /proc/self/status > /dev/null; - then - FWI_CAPTAIN=1 - fi - - if [ $FWI_CAPTAIN ]; then - >&2 echo "generating the input dataset" - ${fwiParams}/bin/ModelGenerator -m "$FWI_PARAMS" "$FWI_FREQ" - fi - - echo >&2 "Current dir: $(pwd)" - echo >&2 "Using PARAMS=$FWI_PARAMS and FREQ=$FWI_FREQ" - '' + optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - ''; - - argv = [ - ''"$FWI_PARAMS"'' - ''"$FWI_FREQ"'' - ] ++ optional (needsBlocksize conf) conf.blocksize ++ [ - "-1" # Fordward steps - "-1" # Backward steps - ioFreq # Write/read frequency - ]; - - post = '' - # Go back to the garlic out directory - cd "$FWI_SRUNDIR" - - if [ $FWI_CAPTAIN ]; then - '' + optionalString (conf.enableCTF) '' - # FIXME: We should specify the path in the nanos6 config, so we - # can avoid the race condition while they are generating the - # traces - sleep 3 - - # Save the traces - mv "$FWI_EXECDIR"/trace_* . - '' + '' - rm -rf "$FWI_EXECDIR" - fi - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: - let - fwiParams = bsc.apps.fwi.params.override { - inherit (conf) nx ny nz; - }; - in - apps.fwi.solver.override { - inherit (conf) gitBranch; - inherit fwiParams; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix deleted file mode 100644 index 15bc484..0000000 --- a/garlic/exp/fwi/granularity.nix +++ /dev/null @@ -1,71 +0,0 @@ -# Regular granularity test for FWI - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+isend+oss+task" ]; - blocksize = range2 1 256; - n = [ {nx=100; nz=100; ny=8000; ntpn=2; nodes=1;} ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-granularity"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz ntpn nodes; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = ntpn; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/io.nix b/garlic/exp/fwi/io.nix deleted file mode 100644 index 0d107d7..0000000 --- a/garlic/exp/fwi/io.nix +++ /dev/null @@ -1,75 +0,0 @@ -# Test FWI variants based on tasks with and without I/O. -# This experiment solves a computationally expensive input which brings the -# storage devices to saturation when I/O is enabled. The same input runs -# without I/O for comparison purposes. Also, a range of block sizes -# deemed as efficient according to the granularity experiment are -# explored. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -, enableExtended ? false -}: - -with lib; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ "garlic/tampi+send+oss+task" ]; - blocksize = [ 1 2 4 8 ]; - n = [ {nx=500; nz=500; ny=16000;} ]; - nodes = if (enableExtended) then range2 1 16 else [ 4 ]; - enableIO = [ false true ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-io"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + (if (enableIO) then "-io1" else "-io0") - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize enableIO nodes; - inherit (c.n) nx ny nz; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - enableCTF = false; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/reuse.nix b/garlic/exp/fwi/reuse.nix deleted file mode 100644 index 9a725e2..0000000 --- a/garlic/exp/fwi/reuse.nix +++ /dev/null @@ -1,91 +0,0 @@ -# This test compares a FWI version using poor data locality (+NOREUSE) versus -# the optimized version (used for all other experiments). Follows a pseudocode -# snippet illustrating the fundamental difference between version. -# -# NOREUSE -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# for (y) for (x) for (z) -# computB(v[y][x][z]); -# for (y) for (x) for (z) -# computC(v[y][x][z]); -# -# Optimized version -# ---------------------- -# for (y) for (x) for (z) -# computA(v[y][x][z]); -# computB(v[y][x][z]); -# computC(v[y][x][z]); - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, callPackage -}: - -with lib; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/mpi+send+oss+task" - "garlic/mpi+send+oss+task+NOREUSE" - ]; - - blocksize = [ 1 2 4 8 ]; - - n = [ {nx=300; ny=2000; nz=300;} ]; # / half node - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "fwi-reuse"; - unitName = "${expName}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - enableCTF = false; - enableIO = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix deleted file mode 100644 index c54370e..0000000 --- a/garlic/exp/fwi/ss.nix +++ /dev/null @@ -1,90 +0,0 @@ -# Strong scaling test for FWI variants based on tasks. This -# experiment explores a range of block sizes deemed as efficient -# according to the granularity experiment. - -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix {}; - inherit (common) getConfigs getResources pipeline; - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/tampi+isend+oss+task" - ] ++ optionals (enableExtended) [ - "garlic/tampi+send+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - "garlic/mpi+send+omp+fork" - # FIXME: the mpi pure version has additional constraints with the - # number of planes in Y. By now is disabled. - #"garlic/mpi+send+seq" - ]; - - blocksize = if (enableExtended) - then range2 1 16 - else [ 2 ]; - - n = [ { nx=100; ny=8000; nz=100; } ]; - - nodes = range2 1 16; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = c: machineConfig // rec { - expName = "fwi-ss"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bs${toString blocksize}" - + "-${toString gitBranch}"; - - inherit (machineConfig) hw; - inherit (c) gitBranch blocksize; - inherit (c.n) nx ny nz; - - # Other FWI parameters - enableIO = true; - enableCTF = false; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - inherit (getResources { inherit gitBranch hw; }) - cpusPerTask ntasksPerNode; - - nodes = c.nodes; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - tempDir = fs.local.temp; - }; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/heat/granularity.nix b/garlic/exp/heat/granularity.nix deleted file mode 100644 index 8ebe5db..0000000 --- a/garlic/exp/heat/granularity.nix +++ /dev/null @@ -1,175 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, writeText -, enablePerf ? false -, enableCTF ? false -, enableHWC ? false -, enableExtended ? false -}: - -# TODO: Finish HWC first -assert (enableHWC == false); - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = with bsc; { - cbs = range2 32 4096; - rbs = range2 32 4096; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "heat"; - unitName = expName + - ".cbs-${toString cbs}" + - ".rbs-${toString rbs}"; - - inherit (machineConfig) hw; - - # heat options - timesteps = 10; - cols = 1024 * 16; # Columns - rows = 1024 * 16; # Rows - inherit (c) cbs rbs; - gitBranch = "garlic/tampi+isend+oss+task"; - - # Repeat the execution of each unit 30 times - loops = 10; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - # Assign one socket to each task (only one process) - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - filterConfigs = c: let - # Too small sizes lead to huge overheads - goodSize = (c.cbs * c.rbs >= 1024); - # When the extended units are not enabled, we only select those in - # the diagonal. - extended = if (enableExtended) then true - else c.cbs == c.rbs; - in - goodSize && extended; - - # Compute the array of configurations - configs = filter (filterConfigs) (stdexp.buildConfigs { - inherit varConf genConf; - }); - - perf = {nextStage, conf, ...}: stages.perf { - inherit nextStage; - perfOptions = "stat -o .garlic/perf.csv -x , " + - "-e cycles,instructions,cache-references,cache-misses"; - }; - - ctf = {nextStage, conf, ...}: let - # Create the nanos6 configuration file - nanos6ConfigFile = writeText "nanos6.toml" '' - version.instrument = "ctf" - turbo.enabled = false - instrument.ctf.converter.enabled = false - '' + optionalString (enableHWC) '' - hardware_counters.papi.enabled = true - hardware_counters.papi.counters = [ - "PAPI_TOT_INS", "PAPI_TOT_CYC", - "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM" - ] - ''; - - in stages.exec { - inherit nextStage; - - # And use it - env = '' - export NANOS6_CONFIG=${nanos6ConfigFile} - ''; - - # FIXME: We should run a hook *after* srun has ended, so we can - # execute it in one process only (not in N ranks). This hack works - # with one process only. Or be able to compute the name of the trace - # directory so we can begin the conversion in parallel - post = assert (conf.nodes * conf.ntasksPerNode == 1); '' - tracedir=$(ls -d trace_* | head -1) - echo "using tracedir=$tracedir" - - offset=$(grep 'offset =' $tracedir/ctf/ust/uid/1000/64-bit/metadata | \ - grep -o '[0-9]*') - echo "offset = $offset" - - start_time=$(awk '/^start_time / {print $2}' stdout.log) - end_time=$(awk '/^end_time / {print $2}' stdout.log) - - begin=$(awk "BEGIN{print $start_time*1e9 - $offset}") - end=$(awk "BEGIN{print $end_time*1e9 - $offset}") - - echo "only events between $begin and $end" - - ${bsc.cn6}/bin/cn6 -s $tracedir - - ${bsc.cn6}/bin/cut $begin $end < $tracedir/prv/trace.prv |\ - ${bsc.cn6}/bin/hcut 1 ${toString conf.cpusPerTask} \ - > $tracedir/prv/trace-cut.prv - - ${bsc.cn6}/bin/dur 6400025 0 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_dead.csv & - - ${bsc.cn6}/bin/dur 6400025 1 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_runtime.csv & - - ${bsc.cn6}/bin/dur 6400025 3 < $tracedir/prv/trace-cut.prv |\ - awk '{s+=$1} END {print s}' >> .garlic/time_mode_task.csv & - - wait - - # Remove the traces at the end, as they are huge - rm -rf $tracedir - ''; - # TODO: To enable HWC we need to first add a taskwait before the - # first get_time() measurement, otherwise we get the HWC of the - # main task, which will be huge. - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ - "--rows" conf.rows - "--cols" conf.cols - "--rbs" conf.rbs - "--cbs" conf.cbs - "--timesteps" conf.timesteps - ]; - - # The next stage is the program - env = '' - ln -sf ${nextStage}/etc/heat.conf heat.conf || true - ''; - }; - - program = {nextStage, conf, ...}: bsc.garlic.apps.heat.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ - (optional enablePerf perf) ++ - (optional enableCTF ctf) ++ - [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/common.nix b/garlic/exp/hpcg/common.nix deleted file mode 100644 index a7c19e3..0000000 --- a/garlic/exp/hpcg/common.nix +++ /dev/null @@ -1,73 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, callPackage -}: - -with lib; - -rec { - - checkInput = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = optionalString (! (conf.enableGen or false)) ( - let - gen = callPackage ./gen.nix { }; - inputTre = gen.getInputTre conf; - exp = inputTre.experiment; - unit = elemAt exp.units 0; - expName = baseNameOf (toString exp); - unitName = baseNameOf (toString unit); - inputPath = "$GARLIC_OUT/${expName}/${unitName}/1"; - in - '' - # Force the generation of the input resultTree as a dependency: - # ${toString inputTre.result} - - # Ensure the input dataset is still available - export HPCG_INPUT_PATH="${toString inputPath}" - - if [ ! -e "$HPCG_INPUT_PATH" ]; then - >&2 echo "Missing input dataset: $HPCG_INPUT_PATH" - exit 1 - fi - '' - ); - }; - - getSizePerTask = cpusPerTask: sizePerCpu: - mapAttrs (name: val: val * cpusPerTask) sizePerCpu; - - exec = {nextStage, conf, ...}: let - actionArg = if (conf.enableGen or false) - then "--store=." - else "--load=\"$HPCG_INPUT_PATH\""; - - in stages.exec { - inherit nextStage; - argv = [ - "--nx=${toString conf.sizePerTask.x}" - "--ny=${toString conf.sizePerTask.y}" - "--nz=${toString conf.sizePerTask.z}" - "--npx=${toString conf.nprocs.x}" - "--npy=${toString conf.nprocs.y}" - "--npz=${toString conf.nprocs.z}" - "--nblocks=${toString conf.nblocks}" - "--ncomms=${toString conf.ncomms}" - # The input symlink is generated by the input stage, which is generated by - # the genInput function. - actionArg - ] ++ optional (conf.disableAspectRatio or false) "--no-ar=1"; - }; - - program = {nextStage, conf, ...}: bsc.apps.hpcg.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ - checkInput - exec program ]; -} diff --git a/garlic/exp/hpcg/gen.nix b/garlic/exp/hpcg/gen.nix deleted file mode 100644 index d34eccc..0000000 --- a/garlic/exp/hpcg/gen.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -rec { - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-gen"; - unitName = expName - + "-nodes${toString nodes}" - + "-spt.z${toString sizePerTask.z}"; - - inherit (targetMachine.config) hw; - - # Inherit options from the current conf - inherit (c) sizePerTask nprocs disableAspectRatio gitBranch - cpusPerTask ntasksPerNode nodes; - - # nblocks and ncomms are ignored from c - ncomms = 1; - nblocks = 1; - - # We only need one run - loops = 1; - - # Generate the input - enableGen = true; - - # Resources - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - getInputTre = conf: stdexp.genExperiment { - configs = [ (genConf conf) ]; - pipeline = common.pipeline; - }; -} diff --git a/garlic/exp/hpcg/granularity.nix b/garlic/exp/hpcg/granularity.nix deleted file mode 100644 index 7c3d2a8..0000000 --- a/garlic/exp/hpcg/granularity.nix +++ /dev/null @@ -1,76 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - blocksPerCpu = range2 0.5 256; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-granularity"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = maxNodes / nodes; - - # hpcg options - inherit (c) blocksPerCpu gitBranch; - baseSizeZ = 16; - nodes = 1; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = floatTruncate (blocksPerCpu * cpusPerTask); - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 3; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/scaling.nix b/garlic/exp/hpcg/scaling.nix deleted file mode 100644 index afb1aee..0000000 --- a/garlic/exp/hpcg/scaling.nix +++ /dev/null @@ -1,78 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -, enableExtended ? false -, enableStrong ? true -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - maxNodes = 16; - - # Initial variable configuration - varConf = { - nodes = range2 1 maxNodes; - baseSizeZ = if (enableExtended) then [ 8 16 ] else [ 16 ]; - blocksPerCpu = if (enableExtended) then range2 1 8 else [ 4 ]; - gitBranch = [ - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = if (enableStrong) then "hpcg-ss" else "hpcg-ws"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-bpc${toString blocksPerCpu}"; - - inherit (targetMachine.config) hw; - - inherit maxNodes; - sizeFactor = if (enableStrong) then maxNodes / nodes else 1; - - # hpcg options - inherit (c) nodes blocksPerCpu gitBranch; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { - x = 2; - y = 2; - z = c.baseSizeZ * sizeFactor; - }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/size.nix b/garlic/exp/hpcg/size.nix deleted file mode 100644 index 1124323..0000000 --- a/garlic/exp/hpcg/size.nix +++ /dev/null @@ -1,65 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - common = callPackage ./common.nix { }; - - inherit (common) pipeline getSizePerTask; - - # Initial variable configuration - varConf = { - sizeFactor = [ 1 2 4 8 16 32 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "hpcg-size"; - unitName = "${expName}" - + "-nodes${toString nodes}" - + "-sf${toString sizeFactor}"; - - inherit (targetMachine.config) hw; - - # hpcg options - inherit (c) sizeFactor; - gitBranch = "garlic/tampi+isend+oss+task"; - nodes = 16; - totalTasks = ntasksPerNode * nodes; - sizePerCpu = { x = 2; y = 2; z = 4 * sizeFactor; }; - sizePerTask = getSizePerTask cpusPerTask sizePerCpu; - nprocs = { x=1; y=1; z=totalTasks; }; - blocksPerCpu = 4; - nblocks = blocksPerCpu * cpusPerTask; - ncomms = 1; - disableAspectRatio = true; - - # Repeat the execution of each unit several times - loops = 5; - - # Resources - qos = "debug"; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix deleted file mode 100644 index 2f2bcc4..0000000 --- a/garlic/exp/index.nix +++ /dev/null @@ -1,108 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -{ - nbody = rec { - granularity = callPackage ./nbody/granularity.nix { }; - ss = callPackage ./nbody/ss.nix { }; - numa = callPackage ./nbody/numa.nix { }; - }; - - saiph = { - granularity = callPackage ./saiph/granularity.nix { }; - ss = callPackage ./saiph/ss.nix { }; - }; - - creams = rec { - ss = callPackage ./creams/ss.nix { }; - granularity = callPackage ./creams/granularity.nix { }; - size = callPackage ./creams/size.nix { }; - granularity16 = callPackage ./creams/granularity16.nix { }; - - # These experiments are the extended versions of the previous - # ones. We split them so we can keep a reasonable execution time - big.granularity = granularity.override { enableExtended = true; }; - big.ss = granularity.override { enableExtended = true; }; - }; - - hpcg = rec { - granularity = callPackage ./hpcg/granularity.nix { }; - ss = callPackage ./hpcg/scaling.nix { }; - ws = ss.override { enableStrong=false; }; - size = callPackage ./hpcg/size.nix { }; - - big.ss = ss.override { enableExtended = true; }; - }; - - heat = rec { - granularity = callPackage ./heat/granularity.nix { }; - cache = granularity.override { enablePerf = true; }; - ctf = granularity.override { enableCTF = true; }; - }; - - bigsort = rec { - genseq = callPackage ./bigsort/genseq.nix { - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - }; - - shuffle = callPackage ./bigsort/shuffle.nix { - inputTre = genseq; - n = toString (1024 * 1024 * 1024 / 8); # 1 GB input size - dram = toString (1024 * 1024 * 1024); # 1 GB chunk - inherit (bsc.garlic.pp) resultFromTrebuchet; - }; - - sort = callPackage ./bigsort/sort.nix { - inputTre = shuffle; - inherit (bsc.garlic.pp) resultFromTrebuchet; - removeOutput = false; - }; - }; - - slurm = { - cpu = callPackage ./slurm/cpu.nix { }; - sigsegv = callPackage ./slurm/sigsegv.nix { }; - exit1 = callPackage ./slurm/exit1.nix { }; - }; - - lulesh = { - test = callPackage ./lulesh/test.nix { }; - }; - - fwi = rec { - granularity = callPackage ./fwi/granularity.nix { }; - ss = callPackage ./fwi/ss.nix { }; - reuse = callPackage ./fwi/reuse.nix { }; - io = callPackage ./fwi/io.nix { }; - - # Extended experiments - big.io = io.override { enableExtended = true; }; - }; - - osu = rec { - latency = callPackage ./osu/latency.nix { }; - latencyShm = latency.override { interNode = false; }; - latencyMt = latency.override { enableMultithread = true; }; - latencyMtShm = latency.override { enableMultithread = true; interNode = true; }; - bw = callPackage ./osu/bw.nix { }; - impi = callPackage ./osu/impi.nix { }; - bwShm = bw.override { interNode = false; }; - mtu = callPackage ./osu/mtu.nix { }; - eager = callPackage ./osu/eager.nix { }; - }; - - examples = { - granularity = callPackage ./examples/granularity.nix { }; - }; - - cn6 = { - timediff = callPackage ./cn6/timediff.nix { }; - nbody = callPackage ./cn6/nbody.nix { }; - }; -} diff --git a/garlic/exp/lulesh/test.nix b/garlic/exp/lulesh/test.nix deleted file mode 100644 index eb9b078..0000000 --- a/garlic/exp/lulesh/test.nix +++ /dev/null @@ -1,85 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - - # Initial variable configuration - varConf = with bsc; { - gitBranch = [ - "garlic/mpi+isend+seq" - "garlic/tampi+isend+oss+taskloop" - "garlic/tampi+isend+oss+taskfor" - "garlic/tampi+isend+oss+task" - "garlic/mpi+isend+seq" - "garlic/mpi+isend+oss+task" - "garlic/mpi+isend+omp+fork" - "garlic/tampi+isend+oss+taskloopfor" - ]; - }; - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "lulesh"; - unitName = "${expName}-test"; - inherit (machineConfig) hw; - - # options - iterations = 10; - size = 30; - gitBranch = c.gitBranch; - - # Repeat the execution of each unit several times - loops = 10; - - # Resources - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - /* Lulesh options: - -q : quiet mode - suppress all stdout - -i : number of cycles to run - -s : length of cube mesh along side - -r : Number of distinct regions (def: 11) - -b : Load balance between regions of a domain (def: 1) - -c : Extra cost of more expensive regions (def: 1) - -f : Number of files to split viz dump into (def: (np+10)/9) - -p : Print out progress - -v : Output viz file (requires compiling with -DVIZ_MESH - -h : This message - */ - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-i" iterations "-s" size ]; - }; - - apps = bsc.garlic.apps; - - program = {nextStage, conf, ...}: apps.lulesh.override { - inherit (conf) gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/common.nix b/garlic/exp/nbody/common.nix deleted file mode 100644 index 8570332..0000000 --- a/garlic/exp/nbody/common.nix +++ /dev/null @@ -1,37 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, stages -, numactl -, garlicTools -}: - -with lib; -with garlicTools; - -rec { - getConfigs = {varConf, genConf}: stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec - ( - { - inherit nextStage; - argv = with conf; [ "-t" timesteps "-p" particles ]; - } - # Use numactl to use the interleave policy if requested (default is - # false) - // optionalAttrs (conf.interleaveMem or false) { - program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}"; - } - ); - - program = {nextStage, conf, ...}: bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; -} diff --git a/garlic/exp/nbody/granularity.nix b/garlic/exp/nbody/granularity.nix deleted file mode 100644 index 4b59bae..0000000 --- a/garlic/exp/nbody/granularity.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 64 2048; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - gitBranch = c.gitBranch; - - expName = "nbody-granularity"; - unitName = expName + - "-${toString gitBranch}" + - "-bs${toString blocksize}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; - nodes = 1; - time = "02:00:00"; - jobName = unitName; - }; - - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/numa.nix b/garlic/exp/nbody/numa.nix deleted file mode 100644 index a860fdc..0000000 --- a/garlic/exp/nbody/numa.nix +++ /dev/null @@ -1,64 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, numactl -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = range2 256 1024; - gitBranch = [ "garlic/tampi+send+oss+task" ]; - attachToSocket = [ true false ]; - interleaveMem = [ true false ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) attachToSocket interleaveMem gitBranch blocksize; - - expName = "nbody-numa"; - unitName = expName + - "-${toString gitBranch}" + - "-bs.${toString blocksize}" + - "-tpn.${toString ntasksPerNode}" + - "-interleave.${if (interleaveMem) then "yes" else "no"}"; - - loops = 10; - - qos = "debug"; - cpusPerTask = if (attachToSocket) - then hw.cpusPerSocket - else hw.cpusPerNode; - ntasksPerNode = if (attachToSocket) - then hw.socketsPerNode - else 1; - nodes = 4; - time = "02:00:00"; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/granularity-oss.nix b/garlic/exp/nbody/old/granularity-oss.nix deleted file mode 100644 index 2dbe81c..0000000 --- a/garlic/exp/nbody/old/granularity-oss.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Initial variable configuration - varConf = { - blocksize = [ 128 256 512 1024 2048 4096 ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 4096 * hw.cpusPerSocket; - timesteps = 10; - blocksize = c.blocksize; - - gitBranch = "garlic/oss+task"; - expName = "nbody-granularity"; - unitName = expName + "-bs${toString blocksize}"; - - loops = 30; - - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - argv = [ "-t" conf.timesteps "-p" conf.particles ]; - }; - - program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { - inherit (conf) blocksize gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/nblocks.nix b/garlic/exp/nbody/old/nblocks.nix deleted file mode 100644 index 4e7dd94..0000000 --- a/garlic/exp/nbody/old/nblocks.nix +++ /dev/null @@ -1,104 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 7 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, gitBranch ? "garlic/tampi+send+oss+task" -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-nblocks"; - unitName = "${self.expName}${toString self.nblocks}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps gitBranch enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - nodes = 1; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/scaling.nix b/garlic/exp/nbody/old/scaling.nix deleted file mode 100644 index 47fd221..0000000 --- a/garlic/exp/nbody/old/scaling.nix +++ /dev/null @@ -1,110 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools - -# Options for the experiment -, enableCTF ? false -# Number of cases tested -, steps ? 6 -# nbody iterations -, timesteps ? 10 -# nbody total number of particles -, particles ? null -, loops ? 10 -, nblocks0 ? null -}: - -with lib; -with garlicTools; - -let - - defaultOpt = var: def: if (var != null) then var else def; - - machineConfig = targetMachine.config; - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list with values 2^n with n from 0 to (steps - 1) inclusive - i = expRange 2 0 (steps - 1); - nodes = [ 1 2 4 8 16 ]; - gitBranch = [ - "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = var: fix (self: var // targetMachine.config // { - expName = "nbody-scaling"; - unitName = self.expName + - "-nb${toString self.nblocks}"+ - "-nodes${toString self.nodes}"; - - inherit (machineConfig) hw; - - # nbody options - particles = defaultOpt particles (4096 * self.hw.cpusPerSocket); - nblocks0 = defaultOpt nblocks0 (self.hw.cpusPerSocket / 2); - # The number of blocks is then computed from the multiplier "i" and - # the initial number of blocks "nblocks0" - nblocks = self.i * self.nblocks0; - - totalTasks = self.ntasksPerNode * self.nodes; - particlesPerTask = self.particles / self.totalTasks; - blocksize = self.particlesPerTask / self.nblocks; - cc = bsc.icc; - mpi = bsc.impi; - cflags = "-g"; - inherit timesteps enableCTF loops; - - # Resources - qos = "debug"; - cpusPerTask = self.hw.cpusPerSocket; - ntasksPerNode = self.hw.socketsPerNode; - jobName = self.unitName; - }); - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - perf = {nextStage, conf, ...}: with conf; stages.perf { - inherit nextStage; - perfOptions = "record --call-graph dwarf -o \\$\\$.perf"; - }; - - ctf = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf,\ - instrument.ctf.converter.enabled=false" - ''; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override ({ - inherit (conf) cc blocksize mpi gitBranch cflags; - }); - - pipeline = stdexp.stdPipeline ++ [ ctf exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/strong-scaling-oss.nix b/garlic/exp/nbody/old/strong-scaling-oss.nix deleted file mode 100644 index 2a6f789..0000000 --- a/garlic/exp/nbody/old/strong-scaling-oss.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcsAndParticles = [ 1 2 4 8 16 32 48 ]; - input = [ - { numParticles=1 ; cpuMask="0x1"; } - { numParticles=2 ; cpuMask="0x3"; } - { numParticles=4 ; cpuMask="0xf"; } - { numParticles=8 ; cpuMask="0xff"; } - { numParticles=16; cpuMask="0xffff"; } - { numParticles=32; cpuMask="0xffffffff"; } - { numParticles=48; cpuMask="0xffffffffffff"; } - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - inherit (c.input) numParticles cpuMask; - particles = 1024 * numParticles * 2; - timesteps = 10; - blocksize = 1024; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${toString numParticles}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-mpi.nix b/garlic/exp/nbody/old/weak-scaling-mpi.nix deleted file mode 100644 index e830a42..0000000 --- a/garlic/exp/nbody/old/weak-scaling-mpi.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - numProcs = [ 1 2 4 8 16 32 48 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) numProcs; - cc = icc; - mpi = impi; - gitBranch = "garlic/mpi+send"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = numProcs; - nodes = 1; - time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "nbody-bs-${toString numProcs}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/old/weak-scaling-oss.nix b/garlic/exp/nbody/old/weak-scaling-oss.nix deleted file mode 100644 index 4c7e574..0000000 --- a/garlic/exp/nbody/old/weak-scaling-oss.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with lib; - -let - # Initial variable configuration - varConf = with bsc; { - cpuMask = [ "0x1" "0x3" "0xf" "0xff" "0xffff" "0xffffffff" "0xffffffffffff" ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; - timesteps = 10; - blocksize = 1024; - inherit (c) cpuMask; - cc = icc; - mpi = impi; - gitBranch = "garlic/oss+task"; - - # Repeat the execution of each unit 30 times - loops = 30; - - # Resources - qos = "debug"; - ntasksPerNode = 1; - nodes = 1; - time = "02:00:00"; - cpuBind = "verbose,mask_cpu:${cpuMask}"; - jobName = "nbody-bs-${cpuMask}-${gitBranch}"; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; - }; - - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/nbody/ss.nix b/garlic/exp/nbody/ss.nix deleted file mode 100644 index d9ebf07..0000000 --- a/garlic/exp/nbody/ss.nix +++ /dev/null @@ -1,59 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -, callPackage -}: - -with lib; -with garlicTools; - -let - # Initial variable configuration - varConf = { - blocksize = [ 128 ]; - nodes = range2 1 16; - gitBranch = [ -# "garlic/mpi+send+oss+task" -# "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - hw = targetMachine.config.hw; - particles = 8 * 1024 * hw.cpusPerSocket; - timesteps = 10; - - inherit (c) blocksize nodes gitBranch; - - expName = "nbody-scaling"; - unitName = expName + - "-${toString gitBranch}" + - "-nodes${toString nodes}"; - - loops = 5; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; - time = "02:00:00"; - cpusPerTask = hw.cpusPerSocket; - jobName = unitName; - }; - - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - - configs = getConfigs { - inherit varConf genConf; - }; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/bw.nix b/garlic/exp/osu/bw.nix deleted file mode 100644 index e2c3f37..0000000 --- a/garlic/exp/osu/bw.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-bw-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/eager.nix b/garlic/exp/osu/eager.nix deleted file mode 100644 index 69ee939..0000000 --- a/garlic/exp/osu/eager.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 20 24 ]; - PSM2_MTU_KB = [ 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/impi.nix b/garlic/exp/osu/impi.nix deleted file mode 100644 index 97f6283..0000000 --- a/garlic/exp/osu/impi.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - threshold = [ 8000 16000 32000 64000 ]; - #threshold = [ 4096 8192 10240 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - mpi = impi; - cpusPerTask = 1; - time = "00:10:00"; - qos = "debug"; - loops = 10; - expName = "osu-impi-rndv"; - unitName = expName + "-${toString threshold}"; - jobName = expName; - inherit (c) threshold; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export PSM2_MQ_RNDV_SHM_THRESH=${toString conf.threshold} - export PSM2_MQ_RNDV_HFI_THRESH=${toString conf.threshold} - export PSM2_MQ_EAGER_SDMA_SZ=${toString conf.threshold} - #export PSM2_MTU=${toString conf.threshold} - export PSM2_TRACEMASK=0x101 - ''; - - program = "${nextStage}/bin/osu_bw"; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/latency.nix b/garlic/exp/osu/latency.nix deleted file mode 100644 index f58eb60..0000000 --- a/garlic/exp/osu/latency.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages - -# Should we test the network (true) or the shared memory (false)? -, interNode ? true -, enableMultithread ? false -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = if interNode then 2 else 1; - ntasksPerNode = if interNode then 1 else 2; - cpusPerTask = if (enableMultithread) then hw.cpusPerSocket else 1; - time = "00:10:00"; - qos = "debug"; - loops = 30; - expName = "osu-latency-${mpi.name}"; - unitName = expName; - jobName = expName; - inherit (c) mpi; - inherit enableMultithread; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = if (enableMultithread) then - "${nextStage}/bin/osu_latency_mt" - else - "${nextStage}/bin/osu_latency"; - - argv = optionals (enableMultithread) [ - "-t" "${toString conf.cpusPerTask}:${toString conf.cpusPerTask}" - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/mtu.nix b/garlic/exp/osu/mtu.nix deleted file mode 100644 index ca99091..0000000 --- a/garlic/exp/osu/mtu.nix +++ /dev/null @@ -1,84 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -}: - -with builtins; -with lib; - -let - - machineConfig = targetMachine.config; - - # Initial variable configuration - varConf = with bsc; { - sizeKB = range 5 25; - mpi = [ impi ]; - #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; - PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 ]; - PSM2_MTU_KB = [ 8 10 ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - inherit (machineConfig) hw; - nodes = 2; - ntasksPerNode = 1; - cpusPerTask = 1; - time = "00:30:00"; - qos = "debug"; - loops = 10; - iterations = 50000; - #FIXME: Notice the switchover is 16000 and MTU is 10240 - PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; - PSM2_MTU = PSM2_MTU_KB * 1024; - expName = "osu-bw"; - unitName = expName + - "-size.${toString sizeKB}K" + - "-mtu.${toString PSM2_MTU_KB}K" + - "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; - jobName = expName; - inherit (c) mpi sizeKB - PSM2_MQ_EAGER_SDMA_SZ_KB - PSM2_MTU_KB; - - size = sizeKB * 1024; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - - program = "${nextStage}/bin/osu_bw"; - - env = '' - export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} - export PSM2_MTU=${toString PSM2_MTU} - export PSM2_TRACEMASK=0x101 - export PSM2_MQ_PRINT_STATS=-1 - ''; - - argv = [ - "-m" "${toString size}:${toString size}" - "-i" iterations - ]; - }; - - program = {nextStage, conf, ...}: bsc.osumb.override { - # Use the specified MPI implementation - inherit (conf) mpi; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/granularity.nix b/garlic/exp/saiph/granularity.nix deleted file mode 100644 index 30b9b28..0000000 --- a/garlic/exp/saiph/granularity.nix +++ /dev/null @@ -1,132 +0,0 @@ -####################################################################### -# Saiph, granularity experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Granularity experiment configuration: -# Single-core run -# MPI binded to sockets: MPI procs = 2 -# Mesh distributed across third dimension to ensure contiguous -# communications -# --> nbgx = 1, nbgy = 1 -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -####################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - # Variable configurations - varConf = with targetMachine.config; { - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = range2 1 (hw.cpusPerNode * 8); - nblz = [ 8 ]; - sizex = [ 3 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-granularity"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - - jobName = unitName; - - # saiph options - totalTasks = ntasksPerNode * nodes; - nodes = 1; - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = totalTasks; # forcing distribution by last dim - - inherit (c) nblx nbly nblz gitBranch sizex; - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # fix a specific commit - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all - # factors - allConfigs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - # Filter to remove non-desired configurations: - # --> tasks/proc < 0.5 - # --> nblz > 50 - isGoodConfig = c: - let - maxNblz = c.cpusPerTask * 2; - in - ! (c.blocksPerCpu < 0.5 || c.nblz > maxNblz); - - configs = filter (isGoodConfig) allConfigs; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString conf.cpusPerTask} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/ss.nix b/garlic/exp/saiph/ss.nix deleted file mode 100644 index 5ddc47a..0000000 --- a/garlic/exp/saiph/ss.nix +++ /dev/null @@ -1,122 +0,0 @@ -###################################################################################### -# Saiph, scalability experiment: -# -# App:Heat 3D - garlic/tampi+isend+oss+task+simd branch -# App details: -# 3D mesh of ~400*400*400 points -# nbgx = global blocks in the X dimension -# nbgy = global blocks in the Y dimension -# nbgz = global blocks in the Z dimension -# --> nbgx*nbgy*nbgz = global distributed blocks -# nbly = local blocks in the Y dimension -# nblz = local blocks in the Z dimension -# --> nbly*nblz = local blocks (#tasks) -# -# Scalability experiment configuration: -# From a single-core granularity experiment, use a suited local blocking set: -# --> nbly*nblz >= 48 (at least 3tasks/proc) -# MPI binded to sockets: MPI procs = 2*nodes -# Mesh distributed across third dimension to ensure contiguous communications -# --> nbgx = 1, nbgy = 1 -# Global distribution limited by global mesh size -# First dimension cannot be locally blocked (simd reasons) -# Second and third dimension local blocking limited by local mesh size -# -###################################################################################### - -# Common packages, tools and options -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - #*** Variable configurations *** - varConf = with targetMachine.config; { - # FIXME: None of those selected nbl* and problem size is able to give good - # efficiency when testing strong scaling. We should find better values. - # Local blocks per dimension - nblx = [ 1 ]; # SIMD - nbly = [ 32 ]; - nblz = [ 8 ]; - sizex = [ 3 6 ]; - gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; - nodes = range2 1 8; - }; - - #*** Generate the complete configuration for each unit *** - genConf = c: targetMachine.config // rec { - - # Experiment, units and job names - expName = "saiph-ss"; - unitName = "${expName}" - + "-N${toString nodes}" - + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" - + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; - jobName = unitName; - - # saiph options - enableManualDist = true; # allows to manually set nbg{x-y-z} - nbgx = 1; - nbgy = 1; - nbgz = nodes * ntasksPerNode; # forcing distribution by last dim - - inherit (c) nblx nbly nblz nodes sizex; - - gitBranch = "garlic/tampi+isend+oss+task+simd"; - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; # fix a specific commit - - blocksPerTask = nblx * nbly * nblz * 1.0; - blocksPerCpu = blocksPerTask / cpusPerTask; - - # Repeat the execution of each unit 10 times - loops = 10; - - # Resources - inherit (targetMachine.config) hw; - - qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket - }; - - #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all factors - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = '' - export OMP_NUM_THREADS=${toString hw.cpusPerSocket} - ''; - }; - - #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: bsc.apps.saiph.override { - inherit (conf) enableManualDist - nbgx nbgy nbgz nblx nbly nblz - sizex - gitBranch gitCommit; - - L3SizeKB = conf.hw.cacheSizeKB.L3; - cachelineBytes = conf.hw.cachelineBytes; - }; - - #*** Add stages to the pipeline *** - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/cpu.nix b/garlic/exp/slurm/cpu.nix deleted file mode 100644 index d2ee632..0000000 --- a/garlic/exp/slurm/cpu.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = with bsc; { - # Create a list of cpus per task by computing the divisors of the number of - # cpus per socket, example: divisors 24 = [ 1 2 3 4 6 8 12 24 ] - cpusPerTask = divisors hw.cpusPerSocket; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "cpu"; - unitName = "${expName}.${toString cpusPerTask}"; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - - # Resources - qos = "debug"; - inherit (c) cpusPerTask; - # As cpusPerTask is a divisor of the cpusPerSocket and thus cpusPerNode, we - # know the remainder is zero: - ntasksPerNode = hw.cpusPerNode / cpusPerTask; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - }; - - program = {nextStage, conf, ...}: bsc.dummy; - - pipeline = stdexp.stdPipeline ++ [ program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/exit1.nix b/garlic/exp/slurm/exit1.nix deleted file mode 100644 index bf60136..0000000 --- a/garlic/exp/slurm/exit1.nix +++ /dev/null @@ -1,69 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - script = [ - "exit 1" - "exit 0" - "kill -SEGV $$" - "kill -TERM $$" - ]; - }; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "exit1"; - unitName = expName + "-" + - builtins.replaceStrings [" " "$"] ["-" "-"] script; - - inherit (machineConfig) hw; - - # Repeat the execution of each unit 30 times - loops = 1; - inherit (c) script; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = 2; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - pre = "sleep 5"; - post = "echo dummy"; - }; - - prog = {conf,...}: stages.script { - inherit (conf) script; - }; - - pipeline = stdexp.stdPipeline ++ [ exec prog ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/slurm/sigsegv.nix b/garlic/exp/slurm/sigsegv.nix deleted file mode 100644 index b77f58b..0000000 --- a/garlic/exp/slurm/sigsegv.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, stdexp -, bsc -, targetMachine -, stages -, garlicTools -}: - -with lib; -with garlicTools; - -let - - machineConfig = targetMachine.config; - - inherit (machineConfig) hw; - - # Initial variable configuration - varConf = { - when = [ "before" "after" "never" ]; - }; - - # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { - expName = "sigsegv"; - unitName = expName + "-" + when; - - inherit (machineConfig) hw; - inherit (c) when; - - loops = 3; - - # Resources - qos = "debug"; - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - nodes = 1; - jobName = unitName; - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: with conf; stages.exec { - inherit nextStage; - env = "date"; - argv = [ conf.when ]; - }; - - program = {nextStage, conf, ...}: bsc.test.sigsegv; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/fig/creams/granularity.R b/garlic/fig/creams/granularity.R deleted file mode 100644 index 524c987..0000000 --- a/garlic/fig/creams/granularity.R +++ /dev/null @@ -1,83 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.iter = time / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodesFactor = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.iter = median(time.iter)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="granul", y="Normalized time", - title="Creams granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(x="granul", y="Time (s)", title="Creams granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/creams/granularity16.R b/garlic/fig/creams/granularity16.R deleted file mode 100644 index 24f3bf7..0000000 --- a/garlic/fig/creams/granularity16.R +++ /dev/null @@ -1,136 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(median.time.elem = median(time.elem)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(log.median.time.elem = log(median.time.elem)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() %>% - group_by(sizeFactor) %>% - mutate(optimal.granul = (median.time.elem == min(median.time.elem))) %>% - ungroup() - -dfopt = df %>% filter(optimal.granul == TRUE) - -dpi = 300 -h = 4 -w = 10 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + - facet_wrap(sizeFactor ~ ., labeller=label_both, nrow=1) + - labs(x="Granularity", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=median.time.elem, color=sizeFactor)) + - geom_line(aes(group=sizeFactor)) + - geom_point(data=dfopt, aes(x=granul, y=median.time.elem)) + - theme_bw() + - labs(x="Granularity", y="Time / k (s)", - color="Size factor k", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("median.time.png", plot=p, width=5, height=5, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=5, height=5, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=granul, y=sizeFactor, fill=log.median.time.elem)) + - geom_raster() + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="Granularity", y="Size factor", fill="Time / k (s)", subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -k=1 -ggsave("heatmap.png", plot=p, width=4.8*k, height=5*k, dpi=300) -ggsave("heatmap.pdf", plot=p, width=4.8*k, height=5*k, dpi=300) diff --git a/garlic/fig/creams/size.R b/garlic/fig/creams/size.R deleted file mode 100644 index 21e3335..0000000 --- a/garlic/fig/creams/size.R +++ /dev/null @@ -1,96 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - config.sizeFactor, - config.nz, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - sizeFactor=config.sizeFactor, - nz=config.nz, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.elem = time / sizeFactor) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(sizeFactor = as.factor(sizeFactor)) %>% - mutate(nz = as.factor(nz)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 3 -w = 6 - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + -# geom_boxplot() + -# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + -# theme_bw() + -# facet_wrap(branch ~ .) + -# labs(x="nodes", y="Normalized time", -# title="Creams strong scaling: normalized time", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=sizeFactor, y=time.elem, color=branch)) + - geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="Size factor k (nz=k*3072)", y="Time / k (s)", - #title="Creams size: time per object", - subtitle=output) + - theme(plot.subtitle=element_text(size=8, family="mono"), - legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/creams/ss.R b/garlic/fig/creams/ss.R deleted file mode 100644 index b367781..0000000 --- a/garlic/fig/creams/ss.R +++ /dev/null @@ -1,128 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.gitBranch, - config.granul, - config.iterations, - time, - total_time) %>% - - rename(nodes=config.nodes, - gitBranch=config.gitBranch, - granul=config.granul, - iterations=config.iterations) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.iter = time.nodes / iterations) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(granul = as.factor(granul)) %>% - mutate(iterations = as.factor(iterations)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=granul, color=iterations)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title="Creams strong scaling: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", title="Creams strong scaling: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=median.time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Median time * nodes (s)", title="Creams strong scaling: median time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("median.time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_boxplot() + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", title="Creams strong scaling: time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.boxplot.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.boxplot.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -#p = ggplot(df, aes(x=nodes, y=time.nodes.iter, color=branch)) + -# geom_point(shape=21, size=3) + -# geom_line(aes(y=median.time.nodes.iter, group=interaction(granul,iterations))) + -# theme_bw() + -# #facet_wrap(branch ~ .) + -# labs(x="nodes", y="Time * nodes / iterations (s)", -# title="Creams strong scaling: time * nodes / iterations", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) -# -#ggsave("time.nodes.iter.png", plot=p, width=w, height=h, dpi=dpi) -#ggsave("time.nodes.iter.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/dev/shell.nix b/garlic/fig/dev/shell.nix deleted file mode 100644 index 840c83e..0000000 --- a/garlic/fig/dev/shell.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ pkgs ? import ../../../default.nix }: - -with pkgs; - -let - rWrapper = pkgs.rWrapper.override { - packages = with pkgs.rPackages; [ tidyverse rjson jsonlite egg ]; - }; -in -stdenv.mkDerivation { - name = "R"; - - buildInputs = [ rWrapper ]; -} diff --git a/garlic/fig/examples/granularity.R b/garlic/fig/examples/granularity.R deleted file mode 100644 index 7577dca..0000000 --- a/garlic/fig/examples/granularity.R +++ /dev/null @@ -1,195 +0,0 @@ -# This R program takes as argument the dataset that contains the results of the -# execution of the heat example experiment and produces some plots. All the -# knowledge to understand how this script works is covered by this nice R book: -# -# Winston Chang, R Graphics Cookbook: Practical Recipes for Visualizing Data, -# O’Reilly Media (2020). 2nd edition -# -# Which can be freely read it online here: https://r-graphics.org/ -# -# Please, search in this book before copying some random (and probably oudated) -# reply on stack overflow. - -# We load some R packages to import the required functions. We mainly use the -# tidyverse packages, which are very good for ploting data, -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Here we simply load the arguments to find the input dataset. If nothing is -# specified we use the file named `input` in the current directory. -# We can run this script directly using: -# Rscript - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Here we build of dataframe from the input dataset by chaining operations using -# the magritte operator `%>%`, which is similar to a UNIX pipe. -# First we read the input file, which is expected to be NDJSON -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - # Then we flatten it, as it may contain dictionaries inside the columns - jsonlite::flatten() %>% - - # Now the dataframe contains all the configuration of the units inside the - # columns named `config.*`, for example `config.cbs`. We first select only - # the columns that we need: - select(config.cbs, config.rbs, unit, time) %>% - - # And then we rename those columns to something shorter: - rename(cbs=config.cbs, rbs=config.rbs) %>% - - # The columns contain the values that we specified in the experiment as - # integers. However, we need to tell R that those values are factors. So we - # apply to those columns the `as.factor()` function: - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - - # The same for the unit (which is the hash that nix has given to each unit) - mutate(unit = as.factor(unit)) %>% - - # Then, we can group our dataset by each unit. This will always work - # independently of the variables that vary from unit to unit. - group_by(unit) %>% - - # And compute some metrics which are applied to each group. For example we - # compute the median time within the runs of a unit: - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - # Then, we remove the grouping. This step is very important, otherwise the - # plotting functions get confused: - ungroup() - - -# These constants will be used when creating the plots. We use high quality -# images with 300 dots per inch and 5 x 5 inches of size by default. -dpi = 300 -h = 5 -w = 5 - - -# --------------------------------------------------------------------- - - -# We plot the median time (of each unit) as we vary the block size. As we vary -# both the cbs and rbs, we plot cbs while fixing rbs at a time. -p = ggplot(df, aes(x=cbs, y=median.time, color=rbs)) + - # We add a point to the median - geom_point() + - - # We also add the lines to connect the points. We need to specify which - # variable will do the grouping, otherwise we will have one line per point. - geom_line(aes(group=rbs)) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Median time (s)", title="Heat granularity: median time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# Another interesting plot is the normalized time, which shows the variance of -# the execution times, and can be used to find problems: -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - - # The boxplots are useful to identify outliers and problems with the - # distribution of time - geom_boxplot() + - - # We add a line to mark the 1% limit above and below the median - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - - # We split the plot into subplots, one for each value of the rbs column - facet_wrap(~ rbs) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Normalized time", title="Heat granularity: normalized time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We plot the time of each run as we vary the block size -p = ggplot(df, aes(x=cbs, y=time, color=rbs)) + - - # We add a points (scatter plot) using circles (shape=21) a bit larger - # than the default (size=3) - geom_point(shape=21, size=3) + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="Time (s)", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - - -# --------------------------------------------------------------------- - - -# We can also plot both cbs and rbs in each dimension by mapping the time with a -# color. The `fill` argument instruct R to use the `median.time` as color -p = ggplot(df, aes(x=cbs, y=rbs, fill=median.time)) + - - # Then we use the geom_raster method to paint rectangles filled with color - geom_raster() + - - # The colors are set using the viridis package, using the plasma palete. Those - # colors are designed to be safe for color impaired people and also when - # converting the figures to grayscale. - scale_fill_viridis(option="plasma") + - - # We also force each tile to be an square - coord_fixed() + - - # The bw theme is recommended for publications - theme_bw() + - - # Here we add the title and the labels of the axes - labs(x="cbs", y="rbs", title="Heat granularity: time", - subtitle=output) + - - # And set the subtitle font size a bit smaller, so it fits nicely - theme(plot.subtitle=element_text(size=8)) - -# Then, we save the plot both in png and pdf -ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R deleted file mode 100644 index da9f1a4..0000000 --- a/garlic/fig/fwi/granularity.R +++ /dev/null @@ -1,74 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=branch)) + - theme_bw() + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(legend.position="bottom") + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/fwi/io.R b/garlic/fig/fwi/io.R deleted file mode 100644 index e10e3dc..0000000 --- a/garlic/fig/fwi/io.R +++ /dev/null @@ -1,120 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(forcats) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.blocksize, config.ioFreq, config.gitBranch, config.nodes, time, unit) %>% - rename( - blocksize=config.blocksize, - enableIO=config.enableIO, - gitBranch=config.gitBranch, - nodes=config.nodes - ) %>% - filter(blocksize == 1) %>% - group_by(unit) %>% - mutate(mtime = median(time)) %>% - mutate(nxmtime = mtime * nodes) %>% - mutate(nxtime = time * nodes) %>% - ungroup() - -df$gitBranch = as.factor(df$gitBranch) -df$enableIO = as.factor(df$enableIO) -df$blocksize = as.factor(df$blocksize) -df$nodes = as.factor(df$nodes) - -ppi=300 -h=5 -w=5 - -#################################################################### -### Line plot (time) -#################################################################### -png("time.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=time, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -#################################################################### -### Line plot (time x nodes) -#################################################################### -png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi) - -p = ggplot(df, aes(x=nodes, y=nxtime, group=enableIO, color=enableIO)) + - geom_point() + - geom_line() + - theme_bw() + - labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling for mpi+send+oss+task", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) - -# Render the plot -print(p) - -# Save the png image -dev.off() - -##################################################################### -#### Line plot (median time) -##################################################################### -#png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() -# -##################################################################### -#### Line plot (nodes x median time) -##################################################################### -#png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi) -# -#p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) + -# geom_point() + -# geom_line() + -# theme_bw() + -# labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling", -# subtitle=output) + -# theme(plot.subtitle=element_text(size=8)) + -# theme(legend.position = c(0.5, 0.88)) -# -## Render the plot -#print(p) -# -## Save the png image -#dev.off() diff --git a/garlic/fig/fwi/ss.R b/garlic/fig/fwi/ss.R deleted file mode 100644 index 428d5c1..0000000 --- a/garlic/fig/fwi/ss.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.blocksize, - config.gitBranch, - config.nodes, - time) %>% - - rename(blocksize=config.blocksize, - nodes=config.nodes, - gitBranch=config.gitBranch) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(unit = as.factor(unit)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - ungroup() - - -dpi = 300 -h = 6 -w = 6 - -main_title = "FWI strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=gitBranch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=gitBranch)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(y="Time (s)", - title=sprintf("%s: time", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", main_title), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/cache.R b/garlic/fig/heat/cache.R deleted file mode 100644 index dca6baf..0000000 --- a/garlic/fig/heat/cache.R +++ /dev/null @@ -1,78 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, perf.cache_misses, perf.instructions, perf.cycles, time) %>% - rename(cbs=config.cbs, rbs=config.rbs) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df=group_by(df, cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.misses = median(perf.cache_misses)) %>% - mutate(log.median.misses = log(median.misses)) %>% - mutate(median.instr= median(perf.instructions)) %>% - mutate(log.median.instr= log(median.instr)) %>% - mutate(median.cycles = median(perf.cycles)) %>% - mutate(median.cpi = median.cycles / median.instr) %>% - mutate(median.ipc = median.instr / median.cycles) %>% - mutate(median.ips = median.instr / median.time) %>% - mutate(median.cps = median.cycles / median.time) %>% - ungroup()# %>% - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.misses", "cache misses") -heatmap_plot(df, "log.median.misses", "cache misses") -heatmap_plot(df, "median.instr", "instructions") -heatmap_plot(df, "log.median.instr", "instructions") -heatmap_plot(df, "median.cycles", "cycles") -heatmap_plot(df, "median.ipc", "IPC") -heatmap_plot(df, "median.cpi", "cycles/instruction") -heatmap_plot(df, "median.ips", "instructions/second") -heatmap_plot(df, "median.cps", "cycles/second") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df_filtered, "median.time", "execution time (seconds)") -heatmap_plot(df_filtered, "log.median.time", "execution time") diff --git a/garlic/fig/heat/granularity.R b/garlic/fig/heat/granularity.R deleted file mode 100644 index 19c3f37..0000000 --- a/garlic/fig/heat/granularity.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.cbs, - config.rbs, - time, - total_time) %>% - - rename(cbs=config.cbs, - rbs=config.rbs) %>% - - # Convert to factors - mutate(cbs = as.factor(cbs)) %>% - mutate(rbs = as.factor(rbs)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 6 -w = 6 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(y="Normalized time", - title="Heat granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=cbs, y=time)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=0)) + - theme_bw() + - labs(y="Time (s)", title="Heat granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/heat/mode.R b/garlic/fig/heat/mode.R deleted file mode 100644 index 5e409bc..0000000 --- a/garlic/fig/heat/mode.R +++ /dev/null @@ -1,122 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(viridis) -library(tidyr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.cbs, config.rbs, - ctf_mode.runtime, - ctf_mode.task, - ctf_mode.dead, - config.cpusPerTask, - time) %>% - rename( - cbs=config.cbs, - rbs=config.rbs, - runtime=ctf_mode.runtime, - task=ctf_mode.task, - dead=ctf_mode.dead, - cpusPerTask=config.cpusPerTask, - ) - -df$cbs = as.factor(df$cbs) -df$rbs = as.factor(df$rbs) - -# Normalize the time by the median -df = df %>% - mutate(runtime = runtime * 1e-9 / cpusPerTask) %>% - mutate(dead = dead * 1e-9 / cpusPerTask) %>% - mutate(task = task * 1e-9 / cpusPerTask) %>% - group_by(cbs, rbs) %>% - mutate(median.time = median(time)) %>% - mutate(log.median.time = log(median.time)) %>% - mutate(median.dead = median(dead)) %>% - mutate(median.runtime = median(runtime)) %>% - mutate(median.task = median(task)) %>% - ungroup() #%>% - -print(df) - -heatmap_plot = function(df, colname, title) { - p = ggplot(df, aes(x=cbs, y=rbs, fill=!!ensym(colname))) + - geom_raster() + - #scale_fill_gradient(high="black", low="white") + - scale_fill_viridis(option="plasma") + - coord_fixed() + - theme_bw() + - theme(axis.text.x=element_text(angle = -45, hjust = 0)) + - theme(plot.subtitle=element_text(size=8)) + - #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + - guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + - labs(x="cbs", y="rbs", - title=sprintf("Heat granularity: %s", title), - subtitle=output) + - theme(legend.position="bottom") - - k=1 - ggsave(sprintf("%s.png", colname), plot=p, width=4.8*k, height=5*k, dpi=300) - ggsave(sprintf("%s.pdf", colname), plot=p, width=4.8*k, height=5*k, dpi=300) -} - -heatmap_plot(df, "median.runtime", "runtime") -heatmap_plot(df, "median.dead", "not used") -heatmap_plot(df, "median.task", "task") - -cutlevel = 0.5 -# To plot the median.time we crop the larger values: -df_filtered = filter(df, between(median.time, - median(time) - (cutlevel * sd(time)), - median(time) + (cutlevel * sd(time)))) - -heatmap_plot(df, "median.time", "execution time (seconds)") -heatmap_plot(df, "log.median.time", "execution time") - -df_square = filter(df, cbs == rbs) %>% - gather(key = time.from, value = acc.time, - c("median.dead", "median.runtime", "median.task")) - -# Colors similar to Paraver -colors <- c("median.dead" = "gray", - "median.runtime" = "blue", - "median.task" = "red") - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_area(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - geom_point(aes(y=median.time, color="black")) + - geom_line(aes(y=median.time, group=0, color="black")) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - scale_color_identity(breaks = c("black"), - labels = c("Total time"), guide = "legend") + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("area.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("area.time.pdf", plot=p, width=6, height=6, dpi=300) - -p = ggplot(df_square, aes(x=cbs, y=acc.time)) + - geom_col(aes(fill=time.from, group=time.from)) + - scale_fill_manual(values = colors) + - theme_bw() + - theme(legend.position=c(0.5, 0.7)) + - labs(x="Blocksize (side)", y="Time (s)", - fill="Estimated", color="Direct measurement", - title="Heat granularity: time distribution", subtitle=output) - -ggsave("col.time.png", plot=p, width=6, height=6, dpi=300) -ggsave("col.time.pdf", plot=p, width=6, height=6, dpi=300) diff --git a/garlic/fig/hpcg/granularity.R b/garlic/fig/hpcg/granularity.R deleted file mode 100644 index 29126da..0000000 --- a/garlic/fig/hpcg/granularity.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.ncomms, - config.hw.cpusPerSocket, - config.blocksPerCpu, - unit, - time) %>% - - rename(nblocks=config.nblocks, - ncomms=config.ncomms, - blocksPerCpu=config.blocksPerCpu) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=blocksPerCpu, y=normalized.time)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - labs(x="Blocks per CPU", y="Normalized time", title="HPCG granularity: normalized time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=blocksPerCpu, y=time)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Blocks per CPU", y="Time (s)", title="HPCG granularity: time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - diff --git a/garlic/fig/hpcg/size.R b/garlic/fig/hpcg/size.R deleted file mode 100644 index a7cc5f1..0000000 --- a/garlic/fig/hpcg/size.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.nodes = time * nodes) %>% - mutate(time.nodes.elem = time.nodes / sizePerCpu.z) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=5 -w=5 - -p = ggplot(df, aes(x=sizePerCpu.z, y=time.nodes.elem)) + - geom_point(shape=21, size=3) + - theme_bw() + - labs(x="Size per CPU in Z", y="Time * nodes / spcz (s)", - title="HPCG size: time * nodes / spcz", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ss.R b/garlic/fig/hpcg/ss.R deleted file mode 100644 index 3978618..0000000 --- a/garlic/fig/hpcg/ss.R +++ /dev/null @@ -1,82 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(time.sizeZ = time / sizePerCpu.z) %>% - mutate(time.nodes = time * nodes) %>% - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time.nodes)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", - title="HPCG strong scalability in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) - -p = ggplot(df, aes(x=nodes, y=time.sizeZ, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time / npcz (s)", title="HPCG strong scalability in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.size.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.size.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/hpcg/ws.R b/garlic/fig/hpcg/ws.R deleted file mode 100644 index d12f35b..0000000 --- a/garlic/fig/hpcg/ws.R +++ /dev/null @@ -1,71 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -args = commandArgs(trailingOnly=TRUE) - -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.nprocs.x, - config.nprocs.y, - config.nprocs.z, - config.blocksPerCpu, - config.sizePerCpu.z, - unit, - time - ) %>% - - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket, - nodes=config.nodes, - blocksPerCpu=config.blocksPerCpu, - sizePerCpu.z=config.sizePerCpu.z, - npx=config.nprocs.x, - npy=config.nprocs.y, - npz=config.nprocs.z - ) %>% - - mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% - mutate(time.sizeZ = time / sizePerCpu.z) %>% - - mutate(nblocks = as.factor(nblocks)) %>% - mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(unit = as.factor(unit)) %>% - mutate(sizePerCpu.z = as.factor(sizePerCpu.z)) %>% - - mutate(timePerNprocs = time * npz) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi=300 -h=7 -w=7 - -p = ggplot(df, aes(x=nodes, y=time, fill=sizePerCpu.z)) + - geom_boxplot() + - theme_bw() + - labs(x="Nodes", y="Time (s)", title="HPCG weak scaling in Z", - color="Size per CPU in Z", - subtitle=output) + - theme(plot.subtitle=element_text(size=8), - legend.position="bottom") - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix deleted file mode 100644 index 0e7ce41..0000000 --- a/garlic/fig/index.nix +++ /dev/null @@ -1,97 +0,0 @@ -{ - super -, self -, bsc -, garlic -, callPackage -}: - -let - rPlot = garlic.pp.rPlot; - exp = garlic.exp; - pp = garlic.pp; - ds = garlic.ds; - fig = garlic.fig; - - stdPlot = rScript: expList: rPlot { - script = rScript; - dataset = pp.mergeDatasets (map (e: ds.std.timetable e.result) expList); - }; - - customPlot = rScript: dataset: rPlot { - script = rScript; - dataset = dataset; - }; - - linkTree = name: tree: self.linkFarm name ( - self.lib.mapAttrsToList ( - name: value: { name=name; path=value; } - ) tree); -in -{ - nbody = with exp.nbody; { - granularity = stdPlot ./nbody/granularity.R [ granularity ]; - ss = stdPlot ./nbody/ss.R [ ss ]; - numa = stdPlot ./nbody/numa.R [ numa ]; - }; - - hpcg = with exp.hpcg; { - ss = stdPlot ./hpcg/ss.R [ ss ]; - ws = stdPlot ./hpcg/ws.R [ ws ]; - size = stdPlot ./hpcg/size.R [ size ]; - granularity = stdPlot ./hpcg/granularity.R [ granularity ]; - }; - - saiph = with exp.saiph; { - granularity = stdPlot ./saiph/granularity.R [ granularity ]; - ss = stdPlot ./saiph/ss.R [ ss ]; - }; - - heat = with exp.heat; { - granularity = stdPlot ./heat/granularity.R [ granularity ]; - cache = customPlot ./heat/cache.R (ds.perf.stat cache.result); - ctf = customPlot ./heat/mode.R (ds.ctf.mode ctf.result); - }; - - creams = with exp.creams; { - ss = stdPlot ./creams/ss.R [ ss ]; - granularity = stdPlot ./creams/granularity.R [ granularity ]; - size = stdPlot ./creams/size.R [ size ]; - granularity16 = stdPlot ./creams/granularity16.R [ granularity16 ]; - - # Extended version (we could use another R script for those plots - big.ss = stdPlot ./creams/ss.R [ big.ss ]; - big.granularity = stdPlot ./creams/granularity.R [ big.granularity ]; - }; - - fwi = with exp.fwi; { - granularity = stdPlot ./fwi/granularity.R [ granularity ]; - reuse = stdPlot ./fwi/granularity.R [ reuse ]; - ss = stdPlot ./fwi/ss.R [ ss ]; - io = stdPlot ./fwi/io.R [ io ]; - }; - - osu = with exp.osu; { - latency = customPlot ./osu/latency.R (ds.osu.latency latency.result); - latencyShm = customPlot ./osu/latency.R (ds.osu.latency latencyShm.result); - latencyMt = customPlot ./osu/latency.R (ds.osu.latency latencyMt.result); - latencyMtShm = customPlot ./osu/latency.R (ds.osu.latency latencyMtShm.result); - - bw = customPlot ./osu/bw.R (ds.osu.bw bw.result); - bwShm = customPlot ./osu/bw.R (ds.osu.bw bwShm.result); - impi = customPlot ./osu/impi.R (ds.osu.bw impi.result); - mtu = customPlot ./osu/mtu.R (ds.osu.bw mtu.result); - eager = customPlot ./osu/eager.R (ds.osu.bw eager.result); - }; - - # The figures used in the article contained in a directory per figure - article = with fig; linkTree "article-fig" { - "osu/latency" = osu.latency; - "osu/bw" = osu.bw; - "osu/mtu" = osu.mtu; - }; - - examples = with exp.examples; { - granularity = stdPlot ./examples/granularity.R [ granularity ]; - }; -} diff --git a/garlic/fig/nbody/granularity.R b/garlic/fig/nbody/granularity.R deleted file mode 100644 index 85ee0b4..0000000 --- a/garlic/fig/nbody/granularity.R +++ /dev/null @@ -1,62 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.particles, unit, time) %>% - rename(blocksize=config.blocksize, particles=config.particles, branch=config.gitBranch) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(particles = as.factor(particles)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 5 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Blocksize", y="Normalized Time", title="NBody Granularity: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time)) + - geom_boxplot() + - theme_bw() + - labs(x="Blocksize", y="Time (s)", title="NBody Granularity: Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/numa.R b/garlic/fig/nbody/numa.R deleted file mode 100644 index b080648..0000000 --- a/garlic/fig/nbody/numa.R +++ /dev/null @@ -1,86 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(unit, - config.blocksize, - config.gitBranch, - config.attachToSocket, - config.interleaveMem, - config.nodes, - unit, - time) %>% - - rename(blocksize=config.blocksize, - gitBranch=config.gitBranch, - nodes=config.nodes, - attachToSocket=config.attachToSocket, - interleaveMem=config.interleaveMem) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(attachToSocket = as.factor(attachToSocket)) %>% - mutate(interleaveMem = as.factor(interleaveMem)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -branch = unique(df$branch) -nodes = unique(df$nodes) - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=normalized.time, color=interleaveMem)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Normalized time", - title=sprintf("NBody NUMA (%s | %d Nodes): Normalized time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksize, y=time, color=interleaveMem)) + - geom_boxplot() + - geom_line(aes(y=median.time)) + - theme_bw() + - facet_wrap(~ attachToSocket, labeller=label_both) + - labs(x="Blocksize", y="Time (s)", - title=sprintf("NBody NUMA (%s | %d Nodes): Time", - branch, nodes), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position="bottom") - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/nbody/old/baseline.R b/garlic/fig/nbody/old/baseline.R deleted file mode 100644 index 5509d8e..0000000 --- a/garlic/fig/nbody/old/baseline.R +++ /dev/null @@ -1,212 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) -library(egg) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, - config.nblocks, - config.hw.cpusPerSocket, - config.nodes, - config.blocksize, - config.particles, - config.gitBranch, - time) %>% - rename(nblocks=config.nblocks, - nodes=config.nodes, - blocksize=config.blocksize, - particles=config.particles, - gitBranch=config.gitBranch, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$nodesFactor = as.factor(df$nodes) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) -df$blocksizeFactor = as.factor(df$blocksize) -df$particlesFactor = as.factor(df$particles) -df$gitBranch = as.factor(df$gitBranch) - -# Normalize the time by the median -D=group_by(df, nblocks, nodesFactor, gitBranch) %>% - mutate(tmedian = median(time)) %>% - mutate(tn = tmedian * nodes) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) %>% - ungroup() %>% - group_by(nodesFactor, gitBranch) %>% - mutate(tmedian_min = min(tmedian)) %>% - ungroup() %>% - group_by(gitBranch) %>% - mutate(tmin_max = max(tmedian_min)) %>% - mutate(tideal = tmin_max / nodes) %>% - ungroup() - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=7.5 -w=7.5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot(aes(fill=nodesFactor)) + - scale_color_manual(values=c("black", "brown")) + - facet_grid(gitBranch ~ .) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - - #theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - theme_bw()+ - theme(plot.subtitle=element_text(size=8)) - - - - -# Render the plot -print(p) -dev.off() - - -p1 = ggplot(D, aes(x=blocksizeFactor, y=time)) + - - labs(x="Blocksize", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - #theme(legend.position = c(0.5, 0.8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocksize.png", width=w*ppi, height=h*ppi, res=ppi) -print(p1) -dev.off() - -p2 = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - - geom_line(aes(y=tmedian, - group=interaction(gitBranch, nodesFactor), - color=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3, shape=21) + - facet_grid(gitBranch ~ .) + - - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) - -png("time-blocks-per-cpu.png", width=w*ppi, height=h*ppi, res=ppi) -print(p2) -dev.off() - -#p = ggarrange(p1, p2, ncol=2) -#png("time-gra.png", width=2*w*ppi, height=h*ppi, res=ppi) -#print(p) -#dev.off() - - - -png("exp-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=df, aes(x=nodesFactor, y=particlesFactor)) + - labs(x="Nodes", y="Particles", title="Nbody: Experiment space") + - geom_line(aes(group=particles)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("gra-space.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(data=D, aes(x=nodesFactor, y=blocksPerCpuFactor)) + - labs(x="Nodes", y="Blocks/CPU", title="Nbody: Granularity space") + - geom_line(aes(group=nodesFactor)) + - geom_point(aes(color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - theme_bw() -print(p) -dev.off() - - -png("performance.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tmedian, - linetype=blocksPerCpuFactor, - group=interaction(gitBranch, blocksPerCpuFactor))) + - geom_line(aes(y=tideal, group=gitBranch), color="red") + - geom_point(aes(y=tmedian, color=nodesFactor), size=3) + - facet_grid(gitBranch ~ .) + - scale_shape_manual(values=c(21, 22)) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() - - -png("time-nodes.png", width=w*ppi, height=h*ppi, res=ppi) -p = ggplot(D, aes(x=nodesFactor)) + - labs(x="Nodes", y="Time * nodes (s)", title="Nbody strong scaling") + - theme_bw() + - geom_line(aes(y=tn, group=gitBranch)) + - facet_grid(gitBranch ~ .) + - scale_y_continuous(trans=log2_trans()) -print(p) -dev.off() diff --git a/garlic/fig/nbody/old/freeCpu.R b/garlic/fig/nbody/old/freeCpu.R deleted file mode 100644 index cf97106..0000000 --- a/garlic/fig/nbody/old/freeCpu.R +++ /dev/null @@ -1,112 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.freeCpu, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - freeCpu=config.freeCpu, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$freeCpu = as.factor(df$freeCpu) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(freeCpu, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=freeCpu)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans(), -# breaks=bs_unique) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~freeCpu) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time, color=freeCpu)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/jemalloc.R b/garlic/fig/nbody/old/jemalloc.R deleted file mode 100644 index 37fc991..0000000 --- a/garlic/fig/nbody/old/jemalloc.R +++ /dev/null @@ -1,113 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the cpu bind, nblocks and time -df = select(dataset, config.enableJemalloc, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - jemalloc=config.enableJemalloc, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) - -df$jemalloc = as.factor(df$jemalloc) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Split by malloc variant -D=df %>% group_by(jemalloc, nblocks) %>% - mutate(tnorm = time / median(time) - 1) - # Add another column: blocksPerCpu (we assume one task per socket, using - # all CPUs) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=nblocks, y=tnorm)) + - - # Labels - labs(x="Num blocks", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="red") + - - # Draw boxplots - geom_boxplot(aes(fill=jemalloc)) + - -# # Use log2 scale in x -# scale_x_continuous(trans=log2_trans()) + -# - scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - - theme(legend.position = c(0.85, 0.85)) #+ - - # Place each variant group in one separate plot - #facet_wrap(~jemalloc) - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpu, y=time, color=jemalloc)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/old/scaling.R b/garlic/fig/nbody/old/scaling.R deleted file mode 100644 index 1d50a37..0000000 --- a/garlic/fig/nbody/old/scaling.R +++ /dev/null @@ -1,111 +0,0 @@ -library(ggplot2) -library(dplyr) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file)) %>% - jsonlite::flatten() - -particles = unique(dataset$config.particles) - -# We only need the nblocks and time -df = select(dataset, config.nblocks, config.hw.cpusPerSocket, time) %>% - rename(nblocks=config.nblocks, - cpusPerSocket=config.hw.cpusPerSocket) - -df = df %>% mutate(blocksPerCpu = nblocks / cpusPerSocket) -df$nblocks = as.factor(df$nblocks) -df$blocksPerCpuFactor = as.factor(df$blocksPerCpu) - -# Normalize the time by the median -D=group_by(df, nblocks) %>% - mutate(tnorm = time / median(time) - 1) %>% - mutate(bad = max(ifelse(abs(tnorm) >= 0.01, 1, 0))) - -D$bad = as.factor(D$bad) - -#D$bad = as.factor(ifelse(abs(D$tnorm) >= 0.01, 2, -# ifelse(abs(D$tnorm) >= 0.005, 1, 0))) - -bs_unique = unique(df$nblocks) -nbs=length(bs_unique) - -print(D) - -ppi=300 -h=5 -w=5 - -png("box.png", width=w*ppi, height=h*ppi, res=ppi) -# -# -# -# Create the plot with the normalized time vs nblocks -p = ggplot(data=D, aes(x=blocksPerCpuFactor, y=tnorm, color=bad)) + - - # Labels - labs(x="Blocks/CPU", y="Normalized time", - title=sprintf("Nbody normalized time. Particles=%d", particles), - subtitle=output) + - - - # Center the title - #theme(plot.title = element_text(hjust = 0.5)) + - - # Black and white mode (useful for printing) - #theme_bw() + - - # Add the maximum allowed error lines - geom_hline(yintercept=c(-0.01, 0.01), - linetype="dashed", color="gray") + - - # Draw boxplots - geom_boxplot() + - scale_color_manual(values=c("black", "brown")) + - - #scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) + - - theme_bw() + - - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = "none") - #theme(legend.position = c(0.85, 0.85)) - - - - -# Render the plot -print(p) - -## Save the png image -dev.off() -# -png("scatter.png", width=w*ppi, height=h*ppi, res=ppi) -# -## Create the plot with the normalized time vs nblocks -p = ggplot(D, aes(x=blocksPerCpuFactor, y=time)) + - - labs(x="Blocks/CPU", y="Time (s)", - title=sprintf("Nbody granularity. Particles=%d", particles), - subtitle=output) + - theme_bw() + - theme(plot.subtitle=element_text(size=8)) + - theme(legend.position = c(0.5, 0.88)) + - - geom_point(shape=21, size=3) + - #scale_x_continuous(trans=log2_trans()) + - scale_y_continuous(trans=log2_trans()) - -# Render the plot -print(p) - -# Save the png image -dev.off() diff --git a/garlic/fig/nbody/ss.R b/garlic/fig/nbody/ss.R deleted file mode 100644 index 2ce9b2c..0000000 --- a/garlic/fig/nbody/ss.R +++ /dev/null @@ -1,66 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) - -# Load the arguments (argv) -args = commandArgs(trailingOnly=TRUE) -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() %>% - select(config.blocksize, config.gitBranch, config.nodes, unit, time) %>% - rename(nodes = config.nodes, blocksize=config.blocksize, branch=config.gitBranch) %>% - - mutate(time.nodes = time * nodes) %>% - - mutate(blocksize = as.factor(blocksize)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(branch = as.factor(branch)) %>% - mutate(unit = as.factor(unit)) %>% - - group_by(unit) %>% - - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - - ungroup() - -dpi = 300 -h = 5 -w = 8 - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, color=branch)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - facet_wrap(~ branch) + - theme_bw() + - labs(x="Nodes", y="Normalized time (s)", title="NBody Scaling: Normalized Time", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=branch)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=branch)) + - theme_bw() + - labs(x="Nodes", y="Time * nodes (s)", title="NBody Scaling: Time * nodes", - subtitle=output) + - theme(plot.subtitle=element_text(size=5)) + - theme(legend.position="bottom") + - theme(legend.text = element_text(size=7)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/osu/bw.R b/garlic/fig/osu/bw.R deleted file mode 100644 index 876519d..0000000 --- a/garlic/fig/osu/bw.R +++ /dev/null @@ -1,59 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, bw) %>% - rename(unitName=config.unitName) %>% - mutate(bw=bw / 1024.0) %>% - mutate(unitName=str_replace(unitName, "osu-bw-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=median.bw)) + - labs(x="Message size", y="Bandwidth (GB/s)", - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=1.5) + - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - geom_hline(yintercept=12.5, color="red") + - annotate("text", x=1, y=12.5 * .95, - label="Max: 12.5GB/s (100Gbps)", - hjust=0, vjust=1, size=3) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12, - ) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(1,0), legend.position = c(0.99, 0.01)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/eager.R b/garlic/fig/osu/eager.R deleted file mode 100644 index aa9e573..0000000 --- a/garlic/fig/osu/eager.R +++ /dev/null @@ -1,63 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=150 -h=6 -w=8 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - labs(x="Message size (KB)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=3) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") + - geom_vline(xintercept = 10, color="red") + - annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MQ_EAGER_SDMA_SZ.f), nrow=3, labeller = "label_both") + - scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - theme_bw() - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/impi.R b/garlic/fig/osu/impi.R deleted file mode 100644 index c62a334..0000000 --- a/garlic/fig/osu/impi.R +++ /dev/null @@ -1,68 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, config.threshold, size, bw) %>% - rename(unitName=config.unitName) %>% - rename(threshold=config.threshold) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$threshold = as.factor(df$threshold) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianBw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -p = ggplot(data=df, aes(x=size, y=bw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU bandwidth benchmark: nodes=%d tasksPerNode=%d cpusPerTask=%d", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_boxplot(aes(color=threshold, group=interaction(threshold, sizeFactor))) + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ppi=300 -h=4 -w=8 -ggsave("boxplot.pdf", plot=p, width=w, height=h, dpi=ppi) -ggsave("boxplot.png", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=medianBw)) + - labs(x="Size (bytes)", y="Bandwidth (MB/s)", - title=sprintf("OSU benchmark: osu_bw", - nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(color=threshold, linetype=threshold)) + - geom_point(aes(color=threshold, shape=threshold)) + - geom_hline(yintercept = 100e3 / 8, color="red") + - annotate("text", x = 8, y = (100e3 / 8) * 0.95, label = "12.5GB/s (100Gb/s)") + - scale_x_continuous(trans=log2_trans()) + - #scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - theme_bw() + - theme(legend.position = c(0.8, 0.2)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/latency.R b/garlic/fig/osu/latency.R deleted file mode 100644 index e116a73..0000000 --- a/garlic/fig/osu/latency.R +++ /dev/null @@ -1,77 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(stringr) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, config.unitName, config.nodes, config.ntasksPerNode, config.cpusPerTask, size, latency) %>% - rename(unitName=config.unitName) %>% - mutate(unitName=str_replace(unitName, "osu-latency-", "")) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(medianLatency = median(latency)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=size, y=medianLatency)) + - labs(x="Message size", y="Median latency (µs)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(linetype=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - n.breaks = 12)+ - scale_shape_discrete(name = "MPI version") + - scale_linetype_discrete(name = "MPI version") + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) + - theme(legend.justification = c(0,1), legend.position = c(0.01, 0.99)) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) - -ggsave("median-lines.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("median-lines.pdf", plot=p, width=w, height=h, dpi=ppi) - -p = ggplot(data=df, aes(x=size, y=latency)) + - labs(x="Size (bytes)", y="Latency (us)", - #title=sprintf("OSU benchmark: osu_latency", nodes, tasksPerNode, cpusPerTask), - subtitle=output) + - geom_line(aes(y=medianLatency, linetype=unitName, group=unitName)) + - geom_point(aes(shape=unitName), size=2) + - scale_y_log10(breaks = breaks, minor_breaks = minor_breaks) + - scale_x_continuous(trans=log2_trans(), - labels=label_bytes("auto_binary"), - breaks=unique(df$size), - minor_breaks=NULL) + - theme_bw() + - theme(plot.subtitle = element_text(color="gray50")) + - theme(axis.text.x = element_text(angle=-45, hjust=0)) + - theme(legend.position = c(0.2, 0.8)) - -ggsave("latency.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("latency.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/mtu.R b/garlic/fig/osu/mtu.R deleted file mode 100644 index 09e2f63..0000000 --- a/garlic/fig/osu/mtu.R +++ /dev/null @@ -1,69 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) - -args=commandArgs(trailingOnly=TRUE) - -# Read the timetable from args[1] -input_file = "input.json" -if (length(args)>0) { input_file = args[1] } -if (length(args)>1) { output = args[2] } else { output = "?" } - -# Load the dataset in NDJSON format -dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - jsonlite::flatten() - -# We only need the nblocks and time -df = select(dataset, - config.unitName, - config.nodes, - config.ntasksPerNode, - config.cpusPerTask, - config.PSM2_MQ_EAGER_SDMA_SZ, - config.PSM2_MTU, - size, bw, config.iterations) %>% - rename(unitName=config.unitName, - iterations=config.iterations, - PSM2_MQ_EAGER_SDMA_SZ.val=config.PSM2_MQ_EAGER_SDMA_SZ, - PSM2_MTU.val=config.PSM2_MTU) %>% - mutate(bw = bw / 1000.0) - -nodes = unique(df$config.nodes) -tasksPerNode = unique(df$config.ntasksPerNode) -cpusPerTask = unique(df$config.cpusPerTask) -df$unitName = as.factor(df$unitName) -df$sizeFactor = as.factor(df$size) -df$sizeKB = df$size / 1024 -df$PSM2_MQ_EAGER_SDMA_SZ = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ.val) -df$PSM2_MTU = as.factor(df$PSM2_MTU.val) - -iterations = unique(df$iterations) - -df = group_by(df, unitName, sizeFactor) %>% - mutate(median.bw = median(bw)) %>% - ungroup() - -breaks = 10^(-10:10) -minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) - -ppi=300 -h=3 -w=6 - -p = ggplot(data=df, aes(x=sizeKB, y=bw)) + - geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ.val/1024), color="blue") + - geom_vline(aes(xintercept = PSM2_MTU.val/1024), color="red") + - labs(x="Message size (KiB)", y="Bandwidth (GB/s)", - #title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), - subtitle=output) + - geom_point(shape=21, size=2) + - #annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + - facet_wrap(vars(PSM2_MTU), nrow=3, labeller = "label_both") + - #scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + - scale_x_continuous(n.breaks = 12) + - theme_bw() + - theme(plot.subtitle = element_text(size=8, family="mono")) - -ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) -ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/saiph/granularity.R b/garlic/fig/saiph/granularity.R deleted file mode 100644 index 66bd7c6..0000000 --- a/garlic/fig/saiph/granularity.R +++ /dev/null @@ -1,95 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D granularity" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nbly, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=blocksPerCpu, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + - scale_x_continuous(trans=log2_trans()) + - labs(y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/fig/saiph/ss.R b/garlic/fig/saiph/ss.R deleted file mode 100644 index 6498f30..0000000 --- a/garlic/fig/saiph/ss.R +++ /dev/null @@ -1,110 +0,0 @@ -library(ggplot2) -library(dplyr, warn.conflicts = FALSE) -library(scales) -library(jsonlite) -library(viridis, warn.conflicts = FALSE) -library(stringr) - -args = commandArgs(trailingOnly=TRUE) - -# Set the input dataset if given in argv[1], or use "input" as default -if (length(args)>0) { input_file = args[1] } else { input_file = "input" } -if (length(args)>1) { output = args[2] } else { output = "?" } - -df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% - - jsonlite::flatten() %>% - - select(unit, - config.nodes, - config.nblx, - config.nbly, - config.nblz, - config.gitBranch, - config.blocksPerCpu, - config.sizex, - time, - total_time) %>% - - rename(nodes=config.nodes, - nblx=config.nblx, - nbly=config.nbly, - nblz=config.nblz, - gitBranch=config.gitBranch, - blocksPerCpu=config.blocksPerCpu, - sizex=config.sizex) %>% - - # Remove the "garlic/" prefix from the gitBranch - mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% - - # Computations before converting to factor - mutate(time.nodes = time * nodes) %>% - - # Convert to factors - mutate(unit = as.factor(unit)) %>% - mutate(nodes = as.factor(nodes)) %>% - mutate(gitBranch = as.factor(gitBranch)) %>% - mutate(nblx = as.factor(nblx)) %>% - mutate(nbly = as.factor(nbly)) %>% - mutate(nblz = as.factor(nblz)) %>% - mutate(sizex = as.factor(sizex)) %>% - mutate(unit = as.factor(unit)) %>% - - # Compute median times - group_by(unit) %>% - mutate(median.time = median(time)) %>% - mutate(median.time.nodes = median(time.nodes)) %>% - mutate(normalized.time = time / median.time - 1) %>% - mutate(log.median.time = log(median.time)) %>% - ungroup() - -dpi = 300 -h = 5 -w = 8 - -maintitle = "Saiph-Heat3D strong scaling" - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=normalized.time, fill=sizex)) + - geom_boxplot() + - geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + - theme_bw() + - facet_wrap(branch ~ .) + - labs(x="nodes", y="Normalized time", - title=sprintf("%s: normalized time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time, group=sizex)) + - theme_bw() + -# facet_wrap(branch ~ .) + - labs(x="nodes", y="Time (s)", - title=sprintf("%s: time", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) - -# --------------------------------------------------------------------- - -p = ggplot(df, aes(x=nodes, y=time.nodes, color=sizex)) + - geom_point(shape=21, size=3) + - geom_line(aes(y=median.time.nodes, group=sizex)) + - theme_bw() + - #facet_wrap(branch ~ .) + - labs(x="nodes", y="Time * nodes (s)", - title=sprintf("%s: time * nodes", maintitle), - subtitle=output) + - theme(plot.subtitle=element_text(size=8)) - -ggsave("time.nodes.png", plot=p, width=w, height=h, dpi=dpi) -ggsave("time.nodes.pdf", plot=p, width=w, height=h, dpi=dpi) diff --git a/garlic/garlicd/default.nix b/garlic/garlicd/default.nix deleted file mode 100644 index b34efdf..0000000 --- a/garlic/garlicd/default.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, nix -, garlicTool -}: - -let - extraPath = "${garlicTool}:${nix}"; -in - stdenv.mkDerivation { - name = "garlicd"; - preferLocalBuild = true; - - phases = [ "unpackPhase" "installPhase" ]; - - src = ./garlicd; - - unpackPhase = '' - cp $src garlicd - ''; - - installPhase = '' - substituteInPlace garlicd \ - --replace @extraPath@ ${extraPath} - mkdir -p $out/bin - cp -a garlicd $out/bin - ''; - } diff --git a/garlic/garlicd/garlicd b/garlic/garlicd/garlicd deleted file mode 100755 index 17e3f9d..0000000 --- a/garlic/garlicd/garlicd +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -set -e - -emsg() { - >&2 echo "garlicd: $@" -} - -msg() { - emsg "$@" - if [ ! -z "$st" ]; then echo "garlicd: $@" >>"$st"; fi -} - -if [ ! -z "$1" ]; then - >&2 echo "usage: garlicd" - exit 1 -fi - -export PATH="@extraPath@:$PATH" - -garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - -if [ -z "$garlic_sandbox" ]; then - emsg "Missing extra-sandbox-paths /garlic mountpoint" - emsg "Check the ~/.config/nix/nix.conf file" - exit 1 -fi - -mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') -mountdir=$(readlink -f "$mountdir_rel") -run="$mountdir/run" -completed="$mountdir/completed" -wipe="$mountdir/wipe" -st="$mountdir/st" - -handle_bad_signal() { - msg "cleaning FIFO pipes" - rm -f "$run" "$completed" "$wipe" - exit 1 -} - -trap handle_bad_signal SIGINT - -for fifo in "$run" "$completed" "$wipe"; do - if [ ! -e "$fifo" ]; then - mkfifo "$fifo" - # FIXME: Use more resctrictive permissions - chmod 666 "$fifo" - fi -done - -while true; do - emsg "--- Waiting for experiments ---" - tre=$(head -1 "$run") - - # Truncate state file - printf "" > "$st" - - msg "Attempting to run: $tre" - msg "Copying files to MN4..." - # It fails if the user doesn't have nix-store, but is already copied - # with the post build hook - nix copy --to ssh://mn1 $tre || true - - set +e - - msg "Launching the experiment..." - garlic -R "$tre" 2>>"$st" - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "Failed to run the experiment :-(" - continue - fi - - msg "Fetching results..." - garlic -FKv "$tre" 2>>"$st" - - if [ "$?" != 0 ]; then - echo ERROR > "$completed" - msg "The experiment failed :-(" - continue - fi - - set -e - - msg "Signalling nix build..." - echo -n "$tre" >> "$completed" - - msg "Waiting for nix to finish the build..." - - tre2=$(head -1 "$wipe") - if [ "$tre" != "$tre2" ]; then - msg "error: trebuchet mismatch" - exit 1 - fi - - msg "Removing temporal files..." - garlic -D "$tre" 2>>"$st" - - echo -n "$tre" >> "$completed" - - msg "Execution completed :-)" -done diff --git a/garlic/index.nix b/garlic/index.nix deleted file mode 100644 index ed8e2d8..0000000 --- a/garlic/index.nix +++ /dev/null @@ -1,138 +0,0 @@ -{ - super -, self -, bsc -, callPackage -}: - -{ - develop = let - commonPackages = with self; [ - coreutils htop procps-ng vim which strace - tmux gdb kakoune universal-ctags bashInteractive - glibcLocales ncurses git screen curl boost - # Add more nixpkgs packages here... - ]; - bscPackages = with bsc; [ - slurm clangOmpss2 icc mcxx perf tampi impi vtk paraver - # Add more bsc packages here... - ]; - packages = commonPackages ++ bscPackages; - in - bsc.garlic.stages.exec rec { - nextStage = bsc.garlic.stages.isolate { - nextStage = bsc.garlic.unsafeDevelop.override { - extraInputs = packages; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - extraMounts = [ "/tmp:$TMPDIR" ]; - }; - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - # This hack uploads all dependencies to MN4 - pre = let - nixPrefix = bsc.garlic.targetMachine.config.nixPrefix; - stageProgram = bsc.garlicTools.stageProgram; - in - '' - # Hack to upload this to MN4: @upload-to-mn@ - - # Create a link to the develop script - ln -fs ${nixPrefix}${stageProgram nextStage} .nix-develop - ''; - post = "\n"; - }; - - unsafeDevelop = callPackage ./develop/default.nix { }; - - # Configuration for the machines - machines = callPackage ./machines.nix { }; - - report = callPackage ./report.nix { - fig = bsc.garlic.fig; - }; - - sedReport = callPackage ./sedReport.nix { - fig = bsc.garlic.fig; - }; - - bundleReport = callPackage ./bundleReport.nix { - fig = bsc.garlic.fig; - }; - - reportTar = callPackage ./reportTar.nix { - fig = bsc.garlic.fig; - }; - - # Use the configuration for the following target machine - targetMachine = bsc.garlic.machines.mn4; - - # Load some helper functions to generate app variants - - stdexp = callPackage ./stdexp.nix { - inherit (bsc.garlic) targetMachine stages pp; - }; - - # Execution stages - stages = { - sbatch = callPackage ./stages/sbatch.nix { }; - srun = callPackage ./stages/srun.nix { }; - baywatch = callPackage ./stages/baywatch.nix { }; - control = callPackage ./stages/control.nix { }; - exec = callPackage ./stages/exec.nix { }; - script = callPackage ./stages/script.nix { }; - extrae = callPackage ./stages/extrae.nix { }; - valgrind = callPackage ./stages/valgrind.nix { }; - perf = callPackage ./stages/perf.nix { }; - isolate = callPackage ./stages/isolate { }; - runexp = callPackage ./stages/runexp { }; - trebuchet = callPackage ./stages/trebuchet.nix { }; - strace = callPackage ./stages/strace.nix { }; - unit = callPackage ./stages/unit.nix { }; - experiment = callPackage ./stages/experiment.nix { }; - }; - - # Tests (move to bsc ?) - mpptest = callPackage ./mpptest { }; - - ppong = callPackage ./ppong { - mpi = bsc.mpi; - }; - - hist = callPackage ./pp/hist { }; - - tool = callPackage ./sh/default.nix { - sshHost = "mn1"; - }; - - # Post processing - pp = { - store = callPackage ./pp/store.nix { }; - rPlot = callPackage ./pp/rplot.nix { }; - mergeDatasets = callPackage ./pp/merge.nix { }; - }; - - garlicd = callPackage ./garlicd/default.nix { - garlicTool = bsc.garlic.tool; - }; - - # Apps for Garlic - apps = callPackage ./apps/index.nix { - inherit self super bsc; - }; - - # Experiments - exp = callPackage ./exp/index.nix { - inherit self super bsc; - }; - - # Dataset generators from resultTree - ds = callPackage ./ds/index.nix { - inherit self super bsc; - }; - - # Figures generated from the datasets - fig = callPackage ./fig/index.nix { - inherit self super bsc; - }; - -} diff --git a/garlic/machines.nix b/garlic/machines.nix deleted file mode 100644 index e2bdc54..0000000 --- a/garlic/machines.nix +++ /dev/null @@ -1,63 +0,0 @@ -{ - stdenv -}: - -{ - # MareNostrum 4 configuration - mn4 = { - config = { - clusterName = "mn4"; - sshHost = "mn1"; - nixPrefix = "/gpfs/projects/bsc15/nix"; - march = "skylake-avx512"; - mtune = "skylake-avx512"; - hw = { - # The rev attribute attemps to capture the hardware - # configuration of the machine, and will rebuild all experiments - # if it changed. It only holds the timestamp at the current - # time, representing the HW configuration at that moment. - rev = 1614253003; - - # Node and socket details - cpusPerNode = 48; - cpusPerSocket = 24; - socketsPerNode = 2; - cachelineBytes = 64; - - # Cache sizes - cacheSizeKB = { - L1d = 32; - L1i = 32; - L2 = 1024; - L3 = 33792; - }; - }; - }; - - # Experimental naming convention for the FS - fs = rec { - topology = { - gpfs = { - projects = "/gpfs/projects/bsc15/garlic"; - scratch = "/gpfs/scratch/bsc15/bsc15557/garlic"; - }; - - ssd = { - # Beware to expand the temp dir at execution time - temp = "$TMPDIR"; - }; - }; - - shared = with topology; { - fast = gpfs.scratch; - reliable = gpfs.projects; - }; - - local = { - temp = topology.ssd.temp; - }; - }; - - # TODO: Add the specific details for SLURM and the interconection here - }; -} diff --git a/garlic/mpptest/default.nix b/garlic/mpptest/default.nix deleted file mode 100644 index c142667..0000000 --- a/garlic/mpptest/default.nix +++ /dev/null @@ -1,16 +0,0 @@ -{ - stdenv -, mpi -, fetchurl -}: - -stdenv.mkDerivation { - name = "mpptest"; - - src = fetchurl { - url = "http://ftp.mcs.anl.gov/pub/mpi/tools/perftest.tar.gz"; - sha256 = "11i22lq3pch3pvmhnbsgxzd8ap4yvpvlhy2f7k8x3krdwjhl0jvl"; - }; - - buildInputs = [ mpi ]; -} diff --git a/garlic/pp/hist/default.nix b/garlic/pp/hist/default.nix deleted file mode 100644 index 4a3d464..0000000 --- a/garlic/pp/hist/default.nix +++ /dev/null @@ -1,25 +0,0 @@ -{ - stdenv -, ministat -}: - -stdenv.mkDerivation { - name = "hist"; - preferLocalBuild = true; - src = ./.; - - dontBuild = true; - dontConfigure = true; - - inherit ministat; - - patchPhase = '' - substituteAllInPlace hist.sh - ''; - - installPhase = '' - mkdir -p $out/bin - cp hist.sh $out/bin/hist - chmod +x $out/bin/hist - ''; -} diff --git a/garlic/pp/hist/hist.sh b/garlic/pp/hist/hist.sh deleted file mode 100755 index f3363b4..0000000 --- a/garlic/pp/hist/hist.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Use it either reading from stdin or by specifing -# multiple files as arguments - -# xeon07$ hist stdout.log -# x -# +------------------------------------------------------------------------+ -# | x | -# | x | -# | x | -# | x | -# | x | -# | xxx | -# | xxx | -# | xxxxx | -# | xxxxxx | -# | xxxxxxx x| -# ||________M_A___________| | -# +------------------------------------------------------------------------+ -# N Min Max Median Avg Stddev -# x 30 3.585183 3.763913 3.591559 3.5973344 0.031719975 -# -# Other ministat options can be passed as well. The -S option splits the results -# in multiple plots. - - -usage() { echo "Usage: hist [-hSAns] [-c confidence] [-w width] files..." 1>&2; exit 1; } - -function stat_files() { - tmpfiles=() - sedcmd="" - - for file in ${files[@]}; do - tmp=$(mktemp) - awk '/^time /{print $2}' "$file" > "$tmp" - sedcmd+="s:$tmp:$file:g;" - tmpfiles+=($tmp) - done - - if [ $split == 1 ]; then - for f in "${tmpfiles[@]}"; do - ministat $ministat_opt $f | sed -e "$sedcmd" - done - else - ministat $ministat_opt ${tmpfiles[@]} | sed -e "$sedcmd" - fi - - rm ${tmpfiles[@]} -} - -split=0 -ministat_opt="-w72" - -while getopts "hSAnsc:w:" o; do - case "${o}" in - S) split=1 ;; - c) ministat_opt+=" -c $OPTARG" ;; - w) ministat_opt+=" -w $OPTARG" ;; - A) ministat_opt+=" -$o" ;; - n) ministat_opt+=" -$o" ;; - s) ministat_opt+=" -$o" ;; - *) usage ;; - esac -done - -shift $((OPTIND-1)) - -ministat=@ministat@/bin -#ministat=/nix/store/sh9b484bnhkajxnblpwix7fhbkid6365-ministat-20150715-1/bin - -export PATH="$PATH:$ministat" - -files=("$@") -if [[ -z "${files[@]}" ]]; then - awk '/^time /{print $2}' | ministat $ministat_opt -else - stat_files -fi diff --git a/garlic/pp/merge.nix b/garlic/pp/merge.nix deleted file mode 100644 index 03caaea..0000000 --- a/garlic/pp/merge.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, lib -}: - -datasets: - -with lib; - -stdenv.mkDerivation { - name = "merged-dataset"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - inherit datasets; - installPhase = '' - mkdir -p $out - n=1 - for d in $datasets; do - ln -s $d $out/$n - let n=n+1 - cat $d/dataset >> $out/dataset - done - ''; -} diff --git a/garlic/pp/rplot.nix b/garlic/pp/rplot.nix deleted file mode 100644 index df154da..0000000 --- a/garlic/pp/rplot.nix +++ /dev/null @@ -1,165 +0,0 @@ -{ - stdenv -, lib -, rWrapper -, rPackages -, fontconfig -, dejavu_fonts -, liberation_ttf -, noto-fonts -, makeFontsConf -, makeFontsCache -, jq -, fetchFromGitHub -, writeText -, runCommand -, glibcLocales -}: - -{ -# The two results to be compared - dataset -, script -, extraRPackages ? [] -}: - -with lib; - -let - scalesPatched = with rPackages; buildRPackage { - name = "scales"; - src = fetchFromGitHub { - owner = "mikmart"; - repo = "scales"; - #ref = "label-bytes"; - rev = "fa7d91c765b6b5d2f682c7c22e0478d96c2ea76c"; - sha256 = "10dsyxp9pxzdmg04xpnrxqhc4qfhbkr3jhx8whfr7z27wgfrr1n3"; - }; - propagatedBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - nativeBuildInputs = [ farver labeling lifecycle munsell R6 RColorBrewer viridisLite ]; - }; - - customR = rWrapper.override { - packages = with rPackages; [ scalesPatched tidyverse viridis egg - Cairo extrafont ] ++ extraRPackages; - }; - - myFonts = [ - dejavu_fonts - #noto-fonts - #liberation_ttf - ]; - - cacheConf = - let - cache = makeFontsCache { fontDirectories = myFonts; }; - in - writeText "fc-00-nixos-cache.conf" '' - - - - - ${concatStringsSep "\n" (map (font: "${font}") myFonts)} - ${optionalString (stdenv.hostPlatform == stdenv.buildPlatform) '' - - ${cache} - ''} - - ''; - - # default fonts configuration file - # priority 52 - defaultFontsConf = - let genDefault = fonts: name: - optionalString (fonts != []) '' - - ${name} - - ${concatStringsSep "" - (map (font: '' - ${font} - '') fonts)} - - - ''; - in - writeText "fc-52-nixos-default-fonts.conf" '' - - - - - ${genDefault [ "DejaVu Sans" ] "sans-serif"} - ${genDefault [ "DejaVu Serif" ] "serif"} - ${genDefault [ "DejaVu Sans Mono" ] "monospace"} - ${genDefault [ "Noto Color Emoji"] "emoji"} - - ''; - - fontConfPath = - let - fixedConf = runCommand "fonts-fixed.conf" { - preferLocalBuild = true; - } '' - head --lines=-2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - - cat >> $out << 'EOF' - - conf.d - EOF - - tail -2 ${fontconfig.out}/etc/fonts/fonts.conf >> $out - ''; - in - runCommand "fontconfig-conf" { - preferLocalBuild = true; - } '' - dst=$out/etc/fonts/conf.d - mkdir -p $dst - # fonts.conf - ln -s ${fixedConf} $dst/../fonts.conf - - # fontconfig default config files - ln -s ${fontconfig.out}/etc/fonts/conf.d/*.conf \ - $dst/ - - # 00-nixos-cache.conf - ln -s ${cacheConf} $dst/00-nixos-cache.conf - - # 52-nixos-default-fonts.conf - ln -s ${defaultFontsConf} $dst/52-nixos-default-fonts.conf - ''; - -in stdenv.mkDerivation { - name = "plot"; - buildInputs = [ customR jq fontconfig glibcLocales ]; - preferLocalBuild = true; - dontPatchShebangs = true; - phases = [ "installPhase" ]; - - installPhase = '' - export FONTCONFIG_PATH=${fontConfPath}/etc/fonts/ - export LANG=en_US.UTF-8 - - mkdir -p $out - cd $out - dataset=$(readlink -f ${dataset}/dataset) - - ln -s $dataset input - Rscript --vanilla ${script} "$dataset" "$out" - - # HACK: replace the \minus for a \hyphen to keep the file paths intact, so - # they can be copied to the terminal directly. The StandardEncoding is not - # working (AdobeStd.enc). - find "$out" -name '*.pdf' | xargs -l1 sed -i 's.45/minus.45/hyphen.g' - - if [ "''${dataset##*.}" == gz ]; then - gunzip --stdout $dataset - else - cat $dataset - fi | jq -c .total_time |\ - awk '{s+=$1} END {printf "%f\n", s/60}' > total_job_time_minutes - ''; -} diff --git a/garlic/pp/store.nix b/garlic/pp/store.nix deleted file mode 100644 index 6005efb..0000000 --- a/garlic/pp/store.nix +++ /dev/null @@ -1,89 +0,0 @@ -{ - stdenv -}: - -{ - trebuchet, - experiment -}: - -with builtins; - -let - experimentName = baseNameOf (experiment); - trebuchetName = baseNameOf (trebuchet); -in - stdenv.mkDerivation { - name = "resultTree"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - installPhase = '' - echo "resultTree: searching for garlicd daemon..." - if [ -e /garlic/run ]; then - echo "resultTree: asking the daemon to run and fetch the experiment" - - echo ${trebuchet} >> /garlic/run - echo "resultTree: waiting for experiment results..." - stdbuf -o 0 tail -f /garlic/st & - stpid=$! - res=$(cat /garlic/completed) - - kill -TERM $stpid - - if [ "$res" == "ERROR" ]; then - echo "resultTree: the experiment failed" - exit 1 - fi - - if [ "$res" != "${trebuchet}" ]; then - echo "resultTree: unknown trebuchet received" - exit 1 - fi - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: assuming results are already in /garlic" - fi - - echo "resultTree: attempting to copy the results from /garlic ..." - - exp=/garlic/cache/${experimentName} - - if [ ! -e "$exp" ]; then - echo "resultTree: $exp: not found" - echo "resultTree: run the experiment and fetch the results running" - echo "resultTree: the following command from the nix-shell" - echo - echo -e "\e[30;48;5;2mgarlic -RFv ${trebuchet}\e[0m" - echo - echo "resultTree: see garlic(1) for more details." - echo "resultTree: cannot continue building $out, aborting" - exit 1 - fi - - echo "resultTree: copying results from /garlic into the nix store..." - - mkdir -p $out - cp -aL $exp $out/ - ln -s ${trebuchet} $out/trebuchet - ln -s ${experiment} $out/experiment - - - if [ -e /garlic/run ]; then - echo "resultTree: removing temp files..." - echo ${trebuchet} >> /garlic/wipe - echo "resultTree: waiting confimation from daemon..." - cat /garlic/completed > /dev/null - else - echo "resultTree: garlicd not detected: /garlic/run not found" - echo "resultTree: ignoring temp files" - fi - - echo "resultTree: successfully copied into the nix store" - - echo " experiment: ${experiment}" - echo " trebuchet: ${trebuchet}" - echo " resultTree: $out" - ''; - } diff --git a/garlic/report.nix b/garlic/report.nix deleted file mode 100644 index 6d7dc46..0000000 --- a/garlic/report.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, sedReport -}: -let - sedCmd = (import sedReport) fig; -in - stdenv.mkDerivation { - name = "report.pdf"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${sedCmd} - make - ''; - installPhase = '' - cp report.pdf $out - ''; - } diff --git a/garlic/report/Makefile b/garlic/report/Makefile deleted file mode 100644 index 81ae94d..0000000 --- a/garlic/report/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -all: report.tex - pdflatex report.tex -o report.pdf - # Run again to fix figure references - pdflatex report.tex -o report.pdf diff --git a/garlic/report/report.tex b/garlic/report/report.tex deleted file mode 100644 index f56c1c9..0000000 --- a/garlic/report/report.tex +++ /dev/null @@ -1,31 +0,0 @@ -\documentclass{article} -\usepackage{graphicx} - -\begin{document} - -\title{Example of Nix + \LaTeX{}} -\author{Rodrigo Arias Mallo} - -\maketitle - -\section{Nbody} -The nbody program has been executed with varying block sizes while the execution -time $t$ is measured, as shown in the figure \ref{fig:nbody.test}. -% -\begin{figure}[h] - \centering - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/scatter.png} - \includegraphics[width=0.45\textwidth]{@fig.nbody.small@/box.png} - \caption{Nbody times with varying block size} - \label{fig:nbody.test} -\end{figure} -% -The normalized time $\hat t$ is computed with the median time $t_m$ using $ \hat -t = t / t_{m} - 1 $. It can be observed that the normalized times exceed the -maximum allowed interval in most cases, except with the largest block sizes. - -Once the experiment \texttt{exp.nbody.test} changes, the hash of the experiment -program will change, therefore the plot will be updated and, lastly, this -report. - -\end{document} diff --git a/garlic/reportTar.nix b/garlic/reportTar.nix deleted file mode 100644 index c2b274b..0000000 --- a/garlic/reportTar.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -, fig -, writeText -, busybox -, jq -, texlive -, bundleReport -}: -let - - genCmd = (import bundleReport) fig; -in - stdenv.mkDerivation { - name = "report.tar.gz"; - src = ./report; - buildInputs = [ jq texlive.combined.scheme-basic ]; - buildPhase = '' - ${genCmd} - ls -ltR - cat report.tex - make - ''; - installPhase = '' - cd .. - tar -czf $out report - ''; - } diff --git a/garlic/sedReport.nix b/garlic/sedReport.nix deleted file mode 100644 index b776681..0000000 --- a/garlic/sedReport.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ - stdenv -, fig -}: -stdenv.mkDerivation { - name = "sedReport"; - src = ./report; - buildPhase = '' - grep -o '@[^ @]*@' report.tex | sed 's/@//g' | sort -u > list - - echo "fig:" > fun.nix - echo "'''" >> fun.nix - sed 's:\(^.*\)$:sed -i "s;@\1@;''${\1};g" report.tex:g' list >> fun.nix - echo "'''" >> fun.nix - ''; - installPhase = '' - cp fun.nix $out - ''; -} diff --git a/garlic/sh/default.nix b/garlic/sh/default.nix deleted file mode 100644 index 27d34d1..0000000 --- a/garlic/sh/default.nix +++ /dev/null @@ -1,41 +0,0 @@ -{ - stdenv -, garlicTools -, sshHost -, rsync -, openssh -, nix -, jq -, ncurses -}: - -with garlicTools; - -let - garlicPrefix = "/mnt/garlic"; -in - stdenv.mkDerivation { - name = "garlic-tool"; - preferLocalBuild = true; - - buildInputs = [ rsync openssh nix jq ncurses ]; - phases = [ "unpackPhase" "installPhase" ]; - - src = ./.; - - inherit garlicPrefix sshHost; - - installPhase = '' - substituteAllInPlace garlic - substituteInPlace garlic \ - --replace @PATH@ $PATH - mkdir -p $out/bin - cp garlic $out/bin - chmod +x $out/bin/garlic - mkdir -p $out/share/man/man1 - cp garlic.1 $out/share/man/man1 - - patchShebangs garlic-* - cp garlic-* $out/bin - ''; - } diff --git a/garlic/sh/fix-figure-subtitle.sh b/garlic/sh/fix-figure-subtitle.sh deleted file mode 100755 index a98614b..0000000 --- a/garlic/sh/fix-figure-subtitle.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -if grep -q 'output = args[2]' "$1"; then exit 0; fi - -sed '/length(args)>0/aif (length(args)>1) { output = args[2] } else { output = "?" }' -i "$1" -sed '/jsonlite::flatten/,$s/input_file/output/g' -i "$1" diff --git a/garlic/sh/garlic b/garlic/sh/garlic deleted file mode 100755 index df3e33f..0000000 --- a/garlic/sh/garlic +++ /dev/null @@ -1,298 +0,0 @@ -#!/bin/bash -e - -garlicPrefix=@garlicPrefix@ -sshHost=@sshHost@ -PATH=@PATH@ - -usage() { echo "Usage: garlic [-RFwv] trebuchet" 1>&2; exit 1; } - -msg() { - >&2 echo "garlic: $@" -} - -findClosure() { - what=$1 - from=$2 - mexp=$(nix-store -qR "$from" | grep -E -- "$what") - n=$(echo "$mexp" | awk 'BEGIN { count=0 } NF { count++ } END { print count }') - if [ $n -eq 0 ]; then - >&2 echo "$exp: $what not found" - exit 1 - fi - if [ $n -gt 1 ]; then - >&2 echo "$exp: multiple $what found" - exit 1 - fi - echo "$mexp" -} - -findExperiment() { - grep -o -- "/nix/store/.*-experiment" "$1" -} - -findOutputDir() { - garlic_sandbox=$(nix show-config |\ - grep extra-sandbox-paths |\ - grep -o '/garlic=[^ ]*' || true) - - if [ -z "$garlic_sandbox" ]; then - >&2 echo "Missing extra-sandbox-paths /garlic mountpoint" - >&2 echo "Check the ~/.config/nix/nix.conf file" - exit 1 - fi - - mountdir_rel=$(echo "$garlic_sandbox" | sed 's@^/garlic=@@g') - mountdir=$(readlink -f "$mountdir_rel") - - echo "$mountdir/cache" -} - -drvFromOutput() { - nix-store -q --deriver $1 -} - -checkTrebuchet() { - if [ ! -e "$trebuchet" ]; then - >&2 echo "$trebuchet: not found" - exit 1 - fi - - if [ ! -f "$trebuchet" ]; then - >&2 echo "$trebuchet: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a trebuchet - if [ -z "$(grep "This trebuchet launches" $trebuchet)" ]; then - >&2 echo "$trebuchet: not a trebuchet" - exit 1 - fi - - return 0 -} - -checkExperiment() { - if [ ! -e "$experiment" ]; then - >&2 echo "$experiment: not found" - exit 1 - fi - - if [ ! -f "$experiment" ]; then - >&2 echo "$experiment: not a file" - exit 1 - fi - - # FIXME: We need a better way to determine a experiment - if [ -z "$(grep "This is an experiment" $experiment)" ]; then - >&2 echo "$experiment: not an experiment" - exit 1 - fi - - return 0 -} - -checkMountpoint() { - if [ ! -e "$garlicPrefix/garlic.control" ]; then - >&2 echo "error: missing $garlicPrefix/garlic.control" - >&2 echo "Is the mountpoint enabled?" - exit 1 - fi -} - -status_line() { - unithash="$1" - name="$2" - status="$3" - - red=$(tput -T ansi setaf 1) - green=$(tput -T ansi setaf 2) - yellow=$(tput -T ansi setaf 3) - color_reset=$(tput -T ansi sgr0) - - case $status in - ok) - color_st="$green" - ;; - run*) - color_st="$yellow" - ;; - exit*) - color_st="$red" - ;; - *) - color_st="" - ;; - esac - - if [ $verbose ]; then - >&2 printf "garlic: %s %s%-9s%s %s\n" \ - "$unithash" "$color_st" "$status" "$color_reset" "$name" - fi -} - -do_fetch() { - expName=$(basename $experiment) - user=$(ssh -G "$sshHost" | awk '/^user /{print $2}') - exp=$garlicPrefix/$user/out/$expName - - if [ ! -e "$exp" ]; then - echo "missing experiment: $exp" - exit 1 - fi - - unitNameList=$(grep '^/nix/store/.*-unit$' $experiment |\ - sort -u |\ - sed 's@^/nix/store/@@') - - cwd=$(pwd) - - repeat=1 - bad=0 - while [ 1 ]; do - repeat=0 - test $verbose && msg "Checking units $(date --rfc-3339=seconds)..." - - declare -A unit_names - - for unit in $unitNameList; do - - unit_hash="${unit%-unit}" - unit_status="?" - unit_name="?" - - if [ -e "$exp/$unit" ]; then - st_file="$exp/$unit/status" - conf_json="$exp/$unit/garlic_config.json" - done_file="$exp/$unit/done" - - if [ -z "${unit_names[$unit_hash]}" ]; then - if [ -e "$conf_json" ]; then - unit_names+=([$unit_hash]=$(jq -r .unitName "$conf_json")) - unit_name="${unit_names[$unit_hash]}" - fi - else - unit_name="${unit_names[$unit_hash]}" - fi - - if [ -e "$st_file" ]; then - unit_status=$(cat "$st_file") - fi - fi - - status_line "$unit_hash" "$unit_name" "$unit_status" - - if [ ! -e "$done_file" ]; then - repeat=1 - elif [ "$unit_status" != "ok" ]; then - bad=1 - fi - done - - if [ $repeat -eq 0 ]; then - break - else - test $verbose && msg "" - fi - - if [ $waitResults -eq 1 ]; then - #echo "waiting 3 seconds to try again" - sleep 3 - else - break - fi - done - - if [ $repeat -eq 1 ]; then - #>&2 echo "$exp: execution incomplete" - exit 1 - fi - - if [ $bad -eq 1 ]; then - msg "Some units failed, aborting" - exit 1 - fi - - cd "$cwd" - - test $verbose && msg "execution complete, fetching results" - - mkdir -p "$outputDir" - - rsync -vrt --copy-links \ - --include='*/*/garlic_config.json' \ - --include='*/*/std*.log' \ - --include='*/*/*/std*.log' \ - --include='*/*/*/.garlic' \ - --include='*/*/*/.garlic/**' \ - --exclude='*/*/*/*' \ - "$exp" "$outputDir" - - if [ ! $enableKeep ]; then - nix-build -E "(with import ./default.nix; \ - garlic.pp.store { \ - trebuchet = (import \"$trebuchetDrv\" ); \ - experiment = (import \"$experimentDrv\"); \ - })" - - rm -rf "$outputDir/$expName" - fi -} - -do_run() { - - >&2 $trebuchet -} - -do_delete() { - expName=$(basename $experiment) - rm -rf $outputDir/$expName -} - -waitResults=1 -verbose= -operation= -target= -enableRun= -enableFetch= -enableKeep= -enableDelete= - -while getopts "vwRFKD" o; do - case "${o}" in - R) enableRun=1 ;; - F) enableFetch=1 ;; - K) enableKeep=1 ;; - D) enableDelete=1 ;; - w) waitResults=0 ;; - v) verbose=1 ;; - *) usage ;; - esac -done -shift $((OPTIND-1)) -trebuchet="$1" - -if [ -z "$enableRun" -a -z "$enableFetch" -a -z "$enableDelete" ]; then - >&2 echo "missing operation" - usage -fi - -if [ -z "$trebuchet" ]; then - >&2 echo "missing experiment" - usage -fi - -checkMountpoint - -checkTrebuchet $trebuchet - -outputDir=$(findOutputDir) - -experiment=$(findExperiment "$trebuchet") -checkExperiment $experiment - -trebuchetDrv=$(drvFromOutput $trebuchet) -experimentDrv=$(drvFromOutput $experiment) - -if [ $enableRun ]; then do_run; fi -if [ $enableFetch ]; then do_fetch; fi -if [ $enableDelete ]; then do_delete; fi diff --git a/garlic/sh/garlic-add-copyright b/garlic/sh/garlic-add-copyright deleted file mode 100755 index 089aafe..0000000 --- a/garlic/sh/garlic-add-copyright +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -if [ -z "$1" ]; then - cat << 'EOF' -Usage: garlic-add-copyright files... - -This tool prepends the content of the copyright file to the list of given files. -The word 'copyright' is used to determine if a file already has a copyright -notice or not. - -Example: - - Given the following HEADER file: - - $ cat HEADER - /* - * This file is part of NBody and is licensed under the terms contained - * in the LICENSE file. - * - * Copyright (C) 2021 Barcelona Supercomputing Center (BSC) - */ - - It can be prepended to all files ending in .c or .h with: - - $ garlic-add-copyright HEADER $(find * -type f | grep -Ei '\.(c|h)$') - -EOF - exit 1 -fi - -header_file="$1" -shift - -if ! grep -qi copyright "$header_file"; then - >&2 echo "The header file '$header_file' doesn't contain the word 'copyright'" - exit 1 -fi - -for f in "${@}"; do - if grep -qi copyright "$f"; then - echo "$f: Contains copyright word, skipping" - continue - fi - - tmp_fn="$f.copyright-being-added" - - cat "$header_file" "$f" > "$tmp_fn" - mv "$tmp_fn" "$f" -done diff --git a/garlic/sh/garlic-git-table b/garlic/sh/garlic-git-table deleted file mode 100755 index 58f4049..0000000 --- a/garlic/sh/garlic-git-table +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -progname="$(basename $0)" - -if [ -z "$1" ]; then - cat >&2 < - -Finds all garlic/* branches and their current commit of the given -repository and builds the gitTable to be used in nix derivations. - -Example: - garlic-git-table ssh://git@bscpm03.bsc.es/garlic/apps/heat.git -EOF - exit 1 -fi - -echo '{' -echo " # Auto-generated with $progname on $(date --rfc-3339=date) for repo:" -echo " # $1" -echo -git ls-remote $1 'garlic/*' |\ - sed 's@refs/heads/@@' |\ - awk '{printf "\"%s\" = \"%s\";\n", $2, $1}' |\ - column -t -o ' ' |\ - sed 's/^/ /' -echo '}' diff --git a/garlic/sh/garlic-propagate-commit b/garlic/sh/garlic-propagate-commit deleted file mode 100755 index 04617e1..0000000 --- a/garlic/sh/garlic-propagate-commit +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -e - -function printHelp() -{ - >&2 echo "Usage: garlic-propagate-commit " - >&2 echo - cat >&2 </dev/null | sed 's/^..//'` == $1 ]]; then - return 0 - else - return 1 - fi -} - - -if [ $# -lt 1 ]; then - printHelp - exit 1 -fi - -# Obtain all the tracked branches -branches=(`git branch | sed 's/^..//'`) -currentBranch=`git rev-parse --abbrev-ref HEAD` - -# Make sure that the commit SHA exists -commit=$1 -if branchContainsCommit $currentBranch $commit; then - echo "Commit $commit exists in current branch, proceeding..." -else - echo "Error: Commit $commit does not exist in the current branch" - exit 1 -fi - -# Distribute the commit for all tracked branches -for branch in ${branches[@]}; -do - if [[ $branch != $currentBranch ]]; then - echo "Trying to add commit $commit to branch $branch" - if branchContainsCommit $branch $commit; then - echo "Branch $branch already contains commit $commit, skipping" - else - git checkout $branch - git cherry-pick $commit - git push - fi - fi -done - -# Return to the original branch -git checkout $currentBranch diff --git a/garlic/sh/garlic-query b/garlic/sh/garlic-query deleted file mode 100755 index 6b0c2de..0000000 --- a/garlic/sh/garlic-query +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash - -function find_garlic_conf() { - grep -o '^cp /nix/store/[^ ]*-garlic_config.json' "$1" | sed 's/^cp //g' -} - -function show_units() { - exp=$1 - - units=$(grep '^/nix/store/.*-unit$' $exp) - - nunits=$(echo "$units" | wc -l) - echo " Experiment: $exp" - echo " Units: $nunits" - echo - echo " Unit file Name" - - for unit in $units; do - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - printf " %s %s %s %s\n" "$unit" "$present" "$unitName" - done -} - -function query_tre() { - tre=$1 - exp=$(grep "^# */nix/store/[^ ]*-experiment$" "$tre" | sed 's/# *//g') - - echo - echo " Trebuchet: $tre" - show_units $exp - echo -} - -function query_exp() { - exp=$1 - - echo - show_units "$exp" - echo -} - -function query_unit() { - unit=$1 - - stages=$(grep '^# */nix/store/.*$' $unit | sed 's/^# */ /g') - gconf=$(find_garlic_conf $unit) - unitName=$(jq -r .unitName "$gconf") - - echo - echo " Unit: $unit" - echo " Name: $unitName" - echo " Stages:" - echo - echo "$stages" - echo - echo " Config: $gconf" - echo - jq . "$gconf" -} - -function query_result() { - tree=$1 - - tre=$(readlink -f $tree/trebuchet) - exp=$(readlink -f $tree/experiment) - - echo - echo " Result tree: $tree" - echo " Trebuchet: $tre" - show_units $exp - echo -} - -element=$1 - -if [ "$1" == "--help" -o -z "$1" ]; then - cat < - -The path may be a trebuchet, experiment, unit or resultTree. -EOF - exit 1 -fi - -# Try prepending the nix store -if [ ! -e $element ]; then - element=/nix/store/$element* -fi - -element=$(readlink -f $element) - -case "$element" in - *-trebuchet) query_tre $element ;; - *-experiment) query_exp $element ;; - *-unit) query_unit $element ;; - *-resultTree) query_result $element ;; - *) echo "unknown: $element"; exit 1 ;; -esac diff --git a/garlic/sh/garlic.1 b/garlic/sh/garlic.1 deleted file mode 100644 index 86915d8..0000000 --- a/garlic/sh/garlic.1 +++ /dev/null @@ -1,61 +0,0 @@ -.\" The following commands are required for all man pages. -.Dd Nov 5, 2020 -.Dt garlic 1 -.Os Linux -.Sh NAME -.Nm garlic -.Nd Garlic benchmark tool -.Sh SYNOPSIS -.Nm garlic -.Op Fl RFvw -.Ar trebuchet -.Sh DESCRIPTION -.Nm -is a program to handle tasks related with the experiments of the Garlic -benchmark. The -.Ar trebuchet -argument is a path to a valid trebuchet stage script. The -specified operations are executed sequentially and at -least one must be selected from: -.Bl -tag -width Ds -compact -.Pp -.It Fl R -launches the -.Ar experiment using the trebuchet. -.Pp -.It Fl F -waits until the -.Ar experiment -is completed and then fetches the results -into the nix store. -.El -.Pp -Additionally, the following options are available: -.Bl -tag -width Ds -compact -.Pp -.It Fl v -be more verbose. -.Pp -.It Fl w -don't wait for the complete results when fetching; fails if they are not -ready. -.El -.\" This next command is for sections 1, 6, 7, and 8 only. -.\" .Sh ENVIRONMENT -.\" .Sh FILES -.Sh EXIT STATUS -.Ex -std -.\" .Sh EXAMPLES -.\" This next command is for sections 1, 4, 6, 7, 8, and 9 only -.\" (fprintf/stderr type diagnostics). -.\" .Sh DIAGNOSTICS -.\" .Sh COMPATIBILITY -.\" This next command is for sections 2, 3, 4, and 9 only -.\" (settings of the errno variable). -.\" .Sh ERRORS -.\" .Sh SEE ALSO -.\" .Sh STANDARDS -.\" .Sh HISTORY -.\" .Sh AUTHORS -.\" .Sh CAVEATS -.\" .Sh BUGS diff --git a/garlic/shell.nix b/garlic/shell.nix deleted file mode 100644 index 9a38076..0000000 --- a/garlic/shell.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ pkgs ? import ./. }: - -with pkgs; -with bsc; - -mkShell { - name = "garlic-shell"; - - buildInputs = - # Packages from garlic - (with garlic; [ tool garlicd ]) ++ - # Packages from bsc - [ groff paraver icc nix openssh git cn6 nix-diff clangOmpss2 gdb ]; - - # inputsFrom to get build dependencies - - shellHook = '' - alias l="ls -l --color=auto -v" - alias ll="ls -l --color=auto -v" - alias lh="ls -hAl --color=auto -v" - alias ls="ls --color=auto -v" - alias ..="cd .." - - export LANG=C - export SHELL=${bash}/bin/bash - - echo Welcome to the garlic shell - ''; -} diff --git a/garlic/stages/baywatch.nix b/garlic/stages/baywatch.nix deleted file mode 100644 index 2b72f4d..0000000 --- a/garlic/stages/baywatch.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ - stdenv -, garlicTools -}: -{ - nextStage -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "baywatch"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${stageProgram nextStage} - echo $? >> .srun.rc.$SLURM_PROCID - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/control.nix b/garlic/stages/control.nix deleted file mode 100644 index 3e04485..0000000 --- a/garlic/stages/control.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, loops ? 30 -}: - -with garlicTools; - -stdenv.mkDerivation { - name = "control"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<"EOF" - #!/bin/sh -e - - function badexit() { - errcode=$? - if [ $errcode != 0 ]; then - printf "exit %d\n" $errcode > "$basedir/status" - echo "exiting with $errcode" - fi - - echo 1 > "$basedir/done" - exit $errcode - } - - trap badexit EXIT - - basedir=$(pwd) - loops=${toString loops} - for n in $(seq 1 $loops); do - export GARLIC_RUN="$n" - echo "run $n/$loops" > status - mkdir "$n" - cd "$n" - mkdir .garlic - date +%s > .garlic/total_time_start - ${stageProgram nextStage} - date +%s > .garlic/total_time_end - cd .. - done - echo "ok" > status - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/exec.nix b/garlic/stages/exec.nix deleted file mode 100644 index 8717041..0000000 --- a/garlic/stages/exec.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, env ? "" -, pre ? "" -, argv ? [] -, post ? "" -, nixPrefix ? "" -, program ? null -}: - -with builtins; -with garlicTools; - -let - argvString = concatStringsSep " " (map (e: toString e) argv); - execMethod = if (post == "") then "exec " else ""; - programPath = if (program != null) then program else (stageProgram nextStage); -in -stdenv.mkDerivation { - name = "exec"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - ${env} - - ${pre} - - ${execMethod}${nixPrefix}${programPath} ${argvString} - - ${post} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/experiment.nix b/garlic/stages/experiment.nix deleted file mode 100644 index 19c0e45..0000000 --- a/garlic/stages/experiment.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - stdenv -, lib -, garlicTools -}: - -{ - units -}: - -with lib; -with garlicTools; - -let - unitsString = builtins.concatStringsSep "\n" - (map (x: "${stageProgram x}") units); - - unitsLinks = builtins.concatStringsSep "\n" - (map (x: "ln -s ../${baseNameOf x} ${baseNameOf x}") units); -in -stdenv.mkDerivation { - name = "experiment"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - inherit units; - - isExperiment = true; - - installPhase = '' - cat > $out << EOF - #!/bin/sh - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "experiment: GARLIC_OUT not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - export GARLIC_EXPERIMENT=$(basename $out) - - if [ -e "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "experiment: skipping, directory exists: \$GARLIC_EXPERIMENT" - exit 0 - fi - - mkdir -p "\$GARLIC_EXPERIMENT" - - cd "\$GARLIC_EXPERIMENT" - ${unitsLinks} - - >&2 echo "experiment: running \$GARLIC_EXPERIMENT" - - # This is an experiment formed by the following units: - ${unitsString} - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/extrae.nix b/garlic/stages/extrae.nix deleted file mode 100644 index dabb31e..0000000 --- a/garlic/stages/extrae.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ - stdenv -, garlicTools -}: - -{ - nextStage -, configFile -, traceLib -, extrae -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "extrae"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <&2 echo Running isolate stage1 -#>&2 echo PATH=$PATH - -if [ -e /nix ]; then - >&2 echo "/nix found, aborting" - exit 1 -fi - -nixhome="@nixPrefix@/nix" -shell="@busybox@/bin/sh" -nixjoin="@nixPrefix@@nixtools@/bin/nix-join" - - -env=( - PATH="@nixPrefix@@busybox@/bin:@busybox@/bin:@extraPath@" - $(env | grep ^SLURM || true) - $(env | grep ^PMI || true) - $(env | grep ^GARLIC || true) - $(env | grep ^USER || true) - $(env | grep ^TMPDIR || true) - $(env | grep '^TERM=' || true) - HOME="/homeless-shelter" -) - -mounts=( - #-m @nixPrefix@ - #FIXME: Use only the strictly neccesary from /etc - -m /original-etc:/etc - # The /etc/hosts file is a symlink to this etc/ - -m /.statelite/tmpfs/etc - -m /sys - -m /dev - -m /proc - # nscd cache: doesn't exist (?) - #-m /var/run/nscd - # Needed for munge auth - -m /var/run/munge - # FIXME: We should only need nix and the output path - -m /gpfs/projects/bsc15 - -m /gpfs/scratch/bsc15 - -m /bin:@nixPrefix@@busybox@/bin - -m "$TMPDIR" - @extraMountOptions@ -) - -symlinks=( - -s /etc/hosts:/original-etc/hosts - -s /etc/passwd:/original-etc/passwd - -s /etc/resolv.conf:/original-etc/resolv.conf - -s /etc/host.conf:/original-etc/host.conf - -s /etc/slurm/slurm.conf:/original-etc/slurm/slurm.conf - -s /etc/services:/original-etc/services -) - -exec $nixjoin -i "${mounts[@]}" "${symlinks[@]}" $nixhome -- \ - env -i "${env[@]}" @out@/bin/stage2 "$@" diff --git a/garlic/stages/isolate/stage2 b/garlic/stages/isolate/stage2 deleted file mode 100644 index 1924890..0000000 --- a/garlic/stages/isolate/stage2 +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -#>&2 echo Running isolate stage2 -#>&2 echo PATH=$PATH - -if [ ! -e /nix ]; then - >&2 echo "/nix not found, aborting" - exit 1 -fi - -if [ -e /usr ]; then - >&2 echo "Environment not isolated, aborting" - exit 1 -fi - -exec @program@ "$@" diff --git a/garlic/stages/perf.nix b/garlic/stages/perf.nix deleted file mode 100644 index cbbe937..0000000 --- a/garlic/stages/perf.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ - stdenv -, perf -, garlicTools -}: - -{ - nextStage -, perfOptions -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "perf"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out <&2 echo "Cannot use runexp inside nix environment!" - exit 1 -fi - -#>&2 echo Running runexp for MN4 -#>&2 echo PATH=$PATH - -user=$(id -un) -group=$(id -gn) - -export GARLIC_OUT="/gpfs/projects/bsc15/garlic/$user/out" -mkdir -p "$GARLIC_OUT" -export GARLIC_INDEX="/gpfs/projects/bsc15/garlic/$user/index" -mkdir -p "$GARLIC_INDEX" -export GARLIC_USER="$user" -cd "$GARLIC_OUT" - -exec @nixPrefix@@program@ diff --git a/garlic/stages/sbatch.nix b/garlic/stages/sbatch.nix deleted file mode 100644 index 34a3167..0000000 --- a/garlic/stages/sbatch.nix +++ /dev/null @@ -1,105 +0,0 @@ -{ - stdenv -, lib -, numactl -, slurm -, garlicTools -}: - -{ - nextStage -, jobName -, chdir ? "." -, nixPrefix ? "" -, binary ? "/bin/run" -, ntasks ? null -, ntasksPerNode ? null -, ntasksPerSocket ? null -, cpusPerTask ? null -, nodes ? null -, exclusive ? true # By default we run in exclusive mode -, qos ? null -, reservation ? null -, time ? null -, output ? "stdout.log" -, error ? "stderr.log" -, extra ? null -, acctgFreq ? null -}: - -with lib; -with garlicTools; - -# sbatch fails silently if we pass garbage, so we assert the types here to avoid -# sending `nodes = [ 1 2 ]` by mistake. -assert (jobName != null) -> isString jobName; -assert (chdir != null) -> isString chdir; -assert (nixPrefix != null) -> isString nixPrefix; -assert (ntasks != null) -> isInt ntasks; -assert (ntasksPerNode != null) -> isInt ntasksPerNode; -assert (ntasksPerSocket != null) -> isInt ntasksPerSocket; -assert (cpusPerTask != null) -> isInt cpusPerTask; -assert (nodes != null) -> isInt nodes; -assert (exclusive != null) -> isBool exclusive; -assert (qos != null) -> isString qos; -assert (reservation != null) -> isString reservation; -assert (time != null) -> isString time; -assert (output != null) -> isString output; -assert (error != null) -> isString error; -assert (extra != null) -> isString extra; - -let - - sbatchOpt = name: value: optionalString (value!=null) - "#SBATCH --${name}=${toString value}\n"; - sbatchEnable = name: value: optionalString (value!=null) - "#SBATCH --${name}\n"; - -in - -stdenv.mkDerivation rec { - name = "sbatch"; - preferLocalBuild = true; - - phases = [ "installPhase" ]; - - #SBATCH --tasks-per-node=48 - #SBATCH --ntasks-per-socket=24 - #SBATCH --cpus-per-task=1 - dontBuild = true; - dontPatchShebangs = true; - programPath = "/run"; - - installPhase = '' - mkdir -p $out - cat > $out/job < $out/run < status - EOF - chmod +x $out/run - ''; -} diff --git a/garlic/stages/script.nix b/garlic/stages/script.nix deleted file mode 100644 index 292a209..0000000 --- a/garlic/stages/script.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ - stdenv -}: - -{ - script -, shell ? "/bin/sh" -, exitOnError ? true -}: - -let - setcmd = if exitOnError then "set -e" else ""; -in -stdenv.mkDerivation { - name = "script"; - preferLocalBuild = true; - phases = [ "installPhase" ]; - installPhase = '' - cat > $out <<'EOF' - #!${shell} - ${setcmd} - - ${script} - - EOF - chmod +x $out - ''; -} diff --git a/garlic/stages/srun.nix b/garlic/stages/srun.nix deleted file mode 100644 index a2481a5..0000000 --- a/garlic/stages/srun.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ - stdenv -, slurm -, garlicTools -}: -{ - nextStage -, cpuBind -, nixPrefix -, preSrun ? "" -, postSrun ? "" -, srunOptions ? "" -, output ? "stdout.log" -, error ? "stderr.log" -}: - -with garlicTools; - -stdenv.mkDerivation rec { - name = "srun"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out <<'EOF' - #!/bin/sh -e - - ${preSrun} - - ${slurm}/bin/srun \ - --mpi=pmi2 \ - --cpu-bind=${cpuBind} \ - --output=${output} \ - --error=${error} \ - ${srunOptions} \ - ${nixPrefix}${stageProgram nextStage} - - >&2 echo srun exit code: $? - - # Ensure that none failed, as srun fails to capture errors - # after MPI_Finalize - for i in $(seq 0 $(($SLURM_NTASKS - 1))); do - if [ ! -e .srun.rc.$i ]; then - >&2 echo "missing exit code for rank $i, aborting" - exit 1 - fi - if ! grep -q '^0$' .srun.rc.$i; then - >&2 echo "non-zero exit for rank $i, aborting" - exit 1 - fi - done - - ${postSrun} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/strace.nix b/garlic/stages/strace.nix deleted file mode 100644 index 22fb4a5..0000000 --- a/garlic/stages/strace.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, strace -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "strace"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out < $out << EOF - #!/bin/sh -e - - ${desc} - - if [ -z "\$GARLIC_OUT" ]; then - >&2 echo "unit: GARLIC_OUT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_EXPERIMENT" ]; then - >&2 echo "unit: GARLIC_EXPERIMENT not defined, aborting" - exit 1 - fi - - if [ -z "\$GARLIC_INDEX" ]; then - >&2 echo "unit: GARLIC_INDEX not defined, aborting" - exit 1 - fi - - cd "\$GARLIC_OUT" - - # Set the experiment unit in the environment - export GARLIC_UNIT=$(basename $out) - - # Create an index entry - rm -f "\$GARLIC_INDEX/${safeUnitName}" \ - "\$GARLIC_INDEX/${safeExpName}" - - ln -Tfs "../out/\$GARLIC_UNIT" \ - "\$GARLIC_INDEX/${safeUnitName}" - - ln -Tfs "../out/\$GARLIC_EXPERIMENT" \ - "\$GARLIC_INDEX/${safeExpName}" - - if [ -e "\$GARLIC_UNIT" ]; then - >&2 echo "unit: skipping, already exists: \$GARLIC_UNIT" - exit 0 - fi - - # And change the working directory - mkdir \$GARLIC_UNIT - cd \$GARLIC_UNIT - - # Copy the configuration for the unit to the output path - cp ${jsonConf} garlic_config.json - - # Finally, execute the first stage: - exec ${firstStage} - EOF - - chmod +x $out - ''; -} diff --git a/garlic/stages/valgrind.nix b/garlic/stages/valgrind.nix deleted file mode 100644 index 9e156c0..0000000 --- a/garlic/stages/valgrind.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ - stdenv -, valgrind -, garlicTools -}: - -{ - nextStage -}: - -with garlicTools; - -let - program = stageProgram nextStage; -in - stdenv.mkDerivation { - name = "valgrind"; - phases = [ "installPhase" ]; - preferLocalBuild = true; - dontPatchShebangs = true; - installPhase = '' - cat > $out < $out''); -} diff --git a/garlic/tools.nix b/garlic/tools.nix deleted file mode 100644 index c092b55..0000000 --- a/garlic/tools.nix +++ /dev/null @@ -1,140 +0,0 @@ -{ - stdenv -, lib -}: - -with lib; - -let - gen = rec { - # genAttrSets "a" ["hello" "world"] - # [ { a = "hello"; } { a = "world"; } ] - genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); - - # addAttrSets "a" [1 2] {e=4;} - # [ { a = 1; e = 4; } { a = 2; e = 4; } ] - addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); - - # attrToList {a=1;} - # [ { name = "a"; value = 1; } ] - attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); - - # mergeConfig [{e=1;}] {name="a"; value=[1 2] - # [ { a = 1; e = 1; } { a = 2; e = 1; } ] - mergeConfig = (arr: new: flatten ( map (x: addAttrSets new.name new.value x) arr)); - - # genConfigs {a=[1 2]; b=[3 4];} - # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] - genConfigs = (config: foldl mergeConfig [{}] (attrToList config)); - - # Generate multiple app versions by override with each config - genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs); - - # Generate app version from an array of apps - genApps = (apps: configs: - flatten (map (app: genApp app configs) apps)); - - /* Returns the path of the executable of a stage */ - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" - else "${stage}"; - - /* Given a trebuchet, returns the experiment */ - getExperimentStage = drv: - if (drv ? isExperiment) && drv.isExperiment then drv - else getExperimentStage drv.nextStage; - - # Computes the exponentiation operation - pow = x: n: fold (a: b: a*b) 1 (map (a: x) (range 1 n)); - - # Generates a list of exponents from a to b inclusive, and raises base to - # each element of the list. - expRange = base: a: b: (map (ex: pow base ex) (range a b)); - - # Generates a range from start to end (inclusive) by multiplying start by 2. - range2 = start: end: - let - _range2 = s: e: if (s > e) then [] else [ s ] ++ (_range2 (s * 2) e); - in - _range2 start end; - - # Generates a list of integers by halving number N until it reaches 1. Is - # sorted from the smallest to largest. - halfList = N: - let - _divList = n: if (n == 0) then [] else (_divList (n / 2)) ++ [ n ]; - in - _divList N; - - # A list of all divisors of n, sorted in increased order: - divisors = n: filter (x: (mod n x == 0)) (range 1 n); - - # Generates a set given a list of keys, where all values are null. - genNullAttr = l: genAttrs l (name: null); - - # From the keys in the lis l, generates a set with the values in the set a, - # if they don't exist, they are not taken. Values set to null are removed. - optionalInherit = l: a: filterAttrs (n: v: v!=null) - (overrideExisting (genNullAttr l) a); - - # Given a float f, truncates it and returns the resulting the integer - floatTruncate = f: let - strFloat = toString f; - slices = splitString "." strFloat; - front = elemAt slices 0; - in - toInt front; - - # Returns the given gitCommit if not null, or the one stored in the - # gitTable for the branch gitBranch. - findCommit = {gitCommit ? null, gitTable ? null, gitBranch}: - assert (gitCommit == null) -> (gitTable != null); - assert (gitTable == null) -> (gitCommit != null); - if (gitCommit != null) then gitCommit - else - assert (assertMsg (gitTable ? "${gitBranch}") - '' - The git branch "${gitBranch}" was not found in the gitTable. - Is the gitTable outdated? - ''); - gitTable."${gitBranch}"; - - # Custom wrapper around fetchGit to be able to quickly specify apps - # and change the repository source for all at once. Returns an - # attributte set with the `src` as well as the selected gitCommit, - # gitBranch and gitURL. - fetchGarlicApp = { - gitBranch, - appName ? null, - gitURL ? null, - gitCommit ? null, - gitTable ? null - }: - assert (appName == null) -> (gitURL != null); - assert (gitURL == null) -> (appName != null); - let - _gitURL = if (gitURL != null) then gitURL - else "ssh://git@bscpm03.bsc.es/garlic/apps/${appName}.git"; - _gitCommit = findCommit { - inherit gitCommit gitTable gitBranch; - }; - _gitBranch = gitBranch; - in - { - src = builtins.fetchGit { - url = _gitURL; - ref = _gitBranch; - rev = _gitCommit; - }; - gitBranch = _gitBranch; - gitCommit = _gitCommit; - - # The gitURL is not stored in the derivation, as we dont really - # care of where the source comes from, as long as is the same - # commit. - gitURL = _gitURL; - }; - }; -in - gen