From 1e07be863ac4d542fb5d0b02ed21a73600433d73 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Tue, 18 Aug 2020 18:28:30 +0200 Subject: [PATCH] Add OSU test benchmarks --- bsc/garlic/argv.nix | 3 +- bsc/garlic/default.nix | 3 ++ bsc/garlic/exp/nbody/mpi.nix | 19 ++++++--- bsc/garlic/exp/osu/latency.nix | 70 ++++++++++++++++++++++++++++++++++ bsc/garlic/sbatch.nix | 2 + bsc/osu/default.nix | 38 ++++++++++++++++++ default.nix | 9 +++-- 7 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 bsc/garlic/exp/osu/latency.nix create mode 100644 bsc/osu/default.nix diff --git a/bsc/garlic/argv.nix b/bsc/garlic/argv.nix index 701b8d7..73adc16 100644 --- a/bsc/garlic/argv.nix +++ b/bsc/garlic/argv.nix @@ -7,6 +7,7 @@ app , env ? "" , argv # bash array as string, example: argv=''(-f "file with spaces" -t 10)'' +, program ? "bin/run" }: stdenv.mkDerivation { @@ -23,7 +24,7 @@ stdenv.mkDerivation { ${env} argv=${argv} - exec ${app}/bin/run \''${argv[@]} + exec ${app}/${program} \''${argv[@]} EOF chmod +x $out/bin/run ''; diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix index c975b25..38d7e73 100644 --- a/bsc/garlic/default.nix +++ b/bsc/garlic/default.nix @@ -36,6 +36,9 @@ let bs = callPackage ./exp/nbody/bs.nix { }; mpi = callPackage ./exp/nbody/mpi.nix { }; }; + osu = { + latency = callPackage ./exp/osu/latency.nix { }; + }; }; }; diff --git a/bsc/garlic/exp/nbody/mpi.nix b/bsc/garlic/exp/nbody/mpi.nix index 64f1279..a618f85 100644 --- a/bsc/garlic/exp/nbody/mpi.nix +++ b/bsc/garlic/exp/nbody/mpi.nix @@ -18,13 +18,17 @@ let config = { cc = [ bsc.icc ]; blocksize = [ 2048 ]; + mpi = [ bsc.impi bsc.openmpi bsc.mpich ]; }; extraConfig = { - particles = 16384; + particles = 32*1024; timesteps = 10; - ntasks = 2; - mpi = bsc.impi; + ntasksPerNode = 2; + nodes = 1; + time = "00:10:00"; + qos = "debug"; + #mpi = bsc.impi; #mpi = bsc.openmpi; gitBranch = "garlic/mpi+send"; gitURL = "ssh://git@bscpm02.bsc.es/garlic/apps/nbody.git"; @@ -37,7 +41,10 @@ let app = app; nixPrefix = "/gpfs/projects/bsc15/nix"; exclusive = false; - ntasks = "${toString conf.ntasks}"; + ntasksPerNode = "${toString conf.ntasksPerNode}"; + nodes = "${toString conf.nodes}"; + time = conf.time; + qos = conf.qos; chdirPrefix = "/home/bsc15/bsc15557/bsc-nixpkgs/out"; }; @@ -61,7 +68,7 @@ let nbody.override { inherit cc mpi blocksize gitBranch gitURL; }; pipeline = conf: -# sbatch conf ( + sbatch conf ( srun ( nixsetupWrapper ( argv conf ( @@ -69,7 +76,7 @@ let ) ) ) -# ) + ) ; # Ideally it should look like this: diff --git a/bsc/garlic/exp/osu/latency.nix b/bsc/garlic/exp/osu/latency.nix new file mode 100644 index 0000000..f3a0145 --- /dev/null +++ b/bsc/garlic/exp/osu/latency.nix @@ -0,0 +1,70 @@ +{ + bsc +, genApp +, genConfigs + +# Wrappers +, launchWrapper +, sbatchWrapper +, srunWrapper +, argvWrapper +, controlWrapper +, nixsetupWrapper +}: + +let + # Set the configuration for the experiment + config = { + mpi = [ bsc.impi bsc.openmpi bsc.mpich ]; + }; + + extraConfig = { + ntasksPerNode = 1; + nodes = 2; + time = "00:10:00"; + qos = "debug"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // extraConfig) (genConfigs config); + + sbatch = conf: app: sbatchWrapper { + app = app; + nixPrefix = "/gpfs/projects/bsc15/nix"; + exclusive = false; + ntasksPerNode = "${toString conf.ntasksPerNode}"; + nodes = "${toString conf.nodes}"; + time = conf.time; + qos = conf.qos; + chdirPrefix = "/home/bsc15/bsc15557/bsc-nixpkgs/out"; + }; + + srun = app: srunWrapper { + app = app; + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + argv = app: + argvWrapper { + app = app; + program = "bin/osu_latency"; + argv = "()"; + env = '' + export I_MPI_THREAD_SPLIT=1 + ''; + }; + + osumbFn = conf: + with conf; + bsc.osumb.override { inherit mpi; }; + + pipeline = conf: srun (nixsetupWrapper (argv (osumbFn conf))); + #pipeline = conf: sbatch conf (srun (nixsetupWrapper (argv bsc.osumb))); + + # Ideally it should look like this: + #pipeline = sbatch nixsetup control argv nbodyFn; + + jobs = map pipeline configs; + +in + launchWrapper jobs diff --git a/bsc/garlic/sbatch.nix b/bsc/garlic/sbatch.nix index f503e23..13092ba 100644 --- a/bsc/garlic/sbatch.nix +++ b/bsc/garlic/sbatch.nix @@ -10,6 +10,7 @@ , argv ? "" , binary ? "/bin/run" , ntasks ? null +, ntasksPerNode ? null , nodes ? null , exclusive ? true # By default we run in exclusive mode , qos ? null @@ -51,6 +52,7 @@ stdenv.mkDerivation rec { #SBATCH --job-name="${name}" '' + sbatchOpt "ntasks" ntasks + + sbatchOpt "ntasks-per-node" ntasksPerNode + sbatchOpt "nodes" nodes + sbatchOpt "chdir" "${chdirPrefix}/$(basename $out)" + sbatchOpt "output" output diff --git a/bsc/osu/default.nix b/bsc/osu/default.nix new file mode 100644 index 0000000..bbe8dc1 --- /dev/null +++ b/bsc/osu/default.nix @@ -0,0 +1,38 @@ +{ + stdenv, + fetchurl, + mpi +}: + +stdenv.mkDerivation rec { + version = "5.6.3"; + name = "osu-micro-benchmarks-${version}"; + + src = fetchurl { + url = "http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-${version}.tar.gz"; + sha256 = "1f5fc252c0k4rd26xh1v5017wfbbsr2w7jm49x8yigc6n32sisn5"; + }; + + doCheck = true; + enableParallelBuilding = true; + buildInputs = [ mpi ]; + configureFlags = [ + "CC=${mpi}/bin/mpicc" + "CXX=${mpi}/bin/mpicxx" + ]; + + postInstall = '' + mkdir -p $out/bin + cp $out/libexec/osu-micro-benchmarks/mpi/one-sided/* $out/bin/ + cp $out/libexec/osu-micro-benchmarks/mpi/collective/* $out/bin/ + cp $out/libexec/osu-micro-benchmarks/mpi/pt2pt/* $out/bin/ + cp $out/libexec/osu-micro-benchmarks/mpi/startup/* $out/bin/ + ''; + + meta = { + description = "OSU Micro-Benchmarks"; + homepage = http://mvapich.cse.ohio-state.edu/benchmarks/; + maintainers = [ ]; + platforms = stdenv.lib.platforms.all; + }; +} diff --git a/default.nix b/default.nix index e28070d..44e919e 100644 --- a/default.nix +++ b/default.nix @@ -9,9 +9,8 @@ let self.bsc = rec { # Load the default implementation - #mpi = pkgs.mpich; - #mpi = pkgs.openmpi; - #mpi = openmpi; # Our OpenMPI variant + #mpi = mpich; + #mpi = openmpi; mpi = intel-mpi; # Load the default compiler @@ -37,6 +36,10 @@ let pscom = callPackage ./bsc/parastation/pscom.nix { }; psmpi = callPackage ./bsc/parastation/psmpi.nix { }; + osumb = callPackage ./bsc/osu/default.nix { }; + + mpich = pkgs.mpich; + # Default Intel MPI version is 2019 (the last one) impi = intel-mpi; intel-mpi = intel-mpi-2019;