From 5cbc8e4fbbc9ab3ad3f5ee6ed2b0bb63094d7317 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Wed, 30 Sep 2020 17:32:49 +0200 Subject: [PATCH 1/5] First attempt to create an experiment with CREAMS: strong scaling from 1 to 16 nodes using the pure MPI version --- garlic/creams/default.nix | 30 ++++- garlic/creams/input.nix | 27 ++++ garlic/exp/creams/SS+mpi+send+seq.nix | 173 ++++++++++++++++++++++++++ garlic/exp/creams/test.nix | 173 ++++++++++++++++++++++++++ garlic/exp/creams/test.nix~ | 169 +++++++++++++++++++++++++ garlic/stages/sbatch.nix | 2 +- overlay.nix | 33 +++-- 7 files changed, 594 insertions(+), 13 deletions(-) create mode 100644 garlic/creams/input.nix create mode 100644 garlic/exp/creams/SS+mpi+send+seq.nix create mode 100644 garlic/exp/creams/test.nix create mode 100644 garlic/exp/creams/test.nix~ diff --git a/garlic/creams/default.nix b/garlic/creams/default.nix index 3e8d822..ac704c4 100644 --- a/garlic/creams/default.nix +++ b/garlic/creams/default.nix @@ -2,23 +2,39 @@ stdenv , nanos6 , mpi +, openmpi +, impi , tampi , mcxx -, icc +, cc +, gitBranch }: +assert (mpi == impi || mpi == openmpi); + +let + mpiName = (if mpi == openmpi then + "OpenMPI" + else + "IntelMPI"); + +in stdenv.mkDerivation rec { name = "creams"; + # src = /home/Computational/pmartin1/creams-simplified; + src = builtins.fetchGit { url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git"; - ref = "MPI+OmpSs-2+TAMPI"; + ref = "${gitBranch}"; }; + programPath = "/bin/creams.exe"; + buildInputs = [ nanos6 mpi - icc + cc tampi mcxx ]; @@ -27,12 +43,18 @@ stdenv.mkDerivation rec { configurePhase = '' export TAMPI_HOME=${tampi} + . etc/bashrc + #export FORTRAN_COMPILER=GNU # GCC compiler + export MPI_LIB=${mpiName} + + echo + + CREAMS_UPDATE_ENVIRONMENT ''; installPhase = '' mkdir -p $out/bin cp -a build/* $out/bin ''; - } diff --git a/garlic/creams/input.nix b/garlic/creams/input.nix new file mode 100644 index 0000000..33405e1 --- /dev/null +++ b/garlic/creams/input.nix @@ -0,0 +1,27 @@ +{ + stdenv +, nodes +, gitBranch +}: + +stdenv.mkDerivation rec { + name = "creams-input"; + + # src = /home/Computational/pmartin1/creams-simplified; + + src = builtins.fetchGit { + url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git"; + ref = "${gitBranch}"; + }; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + pushd SodTubeBenchmark + bash gridScript.sh 0 0 $((${toString nodes}*48)) 0 + popd + + mkdir -p $out + cp -a SodTubeBenchmark $out/ + ''; +} diff --git a/garlic/exp/creams/SS+mpi+send+seq.nix b/garlic/exp/creams/SS+mpi+send+seq.nix new file mode 100644 index 0000000..c4ade80 --- /dev/null +++ b/garlic/exp/creams/SS+mpi+send+seq.nix @@ -0,0 +1,173 @@ +{ + stdenv +, nixpkgs +, pkgs +, genApp +, genConfigs +, runWrappers +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Set variable configuration for the experiment + varConfig = { + # cc = [ self.gcc10 ]; # Does not work + # mpi = [ bsc.openmpi ]; # Does not work + # cc = [ bsc.icc ]; + # mpi = [ bsc.impi ]; + + nodes = [ 1 2 4 8 16 ]; + }; + + # Common configuration + common = { + gitBranch = "garlic/mpi+send+seq"; + + cc = bsc.icc ; + mpi = bsc.impi; + + # Resources + ntasksPerNode = 48; + #ntasksPerSocket = 24; // Add this variable to nix + + # Stage configuration + enableSbatch = true; + enableControl = true; + enableExtrae = false; + enablePerf = false; + enableCtf = false; + + # MN4 path + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // common) (genConfigs varConfig); + + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" else "${stage}"; + + w = runWrappers; + + sbatch = {stage, conf, ...}: with conf; w.sbatch { + program = stageProgram stage; + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-bs"; + inherit nixPrefix nodes ntasksPerNode; + }; + + control = {stage, conf, ...}: with conf; w.control { + program = stageProgram stage; + }; + + srun = {stage, conf, ...}: with conf; w.srun { + program = stageProgram stage; + srunOptions = "--cpu-bind=verbose,rank"; + inherit nixPrefix; + }; + + statspy = {stage, conf, ...}: with conf; w.statspy { + program = stageProgram stage; + }; + + perf = {stage, conf, ...}: with conf; w.perf { + program = stageProgram stage; + perfArgs = "sched record -a"; + }; + + nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + program = stageProgram stage; + nixsetup = "${nixPrefix}/bin/nix-setup"; + }; + + extrae = {stage, conf, ...}: w.extrae { + program = stageProgram stage; + traceLib = "mpi"; # mpi -> libtracempi.so + configFile = ./extrae.xml; + }; + + ctf = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + export NANOS6=ctf + export NANOS6_CTF2PRV=0 + ''; + }; + + bscOverlay = import ../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; + + inputDataset = {stage, conf, ...}: with conf; + let + input = bsc.garlic.creamsInput.override { + inherit gitBranch nodes; + }; + in w.argv + { + program = stageProgram stage; + env = '' + cp -r ${input}/SodTubeBenchmark/* . + + pwd + ls -l + ''; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + creamsFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.creams.override { + inherit cc mpi gitBranch; + }; + + stages = with common; [] + # Use sbatch to request resources first + ++ optional enableSbatch sbatch + + # Repeats the next stages N times + ++ optionals enableControl [ nixsetup control ] + + # Executes srun to launch the program in the requested nodes, and + # immediately after enters the nix environment again, as slurmstepd launches + # the next stages from outside the namespace. + ++ [ srun nixsetup ] + + # Intrumentation with extrae + ++ optional enableExtrae extrae + + # Optionally profile the next stages with perf + ++ optional enablePerf perf + + # Optionally profile nanos6 with the new ctf + ++ optional enableCtf ctf + + # Execute the nbody app with the argv and env vars + ++ [ inputDataset creamsFn ]; + + # List of actual programs to be executed + jobs = map (conf: w.stagen { inherit conf stages; }) configs; + +in + # We simply run each program one after another + w.launch jobs diff --git a/garlic/exp/creams/test.nix b/garlic/exp/creams/test.nix new file mode 100644 index 0000000..c4ade80 --- /dev/null +++ b/garlic/exp/creams/test.nix @@ -0,0 +1,173 @@ +{ + stdenv +, nixpkgs +, pkgs +, genApp +, genConfigs +, runWrappers +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Set variable configuration for the experiment + varConfig = { + # cc = [ self.gcc10 ]; # Does not work + # mpi = [ bsc.openmpi ]; # Does not work + # cc = [ bsc.icc ]; + # mpi = [ bsc.impi ]; + + nodes = [ 1 2 4 8 16 ]; + }; + + # Common configuration + common = { + gitBranch = "garlic/mpi+send+seq"; + + cc = bsc.icc ; + mpi = bsc.impi; + + # Resources + ntasksPerNode = 48; + #ntasksPerSocket = 24; // Add this variable to nix + + # Stage configuration + enableSbatch = true; + enableControl = true; + enableExtrae = false; + enablePerf = false; + enableCtf = false; + + # MN4 path + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // common) (genConfigs varConfig); + + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" else "${stage}"; + + w = runWrappers; + + sbatch = {stage, conf, ...}: with conf; w.sbatch { + program = stageProgram stage; + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-bs"; + inherit nixPrefix nodes ntasksPerNode; + }; + + control = {stage, conf, ...}: with conf; w.control { + program = stageProgram stage; + }; + + srun = {stage, conf, ...}: with conf; w.srun { + program = stageProgram stage; + srunOptions = "--cpu-bind=verbose,rank"; + inherit nixPrefix; + }; + + statspy = {stage, conf, ...}: with conf; w.statspy { + program = stageProgram stage; + }; + + perf = {stage, conf, ...}: with conf; w.perf { + program = stageProgram stage; + perfArgs = "sched record -a"; + }; + + nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + program = stageProgram stage; + nixsetup = "${nixPrefix}/bin/nix-setup"; + }; + + extrae = {stage, conf, ...}: w.extrae { + program = stageProgram stage; + traceLib = "mpi"; # mpi -> libtracempi.so + configFile = ./extrae.xml; + }; + + ctf = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + export NANOS6=ctf + export NANOS6_CTF2PRV=0 + ''; + }; + + bscOverlay = import ../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; + + inputDataset = {stage, conf, ...}: with conf; + let + input = bsc.garlic.creamsInput.override { + inherit gitBranch nodes; + }; + in w.argv + { + program = stageProgram stage; + env = '' + cp -r ${input}/SodTubeBenchmark/* . + + pwd + ls -l + ''; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + creamsFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.creams.override { + inherit cc mpi gitBranch; + }; + + stages = with common; [] + # Use sbatch to request resources first + ++ optional enableSbatch sbatch + + # Repeats the next stages N times + ++ optionals enableControl [ nixsetup control ] + + # Executes srun to launch the program in the requested nodes, and + # immediately after enters the nix environment again, as slurmstepd launches + # the next stages from outside the namespace. + ++ [ srun nixsetup ] + + # Intrumentation with extrae + ++ optional enableExtrae extrae + + # Optionally profile the next stages with perf + ++ optional enablePerf perf + + # Optionally profile nanos6 with the new ctf + ++ optional enableCtf ctf + + # Execute the nbody app with the argv and env vars + ++ [ inputDataset creamsFn ]; + + # List of actual programs to be executed + jobs = map (conf: w.stagen { inherit conf stages; }) configs; + +in + # We simply run each program one after another + w.launch jobs diff --git a/garlic/exp/creams/test.nix~ b/garlic/exp/creams/test.nix~ new file mode 100644 index 0000000..ac598d2 --- /dev/null +++ b/garlic/exp/creams/test.nix~ @@ -0,0 +1,169 @@ +{ + stdenv +, nixpkgs +, pkgs +, genApp +, genConfigs +, runWrappers +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Set variable configuration for the experiment + varConfig = { + mpi = [ bsc.impi bsc.openmpi ]; + nodes = [ 1 ]; + }; + + # Common configuration + common = { + # Compile time nbody config + gitBranch = "garlic/mpi+send+seq"; + + cc = bsc.icc; + + # Resources + ntasksPerNode = 48; + nodes = 1; + + # Stage configuration + enableSbatch = true; + enableControl = true; + enableExtrae = false; + enablePerf = false; + enableCtf = false; + + # MN4 path + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // common) (genConfigs varConfig); + + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" else "${stage}"; + + w = runWrappers; + + sbatch = {stage, conf, ...}: with conf; w.sbatch { + program = stageProgram stage; + exclusive = true; + time = "02:00:00"; + qos = "debug"; + jobName = "nbody-bs"; + inherit nixPrefix nodes ntasksPerNode; + }; + + control = {stage, conf, ...}: with conf; w.control { + program = stageProgram stage; + }; + + srun = {stage, conf, ...}: with conf; w.srun { + program = stageProgram stage; + srunOptions = "--cpu-bind=verbose,rank"; + inherit nixPrefix; + }; + + statspy = {stage, conf, ...}: with conf; w.statspy { + program = stageProgram stage; + }; + + perf = {stage, conf, ...}: with conf; w.perf { + program = stageProgram stage; + perfArgs = "sched record -a"; + }; + + nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + program = stageProgram stage; + nixsetup = "${nixPrefix}/bin/nix-setup"; + }; + + extrae = {stage, conf, ...}: w.extrae { + program = stageProgram stage; + traceLib = "mpi"; # mpi -> libtracempi.so + configFile = ./extrae.xml; + }; + + ctf = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + export NANOS6=ctf + export NANOS6_CTF2PRV=0 + ''; + }; + + bscOverlay = import ../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; + + inputDataset = {stage, conf, ...}: with conf; + let + input = bsc.garlic.creamsInput.override { + inherit gitBranch nodes; + }; + in w.argv + { + program = stageProgram stage; + env = '' + cp -r ${input}/SodTubeBenchmark/* . + + pwd + ls -l + ''; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + creamsFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.creams.override { + inherit cc mpi gitBranch; + }; + + stages = with common; [] + # Use sbatch to request resources first + ++ optional enableSbatch sbatch + + # Repeats the next stages N times + ++ optionals enableControl [ nixsetup control ] + + # Executes srun to launch the program in the requested nodes, and + # immediately after enters the nix environment again, as slurmstepd launches + # the next stages from outside the namespace. + ++ [ srun nixsetup ] + + # Intrumentation with extrae + ++ optional enableExtrae extrae + + # Optionally profile the next stages with perf + ++ optional enablePerf perf + + # Optionally profile nanos6 with the new ctf + ++ optional enableCtf ctf + + # Execute the nbody app with the argv and env vars + ++ [ inputDataset creamsFn ]; + + # List of actual programs to be executed + jobs = map (conf: w.stagen { inherit conf stages; }) configs; + +in + # We simply run each program one after another + w.launch jobs diff --git a/garlic/stages/sbatch.nix b/garlic/stages/sbatch.nix index c9d0bb9..d2a936d 100644 --- a/garlic/stages/sbatch.nix +++ b/garlic/stages/sbatch.nix @@ -28,7 +28,7 @@ with stdenv.lib; let sbatchOpt = name: value: optionalString (value!=null) - "#SBATCH --${name}=${value}\n"; + "#SBATCH --${name}=${toString value}\n"; sbatchEnable = name: value: optionalString (value!=null) "#SBATCH --${name}\n"; diff --git a/overlay.nix b/overlay.nix index ce8ed51..c01e311 100644 --- a/overlay.nix +++ b/overlay.nix @@ -143,14 +143,21 @@ let # tampi = tampi; # }; # -# creams = callPackage ./garlic/creams { -# stdenv = pkgs.gcc9Stdenv; -# mpi = intel-mpi; -# tampi = tampi.override { -# mpi = intel-mpi; -# }; -# }; -# + creams = callPackage ./garlic/creams { + gitBranch = "garlic/mpi+send+seq"; + + #cc = self.gcc10; # Does not work + #mpi = self.bsc.openmpi-mn4; # Does not work + + cc = self.bsc.icc; + mpi = self.bsc.mpi; + }; + + creamsInput = callPackage ./garlic/creams/input.nix { + gitBranch = "garlic/mpi+send+seq"; + nodes = 1; + }; + # lulesh = callPackage ./garlic/lulesh { # mpi = intel-mpi; # }; @@ -231,6 +238,16 @@ let }; }; + creams = { + SS_mpi_send_seq = callPackage ./garlic/exp/creams/SS+mpi+send+seq.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; + }; + osu = rec { latency-internode = callPackage ./garlic/exp/osu/latency.nix { }; latency-intranode = callPackage ./garlic/exp/osu/latency.nix { From 6ae71cc5e9dc065b64453228b220e45d98c0912a Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Thu, 1 Oct 2020 16:48:35 +0200 Subject: [PATCH 2/5] Improvement the experiment based on CREAMS --- garlic/creams/default.nix | 11 +- garlic/creams/input.nix | 12 +- garlic/exp/creams/SS+mpi+send+seq.nix | 40 +++--- garlic/exp/creams/test.nix | 173 -------------------------- overlay.nix | 11 +- 5 files changed, 44 insertions(+), 203 deletions(-) delete mode 100644 garlic/exp/creams/test.nix diff --git a/garlic/creams/default.nix b/garlic/creams/default.nix index ac704c4..b1dd26e 100644 --- a/garlic/creams/default.nix +++ b/garlic/creams/default.nix @@ -6,6 +6,8 @@ , impi , tampi , mcxx +, gnuDef +, intelDef , cc , gitBranch }: @@ -18,12 +20,16 @@ let else "IntelMPI"); + compName = (if cc == intelDef then + "Intel" + else + "GNU"); + in stdenv.mkDerivation rec { name = "creams"; # src = /home/Computational/pmartin1/creams-simplified; - src = builtins.fetchGit { url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git"; ref = "${gitBranch}"; @@ -45,7 +51,8 @@ stdenv.mkDerivation rec { export TAMPI_HOME=${tampi} . etc/bashrc - #export FORTRAN_COMPILER=GNU # GCC compiler + + export FORTRAN_COMPILER=${compName} export MPI_LIB=${mpiName} echo diff --git a/garlic/creams/input.nix b/garlic/creams/input.nix index 33405e1..30da776 100644 --- a/garlic/creams/input.nix +++ b/garlic/creams/input.nix @@ -1,6 +1,7 @@ { stdenv -, nodes +, granul ? 0 +, nprocz ? 0 , gitBranch }: @@ -8,17 +9,20 @@ stdenv.mkDerivation rec { name = "creams-input"; # src = /home/Computational/pmartin1/creams-simplified; - src = builtins.fetchGit { url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git"; ref = "${gitBranch}"; }; - phases = [ "unpackPhase" "installPhase" ]; + phases = [ "unpackPhase" "patchPhase" "installPhase" ]; + + patchPhase = '' + patchShebangs SodTubeBenchmark/gridScript.sh + ''; installPhase = '' pushd SodTubeBenchmark - bash gridScript.sh 0 0 $((${toString nodes}*48)) 0 + ./gridScript.sh 0 0 ${toString nprocz} ${toString granul} popd mkdir -p $out diff --git a/garlic/exp/creams/SS+mpi+send+seq.nix b/garlic/exp/creams/SS+mpi+send+seq.nix index c4ade80..da49d46 100644 --- a/garlic/exp/creams/SS+mpi+send+seq.nix +++ b/garlic/exp/creams/SS+mpi+send+seq.nix @@ -14,21 +14,23 @@ let # Set variable configuration for the experiment varConfig = { - # cc = [ self.gcc10 ]; # Does not work - # mpi = [ bsc.openmpi ]; # Does not work - # cc = [ bsc.icc ]; - # mpi = [ bsc.impi ]; + cc = [ bsc.icc ]; # [ bsc.icc pkgs.gfortran10 ]; - nodes = [ 1 2 4 8 16 ]; + mpi = [ bsc.impi ]; # [ bsc.impi bsc.openmpi-mn4 ]; + + input = [ + { nodes=1 ; nprocz=48 ; granul=0; } + { nodes=2 ; nprocz=96 ; granul=0; } + { nodes=4 ; nprocz=192; granul=0; } + { nodes=8 ; nprocz=384; granul=0; } + { nodes=16; nprocz=768; granul=0; } + ]; }; # Common configuration common = { gitBranch = "garlic/mpi+send+seq"; - cc = bsc.icc ; - mpi = bsc.impi; - # Resources ntasksPerNode = 48; #ntasksPerSocket = 24; // Add this variable to nix @@ -56,10 +58,12 @@ let sbatch = {stage, conf, ...}: with conf; w.sbatch { program = stageProgram stage; exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "nbody-bs"; - inherit nixPrefix nodes ntasksPerNode; + time = "10:00:00"; + ####qos = "debug"; + jobName = "creams-ss-mpi+send+seq"; + inherit nixPrefix ntasksPerNode; + + nodes = input.nodes; }; control = {stage, conf, ...}: with conf; w.control { @@ -109,19 +113,19 @@ let ]; }; - inputDataset = {stage, conf, ...}: with conf; + inputDataset = {stage, conf, ...}: let input = bsc.garlic.creamsInput.override { - inherit gitBranch nodes; + gitBranch = conf.gitBranch; + granul = conf.input.granul; + nprocz = conf.input.nprocz; }; in w.argv { program = stageProgram stage; env = '' cp -r ${input}/SodTubeBenchmark/* . - - pwd - ls -l + chmod +w -R . ''; }; @@ -162,7 +166,7 @@ let # Optionally profile nanos6 with the new ctf ++ optional enableCtf ctf - # Execute the nbody app with the argv and env vars + # Execute the app with the argv and env vars ++ [ inputDataset creamsFn ]; # List of actual programs to be executed diff --git a/garlic/exp/creams/test.nix b/garlic/exp/creams/test.nix deleted file mode 100644 index c4ade80..0000000 --- a/garlic/exp/creams/test.nix +++ /dev/null @@ -1,173 +0,0 @@ -{ - stdenv -, nixpkgs -, pkgs -, genApp -, genConfigs -, runWrappers -}: - -with stdenv.lib; - -let - bsc = pkgs.bsc; - - # Set variable configuration for the experiment - varConfig = { - # cc = [ self.gcc10 ]; # Does not work - # mpi = [ bsc.openmpi ]; # Does not work - # cc = [ bsc.icc ]; - # mpi = [ bsc.impi ]; - - nodes = [ 1 2 4 8 16 ]; - }; - - # Common configuration - common = { - gitBranch = "garlic/mpi+send+seq"; - - cc = bsc.icc ; - mpi = bsc.impi; - - # Resources - ntasksPerNode = 48; - #ntasksPerSocket = 24; // Add this variable to nix - - # Stage configuration - enableSbatch = true; - enableControl = true; - enableExtrae = false; - enablePerf = false; - enableCtf = false; - - # MN4 path - nixPrefix = "/gpfs/projects/bsc15/nix"; - }; - - # Compute the cartesian product of all configurations - configs = map (conf: conf // common) (genConfigs varConfig); - - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" else "${stage}"; - - w = runWrappers; - - sbatch = {stage, conf, ...}: with conf; w.sbatch { - program = stageProgram stage; - exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "nbody-bs"; - inherit nixPrefix nodes ntasksPerNode; - }; - - control = {stage, conf, ...}: with conf; w.control { - program = stageProgram stage; - }; - - srun = {stage, conf, ...}: with conf; w.srun { - program = stageProgram stage; - srunOptions = "--cpu-bind=verbose,rank"; - inherit nixPrefix; - }; - - statspy = {stage, conf, ...}: with conf; w.statspy { - program = stageProgram stage; - }; - - perf = {stage, conf, ...}: with conf; w.perf { - program = stageProgram stage; - perfArgs = "sched record -a"; - }; - - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { - program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; - }; - - extrae = {stage, conf, ...}: w.extrae { - program = stageProgram stage; - traceLib = "mpi"; # mpi -> libtracempi.so - configFile = ./extrae.xml; - }; - - ctf = {stage, conf, ...}: w.argv { - program = stageProgram stage; - env = '' - export NANOS6=ctf - export NANOS6_CTF2PRV=0 - ''; - }; - - bscOverlay = import ../../../overlay.nix; - - genPkgs = newOverlay: nixpkgs { - overlays = [ - bscOverlay - newOverlay - ]; - }; - - inputDataset = {stage, conf, ...}: with conf; - let - input = bsc.garlic.creamsInput.override { - inherit gitBranch nodes; - }; - in w.argv - { - program = stageProgram stage; - env = '' - cp -r ${input}/SodTubeBenchmark/* . - - pwd - ls -l - ''; - }; - - # We may be able to use overlays by invoking the fix function directly, but we - # have to get the definition of the bsc packages and the garlic ones as - # overlays. - - creamsFn = {stage, conf, ...}: with conf; - let - # We set the mpi implementation to the one specified in the conf, so all - # packages in bsc will use that one. - customPkgs = genPkgs (self: super: { - bsc = super.bsc // { mpi = conf.mpi; }; - }); - in - customPkgs.bsc.garlic.creams.override { - inherit cc mpi gitBranch; - }; - - stages = with common; [] - # Use sbatch to request resources first - ++ optional enableSbatch sbatch - - # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] - - # Executes srun to launch the program in the requested nodes, and - # immediately after enters the nix environment again, as slurmstepd launches - # the next stages from outside the namespace. - ++ [ srun nixsetup ] - - # Intrumentation with extrae - ++ optional enableExtrae extrae - - # Optionally profile the next stages with perf - ++ optional enablePerf perf - - # Optionally profile nanos6 with the new ctf - ++ optional enableCtf ctf - - # Execute the nbody app with the argv and env vars - ++ [ inputDataset creamsFn ]; - - # List of actual programs to be executed - jobs = map (conf: w.stagen { inherit conf stages; }) configs; - -in - # We simply run each program one after another - w.launch jobs diff --git a/overlay.nix b/overlay.nix index c01e311..80a2aaa 100644 --- a/overlay.nix +++ b/overlay.nix @@ -144,18 +144,17 @@ let # }; # creams = callPackage ./garlic/creams { + gnuDef = self.gfortran10 ; # Default GNU compiler version + intelDef = self.bsc.icc ; # Default Intel compiler version + gitBranch = "garlic/mpi+send+seq"; - #cc = self.gcc10; # Does not work - #mpi = self.bsc.openmpi-mn4; # Does not work - - cc = self.bsc.icc; - mpi = self.bsc.mpi; + cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10; + mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4; }; creamsInput = callPackage ./garlic/creams/input.nix { gitBranch = "garlic/mpi+send+seq"; - nodes = 1; }; # lulesh = callPackage ./garlic/lulesh { From c85b2976ef13da5b50dfc2a2192643637e473622 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Fri, 2 Oct 2020 16:27:22 +0200 Subject: [PATCH 3/5] Fix non-hybrid strong scalability experiments --- .../{SS+mpi+send+seq.nix => SS+nohybrid.nix} | 23 ++- garlic/exp/creams/test.nix~ | 169 ------------------ overlay.nix | 4 +- 3 files changed, 13 insertions(+), 183 deletions(-) rename garlic/exp/creams/{SS+mpi+send+seq.nix => SS+nohybrid.nix} (89%) delete mode 100644 garlic/exp/creams/test.nix~ diff --git a/garlic/exp/creams/SS+mpi+send+seq.nix b/garlic/exp/creams/SS+nohybrid.nix similarity index 89% rename from garlic/exp/creams/SS+mpi+send+seq.nix rename to garlic/exp/creams/SS+nohybrid.nix index da49d46..a7efa5b 100644 --- a/garlic/exp/creams/SS+mpi+send+seq.nix +++ b/garlic/exp/creams/SS+nohybrid.nix @@ -19,18 +19,18 @@ let mpi = [ bsc.impi ]; # [ bsc.impi bsc.openmpi-mn4 ]; input = [ - { nodes=1 ; nprocz=48 ; granul=0; } - { nodes=2 ; nprocz=96 ; granul=0; } - { nodes=4 ; nprocz=192; granul=0; } - { nodes=8 ; nprocz=384; granul=0; } - { nodes=16; nprocz=768; granul=0; } + { nodes=1 ; nprocz=48 ; granul=0; time= "10:00:00"; } + { nodes=2 ; nprocz=96 ; granul=0; time= "05:00:00"; } + { nodes=4 ; nprocz=192; granul=0; time= "03:00:00"; } + { nodes=8 ; nprocz=384; granul=0; time= "02:00:00"; } + { nodes=16; nprocz=768; granul=0; time= "01:00:00"; } ]; + + gitBranch = [ "garlic/mpi+send+seq" ]; }; # Common configuration common = { - gitBranch = "garlic/mpi+send+seq"; - # Resources ntasksPerNode = 48; #ntasksPerSocket = 24; // Add this variable to nix @@ -56,14 +56,13 @@ let w = runWrappers; sbatch = {stage, conf, ...}: with conf; w.sbatch { + nodes = input.nodes; program = stageProgram stage; exclusive = true; - time = "10:00:00"; - ####qos = "debug"; - jobName = "creams-ss-mpi+send+seq"; + time = input.time; + #qos = "debug"; + jobName = "creams-ss-${toString input.nodes}-${toString gitBranch}"; inherit nixPrefix ntasksPerNode; - - nodes = input.nodes; }; control = {stage, conf, ...}: with conf; w.control { diff --git a/garlic/exp/creams/test.nix~ b/garlic/exp/creams/test.nix~ deleted file mode 100644 index ac598d2..0000000 --- a/garlic/exp/creams/test.nix~ +++ /dev/null @@ -1,169 +0,0 @@ -{ - stdenv -, nixpkgs -, pkgs -, genApp -, genConfigs -, runWrappers -}: - -with stdenv.lib; - -let - bsc = pkgs.bsc; - - # Set variable configuration for the experiment - varConfig = { - mpi = [ bsc.impi bsc.openmpi ]; - nodes = [ 1 ]; - }; - - # Common configuration - common = { - # Compile time nbody config - gitBranch = "garlic/mpi+send+seq"; - - cc = bsc.icc; - - # Resources - ntasksPerNode = 48; - nodes = 1; - - # Stage configuration - enableSbatch = true; - enableControl = true; - enableExtrae = false; - enablePerf = false; - enableCtf = false; - - # MN4 path - nixPrefix = "/gpfs/projects/bsc15/nix"; - }; - - # Compute the cartesian product of all configurations - configs = map (conf: conf // common) (genConfigs varConfig); - - stageProgram = stage: - if stage ? programPath - then "${stage}${stage.programPath}" else "${stage}"; - - w = runWrappers; - - sbatch = {stage, conf, ...}: with conf; w.sbatch { - program = stageProgram stage; - exclusive = true; - time = "02:00:00"; - qos = "debug"; - jobName = "nbody-bs"; - inherit nixPrefix nodes ntasksPerNode; - }; - - control = {stage, conf, ...}: with conf; w.control { - program = stageProgram stage; - }; - - srun = {stage, conf, ...}: with conf; w.srun { - program = stageProgram stage; - srunOptions = "--cpu-bind=verbose,rank"; - inherit nixPrefix; - }; - - statspy = {stage, conf, ...}: with conf; w.statspy { - program = stageProgram stage; - }; - - perf = {stage, conf, ...}: with conf; w.perf { - program = stageProgram stage; - perfArgs = "sched record -a"; - }; - - nixsetup = {stage, conf, ...}: with conf; w.nixsetup { - program = stageProgram stage; - nixsetup = "${nixPrefix}/bin/nix-setup"; - }; - - extrae = {stage, conf, ...}: w.extrae { - program = stageProgram stage; - traceLib = "mpi"; # mpi -> libtracempi.so - configFile = ./extrae.xml; - }; - - ctf = {stage, conf, ...}: w.argv { - program = stageProgram stage; - env = '' - export NANOS6=ctf - export NANOS6_CTF2PRV=0 - ''; - }; - - bscOverlay = import ../../../overlay.nix; - - genPkgs = newOverlay: nixpkgs { - overlays = [ - bscOverlay - newOverlay - ]; - }; - - inputDataset = {stage, conf, ...}: with conf; - let - input = bsc.garlic.creamsInput.override { - inherit gitBranch nodes; - }; - in w.argv - { - program = stageProgram stage; - env = '' - cp -r ${input}/SodTubeBenchmark/* . - - pwd - ls -l - ''; - }; - - # We may be able to use overlays by invoking the fix function directly, but we - # have to get the definition of the bsc packages and the garlic ones as - # overlays. - - creamsFn = {stage, conf, ...}: with conf; - let - # We set the mpi implementation to the one specified in the conf, so all - # packages in bsc will use that one. - customPkgs = genPkgs (self: super: { - bsc = super.bsc // { mpi = conf.mpi; }; - }); - in - customPkgs.bsc.garlic.creams.override { - inherit cc mpi gitBranch; - }; - - stages = with common; [] - # Use sbatch to request resources first - ++ optional enableSbatch sbatch - - # Repeats the next stages N times - ++ optionals enableControl [ nixsetup control ] - - # Executes srun to launch the program in the requested nodes, and - # immediately after enters the nix environment again, as slurmstepd launches - # the next stages from outside the namespace. - ++ [ srun nixsetup ] - - # Intrumentation with extrae - ++ optional enableExtrae extrae - - # Optionally profile the next stages with perf - ++ optional enablePerf perf - - # Optionally profile nanos6 with the new ctf - ++ optional enableCtf ctf - - # Execute the nbody app with the argv and env vars - ++ [ inputDataset creamsFn ]; - - # List of actual programs to be executed - jobs = map (conf: w.stagen { inherit conf stages; }) configs; - -in - # We simply run each program one after another - w.launch jobs diff --git a/overlay.nix b/overlay.nix index 80a2aaa..a520473 100644 --- a/overlay.nix +++ b/overlay.nix @@ -237,8 +237,8 @@ let }; }; - creams = { - SS_mpi_send_seq = callPackage ./garlic/exp/creams/SS+mpi+send+seq.nix { + creamsSS = { + nohybrid = callPackage ./garlic/exp/creams/SS+nohybrid.nix { pkgs = self // self.bsc.garlic; nixpkgs = import ; genApp = self.bsc.garlic.genApp; From b403fbefe139fdeff2aa9a751f41f338b4eb5639 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Fri, 2 Oct 2020 17:48:00 +0200 Subject: [PATCH 4/5] Add hybrid strong scalability experiments --- garlic/exp/creams/SS+hybrid.nix | 180 ++++++++++++++++++++++++++++++++ overlay.nix | 7 ++ 2 files changed, 187 insertions(+) create mode 100644 garlic/exp/creams/SS+hybrid.nix diff --git a/garlic/exp/creams/SS+hybrid.nix b/garlic/exp/creams/SS+hybrid.nix new file mode 100644 index 0000000..987f103 --- /dev/null +++ b/garlic/exp/creams/SS+hybrid.nix @@ -0,0 +1,180 @@ +{ + stdenv +, nixpkgs +, pkgs +, genApp +, genConfigs +, runWrappers +}: + +with stdenv.lib; + +let + bsc = pkgs.bsc; + + # Set variable configuration for the experiment + varConfig = { + cc = [ bsc.icc ]; # [ bsc.icc pkgs.gfortran10 ]; + + mpi = [ bsc.impi ]; # [ bsc.impi bsc.openmpi-mn4 ]; + + input = [ + { nodes=1 ; nprocz=2 ; granul=37; time= "10:00:00"; } + { nodes=2 ; nprocz=4 ; granul=19; time= "05:00:00"; } + { nodes=4 ; nprocz=8 ; granul=10; time= "03:00:00"; } + { nodes=8 ; nprocz=16; granul=9 ; time= "02:00:00"; } + { nodes=16; nprocz=32; granul=9 ; time= "01:00:00"; } + ]; + + gitBranch = [ "garlic/mpi+isend+oss+task" + "garlic/mpi+send+omp+fork" + "garlic/mpi+send+oss+task" + "garlic/tampi+isend+oss+task" + ]; + }; + + # Common configuration + common = { + # Resources + ntasksPerNode = 2; + #ntasksPerSocket = 1; // Add this variable to nix + + # Stage configuration + enableSbatch = true; + enableControl = true; + enableExtrae = false; + enablePerf = false; + enableCtf = false; + + # MN4 path + nixPrefix = "/gpfs/projects/bsc15/nix"; + }; + + # Compute the cartesian product of all configurations + configs = map (conf: conf // common) (genConfigs varConfig); + + stageProgram = stage: + if stage ? programPath + then "${stage}${stage.programPath}" else "${stage}"; + + w = runWrappers; + + sbatch = {stage, conf, ...}: with conf; w.sbatch { + nodes = input.nodes; + program = stageProgram stage; + exclusive = true; + time = input.time; + #qos = "debug"; + jobName = "creams-ss-${toString input.nodes}-${toString gitBranch}"; + inherit nixPrefix ntasksPerNode; + }; + + control = {stage, conf, ...}: with conf; w.control { + program = stageProgram stage; + }; + + srun = {stage, conf, ...}: with conf; w.srun { + program = stageProgram stage; + srunOptions = "--cpu-bind=verbose,socket"; + inherit nixPrefix; + }; + + statspy = {stage, conf, ...}: with conf; w.statspy { + program = stageProgram stage; + }; + + perf = {stage, conf, ...}: with conf; w.perf { + program = stageProgram stage; + perfArgs = "sched record -a"; + }; + + nixsetup = {stage, conf, ...}: with conf; w.nixsetup { + program = stageProgram stage; + nixsetup = "${nixPrefix}/bin/nix-setup"; + }; + + extrae = {stage, conf, ...}: w.extrae { + program = stageProgram stage; + traceLib = "mpi"; # mpi -> libtracempi.so + configFile = ./extrae.xml; + }; + + ctf = {stage, conf, ...}: w.argv { + program = stageProgram stage; + env = '' + export NANOS6=ctf + export NANOS6_CTF2PRV=0 + ''; + }; + + bscOverlay = import ../../../overlay.nix; + + genPkgs = newOverlay: nixpkgs { + overlays = [ + bscOverlay + newOverlay + ]; + }; + + inputDataset = {stage, conf, ...}: + let + input = bsc.garlic.creamsInput.override { + gitBranch = conf.gitBranch; + granul = conf.input.granul; + nprocz = conf.input.nprocz; + }; + in w.argv + { + program = stageProgram stage; + env = '' + cp -r ${input}/SodTubeBenchmark/* . + chmod +w -R . + ''; + }; + + # We may be able to use overlays by invoking the fix function directly, but we + # have to get the definition of the bsc packages and the garlic ones as + # overlays. + + creamsFn = {stage, conf, ...}: with conf; + let + # We set the mpi implementation to the one specified in the conf, so all + # packages in bsc will use that one. + customPkgs = genPkgs (self: super: { + bsc = super.bsc // { mpi = conf.mpi; }; + }); + in + customPkgs.bsc.garlic.creams.override { + inherit cc mpi gitBranch; + }; + + stages = with common; [] + # Use sbatch to request resources first + ++ optional enableSbatch sbatch + + # Repeats the next stages N times + ++ optionals enableControl [ nixsetup control ] + + # Executes srun to launch the program in the requested nodes, and + # immediately after enters the nix environment again, as slurmstepd launches + # the next stages from outside the namespace. + ++ [ srun nixsetup ] + + # Intrumentation with extrae + ++ optional enableExtrae extrae + + # Optionally profile the next stages with perf + ++ optional enablePerf perf + + # Optionally profile nanos6 with the new ctf + ++ optional enableCtf ctf + + # Execute the app with the argv and env vars + ++ [ inputDataset creamsFn ]; + + # List of actual programs to be executed + jobs = map (conf: w.stagen { inherit conf stages; }) configs; + +in + # We simply run each program one after another + w.launch jobs diff --git a/overlay.nix b/overlay.nix index a520473..4edc2d0 100644 --- a/overlay.nix +++ b/overlay.nix @@ -245,6 +245,13 @@ let genConfigs = self.bsc.garlic.genConfigs; runWrappers = self.bsc.garlic.runWrappers; }; + hybrid = callPackage ./garlic/exp/creams/SS+hybrid.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; }; osu = rec { From 231672a22226d93222c8588b1b51abba87d90176 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Date: Fri, 2 Oct 2020 18:28:13 +0200 Subject: [PATCH 5/5] Rename files to improve consistency --- .../creams/{SS+hybrid.nix => ss+hybrid.nix} | 0 .../creams/{SS+nohybrid.nix => ss+pure.nix} | 0 overlay.nix | 30 ++++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) rename garlic/exp/creams/{SS+hybrid.nix => ss+hybrid.nix} (100%) rename garlic/exp/creams/{SS+nohybrid.nix => ss+pure.nix} (100%) diff --git a/garlic/exp/creams/SS+hybrid.nix b/garlic/exp/creams/ss+hybrid.nix similarity index 100% rename from garlic/exp/creams/SS+hybrid.nix rename to garlic/exp/creams/ss+hybrid.nix diff --git a/garlic/exp/creams/SS+nohybrid.nix b/garlic/exp/creams/ss+pure.nix similarity index 100% rename from garlic/exp/creams/SS+nohybrid.nix rename to garlic/exp/creams/ss+pure.nix diff --git a/overlay.nix b/overlay.nix index 4edc2d0..2a3283f 100644 --- a/overlay.nix +++ b/overlay.nix @@ -237,20 +237,22 @@ let }; }; - creamsSS = { - nohybrid = callPackage ./garlic/exp/creams/SS+nohybrid.nix { - pkgs = self // self.bsc.garlic; - nixpkgs = import ; - genApp = self.bsc.garlic.genApp; - genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; - }; - hybrid = callPackage ./garlic/exp/creams/SS+hybrid.nix { - pkgs = self // self.bsc.garlic; - nixpkgs = import ; - genApp = self.bsc.garlic.genApp; - genConfigs = self.bsc.garlic.genConfigs; - runWrappers = self.bsc.garlic.runWrappers; + creams = { + ss = { + pure = callPackage ./garlic/exp/creams/ss+pure.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; + hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { + pkgs = self // self.bsc.garlic; + nixpkgs = import ; + genApp = self.bsc.garlic.genApp; + genConfigs = self.bsc.garlic.genConfigs; + runWrappers = self.bsc.garlic.runWrappers; + }; }; };