From b60a46b6836feb121d14a0d5f9d015c4e6cb1da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Pe=C3=B1acoba?= Date: Fri, 26 Mar 2021 16:20:04 +0100 Subject: [PATCH] hpcg: add weakscaling over some nblocks to check which axis is better --- garlic/exp/hpcg/oss.slices.weakscaling.nix | 104 +++++++++++++++++++ garlic/exp/hpcg/slices.nix | 17 +++- garlic/exp/index.nix | 10 +- garlic/fig/hpcg/oss.slices.weakscaling.R | 110 +++++++++++++++++++++ garlic/fig/index.nix | 8 +- 5 files changed, 240 insertions(+), 9 deletions(-) create mode 100644 garlic/exp/hpcg/oss.slices.weakscaling.nix create mode 100644 garlic/fig/hpcg/oss.slices.weakscaling.R diff --git a/garlic/exp/hpcg/oss.slices.weakscaling.nix b/garlic/exp/hpcg/oss.slices.weakscaling.nix new file mode 100644 index 0000000..4ff467d --- /dev/null +++ b/garlic/exp/hpcg/oss.slices.weakscaling.nix @@ -0,0 +1,104 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +, genInput +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = { + n = [ { x = 192; y = 192; z = 192; } ]; + nprocs = [ + { x = 2; y = 1; z = 1; } + { x = 4; y = 1; z = 1; } + { x = 8; y = 1; z = 1; } + { x = 16; y = 1; z = 1; } + { x = 32; y = 1; z = 1; } + + { x = 1; y = 2; z = 1; } + { x = 1; y = 4; z = 1; } + { x = 1; y = 8; z = 1; } + { x = 1; y = 16; z = 1; } + { x = 1; y = 32; z = 1; } + + { x = 1; y = 1; z = 2; } + { x = 1; y = 1; z = 4; } + { x = 1; y = 1; z = 8; } + { x = 1; y = 1; z = 16; } + { x = 1; y = 1; z = 32; } + + ]; + # nblocks = [ 12 24 48 96 192 384 768 1536 ]; + nblocks = [ 384 768 1536 ]; + ncommblocks = [ 1 ]; + # nodes = [ 1 ]; + # nodes = [ 1 2 4 8 16 ]; + }; + + # Generate the complete configuration for each unit + genConf = c: targetMachine.config // rec { + expName = "hpcg.oss"; + unitName = "${expName}.nb${toString nblocks}"; + + inherit (targetMachine.config) hw; + + # hpcg options + inherit (c) n nprocs nblocks ncommblocks; + + gitBranch = "garlic/tampi+isend+oss+task"; + + # Repeat the execution of each unit 30 times + loops = 10; + + disableAspectRatio = true; + + # Resources + qos = "debug"; + ntasksPerNode = hw.socketsPerNode; + time = "02:00:00"; + # task in one socket + cpusPerTask = hw.cpusPerSocket; + nodes = (nprocs.x * nprocs.y * nprocs.z) / ntasksPerNode; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + input = genInput configs; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + argv = [ + "--nx=${toString conf.n.x}" + "--ny=${toString conf.n.y}" + "--nz=${toString conf.n.z}" + # Distribute all processes in X axis + "--npx=${toString conf.nprocs.x}" + "--npy=${toString conf.nprocs.y}" + "--npz=${toString conf.nprocs.z}" + "--nblocks=${toString conf.nblocks}" + "--ncomms=${toString conf.ncommblocks}" + # The input symlink is generated by the input stage, which is generated by + # the genInput function. + "--load=input" + # Disable HPCG Aspect Ratio to run any mpi layout + ] ++ optional (conf.disableAspectRatio) "--no-ar=1"; + }; + + program = {nextStage, conf, ...}: bsc.apps.hpcg.override { + inherit (conf) gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ input exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/slices.nix b/garlic/exp/hpcg/slices.nix index cff1f75..4ff467d 100644 --- a/garlic/exp/hpcg/slices.nix +++ b/garlic/exp/hpcg/slices.nix @@ -19,9 +19,22 @@ let { x = 8; y = 1; z = 1; } { x = 16; y = 1; z = 1; } { x = 32; y = 1; z = 1; } + + { x = 1; y = 2; z = 1; } + { x = 1; y = 4; z = 1; } + { x = 1; y = 8; z = 1; } + { x = 1; y = 16; z = 1; } + { x = 1; y = 32; z = 1; } + + { x = 1; y = 1; z = 2; } + { x = 1; y = 1; z = 4; } + { x = 1; y = 1; z = 8; } + { x = 1; y = 1; z = 16; } + { x = 1; y = 1; z = 32; } + ]; # nblocks = [ 12 24 48 96 192 384 768 1536 ]; - nblocks = [ 384 ]; + nblocks = [ 384 768 1536 ]; ncommblocks = [ 1 ]; # nodes = [ 1 ]; # nodes = [ 1 2 4 8 16 ]; @@ -40,7 +53,7 @@ let gitBranch = "garlic/tampi+isend+oss+task"; # Repeat the execution of each unit 30 times - loops = 3; + loops = 10; disableAspectRatio = true; diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 910cffe..02ad682 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -65,13 +65,13 @@ inherit genInput; }; - ossScalability = callPackage ./hpcg/oss.scalability.192.nix { - inherit genInput; - }; - - # slices = callPackage ./hpcg/slices.nix { + # ossScalability = callPackage ./hpcg/oss.scalability.192.nix { # inherit genInput; # }; + + ossSlicesWeakscaling = callPackage ./hpcg/oss.slices.weakscaling.nix { + inherit genInput; + }; }; heat = rec { diff --git a/garlic/fig/hpcg/oss.slices.weakscaling.R b/garlic/fig/hpcg/oss.slices.weakscaling.R new file mode 100644 index 0000000..7109b0f --- /dev/null +++ b/garlic/fig/hpcg/oss.slices.weakscaling.R @@ -0,0 +1,110 @@ +# This R program takes as argument the dataset that contains the results of the +# execution of the heat example experiment and produces some plots. All the +# knowledge to understand how this script works is covered by this nice R book: +# +# Winston Chang, R Graphics Cookbook: Practical Recipes for Visualizing Data, +# O’Reilly Media (2020). 2nd edition +# +# Which can be freely read it online here: https://r-graphics.org/ +# +# Please, search in this book before copying some random (and probably oudated) +# reply on stack overflow. + +# We load some R packages to import the required functions. We mainly use the +# tidyverse packages, which are very good for ploting data, +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) +library(viridis, warn.conflicts = FALSE) + +# Here we simply load the arguments to find the input dataset. If nothing is +# specified we use the file named `input` in the current directory. +# We can run this script directly using: +# Rscript + +# Load the arguments (argv) +args = commandArgs(trailingOnly=TRUE) + +# Set the input dataset if given in argv[1], or use "input" as default +if (length(args)>0) { input_file = args[1] } else { input_file = "input" } + +df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + + # Then we flatten it, as it may contain dictionaries inside the columns + jsonlite::flatten() %>% + + # Now the dataframe contains all the configuration of the units inside the + # columns named `config.*`, for example `config.cbs`. We first select only + # the columns that we need: + select(config.nblocks, + config.ncommblocks, + config.hw.cpusPerSocket, + config.nodes, + config.nprocs.x, + config.nprocs.y, + config.nprocs.z, + unit, + time + ) %>% + + # And then we rename those columns to something shorter: + rename(nblocks=config.nblocks, + ncommblocks=config.ncommblocks, + cpusPerSocket=config.hw.cpusPerSocket, + nodes=config.nodes, + npx=config.nprocs.x, + npy=config.nprocs.y, + npz=config.nprocs.z + ) %>% + + mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>% + + mutate(blocksPerCpu = nblocks / cpusPerSocket) %>% + + mutate(nblocks = as.factor(nblocks)) %>% + mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>% + mutate(nodes = as.factor(nodes)) %>% + mutate(unit = as.factor(unit)) %>% + + group_by(unit) %>% + + # And compute some metrics which are applied to each group. For example we + # compute the median time within the runs of a unit: + mutate(median.time = median(time)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + + # Then, we remove the grouping. This step is very important, otherwise the + # plotting functions get confused: + ungroup() + +dpi=300 +h=5 +w=5 +w=3*w + +# We plot the time of each run as we vary the block size +p = ggplot(df, aes(x=blocksPerCpu, y=time, color=axisColor)) + + + # We add a points (scatter plot) using circles (shape=21) a bit larger + # than the default (size=3) + geom_point(shape=21, size=3) + + + facet_wrap(~ nodes, labeller="label_both") + + + # The bw theme is recommended for publications + theme_bw() + + + # Here we add the title and the labels of the axes + labs(x="Blocks Per CPU", y="Time (s)", title="HPCG weak scalability: time", + color="Axis", + subtitle=input_file) + + + # And set the subtitle font size a bit smaller, so it fits nicely + theme(plot.subtitle=element_text(size=8)) + +# Then, we save the plot both in png and pdf +ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) + diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index b4ade9a..b87c9ce 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -38,9 +38,13 @@ in }; hpcg = with exp.hpcg; { + # /nix/store/8dr191vch1nw7vfz8nj36d5nhwnbdnf3-plot ossGranularity = stdPlot ./hpcg/oss.granularity.R [ ossGranularity ]; - ossScalability = stdPlot ./hpcg/oss.scalability.R [ ossScalability ]; - # slices = stdPlot ./hpcg/oss.R [ slices ]; + + # ossScalability = stdPlot ./hpcg/oss.scalability.R [ ossScalability ]; + + # /nix/store/a3x76fbnfbacn2xhz3q65fklfp0qbb6p-plot + ossWeakscalingPerAxisPerBlock = stdPlot ./hpcg/oss.slices.weakscaling.R [ ossSlicesWeakscaling ]; }; saiph = with exp.saiph; {