From b6f563f6216abcf490525ccea01ca2d2f2f6dbba Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Sun, 9 May 2021 11:36:43 +0200 Subject: [PATCH] creams: add granularity experiment with 16 nodes --- garlic/exp/creams/granularity16.nix | 131 +++++++++++++++++++++++++++ garlic/exp/index.nix | 1 + garlic/fig/creams/granularity16.R | 136 ++++++++++++++++++++++++++++ garlic/fig/index.nix | 1 + 4 files changed, 269 insertions(+) create mode 100644 garlic/exp/creams/granularity16.nix create mode 100644 garlic/fig/creams/granularity16.R diff --git a/garlic/exp/creams/granularity16.nix b/garlic/exp/creams/granularity16.nix new file mode 100644 index 0000000..a43d2bb --- /dev/null +++ b/garlic/exp/creams/granularity16.nix @@ -0,0 +1,131 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +, garlicTools +, enableExtended ? false +}: + +with stdenv.lib; +with garlicTools; + +let + # Initial variable configuration + varConf = { + + #nodes = range2 1 16; + nodes = [ 16 ]; + sizeFactor = [ 1 2 4 8 16 32 ]; + granul = [ 1 2 4 8 16 ]; + + # Max. number of iterations + iterations = [ 20 ] ++ optionals (enableExtended) [ 10 ]; + + gitBranch = [ + "garlic/tampi+isend+oss+task" + #"garlic/mpi+send+omp+fork" + #"garlic/mpi+send+omp+task" + #"garlic/mpi+send+seq" + ] ++ optionals (enableExtended) [ + "garlic/mpi+send+oss+task" + "garlic/mpi+isend+omp+task" + "garlic/mpi+isend+oss+task" + ]; + }; + + # We use these auxiliary functions to assign different configurations + # depending on the git branch. + getGranul = branch: oldGranul: + if (branch == "garlic/mpi+send+seq") + then 999999 else oldGranul; + + getCpusPerTask = branch: hw: + if (branch == "garlic/mpi+send+seq") + then 1 else hw.cpusPerSocket; + + getNtasksPerNode = branch: hw: + if (branch == "garlic/mpi+send+seq") + then hw.cpusPerNode else hw.socketsPerNode; + + # Generate the complete configuration for each unit + genConf = c: targetMachine.config // rec { + + expName = "creams-granularity16"; + unitName = "${expName}" + + "-granul.${toString granul}" + + "-sf.${toString sizeFactor}"; + + inherit (targetMachine.config) hw; + + # Options for creams + inherit (c) iterations gitBranch nodes sizeFactor; + granul = getGranul gitBranch c.granul; + nprocz = ntasksPerNode * nodes; + baseSizePerCpu = 2; + baseSize = baseSizePerCpu * cpusPerTask * ntasksPerNode * nodes; + + nz = baseSize * sizeFactor; + + # Repeat the execution of each unit 10 times + loops = 10; + + # Resources + qos = "debug"; + time = "02:00:00"; + ntasksPerNode = getNtasksPerNode gitBranch hw; + cpusPerTask = getCpusPerTask gitBranch hw; + jobName = unitName; + }; + + # Compute the array of configurations + configs = unique ( + filter (c: !(c.granul == 1 && c.sizeFactor >= 32))(stdexp.buildConfigs { + inherit varConf genConf; + })); + + # Custom srun stage to copy the creams input dataset + customSrun = {nextStage, conf, ...}: + let + input = bsc.garlic.apps.creamsInput.override { + inherit (conf) gitBranch granul nprocz nz; + }; + in + stdexp.stdStages.srun { + inherit nextStage conf; + # Now we add some commands to execute before calling srun. These will + # only run in one rank (the first in the list of allocated nodes) + preSrun = '' + cp -r ${input}/SodTubeBenchmark/* . + chmod +w -R . + sed -i '/maximum number of iterations/s/50/${toString conf.iterations}/' input.dat + rm -f nanos6.toml + ''; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + env = '' + export NANOS6_CONFIG_OVERRIDE="version.dependencies=regions" + ''; + + # Remove restarts as is not needed and is huge + post = '' + rm -rf restarts || true + ''; + }; + + # Creams program + creams = {nextStage, conf, ...}: bsc.apps.creams.override { + inherit (conf) gitBranch; + }; + + pipeline = stdexp.stdPipelineOverride { + # Replace the stdandard srun stage with our own + overrides = { srun = customSrun; }; + } ++ [ exec creams ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index f531d34..ff54e7d 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -22,6 +22,7 @@ ss = callPackage ./creams/ss.nix { }; granularity = callPackage ./creams/granularity.nix { }; size = callPackage ./creams/size.nix { }; + granularity16 = callPackage ./creams/granularity16.nix { }; # These experiments are the extended versions of the previous # ones. We split them so we can keep a reasonable execution time diff --git a/garlic/fig/creams/granularity16.R b/garlic/fig/creams/granularity16.R new file mode 100644 index 0000000..24f3bf7 --- /dev/null +++ b/garlic/fig/creams/granularity16.R @@ -0,0 +1,136 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) +library(viridis, warn.conflicts = FALSE) +library(stringr) + +args = commandArgs(trailingOnly=TRUE) + +# Set the input dataset if given in argv[1], or use "input" as default +if (length(args)>0) { input_file = args[1] } else { input_file = "input" } +if (length(args)>1) { output = args[2] } else { output = "?" } + +df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + + jsonlite::flatten() %>% + + select(unit, + config.nodes, + config.gitBranch, + config.granul, + config.iterations, + config.sizeFactor, + config.nz, + time, + total_time) %>% + + rename(nodes=config.nodes, + gitBranch=config.gitBranch, + granul=config.granul, + sizeFactor=config.sizeFactor, + nz=config.nz, + iterations=config.iterations) %>% + + # Remove the "garlic/" prefix from the gitBranch + mutate(branch = str_replace(gitBranch, "garlic/", "")) %>% + + # Computations before converting to factor + mutate(time.nodes = time * nodes) %>% + mutate(time.elem = time / sizeFactor) %>% + mutate(time.nodes.iter = time.nodes / iterations) %>% + + # Convert to factors + mutate(unit = as.factor(unit)) %>% + mutate(nodes = as.factor(nodes)) %>% + mutate(gitBranch = as.factor(gitBranch)) %>% + mutate(granul = as.factor(granul)) %>% + mutate(iterations = as.factor(iterations)) %>% + mutate(sizeFactor = as.factor(sizeFactor)) %>% + mutate(nz = as.factor(nz)) %>% + mutate(unit = as.factor(unit)) %>% + + # Compute median times + group_by(unit) %>% + mutate(median.time = median(time)) %>% + mutate(median.time.nodes = median(time.nodes)) %>% + mutate(median.time.elem = median(time.elem)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + mutate(log.median.time.elem = log(median.time.elem)) %>% + mutate(median.time.nodes.iter = median(time.nodes.iter)) %>% + ungroup() %>% + group_by(sizeFactor) %>% + mutate(optimal.granul = (median.time.elem == min(median.time.elem))) %>% + ungroup() + +dfopt = df %>% filter(optimal.granul == TRUE) + +dpi = 300 +h = 4 +w = 10 + +# --------------------------------------------------------------------- + +#p = ggplot(df, aes(x=sizeFactor, y=normalized.time, fill=granul, color=iterations)) + +# geom_boxplot() + +# geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + +# theme_bw() + +# facet_wrap(branch ~ .) + +# labs(x="nodes", y="Normalized time", +# title="Creams strong scaling: normalized time", +# subtitle=output) + +# theme(plot.subtitle=element_text(size=8)) +# +#ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) +#ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=granul, y=time.elem, color=branch)) + + geom_point(shape=21, size=3) + +# geom_line(aes(y=median.time, group=gitBranch)) + + theme_bw() + + facet_wrap(sizeFactor ~ ., labeller=label_both, nrow=1) + + labs(x="Granularity", y="Time / k (s)", + #title="Creams size: time per object", + subtitle=output) + + theme(plot.subtitle=element_text(size=8, family="mono"), + legend.position="bottom") + +ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=granul, y=median.time.elem, color=sizeFactor)) + + geom_line(aes(group=sizeFactor)) + + geom_point(data=dfopt, aes(x=granul, y=median.time.elem)) + + theme_bw() + + labs(x="Granularity", y="Time / k (s)", + color="Size factor k", + subtitle=output) + + theme(plot.subtitle=element_text(size=8, family="mono"), + legend.position="bottom") + +ggsave("median.time.png", plot=p, width=5, height=5, dpi=dpi) +ggsave("median.time.pdf", plot=p, width=5, height=5, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=granul, y=sizeFactor, fill=log.median.time.elem)) + + geom_raster() + + scale_fill_viridis(option="plasma") + + coord_fixed() + + theme_bw() + + theme(axis.text.x=element_text(angle = -45, hjust = 0)) + + theme(plot.subtitle=element_text(size=8)) + + #guides(fill = guide_colorbar(barwidth=15, title.position="top")) + + guides(fill = guide_colorbar(barwidth=12, title.vjust=0.8)) + + labs(x="Granularity", y="Size factor", fill="Time / k (s)", subtitle=output) + + theme(plot.subtitle=element_text(size=8, family="mono"), + legend.position="bottom") + +k=1 +ggsave("heatmap.png", plot=p, width=4.8*k, height=5*k, dpi=300) +ggsave("heatmap.pdf", plot=p, width=4.8*k, height=5*k, dpi=300) diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index a54ed9f..0e7ce41 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -57,6 +57,7 @@ in ss = stdPlot ./creams/ss.R [ ss ]; granularity = stdPlot ./creams/granularity.R [ granularity ]; size = stdPlot ./creams/size.R [ size ]; + granularity16 = stdPlot ./creams/granularity16.R [ granularity16 ]; # Extended version (we could use another R script for those plots big.ss = stdPlot ./creams/ss.R [ big.ss ];