nbody: add strong scaling experiment

garlic/exp/index.nix

@@ -10,6 +10,7 @@
   nbody = rec {
     granularity = callPackage ./nbody/granularity-mpi.nix { };
     nodesorsockets = callPackage ./nbody/nodes-or-sockets-mpi.nix { };
+    scaling = callPackage ./nbody/strong-scaling-mpi.nix { };
   };
 
   saiph = {

garlic/exp/nbody/strong-scaling-mpi.nix

@@ -4,37 +4,44 @@
 , bsc
 , targetMachine
 , stages
+, garlicTools
 }:
 
 with stdenv.lib;
+with garlicTools;
 
 let
   # Initial variable configuration
   varConf = with bsc; {
-    numProcsAndParticles = [ 1 2 4 8 16 32 48 ];
+    blocksize = [ 512 ];
+    nodes = [ 1 2 4 8 16 ];
+    gitBranch = [
+      "garlic/mpi+send+oss+task" 
+      "garlic/tampi+send+oss+task" 
+      "garlic/tampi+isend+oss+task"
+    ];
   };
 
   # Generate the complete configuration for each unit
-  genConf = with bsc; c: targetMachine.config // rec {
+  genConf = c: targetMachine.config // rec {
-    # nbody options
+    hw = targetMachine.config.hw;
-    inherit (c) numProcsAndParticles;
+    particles = 16 * 4096 * hw.cpusPerSocket;
-    particles = 1024 * numProcsAndParticles * 2;
     timesteps = 10;
-    blocksize = 1024;
+    blocksize = c.blocksize;
-    cc = icc;
+    numNodes  = c.nodes;
-    mpi = impi;
+    gitBranch = c.gitBranch;
-    gitBranch = "garlic/mpi+send";
+
+    expName = "nbody-scaling";
+    unitName = expName + "-${toString gitBranch}" + "-nodes${toString numNodes}";
 
-    # Repeat the execution of each unit 30 times
     loops = 30;
 
-    # Resources
+    nodes = numNodes;
     qos = "debug";
-    ntasksPerNode = numProcsAndParticles;
+    ntasksPerNode = 2;
-    nodes = 1;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
+    cpusPerTask = hw.cpusPerSocket;
-    jobName = "nbody-bs-${toString numProcsAndParticles}-${gitBranch}";
+    jobName = unitName;
   };
 
   # Compute the array of configurations
@@ -42,17 +49,13 @@ let
     inherit varConf genConf;
   };
 
-  exec = {nextStage, conf, ...}: with conf; stages.exec {
+  exec = {nextStage, conf, ...}: stages.exec {
     inherit nextStage;
-    argv = [ "-t" timesteps "-p" particles ];
+    argv = [ "-t" conf.timesteps "-p" conf.particles ];
   };
 
-  program = {nextStage, conf, ...}: with conf;
+  program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override {
-  let
+    inherit (conf) blocksize gitBranch;
-    customPkgs = stdexp.replaceMpi conf.mpi;
-  in
-    customPkgs.apps.nbody.override {
-      inherit cc blocksize mpi gitBranch;
   };
 
   pipeline = stdexp.stdPipeline ++ [ exec program ];

garlic/fig/index.nix

@@ -32,6 +32,7 @@ in
   nbody = with exp.nbody; {
     granularity = stdPlot ./nbody/granularity.R [ granularity ];
     nodesorsockets = stdPlot ./nbody/nodes-or-sockets.R [ nodesorsockets ];
+    scaling = stdPlot ./nbody/scaling.R [ scaling ];
   };
 
   hpcg = with exp.hpcg; {

garlic/fig/nbody/scaling.R

@@ -0,0 +1,93 @@
+library(ggplot2)
+library(dplyr, warn.conflicts = FALSE)
+library(scales)
+library(jsonlite)
+library(viridis, warn.conflicts = FALSE)
+
+# Load the arguments (argv)
+args = commandArgs(trailingOnly=TRUE)
+if (length(args)>0) { input_file = args[1] } else { input_file = "input" }
+
+df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>%
+  jsonlite::flatten() %>%
+  select(config.blocksize, config.gitBranch, config.numNodes, unit, time) %>%
+  rename(nodes = config.numNodes, blocksize=config.blocksize, branch=config.gitBranch) %>%
+
+  mutate(blocksize = as.factor(blocksize)) %>%
+  mutate(nodes = as.factor(nodes)) %>%
+  mutate(branch = as.factor(branch)) %>%
+  mutate(unit = as.factor(unit)) %>%
+
+  group_by(unit) %>%
+
+  mutate(median.time = median(time)) %>%
+  mutate(normalized.time = time / median.time - 1) %>%
+  mutate(log.median.time = log(median.time)) %>%
+
+  ungroup()
+
+dpi = 300
+h = 5
+w = 8
+
+# ---------------------------------------------------------------------
+
+p = ggplot(df, aes(x=nodes, y=median.time, color=branch)) +
+  geom_point() +
+  geom_line(aes(group=branch)) +
+  theme_bw() +
+  labs(x="Nodes", y="Median time (s)", title="NBody Scaling: Median Time", 
+    subtitle=input_file) + 
+  theme(plot.subtitle=element_text(size=5)) +
+  theme(legend.position="bottom") +
+  theme(legend.text = element_text(size=7))
+
+ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi)
+ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi)
+
+# ---------------------------------------------------------------------
+
+p = ggplot(df, aes(x=nodes, y=normalized.time, color=branch)) +
+  geom_boxplot() +
+  geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") +
+  facet_wrap(~ branch) +
+  theme_bw() +
+  labs(x="Nodes", y="Normalized time (s)", title="NBody Scaling: Normalized Time", 
+    subtitle=input_file) + 
+  theme(plot.subtitle=element_text(size=5)) +
+  theme(legend.position="bottom") +
+  theme(legend.text = element_text(size=7))
+
+ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi)
+ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi)
+
+# ---------------------------------------------------------------------
+
+p = ggplot(df, aes(x=nodes, y=time, color=branch)) +
+  geom_point(shape=21, size=3) +
+  theme_bw() +
+  labs(x="Nodes", y="Time (s)", title="NBody Scaling: Time", 
+    subtitle=input_file) + 
+  theme(plot.subtitle=element_text(size=5)) +
+  theme(legend.position="bottom") +
+  theme(legend.text = element_text(size=7))
+
+ggsave("time.png", plot=p, width=w, height=h, dpi=dpi)
+ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi)
+
+
+# ---------------------------------------------------------------------
+
+p = ggplot(df, aes(x=nodes, y=branch, fill=median.time)) +
+  geom_raster() + 
+  scale_fill_viridis(option="plasma") +
+  coord_fixed() +
+  theme_bw() +
+  labs(x="Nodes", y="Branch", title="NBody Scaling: Time", 
+    subtitle=input_file) + 
+  theme(plot.subtitle=element_text(size=5)) +
+  theme(legend.position="bottom") +
+  theme(legend.text = element_text(size=7))
+
+ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi)
+ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi)