hpcg: add weakscaling over some nblocks to check which axis is better

garlic/exp/hpcg/oss.slices.weakscaling.nix

@@ -0,0 +1,104 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+, genInput
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = {
+    n = [ { x = 192; y = 192; z = 192; } ];
+    nprocs = [
+        { x = 2; y = 1; z = 1; }
+        { x = 4; y = 1; z = 1; }
+        { x = 8; y = 1; z = 1; }
+        { x = 16; y = 1; z = 1; }
+        { x = 32; y = 1; z = 1; }
+
+        { x = 1; y = 2; z = 1; }
+        { x = 1; y = 4; z = 1; }
+        { x = 1; y = 8; z = 1; }
+        { x = 1; y = 16; z = 1; }
+        { x = 1; y = 32; z = 1; }
+
+        { x = 1; y = 1; z = 2; }
+        { x = 1; y = 1; z = 4; }
+        { x = 1; y = 1; z = 8; }
+        { x = 1; y = 1; z = 16; }
+        { x = 1; y = 1; z = 32; }
+
+    ];
+    # nblocks = [ 12 24 48 96 192 384 768 1536 ];
+    nblocks = [ 384 768 1536 ];
+    ncommblocks = [ 1 ];
+    # nodes = [ 1 ];
+    # nodes = [ 1 2 4 8 16 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = c: targetMachine.config // rec {
+    expName = "hpcg.oss";
+    unitName = "${expName}.nb${toString nblocks}";
+
+    inherit (targetMachine.config) hw;
+
+    # hpcg options
+    inherit (c) n nprocs nblocks ncommblocks;
+
+    gitBranch = "garlic/tampi+isend+oss+task";
+
+    # Repeat the execution of each unit 30 times
+    loops = 10;
+
+    disableAspectRatio = true;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = hw.socketsPerNode;
+    time = "02:00:00";
+    # task in one socket
+    cpusPerTask = hw.cpusPerSocket;
+    nodes = (nprocs.x * nprocs.y * nprocs.z) / ntasksPerNode;
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  input = genInput configs;
+
+  exec = {nextStage, conf, ...}: stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString conf.n.x}"
+      "--ny=${toString conf.n.y}"
+      "--nz=${toString conf.n.z}"
+      # Distribute all processes in X axis
+      "--npx=${toString conf.nprocs.x}"
+      "--npy=${toString conf.nprocs.y}"
+      "--npz=${toString conf.nprocs.z}"
+      "--nblocks=${toString conf.nblocks}"
+      "--ncomms=${toString conf.ncommblocks}"
+      # The input symlink is generated by the input stage, which is generated by
+      # the genInput function.
+      "--load=input"
+      # Disable HPCG Aspect Ratio to run any mpi layout
+    ] ++ optional (conf.disableAspectRatio) "--no-ar=1";
+  };
+
+  program = {nextStage, conf, ...}: bsc.apps.hpcg.override {
+    inherit (conf) gitBranch;
+  };
+
+  pipeline = stdexp.stdPipeline ++ [ input exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/hpcg/slices.nix

@@ -19,9 +19,22 @@ let
         { x = 8; y = 1; z = 1; }
         { x = 16; y = 1; z = 1; }
         { x = 32; y = 1; z = 1; }
+
+        { x = 1; y = 2; z = 1; }
+        { x = 1; y = 4; z = 1; }
+        { x = 1; y = 8; z = 1; }
+        { x = 1; y = 16; z = 1; }
+        { x = 1; y = 32; z = 1; }
+
+        { x = 1; y = 1; z = 2; }
+        { x = 1; y = 1; z = 4; }
+        { x = 1; y = 1; z = 8; }
+        { x = 1; y = 1; z = 16; }
+        { x = 1; y = 1; z = 32; }
+
     ];
     # nblocks = [ 12 24 48 96 192 384 768 1536 ];
-    nblocks = [ 384 ];
+    nblocks = [ 384 768 1536 ];
     ncommblocks = [ 1 ];
     # nodes = [ 1 ];
     # nodes = [ 1 2 4 8 16 ];
@@ -40,7 +53,7 @@ let
     gitBranch = "garlic/tampi+isend+oss+task";
 
     # Repeat the execution of each unit 30 times
-    loops = 3;
+    loops = 10;
 
     disableAspectRatio = true;
 

garlic/exp/index.nix

@@ -65,13 +65,13 @@
       inherit genInput;
     };
 
-    ossScalability = callPackage ./hpcg/oss.scalability.192.nix {
+    # ossScalability = callPackage ./hpcg/oss.scalability.192.nix {
-      inherit genInput;
-    };
-
-    # slices = callPackage ./hpcg/slices.nix {
     #   inherit genInput;
     # };
+
+    ossSlicesWeakscaling = callPackage ./hpcg/oss.slices.weakscaling.nix {
+      inherit genInput;
+    };
   };
 
   heat = rec {

garlic/fig/hpcg/oss.slices.weakscaling.R

@@ -0,0 +1,110 @@
+# This R program takes as argument the dataset that contains the results of the
+# execution of the heat example experiment and produces some plots. All the
+# knowledge to understand how this script works is covered by this nice R book:
+#
+# Winston Chang, R Graphics Cookbook: Practical Recipes for Visualizing Data,
+# O’Reilly Media (2020). 2nd edition
+#
+# Which can be freely read it online here: https://r-graphics.org/
+#
+# Please, search in this book before copying some random (and probably oudated)
+# reply on stack overflow.
+
+# We load some R packages to import the required functions. We mainly use the
+# tidyverse packages, which are very good for ploting data,
+library(ggplot2)
+library(dplyr, warn.conflicts = FALSE)
+library(scales)
+library(jsonlite)
+library(viridis, warn.conflicts = FALSE)
+
+# Here we simply load the arguments to find the input dataset. If nothing is
+# specified we use the file named `input` in the current directory.
+# We can run this script directly using:
+# Rscript <path-to-this-script> <input-dataset>
+
+# Load the arguments (argv)
+args = commandArgs(trailingOnly=TRUE)
+
+# Set the input dataset if given in argv[1], or use "input" as default
+if (length(args)>0) { input_file = args[1] } else { input_file = "input" }
+
+df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>%
+
+  # Then we flatten it, as it may contain dictionaries inside the columns
+  jsonlite::flatten() %>%
+
+  # Now the dataframe contains all the configuration of the units inside the
+  # columns named `config.*`, for example `config.cbs`. We first select only
+  # the columns that we need:
+  select(config.nblocks,
+         config.ncommblocks,
+         config.hw.cpusPerSocket,
+         config.nodes,
+         config.nprocs.x,
+         config.nprocs.y,
+         config.nprocs.z,
+         unit,
+         time
+         ) %>%
+
+  # And then we rename those columns to something shorter:
+  rename(nblocks=config.nblocks,
+         ncommblocks=config.ncommblocks,
+         cpusPerSocket=config.hw.cpusPerSocket,
+         nodes=config.nodes,
+         npx=config.nprocs.x,
+         npy=config.nprocs.y,
+         npz=config.nprocs.z
+         ) %>%
+
+  mutate(axisColor=as.factor(ifelse(npx != 1, "X", ifelse(npy != 1, "Y", "Z")))) %>%
+
+  mutate(blocksPerCpu = nblocks / cpusPerSocket) %>%
+
+  mutate(nblocks = as.factor(nblocks)) %>%
+  mutate(blocksPerCpu = as.factor(blocksPerCpu)) %>%
+  mutate(nodes = as.factor(nodes)) %>%
+  mutate(unit = as.factor(unit)) %>%
+
+  group_by(unit) %>%
+
+  # And compute some metrics which are applied to each group. For example we
+  # compute the median time within the runs of a unit:
+  mutate(median.time = median(time)) %>%
+  mutate(normalized.time = time / median.time - 1) %>%
+  mutate(log.median.time = log(median.time)) %>%
+
+  # Then, we remove the grouping. This step is very important, otherwise the
+  # plotting functions get confused:
+  ungroup()
+
+dpi=300
+h=5
+w=5
+w=3*w
+
+# We plot the time of each run as we vary the block size
+p = ggplot(df, aes(x=blocksPerCpu, y=time, color=axisColor)) +
+
+  # We add a points (scatter plot) using circles (shape=21) a bit larger
+  # than the default (size=3)
+  geom_point(shape=21, size=3) +
+
+  facet_wrap(~ nodes, labeller="label_both") +
+
+  # The bw theme is recommended for publications
+  theme_bw() +
+
+  # Here we add the title and the labels of the axes
+  labs(x="Blocks Per CPU", y="Time (s)", title="HPCG weak scalability: time",
+    color="Axis",
+    subtitle=input_file) +
+
+  # And set the subtitle font size a bit smaller, so it fits nicely
+  theme(plot.subtitle=element_text(size=8))
+
+# Then, we save the plot both in png and pdf
+ggsave("time.png", plot=p, width=w, height=h, dpi=dpi)
+ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi)
+

garlic/fig/index.nix

@@ -38,9 +38,13 @@ in
   };
 
   hpcg = with exp.hpcg; {
+    # /nix/store/8dr191vch1nw7vfz8nj36d5nhwnbdnf3-plot
     ossGranularity = stdPlot ./hpcg/oss.granularity.R [ ossGranularity ];
-    ossScalability = stdPlot ./hpcg/oss.scalability.R [ ossScalability ];
+
-    # slices = stdPlot ./hpcg/oss.R [ slices ];
+    # ossScalability = stdPlot ./hpcg/oss.scalability.R [ ossScalability ];
+
+    # /nix/store/a3x76fbnfbacn2xhz3q65fklfp0qbb6p-plot
+    ossWeakscalingPerAxisPerBlock = stdPlot ./hpcg/oss.slices.weakscaling.R [ ossSlicesWeakscaling ];
   };
 
   saiph = with exp.saiph; {