diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 4183763..8af4d98 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -116,6 +116,8 @@ bw = callPackage ./osu/bw.nix { }; impi = callPackage ./osu/impi.nix { }; bwShm = bw.override { interNode = false; }; + mtu = callPackage ./osu/mtu.nix { }; + eager = callPackage ./osu/eager.nix { }; }; examples = { diff --git a/garlic/exp/osu/eager.nix b/garlic/exp/osu/eager.nix new file mode 100644 index 0000000..d1d89fe --- /dev/null +++ b/garlic/exp/osu/eager.nix @@ -0,0 +1,83 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with builtins; +with stdenv.lib; + +let + + machineConfig = targetMachine.config; + + # Initial variable configuration + varConf = with bsc; { + sizeKB = range 5 25; + mpi = [ impi ]; + #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; + PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 20 24 ]; + PSM2_MTU_KB = [ 10 ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + inherit (machineConfig) hw; + nodes = 2; + ntasksPerNode = 1; + cpusPerTask = 1; + time = "00:30:00"; + qos = "debug"; + loops = 10; + iterations = 50000; + #FIXME: Notice the switchover is 16000 and MTU is 10240 + PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; + PSM2_MTU = PSM2_MTU_KB * 1024; + expName = "osu-bw"; + unitName = expName + + "-size.${toString sizeKB}K" + + "-mtu.${toString PSM2_MTU_KB}K" + + "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; + jobName = expName; + inherit (c) mpi sizeKB + PSM2_MQ_EAGER_SDMA_SZ_KB + PSM2_MTU_KB; + + size = sizeKB * 1024; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + + program = "${nextStage}/bin/osu_bw"; + + env = '' + export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} + export PSM2_MTU=${toString PSM2_MTU} + export PSM2_TRACEMASK=0x101 + export PSM2_MQ_PRINT_STATS=-1 + ''; + + argv = [ + "-m" "${toString size}:${toString size}" + "-i" iterations + ]; + }; + + program = {nextStage, conf, ...}: bsc.osumb.override { + # Use the specified MPI implementation + inherit (conf) mpi; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/osu/mtu.nix b/garlic/exp/osu/mtu.nix new file mode 100644 index 0000000..ac58711 --- /dev/null +++ b/garlic/exp/osu/mtu.nix @@ -0,0 +1,83 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with builtins; +with stdenv.lib; + +let + + machineConfig = targetMachine.config; + + # Initial variable configuration + varConf = with bsc; { + sizeKB = range 5 25; + mpi = [ impi ]; + #mpi = [ impi bsc.openmpi mpich ]; #psmpi ]; + PSM2_MQ_EAGER_SDMA_SZ_KB = [ 16 ]; + PSM2_MTU_KB = [ 8 10 ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + inherit (machineConfig) hw; + nodes = 2; + ntasksPerNode = 1; + cpusPerTask = 1; + time = "00:30:00"; + qos = "debug"; + loops = 10; + iterations = 50000; + #FIXME: Notice the switchover is 16000 and MTU is 10240 + PSM2_MQ_EAGER_SDMA_SZ = PSM2_MQ_EAGER_SDMA_SZ_KB * 1000; + PSM2_MTU = PSM2_MTU_KB * 1024; + expName = "osu-bw"; + unitName = expName + + "-size.${toString sizeKB}K" + + "-mtu.${toString PSM2_MTU_KB}K" + + "-sdma.${toString PSM2_MQ_EAGER_SDMA_SZ_KB}K"; + jobName = expName; + inherit (c) mpi sizeKB + PSM2_MQ_EAGER_SDMA_SZ_KB + PSM2_MTU_KB; + + size = sizeKB * 1024; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + + program = "${nextStage}/bin/osu_bw"; + + env = '' + export PSM2_MQ_EAGER_SDMA_SZ=${toString PSM2_MQ_EAGER_SDMA_SZ} + export PSM2_MTU=${toString PSM2_MTU} + export PSM2_TRACEMASK=0x101 + export PSM2_MQ_PRINT_STATS=-1 + ''; + + argv = [ + "-m" "${toString size}:${toString size}" + "-i" iterations + ]; + }; + + program = {nextStage, conf, ...}: bsc.osumb.override { + # Use the specified MPI implementation + inherit (conf) mpi; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index 7aef5eb..4810dcb 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -77,6 +77,8 @@ in bw = customPlot ./osu/bw.R (ds.osu.bw bw.result); bwShm = customPlot ./osu/bw.R (ds.osu.bw bwShm.result); impi = customPlot ./osu/impi.R (ds.osu.bw impi.result); + mtu = customPlot ./osu/mtu.R (ds.osu.bw mtu.result); + eager = customPlot ./osu/eager.R (ds.osu.bw eager.result); }; # The figures used in the article contained in a directory per figure diff --git a/garlic/fig/osu/eager.R b/garlic/fig/osu/eager.R new file mode 100644 index 0000000..2ec73bc --- /dev/null +++ b/garlic/fig/osu/eager.R @@ -0,0 +1,62 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) + +args=commandArgs(trailingOnly=TRUE) + +# Read the timetable from args[1] +input_file = "input.json" +if (length(args)>0) { input_file = args[1] } + +# Load the dataset in NDJSON format +dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() + +# We only need the nblocks and time +df = select(dataset, + config.unitName, + config.nodes, + config.ntasksPerNode, + config.cpusPerTask, + config.PSM2_MQ_EAGER_SDMA_SZ, + size, bw, config.iterations) %>% + rename(unitName=config.unitName, + iterations=config.iterations, + PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ) + +nodes = unique(df$config.nodes) +tasksPerNode = unique(df$config.ntasksPerNode) +cpusPerTask = unique(df$config.cpusPerTask) +df$unitName = as.factor(df$unitName) +df$sizeFactor = as.factor(df$size) +df$sizeKB = df$size / 1024 +df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ) + +iterations = unique(df$iterations) + +df = group_by(df, unitName, sizeFactor) %>% + mutate(medianBw = median(bw)) %>% + ungroup() + +breaks = 10^(-10:10) +minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) + +ppi=150 +h=6 +w=8 + +p = ggplot(data=df, aes(x=sizeKB, y=bw)) + + labs(x="Message size (KB)", y="Bandwidth (MB/s)", + title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), + subtitle=input_file) + + geom_point(shape=21, size=3) + + geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") + + geom_vline(xintercept = 10, color="red") + + annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + + facet_wrap(vars(PSM2_MQ_EAGER_SDMA_SZ.f), nrow=3, labeller = "label_both") + + scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + + theme_bw() + +ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) +ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi) diff --git a/garlic/fig/osu/mtu.R b/garlic/fig/osu/mtu.R new file mode 100644 index 0000000..1344126 --- /dev/null +++ b/garlic/fig/osu/mtu.R @@ -0,0 +1,65 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) + +args=commandArgs(trailingOnly=TRUE) + +# Read the timetable from args[1] +input_file = "input.json" +if (length(args)>0) { input_file = args[1] } + +# Load the dataset in NDJSON format +dataset = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() + +# We only need the nblocks and time +df = select(dataset, + config.unitName, + config.nodes, + config.ntasksPerNode, + config.cpusPerTask, + config.PSM2_MQ_EAGER_SDMA_SZ, + config.PSM2_MTU, + size, bw, config.iterations) %>% + rename(unitName=config.unitName, + iterations=config.iterations, + PSM2_MQ_EAGER_SDMA_SZ=config.PSM2_MQ_EAGER_SDMA_SZ, + PSM2_MTU=config.PSM2_MTU) + +nodes = unique(df$config.nodes) +tasksPerNode = unique(df$config.ntasksPerNode) +cpusPerTask = unique(df$config.cpusPerTask) +df$unitName = as.factor(df$unitName) +df$sizeFactor = as.factor(df$size) +df$sizeKB = df$size / 1024 +df$PSM2_MQ_EAGER_SDMA_SZ.f = as.factor(df$PSM2_MQ_EAGER_SDMA_SZ) +df$PSM2_MTU.f = as.factor(df$PSM2_MTU) + +iterations = unique(df$iterations) + +df = group_by(df, unitName, sizeFactor) %>% + mutate(medianBw = median(bw)) %>% + ungroup() + +breaks = 10^(-10:10) +minor_breaks <- rep(1:9, 21)*(10^rep(-10:10, each=9)) + +ppi=150 +h=6 +w=8 + +p = ggplot(data=df, aes(x=sizeKB, y=bw)) + + labs(x="Message size (KB)", y="Bandwidth (MB/s)", + title=sprintf("OSU benchmark: osu_bw --iterations %d", iterations), + subtitle=input_file) + + geom_point(shape=21, size=3) + + geom_vline(aes(xintercept = PSM2_MQ_EAGER_SDMA_SZ/1024), color="blue") + + geom_vline(aes(xintercept = PSM2_MTU / 1024), color="red") + + #annotate("text", x = 10.2, y = 8.5e3, label = "MTU = 10KB", color="red", hjust=0) + + facet_wrap(vars(PSM2_MTU.f), nrow=3, labeller = "label_both") + + scale_x_continuous(breaks = unique(df$sizeKB), minor_breaks=NULL) + + theme_bw() + +ggsave("bw.png", plot=p, width=w, height=h, dpi=ppi) +ggsave("bw.pdf", plot=p, width=w, height=h, dpi=ppi)