Add experiments with jemalloc and CPU affinity

garlic/exp/nbody/tampi.nix

@@ -4,6 +4,8 @@
 , bsc
 , targetMachine
 , stages
+, enableJemalloc ? false
+, enableFreeCpu ? false
 }:
 
 with stdenv.lib;
@@ -35,7 +37,7 @@ let
     mpi = impi;
     gitBranch = "garlic/tampi+send+oss+task";
     cflags = "-g";
-
+    
     # Repeat the execution of each unit 30 times
     loops = 10;
 
@@ -44,7 +46,9 @@ let
     ntasksPerNode = hw.socketsPerNode;
     nodes = 1;
     time = "02:00:00";
-    cpuBind = "sockets,verbose";
+    cpuBind = if (enableFreeCpu)
+      then "verbose,mask_cpu:0x7fffff,0x7fffff000000"
+      else "sockets,verbose";
     jobName = "bs-${toString blocksize}-${gitBranch}-nbody";
   };
 
@@ -67,9 +71,13 @@ let
   let
     customPkgs = stdexp.replaceMpi conf.mpi;
   in
-    customPkgs.apps.nbody.override {
+    customPkgs.apps.nbody.override ({
       inherit cc blocksize mpi gitBranch cflags;
-    };
+    } // optionalAttrs enableJemalloc {
+      mcxx = bsc.mcxx.override {
+        nanos6 = bsc.nanos6Jemalloc;
+      };
+    });
 
   pipeline = stdexp.stdPipeline ++ [ exec program ];
 

garlic/fig/nbody/freeCpu/default.nix

@@ -0,0 +1,67 @@
+{
+  stdenv
+, gnuplot
+, jq
+, garlicTools
+, resultFromTrebuchet
+, writeText
+, rWrapper
+, rPackages
+
+# The two results to be compared
+, resDefault
+, resFreeCpu
+}:
+
+with garlicTools;
+with stdenv.lib;
+
+let
+  customR = rWrapper.override {
+    packages = with rPackages; [ tidyverse ];
+  };
+
+  plotScript = ./plot.R;
+
+in stdenv.mkDerivation {
+  name = "plot";
+  buildInputs = [ jq gnuplot customR ];
+  preferLocalBuild = true;
+  dontPatchShebangs = true;
+
+  src = ./.;
+
+  buildPhase = ''
+    echo default = ${resDefault}
+    echo freeCpu = ${resFreeCpu}
+
+    substituteAllInPlace plot.R
+    sed -ie "s:@expResult@:$out:g" plot.R
+
+    for unit in ${resDefault}/*/*; do
+      name=$(basename $unit)
+      log="$unit/stdout.log"
+      conf="$unit/garlic_config.json"
+      bs=$(jq .blocksize $conf)
+      awk "/^time /{print \"default\", $bs, \$2}" $log >> data.csv
+    done
+
+    for unit in ${resFreeCpu}/*/*; do
+      name=$(basename $unit)
+      log="$unit/stdout.log"
+      conf="$unit/garlic_config.json"
+      bs=$(jq .blocksize $conf)
+      awk "/^time /{print \"freeCpu\", $bs, \$2}" $log >> data.csv
+    done
+
+    Rscript plot.R
+  '';
+
+  installPhase = ''
+    mkdir $out
+    ln -s ${resFreeCpu} $out/resFreeCpu
+    ln -s ${resDefault} $out/resDefault
+    cp *.png $out/
+    cp *.csv $out/
+  '';
+}

garlic/fig/nbody/freeCpu/plot.R

@@ -0,0 +1,95 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+
+# Load the dataset
+#df=read.table("/nix/store/zcyazjbcjn2lhxrpa3bs5y7rw3bbcgnr-plot/data.csv",
+df=read.table("data.csv",
+	      col.names=c("variant", "blocksize", "time"))
+
+# Use the blocksize as factor
+df$blocksize = as.factor(df$blocksize)
+
+# Split by malloc variant
+
+D=df %>% group_by(variant, blocksize) %>%
+	mutate(tnorm = time / median(time) - 1)
+
+
+bs_unique = unique(df$blocksize)
+nbs=length(bs_unique)
+
+
+print(D)
+
+ppi=300
+h=5
+w=5
+
+png("box.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+#
+#
+# Create the plot with the normalized time vs blocksize
+p = ggplot(data=D, aes(x=blocksize, y=tnorm)) +
+
+	# Labels
+	labs(x="Block size", y="Normalized time",
+              title="Nbody normalized time",
+              subtitle="@expResult@/data.csv") +
+
+	# Center the title
+	#theme(plot.title = element_text(hjust = 0.5)) +
+
+	# Black and white mode (useful for printing)
+	#theme_bw() +
+
+	# Add the maximum allowed error lines
+	geom_hline(yintercept=c(-0.01, 0.01),
+		linetype="dashed", color="red") +
+
+	# Draw boxplots
+	geom_boxplot(aes(fill=variant)) +
+
+#	# Use log2 scale in x
+#	scale_x_continuous(trans=log2_trans(),
+#			   breaks=bs_unique) +
+#
+	scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) +
+
+	theme_bw() +
+
+	theme(plot.subtitle=element_text(size=10)) +
+
+	theme(legend.position = c(0.85, 0.85)) #+
+
+	# Place each variant group in one separate plot
+	#facet_wrap(~variant)
+
+
+
+# Render the plot
+print(p)
+
+## Save the png image
+dev.off()
+#
+png("scatter.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+## Create the plot with the normalized time vs blocksize
+p = ggplot(D, aes(x=blocksize, y=time, color=variant)) +
+
+	labs(x="Block size", y="Time (s)",
+              title="Nbody granularity",
+              subtitle="@expResult@") +
+	theme_bw() +
+
+	geom_point(shape=21, size=3) +
+	#scale_x_continuous(trans=log2_trans()) +
+	scale_y_continuous(trans=log2_trans())
+
+# Render the plot
+print(p)
+
+# Save the png image
+dev.off()

garlic/fig/nbody/jemalloc/default.nix

@@ -0,0 +1,68 @@
+{
+  stdenv
+, gnuplot
+, jq
+, garlicTools
+, resultFromTrebuchet
+, writeText
+, rWrapper
+, rPackages
+
+# The two results to be compared
+, resDefault
+, resJemalloc
+}:
+
+with garlicTools;
+with stdenv.lib;
+
+let
+  customR = rWrapper.override {
+    packages = with rPackages; [ tidyverse ];
+  };
+
+  plotScript = ./plot.R;
+
+in stdenv.mkDerivation {
+  name = "plot";
+  buildInputs = [ jq gnuplot customR ];
+  preferLocalBuild = true;
+  dontPatchShebangs = true;
+
+  inherit resDefault resJemalloc;
+
+  src = ./.;
+
+  buildPhase = ''
+    echo default = ${resJemalloc}
+    echo jemalloc = ${resJemalloc}
+
+    substituteAllInPlace plot.R
+
+    for unit in ${resDefault}/*/*; do
+      name=$(basename $unit)
+      log="$unit/stdout.log"
+      conf="$unit/garlic_config.json"
+      bs=$(jq .blocksize $conf)
+      awk "/^time /{print \"default\", $bs, \$2}" $log >> data.csv
+    done
+
+    for unit in ${resJemalloc}/*/*; do
+      name=$(basename $unit)
+      log="$unit/stdout.log"
+      conf="$unit/garlic_config.json"
+      bs=$(jq .blocksize $conf)
+      awk "/^time /{print \"jemalloc\", $bs, \$2}" $log >> data.csv
+    done
+
+    #Rscript plot.R
+  '';
+
+  installPhase = ''
+    mkdir $out
+    ln -s ${resJemalloc} $out/resJemalloc
+    ln -s ${resDefault} $out/resDefault
+    #cp *.png $out/
+    cp *.csv $out/
+  '';
+}

garlic/fig/nbody/jemalloc/plot.R

@@ -0,0 +1,92 @@
+library(ggplot2)
+library(dplyr)
+library(scales)
+
+# Load the dataset
+df=read.table("/nix/store/vvfcimwp8mkv6kc5fs3rbyjy8grgpmmb-plot/data.csv",
+	      col.names=c("variant", "blocksize", "time"))
+
+# Use the blocksize as factor
+df$blocksize = as.factor(df$blocksize)
+
+# Split by malloc variant
+
+D=df %>% group_by(variant, blocksize) %>%
+	mutate(tnorm = time / median(time) - 1)
+
+
+bs_unique = unique(df$blocksize)
+nbs=length(bs_unique)
+
+
+print(D)
+
+ppi=300
+h=5
+w=5
+
+png("box.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+#
+#
+# Create the plot with the normalized time vs blocksize
+p = ggplot(data=D, aes(x=blocksize, y=tnorm)) +
+
+	# Labels
+	labs(x="Block size", y="Normalized time",
+              title="Nbody normalized time",
+              subtitle="@expResult@") +
+
+	# Center the title
+	#theme(plot.title = element_text(hjust = 0.5)) +
+
+	# Black and white mode (useful for printing)
+	#theme_bw() +
+
+	# Add the maximum allowed error lines
+	geom_hline(yintercept=c(-0.01, 0.01),
+		linetype="dashed", color="red") +
+
+	# Draw boxplots
+	geom_boxplot(aes(fill=variant)) +
+
+#	# Use log2 scale in x
+#	scale_x_continuous(trans=log2_trans(),
+#			   breaks=bs_unique) +
+#
+	scale_y_continuous(breaks = scales::pretty_breaks(n = 10)) +
+
+	theme_bw() +
+
+	theme(legend.position = c(0.85, 0.85)) #+
+
+	# Place each variant group in one separate plot
+	#facet_wrap(~variant)
+
+
+
+# Render the plot
+print(p)
+
+## Save the png image
+dev.off()
+#
+#png("scatter.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+## Create the plot with the normalized time vs blocksize
+#p = ggplot(D, aes(x=blocksize, y=time, color=variant)) +
+#
+#	labs(x="Block size", y="Time (s)",
+#              title="Nbody granularity",
+#              subtitle="@expResult@") +
+#	theme_bw() +
+#
+#	geom_point(shape=21, size=3) +
+#	scale_x_continuous(trans=log2_trans()) +
+#	scale_y_continuous(trans=log2_trans())
+#
+## Render the plot
+#print(p)
+#
+## Save the png image
+#dev.off()

garlic/fig/nbody/test/plot.R

@@ -9,19 +9,28 @@ bs_unique = unique(df$blocksize)
 nbs=length(bs_unique)
 
 # Normalize the time by the median
-D=group_by(df, blocksize) %>% mutate(tnorm = time / median(time) - 1)
+D=group_by(df, blocksize) %>%
+	mutate(tnorm = time / median(time) - 1) # %>%
+#	mutate(bad = (abs(tnorm) >= 0.01)) %>%
+#	mutate(color = ifelse(bad,"red","black"))
 
-ppi=300
+D$bad = cut(abs(D$tnorm), breaks=c(-Inf, 0.01, +Inf), labels=c("good", "bad"))
-h=5
-w=5
-png("box.png", width=w*ppi, height=h*ppi, res=ppi)
 
+print(D)
+
+#ppi=300
+#h=5
+#w=5
+#png("box.png", width=w*ppi, height=h*ppi, res=ppi)
+#
+#
+#
 # Create the plot with the normalized time vs blocksize
 p = ggplot(D, aes(x=blocksize, y=tnorm)) +
 
 	# Labels
-	labs(x="Blocksize", y="Normalized time",
+	labs(x="Block size", y="Normalized time",
-              title="Nbody granularity",
+              title="Nbody normalized time",
               subtitle="@expResult@") +
 
 	# Center the title
@@ -43,32 +52,31 @@ p = ggplot(D, aes(x=blocksize, y=tnorm)) +
 	geom_hline(yintercept=c(-0.01, 0.01),
 		linetype="dashed", color="red")
 
-# Render the plot
-print(p)
 
-# Save the png image
-dev.off()
-
-D=group_by(df, blocksize) %>% mutate(tnorm = time / median(time) - 1)
-
-png("scatter.png", width=w*ppi, height=h*ppi, res=ppi)
-
-# Create the plot with the normalized time vs blocksize
-p = ggplot(D, aes(x=blocksize, y=time)) +
-
-	labs(x="Blocksize", y="Time (s)",
-              title="Nbody granularity",
-              subtitle="@expResult@") +
-
-	geom_point(
-		   #position=position_jitter(width=0.2, heigh=0)
-		   shape=21, size=1.5) +
-	scale_x_continuous(trans=log2_trans(),
-			   breaks=bs_unique) +
-	scale_y_continuous(trans=log2_trans())
 
 # Render the plot
 print(p)
+#
+## Save the png image
+#dev.off()
+#
+#png("scatter.png", width=w*ppi, height=h*ppi, res=ppi)
+
+## Create the plot with the normalized time vs blocksize
+#p = ggplot(D, aes(x=blocksize, y=time, color=bad)) +
+#
+#	labs(x="Blocksize", y="Time (s)",
+#              title="Nbody granularity",
+#              subtitle="@expResult@") +
+#
+#	geom_point(shape=21, size=1.5) +
+#	scale_color_manual(values=c("black", "red")) +
+#	scale_x_continuous(trans=log2_trans(),
+#			   breaks=bs_unique) +
+#	scale_y_continuous(trans=log2_trans())
+#
+## Render the plot
+#print(p)
 
 # Save the png image
-dev.off()
+#dev.off()

overlay.nix

@@ -282,6 +282,12 @@ let
         nbody = {
           test  = callPackage ./garlic/exp/nbody/test.nix { };
           tampi = callPackage ./garlic/exp/nbody/tampi.nix { };
+          freeCpu = callPackage ./garlic/exp/nbody/tampi.nix {
+            enableFreeCpu = true;
+          };
+          jemalloc = callPackage ./garlic/exp/nbody/tampi.nix {
+            enableJemalloc = true;
+          };
         };
 
         saiph = {
@@ -307,6 +313,12 @@ let
       # Post processing tools
       hist = callPackage ./garlic/postprocess/hist { };
       getExpResult = callPackage ./garlic/postprocess/result.nix { };
+      resultFromTrebuchet = trebuchetStage: self.garlic.getExpResult {
+        garlicTemp = "/tmp/garlic-temp";
+        inherit trebuchetStage;
+        experimentStage = with self.bsc.garlicTools;
+          getExperimentStage trebuchetStage;
+      };
       fetchExperiment = callPackage ./garlic/postprocess/fetch.nix { };
 
       # Figures generated from the experiments
@@ -317,6 +329,18 @@ let
               self.bsc.garlic.exp.nbody.tampi
             ];
           };
+          jemalloc = callPackage ./garlic/fig/nbody/jemalloc/default.nix {
+            resDefault = self.garlic.resultFromTrebuchet
+              self.bsc.garlic.exp.nbody.tampi;
+            resJemalloc = self.garlic.resultFromTrebuchet
+              self.bsc.garlic.exp.nbody.jemalloc;
+          };
+          freeCpu = callPackage ./garlic/fig/nbody/freeCpu/default.nix {
+            resDefault = self.garlic.resultFromTrebuchet
+              self.bsc.garlic.exp.nbody.tampi;
+            resFreeCpu = self.garlic.resultFromTrebuchet
+              self.bsc.garlic.exp.nbody.freeCpu;
+          };
         };
       };
     };