diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index a70556b..5645c62 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -32,8 +32,8 @@ }; saiph = { - granularity-saiph = callPackage ./saiph/granularity-saiph.nix { }; - scalability-saiph = callPackage ./saiph/scalability-saiph.nix { }; + granularity = callPackage ./saiph/granularity.nix { }; + ss = callPackage ./saiph/ss.nix { }; }; creams = rec { diff --git a/garlic/exp/saiph/granularity-saiph.nix b/garlic/exp/saiph/granularity.nix similarity index 52% rename from garlic/exp/saiph/granularity-saiph.nix rename to garlic/exp/saiph/granularity.nix index 0c794a2..6387bf6 100644 --- a/garlic/exp/saiph/granularity-saiph.nix +++ b/garlic/exp/saiph/granularity.nix @@ -1,4 +1,4 @@ -###################################################################################### +####################################################################### # Saiph, granularity experiment: # # App:Heat 3D - garlic/tampi+isend+oss+task+simd branch @@ -15,12 +15,13 @@ # Granularity experiment configuration: # Single-core run # MPI binded to sockets: MPI procs = 2 -# Mesh distributed across third dimension to ensure contiguous communications +# Mesh distributed across third dimension to ensure contiguous +# communications # --> nbgx = 1, nbgy = 1 # First dimension cannot be locally blocked (simd reasons) # Second and third dimension local blocking limited by local mesh size # -###################################################################################### +####################################################################### # Common packages, tools and options { @@ -29,81 +30,102 @@ , bsc , targetMachine , stages +, garlicTools }: with stdenv.lib; +with garlicTools; let - #*** Variable configurations *** - varConf = with bsc; { + # Variable configurations + varConf = with targetMachine.config; { # Local blocks per dimension - nbl1 = [ 1 2 3 4 6 12 24 48 96 ]; - nbl2 = [ 1 2 3 4 6 12 24 48 96 ]; + nblx = [ 1 ]; # SIMD + nbly = range2 1 (hw.cpusPerNode * 8); + nblz = [ 8 ]; + sizex = [ 3 ]; + gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; }; - #*** Generate the complete configuration for each unit *** - genConf = with bsc; c: targetMachine.config // rec { + # Generate the complete configuration for each unit + genConf = c: targetMachine.config // rec { # Experiment, units and job names expName = "saiph-granularity"; - unitName = "${expName}-N${toString nodes}" + "nbg_${toString nbgx}-${toString nbgy}-${toString nbgz}" + "nbl_1-${toString nbly}-${toString nblz}"; - jobName = "${unitName}"; + unitName = "${expName}" + + "-N${toString nodes}" + + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" + + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; + + jobName = unitName; # saiph options + totalTasks = ntasksPerNode * nodes; nodes = 1; enableManualDist = true; # allows to manually set nbg{x-y-z} nbgx = 1; nbgy = 1; - nbgz = nodes*2; # forcing distribution by last dim - nblx = 1; # simd reasons - nbly = c.nbl1; # takes values from varConf - nblz = c.nbl2; # takes values from varConf - mpi = impi; - gitBranch = "garlic/tampi+isend+oss+task+simd"; - gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; # fix a specific commit - rev = "0"; + nbgz = totalTasks; # forcing distribution by last dim - # Repeat the execution of each unit 30 times - loops = 30; + inherit (c) nblx nbly nblz gitBranch sizex; + + blocksPerTask = nblx * nbly * nblz * 1.0; + blocksPerCpu = blocksPerTask / cpusPerTask; + + # fix a specific commit + gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; + + # Repeat the execution of each unit 10 times + loops = 10; # Resources inherit (targetMachine.config) hw; qos = "debug"; - ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets - cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket + ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets + cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket }; #*** Compute the final set of configurations *** - # Compute the array of configurations: cartesian product of all factors + # Compute the array of configurations: cartesian product of all + # factors allConfigs = stdexp.buildConfigs { inherit varConf genConf; }; + # Filter to remove non-desired configurations: # --> tasks/proc < 0.5 # --> nblz > 50 - configs = filter (el: if ((builtins.mul el.nbly el.nblz) < (builtins.mul 0.5 el.cpusPerTask) || el.nblz > 50) then false else true) allConfigs; + isGoodConfig = c: + let + maxNblz = c.cpusPerTask * 2; + in + ! (c.blocksPerCpu < 0.5 || c.nblz > maxNblz); + + configs = filter (isGoodConfig) allConfigs; #*** Sets the env/argv of the program *** - exec = {nextStage, conf, ...}: with conf; stages.exec { + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; env = '' - export OMP_NUM_THREADS=${toString hw.cpusPerSocket} + export OMP_NUM_THREADS=${toString conf.cpusPerTask} ''; }; #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.saiph.override { - inherit (conf) enableManualDist nbgx nbgy nbgz nblx nbly nblz mpi gitBranch gitCommit; - }; + program = {nextStage, conf, ...}: bsc.apps.saiph.override { + inherit (conf) enableManualDist + nbgx nbgy nbgz nblx nbly nblz + sizex + gitBranch gitCommit; + + L3SizeKB = conf.hw.cacheSizeKB.L3; + cachelineBytes = conf.hw.cachelineBytes; + }; #*** Add stages to the pipeline *** pipeline = stdexp.stdPipeline ++ [ exec program ]; in - stdexp.genExperiment { inherit configs pipeline; } + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/saiph/scalability-saiph.nix b/garlic/exp/saiph/ss.nix similarity index 64% rename from garlic/exp/saiph/scalability-saiph.nix rename to garlic/exp/saiph/ss.nix index 5ec5d1a..4fbb24e 100644 --- a/garlic/exp/saiph/scalability-saiph.nix +++ b/garlic/exp/saiph/ss.nix @@ -31,62 +31,68 @@ , bsc , targetMachine , stages +, garlicTools }: with stdenv.lib; +with garlicTools; let #*** Variable configurations *** - varConf = with bsc; { + varConf = with targetMachine.config; { + # FIXME: None of those selected nbl* and problem size is able to give good + # efficiency when testing strong scaling. We should find better values. # Local blocks per dimension - nbl1 = [ 1 2 3 4 6 12 24 48 96 ]; - nbl2 = [ 1 2 3 4 6 12 24 48 96 ]; - # Number of nodes - nodes = [ 1 2 4 8 ]; + nblx = [ 1 ]; # SIMD + nbly = [ 32 ]; + nblz = [ 8 ]; + sizex = [ 3 6 ]; + gitBranch = [ "garlic/tampi+isend+oss+task+simd" ]; + nodes = range2 1 8; }; #*** Generate the complete configuration for each unit *** - genConf = with bsc; c: targetMachine.config // rec { + genConf = c: targetMachine.config // rec { # Experiment, units and job names - expName = "saiph-scalability"; - unitName = "${expName}-N${toString nodes}" + "nbg_${toString nbgx}-${toString nbgy}-${toString nbgz}" + "nbl_1-${toString nbly}-${toString nblz}"; - jobName = "${unitName}"; + expName = "saiph-ss"; + unitName = "${expName}" + + "-N${toString nodes}" + + "-nbg.x${toString nbgx}.y${toString nbgy}.z${toString nbgz}" + + "-nbl.x${toString nblx}.y${toString nbly}.z${toString nblz}"; + jobName = unitName; # saiph options - nodes = c.nodes; # takes values from varConf enableManualDist = true; # allows to manually set nbg{x-y-z} nbgx = 1; nbgy = 1; - nbgz = nodes*2; # forcing distribution by last dim - nblx = 1; # simd reasons - nbly = c.nbl1; # takes values from varConf - nblz = c.nbl2; # takes values from varConf - mpi = impi; + nbgz = nodes * ntasksPerNode; # forcing distribution by last dim + + inherit (c) nblx nbly nblz nodes sizex; + gitBranch = "garlic/tampi+isend+oss+task+simd"; gitCommit = "8052494d7dc62bef95ebaca9938e82fb029686f6"; # fix a specific commit - rev = "0"; - # Repeat the execution of each unit 30 times - loops = 30; + + blocksPerTask = nblx * nbly * nblz * 1.0; + blocksPerCpu = blocksPerTask / cpusPerTask; + + # Repeat the execution of each unit 10 times + loops = 10; # Resources inherit (targetMachine.config) hw; - qos = "bsc_cs"; + + qos = "debug"; ntasksPerNode = hw.socketsPerNode; # MPI binded to sockets cpusPerTask = hw.cpusPerSocket; # Using the 24 CPUs of each socket }; #*** Compute the final set of configurations *** # Compute the array of configurations: cartesian product of all factors - allConfigs = stdexp.buildConfigs { + configs = stdexp.buildConfigs { inherit varConf genConf; }; - # Filter to remove non-desired configurations: - # --> tasks/proc < 3 - # --> nblz > 25 - configs = filter (el: if ((builtins.mul el.nbly el.nblz) < (builtins.mul 3 el.cpusPerTask) || el.nblz > 25) then false else true) allConfigs; - #*** Sets the env/argv of the program *** exec = {nextStage, conf, ...}: with conf; stages.exec { @@ -97,17 +103,19 @@ let }; #*** Configure the program according to the app *** - program = {nextStage, conf, ...}: - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.saiph.override { - inherit (conf) enableManualDist nbgx nbgy nbgz nblx nbly nblz mpi gitBranch gitCommit; - }; + program = {nextStage, conf, ...}: bsc.apps.saiph.override { + inherit (conf) enableManualDist + nbgx nbgy nbgz nblx nbly nblz + sizex + gitBranch gitCommit; + + L3SizeKB = conf.hw.cacheSizeKB.L3; + cachelineBytes = conf.hw.cachelineBytes; + }; #*** Add stages to the pipeline *** pipeline = stdexp.stdPipeline ++ [ exec program ]; in - stdexp.genExperiment { inherit configs pipeline; } + stdexp.genExperiment { inherit configs pipeline; }