diff --git a/garlic/exp/fwi/strong_scaling_task.nix b/garlic/exp/fwi/ss.nix similarity index 52% rename from garlic/exp/fwi/strong_scaling_task.nix rename to garlic/exp/fwi/ss.nix index 4ad7833..7329d1d 100644 --- a/garlic/exp/fwi/strong_scaling_task.nix +++ b/garlic/exp/fwi/ss.nix @@ -8,56 +8,54 @@ , bsc , targetMachine , stages +, garlicTools +, enableExtended ? false }: with stdenv.lib; +with garlicTools; let inherit (targetMachine) fs; + # We split these into a separate group so we can remove the blocksize + # later. + forkJoinBranches = [ + "garlic/mpi+send+omp+fork" + ]; + # Initial variable configuration varConf = { gitBranch = [ - "garlic/tampi+send+oss+task" - "garlic/tampi+isend+oss+task" - "garlic/mpi+send+omp+task" - "garlic/mpi+send+oss+task" - ]; + "garlic/tampi+isend+oss+task" + ] ++ optionals (enableExtended) ([ + "garlic/tampi+send+oss+task" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + ] ++ forkJoinBranches); - blocksize = [ 1 2 4 8 16 ]; + blocksize = if (enableExtended) + then range2 1 16 + else [ 2 ]; - n = [ - {nx=100; nz=100; ny=8000;} - ]; - - nodes = [ 1 2 4 8 16 ]; + n = [ {nx=100; nz=100; ny=8000;} ]; + nodes = range2 1 16; }; -# The c value contains something like: -# { -# n = { nx=500; ny=500; nz=500; } -# blocksize = 1; -# gitBranch = "garlic/tampi+send+oss+task"; -# } - machineConfig = targetMachine.config; # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "fwi"; - unitName = "${expName}-test"; + genConf = c: targetMachine.config // rec { + expName = "fwi-ss"; + unitName = "${expName}" + + "-nodes${toString nodes}" + + "-bs${toString blocksize}" + + "-${toString gitBranch}"; + inherit (machineConfig) hw; - - cc = icc; inherit (c) gitBranch blocksize; - - #nx = c.n.nx; - #ny = c.n.ny; - #nz = c.n.nz; - - # Same but shorter: inherit (c.n) nx ny nz; fwiInput = bsc.apps.fwi.input.override { @@ -69,53 +67,62 @@ let # Repeat the execution of each unit several times loops = 10; - #loops = 1; # Resources cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 2; + ntasksPerNode = hw.socketsPerNode; nodes = c.nodes; qos = "debug"; time = "02:00:00"; jobName = unitName; - tracing = "no"; + enableCTF = false; # Enable permissions to write in the local storage extraMounts = [ fs.local.temp ]; }; + # Returns true if the given config is in the forkJoinBranches list + isForkJoin = c: any (e: c.gitBranch == e) forkJoinBranches; + + # Set the blocksize to null for the fork join branch + fixBlocksize = c: if (isForkJoin c) then (c // { blocksize = null; }) else c; + # Compute the array of configurations - configs = stdexp.buildConfigs { + allConfigs = stdexp.buildConfigs { inherit varConf genConf; }; + # The unique function ensures that we only run one config for the fork + # join branch, even if we have multiple blocksizes. + configs = unique (map fixBlocksize allConfigs); + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; pre = '' - CDIR=$PWD - if [[ "${conf.tracing}" == "yes" ]]; then - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - fi + CDIR=$(pwd) EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - mkdir -p $EXECDIR - cd $EXECDIR + mkdir -p "$EXECDIR" + cd "$EXECDIR" ln -fs ${conf.fwiInput}/InputModels InputModels || true + '' + optionalString (conf.enableCTF) '' + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" ''; argv = [ "${conf.fwiInput}/fwi_params.txt" "${conf.fwiInput}/fwi_frequencies.txt" - conf.blocksize + ] + ++ optional (isForkJoin conf) conf.blocksize + ++ [ "-1" # Fordward steps "-1" # Backward steps conf.ioFreq # Write/read frequency ]; post = '' rm -rf Results || true - if [[ "${conf.tracing}" == "yes" ]]; then - mv trace_* $CDIR - fi + '' + optionalString (conf.enableCTF) '' + mv trace_* "$CDIR" ''; }; @@ -123,7 +130,7 @@ let # FWI program program = {nextStage, conf, ...}: apps.fwi.solver.override { - inherit (conf) cc gitBranch fwiInput; + inherit (conf) gitBranch fwiInput; }; pipeline = stdexp.stdPipeline ++ [ exec program ]; diff --git a/garlic/exp/fwi/strong_scaling_forkjoin.nix b/garlic/exp/fwi/strong_scaling_forkjoin.nix deleted file mode 100644 index 902eaf6..0000000 --- a/garlic/exp/fwi/strong_scaling_forkjoin.nix +++ /dev/null @@ -1,128 +0,0 @@ -# Strong scaling test for FWI variants based on forkjoint. This -# experiment does not rely on block sizes. - -{ - stdenv -, stdexp -, bsc -, targetMachine -, stages -}: - -with stdenv.lib; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/mpi+send+omp+fork" - ]; - - blocksize = [ 0 ]; - - n = [ - {nx=100; nz=100; ny=8000;} - ]; - - nodes = [ 1 2 4 8 16 ]; - - }; - -# The c value contains something like: -# { -# n = { nx=500; ny=500; nz=500; } -# blocksize = 1; -# gitBranch = "garlic/tampi+send+oss+task"; -# } - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "fwi"; - unitName = "${expName}-test"; - inherit (machineConfig) hw; - - cc = icc; - inherit (c) gitBranch blocksize; - - #nx = c.n.nx; - #ny = c.n.ny; - #nz = c.n.nz; - - # Same but shorter: - inherit (c.n) nx ny nz; - - fwiInput = bsc.apps.fwi.input.override { - inherit (c.n) nx ny nz; - }; - - # Other FWI parameters - ioFreq = -1; - - # Repeat the execution of each unit several times - loops = 10; - #loops = 1; - - # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = 2; - nodes = c.nodes; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - tracing = "no"; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = '' - CDIR=$PWD - if [[ "${conf.tracing}" == "yes" ]]; then - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - fi - EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - mkdir -p $EXECDIR - cd $EXECDIR - ln -fs ${conf.fwiInput}/InputModels InputModels || true - ''; - argv = [ - "${conf.fwiInput}/fwi_params.txt" - "${conf.fwiInput}/fwi_frequencies.txt" - "-1" # Fordward steps - "-1" # Backward steps - conf.ioFreq # Write/read frequency - ]; - post = '' - rm -rf Results || true - if [[ "${conf.tracing}" == "yes" ]]; then - mv trace_* $CDIR - fi - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: apps.fwi.solver.override { - inherit (conf) cc gitBranch fwiInput; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index f6a518d..01275fb 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -98,9 +98,9 @@ }; fwi = { - granularity = callPackage ./fwi/granularity.nix { }; - strong_scaling_task = callPackage ./fwi/strong_scaling_task.nix { }; - strong_scaling_forkjoin = callPackage ./fwi/strong_scaling_forkjoin.nix { }; + granularity = callPackage ./fwi/granularity.nix { }; + ss = callPackage ./fwi/ss.nix { }; + strong_scaling_mpionly = callPackage ./fwi/strong_scaling_mpionly.nix { }; data_reuse = callPackage ./fwi/data_reuse.nix { }; strong_scaling_io = callPackage ./fwi/strong_scaling_io.nix { };