diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix index ebaf371..b220df2 100644 --- a/garlic/exp/fwi/common.nix +++ b/garlic/exp/fwi/common.nix @@ -31,6 +31,15 @@ rec { # join branch, even if we have multiple blocksizes. unique (map fixBlocksize allConfigs); + getResources = {gitBranch, hw}: + if (gitBranch == "garlic/mpi+send+seq") then { + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = hw.socketsPerNode; + } else { + cpusPerTask = 1; + ntasksPerNode = hw.cpusPerNode; + }; + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix index 25ff81d..2302d43 100644 --- a/garlic/exp/fwi/ss.nix +++ b/garlic/exp/fwi/ss.nix @@ -17,6 +17,8 @@ with stdenv.lib; with garlicTools; let + common = callPackage ./common.nix {}; + inherit (common) getConfigs getResources pipeline; inherit (targetMachine) fs; @@ -29,13 +31,16 @@ let "garlic/mpi+send+omp+task" "garlic/mpi+send+oss+task" "garlic/mpi+send+omp+fork" + # FIXME: the mpi pure version has additional constraints with the + # number of planes in Y. By now is disabled. + #"garlic/mpi+send+seq" ]; blocksize = if (enableExtended) then range2 1 16 else [ 2 ]; - n = [ {nx=100; nz=100; ny=8000;} ]; + n = [ { nx=100; ny=8000; nz=100; } ]; nodes = range2 1 16; }; @@ -65,8 +70,9 @@ let loops = 10; # Resources - cpusPerTask = hw.cpusPerSocket; - ntasksPerNode = hw.socketsPerNode; + inherit (getResources { inherit gitBranch hw; }) + cpusPerTask ntasksPerNode; + nodes = c.nodes; qos = "debug"; time = "02:00:00"; @@ -79,10 +85,6 @@ let tempDir = fs.local.temp; }; - common = callPackage ./common.nix {}; - - inherit (common) getConfigs pipeline; - configs = getConfigs { inherit varConf genConf; }; diff --git a/garlic/exp/fwi/strong_scaling_mpionly.nix b/garlic/exp/fwi/strong_scaling_mpionly.nix deleted file mode 100644 index 94ffd8d..0000000 --- a/garlic/exp/fwi/strong_scaling_mpionly.nix +++ /dev/null @@ -1,130 +0,0 @@ -# Strong scaling test for FWI variants based exclusively on MPI. This -# experiment does not rely on block sizes. An MPI process is instantiated per -# core. - -{ - stdenv -, stdexp -, bsc -, targetMachine -, stages -}: - -with stdenv.lib; - -let - - inherit (targetMachine) fs; - - # Initial variable configuration - varConf = { - gitBranch = [ - "garlic/mpi+send+seq" - ]; - - blocksize = [ 0 ]; - - n = [ - {nx=100; nz=100; ny=8000;} - ]; - - # Not enough planes for 4, 8 and 16 nodes - nodes = [ 1 2 ]; - - }; - -# The c value contains something like: -# { -# n = { nx=500; ny=500; nz=500; } -# blocksize = 1; -# gitBranch = "garlic/tampi+send+oss+task"; -# } - - machineConfig = targetMachine.config; - - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "fwi"; - unitName = "${expName}-test"; - inherit (machineConfig) hw; - - cc = icc; - inherit (c) gitBranch blocksize; - - #nx = c.n.nx; - #ny = c.n.ny; - #nz = c.n.nz; - - # Same but shorter: - inherit (c.n) nx ny nz; - - fwiInput = bsc.apps.fwi.input.override { - inherit (c.n) nx ny nz; - }; - - # Other FWI parameters - ioFreq = -1; - - # Repeat the execution of each unit several times - loops = 10; - #loops = 1; - - # Resources - cpusPerTask = 1; - ntasksPerNode = hw.cpusPerNode; - nodes = c.nodes; - qos = "debug"; - time = "02:00:00"; - jobName = unitName; - - tracing = "no"; - - # Enable permissions to write in the local storage - extraMounts = [ fs.local.temp ]; - - }; - - # Compute the array of configurations - configs = stdexp.buildConfigs { - inherit varConf genConf; - }; - - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = '' - CDIR=$PWD - if [[ "${conf.tracing}" == "yes" ]]; then - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - fi - EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - mkdir -p $EXECDIR - cd $EXECDIR - ln -fs ${conf.fwiInput}/InputModels InputModels || true - ''; - argv = [ - "${conf.fwiInput}/fwi_params.txt" - "${conf.fwiInput}/fwi_frequencies.txt" - "-1" # Fordward steps - "-1" # Backward steps - conf.ioFreq # Write/read frequency - ]; - post = '' - rm -rf Results || true - if [[ "${conf.tracing}" == "yes" ]]; then - mv trace_* $CDIR - fi - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: apps.fwi.solver.override { - inherit (conf) cc gitBranch fwiInput; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; - -in - - stdexp.genExperiment { inherit configs pipeline; }