From 9aa07993b2088db832a0c414760cc385e8b628eb Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 12 Apr 2021 14:41:26 +0200 Subject: [PATCH] fwi: refactor ss and granularity experiments A common.nix file contains the shared stages --- garlic/exp/fwi/common.nix | 59 ++++++++++++++++++++++++++ garlic/exp/fwi/granularity.nix | 77 +++++++--------------------------- garlic/exp/fwi/ss.nix | 61 ++++----------------------- 3 files changed, 84 insertions(+), 113 deletions(-) create mode 100644 garlic/exp/fwi/common.nix diff --git a/garlic/exp/fwi/common.nix b/garlic/exp/fwi/common.nix new file mode 100644 index 0000000..5ac38ea --- /dev/null +++ b/garlic/exp/fwi/common.nix @@ -0,0 +1,59 @@ +{ + stdenv +, stdexp +, bsc +, stages +}: + +with stdenv.lib; + +rec { + + # We split these into a separate group so we can remove the blocksize + # later. + forkJoinBranches = [ "garlic/mpi+send+omp+fork" ]; + + # Returns true if the given config is in the forkJoinBranches list + isForkJoin = c: any (e: c.gitBranch == e) forkJoinBranches; + + # Set the blocksize to null for the fork join branch + fixBlocksize = c: if (isForkJoin c) then (c // { blocksize = null; }) else c; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + + pre = '' + CDIR=$(pwd) + EXECDIR="${conf.tempDir}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p "$EXECDIR" + cd "$EXECDIR" + ln -fs ${conf.fwiInput}/InputModels InputModels || true + '' + optionalString (conf.enableCTF) '' + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + ''; + + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + ] ++ optional (! isForkJoin conf) conf.blocksize ++ [ + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + + post = '' + rm -rf Results || true + '' + optionalString (conf.enableCTF) '' + mv trace_* "$CDIR" + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; +} diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix index 7773b3b..df3ceb2 100644 --- a/garlic/exp/fwi/granularity.nix +++ b/garlic/exp/fwi/granularity.nix @@ -6,9 +6,12 @@ , bsc , targetMachine , stages +, garlicTools +, callPackage }: with stdenv.lib; +with garlicTools; let @@ -27,38 +30,24 @@ let # "garlic/seq" ]; - blocksize = [ 1 2 4 8 16 32 64 128 256 ]; + blocksize = range2 1 256; - n = [ - {nx=100; nz=100; ny=8000; ntpn=2; nn=1;} - ]; + n = [ {nx=100; nz=100; ny=8000; ntpn=2; nodes=1;} ]; }; -# The c value contains something like: -# { -# n = { nx=500; ny=500; nz=500; } -# blocksize = 1; -# gitBranch = "garlic/tampi+send+oss+task"; -# } - machineConfig = targetMachine.config; # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "fwi"; - unitName = "${expName}-test"; + genConf = c: targetMachine.config // rec { + expName = "fwi-granularity"; + unitName = "${expName}" + + "-bs${toString blocksize}" + + "-${toString gitBranch}"; + inherit (machineConfig) hw; - - cc = icc; inherit (c) gitBranch blocksize; - - #nx = c.n.nx; - #ny = c.n.ny; - #nz = c.n.nz; - - # Same but shorter: - inherit (c.n) nx ny nz ntpn nn; + inherit (c.n) nx ny nz ntpn nodes; fwiInput = bsc.apps.fwi.input.override { inherit (c.n) nx ny nz; @@ -69,20 +58,19 @@ let # Repeat the execution of each unit several times loops = 10; - #loops = 1; # Resources cpusPerTask = hw.cpusPerSocket; ntasksPerNode = ntpn; - nodes = nn; qos = "debug"; time = "02:00:00"; jobName = unitName; - tracing = "no"; + enableCTF = false; # Enable permissions to write in the local storage extraMounts = [ fs.local.temp ]; + tempDir = fs.local.temp; }; @@ -91,42 +79,9 @@ let inherit varConf genConf; }; - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = '' - CDIR=$PWD - if [[ "${conf.tracing}" == "yes" ]]; then - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - fi - EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - mkdir -p $EXECDIR - cd $EXECDIR - ln -fs ${conf.fwiInput}/InputModels InputModels || true - ''; - argv = [ - "${conf.fwiInput}/fwi_params.txt" - "${conf.fwiInput}/fwi_frequencies.txt" - conf.blocksize - "-1" # Fordward steps - "-1" # Backward steps - conf.ioFreq # Write/read frequency - ]; - post = '' - rm -rf Results || true - if [[ "${conf.tracing}" == "yes" ]]; then - mv trace_* $CDIR - fi - ''; - }; + common = callPackage ./common.nix {}; - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: apps.fwi.solver.override { - inherit (conf) cc gitBranch fwiInput; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; + inherit (common) fixBlocksize pipeline; in diff --git a/garlic/exp/fwi/ss.nix b/garlic/exp/fwi/ss.nix index 7329d1d..ff8c4ec 100644 --- a/garlic/exp/fwi/ss.nix +++ b/garlic/exp/fwi/ss.nix @@ -9,6 +9,7 @@ , targetMachine , stages , garlicTools +, callPackage , enableExtended ? false }: @@ -17,23 +18,20 @@ with garlicTools; let - inherit (targetMachine) fs; + common = callPackage ./common.nix {}; - # We split these into a separate group so we can remove the blocksize - # later. - forkJoinBranches = [ - "garlic/mpi+send+omp+fork" - ]; + inherit (targetMachine) fs; # Initial variable configuration varConf = { gitBranch = [ "garlic/tampi+isend+oss+task" - ] ++ optionals (enableExtended) ([ + ] ++ optionals (enableExtended) [ "garlic/tampi+send+oss+task" "garlic/mpi+send+omp+task" "garlic/mpi+send+oss+task" - ] ++ forkJoinBranches); + "garlic/mpi+send+omp+fork" + ]; blocksize = if (enableExtended) then range2 1 16 @@ -47,7 +45,7 @@ let machineConfig = targetMachine.config; # Generate the complete configuration for each unit - genConf = c: targetMachine.config // rec { + genConf = c: machineConfig // rec { expName = "fwi-ss"; unitName = "${expName}" + "-nodes${toString nodes}" @@ -80,15 +78,9 @@ let # Enable permissions to write in the local storage extraMounts = [ fs.local.temp ]; - + tempDir = fs.local.temp; }; - # Returns true if the given config is in the forkJoinBranches list - isForkJoin = c: any (e: c.gitBranch == e) forkJoinBranches; - - # Set the blocksize to null for the fork join branch - fixBlocksize = c: if (isForkJoin c) then (c // { blocksize = null; }) else c; - # Compute the array of configurations allConfigs = stdexp.buildConfigs { inherit varConf genConf; @@ -98,42 +90,7 @@ let # join branch, even if we have multiple blocksizes. configs = unique (map fixBlocksize allConfigs); - exec = {nextStage, conf, ...}: stages.exec { - inherit nextStage; - pre = '' - CDIR=$(pwd) - EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" - mkdir -p "$EXECDIR" - cd "$EXECDIR" - ln -fs ${conf.fwiInput}/InputModels InputModels || true - '' + optionalString (conf.enableCTF) '' - export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" - ''; - argv = [ - "${conf.fwiInput}/fwi_params.txt" - "${conf.fwiInput}/fwi_frequencies.txt" - ] - ++ optional (isForkJoin conf) conf.blocksize - ++ [ - "-1" # Fordward steps - "-1" # Backward steps - conf.ioFreq # Write/read frequency - ]; - post = '' - rm -rf Results || true - '' + optionalString (conf.enableCTF) '' - mv trace_* "$CDIR" - ''; - }; - - apps = bsc.garlic.apps; - - # FWI program - program = {nextStage, conf, ...}: apps.fwi.solver.override { - inherit (conf) gitBranch fwiInput; - }; - - pipeline = stdexp.stdPipeline ++ [ exec program ]; + inherit (common) fixBlocksize pipeline; in