From 9cba2d609c0fddb8dab7722d8b01af098071f3e1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Fri, 24 Jul 2020 15:30:28 +0200 Subject: [PATCH 1/3] Working proof of concept for garlic experiments --- bsc/garlic/default.nix | 14 +++++++++-- bsc/garlic/experiments/default.nix | 39 ++++++++++++++++++++++++++++++ bsc/garlic/ppong/default.nix | 18 +++++++++++--- 3 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 bsc/garlic/experiments/default.nix diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix index c22f3b0..e2c6475 100644 --- a/bsc/garlic/default.nix +++ b/bsc/garlic/default.nix @@ -7,8 +7,18 @@ let callPackage = pkgs.lib.callPackageWith (pkgs // bsc // garlic); callPackages = pkgs.lib.callPackagesWith (pkgs // bsc // garlic); garlic = rec { - mpptest = callPackage ./mpptest/default.nix { }; - ppong = callPackage ./ppong/default.nix { }; + + mpptest = callPackage ./mpptest { }; + + ppong = callPackage ./ppong { }; + + experiments = callPackage ./experiments { + apps = [ + (ppong.override { mpi=bsc.intel-mpi;}) + (ppong.override { mpi=pkgs.mpich;}) + ]; + }; + }; in garlic diff --git a/bsc/garlic/experiments/default.nix b/bsc/garlic/experiments/default.nix new file mode 100644 index 0000000..64754e1 --- /dev/null +++ b/bsc/garlic/experiments/default.nix @@ -0,0 +1,39 @@ +{ + stdenv +, mpi +, fetchurl +, apps +}: + +stdenv.mkDerivation { + name = "garlic-experiments"; + + src = ./.; + + buildInputs = [] ++ apps; + apps = apps; + + buildPhase = '' + for app in $apps; do + test -e $app/bin/run || (echo $app/bin/run not found; exit 1) + done + ''; + + installPhase = '' + mkdir -p $out/apps + for app in $apps; do + ln -s $app $out/apps/$(basename $app) + done + + mkdir -p $out/bin + cat > $out/bin/run < Date: Fri, 24 Jul 2020 18:34:18 +0200 Subject: [PATCH 2/3] Add config generation --- bsc/garlic/experiments/config.nix | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 bsc/garlic/experiments/config.nix diff --git a/bsc/garlic/experiments/config.nix b/bsc/garlic/experiments/config.nix new file mode 100644 index 0000000..7681a8c --- /dev/null +++ b/bsc/garlic/experiments/config.nix @@ -0,0 +1,61 @@ +let + lib = import ; + + inputParams = { + # MPI implementation + mpi = [ + "impi" + "mpich" + ]; + + # Gcc compiler + gcc = [ + "gcc9" + "gcc7" + ]; + + # Additional cflags + cflags = [ + ["-O3" "-fnobugs"] + ["-Ofast"] + ]; + + # Which git branches +# branches = [ +# "mpi+seq" +# "seq" +# ]; + }; + + apps = [ + "dummy" + ]; + + # genAttrSets "a" ["hello" "world"] + # [ { a = "hello"; } { a = "world"; } ] + genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); + + # addAttrSets "a" [1 2] {e=4;} + # [ { a = 1; e = 4; } { a = 2; e = 4; } ] + addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); + + # attrToList {a=1;} + # [ { name = "a"; value = 1; } ] + attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); + + # mergeConfig [{e=1;}] {name="a"; value=[1 2] + # [ { a = 1; e = 1; } { a = 2; e = 1; } ] + mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr)); + + # genConfigs {a=[1 2]; b=[3 4];} + # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] + genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config)); + + + # Generates all configs from inputParams + allConfigs = (genConfigs inputParams); + +in + { + inherit allConfigs; + } From 979888eede1d53c1c31eed7380a8d5c1fe5737d1 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Mon, 27 Jul 2020 11:14:33 +0200 Subject: [PATCH 3/3] Add generators for experiments --- bsc/garlic/default.nix | 19 +++++++++++++------ bsc/garlic/gen.nix | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 bsc/garlic/gen.nix diff --git a/bsc/garlic/default.nix b/bsc/garlic/default.nix index e2c6475..aa5ff95 100644 --- a/bsc/garlic/default.nix +++ b/bsc/garlic/default.nix @@ -6,19 +6,26 @@ let callPackage = pkgs.lib.callPackageWith (pkgs // bsc // garlic); callPackages = pkgs.lib.callPackagesWith (pkgs // bsc // garlic); + + # Load some helper functions to generate app variants + inherit (import ./gen.nix) genApps genConfigs; + garlic = rec { mpptest = callPackage ./mpptest { }; ppong = callPackage ./ppong { }; - experiments = callPackage ./experiments { - apps = [ - (ppong.override { mpi=bsc.intel-mpi;}) - (ppong.override { mpi=pkgs.mpich;}) - ]; + exp = { + mpiImpl = callPackage ./experiments { + apps = genApps [ ppong ] ( + genConfigs { + mpi = [ bsc.intel-mpi pkgs.mpich pkgs.openmpi ]; + } + ); + }; }; - }; + in garlic diff --git a/bsc/garlic/gen.nix b/bsc/garlic/gen.nix new file mode 100644 index 0000000..adac866 --- /dev/null +++ b/bsc/garlic/gen.nix @@ -0,0 +1,34 @@ +let + lib = import ; + + gen = rec { + # genAttrSets "a" ["hello" "world"] + # [ { a = "hello"; } { a = "world"; } ] + genAttrSets = (name: arr: (map (x: {${name}=x; })) arr); + + # addAttrSets "a" [1 2] {e=4;} + # [ { a = 1; e = 4; } { a = 2; e = 4; } ] + addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr); + + # attrToList {a=1;} + # [ { name = "a"; value = 1; } ] + attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set)); + + # mergeConfig [{e=1;}] {name="a"; value=[1 2] + # [ { a = 1; e = 1; } { a = 2; e = 1; } ] + mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr)); + + # genConfigs {a=[1 2]; b=[3 4];} + # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ] + genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config)); + + # Generate multiple app versions by override with each config + genApp = (app: configs: map (conf: app.override conf) configs); + + # Generate app version from an array of apps + genApps = (apps: configs: + lib.flatten (map (app: genApp app configs) apps)); + + }; +in + gen