Simplify paths

garlic/stages/argv.nix

@@ -0,0 +1,30 @@
+{
+  stdenv
+, bash
+}:
+
+{
+  program
+, env ? ""
+
+# bash array as string, example: argv=''(-f "file with spaces" -t 10)''
+, argv ? "()"
+}:
+
+stdenv.mkDerivation {
+  name = "argv";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!${bash}/bin/bash
+    # Requires /nix to use bash
+    
+    ${env}
+
+    argv=${argv}
+    exec ${program} \''${argv[@]}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/control.nix

@@ -0,0 +1,25 @@
+{
+  stdenv
+}:
+
+{
+  program
+, loops ? 30
+}:
+
+stdenv.mkDerivation {
+  name = "control";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+    #set -e
+    for n in {1..${toString loops}}; do
+      ${program}
+    done
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/default.nix

@@ -0,0 +1,64 @@
+{
+  pkgs
+, callPackage
+, callPackages
+}:
+
+let
+
+  garlic = {
+
+    # Load some helper functions to generate app variants
+    inherit (import ./gen.nix) genApps genApp genConfigs;
+
+    mpptest = callPackage ./mpptest { };
+
+    ppong = callPackage ./ppong {
+      mpi = pkgs.mpi;
+    };
+
+    nbody = callPackage ./nbody {
+      cc = pkgs.icc;
+      mpi = pkgs.impi;
+      tampi = pkgs.tampi;
+      gitBranch = "garlic/seq";
+    };
+
+    runWrappers = {
+      sbatch  = callPackage ./sbatch.nix { };
+      srun    = callPackage ./srun.nix { };
+      launch  = callPackage ./launcher.nix { };
+      control = callPackage ./control.nix { };
+      nixsetup= callPackage ./nix-setup.nix { };
+      argv    = callPackage ./argv.nix { };
+      statspy = callPackage ./statspy.nix { };
+      extrae  = callPackage ./extrae.nix { };
+      stagen  = callPackage ./stagen.nix { };
+    };
+
+    # Perf is tied to a linux kernel specific version
+    linuxPackages = pkgs.linuxPackages_4_4;
+    perfWrapper = callPackage ./perf.nix {
+      perf = pkgs.linuxPackages.perf;
+    };
+
+    exp = {
+      noise = callPackage ./exp/noise.nix { };
+      nbody = {
+        bs = callPackage ./exp/nbody/bs.nix {
+          pkgs = pkgs // garlic;
+        };
+        mpi = callPackage ./exp/nbody/mpi.nix { };
+      };
+      osu = rec {
+        latency-internode = callPackage ./exp/osu/latency.nix { };
+        latency-intranode = callPackage ./exp/osu/latency.nix {
+          interNode = false;
+        };
+        latency = latency-internode;
+      };
+    };
+  };
+
+in
+  garlic

garlic/stages/extrae.nix

@@ -0,0 +1,37 @@
+{
+  stdenv
+, bash
+, extrae
+#, writeShellScriptBin
+}:
+
+{
+  program
+, configFile
+, traceLib
+}:
+
+#writeShellScriptBin "extraeWrapper" ''
+#  export EXTRAE_HOME=${extrae}
+#  export LD_PRELOAD=${extrae}/lib/lib${traceLib}trace.so:$LD_PRELOAD
+#  export EXTRAE_CONFIG_FILE=${configFile}
+#  exec ${program}
+#''
+
+stdenv.mkDerivation {
+  name = "extrae";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!${bash}/bin/bash
+    # Requires /nix to use bash
+    
+    export EXTRAE_HOME=${extrae}
+    export LD_PRELOAD=${extrae}/lib/lib${traceLib}trace.so:$LD_PRELOAD
+    export EXTRAE_CONFIG_FILE=${configFile}
+    exec ${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/gen.nix

@@ -0,0 +1,34 @@
+let
+  lib = import <nixpkgs/lib>;
+
+  gen = rec {
+    # genAttrSets "a" ["hello" "world"]
+    # [ { a = "hello"; } { a = "world"; } ]
+    genAttrSets = (name: arr: (map (x: {${name}=x; })) arr);
+
+    # addAttrSets "a" [1 2] {e=4;}
+    # [ { a = 1; e = 4; } { a = 2; e = 4; } ]
+    addAttrSets = (name: arr: set: (map (x: set // {${name}=x; })) arr);
+
+    # attrToList {a=1;}
+    # [ { name = "a"; value = 1; } ]
+    attrToList = (set: map (name: {name=name; value=set.${name};} ) (builtins.attrNames set));
+
+    # mergeConfig [{e=1;}] {name="a"; value=[1 2]
+    # [ { a = 1; e = 1; } { a = 2; e = 1; } ]
+    mergeConfig = (arr: new: lib.flatten ( map (x: addAttrSets new.name new.value x) arr));
+
+    # genConfigs {a=[1 2]; b=[3 4];}
+    # [ { a = 1; b = 3; } { a = 1; b = 4; } { a = 2; b = 3; } { a = 2; b = 4; } ]
+    genConfigs = (config: lib.foldl mergeConfig [{}] (attrToList config));
+
+    # Generate multiple app versions by override with each config
+    genApp = (app: configs: map (conf: app.override conf // {conf=conf;}) configs);
+
+    # Generate app version from an array of apps
+    genApps = (apps: configs:
+      lib.flatten (map (app: genApp app configs) apps));
+
+  };
+in
+  gen

garlic/stages/launcher.nix

@@ -0,0 +1,42 @@
+{
+  stdenv
+}:
+
+apps: # Each app must be unique
+
+stdenv.mkDerivation {
+  name = "launcher";
+  preferLocalBuild = true;
+
+  buildInputs = [] ++ apps;
+  apps = apps;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+
+  installPhase = ''
+    mkdir -p $out/apps
+    for j in $apps; do
+      target=$out/apps/$(basename $j)
+      if [ -e $target ]; then
+        echo "Duplicated app: $j"
+        echo
+        echo "Provided apps: "
+        printf "%s\n" $apps
+        echo
+        exit 1
+      fi
+      ln -s $j $target
+    done
+
+    mkdir -p $out/bin
+    cat > $out/bin/run <<EOF
+    #!/bin/sh
+
+    for j in $out/apps/*; do
+      \$j/bin/run
+    done
+    EOF
+
+    chmod +x $out/bin/run
+  '';
+}

garlic/stages/nix-setup.nix

@@ -0,0 +1,28 @@
+{
+  stdenv
+}:
+
+{
+  program
+}:
+
+stdenv.mkDerivation {
+  name = "nixsetup";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+
+    # We need to enter the nix namespace first, in order to have /nix
+    # available, so we use this hack:
+    if [ ! -e /nix ]; then
+      exec nix-setup \$0
+    fi
+
+    exec ${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/perf.nix

@@ -0,0 +1,24 @@
+{
+  stdenv
+, bash
+, perf
+}:
+
+{
+  program
+, perfArgs ? "record -a"
+}:
+
+stdenv.mkDerivation {
+  name = "perfWrapper";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  installPhase = ''
+    cat > $out <<EOF
+    #!${bash}/bin/bash
+
+    exec ${perf}/bin/perf ${perfArgs} ${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/sbatch.nix

@@ -0,0 +1,84 @@
+{
+  stdenv
+, numactl
+}:
+
+{
+  program
+, jobName
+, chdirPrefix ? "."
+, nixPrefix ? ""
+, binary ? "/bin/run"
+, ntasks ? null
+, ntasksPerNode ? null
+, nodes ? null
+, exclusive ? true # By default we run in exclusive mode
+, qos ? null
+, reservation ? null
+, time ? null
+, output ? "job_%j.out"
+, error ? "job_%j.err"
+, contiguous ? null
+, extra ? null
+, acctgFreq ? null
+}:
+
+with stdenv.lib;
+let
+
+  sbatchOpt = name: value: optionalString (value!=null)
+    "#SBATCH --${name}=${value}\n";
+  sbatchEnable = name: value: optionalString (value!=null)
+    "#SBATCH --${name}\n";
+
+in
+
+stdenv.mkDerivation rec {
+  name = "sbatch";
+  preferLocalBuild = true;
+
+  phases = [ "installPhase" ];
+
+  #SBATCH --tasks-per-node=48
+  #SBATCH --ntasks-per-socket=24
+  #SBATCH --cpus-per-task=1
+  dontBuild = true;
+  dontPatchShebangs = true;
+  programPath = "/${name}";
+
+  installPhase = ''
+    mkdir -p $out
+    cat > $out/job <<EOF
+    #!/bin/sh
+    #SBATCH --job-name="${jobName}"
+    ''
+    + sbatchOpt "ntasks" ntasks
+    + sbatchOpt "ntasks-per-node" ntasksPerNode
+    + sbatchOpt "nodes" nodes
+    + sbatchOpt "chdir" "${chdirPrefix}/$(basename $out)"
+    + sbatchOpt "output" output
+    + sbatchOpt "error" error
+    + sbatchEnable "exclusive" exclusive
+    + sbatchOpt "time" time
+    + sbatchOpt "qos" qos
+    + sbatchOpt "reservation" reservation
+    + sbatchOpt "acctg-freq" acctgFreq
+    + optionalString (extra!=null) extra
+    +
+    ''
+    exec ${nixPrefix}${program}
+    EOF
+    
+    cat > $out/${name} <<EOF
+    #!/bin/sh
+    if [ -e "${chdirPrefix}/$(basename $out)" ]; then
+      >&2 echo "Execution aborted: '${chdirPrefix}/$(basename $out)' already exists"
+      exit 1
+    fi
+    mkdir -p "${chdirPrefix}/$(basename $out)"
+    echo sbatch ${nixPrefix}$out/job
+    sbatch ${nixPrefix}$out/job
+    EOF
+    chmod +x $out/${name}
+  '';
+}

garlic/stages/srun.nix

@@ -0,0 +1,22 @@
+{
+  stdenv
+}:
+{
+  program
+, nixPrefix ? ""
+, srunOptions ? ""
+}:
+
+stdenv.mkDerivation rec {
+  name = "srun";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  dontPatchShebangs = true;
+  installPhase = ''
+    cat > $out <<EOF
+    #!/bin/sh
+    exec srun --mpi=pmi2 ${srunOptions} ${nixPrefix}${program}
+    EOF
+    chmod +x $out
+  '';
+}

garlic/stages/stagen.nix

@@ -0,0 +1,55 @@
+{
+  stdenv
+, bash
+, extrae
+, writeShellScriptBin
+, jq
+}:
+
+{
+  stages
+, conf
+, experimentName ? "run"
+}:
+
+with stdenv.lib;
+
+let
+
+  dStages = foldr (stageFn: {conf, prevStage, stages}: {
+    conf = conf;
+    prevStage = stageFn {stage=prevStage; conf=conf;};
+    stages = [ (stageFn {stage=prevStage; conf=conf;}) ] ++ stages;
+  })
+    {conf=conf; stages=[]; prevStage=null;} stages;
+
+  stageProgram = stage:
+    if stage ? programPath
+    then "${stage}${stage.programPath}" else "${stage}";
+
+  linkStages = imap1 (i: s: {
+    name = "${toString i}-${baseNameOf s.name}";
+    path = stageProgram s;
+  }) dStages.stages;
+
+  createLinks = builtins.concatStringsSep "\n"
+    (map (x: "ln -s ${x.path} $out/bin/${x.name}") linkStages);
+
+  firstStageLink = (x: x.name) (elemAt linkStages 0);
+in
+stdenv.mkDerivation {
+  name = "stagen";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  buildInputs = [ jq ];
+  installPhase = ''
+    mkdir -p $out/bin
+    ${createLinks}
+    ln -s ${firstStageLink} $out/bin/${experimentName}
+    cat > $out/config.raw << EOF
+    ${builtins.toJSON conf}
+    EOF
+    jq . $out/config.raw > $out/config.json
+    rm $out/config.raw
+  '';
+}

garlic/stages/statspy.nix

@@ -0,0 +1,29 @@
+{
+  stdenv
+, bash
+}:
+
+{
+  program
+, outputDir ? "."
+}:
+
+stdenv.mkDerivation {
+  name = "statspy";
+  preferLocalBuild = true;
+  phases = [ "installPhase" ];
+  programPath = "/bin/${name}";
+  installPhase = ''
+    mkdir -p $out/bin
+    cat > $out/bin/${name} <<EOF
+    #!${bash}/bin/bash
+
+    mkdir -p ${outputDir}
+    cat /proc/[0-9]*/stat | sort -n > ${outputDir}/statspy.\$(date +%s.%3N).begin
+    ${program}
+    cat /proc/[0-9]*/stat | sort -n > ${outputDir}/statspy.\$(date +%s.%3N).end
+
+    EOF
+    chmod +x $out/bin/${name}
+  '';
+}