bscpkgs/garlic/exp/cn6/timediff.nix

{
  stdenv
, lib
, stdexp
, bsc
, pkgs
, targetMachine
, stages
, garlicTools
, writeText
, enableHWC ? false
}:

with lib;
with garlicTools;

let
  # Initial variable configuration
  varConf = {
    nodes = [ 1 2 4 8 ];
  };

  machineConfig = targetMachine.config;

  genConf = c: targetMachine.config // rec {
    expName = "timediff";
    unitName = expName + "-nodes${toString nodes}";

    inherit (machineConfig) hw;

    loops = 1;

    # Resources
    cpusPerTask = hw.cpusPerSocket;
    ntasksPerNode = hw.socketsPerNode;
    nodes = c.nodes;

    qos = "debug";
    time = "02:00:00";

    jobName = unitName;
  };

  configs = stdexp.buildConfigs {
    inherit varConf genConf;
  };

  # Custom BSC packages
  bsc' = bsc.extend (self: super: {

    # For nanos6 we use my fork for distributed instrumentation at the
    # latest commit
    nanos6 = (super.nanos6Git.override {
      enableJemalloc = true;
    }).overrideAttrs (old: rec {

      src = builtins.fetchGit {
        url = "git@bscpm03.bsc.es:rarias/nanos6.git";
        ref = "rodrigo";
        rev = "5cbeabb4e0446c2c293cc3005f76e6139465caee";
      };
      
      dontStrip = false;
      version = src.shortRev;

      # Disable all unused instrumentations for faster builds
      configureFlags = old.configureFlags ++ [
        "--disable-extrae-instrumentation"
        "--disable-lint-instrumentation"
        "--disable-graph-instrumentation"
        "--disable-stats-instrumentation"
      ];
    });

    # Use clang from master
    clangOmpss2Unwrapped = super.clangOmpss2Unwrapped.overrideAttrs (old: rec {
      version = src.shortRev;
      src = builtins.fetchGit {
        url = "ssh://git@bscpm03.bsc.es/llvm-ompss/llvm-mono.git";
        ref = "master";
        rev = "ce47d99d2b2b968c87187cc7818cc5040b082d6c";
      };
    });

    # Use mcxx from master
    mcxx = super.mcxxGit;

    # We also need the instrumented version of TAMPI
    tampi = super.tampiGit.overrideAttrs (old: rec {
      version = src.shortRev;
      dontStrip = true;
      NIX_CFLAGS = "-O0 -g";
      src = builtins.fetchGit {
        url = "ssh://git@bscpm03.bsc.es/rarias/tampi.git";
        ref = "instrument";
        rev = "6e4294299bf761a1cc31f4181d9479cefa1c7f3e";
      };
    });

    # We use the latest commit in master as src for cn6
    cn6Git = ((super.cn6.overrideAttrs (old: rec {
      version = src.shortRev;
      src = builtins.fetchGit {
        url = "ssh://git@bscpm03.bsc.es/rarias/cn6.git";
        ref = "master";
        rev = "1d23d01d60164b8641746d5a204128a9d31b9650";
      };
    })).override { enableTest = true; });

    cn6 = self.cn6Git;
  });


  ctf = {nextStage, conf, ...}: let
    # Create the nanos6 configuration file
    nanos6ConfigFile = writeText "nanos6.toml" ''
      version.instrument = "ctf"
      turbo.enabled = false
      instrument.ctf.converter.enabled = false
    '' + optionalString (enableHWC) ''
      hardware_counters.papi.enabled = true
      hardware_counters.papi.counters = [
        "PAPI_TOT_INS", "PAPI_TOT_CYC",
        "PAPI_L1_TCM", "PAPI_L2_TCM", "PAPI_L3_TCM"
      ]
    '';

  in stages.exec {
    inherit nextStage;

    # And use it
    env = ''
      export NANOS6_CONFIG=${nanos6ConfigFile}
    '';

    post = ''
      rank=$SLURM_PROCID
      tracedir=trace_timediff_mpi

      # Convert CTF trace to PRV
      ${bsc'.cn6}/bin/cn6 $tracedir/$rank

      # Merge on rank 0 only
      if [ $rank != 0 ]; then
        exit 0;
      fi

      # Wait a bit for all ranks to finish the conversion
      sleep 5

      # Run the merger
      ${bsc'.cn6}/bin/merge-prv $tracedir

      # We need some tools the path
      export PATH="$PATH:${bsc'.babeltrace2}/bin:${pkgs.ministat}/bin"

      ${bsc'.cn6}/bin/sync-err.sh $tracedir
    '';
  };

  exec = {nextStage, conf, ...}: stages.exec {
    inherit nextStage;
    program = "${bsc'.cn6}/bin/timediff_mpi";
    argv = [ conf.cpusPerTask ];
  };

  pipeline = stdexp.stdPipeline ++ [ ctf exec ];

in
 
  stdexp.genExperiment { inherit configs pipeline; }