bscpkgs/garlic/exp/hpcg/slices.nix

{
  stdenv
, stdexp
, bsc
, targetMachine
, stages
, genInput
}:

with stdenv.lib;

let
  # Initial variable configuration
  varConf = {
    n = [ { x = 192; y = 192; z = 192; } ];
    nprocs = [
        { x = 2; y = 1; z = 1; }
        { x = 4; y = 1; z = 1; }
        { x = 8; y = 1; z = 1; }
        { x = 16; y = 1; z = 1; }
        { x = 32; y = 1; z = 1; }

        { x = 1; y = 2; z = 1; }
        { x = 1; y = 4; z = 1; }
        { x = 1; y = 8; z = 1; }
        { x = 1; y = 16; z = 1; }
        { x = 1; y = 32; z = 1; }

        { x = 1; y = 1; z = 2; }
        { x = 1; y = 1; z = 4; }
        { x = 1; y = 1; z = 8; }
        { x = 1; y = 1; z = 16; }
        { x = 1; y = 1; z = 32; }

    ];
    # nblocks = [ 12 24 48 96 192 384 768 1536 ];
    nblocks = [ 384 768 1536 ];
    ncommblocks = [ 1 ];
    # nodes = [ 1 ];
    # nodes = [ 1 2 4 8 16 ];
  };

  # Generate the complete configuration for each unit
  genConf = c: targetMachine.config // rec {
    expName = "hpcg.oss";
    unitName = "${expName}.nb${toString nblocks}";

    inherit (targetMachine.config) hw;

    # hpcg options
    inherit (c) n nprocs nblocks ncommblocks;

    gitBranch = "garlic/tampi+isend+oss+task";

    # Repeat the execution of each unit 30 times
    loops = 10;

    disableAspectRatio = true;

    # Resources
    qos = "debug";
    ntasksPerNode = hw.socketsPerNode;
    time = "02:00:00";
    # task in one socket
    cpusPerTask = hw.cpusPerSocket;
    nodes = (nprocs.x * nprocs.y * nprocs.z) / ntasksPerNode;
    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
  };

  # Compute the array of configurations
  configs = stdexp.buildConfigs {
    inherit varConf genConf;
  };

  input = genInput configs;

  exec = {nextStage, conf, ...}: stages.exec {
    inherit nextStage;
    argv = [
      "--nx=${toString conf.n.x}"
      "--ny=${toString conf.n.y}"
      "--nz=${toString conf.n.z}"
      # Distribute all processes in X axis
      "--npx=${toString conf.nprocs.x}"
      "--npy=${toString conf.nprocs.y}"
      "--npz=${toString conf.nprocs.z}"
      "--nblocks=${toString conf.nblocks}"
      "--ncomms=${toString conf.ncommblocks}"
      # The input symlink is generated by the input stage, which is generated by
      # the genInput function.
      "--load=input"
      # Disable HPCG Aspect Ratio to run any mpi layout
    ] ++ optional (conf.disableAspectRatio) "--no-ar=1";
  };

  program = {nextStage, conf, ...}: bsc.apps.hpcg.override {
    inherit (conf) gitBranch;
  };

  pipeline = stdexp.stdPipeline ++ [ input exec program ];

in

  stdexp.genExperiment { inherit configs pipeline; }