Merge branch 'hpcg' into 'master'

Hpcg

See merge request rarias/bscpkgs!4

garlic/apps/hpcg/default.nix

@@ -1,46 +1,44 @@
 {
   stdenv
-, nanos6
+, cc
-, mpi
+, nanos6 ? null
-, mcxx
+, mcxx ? null
-, tampi
+, mpi ? null
-, icc
+, gitBranch
 }:
 
+with stdenv.lib;
 stdenv.mkDerivation rec {
   name = "hpcg";
 
   src = builtins.fetchGit {
-    url = "ssh://git@bscpm02.bsc.es/rpenacob/hpcg.git";
+    url = "ssh://git@bscpm02.bsc.es/rpenacob/garlic-hpcg.git";
-    ref = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete";
+    ref = "${gitBranch}";
   };
 
   prePatch = ''
     #export NIX_DEBUG=6
   '';
 
-  patches = [ ./tampi.patch ];
-
   buildInputs = [
-    nanos6
+    cc
-    mpi
+  ]
-    icc
+  ++ optional (mcxx != null) mcxx
-    tampi
+  ++ optional (nanos6 != null) nanos6
-    mcxx
+  ++ optional (mpi != null) mpi;
+
+  makeFlags = [
+    "CC=${cc.cc.CC}"
+    "CXX=${cc.cc.CXX}"
   ];
 
   enableParallelBuilding = true;
 
-  configurePhase = ''
-    export TAMPI_HOME=${tampi}
-    mkdir build
-    cd build
-    ../configure MPI_ICPC_OSS
-  '';
-
   installPhase = ''
     mkdir -p $out/bin
     cp bin/* $out/bin/
   '';
 
+  programPath = "/bin/xhpcg";
+
 }

garlic/exp/hpcg/extrae.xml

@@ -0,0 +1,210 @@
+<?xml version='1.0'?>
+
+<!-- Here comes the Extrae configuration.
+     As a general rule, "enabled" means that the feature is enabled :)  If
+     it's not enabled, then the value can be set to some default.
+-->
+
+<!-- Must we activate the tracing? Which is the tracing mode? (detail/bursts) Where is it located? Which kind of trace? Version of the XML parser?-->
+<trace enabled="yes"
+ initial-mode="detail"
+ type="paraver"
+>
+
+  <!-- Configuration of some MPI dependant values -->
+  <mpi enabled="yes">
+    <!-- Gather counters in the MPI routines? -->
+    <counters enabled="yes" />
+  </mpi>
+
+  <!-- Emit information of the callstack -->
+  <callers enabled="yes">
+    <!-- At MPI calls, select depth level -->
+    <mpi enabled="yes">1-3</mpi>
+    <!-- At sampling points, select depth level -->
+		<sampling enabled="yes">1-5</sampling>
+		<!-- At dynamic memory system calls -->
+		<dynamic-memory enabled="no">1-3</dynamic-memory>
+		<!-- At I/O system calls -->
+		<input-output enabled="no">1-3</input-output>
+		<!-- At other system calls -->
+	  <syscall enabled="no">1-3</syscall>
+  </callers>
+
+  <!-- Configuration of some OpenMP dependant values -->
+  <openmp enabled="no" ompt="no">
+    <!-- If the library instruments OpenMP, shall we gather info about locks?
+         Obtaining such information can make the final trace quite large.
+    -->
+		<locks enabled="no" />
+		<!-- Gather info about taskloops? -->
+		<taskloop enabled="no" />
+    <!-- Gather counters in the OpenMP routines? -->
+    <counters enabled="yes" />
+  </openmp>
+
+  <!-- Configuration of some pthread dependant values -->
+  <pthread enabled="no">
+    <!-- If the library instruments pthread, shall we gather info about locks,
+	       mutexs and conds?
+         Obtaining such information can make the final trace quite large.
+    -->
+    <locks enabled="no" />
+    <!-- Gather counters in the pthread routines? -->
+    <counters enabled="yes" />
+  </pthread>
+
+  <!-- Configuration of User Functions -->
+  <user-functions enabled="no" list="/home/bsc41/bsc41273/user-functions.dat" exclude-automatic-functions="no">
+    <!-- Gather counters on the UF routines? -->
+    <counters enabled="yes" />
+  </user-functions>
+
+  <!-- Configure which software/hardware counters must be collected -->
+  <counters enabled="yes">
+    <!-- Configure the CPU hardware counters. You can define here as many sets
+         as you want. You can also define if MPI/OpenMP calls must report such
+         counters.
+         Starting-set property defines which set is chosen from every task.
+         Possible values are:
+           - cyclic : The sets are distributed in a cyclic fashion among all
+           tasks. So Task 0 takes set 1, Task 1 takes set 2,...
+           - block  : The sets are distributed in block fashion among all tasks.
+           Task [0..i-1] takes set 1, Task [i..2*i-1] takes set 2, ...
+           - Number : All the tasks will start with the given set 
+           (from 1..N).
+    -->
+    <cpu enabled="yes" starting-set-distribution="1">
+      <!-- In this example, we configure two sets of counters. The first will 
+           be changed into the second after 5 calls to some collective
+           operation on MPI_COMM_WORLD. Once the second is activated, it will
+           turn to the first after 5seconds (aprox. depending on the MPI calls
+           granularity)
+           If you want that any set be counting forever, just don't set
+           changeat-globalops, or, changeat-time.
+
+           Each set has it's own properties.
+           domain -> in which domain must PAPI obtain the information (see
+                       PAPI info)
+           changeat-globalops=num -> choose the next set after num
+                       MPI_COMM_WORLD operations
+           changeat-time=numTime -> choose the next set after num Time
+                       (for example 5s, 15m (for ms), 10M (for minutes),..)
+      -->
+      <set enabled="yes" domain="all">
+        PAPI_TOT_INS,PAPI_TOT_CYC
+      </set>
+    </cpu>
+
+    <!-- Do we want to gather information of the network counters?
+         Nowadays we can gather information about MX/GM cards.
+     -->
+    <network enabled="no" />
+
+    <!-- Obtain resource usage information -->
+    <resource-usage enabled="no" />
+
+    <!-- Obtain malloc statistics -->
+    <memory-usage enabled="no" />
+  </counters>
+
+  <!-- Define the characteristics of the tracing storage. If not defined,
+       or set, the tracing will send the traces to the current directory
+       with a default output name.
+  -->
+  <storage enabled="no">
+    <!-- The intermediate files will take the name of the application -->
+    <trace-prefix enabled="yes">TRACE</trace-prefix>
+    <!-- Stop the tracing when the intermediate file reaches this amount of MBs -->
+    <size enabled="no">5</size>
+    <!-- Where must we store the MPIT files while the app runs? -->
+    <temporal-directory enabled="yes">/scratch</temporal-directory>
+    <!-- Where must we store the MPIT files once the app ends? -->
+    <final-directory enabled="yes">/gpfs/scratch/bsc41/bsc41273</final-directory>
+  </storage>
+
+  <!-- Buffer configuration -->
+  <buffer enabled="yes">
+    <!-- How many events can we handle before any flush -->
+    <size enabled="yes">5000000</size>
+    <!-- Use the event buffer in a circular manner? You can use this option to
+         trace the last set of events. Needs MPI global routines operating on
+         MPI_COMM_WORLD communicator to be merged
+    -->
+    <circular enabled="no" />
+  </buffer>
+
+  <!-- Control tracing -->
+  <trace-control enabled="no">
+    <!-- We can start the application with a "latent tracing" and wake it up
+         once a control file is created. Use the property 'frequency' to
+         choose at which frequency this check must be done. If not supplied,
+         it will be checked every 100 global operations on MPI_COMM_WORLD.
+    -->
+    <file enabled="no" frequency="5M">/gpfs/scratch/bsc41/bsc41273/control</file>
+    <!-- 
+    -->
+    <global-ops enabled="no"></global-ops>
+  </trace-control>
+
+  <others enabled="yes">
+    <!-- Want to force a minimum amount of time of tracing? Here we force 10
+         minutes -->
+    <minimum-time enabled="no">10M</minimum-time>
+	<!-- Capture the following signals to finish cleanly -->
+    <finalize-on-signal enabled="yes" 
+      SIGUSR1="no" SIGUSR2="no" SIGINT="yes"
+      SIGQUIT="yes" SIGTERM="yes" SIGXCPU="yes"
+      SIGFPE="yes" SIGSEGV="yes" SIGABRT="yes"
+    />
+	<!-- Use instrumentation poitns to flush sampling buffer -->
+    <flush-sampling-buffer-at-instrumentation-point enabled="yes" />
+  </others>
+
+
+  <!-- Bursts library enabled? This requires an special library! -->
+  <bursts enabled="no">
+    <!-- Specify the threshold. This is mandatory! In this example, the
+         threshold is limitted to 500 microseconds
+     -->
+    <threshold enabled="yes">500u</threshold>
+    <!-- Report MPI statistics? -->
+    <mpi-statistics enabled="yes" />
+  </bursts>
+
+  <!-- Enable sampling capabilities using system clock.
+       Type may refer to: default, real, prof and virtual.
+       Period stands for the sampling period (50ms here)
+       plus a variability of 10ms, which means periods from 
+       45 to 55ms.
+  -->
+  <sampling enabled="no" type="default" period="50m" variability="10m" />
+
+  <!-- Enable dynamic memory instrumentation (experimental) -->
+  <dynamic-memory enabled="no" />
+
+  <!-- Enable I/O (read, write) instrumentation (experimental) -->
+  <input-output enabled="no" internals="no"/>
+
+	<!-- Enable system calls instrumentation (experimental) -->
+  <syscall enabled="no" />
+
+  <!-- Do merge the intermediate tracefiles into the final tracefile?
+       Named according to the binary name
+       options: 
+       synchronization = { default, task, node, no } (default is node)
+       max-memory = Number (in Mbytes) max memory used in merge step
+       joint-states = { yes, no } generate joint states?
+       keep-mpits = { yes, no } keep mpit files after merge?
+  -->
+  <merge enabled="yes" 
+    synchronization="default"
+    tree-fan-out="16"
+    max-memory="512"
+    joint-states="yes"
+    keep-mpits="yes"
+    sort-addresses="yes"
+    overwrite="yes"
+  />
+
+</trace>

garlic/exp/hpcg/mpi+omp.nix

@@ -0,0 +1,70 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ { x = 128; y = 192; z = 192; } ];
+    nblocks = [ 6 12 24 48 96 192 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = bsc.icc;
+    mpi = bsc.impi;
+    gitBranch = "garlic/mpi+omp";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 4;
+    time = "02:00:00";
+    # Each task in different socket
+    cpuBind = "verbose,mask_cpu:0x3f";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      OMP_PROC_BIND=true
+      OMP_NUM_THREADS=6
+    '';
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+      "--nblocks=${toString nblocks}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc mpi gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/hpcg/mpi.nix

@@ -0,0 +1,64 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ { x = 96; y = 96; z = 96; } ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = bsc.icc;
+    mpi = bsc.impi;
+    gitBranch = "garlic/mpi";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 24;
+    time = "02:00:00";
+    # Each task in different socket
+    cpuBind = "verbose,mask_cpu:0x1";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc mpi gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/hpcg/omp.nix

@@ -0,0 +1,71 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ { x = 256; y = 288; z = 288; } ];
+    nblocks = [ 12 24 48 96 192 384 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    nblocks = c.nblocks;
+    cc = bsc.icc;
+    mpi = null; # TODO: Remove this for omp
+    gitBranch = "garlic/omp";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 1;
+    time = "02:00:00";
+    # task in one socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = ''
+      OMP_PROC_BIND=true
+      OMP_NUM_THREADS=24
+    '';
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+      "--nblocks=${toString nblocks}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/hpcg/oss.nix

@@ -0,0 +1,70 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ { x = 256; y = 288; z = 288; } ];
+    nblocks = [ 12 24 48 96 192 384 ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    nblocks = c.nblocks;
+    cc = bsc.icc;
+    mcxx = bsc.mcxx;
+    nanos6 = bsc.nanos6;
+    mpi = null; # TODO: Remove this for oss
+    gitBranch = "garlic/oss";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 1;
+    time = "02:00:00";
+    # task in one socket
+    cpuBind = "verbose,mask_cpu:0xffffff";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    env = "NANOS6_DEPENDENCIES=discrete";
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+      "--nblocks=${toString nblocks}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc nanos6 mcxx gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

garlic/exp/hpcg/serial.nix

@@ -0,0 +1,63 @@
+{
+  stdenv
+, stdexp
+, bsc
+, targetMachine
+, stages
+}:
+
+with stdenv.lib;
+
+let
+  # Initial variable configuration
+  varConf = with bsc; {
+    n = [ { x = 256; y = 288; z = 288; } ];
+  };
+
+  # Generate the complete configuration for each unit
+  genConf = with bsc; c: targetMachine.config // rec {
+    # hpcg options
+    n = c.n;
+    cc = icc;
+    mpi = null; # TODO: Remove this for serial
+    gitBranch = "garlic/seq";
+
+    # Repeat the execution of each unit 30 times
+    loops = 30;
+
+    # Resources
+    qos = "debug";
+    ntasksPerNode = 1;
+    nodes = 1;
+    time = "02:00:00";
+    cpuBind = "verbose,mask_cpu:0x1";
+    jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
+  };
+
+  # Compute the array of configurations
+  configs = stdexp.buildConfigs {
+    inherit varConf genConf;
+  };
+
+  exec = {nextStage, conf, ...}: with conf; stages.exec {
+    inherit nextStage;
+    argv = [
+      "--nx=${toString n.x}"
+      "--ny=${toString n.y}"
+      "--nz=${toString n.z}"
+    ];
+  };
+
+  program = {nextStage, conf, ...}: with conf;
+  let
+    customPkgs = stdexp.replaceMpi conf.mpi;
+  in
+    customPkgs.apps.hpcg.override {
+      inherit cc gitBranch;
+    };
+
+  pipeline = stdexp.stdPipeline ++ [ exec program ];
+
+in
+
+  stdexp.genExperiment { inherit configs pipeline; }

overlay.nix

@@ -172,19 +172,6 @@ let
       # Apps for Garlic
 
       apps = {
-        creams = callPackage ./garlic/apps/creams/default.nix {
-          gnuDef   = self.gfortran10 ; # Default GNU   compiler version
-          intelDef = self.bsc.icc    ; # Default Intel compiler version
-
-          gitBranch = "garlic/mpi+send+seq";
-
-          cc  = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
-          mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
-        };
-
-        creamsInput = callPackage ./garlic/apps/creams/input.nix {
-          gitBranch = "garlic/mpi+send+seq";
-        };
 
         nbody = callPackage ./garlic/apps/nbody/default.nix {
           cc = self.bsc.icc;
@@ -198,6 +185,40 @@ let
           cc = self.bsc.clangOmpss2;
         };
 
+        creams = callPackage ./garlic/apps/creams/default.nix {
+          gnuDef   = self.gfortran10 ; # Default GNU   compiler version
+          intelDef = self.bsc.icc    ; # Default Intel compiler version
+          gitBranch = "garlic/mpi+send+seq";
+          cc  = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
+          mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
+        };
+
+        creamsInput = callPackage ./garlic/apps/creams/input.nix {
+          gitBranch = "garlic/mpi+send+seq";
+        };
+
+        hpcg = callPackage ./garlic/apps/hpcg/default.nix {
+          cc = self.bsc.icc;
+          mcxx = self.bsc.mcxx;
+          nanos6 = self.bsc.nanos6;
+          gitBranch = "garlic/oss";
+
+          # cc = self.bsc.icc;
+          # gitBranch = "garlic/seq";
+
+          # cc = self.bsc.icc;
+          # mpi = self.bsc.mpi;
+          # gitBranch = "garlic/mpi";
+
+          # cc = self.bsc.icc;
+          # gitBranch = "garlic/omp";
+
+          # cc = self.bsc.icc;
+          # mpi = self.bsc.mpi;
+          # gitBranch = "garlic/mpi+omp";
+
+        };
+
 #        heat = callPackage ./garlic/apps/heat {
 #          stdenv = pkgs.gcc7Stdenv;
 #          mpi = intel-mpi;
@@ -208,8 +229,6 @@ let
 #          mpi = intel-mpi;
 #        };
 #  
-#        hpcg = callPackage ./garlic/apps/hpcg { };
-#  
 #        hpccg = callPackage ./garlic/apps/hpccg { };
 #  
 #        fwi = callPackage ./garlic/apps/fwi { };
@@ -256,12 +275,20 @@ let
             hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { };
           };
         };
-      };
-    };
 
-    test = {
+        hpcg = {
-      exec = callPackage ./test/garlic/exec.nix {
+          serial = callPackage ./garlic/exp/hpcg/serial.nix { };
-        exec = self.bsc.garlic.stages.exec;
+          mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
+          omp = callPackage ./garlic/exp/hpcg/omp.nix { };
+          mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
+          oss = callPackage ./garlic/exp/hpcg/oss.nix { };
+        };
+
+        test = {
+          exec = callPackage ./test/garlic/exec.nix {
+            exec = self.bsc.garlic.stages.exec;
+          };
+        };
       };
     };
   };