rarias/bscpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'hpcg' into 'master'

Browse Source 
Hpcg

See merge request rarias/bscpkgs!4
This commit is contained in:
Rodrigo Arias 2020-10-30 16:24:24 +01:00
commit dad70761ad
8 changed files with 614 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
garlic/apps/hpcg/default.nix
View File

@ -1,46 +1,44 @@
{
stdenv
, nanos6
, mpi
, mcxx
, tampi
, icc
, cc
, nanos6 ? null
, mcxx ? null
, mpi ? null
, gitBranch
}:
with stdenv.lib;
stdenv.mkDerivation rec {
name = "hpcg";
src = builtins.fetchGit {
url = "ssh://git@bscpm02.bsc.es/rpenacob/hpcg.git";
ref = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete";
url = "ssh://git@bscpm02.bsc.es/rpenacob/garlic-hpcg.git";
ref = "${gitBranch}";
};
prePatch = ''
#export NIX_DEBUG=6
'';
patches = [ ./tampi.patch ];
buildInputs = [
nanos6
mpi
icc
tampi
mcxx
cc
]
++ optional (mcxx != null) mcxx
++ optional (nanos6 != null) nanos6
++ optional (mpi != null) mpi;
makeFlags = [
"CC=${cc.cc.CC}"
"CXX=${cc.cc.CXX}"
];
enableParallelBuilding = true;
configurePhase = ''
export TAMPI_HOME=${tampi}
mkdir build
cd build
../configure MPI_ICPC_OSS
'';
installPhase = ''
mkdir -p $out/bin
cp bin/* $out/bin/
'';
programPath = "/bin/xhpcg";
}

210
garlic/exp/hpcg/extrae.xml Normal file
View File

@ -0,0 +1,210 @@
<?xml version='1.0'?>
<!-- Here comes the Extrae configuration.
As a general rule, "enabled" means that the feature is enabled :) If
it's not enabled, then the value can be set to some default.
-->
<!-- Must we activate the tracing? Which is the tracing mode? (detail/bursts) Where is it located? Which kind of trace? Version of the XML parser?-->
<trace enabled="yes"
initial-mode="detail"
type="paraver"
>
<!-- Configuration of some MPI dependant values -->
<mpi enabled="yes">
<!-- Gather counters in the MPI routines? -->
<counters enabled="yes" />
</mpi>
<!-- Emit information of the callstack -->
<callers enabled="yes">
<!-- At MPI calls, select depth level -->
<mpi enabled="yes">1-3</mpi>
<!-- At sampling points, select depth level -->
<sampling enabled="yes">1-5</sampling>
<!-- At dynamic memory system calls -->
<dynamic-memory enabled="no">1-3</dynamic-memory>
<!-- At I/O system calls -->
<input-output enabled="no">1-3</input-output>
<!-- At other system calls -->
<syscall enabled="no">1-3</syscall>
</callers>
<!-- Configuration of some OpenMP dependant values -->
<openmp enabled="no" ompt="no">
<!-- If the library instruments OpenMP, shall we gather info about locks?
Obtaining such information can make the final trace quite large.
-->
<locks enabled="no" />
<!-- Gather info about taskloops? -->
<taskloop enabled="no" />
<!-- Gather counters in the OpenMP routines? -->
<counters enabled="yes" />
</openmp>
<!-- Configuration of some pthread dependant values -->
<pthread enabled="no">
<!-- If the library instruments pthread, shall we gather info about locks,
mutexs and conds?
Obtaining such information can make the final trace quite large.
-->
<locks enabled="no" />
<!-- Gather counters in the pthread routines? -->
<counters enabled="yes" />
</pthread>
<!-- Configuration of User Functions -->
<user-functions enabled="no" list="/home/bsc41/bsc41273/user-functions.dat" exclude-automatic-functions="no">
<!-- Gather counters on the UF routines? -->
<counters enabled="yes" />
</user-functions>
<!-- Configure which software/hardware counters must be collected -->
<counters enabled="yes">
<!-- Configure the CPU hardware counters. You can define here as many sets
as you want. You can also define if MPI/OpenMP calls must report such
counters.
Starting-set property defines which set is chosen from every task.
Possible values are:
- cyclic : The sets are distributed in a cyclic fashion among all
tasks. So Task 0 takes set 1, Task 1 takes set 2,...
- block : The sets are distributed in block fashion among all tasks.
Task [0..i-1] takes set 1, Task [i..2*i-1] takes set 2, ...
- Number : All the tasks will start with the given set
(from 1..N).
-->
<cpu enabled="yes" starting-set-distribution="1">
<!-- In this example, we configure two sets of counters. The first will
be changed into the second after 5 calls to some collective
operation on MPI_COMM_WORLD. Once the second is activated, it will
turn to the first after 5seconds (aprox. depending on the MPI calls
granularity)
If you want that any set be counting forever, just don't set
changeat-globalops, or, changeat-time.
Each set has it's own properties.
domain -> in which domain must PAPI obtain the information (see
PAPI info)
changeat-globalops=num -> choose the next set after num
MPI_COMM_WORLD operations
changeat-time=numTime -> choose the next set after num Time
(for example 5s, 15m (for ms), 10M (for minutes),..)
-->
<set enabled="yes" domain="all">
PAPI_TOT_INS,PAPI_TOT_CYC
</set>
</cpu>
<!-- Do we want to gather information of the network counters?
Nowadays we can gather information about MX/GM cards.
-->
<network enabled="no" />
<!-- Obtain resource usage information -->
<resource-usage enabled="no" />
<!-- Obtain malloc statistics -->
<memory-usage enabled="no" />
</counters>
<!-- Define the characteristics of the tracing storage. If not defined,
or set, the tracing will send the traces to the current directory
with a default output name.
-->
<storage enabled="no">
<!-- The intermediate files will take the name of the application -->
<trace-prefix enabled="yes">TRACE</trace-prefix>
<!-- Stop the tracing when the intermediate file reaches this amount of MBs -->
<size enabled="no">5</size>
<!-- Where must we store the MPIT files while the app runs? -->
<temporal-directory enabled="yes">/scratch</temporal-directory>
<!-- Where must we store the MPIT files once the app ends? -->
<final-directory enabled="yes">/gpfs/scratch/bsc41/bsc41273</final-directory>
</storage>
<!-- Buffer configuration -->
<buffer enabled="yes">
<!-- How many events can we handle before any flush -->
<size enabled="yes">5000000</size>
<!-- Use the event buffer in a circular manner? You can use this option to
trace the last set of events. Needs MPI global routines operating on
MPI_COMM_WORLD communicator to be merged
-->
<circular enabled="no" />
</buffer>
<!-- Control tracing -->
<trace-control enabled="no">
<!-- We can start the application with a "latent tracing" and wake it up
once a control file is created. Use the property 'frequency' to
choose at which frequency this check must be done. If not supplied,
it will be checked every 100 global operations on MPI_COMM_WORLD.
-->
<file enabled="no" frequency="5M">/gpfs/scratch/bsc41/bsc41273/control</file>
<!--
-->
<global-ops enabled="no"></global-ops>
</trace-control>
<others enabled="yes">
<!-- Want to force a minimum amount of time of tracing? Here we force 10
minutes -->
<minimum-time enabled="no">10M</minimum-time>
<!-- Capture the following signals to finish cleanly -->
<finalize-on-signal enabled="yes"
SIGUSR1="no" SIGUSR2="no" SIGINT="yes"
SIGQUIT="yes" SIGTERM="yes" SIGXCPU="yes"
SIGFPE="yes" SIGSEGV="yes" SIGABRT="yes"
/>
<!-- Use instrumentation poitns to flush sampling buffer -->
<flush-sampling-buffer-at-instrumentation-point enabled="yes" />
</others>
<!-- Bursts library enabled? This requires an special library! -->
<bursts enabled="no">
<!-- Specify the threshold. This is mandatory! In this example, the
threshold is limitted to 500 microseconds
-->
<threshold enabled="yes">500u</threshold>
<!-- Report MPI statistics? -->
<mpi-statistics enabled="yes" />
</bursts>
<!-- Enable sampling capabilities using system clock.
Type may refer to: default, real, prof and virtual.
Period stands for the sampling period (50ms here)
plus a variability of 10ms, which means periods from
45 to 55ms.
-->
<sampling enabled="no" type="default" period="50m" variability="10m" />
<!-- Enable dynamic memory instrumentation (experimental) -->
<dynamic-memory enabled="no" />
<!-- Enable I/O (read, write) instrumentation (experimental) -->
<input-output enabled="no" internals="no"/>
<!-- Enable system calls instrumentation (experimental) -->
<syscall enabled="no" />
<!-- Do merge the intermediate tracefiles into the final tracefile?
Named according to the binary name
options:
synchronization = { default, task, node, no } (default is node)
max-memory = Number (in Mbytes) max memory used in merge step
joint-states = { yes, no } generate joint states?
keep-mpits = { yes, no } keep mpit files after merge?
-->
<merge enabled="yes"
synchronization="default"
tree-fan-out="16"
max-memory="512"
joint-states="yes"
keep-mpits="yes"
sort-addresses="yes"
overwrite="yes"
/>
</trace>

70
garlic/exp/hpcg/mpi+omp.nix Normal file
View File

@ -0,0 +1,70 @@
{
stdenv
, stdexp
, bsc
, targetMachine
, stages
}:
with stdenv.lib;
let
# Initial variable configuration
varConf = with bsc; {
n = [ { x = 128; y = 192; z = 192; } ];
nblocks = [ 6 12 24 48 96 192 ];
};
# Generate the complete configuration for each unit
genConf = with bsc; c: targetMachine.config // rec {
# hpcg options
n = c.n;
cc = bsc.icc;
mpi = bsc.impi;
gitBranch = "garlic/mpi+omp";
# Repeat the execution of each unit 30 times
loops = 30;
# Resources
qos = "debug";
ntasksPerNode = 1;
nodes = 4;
time = "02:00:00";
# Each task in different socket
cpuBind = "verbose,mask_cpu:0x3f";
jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
};
# Compute the array of configurations
configs = stdexp.buildConfigs {
inherit varConf genConf;
};
exec = {nextStage, conf, ...}: with conf; stages.exec {
inherit nextStage;
env = ''
OMP_PROC_BIND=true
OMP_NUM_THREADS=6
'';
argv = [
"--nx=${toString n.x}"
"--ny=${toString n.y}"
"--nz=${toString n.z}"
"--nblocks=${toString nblocks}"
];
};
program = {nextStage, conf, ...}: with conf;
let
customPkgs = stdexp.replaceMpi conf.mpi;
in
customPkgs.apps.hpcg.override {
inherit cc mpi gitBranch;
};
pipeline = stdexp.stdPipeline ++ [ exec program ];
in
stdexp.genExperiment { inherit configs pipeline; }

64
garlic/exp/hpcg/mpi.nix Normal file
View File

@ -0,0 +1,64 @@
{
stdenv
, stdexp
, bsc
, targetMachine
, stages
}:
with stdenv.lib;
let
# Initial variable configuration
varConf = with bsc; {
n = [ { x = 96; y = 96; z = 96; } ];
};
# Generate the complete configuration for each unit
genConf = with bsc; c: targetMachine.config // rec {
# hpcg options
n = c.n;
cc = bsc.icc;
mpi = bsc.impi;
gitBranch = "garlic/mpi";
# Repeat the execution of each unit 30 times
loops = 30;
# Resources
qos = "debug";
ntasksPerNode = 1;
nodes = 24;
time = "02:00:00";
# Each task in different socket
cpuBind = "verbose,mask_cpu:0x1";
jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
};
# Compute the array of configurations
configs = stdexp.buildConfigs {
inherit varConf genConf;
};
exec = {nextStage, conf, ...}: with conf; stages.exec {
inherit nextStage;
argv = [
"--nx=${toString n.x}"
"--ny=${toString n.y}"
"--nz=${toString n.z}"
];
};
program = {nextStage, conf, ...}: with conf;
let
customPkgs = stdexp.replaceMpi conf.mpi;
in
customPkgs.apps.hpcg.override {
inherit cc mpi gitBranch;
};
pipeline = stdexp.stdPipeline ++ [ exec program ];
in
stdexp.genExperiment { inherit configs pipeline; }

71
garlic/exp/hpcg/omp.nix Normal file
View File

@ -0,0 +1,71 @@
{
stdenv
, stdexp
, bsc
, targetMachine
, stages
}:
with stdenv.lib;
let
# Initial variable configuration
varConf = with bsc; {
n = [ { x = 256; y = 288; z = 288; } ];
nblocks = [ 12 24 48 96 192 384 ];
};
# Generate the complete configuration for each unit
genConf = with bsc; c: targetMachine.config // rec {
# hpcg options
n = c.n;
nblocks = c.nblocks;
cc = bsc.icc;
mpi = null; # TODO: Remove this for omp
gitBranch = "garlic/omp";
# Repeat the execution of each unit 30 times
loops = 30;
# Resources
qos = "debug";
ntasksPerNode = 1;
nodes = 1;
time = "02:00:00";
# task in one socket
cpuBind = "verbose,mask_cpu:0xffffff";
jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
};
# Compute the array of configurations
configs = stdexp.buildConfigs {
inherit varConf genConf;
};
exec = {nextStage, conf, ...}: with conf; stages.exec {
inherit nextStage;
env = ''
OMP_PROC_BIND=true
OMP_NUM_THREADS=24
'';
argv = [
"--nx=${toString n.x}"
"--ny=${toString n.y}"
"--nz=${toString n.z}"
"--nblocks=${toString nblocks}"
];
};
program = {nextStage, conf, ...}: with conf;
let
customPkgs = stdexp.replaceMpi conf.mpi;
in
customPkgs.apps.hpcg.override {
inherit cc gitBranch;
};
pipeline = stdexp.stdPipeline ++ [ exec program ];
in
stdexp.genExperiment { inherit configs pipeline; }

70
garlic/exp/hpcg/oss.nix Normal file
View File

@ -0,0 +1,70 @@
{
stdenv
, stdexp
, bsc
, targetMachine
, stages
}:
with stdenv.lib;
let
# Initial variable configuration
varConf = with bsc; {
n = [ { x = 256; y = 288; z = 288; } ];
nblocks = [ 12 24 48 96 192 384 ];
};
# Generate the complete configuration for each unit
genConf = with bsc; c: targetMachine.config // rec {
# hpcg options
n = c.n;
nblocks = c.nblocks;
cc = bsc.icc;
mcxx = bsc.mcxx;
nanos6 = bsc.nanos6;
mpi = null; # TODO: Remove this for oss
gitBranch = "garlic/oss";
# Repeat the execution of each unit 30 times
loops = 30;
# Resources
qos = "debug";
ntasksPerNode = 1;
nodes = 1;
time = "02:00:00";
# task in one socket
cpuBind = "verbose,mask_cpu:0xffffff";
jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
};
# Compute the array of configurations
configs = stdexp.buildConfigs {
inherit varConf genConf;
};
exec = {nextStage, conf, ...}: with conf; stages.exec {
inherit nextStage;
env = "NANOS6_DEPENDENCIES=discrete";
argv = [
"--nx=${toString n.x}"
"--ny=${toString n.y}"
"--nz=${toString n.z}"
"--nblocks=${toString nblocks}"
];
};
program = {nextStage, conf, ...}: with conf;
let
customPkgs = stdexp.replaceMpi conf.mpi;
in
customPkgs.apps.hpcg.override {
inherit cc nanos6 mcxx gitBranch;
};
pipeline = stdexp.stdPipeline ++ [ exec program ];
in
stdexp.genExperiment { inherit configs pipeline; }

63
garlic/exp/hpcg/serial.nix Normal file
View File

@ -0,0 +1,63 @@
{
stdenv
, stdexp
, bsc
, targetMachine
, stages
}:
with stdenv.lib;
let
# Initial variable configuration
varConf = with bsc; {
n = [ { x = 256; y = 288; z = 288; } ];
};
# Generate the complete configuration for each unit
genConf = with bsc; c: targetMachine.config // rec {
# hpcg options
n = c.n;
cc = icc;
mpi = null; # TODO: Remove this for serial
gitBranch = "garlic/seq";
# Repeat the execution of each unit 30 times
loops = 30;
# Resources
qos = "debug";
ntasksPerNode = 1;
nodes = 1;
time = "02:00:00";
cpuBind = "verbose,mask_cpu:0x1";
jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}";
};
# Compute the array of configurations
configs = stdexp.buildConfigs {
inherit varConf genConf;
};
exec = {nextStage, conf, ...}: with conf; stages.exec {
inherit nextStage;
argv = [
"--nx=${toString n.x}"
"--ny=${toString n.y}"
"--nz=${toString n.z}"
];
};
program = {nextStage, conf, ...}: with conf;
let
customPkgs = stdexp.replaceMpi conf.mpi;
in
customPkgs.apps.hpcg.override {
inherit cc gitBranch;
};
pipeline = stdexp.stdPipeline ++ [ exec program ];
in
stdexp.genExperiment { inherit configs pipeline; }

59
overlay.nix
View File

@ -172,19 +172,6 @@ let
# Apps for Garlic
apps = {
creams = callPackage ./garlic/apps/creams/default.nix {
gnuDef = self.gfortran10 ; # Default GNU compiler version
intelDef = self.bsc.icc ; # Default Intel compiler version
gitBranch = "garlic/mpi+send+seq";
cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
};
creamsInput = callPackage ./garlic/apps/creams/input.nix {
gitBranch = "garlic/mpi+send+seq";
};
nbody = callPackage ./garlic/apps/nbody/default.nix {
cc = self.bsc.icc;
@ -198,6 +185,40 @@ let
cc = self.bsc.clangOmpss2;
};
creams = callPackage ./garlic/apps/creams/default.nix {
gnuDef = self.gfortran10 ; # Default GNU compiler version
intelDef = self.bsc.icc ; # Default Intel compiler version
gitBranch = "garlic/mpi+send+seq";
cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
};
creamsInput = callPackage ./garlic/apps/creams/input.nix {
gitBranch = "garlic/mpi+send+seq";
};
hpcg = callPackage ./garlic/apps/hpcg/default.nix {
cc = self.bsc.icc;
mcxx = self.bsc.mcxx;
nanos6 = self.bsc.nanos6;
gitBranch = "garlic/oss";
# cc = self.bsc.icc;
# gitBranch = "garlic/seq";
# cc = self.bsc.icc;
# mpi = self.bsc.mpi;
# gitBranch = "garlic/mpi";
# cc = self.bsc.icc;
# gitBranch = "garlic/omp";
# cc = self.bsc.icc;
# mpi = self.bsc.mpi;
# gitBranch = "garlic/mpi+omp";
};
# heat = callPackage ./garlic/apps/heat {
# stdenv = pkgs.gcc7Stdenv;
# mpi = intel-mpi;
@ -208,8 +229,6 @@ let
# mpi = intel-mpi;
# };
#
# hpcg = callPackage ./garlic/apps/hpcg { };
#
# hpccg = callPackage ./garlic/apps/hpccg { };
#
# fwi = callPackage ./garlic/apps/fwi { };
@ -256,7 +275,13 @@ let
hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { };
};
};
};
hpcg = {
serial = callPackage ./garlic/exp/hpcg/serial.nix { };
mpi = callPackage ./garlic/exp/hpcg/mpi.nix { };
omp = callPackage ./garlic/exp/hpcg/omp.nix { };
mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { };
oss = callPackage ./garlic/exp/hpcg/oss.nix { };
};
test = {
@ -265,6 +290,8 @@ let
};
};
};
};
};
in
{