Improvement the experiment based on CREAMS
This commit is contained in:
Pedro Martinez
parent
5cbc8e4fbb
commit
6ae71cc5e9
@ -6,6 +6,8 @@
|
||||
, impi
|
||||
, tampi
|
||||
, mcxx
|
||||
, gnuDef
|
||||
, intelDef
|
||||
, cc
|
||||
, gitBranch
|
||||
}:
|
||||
@ -18,12 +20,16 @@ let
|
||||
else
|
||||
"IntelMPI");
|
||||
|
||||
compName = (if cc == intelDef then
|
||||
"Intel"
|
||||
else
|
||||
"GNU");
|
||||
|
||||
in
|
||||
stdenv.mkDerivation rec {
|
||||
name = "creams";
|
||||
|
||||
# src = /home/Computational/pmartin1/creams-simplified;
|
||||
|
||||
src = builtins.fetchGit {
|
||||
url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git";
|
||||
ref = "${gitBranch}";
|
||||
@ -45,7 +51,8 @@ stdenv.mkDerivation rec {
|
||||
export TAMPI_HOME=${tampi}
|
||||
|
||||
. etc/bashrc
|
||||
#export FORTRAN_COMPILER=GNU # GCC compiler
|
||||
|
||||
export FORTRAN_COMPILER=${compName}
|
||||
export MPI_LIB=${mpiName}
|
||||
|
||||
echo
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
{
|
||||
stdenv
|
||||
, nodes
|
||||
, granul ? 0
|
||||
, nprocz ? 0
|
||||
, gitBranch
|
||||
}:
|
||||
|
||||
@ -8,17 +9,20 @@ stdenv.mkDerivation rec {
|
||||
name = "creams-input";
|
||||
|
||||
# src = /home/Computational/pmartin1/creams-simplified;
|
||||
|
||||
src = builtins.fetchGit {
|
||||
url = "ssh://git@bscpm02.bsc.es/pmartin1/creams-simplified.git";
|
||||
ref = "${gitBranch}";
|
||||
};
|
||||
|
||||
phases = [ "unpackPhase" "installPhase" ];
|
||||
phases = [ "unpackPhase" "patchPhase" "installPhase" ];
|
||||
|
||||
patchPhase = ''
|
||||
patchShebangs SodTubeBenchmark/gridScript.sh
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
pushd SodTubeBenchmark
|
||||
bash gridScript.sh 0 0 $((${toString nodes}*48)) 0
|
||||
./gridScript.sh 0 0 ${toString nprocz} ${toString granul}
|
||||
popd
|
||||
|
||||
mkdir -p $out
|
||||
|
||||
@ -14,21 +14,23 @@ let
|
||||
|
||||
# Set variable configuration for the experiment
|
||||
varConfig = {
|
||||
# cc = [ self.gcc10 ]; # Does not work
|
||||
# mpi = [ bsc.openmpi ]; # Does not work
|
||||
# cc = [ bsc.icc ];
|
||||
# mpi = [ bsc.impi ];
|
||||
cc = [ bsc.icc ]; # [ bsc.icc pkgs.gfortran10 ];
|
||||
|
||||
nodes = [ 1 2 4 8 16 ];
|
||||
mpi = [ bsc.impi ]; # [ bsc.impi bsc.openmpi-mn4 ];
|
||||
|
||||
input = [
|
||||
{ nodes=1 ; nprocz=48 ; granul=0; }
|
||||
{ nodes=2 ; nprocz=96 ; granul=0; }
|
||||
{ nodes=4 ; nprocz=192; granul=0; }
|
||||
{ nodes=8 ; nprocz=384; granul=0; }
|
||||
{ nodes=16; nprocz=768; granul=0; }
|
||||
];
|
||||
};
|
||||
|
||||
# Common configuration
|
||||
common = {
|
||||
gitBranch = "garlic/mpi+send+seq";
|
||||
|
||||
cc = bsc.icc ;
|
||||
mpi = bsc.impi;
|
||||
|
||||
# Resources
|
||||
ntasksPerNode = 48;
|
||||
#ntasksPerSocket = 24; // Add this variable to nix
|
||||
@ -56,10 +58,12 @@ let
|
||||
sbatch = {stage, conf, ...}: with conf; w.sbatch {
|
||||
program = stageProgram stage;
|
||||
exclusive = true;
|
||||
time = "02:00:00";
|
||||
qos = "debug";
|
||||
jobName = "nbody-bs";
|
||||
inherit nixPrefix nodes ntasksPerNode;
|
||||
time = "10:00:00";
|
||||
####qos = "debug";
|
||||
jobName = "creams-ss-mpi+send+seq";
|
||||
inherit nixPrefix ntasksPerNode;
|
||||
|
||||
nodes = input.nodes;
|
||||
};
|
||||
|
||||
control = {stage, conf, ...}: with conf; w.control {
|
||||
@ -109,19 +113,19 @@ let
|
||||
];
|
||||
};
|
||||
|
||||
inputDataset = {stage, conf, ...}: with conf;
|
||||
inputDataset = {stage, conf, ...}:
|
||||
let
|
||||
input = bsc.garlic.creamsInput.override {
|
||||
inherit gitBranch nodes;
|
||||
gitBranch = conf.gitBranch;
|
||||
granul = conf.input.granul;
|
||||
nprocz = conf.input.nprocz;
|
||||
};
|
||||
in w.argv
|
||||
{
|
||||
program = stageProgram stage;
|
||||
env = ''
|
||||
cp -r ${input}/SodTubeBenchmark/* .
|
||||
|
||||
pwd
|
||||
ls -l
|
||||
chmod +w -R .
|
||||
'';
|
||||
};
|
||||
|
||||
@ -162,7 +166,7 @@ let
|
||||
# Optionally profile nanos6 with the new ctf
|
||||
++ optional enableCtf ctf
|
||||
|
||||
# Execute the nbody app with the argv and env vars
|
||||
# Execute the app with the argv and env vars
|
||||
++ [ inputDataset creamsFn ];
|
||||
|
||||
# List of actual programs to be executed
|
||||
|
||||
@ -1,173 +0,0 @@
|
||||
{
|
||||
stdenv
|
||||
, nixpkgs
|
||||
, pkgs
|
||||
, genApp
|
||||
, genConfigs
|
||||
, runWrappers
|
||||
}:
|
||||
|
||||
with stdenv.lib;
|
||||
|
||||
let
|
||||
bsc = pkgs.bsc;
|
||||
|
||||
# Set variable configuration for the experiment
|
||||
varConfig = {
|
||||
# cc = [ self.gcc10 ]; # Does not work
|
||||
# mpi = [ bsc.openmpi ]; # Does not work
|
||||
# cc = [ bsc.icc ];
|
||||
# mpi = [ bsc.impi ];
|
||||
|
||||
nodes = [ 1 2 4 8 16 ];
|
||||
};
|
||||
|
||||
# Common configuration
|
||||
common = {
|
||||
gitBranch = "garlic/mpi+send+seq";
|
||||
|
||||
cc = bsc.icc ;
|
||||
mpi = bsc.impi;
|
||||
|
||||
# Resources
|
||||
ntasksPerNode = 48;
|
||||
#ntasksPerSocket = 24; // Add this variable to nix
|
||||
|
||||
# Stage configuration
|
||||
enableSbatch = true;
|
||||
enableControl = true;
|
||||
enableExtrae = false;
|
||||
enablePerf = false;
|
||||
enableCtf = false;
|
||||
|
||||
# MN4 path
|
||||
nixPrefix = "/gpfs/projects/bsc15/nix";
|
||||
};
|
||||
|
||||
# Compute the cartesian product of all configurations
|
||||
configs = map (conf: conf // common) (genConfigs varConfig);
|
||||
|
||||
stageProgram = stage:
|
||||
if stage ? programPath
|
||||
then "${stage}${stage.programPath}" else "${stage}";
|
||||
|
||||
w = runWrappers;
|
||||
|
||||
sbatch = {stage, conf, ...}: with conf; w.sbatch {
|
||||
program = stageProgram stage;
|
||||
exclusive = true;
|
||||
time = "02:00:00";
|
||||
qos = "debug";
|
||||
jobName = "nbody-bs";
|
||||
inherit nixPrefix nodes ntasksPerNode;
|
||||
};
|
||||
|
||||
control = {stage, conf, ...}: with conf; w.control {
|
||||
program = stageProgram stage;
|
||||
};
|
||||
|
||||
srun = {stage, conf, ...}: with conf; w.srun {
|
||||
program = stageProgram stage;
|
||||
srunOptions = "--cpu-bind=verbose,rank";
|
||||
inherit nixPrefix;
|
||||
};
|
||||
|
||||
statspy = {stage, conf, ...}: with conf; w.statspy {
|
||||
program = stageProgram stage;
|
||||
};
|
||||
|
||||
perf = {stage, conf, ...}: with conf; w.perf {
|
||||
program = stageProgram stage;
|
||||
perfArgs = "sched record -a";
|
||||
};
|
||||
|
||||
nixsetup = {stage, conf, ...}: with conf; w.nixsetup {
|
||||
program = stageProgram stage;
|
||||
nixsetup = "${nixPrefix}/bin/nix-setup";
|
||||
};
|
||||
|
||||
extrae = {stage, conf, ...}: w.extrae {
|
||||
program = stageProgram stage;
|
||||
traceLib = "mpi"; # mpi -> libtracempi.so
|
||||
configFile = ./extrae.xml;
|
||||
};
|
||||
|
||||
ctf = {stage, conf, ...}: w.argv {
|
||||
program = stageProgram stage;
|
||||
env = ''
|
||||
export NANOS6=ctf
|
||||
export NANOS6_CTF2PRV=0
|
||||
'';
|
||||
};
|
||||
|
||||
bscOverlay = import ../../../overlay.nix;
|
||||
|
||||
genPkgs = newOverlay: nixpkgs {
|
||||
overlays = [
|
||||
bscOverlay
|
||||
newOverlay
|
||||
];
|
||||
};
|
||||
|
||||
inputDataset = {stage, conf, ...}: with conf;
|
||||
let
|
||||
input = bsc.garlic.creamsInput.override {
|
||||
inherit gitBranch nodes;
|
||||
};
|
||||
in w.argv
|
||||
{
|
||||
program = stageProgram stage;
|
||||
env = ''
|
||||
cp -r ${input}/SodTubeBenchmark/* .
|
||||
|
||||
pwd
|
||||
ls -l
|
||||
'';
|
||||
};
|
||||
|
||||
# We may be able to use overlays by invoking the fix function directly, but we
|
||||
# have to get the definition of the bsc packages and the garlic ones as
|
||||
# overlays.
|
||||
|
||||
creamsFn = {stage, conf, ...}: with conf;
|
||||
let
|
||||
# We set the mpi implementation to the one specified in the conf, so all
|
||||
# packages in bsc will use that one.
|
||||
customPkgs = genPkgs (self: super: {
|
||||
bsc = super.bsc // { mpi = conf.mpi; };
|
||||
});
|
||||
in
|
||||
customPkgs.bsc.garlic.creams.override {
|
||||
inherit cc mpi gitBranch;
|
||||
};
|
||||
|
||||
stages = with common; []
|
||||
# Use sbatch to request resources first
|
||||
++ optional enableSbatch sbatch
|
||||
|
||||
# Repeats the next stages N times
|
||||
++ optionals enableControl [ nixsetup control ]
|
||||
|
||||
# Executes srun to launch the program in the requested nodes, and
|
||||
# immediately after enters the nix environment again, as slurmstepd launches
|
||||
# the next stages from outside the namespace.
|
||||
++ [ srun nixsetup ]
|
||||
|
||||
# Intrumentation with extrae
|
||||
++ optional enableExtrae extrae
|
||||
|
||||
# Optionally profile the next stages with perf
|
||||
++ optional enablePerf perf
|
||||
|
||||
# Optionally profile nanos6 with the new ctf
|
||||
++ optional enableCtf ctf
|
||||
|
||||
# Execute the nbody app with the argv and env vars
|
||||
++ [ inputDataset creamsFn ];
|
||||
|
||||
# List of actual programs to be executed
|
||||
jobs = map (conf: w.stagen { inherit conf stages; }) configs;
|
||||
|
||||
in
|
||||
# We simply run each program one after another
|
||||
w.launch jobs
|
||||
11
overlay.nix
11
overlay.nix
@ -144,18 +144,17 @@ let
|
||||
# };
|
||||
#
|
||||
creams = callPackage ./garlic/creams {
|
||||
gnuDef = self.gfortran10 ; # Default GNU compiler version
|
||||
intelDef = self.bsc.icc ; # Default Intel compiler version
|
||||
|
||||
gitBranch = "garlic/mpi+send+seq";
|
||||
|
||||
#cc = self.gcc10; # Does not work
|
||||
#mpi = self.bsc.openmpi-mn4; # Does not work
|
||||
|
||||
cc = self.bsc.icc;
|
||||
mpi = self.bsc.mpi;
|
||||
cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10;
|
||||
mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4;
|
||||
};
|
||||
|
||||
creamsInput = callPackage ./garlic/creams/input.nix {
|
||||
gitBranch = "garlic/mpi+send+seq";
|
||||
nodes = 1;
|
||||
};
|
||||
|
||||
# lulesh = callPackage ./garlic/lulesh {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user