diff --git a/garlic/apps/hpcg/default.nix b/garlic/apps/hpcg/default.nix index f311804..3d62904 100644 --- a/garlic/apps/hpcg/default.nix +++ b/garlic/apps/hpcg/default.nix @@ -1,46 +1,44 @@ { stdenv -, nanos6 -, mpi -, mcxx -, tampi -, icc +, cc +, nanos6 ? null +, mcxx ? null +, mpi ? null +, gitBranch }: +with stdenv.lib; stdenv.mkDerivation rec { name = "hpcg"; src = builtins.fetchGit { - url = "ssh://git@bscpm02.bsc.es/rpenacob/hpcg.git"; - ref = "symgs_coloring_more_than_one_block_per_task_halos_blocking_discreete"; + url = "ssh://git@bscpm02.bsc.es/rpenacob/garlic-hpcg.git"; + ref = "${gitBranch}"; }; prePatch = '' #export NIX_DEBUG=6 ''; - patches = [ ./tampi.patch ]; - buildInputs = [ - nanos6 - mpi - icc - tampi - mcxx + cc + ] + ++ optional (mcxx != null) mcxx + ++ optional (nanos6 != null) nanos6 + ++ optional (mpi != null) mpi; + + makeFlags = [ + "CC=${cc.cc.CC}" + "CXX=${cc.cc.CXX}" ]; enableParallelBuilding = true; - configurePhase = '' - export TAMPI_HOME=${tampi} - mkdir build - cd build - ../configure MPI_ICPC_OSS - ''; - installPhase = '' mkdir -p $out/bin cp bin/* $out/bin/ ''; + programPath = "/bin/xhpcg"; + } diff --git a/garlic/exp/hpcg/extrae.xml b/garlic/exp/hpcg/extrae.xml new file mode 100644 index 0000000..45a7574 --- /dev/null +++ b/garlic/exp/hpcg/extrae.xml @@ -0,0 +1,210 @@ + + + + + + + + + + + + + + + + + 1-3 + + 1-5 + + 1-3 + + 1-3 + + 1-3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PAPI_TOT_INS,PAPI_TOT_CYC + + + + + + + + + + + + + + + + + TRACE + + 5 + + /scratch + + /gpfs/scratch/bsc41/bsc41273 + + + + + + 5000000 + + + + + + + + /gpfs/scratch/bsc41/bsc41273/control + + + + + + + 10M + + + + + + + + + + + 500u + + + + + + + + + + + + + + + + + + + + diff --git a/garlic/exp/hpcg/mpi+omp.nix b/garlic/exp/hpcg/mpi+omp.nix new file mode 100644 index 0000000..dfe4696 --- /dev/null +++ b/garlic/exp/hpcg/mpi+omp.nix @@ -0,0 +1,70 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = with bsc; { + n = [ { x = 128; y = 192; z = 192; } ]; + nblocks = [ 6 12 24 48 96 192 ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + cc = bsc.icc; + mpi = bsc.impi; + gitBranch = "garlic/mpi+omp"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + qos = "debug"; + ntasksPerNode = 1; + nodes = 4; + time = "02:00:00"; + # Each task in different socket + cpuBind = "verbose,mask_cpu:0x3f"; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + OMP_PROC_BIND=true + OMP_NUM_THREADS=6 + ''; + argv = [ + "--nx=${toString n.x}" + "--ny=${toString n.y}" + "--nz=${toString n.z}" + "--nblocks=${toString nblocks}" + ]; + }; + + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { + inherit cc mpi gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/mpi.nix b/garlic/exp/hpcg/mpi.nix new file mode 100644 index 0000000..a2a2ec4 --- /dev/null +++ b/garlic/exp/hpcg/mpi.nix @@ -0,0 +1,64 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = with bsc; { + n = [ { x = 96; y = 96; z = 96; } ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + cc = bsc.icc; + mpi = bsc.impi; + gitBranch = "garlic/mpi"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + qos = "debug"; + ntasksPerNode = 1; + nodes = 24; + time = "02:00:00"; + # Each task in different socket + cpuBind = "verbose,mask_cpu:0x1"; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + argv = [ + "--nx=${toString n.x}" + "--ny=${toString n.y}" + "--nz=${toString n.z}" + ]; + }; + + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { + inherit cc mpi gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/omp.nix b/garlic/exp/hpcg/omp.nix new file mode 100644 index 0000000..9960302 --- /dev/null +++ b/garlic/exp/hpcg/omp.nix @@ -0,0 +1,71 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = with bsc; { + n = [ { x = 256; y = 288; z = 288; } ]; + nblocks = [ 12 24 48 96 192 384 ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + nblocks = c.nblocks; + cc = bsc.icc; + mpi = null; # TODO: Remove this for omp + gitBranch = "garlic/omp"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + qos = "debug"; + ntasksPerNode = 1; + nodes = 1; + time = "02:00:00"; + # task in one socket + cpuBind = "verbose,mask_cpu:0xffffff"; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = '' + OMP_PROC_BIND=true + OMP_NUM_THREADS=24 + ''; + argv = [ + "--nx=${toString n.x}" + "--ny=${toString n.y}" + "--nz=${toString n.z}" + "--nblocks=${toString nblocks}" + ]; + }; + + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { + inherit cc gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/oss.nix b/garlic/exp/hpcg/oss.nix new file mode 100644 index 0000000..5084504 --- /dev/null +++ b/garlic/exp/hpcg/oss.nix @@ -0,0 +1,70 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = with bsc; { + n = [ { x = 256; y = 288; z = 288; } ]; + nblocks = [ 12 24 48 96 192 384 ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + nblocks = c.nblocks; + cc = bsc.icc; + mcxx = bsc.mcxx; + nanos6 = bsc.nanos6; + mpi = null; # TODO: Remove this for oss + gitBranch = "garlic/oss"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + qos = "debug"; + ntasksPerNode = 1; + nodes = 1; + time = "02:00:00"; + # task in one socket + cpuBind = "verbose,mask_cpu:0xffffff"; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + env = "NANOS6_DEPENDENCIES=discrete"; + argv = [ + "--nx=${toString n.x}" + "--ny=${toString n.y}" + "--nz=${toString n.z}" + "--nblocks=${toString nblocks}" + ]; + }; + + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { + inherit cc nanos6 mcxx gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/hpcg/serial.nix b/garlic/exp/hpcg/serial.nix new file mode 100644 index 0000000..3c58b42 --- /dev/null +++ b/garlic/exp/hpcg/serial.nix @@ -0,0 +1,63 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + # Initial variable configuration + varConf = with bsc; { + n = [ { x = 256; y = 288; z = 288; } ]; + }; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + # hpcg options + n = c.n; + cc = icc; + mpi = null; # TODO: Remove this for serial + gitBranch = "garlic/seq"; + + # Repeat the execution of each unit 30 times + loops = 30; + + # Resources + qos = "debug"; + ntasksPerNode = 1; + nodes = 1; + time = "02:00:00"; + cpuBind = "verbose,mask_cpu:0x1"; + jobName = "hpcg-${toString n.x}-${toString n.y}-${toString n.z}-${gitBranch}"; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: with conf; stages.exec { + inherit nextStage; + argv = [ + "--nx=${toString n.x}" + "--ny=${toString n.y}" + "--nz=${toString n.z}" + ]; + }; + + program = {nextStage, conf, ...}: with conf; + let + customPkgs = stdexp.replaceMpi conf.mpi; + in + customPkgs.apps.hpcg.override { + inherit cc gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/overlay.nix b/overlay.nix index 69b9a3e..c695def 100644 --- a/overlay.nix +++ b/overlay.nix @@ -172,19 +172,6 @@ let # Apps for Garlic apps = { - creams = callPackage ./garlic/apps/creams/default.nix { - gnuDef = self.gfortran10 ; # Default GNU compiler version - intelDef = self.bsc.icc ; # Default Intel compiler version - - gitBranch = "garlic/mpi+send+seq"; - - cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10; - mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4; - }; - - creamsInput = callPackage ./garlic/apps/creams/input.nix { - gitBranch = "garlic/mpi+send+seq"; - }; nbody = callPackage ./garlic/apps/nbody/default.nix { cc = self.bsc.icc; @@ -198,6 +185,40 @@ let cc = self.bsc.clangOmpss2; }; + creams = callPackage ./garlic/apps/creams/default.nix { + gnuDef = self.gfortran10 ; # Default GNU compiler version + intelDef = self.bsc.icc ; # Default Intel compiler version + gitBranch = "garlic/mpi+send+seq"; + cc = self.bsc.icc; # self.bsc.icc OR self.gfortran10; + mpi = self.bsc.mpi; # self.bsc.mpi OR self.bsc.openmpi-mn4; + }; + + creamsInput = callPackage ./garlic/apps/creams/input.nix { + gitBranch = "garlic/mpi+send+seq"; + }; + + hpcg = callPackage ./garlic/apps/hpcg/default.nix { + cc = self.bsc.icc; + mcxx = self.bsc.mcxx; + nanos6 = self.bsc.nanos6; + gitBranch = "garlic/oss"; + + # cc = self.bsc.icc; + # gitBranch = "garlic/seq"; + + # cc = self.bsc.icc; + # mpi = self.bsc.mpi; + # gitBranch = "garlic/mpi"; + + # cc = self.bsc.icc; + # gitBranch = "garlic/omp"; + + # cc = self.bsc.icc; + # mpi = self.bsc.mpi; + # gitBranch = "garlic/mpi+omp"; + + }; + # heat = callPackage ./garlic/apps/heat { # stdenv = pkgs.gcc7Stdenv; # mpi = intel-mpi; @@ -208,8 +229,6 @@ let # mpi = intel-mpi; # }; # -# hpcg = callPackage ./garlic/apps/hpcg { }; -# # hpccg = callPackage ./garlic/apps/hpccg { }; # # fwi = callPackage ./garlic/apps/fwi { }; @@ -256,12 +275,20 @@ let hybrid = callPackage ./garlic/exp/creams/ss+hybrid.nix { }; }; }; - }; - }; - test = { - exec = callPackage ./test/garlic/exec.nix { - exec = self.bsc.garlic.stages.exec; + hpcg = { + serial = callPackage ./garlic/exp/hpcg/serial.nix { }; + mpi = callPackage ./garlic/exp/hpcg/mpi.nix { }; + omp = callPackage ./garlic/exp/hpcg/omp.nix { }; + mpi_omp = callPackage ./garlic/exp/hpcg/mpi+omp.nix { }; + oss = callPackage ./garlic/exp/hpcg/oss.nix { }; + }; + + test = { + exec = callPackage ./test/garlic/exec.nix { + exec = self.bsc.garlic.stages.exec; + }; + }; }; }; };