From ecc01e43142e55f5b25869b7b862a677b02380cc Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Mon, 17 Aug 2020 18:55:01 +0200 Subject: [PATCH] Add old SLURM, pmix and pmi2 versions --- bsc/openmpi/default.nix | 17 ++++-- bsc/openmpi/with-slurm.nix | 117 +++++++++++++++++++++++++++++++++++++ bsc/pmix/pmix2.nix | 48 +++++++++++++++ bsc/slurm/default.nix | 6 ++ bsc/slurm/pmi2.nix | 82 ++++++++++++++++++++++++++ default.nix | 23 ++++++-- 6 files changed, 281 insertions(+), 12 deletions(-) create mode 100644 bsc/openmpi/with-slurm.nix create mode 100644 bsc/pmix/pmix2.nix create mode 100644 bsc/slurm/pmi2.nix diff --git a/bsc/openmpi/default.nix b/bsc/openmpi/default.nix index 1cea9bb..5787d8e 100644 --- a/bsc/openmpi/default.nix +++ b/bsc/openmpi/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl, fetchpatch, gfortran, perl, libnl , rdma-core, zlib, numactl, libevent, hwloc, targetPackages, symlinkJoin -, libpsm2, libfabric +, libpsm2, libfabric, pmix, pmi2, ucx # Enable CUDA support , cudaSupport ? false, cudatoolkit ? null @@ -22,7 +22,7 @@ assert !cudaSupport || cudatoolkit != null; let - version = "4.0.3"; + version = "4.0.4"; cudatoolkit_joined = symlinkJoin { name = "${cudatoolkit.name}-unsplit"; @@ -34,7 +34,7 @@ in stdenv.mkDerivation rec { src = with stdenv.lib.versions; fetchurl { url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2"; - sha256 = "00zxcw99gr5n693cmcmn4f6a47vx1ywna895p0x7p163v37gw0hl"; + sha256 = "1i0slg2dxjdgw513aml1n9dsbdxn2fimi2b5712d5r9z4ar4xqj7"; }; postPatch = '' @@ -49,7 +49,7 @@ in stdenv.mkDerivation rec { ''; buildInputs = with stdenv; [ gfortran zlib ] - ++ lib.optionals isLinux [ libnl numactl ] + ++ lib.optionals isLinux [ libnl numactl pmix ucx ] ++ lib.optionals cudaSupport [ cudatoolkit ] ++ [ libevent hwloc ] ++ lib.optional (isLinux || isFreeBSD) rdma-core @@ -58,8 +58,13 @@ in stdenv.mkDerivation rec { nativeBuildInputs = [ perl ]; configureFlags = with stdenv; lib.optional (!cudaSupport) "--disable-mca-dso" - ++ lib.optional isLinux "--with-libnl=${libnl.dev}" - ++ lib.optional enableSGE "--with-sge" + ++ lib.optionals isLinux [ + "--with-libnl=${libnl.dev}" + "--with-pmix=${pmix}" + "--with-pmix-libdir=${pmix}/lib" + "--with-pmi=${pmi2}" + "--with-pmi-libdir=${pmi2}/lib" + ] ++ lib.optional enableSGE "--with-sge" ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default" # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build # https://github.com/openucx/ucx diff --git a/bsc/openmpi/with-slurm.nix b/bsc/openmpi/with-slurm.nix new file mode 100644 index 0000000..f987c00 --- /dev/null +++ b/bsc/openmpi/with-slurm.nix @@ -0,0 +1,117 @@ +{ stdenv, fetchurl, fetchpatch, gfortran, perl, libnl +, rdma-core, zlib, numactl, libevent, hwloc, targetPackages, symlinkJoin +, libpsm2, libfabric + +# Enable CUDA support +, cudaSupport ? false, cudatoolkit ? null + +# Enable the Sun Grid Engine bindings +, enableSGE ? false + +# Pass PATH/LD_LIBRARY_PATH to point to current mpirun by default +, enablePrefix ? false + +# Enable libfabric support (necessary for Omnipath networks) on x86_64 linux +, fabricSupport ? stdenv.isLinux && stdenv.isx86_64 + +# Enable mpi_cxx.so +, enableCxx ? false + +, slurm + +}: + +assert !cudaSupport || cudatoolkit != null; + +let + version = "4.0.3"; + + cudatoolkit_joined = symlinkJoin { + name = "${cudatoolkit.name}-unsplit"; + paths = [ cudatoolkit.out cudatoolkit.lib ]; + }; +in stdenv.mkDerivation rec { + pname = "openmpi"; + inherit version; + + src = with stdenv.lib.versions; fetchurl { + url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2"; + sha256 = "00zxcw99gr5n693cmcmn4f6a47vx1ywna895p0x7p163v37gw0hl"; + }; + + postPatch = '' + patchShebangs ./ + + # Ensure build is reproducible + ts=`date -d @$SOURCE_DATE_EPOCH` + sed -i 's/OPAL_CONFIGURE_USER=.*/OPAL_CONFIGURE_USER="nixbld"/' configure + sed -i 's/OPAL_CONFIGURE_HOST=.*/OPAL_CONFIGURE_HOST="localhost"/' configure + sed -i "s/OPAL_CONFIGURE_DATE=.*/OPAL_CONFIGURE_DATE=\"$ts\"/" configure + find -name "Makefile.in" -exec sed -i "s/\`date\`/$ts/" \{} \; + ''; + + buildInputs = with stdenv; [ gfortran zlib ] + ++ lib.optionals isLinux [ libnl numactl ] + ++ lib.optionals cudaSupport [ cudatoolkit ] + ++ [ libevent hwloc ] + ++ lib.optional (isLinux || isFreeBSD) rdma-core + ++ lib.optional fabricSupport [ libpsm2 libfabric ] + ++ [ slurm ]; + + nativeBuildInputs = [ perl ]; + + configureFlags = with stdenv; lib.optional (!cudaSupport) "--disable-mca-dso" + ++ lib.optional isLinux "--with-libnl=${libnl.dev}" + ++ lib.optional enableSGE "--with-sge" + ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default" + # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build + # https://github.com/openucx/ucx + # https://www.open-mpi.org/faq/?category=buildcuda + ++ lib.optionals cudaSupport [ "--with-cuda=${cudatoolkit_joined}" "--enable-dlopen" ] + ++ lib.optionals fabricSupport [ "--with-psm2=${libpsm2}" "--with-libfabric=${libfabric}" ] + ++ lib.optional enableCxx "--enable-mpi-cxx" + ++ [ "--with-slurm=${slurm}" "--with-pmi" "--enable-static" "--disable-dlopen" ] + ; + + enableParallelBuilding = true; + + postInstall = '' + rm -f $out/lib/*.la + ''; + + postFixup = '' + # default compilers should be indentical to the + # compilers at build time + + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $out/share/openmpi/mpicc-wrapper-data.txt + + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \ + $out/share/openmpi/ortecc-wrapper-data.txt + + sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \ + $out/share/openmpi/mpic++-wrapper-data.txt + + sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:' \ + $out/share/openmpi/mpifort-wrapper-data.txt + ''; + + doCheck = true; + + passthru = { + inherit cudaSupport cudatoolkit; + }; + + meta = with stdenv.lib; { + homepage = "https://www.open-mpi.org/"; + description = "Open source MPI-3 implementation"; + longDescription = "The Open MPI Project is an open source MPI-3 implementation that is developed and maintained by a consortium of academic, research, and industry partners. Open MPI is therefore able to combine the expertise, technologies, and resources from all across the High Performance Computing community in order to build the best MPI library available. Open MPI offers advantages for system and software vendors, application developers and computer science researchers."; + maintainers = with maintainers; [ markuskowa ]; + license = licenses.bsd3; + platforms = platforms.unix; + # In order to use OpenMPI with Extrae we need to keep the Extrae libraries + # rather than the ones from OpenMPI. This happens with libompitrace.so, so + # we set to a lower priority (higher value) the OpenMPI package. + priority = 10; + }; +} diff --git a/bsc/pmix/pmix2.nix b/bsc/pmix/pmix2.nix new file mode 100644 index 0000000..6598fce --- /dev/null +++ b/bsc/pmix/pmix2.nix @@ -0,0 +1,48 @@ +{ stdenv, fetchFromGitHub, perl, autoconf, automake +, libtool, flex, libevent, hwloc, munge, zlib +} : + +let + version = "2.2.4"; + +in stdenv.mkDerivation { + pname = "pmix"; + inherit version; + + src = fetchFromGitHub { + repo = "openpmix"; + owner = "openpmix"; + rev = "v${version}"; + sha256 = "1wc4sbnbg20lp6l6pk1sawrf5wrdajcijd1cmrpp1d6h9nv23ggv"; + }; + + postPatch = '' + patchShebangs ./autogen.pl + patchShebangs ./config + ''; + + nativeBuildInputs = [ perl autoconf automake libtool flex ]; + + buildInputs = [ libevent hwloc munge zlib ]; + + configureFlags = [ + "--with-libevent=${libevent.dev}" + "--with-munge=${munge}" + "--with-hwloc=${hwloc.dev}" + ]; + + preConfigure = '' + ./autogen.pl + ''; + + enableParallelBuilding = true; + + meta = with stdenv.lib; { + description = "Process Management Interface for HPC environments"; + homepage = "https://openpmix.github.io/"; + license = licenses.bsd3; + maintainers = [ maintainers.markuskowa ]; + platforms = platforms.linux; + }; +} + diff --git a/bsc/slurm/default.nix b/bsc/slurm/default.nix index d75fd78..a82a52f 100644 --- a/bsc/slurm/default.nix +++ b/bsc/slurm/default.nix @@ -58,6 +58,12 @@ stdenv.mkDerivation rec { patchShebangs ./configure ''; +# postBuild = '' +# pushd contrib/pmi2 +# make -j install +# popd +# ''; + postInstall = '' rm -f $out/lib/*.la $out/lib/slurm/*.la ''; diff --git a/bsc/slurm/pmi2.nix b/bsc/slurm/pmi2.nix new file mode 100644 index 0000000..36f337f --- /dev/null +++ b/bsc/slurm/pmi2.nix @@ -0,0 +1,82 @@ +{ stdenv, fetchFromGitHub, pkgconfig, libtool, curl +, python, munge, perl, pam, openssl +, ncurses, libmysqlclient, gtk2, lua, hwloc, numactl +, readline, freeipmi, libssh2, xorg +, pmix +# enable internal X11 support via libssh2 +, enableX11 ? true +}: + +stdenv.mkDerivation rec { + name = "slurm-libpmi2-${version}"; + version = "17.11.9-2"; + + # N.B. We use github release tags instead of https://www.schedmd.com/downloads.php + # because the latter does not keep older releases. + src = fetchFromGitHub { + owner = "SchedMD"; + repo = "slurm"; + # The release tags use - instead of . + rev = "${builtins.replaceStrings ["."] ["-"] name}"; + sha256 = "1lq4ac6yjai6wh979dciw8v3d99zbd3w36rfh0vpncqm672fg1qy"; + }; + + outputs = [ "out" ]; + + prePatch = stdenv.lib.optional enableX11 '' + substituteInPlace src/common/x11_util.c \ + --replace '"/usr/bin/xauth"' '"${xorg.xauth}/bin/xauth"' + ''; + + # nixos test fails to start slurmd with 'undefined symbol: slurm_job_preempt_mode' + # https://groups.google.com/forum/#!topic/slurm-devel/QHOajQ84_Es + # this doesn't fix tests completely at least makes slurmd to launch + hardeningDisable = [ "bindnow" ]; + + nativeBuildInputs = [ pkgconfig libtool ]; + buildInputs = [ + curl python munge perl pam openssl + libmysqlclient ncurses gtk2 + lua hwloc numactl readline freeipmi + pmix + ] ++ stdenv.lib.optionals enableX11 [ libssh2 xorg.xauth ]; + + configureFlags = with stdenv.lib; + [ "--with-munge=${munge}" + "--with-ssl=${openssl.dev}" + "--with-hwloc=${hwloc.dev}" + "--with-freeipmi=${freeipmi}" + "--sysconfdir=/etc/slurm" + "--with-pmix=${pmix}" + ] ++ (optional (gtk2 == null) "--disable-gtktest") + ++ (optional enableX11 "--with-libssh2=${libssh2.dev}"); + + + preConfigure = '' + patchShebangs ./doc/html/shtml2html.py + patchShebangs ./doc/man/man2html.py + patchShebangs ./configure + ''; + + preBuild = ''cd contribs/pmi2''; + + #buildPhase = '' + # pushd contrib/pmi2 + # make -j install SHELL=${SHELL} + # popd + #''; + + postInstall = '' + rm -f $out/lib/*.la $out/lib/slurm/*.la + ''; + + enableParallelBuilding = true; + + meta = with stdenv.lib; { + homepage = http://www.schedmd.com/; + description = "Simple Linux Utility for Resource Management"; + platforms = platforms.linux; + license = licenses.gpl2; + maintainers = with maintainers; [ jagajaga markuskowa ]; + }; +} diff --git a/default.nix b/default.nix index 0fed8c4..e28070d 100644 --- a/default.nix +++ b/default.nix @@ -33,11 +33,6 @@ let # BSC Packages # --------------------------------------------------------- # - # Custom OpenMPI with mpi_cxx enabled for TAMPI - openmpi = callPackage ./bsc/openmpi/default.nix { - enableCxx = true; - }; - # ParaStation MPI pscom = callPackage ./bsc/parastation/pscom.nix { }; psmpi = callPackage ./bsc/parastation/psmpi.nix { }; @@ -66,7 +61,23 @@ let intel-license = callPackage bsc/intel-compiler/license.nix { }; - slurm17 = callPackage ./bsc/slurm/default.nix { }; + pmix2 = callPackage ./bsc/pmix/pmix2.nix { }; + + slurm17 = callPackage ./bsc/slurm/default.nix { + pmix = pmix2; + }; + + slurm17-libpmi2 = callPackage ./bsc/slurm/pmi2.nix { + pmix = pmix2; + }; + + openmpi-mn4 = callPackage ./bsc/openmpi/default.nix { + pmix = pmix2; + pmi2 = slurm17-libpmi2; + enableCxx = true; + }; + + openmpi = openmpi-mn4; fftw = callPackage ./bsc/fftw/default.nix { mpi = mpi;