From 115e9beb59c81d1b2f4f9be1fc3b38fe1b536c0a Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 3 Mar 2023 18:18:51 +0100 Subject: [PATCH] Add Intel oneAPI 2023 with compilation tests --- bsc/intel-oneapi/{ifort.nix => 2023.nix} | 179 +++++++++++++----- bsc/intel-oneapi/{default.nix => wrapper.nix} | 0 overlay.nix | 64 +++++-- test/ci.nix | 2 +- test/compilers/hello-c.nix | 68 +++++++ test/compilers/hello-cpp.nix | 36 ++++ test/compilers/hello-f.nix | 35 ++++ test/compilers/suffle-c.nix | 50 +++++ 8 files changed, 378 insertions(+), 56 deletions(-) rename bsc/intel-oneapi/{ifort.nix => 2023.nix} (60%) rename bsc/intel-oneapi/{default.nix => wrapper.nix} (100%) create mode 100644 test/compilers/hello-c.nix create mode 100644 test/compilers/hello-cpp.nix create mode 100644 test/compilers/hello-f.nix create mode 100644 test/compilers/suffle-c.nix diff --git a/bsc/intel-oneapi/ifort.nix b/bsc/intel-oneapi/2023.nix similarity index 60% rename from bsc/intel-oneapi/ifort.nix rename to bsc/intel-oneapi/2023.nix index d9879f1..f537ca1 100644 --- a/bsc/intel-oneapi/ifort.nix +++ b/bsc/intel-oneapi/2023.nix @@ -13,7 +13,9 @@ , symlinkJoin , libfabric , gcc +, gcc7 , wrapCCWith +, linuxHeaders }: # The distribution of intel packages is a mess. We are doing the installation @@ -28,25 +30,24 @@ let - # As of 2022-11-10 this is the last release for hpckit and all other - # components v = { - hpckit = "2022.3.0"; - compiler = "2022.2.0"; - tbb = "2021.7.1"; - mpi = "2021.7.0"; + hpckit = "2023.0.0"; + compiler = "2023.0.0"; + tbb = "2021.8.0"; + mpi = "2021.8.0"; }; aptPackageIndex = stdenv.mkDerivation { name = "intel-oneapi-packages"; srcs = [ + # Set the hashes to "" to fetch them (fetchurl { url = "https://apt.repos.intel.com/oneapi/dists/all/main/binary-amd64/Packages"; - sha256 = "sha256-swUGn097D5o1giK2l+3H4xFcUXSAUYtavQsPyiJlr2A="; + sha256 = "sha256-u0fKJRNkG19xlO3h5+uJgfeAArk+gMATOQQST2RM5pg="; }) (fetchurl { url = "https://apt.repos.intel.com/oneapi/dists/all/main/binary-all/Packages"; - sha256 = "sha256-Ewpy0l0fXiJDG0FkAGykqizW1bm+/lcvI2OREyqzOLM="; + sha256 = "sha256-m3UCjGzq1piHc2Qh/ejM4qBsqDQOVjm3U8E4USi5MwY="; }) ]; phases = [ "installPhase" ]; @@ -75,9 +76,9 @@ let getUrl = pkgList: name: let matches = lib.filter (x: name == x.Package) pkgList; - match = assert lib.length matches == 1; lib.elemAt matches 0; - #n = lib.length matches; - #match = builtins.trace (name + " -- ${builtins.toString n}") (lib.elemAt matches 0); + #match = assert lib.length matches == 1; lib.elemAt matches 0; + n = lib.length matches; + match = builtins.trace (name + " -- n=${builtins.toString n}") (lib.elemAt matches 0); in apthost + match.Filename; @@ -202,7 +203,7 @@ let phases = [ "installPhase" "fixupPhase" ]; dontStrip = true; - autoPatchelfIgnoreMissingDeps = [ "libsycl.so.5" ]; + autoPatchelfIgnoreMissingDeps = [ "libsycl.so.6" ]; installPhase = '' mkdir -p $out/{bin,lib,include} @@ -219,6 +220,7 @@ let # Libraries rsync -a lib/ $out/lib/ + rsync -a lib/x64/ $out/lib/ rsync -a compiler/lib/intel64_lin/ $out/lib/ chmod +w $out/lib cp bin/intel64/libcilkrts.so.5 $out/lib/ @@ -245,6 +247,8 @@ let #"intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-runtime-${version}" ]; + langFortran = true; + buildInputs = [ intel-mpi intel-compiler-shared @@ -258,9 +262,6 @@ let nativeBuildInputs = [ autoPatchelfHook ]; - # The gcc package is required for building other programs - propagatedBuildInputs = [ stdenv.cc intel-compiler-shared ]; - phases = [ "installPhase" "fixupPhase" ]; dontStrip = true; @@ -295,17 +296,41 @@ let ''; }; - - intel-compiler-classic = stdenv.mkDerivation rec { + intel-compiler = stdenv.mkDerivation rec { version = v.compiler; - pname = "intel-compiler-classic"; + pname = "intel-compiler"; src = joinDebs pname [ + # C/C++ + "intel-oneapi-dpcpp-cpp-${version}" + "intel-oneapi-compiler-dpcpp-cpp-${version}" + "intel-oneapi-compiler-dpcpp-cpp-common-${version}" + "intel-oneapi-compiler-dpcpp-cpp-runtime-${version}" "intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-${version}" + "intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-common-${version}" "intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-runtime-${version}" "intel-oneapi-compiler-dpcpp-cpp-classic-fortran-shared-runtime-${version}" - "intel-oneapi-dpcpp-cpp-${version}" - "intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-common-${version}" ]; + # From https://aur.archlinux.org/packages/intel-oneapi-compiler: + # - intel-oneapi-compiler-cpp-eclipse-cfg-2023.0.0-25370_all.deb + # + intel-oneapi-compiler-dpcpp-cpp-2023.0.0-2023.0.0-25370_amd64.deb + # x intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.0.0-2023.0.0-25370_amd64.deb (empty) + # + intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-2023.0.0-25370_amd64.deb + # + intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-common-2023.0.0-2023.0.0-25370_all.deb + # + intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic-runtime-2023.0.0-2023.0.0-25370_amd64.deb + # + intel-oneapi-compiler-dpcpp-cpp-classic-fortran-shared-runtime-2023.0.0-2023.0.0-25370_amd64.deb + # + intel-oneapi-compiler-dpcpp-cpp-common-2023.0.0-2023.0.0-25370_all.deb + # + intel-oneapi-compiler-dpcpp-cpp-runtime-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-compiler-dpcpp-eclipse-cfg-2023.0.0-25370_all.deb + # - intel-oneapi-compiler-fortran-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-compiler-fortran-common-2023.0.0-2023.0.0-25370_all.deb + # - intel-oneapi-compiler-fortran-runtime-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-compiler-fortran-runtime-2023.0.0-25370_amd64.deb + # - intel-oneapi-compiler-shared-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-compiler-shared-common-2023.0.0-2023.0.0-25370_all.deb + # - intel-oneapi-compiler-shared-runtime-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-dpcpp-cpp-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-openmp-2023.0.0-2023.0.0-25370_amd64.deb + # - intel-oneapi-openmp-common-2023.0.0-2023.0.0-25370_all.deb buildInputs = [ intel-compiler-shared @@ -318,9 +343,7 @@ let ]; nativeBuildInputs = [ autoPatchelfHook ]; - - # The gcc package is required for building other programs - propagatedBuildInputs = [ stdenv.cc intel-compiler-shared ]; + autoPatchelfIgnoreMissingDeps = [ "libtbb.so.12" "libtbbmalloc.so.2" "libze_loader.so.1" ]; phases = [ "installPhase" "fixupPhase" ]; @@ -337,10 +360,18 @@ let pushd linux # Binaries rsync -a bin/ $out/bin/ + rsync -a bin-llvm/ $out/bin/ rsync -a bin/intel64/ $out/bin/ # Libraries rsync -a --exclude oclfpga lib/ $out/lib/ + rsync -a compiler/lib/intel64_lin/ $out/lib/ + + # Headers + rsync -a compiler/include/ $out/include/ # Intrinsics for icc + rsync -a include/ $out/include/ + chmod +w $out/include + ln -s $out/lib/clang/16.0.0/include/ $out/include/icx # For icx popd # Manuals @@ -349,35 +380,97 @@ let ''; }; - intel-compiler-classic-wrapper = + wrapIntel = { cc, mygcc, extraBuild ? "", extraInstall ? "" }: let targetConfig = stdenv.targetPlatform.config; - inherit gcc; - in wrapCCWith rec { - cc = intel-compiler-classic; + in (wrapCCWith { + cc = cc; extraBuildCommands = '' - echo "-B${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-cflags echo "-isystem ${cc}/include" >> $out/nix-support/cc-cflags echo "-isystem ${cc}/include/intel64" >> $out/nix-support/cc-cflags - echo "-L${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-ldflags - echo "-L${gcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags + + echo "-L${mygcc.cc}/lib/gcc/${targetConfig}/${mygcc.version}" >> $out/nix-support/cc-ldflags + echo "-L${mygcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags echo "-L${intel-compiler-shared}/lib" >> $out/nix-support/cc-ldflags + echo "-L${cc}/lib" >> $out/nix-support/cc-ldflags - cat "${cc}/nix-support/propagated-build-inputs" >> \ - $out/nix-support/propagated-build-inputs + # Need the gcc in the path + echo 'export "PATH=${mygcc}/bin:$PATH"' >> $out/nix-support/cc-wrapper-hook - # Create the wrappers for icc and icpc - wrap icc $wrapper $ccPath/icc - wrap icpc $wrapper $ccPath/icpc - wrap mcpcom $wrapper $ccPath/mcpcom - ''; - }; + # Disable hardening by default + echo "" > $out/nix-support/add-hardening.sh + '' + extraBuild; + }).overrideAttrs (old: { + installPhase = old.installPhase + extraInstall; + }); + + icx-wrapper = wrapIntel rec { + cc = intel-compiler; + mygcc = gcc; + extraBuild = '' + wrap icx $wrapper $ccPath/icx + wrap icpx $wrapper $ccPath/icpx + echo "-isystem ${cc}/include/icx" >> $out/nix-support/cc-cflags + echo "--gcc-toolchain=${mygcc.cc}" >> $out/nix-support/cc-cflags + ''; + extraInstall = '' + export named_cc="icx" + export named_cxx="icpx" + ''; + }; + + # Legacy + icc-wrapper = wrapIntel rec { + cc = intel-compiler; + # Intel icc classic compiler tries to behave like the gcc found in $PATH. + # EVEN if it doesn't support some of the features. See: + # https://community.intel.com/t5/Intel-C-Compiler/builtin-shuffle-GCC-compatibility-and-has-builtin/td-p/1143619 + mygcc = gcc; + extraBuild = '' + wrap icc $wrapper $ccPath/icc + wrap icpc $wrapper $ccPath/icpc + echo "-isystem ${cc}/include/icc" >> $out/nix-support/cc-cflags + ''; + extraInstall = '' + export named_cc="icc" + export named_cxx="icpc" + ''; + }; + + ifort-wrapper = wrapIntel rec { + cc = intel-compiler-fortran; + mygcc = gcc; + extraBuild = '' + wrap ifort $wrapper $ccPath/ifort + ''; + extraInstall = '' + export named_fc="ifort" + ''; + }; + + stdenv-icc = stdenv.override { + cc = icc-wrapper; + allowedRequisites = null; + }; + + stdenv-icx = stdenv.override { + cc = icx-wrapper; + allowedRequisites = null; + }; + + stdenv-ifort = stdenv.override { + cc = ifort-wrapper; + allowedRequisites = null; + }; in { - inherit - intel-compiler-classic - intel-compiler-classic-wrapper - intel-compiler-fortran - intel-compiler-shared; + inherit aptPackages; + icx = icx-wrapper; + icc = icc-wrapper; + ifort = ifort-wrapper; + + stdenv = stdenv-icx; + stdenv-icc = stdenv-icc; + stdenv-ifort = stdenv-ifort; } diff --git a/bsc/intel-oneapi/default.nix b/bsc/intel-oneapi/wrapper.nix similarity index 100% rename from bsc/intel-oneapi/default.nix rename to bsc/intel-oneapi/wrapper.nix diff --git a/overlay.nix b/overlay.nix index f2065e3..de80618 100644 --- a/overlay.nix +++ b/overlay.nix @@ -39,13 +39,24 @@ let icc2021Unwrapped = callPackage ./bsc/intel-compiler/icc2021.nix { }; - #icc2022Unwrapped = callPackage ./bsc/intel-oneapi/2022.3.1.nix { }; + intel-oneapi-2023 = callPackage ./bsc/intel-oneapi/2023.nix { + libffi = self.libffi_3_3; + }; - intel-oneapi = callPackage ./bsc/intel-oneapi/ifort.nix { }; - ifort2022Unwrapped = bsc.intel-oneapi.intel-compiler-fortran; - icc2022Unwrapped = bsc.intel-oneapi.intel-compiler-classic; + intel2023 = { + inherit (bsc.intel-oneapi-2023) + stdenv icx stdenv-ifort ifort + # Deprecated in mid 2023 + stdenv-icc icc; + }; - #oneapi2022 = callPackage ./bsc/intel-oneapi/oneapi.nix { }; + intel2022 = { + icc = bsc.icc2021; + }; + + intel2021 = { + icc = bsc.icc2021; + }; # A wrapper script that puts all the flags and environment vars # properly and calls the intel compiler binary @@ -63,12 +74,9 @@ let intelLicense = bsc.intelLicense; }; - ifort = bsc.ifort2022; - - icc2022 = bsc.intel-oneapi.intel-compiler-classic-wrapper; - - - icc = bsc.icc2022; + icx = bsc.intel2023.icx; + icc = bsc.intel2023.icc; + ifort = bsc.intel2023.ifort; # We need to set the cc.CC and cc.CXX attributes, in order to # determine the name of the compiler @@ -118,6 +126,12 @@ let nanos6 = bsc.nanos6Release; nanos6Release = callPackage ./bsc/nanos6/default.nix { }; nanos6Git = callPackage ./bsc/nanos6/git.nix { }; + nanos6-icx = bsc.nanos6.override { + stdenv = bsc.intel2023.stdenv; + }; + nanos6-icc = bsc.nanos6.override { + stdenv = bsc.intel2023.stdenv-icc; + }; nanos6Debug = bsc.nanos6.overrideAttrs (old: { dontStrip = true; @@ -283,11 +297,37 @@ let inherit self super bsc callPackage; }; - test = { + test = rec { # hwloc = callPackage ./test/bugs/hwloc.nix { }; sigsegv = callPackage ./test/reproducers/sigsegv.nix { }; + compilers.hello-c = callPackage ./test/compilers/hello-c.nix { }; + compilers.hello-cpp = callPackage ./test/compilers/hello-cpp.nix { }; + compilers.hello-f = callPackage ./test/compilers/hello-f.nix { }; + compilers.intel2023.icx.c = compilers.hello-c.override { + stdenv = bsc.intel2023.stdenv; + }; + compilers.intel2023.icc.c = compilers.hello-c.override { + stdenv = bsc.intel2023.stdenv-icc; + }; + compilers.intel2023.icx.cpp = compilers.hello-cpp.override { + stdenv = bsc.intel2023.stdenv; + }; + compilers.intel2023.icc.cpp = compilers.hello-cpp.override { + stdenv = bsc.intel2023.stdenv-icc; + }; + compilers.intel2023.ifort = compilers.hello-f.override { + stdenv = bsc.intel2023.stdenv-ifort; + }; }; + testAll = with bsc.test; [ + compilers.intel2023.icx.c + compilers.intel2023.icc.c + compilers.intel2023.icx.cpp + compilers.intel2023.icc.cpp + compilers.intel2023.ifort + ]; + ci = import ./test/ci.nix { inherit self super bsc callPackage; }; diff --git a/test/ci.nix b/test/ci.nix index 67349d1..a409aa4 100644 --- a/test/ci.nix +++ b/test/ci.nix @@ -26,7 +26,7 @@ stdenv.mkDerivation rec { wxparaver # Runtimes nanos6 - ]; + ] ++ bsc.testAll; buildPhase = '' if [ -e /boot ]; then diff --git a/test/compilers/hello-c.nix b/test/compilers/hello-c.nix new file mode 100644 index 0000000..f1febd0 --- /dev/null +++ b/test/compilers/hello-c.nix @@ -0,0 +1,68 @@ +{ stdenv, writeText, which, strace }: + +let + hello_c = writeText "hello.c" '' + #include + #include + #include + + int main() + { + printf("Hello world!\n"); + return 0; + } + ''; + shuffle_c = writeText "shuffle.c" '' + #include + + typedef int v4si __attribute__ ((vector_size (16))); + + int main(void) { + v4si a = {1,2,3,4}; + v4si b = {5,6,7,8}; + v4si mask1 = {0,1,1,3}; + v4si mask2 = {0,4,2,5}; + v4si res; + + #if defined(__GNUC__) && (__GNUC__ >= 7) + res = __builtin_shuffle (a, mask1); /* res is {1,2,2,4} */ + res = __builtin_shuffle (a, b, mask2); /* res is {1,5,3,6} */ + + printf("%d %d %d %d\n", res[0], res[1], res[2], res[3]); + #endif + + return 0; + } + ''; +in + +stdenv.mkDerivation rec { + version = "0.0.1"; + name = "hello-c"; + buildInputs = [ stdenv which strace ]; + src = hello_c; + dontUnpack = true; + dontConfigure = true; + NIX_DEBUG = 0; + buildPhase = '' + set -x + echo CC=$CC + which $CC + $CC -v + + cp ${hello_c} hello.c + $CC -v hello.c -o hello + ./hello + + # Only gcc + #cp ${shuffle_c} shuffle.c + #$CC shuffle.c -o shuffle + #./shuffle + + set +x + ''; + + installPhase = '' + touch $out + ''; +} diff --git a/test/compilers/hello-cpp.nix b/test/compilers/hello-cpp.nix new file mode 100644 index 0000000..bd08e50 --- /dev/null +++ b/test/compilers/hello-cpp.nix @@ -0,0 +1,36 @@ +{ stdenv, writeText, which, strace }: + +let + hello_cpp = writeText "hello.cpp" '' + #include + + int main() + { + printf("Hello world!\n"); + return 0; + } + ''; +in + +stdenv.mkDerivation rec { + version = "0.0.1"; + name = "hello-cpp"; + buildInputs = [ stdenv which strace ]; + src = hello_cpp; + dontUnpack = true; + dontConfigure = true; + NIX_DEBUG = 0; + buildPhase = '' + cp $src hello.cpp + set -x + echo CXX=$CXX + which $CXX + $CXX hello.cpp -o hello + ./hello + set +x + ''; + + installPhase = '' + touch $out + ''; +} diff --git a/test/compilers/hello-f.nix b/test/compilers/hello-f.nix new file mode 100644 index 0000000..eeef62a --- /dev/null +++ b/test/compilers/hello-f.nix @@ -0,0 +1,35 @@ +{ stdenv, writeText, which, strace }: + +let + hello_f90 = writeText "hello.f90" '' + program hello + print *, 'Hello, World!' + end program hello + ''; +in + +stdenv.mkDerivation rec { + version = "0.0.1"; + name = "hello-f90"; + buildInputs = [ stdenv which strace ]; + src = hello_f90; + dontUnpack = true; + dontConfigure = true; + NIX_DEBUG = 0; + buildPhase = '' + set -x + echo FC=$FC + which $FC + $FC -v + + cp ${hello_f90} hello.f90 + $FC hello.f90 -o hello + ./hello + + set +x + ''; + + installPhase = '' + touch $out + ''; +} diff --git a/test/compilers/suffle-c.nix b/test/compilers/suffle-c.nix new file mode 100644 index 0000000..fdbb629 --- /dev/null +++ b/test/compilers/suffle-c.nix @@ -0,0 +1,50 @@ +{ stdenv, writeText, which, strace }: + +let + shuffle_c = writeText "shuffle.c" '' + #include + + typedef int v4si __attribute__ ((vector_size (16))); + + int main(void) { + v4si a = {1,2,3,4}; + v4si b = {5,6,7,8}; + v4si mask1 = {0,1,1,3}; + v4si mask2 = {0,4,2,5}; + v4si res; + + #if defined(__GNUC__) && (__GNUC__ >= 7) + res = __builtin_shuffle (a, mask1); /* res is {1,2,2,4} */ + res = __builtin_shuffle (a, b, mask2); /* res is {1,5,3,6} */ + + printf("%d %d %d %d\n", res[0], res[1], res[2], res[3]); + #endif + + return 0; + } + ''; +in + +stdenv.mkDerivation rec { + version = "0.0.1"; + name = "hello-c"; + buildInputs = [ stdenv which strace ]; + src = hello_c; + dontUnpack = true; + dontConfigure = true; + NIX_DEBUG = 0; + buildPhase = '' + cp $src hello.c + set -x + echo CC=$CC + which $CC + #strace -ff -e trace=execve -s99999999 $CC hello.c -o hello + $CC hello.c -o hello + ./hello + set +x + ''; + + installPhase = '' + touch $out + ''; +}