Merge all branches

pkgs/intel-oneapi/2023.nix

@@ -45,40 +45,31 @@ let
     installPhase = ''
       awk -F': ' '\
         BEGIN   { print "[ {" } \
-        NR>1 && /^Package: / { print "} {"; } \
+        NF==0 { empty=1; } \
+        NF && empty { print "} {"; empty=0; } \
         /: /    { printf "%s = \"%s\";\n", $1, $2 } \
         END     { print "} ]" }' $srcs > $out
     '';
   };
 
-  aptPackages = import aptPackageIndex;
-
-  apthost = "https://apt.repos.intel.com/oneapi/";
-
-  getSum = pkgList: name:
+  findMatch = name:
   let
-    matches = lib.filter (x: name == x.Package) pkgList;
-    #n = lib.length matches;
-    #match = builtins.trace (name + " -- ${builtins.toString n}") (lib.elemAt matches 0);
-    match = lib.elemAt matches 0;
-  in
-    match.SHA256;
-
-  getUrl = pkgList: name:
-  let
-    matches = lib.filter (x: name == x.Package) pkgList;
-    #match = assert lib.length matches == 1; lib.elemAt matches 0;
+    aptPackages = import aptPackageIndex;
+    matches = lib.filter (x: name == x.Package) aptPackages;
     n = lib.length matches;
-    match =
-      #builtins.trace (name + " -- n=${builtins.toString n}")
-      (lib.elemAt matches 0);
+    match = builtins.traceVerbose (name + " -- ${builtins.toString n}") (builtins.head matches);
+
+    apthost = "https://apt.repos.intel.com/oneapi/";
   in
-    apthost + match.Filename;
+    {
+      url = apthost + match.Filename;
+      sha256 = match.SHA256;
+    };
 
   uncompressDebs = debs: name: stdenv.mkDerivation {
     name = name;
     srcs = debs;
-    buildInputs = [ dpkg ];
+    nativeBuildInputs = [ dpkg ];
     phases = [ "installPhase" ];
     installPhase = ''
       mkdir -p $out
@@ -91,10 +82,7 @@ let
 
   joinDebs = name: names:
   let
-    urls = builtins.map (x: getUrl aptPackages x) names;
-    sums = builtins.map (x: getSum aptPackages x) names;
-    getsrc = url: sha256: builtins.fetchurl { inherit url sha256; };
-    debs = lib.zipListsWith getsrc urls sums;
+    debs = builtins.map (x: builtins.fetchurl (findMatch x)) names;
   in
     uncompressDebs debs "${name}-source";
 
@@ -108,14 +96,17 @@ let
       "intel-oneapi-mpi-${version}"
     ];
 
-    buildInputs = [
+    nativeBuildInputs = [
+      autoPatchelfHook
       rsync
+    ];
+
+    buildInputs = [
       libfabric
       zlib
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
     installPhase = ''
@@ -154,7 +145,6 @@ let
 
     buildInputs = [
       intel-mpi
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -162,7 +152,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
 
@@ -192,7 +185,6 @@ let
     buildInputs = [
       intel-mpi
       intel-tbb
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -201,7 +193,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
 
@@ -254,7 +249,6 @@ let
     buildInputs = [
       intel-mpi
       intel-compiler-shared
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -262,7 +256,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
 
     phases = [ "installPhase" "fixupPhase" ];
 
@@ -337,7 +334,6 @@ let
 
     buildInputs = [
       intel-compiler-shared
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -345,7 +341,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     autoPatchelfIgnoreMissingDeps = [ "libtbb.so.12" "libtbbmalloc.so.2" "libze_loader.so.1" ];
 
     phases = [ "installPhase" "fixupPhase" ];
@@ -442,7 +441,7 @@ let
     '';
   };
 
-  ifort-wrapper = wrapIntel rec {
+  ifort-wrapper = wrapIntel {
     cc = intel-compiler-fortran;
     mygcc = gcc;
     extraBuild = ''

pkgs/intel-oneapi/packages.nix

@@ -0,0 +1,205 @@
+{ lib
+, stdenv
+, callPackage
+, dpkg
+, fetchurl
+, recurseIntoAttrs
+
+, sqlite
+, elfutils
+}:
+
+let
+  inherit (builtins)
+    elem attrNames attrValues concatMap filter fromJSON getAttr groupBy head isNull listToAttrs map mapAttrs readFile replaceStrings splitVersion ;
+  inherit (lib)
+    converge findFirst groupBy' hasPrefix optional pipe take toInt toList versionAtLeast versionOlder ;
+
+  aptData = fromJSON (readFile ./packages.json);
+
+  # Compare versions in debian control file syntax
+  # See: https://www.debian.org/doc/debian-policy/ch-relationships.html#syntax-of-relationship-fields
+  #
+  # NOTE: this is not a proper version comparison
+  #
+  # A proper version solver, should aggregate dependencies with the same name
+  # and compute the constraint (e.g. a (>= 2) a (<< 5) -> 2 <= a << 5)
+  #
+  # But in the intel repo, there are no such "duplicated" dependencies to specify
+  # upper limits, which leads to issues when intel-hpckit-2021 depends on things
+  # like intel-basekit >= 2021.1.0-2403 and we end up installing the newest
+  # basekit instead of the one from 2021.
+  #
+  # To mitigate this, >= is set to take the latest version with matching major
+  # and minor (only revision and patch are allowed to change)
+  compareVersions = got: kind: want:
+    let
+      g0 = take 2 (splitVersion got);
+      w0 = take 2 (splitVersion want);
+    in if isNull want then true
+    else if kind == "=" then got == want
+    else if kind == "<<" then versionOlder got want
+    else if kind == "<=" then versionAtLeast want got
+    else if kind == ">>" then versionOlder want got
+    else if kind == ">=" then (g0 == w0) && versionAtLeast got want # always match major version
+    else throw "unknown operation: ${kind}";
+
+  findMatching = { pname, kind, version }:
+    findFirst (x: pname == x.pname && compareVersions x.version kind version) null aptData;
+
+  isIntel = pkg: (hasPrefix "intel-" pkg.pname);
+
+  expandDeps = pkg: (map findMatching (filter isIntel pkg.dependencies)) ++ (optional (pkg.size != 0) pkg);
+
+  # get the oldest by major version. If they have the same major version, take
+  # the newest. This prevents most issues with resolutions
+  # versionOlder b a -> true if b is older than a (b `older` a)
+  getNewerInMajor = a: b: let
+    va = a.version;
+    vb = b.version;
+    va0 = head (splitVersion va);
+    vb0 = head (splitVersion vb);
+  in
+  if isNull a then b
+  else if va0 != vb0 then
+    if va0 > vb0 then b else a
+  else if versionOlder vb va then a else b;
+  removeDups = l: attrValues (groupBy' getNewerInMajor null (getAttr "provides") l);
+
+  _resolveDeps = converge (l: removeDups (concatMap expandDeps l));
+  resolveDeps = pkg: let deps = _resolveDeps (toList pkg);
+      namedDeps = (map (x: "${x.pname}-${x.version}") deps);
+    in builtins.traceVerbose (builtins.deepSeq namedDeps namedDeps) deps;
+
+  blacklist = [
+    "intel-basekit-env"
+    "intel-basekit-getting-started"
+    "intel-hpckit-env"
+    "intel-hpckit-getting-started"
+    "intel-oneapi-advisor"
+    "intel-oneapi-common-licensing"
+    "intel-oneapi-common-oneapi-vars"
+    "intel-oneapi-common-vars"
+    "intel-oneapi-compiler-cpp-eclipse-cfg"
+    "intel-oneapi-compiler-dpcpp-eclipse-cfg"
+    "intel-oneapi-condaindex"
+    "intel-oneapi-dev-utilities-eclipse-cfg"
+    "intel-oneapi-dpcpp-ct-eclipse-cfg"
+    "intel-oneapi-eclipse-ide"
+    "intel-oneapi-hpc-toolkit-getting-started"
+    "intel-oneapi-icc-eclipse-plugin-cpp"
+    "intel-oneapi-vtune"
+    "intel-oneapi-vtune-eclipse-plugin-vtune"
+  ];
+
+  isInBlacklist = pkg: elem pkg.provides blacklist;
+  removeBlacklist = filter (e: !(isInBlacklist e));
+
+  dpkgExtractAll = pname: version: {srcs, deps}: stdenv.mkDerivation {
+    inherit pname version srcs;
+
+    nativeBuildInputs = [ dpkg ];
+    phases = [ "installPhase" ];
+
+    passthru = { inherit deps; };
+
+    installPhase = ''
+      mkdir -p $out
+      for src in $srcs; do
+        echo "Unpacking $src"
+        dpkg -x $src $out
+      done
+    '';
+  };
+
+  apthost = "https://apt.repos.intel.com/oneapi/";
+  fetchDeb = p: fetchurl { url = apthost + p.filename; inherit (p) sha256; };
+
+  buildIntel = pkg: pipe pkg [
+    resolveDeps
+    removeBlacklist
+    (l: {srcs = map fetchDeb l; deps = l; })
+    (dpkgExtractAll "${pkg.provides}-extracted" pkg.version)
+  ];
+
+  findHpcKit = year: findMatching { pname = "intel-hpckit"; kind = "<<"; version = toString (year+1); };
+  years = map toInt (attrNames components);
+
+  patchIntel = callPackage ./patch_intel.nix { };
+
+  # Version information for each hpckit. This is used to normalize the paths
+  # so that files are in $out/{bin,lib,include...} instead of all over the place
+  # in $out/opt/intel/oneapi/*/*/{...}.
+  #
+  # The most important is the compiler component, which is used to build the
+  # stdenv for the hpckit.
+  #
+  # NOTE: this have to be manually specified, so we can avoid IFD. To add a
+  # new version, add a new field with an empty attrset, (e.g. "2026" = {}; ),
+  # build hpckit_2026.unpatched and use the values from
+  # result/opt/intel/oneapi/* to populate the attrset.
+  #
+  # WARN: if there are more than one version in the folders of the unpatched
+  # components, our dependency resolution hacks have probably failed and the
+  # package set may be broken.
+  components = {
+    "2025" = {
+      ishmem = "1.3";
+      pti = "0.12";
+      tcm = "1.3";
+      umf = "0.10";
+
+      ccl = "2021.15";
+      compiler = "2025.1";
+      dal = "2025.5";
+      debugger = "2025.1";
+      dev-utilities = "2025.1";
+      dnnl = "2025.1";
+      dpcpp-ct = "2025.1";
+      dpl = "2022.8";
+      ipp = "2022.1";
+      ippcp = "2025.1";
+      mkl = "2025.1";
+      mpi = "2021.15";
+      tbb = "2022.1";
+    };
+    "2024" = {
+      tcm = "1.1";
+
+      ccl = "2021.13";
+      compiler = "2024.2";
+      dal = "2024.6";
+      debugger = "2024.2";
+      dev-utilities = "2024.2";
+      diagnostics = "2024.2";
+      dnnl = "2024.2";
+      dpcpp-ct = "2024.2";
+      dpl = "2022.6";
+      ipp = "2021.12";
+      ippcp = "2021.12";
+      mkl = "2024.2";
+      mpi = "2021.13";
+      tbb = "2021.13";
+
+      extraPackages = [
+        sqlite
+        elfutils
+      ];
+    };
+  };
+
+  replaceDots = replaceStrings ["."] ["_"];
+
+in recurseIntoAttrs (listToAttrs (map (year: let
+      year_str = toString year;
+    in {
+      name = "hpckit_${year_str}";
+      value = patchIntel {unpatched = buildIntel (findHpcKit year); components = components.${year_str}; };
+  }) years)) // {
+  apt = pipe aptData [
+    (groupBy (p: replaceDots p.provides))
+    (mapAttrs (_: l: listToAttrs (map (pkg: { name = replaceDots ("v" + pkg.version); value = pkg; }) l)))
+  ] ;
+
+  inherit resolveDeps patchIntel buildIntel;
+}

pkgs/intel-oneapi/patch_intel.nix

@@ -0,0 +1,177 @@
+{ stdenv
+, stdenvNoCC
+, lib
+, symlinkJoin
+, autoPatchelfHook
+, wrapCCWith
+, overrideCC
+, gcc13
+, hwloc
+, libelf
+, libffi_3_3
+, libpsm2
+, libuuid
+, libxml2
+, numactl
+, ocl-icd
+, openssl
+, python3
+, rdma-core
+, ucx
+, zlib
+, makeOverridable
+, recurseIntoAttrs
+}:
+
+makeOverridable (
+{
+  unpatched,
+  components ? { },
+  extraPackages ? components.extraPackages or []
+}:
+
+let
+  inherit (builtins) attrValues filter mapAttrs removeAttrs;
+
+  gcc = gcc13;
+
+  __components = removeAttrs components ["extraPackages"];
+  _components = __components;
+  # _components = lib.traceSeqN 2 {
+  #   inherit unpatched __components;
+  #   deps = builtins.map (x: "${x.pname}-${x.version}") unpatched.deps;
+  # } __components;
+
+  wrapIntel = { cc, extraBuildCommands ? "", extraInstall ? "", ... }@args:
+    let
+      targetConfig = stdenv.targetPlatform.config;
+    in (wrapCCWith {
+      inherit cc;
+      extraBuildCommands = ''
+        echo "-L${gcc.cc}/lib/gcc/${targetConfig}/${gcc.version}" >> $out/nix-support/cc-ldflags
+        echo "-L${gcc.cc.lib}/lib" >> $out/nix-support/cc-ldflags
+        echo "-L${cc}/lib" >> $out/nix-support/cc-ldflags
+
+        # echo "--gcc-toolchain=${gcc.cc}" >> $out/nix-support/libcxx-cxxflags
+        echo "--gcc-toolchain=${gcc.cc}" >> $out/nix-support/cc-cflags
+
+        # For some reason, If we don't resolve the realpath things go wrong
+        for stddef in ${cc}/lib/clang/*/include/stddef.h ; do
+          dir=$(dirname $(realpath "$stddef"))
+          echo "-isystem $dir" >> $out/nix-support/cc-cflags
+        done
+
+        echo "-isystem ${cc}/include" >> $out/nix-support/cc-cflags
+        echo "-isystem ${cc}/include/intel64" >> $out/nix-support/cc-cflags
+
+        for dir in ${gcc.cc}/lib/gcc/${targetConfig}/*/include; do
+          echo "-isystem $dir" >> $out/nix-support/cc-cflags
+        done
+
+        for dir in ${gcc.cc}/include/c++/*; do
+          echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags
+        done
+        for dir in ${gcc.cc}/include/c++/*/${targetConfig}; do
+          echo "-isystem $dir" >> $out/nix-support/libcxx-cxxflags
+        done
+
+        # FIXME: We should find a better way to modify the PATH instead of using
+        # this ugly hack. See https://jungle.bsc.es/git/rarias/bscpkgs/issues/9
+        echo 'path_backup="${gcc.cc}/bin:$path_backup"' >> $out/nix-support/cc-wrapper-hook
+
+        # Disable hardening by default
+        echo "" > $out/nix-support/add-hardening.sh
+      '' + extraBuildCommands;
+    } // (removeAttrs args ["cc" "extraBuildCommands" "extraInstall"])
+    ).overrideAttrs (old: {
+      installPhase = old.installPhase + extraInstall;
+    });
+
+in
+stdenvNoCC.mkDerivation (finalAttrs: {
+  pname = lib.removeSuffix "-extracted" unpatched.pname;
+  inherit (unpatched) version;
+  src = unpatched;
+
+  phases = [ "installPhase" "fixupPhase" ];
+  buildInputs = [
+    libffi_3_3
+    libelf
+    libxml2
+    hwloc
+    numactl
+    libuuid
+    libpsm2
+    zlib
+    ocl-icd
+    rdma-core
+    ucx
+    openssl
+    python3
+    stdenv.cc.cc.lib
+  ] ++ extraPackages;
+
+  autoPatchelfIgnoreMissingDeps = [ "libhwloc.so.5" "libcuda.so.1" "libze_loader.so.1" ];
+
+  dontCheckForBrokenSymlinks = true;
+
+  nativeBuildInputs = [ autoPatchelfHook ];
+  installPhase = ''
+    cp -r $src/opt/intel/oneapi/ $out
+  '';
+
+  passthru = let
+    pkgs = mapAttrs
+      (folder: version: symlinkJoin {
+        pname = "intel-${folder}";
+        inherit version;
+        paths = ["${finalAttrs.finalPackage}/${folder}/${version}"];
+      })
+      _components;
+  in pkgs // {
+    inherit unpatched;
+    pkgs = recurseIntoAttrs pkgs;
+    components = _components;
+
+    # This contains all packages properly symlinked into toplevel directories
+    # in $out.
+    #
+    # NOTE: there are clashes with packages that have symlinks outside their
+    # scope (libtcm and env/vars.sh)
+    all = symlinkJoin {
+      pname = finalAttrs.finalPackage + "-symlinked";
+      inherit (finalAttrs.finalPackage) version;
+      paths = filter lib.isDerivation (attrValues finalAttrs.finalPackage.pkgs);
+    };
+
+    stdenv = overrideCC stdenv finalAttrs.finalPackage.cc;
+
+    cc = wrapIntel {
+      cc = finalAttrs.finalPackage.pkgs.compiler;
+      extraBuildCommands = ''
+        wrap icx  $wrapper $ccPath/icx
+        wrap icpx $wrapper $ccPath/icpx
+        wrap ifx  $wrapper $ccPath/ifx
+
+        ln -s $out/bin/icpx $out/bin/c++
+        ln -s $out/bin/icx  $out/bin/cc
+
+        # Use this to detect when a compiler subprocess is called
+        # from icpx (--fsycl-host-compiler)
+        echo 'export "NIX_CC_WRAPPER_INTEL=1"' >> $out/nix-support/cc-wrapper-hook
+
+        sed -i 's/.*isCxx=0/isCxx=1/' $out/bin/icpx
+
+        # oneMath looks for sycl libraries in bin/../lib
+        ln -s ${finalAttrs.finalPackage.pkgs.compiler}/lib $out/lib
+        ln -s ${finalAttrs.finalPackage.pkgs.compiler}/include $out/include
+      '';
+      extraInstall = ''
+        export named_cc="icx"
+        export named_cxx="icpx"
+        export named_fc="ifx"
+      '';
+    };
+  };
+
+}))

pkgs/intel-oneapi/process.jq

@@ -0,0 +1,29 @@
+#!/usr/bin/env -S jq -f
+
+def extract_fields: {
+        pname : .Package,
+        version : .Version,
+        provides : .Package | sub("[0-9.-]*$"; ""),
+        filename : .Filename,
+        size : ."Installed-Size" | tonumber,
+        sha256 : .SHA256,
+        dependencies : .Depends,
+    } ;
+
+# parses dependencies into a list of [{.pname, .kind, .version}]
+# some dependencies do not have a version specified, in which case, version = version = null
+#
+# example dependencies:
+# intel-oneapi-common-vars (>= 2023.0.0-25325), intel-oneapi-common-licensing-2023.0.0
+
+def split_dependencies : map(try(.dependencies |= split(",\\s?"; "")) // .dependencies |= []) ;
+
+def match_version : capture("(?<pname>[a-zA-Z0-9_\\-.]*) *(\\((?<kind>[<>=]*) *(?<version>.*)\\))?"; "") ;
+
+def parse_dependencies : map_values(.dependencies.[] |= match_version) ;
+
+def sort_version_decreasing : sort_by(.version | split("[-.]"; "") | map(tonumber)) | reverse ;
+
+map(extract_fields) | split_dependencies | parse_dependencies | sort_version_decreasing
+
+# [.[] | select(.pname == "intel-hpckit") | .version]

pkgs/intel-oneapi/toJson.awk

@@ -0,0 +1,29 @@
+#!/usr/bin/env -S awk -f
+
+BEGIN   {
+    FS=": "
+
+    prev_empty=1
+    t="  "
+
+    print "[ {"
+}
+
+!NF { # empty line, update separator so next non empty line closes the dict
+    prev_empty=1
+    t="},\n{ "
+    next # skip line (we won't match anything else)
+}
+
+{
+    printf t "\"%s\" : \"%s\"\n", $1, $2
+
+    if (prev_empty) {
+        # we were the first after a group of empty lines, following ones have to
+        # have a comma
+        prev_empty=0
+        t=", "
+    }
+}
+
+END     { print "} ]" }

pkgs/intel-oneapi/update.sh

@@ -2,3 +2,5 @@
 
 curl https://apt.repos.intel.com/oneapi/dists/all/main/binary-amd64/Packages -o amd64-packages
 curl https://apt.repos.intel.com/oneapi/dists/all/main/binary-all/Packages -o all-packages
+
+cat amd64-packages all-packages | ./toJson.awk | ./process.jq >packages.json