forked from rarias/jungle
Only save relevant packages from intel repo
Refactored the parsing and dependency resolution logic into a single Haskell script. Reviewed-by: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
This commit is contained in:
@@ -7,145 +7,19 @@
|
||||
|
||||
sqlite,
|
||||
elfutils,
|
||||
|
||||
intelAptRepoURL ? "https://apt.repos.intel.com/oneapi/",
|
||||
}:
|
||||
|
||||
let
|
||||
inherit (builtins)
|
||||
attrNames
|
||||
attrValues
|
||||
concatMap
|
||||
elem
|
||||
filter
|
||||
fromJSON
|
||||
getAttr
|
||||
groupBy
|
||||
head
|
||||
isNull
|
||||
listToAttrs
|
||||
map
|
||||
mapAttrs
|
||||
readFile
|
||||
replaceStrings
|
||||
splitVersion
|
||||
;
|
||||
inherit (lib)
|
||||
converge
|
||||
findFirst
|
||||
groupBy'
|
||||
hasPrefix
|
||||
optional
|
||||
pipe
|
||||
take
|
||||
toInt
|
||||
toList
|
||||
versionAtLeast
|
||||
versionOlder
|
||||
;
|
||||
inherit (builtins) map mapAttrs readFile;
|
||||
|
||||
aptData = fromJSON (readFile ./packages.json);
|
||||
|
||||
# Compare versions in debian control file syntax
|
||||
# See: https://www.debian.org/doc/debian-policy/ch-relationships.html#syntax-of-relationship-fields
|
||||
#
|
||||
# NOTE: this is not a proper version comparison
|
||||
#
|
||||
# A proper version solver, should aggregate dependencies with the same name
|
||||
# and compute the constraint (e.g. a (>= 2) a (<< 5) -> 2 <= a << 5)
|
||||
#
|
||||
# But in the intel repo, there are no such "duplicated" dependencies to specify
|
||||
# upper limits, which leads to issues when intel-hpckit-2021 depends on things
|
||||
# like intel-basekit >= 2021.1.0-2403 and we end up installing the newest
|
||||
# basekit instead of the one from 2021.
|
||||
#
|
||||
# To mitigate this, >= is set to take the latest version with matching major
|
||||
# and minor (only revision and patch are allowed to change)
|
||||
compareVersions =
|
||||
got: kind: want:
|
||||
let
|
||||
g0 = take 2 (splitVersion got);
|
||||
w0 = take 2 (splitVersion want);
|
||||
in
|
||||
if isNull want then
|
||||
true
|
||||
else if kind == "=" then
|
||||
got == want
|
||||
else if kind == "<<" then
|
||||
versionOlder got want
|
||||
else if kind == "<=" then
|
||||
versionAtLeast want got
|
||||
else if kind == ">>" then
|
||||
versionOlder want got
|
||||
else if kind == ">=" then
|
||||
(g0 == w0) && versionAtLeast got want # always match major version
|
||||
else
|
||||
throw "unknown operation: ${kind}";
|
||||
|
||||
findMatching =
|
||||
{
|
||||
pname,
|
||||
kind,
|
||||
version,
|
||||
}:
|
||||
findFirst (x: pname == x.pname && compareVersions x.version kind version) null aptData;
|
||||
|
||||
isIntel = pkg: (hasPrefix "intel-" pkg.pname);
|
||||
|
||||
expandDeps =
|
||||
pkg: (map findMatching (filter isIntel pkg.dependencies)) ++ (optional (pkg.size != 0) pkg);
|
||||
|
||||
# get the oldest by major version. If they have the same major version, take
|
||||
# the newest. This prevents most issues with resolutions
|
||||
# versionOlder b a -> true if b is older than a (b `older` a)
|
||||
getNewerInMajor =
|
||||
a: b:
|
||||
let
|
||||
va = a.version;
|
||||
vb = b.version;
|
||||
va0 = head (splitVersion va);
|
||||
vb0 = head (splitVersion vb);
|
||||
in
|
||||
if isNull a then
|
||||
b
|
||||
else if va0 != vb0 then
|
||||
if va0 > vb0 then b else a
|
||||
else if versionOlder vb va then
|
||||
a
|
||||
else
|
||||
b;
|
||||
removeDups = l: attrValues (groupBy' getNewerInMajor null (getAttr "provides") l);
|
||||
|
||||
_resolveDeps = converge (l: removeDups (concatMap expandDeps l));
|
||||
resolveDeps =
|
||||
pkg:
|
||||
let
|
||||
deps = _resolveDeps (toList pkg);
|
||||
namedDeps = (map (x: "${x.pname}-${x.version}") deps);
|
||||
in
|
||||
builtins.traceVerbose (builtins.deepSeq namedDeps namedDeps) deps;
|
||||
|
||||
blacklist = [
|
||||
"intel-basekit-env"
|
||||
"intel-basekit-getting-started"
|
||||
"intel-hpckit-env"
|
||||
"intel-hpckit-getting-started"
|
||||
"intel-oneapi-advisor"
|
||||
"intel-oneapi-common-licensing"
|
||||
"intel-oneapi-common-oneapi-vars"
|
||||
"intel-oneapi-common-vars"
|
||||
"intel-oneapi-compiler-cpp-eclipse-cfg"
|
||||
"intel-oneapi-compiler-dpcpp-eclipse-cfg"
|
||||
"intel-oneapi-condaindex"
|
||||
"intel-oneapi-dev-utilities-eclipse-cfg"
|
||||
"intel-oneapi-dpcpp-ct-eclipse-cfg"
|
||||
"intel-oneapi-eclipse-ide"
|
||||
"intel-oneapi-hpc-toolkit-getting-started"
|
||||
"intel-oneapi-icc-eclipse-plugin-cpp"
|
||||
"intel-oneapi-vtune"
|
||||
"intel-oneapi-vtune-eclipse-plugin-vtune"
|
||||
];
|
||||
|
||||
isInBlacklist = pkg: elem pkg.provides blacklist;
|
||||
removeBlacklist = filter (e: !(isInBlacklist e));
|
||||
fetchDeb =
|
||||
p:
|
||||
fetchurl {
|
||||
url = intelAptRepoURL + p.file;
|
||||
inherit (p) sha256;
|
||||
};
|
||||
|
||||
dpkgExtractAll =
|
||||
pname: version:
|
||||
@@ -167,35 +41,13 @@ let
|
||||
'';
|
||||
};
|
||||
|
||||
apthost = "https://apt.repos.intel.com/oneapi/";
|
||||
fetchDeb =
|
||||
p:
|
||||
fetchurl {
|
||||
url = apthost + p.filename;
|
||||
inherit (p) sha256;
|
||||
extractKit =
|
||||
name: version: pkg:
|
||||
dpkgExtractAll "${name}-extracted" version {
|
||||
srcs = map fetchDeb pkg;
|
||||
deps = pkg;
|
||||
};
|
||||
|
||||
buildIntel =
|
||||
pkg:
|
||||
pipe pkg [
|
||||
resolveDeps
|
||||
removeBlacklist
|
||||
(l: {
|
||||
srcs = map fetchDeb l;
|
||||
deps = l;
|
||||
})
|
||||
(dpkgExtractAll "${pkg.provides}-extracted" pkg.version)
|
||||
];
|
||||
|
||||
findHpcKit =
|
||||
year:
|
||||
findMatching {
|
||||
pname = "intel-hpckit";
|
||||
kind = "<<";
|
||||
version = toString (year + 1);
|
||||
};
|
||||
years = map toInt (attrNames components);
|
||||
|
||||
patchIntel = callPackage ./patch_intel.nix { };
|
||||
|
||||
# Version information for each hpckit. This is used to normalize the paths
|
||||
@@ -214,29 +66,28 @@ let
|
||||
# components, our dependency resolution hacks have probably failed and the
|
||||
# package set may be broken.
|
||||
components = {
|
||||
"2025" = {
|
||||
ishmem = "1.4";
|
||||
pti = "0.13";
|
||||
tcm = "1.4";
|
||||
umf = "0.11";
|
||||
# hpckit-2023 = {
|
||||
# ccl = "2021.10.0";
|
||||
# compiler = "2023.2.4";
|
||||
# # conda_channel = "linux-64"; # TODO: which package pulls this?
|
||||
# dal = "2023.2.0";
|
||||
# debugger = "2023.2.0";
|
||||
# dev-utilities = "2021.10.0";
|
||||
# diagnostics = "2022.4.0";
|
||||
# dnnl = "2023.2.0";
|
||||
# dpcpp-ct = "2023.2.0";
|
||||
# dpl = "2022.2.0";
|
||||
# ipp = "2021.9.0";
|
||||
# ippcp = "2021.8.0";
|
||||
# itac = "2021.10.0";
|
||||
# mkl = "2023.2.0";
|
||||
# mpi = "2021.10.0";
|
||||
# tbb = "2021.10.0";
|
||||
#
|
||||
# llvmMajorVersion = 14;
|
||||
# };
|
||||
|
||||
ccl = "2021.16";
|
||||
compiler = "2025.2";
|
||||
dal = "2025.8";
|
||||
debugger = "2025.2";
|
||||
dev-utilities = "2025.2";
|
||||
dnnl = "2025.2";
|
||||
dpcpp-ct = "2025.2";
|
||||
dpl = "2022.9";
|
||||
ipp = "2022.2";
|
||||
ippcp = "2025.2";
|
||||
mkl = "2025.2";
|
||||
mpi = "2021.16";
|
||||
tbb = "2022.2";
|
||||
|
||||
llvmMajorVersion = 21;
|
||||
};
|
||||
"2024" = {
|
||||
hpckit-2024 = {
|
||||
tcm = "1.1";
|
||||
|
||||
ccl = "2021.13";
|
||||
@@ -261,41 +112,38 @@ let
|
||||
elfutils
|
||||
];
|
||||
};
|
||||
|
||||
hpckit-2025 = {
|
||||
ishmem = "1.5";
|
||||
tcm = "1.4";
|
||||
umf = "1.0";
|
||||
|
||||
ccl = "2021.17";
|
||||
compiler = "2025.3";
|
||||
dal = "2025.10";
|
||||
debugger = "2025.3";
|
||||
dev-utilities = "2025.3";
|
||||
dnnl = "2025.3";
|
||||
dpcpp-ct = "2025.3";
|
||||
dpl = "2022.10";
|
||||
ipp = "2022.3";
|
||||
ippcp = "2025.3";
|
||||
mkl = "2025.3";
|
||||
mpi = "2021.17";
|
||||
tbb = "2022.3";
|
||||
|
||||
llvmMajorVersion = 21;
|
||||
};
|
||||
};
|
||||
|
||||
replaceDots = replaceStrings [ "." ] [ "_" ];
|
||||
|
||||
in
|
||||
lib.recurseIntoAttrs (
|
||||
listToAttrs (
|
||||
map (
|
||||
year:
|
||||
let
|
||||
year_str = toString year;
|
||||
in
|
||||
{
|
||||
name = "hpckit_${year_str}";
|
||||
value = patchIntel {
|
||||
unpatched = buildIntel (findHpcKit year);
|
||||
components = components.${year_str};
|
||||
};
|
||||
}
|
||||
) years
|
||||
)
|
||||
)
|
||||
mapAttrs (
|
||||
name: value:
|
||||
patchIntel {
|
||||
unpatched = extractKit (lib.getName name) (lib.getVersion name) value;
|
||||
components = components.${name};
|
||||
}
|
||||
) (fromTOML (readFile ./packages.toml))
|
||||
// {
|
||||
apt = pipe aptData [
|
||||
(groupBy (p: replaceDots p.provides))
|
||||
(mapAttrs (
|
||||
_: l:
|
||||
listToAttrs (
|
||||
map (pkg: {
|
||||
name = replaceDots ("v" + pkg.version);
|
||||
value = pkg;
|
||||
}) l
|
||||
)
|
||||
))
|
||||
];
|
||||
|
||||
inherit resolveDeps patchIntel buildIntel;
|
||||
inherit patchIntel extractKit;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user