From 86edeba765459773b00d98116a8dfb1b62a1af62 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 13 Oct 2025 17:25:13 +0200 Subject: [PATCH 1/2] Add nix patch to implement builtins.catchAll --- overlay.nix | 1 + pkgs/nix/add-catchAll.patch | 64 +++++++++++ pkgs/nix/default.nix | 224 +----------------------------------- 3 files changed, 71 insertions(+), 218 deletions(-) create mode 100644 pkgs/nix/add-catchAll.patch diff --git a/overlay.nix b/overlay.nix index 8ffb4bcd..407f1045 100644 --- a/overlay.nix +++ b/overlay.nix @@ -36,6 +36,7 @@ let mpich = callPackage ./pkgs/mpich/default.nix { mpich = prev.mpich; }; nanos6 = callPackage ./pkgs/nanos6/default.nix { }; nanos6Debug = final.nanos6.override { enableDebug = true; }; + nix = callPackage ./pkgs/nix/default.nix { nix = prev.nix; }; nixtools = callPackage ./pkgs/nixtools/default.nix { }; nixgen = callPackage ./pkgs/nixgen/default.nix { }; # Broken because of pkgsStatic.libcap diff --git a/pkgs/nix/add-catchAll.patch b/pkgs/nix/add-catchAll.patch new file mode 100644 index 00000000..8ef57f89 --- /dev/null +++ b/pkgs/nix/add-catchAll.patch @@ -0,0 +1,64 @@ +From 3aa73c21e3afc91522a6121b0d591af6925b4ba6 Mon Sep 17 00:00:00 2001 +From: Rodrigo Arias Mallo +Date: Mon, 13 Oct 2025 16:05:30 +0200 +Subject: [PATCH] Add builtins.catchAll to catch all types of errors + +Allows fetching multiple Git repositories with builtin.fetchGit and +catching any errors thrown by the builtin, in opposition to the builtin +tryEval. +--- + src/libexpr/primops.cc | 38 ++++++++++++++++++++++++++++++++++++++ + 1 file changed, 38 insertions(+) + +diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc +index 36a67a39d..3b26f9f43 100644 +--- a/src/libexpr/primops.cc ++++ b/src/libexpr/primops.cc +@@ -849,6 +849,44 @@ static RegisterPrimOp primop_tryEval({ + .fun = prim_tryEval, + }); + ++/* Like tryEval but catch all errors. Success => {success=true; value=something;}, ++ * else => {success=false; value=false;} */ ++static void prim_catchAll(EvalState & state, const PosIdx pos, Value * * args, Value & v) ++{ ++ auto attrs = state.buildBindings(3); ++ try { ++ state.forceValue(*args[0], pos); ++ attrs.insert(state.sValue, args[0]); ++ attrs.alloc("success").mkBool(true); ++ attrs.alloc("msg").mkNull(); ++ } catch (Error & e) { ++ attrs.alloc(state.sValue).mkBool(false); ++ attrs.alloc("success").mkBool(false); ++ attrs.alloc("msg").mkString(e.msg()); ++ } ++ v.mkAttrs(attrs); ++} ++ ++static RegisterPrimOp primop_catchAll({ ++ .name = "__catchAll", ++ .args = {"e"}, ++ .doc = R"( ++ Try to shallowly evaluate *e*. Return a set containing the ++ attributes `success` (`true` if *e* evaluated successfully, ++ `false` if an error was thrown) and `value`, equalling *e* if ++ successful and `false` otherwise. In contrast with `tryEval`, ++ `catchAll` will prevent all errors from being thrown, including ++ for those created by `abort` and type errors generated by ++ builtins. Also note that this doesn't evaluate *e* deeply, so ++ `let e = { x = throw ""; }; in (builtins.catchAll e).success` ++ will be `true`. Using `builtins.deepSeq` one can get the expected ++ result: `let e = { x = throw ""; }; in ++ (builtins.catchAll (builtins.deepSeq e e)).success` will be ++ `false`. ++ )", ++ .fun = prim_catchAll, ++}); ++ + /* Return an environment variable. Use with care. */ + static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) + { +-- +2.51.0 + diff --git a/pkgs/nix/default.nix b/pkgs/nix/default.nix index 31c5f9e1..1bb2031b 100644 --- a/pkgs/nix/default.nix +++ b/pkgs/nix/default.nix @@ -1,219 +1,7 @@ -{ lib, fetchurl, fetchFromGitHub, callPackage -, storeDir ? "/nix/store" -, stateDir ? "/nix/var" -, confDir ? "/etc" -, boehmgc -, stdenv, llvmPackages_6 -}: +{ nix }: -let - -common = - { lib, stdenv, fetchpatch, perl, curl, bzip2, sqlite, openssl ? null, xz - , bash, coreutils, gzip, gnutar - , pkg-config, boehmgc, perlPackages, libsodium, brotli, boost, editline, nlohmann_json - , autoreconfHook, autoconf-archive, bison, flex, libxml2, libxslt, docbook5, docbook_xsl_ns - , jq, libarchive, rustc, cargo - # Used by tests - , gmock - , busybox-sandbox-shell - , storeDir - , stateDir - , confDir - , withLibseccomp ? lib.any (lib.meta.platformMatch stdenv.hostPlatform) libseccomp.meta.platforms, libseccomp - , withAWS ? stdenv.isLinux || stdenv.isDarwin, aws-sdk-cpp - - , name, suffix ? "", src, crates ? null - - }: - let - sh = busybox-sandbox-shell; - nix = stdenv.mkDerivation rec { - inherit name src; - version = lib.getVersion name; - - is24 = lib.versionAtLeast version "2.4pre"; - isExactly23 = lib.versionAtLeast version "2.3" && lib.versionOlder version "2.4"; - - VERSION_SUFFIX = suffix; - - outputs = [ "out" "dev" "man" "doc" ]; - - nativeBuildInputs = - [ pkg-config ] - ++ lib.optionals is24 [ autoreconfHook autoconf-archive bison flex libxml2 libxslt - docbook5 docbook_xsl_ns jq gmock ]; - - buildInputs = - [ curl openssl sqlite xz bzip2 nlohmann_json - brotli boost editline - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optionals is24 [ libarchive rustc cargo ] - ++ lib.optional withLibseccomp libseccomp - ++ lib.optional withAWS - ((aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }).overrideDerivation (args: { - patches = args.patches or [] ++ [(fetchpatch { - url = "https://github.com/edolstra/aws-sdk-cpp/commit/7d58e303159b2fb343af9a1ec4512238efa147c7.patch"; - sha256 = "103phn6kyvs1yc7fibyin3lgxz699qakhw671kl207484im55id1"; - })]; - })); - - propagatedBuildInputs = [ boehmgc ]; - - # Seems to be required when using std::atomic with 64-bit types - NIX_LDFLAGS = lib.optionalString (stdenv.hostPlatform.system == "armv5tel-linux" || stdenv.hostPlatform.system == "armv6l-linux") "-latomic"; - - preConfigure = - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - '' - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - '' + - # Unpack the Rust crates. - lib.optionalString is24 '' - tar xvf ${crates} -C nix-rust/ - mv nix-rust/nix-vendored-crates* nix-rust/vendor - '' + - # For Nix-2.3, patch around an issue where the Nix configure step pulls in the - # build system's bash and other utilities when cross-compiling - lib.optionalString (stdenv.buildPlatform != stdenv.hostPlatform && isExactly23) '' - mkdir tmp/ - substitute corepkgs/config.nix.in tmp/config.nix.in \ - --subst-var-by bash ${bash}/bin/bash \ - --subst-var-by coreutils ${coreutils}/bin \ - --subst-var-by bzip2 ${bzip2}/bin/bzip2 \ - --subst-var-by gzip ${gzip}/bin/gzip \ - --subst-var-by xz ${xz}/bin/xz \ - --subst-var-by tar ${gnutar}/bin/tar \ - --subst-var-by tr ${coreutils}/bin/tr - mv tmp/config.nix.in corepkgs/config.nix.in - ''; - - configureFlags = - [ "--with-store-dir=${storeDir}" - "--localstatedir=${stateDir}" - "--sysconfdir=${confDir}" - "--disable-init-state" - "--enable-gc" - ] - ++ lib.optionals stdenv.isLinux [ - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optional ( - stdenv.hostPlatform != stdenv.buildPlatform && stdenv.hostPlatform ? nix && stdenv.hostPlatform.nix ? system - ) ''--with-system=${stdenv.hostPlatform.nix.system}'' - # RISC-V support in progress https://github.com/seccomp/libseccomp/pull/50 - ++ lib.optional (!withLibseccomp) "--disable-seccomp-sandboxing"; - - makeFlags = [ "profiledir=$(out)/etc/profile.d" ]; - - installFlags = [ "sysconfdir=$(out)/etc" ]; - - doInstallCheck = false; - - # socket path becomes too long otherwise - #preInstallCheck = lib.optional stdenv.isDarwin '' - # export TMPDIR=$NIX_BUILD_TOP - #''; - - separateDebugInfo = stdenv.isLinux; - - enableParallelBuilding = true; - - meta = { - description = "Powerful package manager that makes package management reliable and reproducible"; - longDescription = '' - Nix is a powerful package manager for Linux and other Unix systems that - makes package management reliable and reproducible. It provides atomic - upgrades and rollbacks, side-by-side installation of multiple versions of - a package, multi-user package management and easy setup of build - environments. - ''; - homepage = "https://nixos.org/"; - license = lib.licenses.lgpl2Plus; - maintainers = [ lib.maintainers.eelco ]; - platforms = lib.platforms.unix; - outputsToInstall = [ "out" "man" ]; - }; - - passthru = { - perl-bindings = stdenv.mkDerivation { - pname = "nix-perl"; - inherit version; - - inherit src; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - - # This is not cross-compile safe, don't have time to fix right now - # but noting for future travellers. - nativeBuildInputs = - [ perl pkg-config curl nix libsodium boost autoreconfHook autoconf-archive ]; - - configureFlags = - [ "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" - ]; - - preConfigure = "export NIX_STATE_DIR=$TMPDIR"; - - preBuild = "unset NIX_INDENT_MAKE"; - }; - }; - }; - in nix; - -in rec { - - nix = nixUnstable; - - nixUnstable = lib.lowPrio (callPackage common rec { - name = "nix-2.4${suffix}"; - suffix = "pre7534_b92f58f6"; - - #src = /home/Computational/rarias/nix/nix-rodarima; - src = fetchFromGitHub { - owner = "rodarima"; - repo = "nix"; - rev = "3a642187c33ed46d952d3a50a83b2576b704fab7"; - sha256 = "0s8is2czpkcj1x1kcjqgbnsbbl03w3fwjjiclsd44zh1ij3wb90s"; - }; - - crates = fetchurl { - url = "https://hydra.nixos.org/build/118797694/download/1/nix-vendored-crates-2.4pre7534_b92f58f6.tar.xz"; - sha256 = "a4c2612bbd81732bbb899bc0c230e07b16f6b6150ffbb19c4907dedbbc2bf9fc"; - }; - - inherit storeDir stateDir confDir boehmgc; - }); - - nixFlakes = lib.lowPrio (callPackage common rec { - name = "nix-2.4${suffix}"; - suffix = "pre20200521_00b562c"; - - src = fetchFromGitHub { - owner = "NixOS"; - repo = "nix"; - rev = "00b562c87ec4c3bbe514f5dc1f4d1c41f66f66bf"; - sha256 = "0s8is2czpkcj1x1kcjqgbnsbbl03w3fwjjiclsd44zh1ij3wb90s"; - }; - - crates = fetchurl { - url = "https://hydra.nixos.org/build/118093786/download/1/nix-vendored-crates-2.4pre20200501_941f952.tar.xz"; - sha256 = "060f4n5srdbb8vsj0m14aqch7im79a4h5g3nrs41p1xc602vhcdl"; - }; - - inherit storeDir stateDir confDir boehmgc; - }); - -} +nix.overrideAttrs (old: { + patches = (old.patches or []) ++ [ + ./add-catchAll.patch + ]; +}) -- 2.49.0 From 9b491e8d011810e7c575cb3a94558279b6641b9c Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Thu, 16 Oct 2025 10:29:08 +0200 Subject: [PATCH 2/2] Add fetchGitMirror function --- overlay.nix | 26 ++++++++++++-------------- pkgs/bench6/default.nix | 9 ++++++--- pkgs/lib.nix | 30 ++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 17 deletions(-) create mode 100644 pkgs/lib.nix diff --git a/overlay.nix b/overlay.nix index 407f1045..186aae3e 100644 --- a/overlay.nix +++ b/overlay.nix @@ -1,9 +1,8 @@ final: /* Future last stage */ prev: /* Previous stage */ -with final.lib; - let + lib = prev.lib; callPackage = final.callPackage; bscPkgs = { @@ -98,19 +97,22 @@ let }; }; + # Load our custom lib functions with import, callPackage fails. + lib' = import ./pkgs/lib.nix { lib = prev.lib; }; + # For now, only build toplevel packages in CI/Hydra - pkgsTopLevel = filterAttrs (_: isDerivation) bscPkgs; + pkgsTopLevel = lib.filterAttrs (_: lib.isDerivation) bscPkgs; # Native build in that platform doesn't imply cross build works canCrossCompile = platform: default: pkg: - (isDerivation pkg) && + (lib.isDerivation pkg) && # If meta.cross is undefined, use default (pkg.meta.cross or default) && - (meta.availableOn final.pkgsCross.${platform}.stdenv.hostPlatform pkg); + (lib.meta.availableOn final.pkgsCross.${platform}.stdenv.hostPlatform pkg); # For now only RISC-V - crossSet = genAttrs [ "riscv64" ] (platform: - filterAttrs (_: canCrossCompile platform true) + crossSet = lib.genAttrs [ "riscv64" ] (platform: + lib.filterAttrs (_: canCrossCompile platform true) final.pkgsCross.${platform}.bsc.pkgsTopLevel); buildList = name: paths: @@ -126,21 +128,17 @@ let ''; pkgsList = buildList "ci-pkgs" (builtins.attrValues pkgsTopLevel); - testsList = buildList "ci-tests" (collect isDerivation tests); + testsList = buildList "ci-tests" (lib.collect lib.isDerivation tests); allList = buildList' "ci-all" [ pkgsList testsList ]; # For now only RISC-V crossList = buildList "ci-cross" - (filter + (lib.filter (canCrossCompile "riscv64" false) # opt-in (pkgs with: meta.cross = true) (builtins.attrValues crossSet.riscv64)); in bscPkgs // { - lib = prev.lib // { - maintainers = prev.lib.maintainers // { - bsc = import ./pkgs/maintainers.nix; - }; - }; + lib = lib'; # Prevent accidental usage of bsc-ci attribute bsc-ci = throw "the bsc-ci attribute is deprecated, use bsc.ci"; diff --git a/pkgs/bench6/default.nix b/pkgs/bench6/default.nix index 93ad8113..2dcf0e1a 100644 --- a/pkgs/bench6/default.nix +++ b/pkgs/bench6/default.nix @@ -14,16 +14,19 @@ , openblas , ovni , gitBranch ? "master" -, gitURL ? "ssh://git@bscpm04.bsc.es/rarias/bench6.git" , gitCommit ? "bf29a53113737c3aa74d2fe3d55f59868faea7b4" +, gitUrls ? [ + "ssh://git@bscpm04.bsc.es/rarias/bench6.git" + "https://github.com/rodarima/bench6.git" +] }: stdenv.mkDerivation rec { pname = "bench6"; version = "${src.shortRev}"; - src = builtins.fetchGit { - url = gitURL; + src = lib.fetchGitMirror { + urls = gitUrls; ref = gitBranch; rev = gitCommit; }; diff --git a/pkgs/lib.nix b/pkgs/lib.nix new file mode 100644 index 00000000..97cbe978 --- /dev/null +++ b/pkgs/lib.nix @@ -0,0 +1,30 @@ +{ lib }: + +let + # If not supported, fall back to tryEval, which will fail in the first case. + safeCatchAll = if (builtins ? catchAll) + then builtins.catchAll + else e: (builtins.tryEval e) // { msg = ""; }; +in lib.extend (_: lib: { + # Same as fetchGit but accepts a list of mirror urls + fetchGitMirror = { urls, ... } @ args: + let + cleanArgs = lib.removeAttrs args [ "urls" ]; + fetchUrl = url: builtins.fetchGit (cleanArgs // { inherit url; }); + safeFetch = url: safeCatchAll (fetchUrl url); + complain = url: + let + r = safeFetch url; + in + if (r.success) then r + else lib.warn "cannot fetch ${url}, trying next + mirror:${builtins.replaceStrings ["\n" ] ["\n> "] ("\n"+r.msg)}" r; + fetchList = lib.map (url: complain url) urls; + bad = throw "cannot fetch from any mirror"; + good = lib.findFirst (e: e.success) bad fetchList; + in good.value; + + maintainers = lib.maintainers // { + bsc = import ./maintainers.nix; + }; +}) -- 2.49.0