Move bsc-ci test into let

Builds the .#bsc-ci.all target on each PR. Causes all packages to be
built in hut, populating the nix cache.

Reviewed-by: Aleix Boné <abonerib@bsc.es>
Tested-by: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>

.gitea/workflows/ci.yaml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:all:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link --print-out-paths .#bsc-ci.all

flake.nix

@@ -17,7 +17,7 @@
       legacyPackages.${system} = pkgs;
 
       hydraJobs = {
-        inherit (self.legacyPackages.${system}.bsc-ci) test pkgs;
+        inherit (self.legacyPackages.${system}.bsc-ci) test pkgs cross;
       };
 
       # propagate nixpkgs lib, so we can do bscpkgs.lib

overlay.nix

@@ -6,9 +6,6 @@ with final.lib;
 let
   callPackage = final.callPackage;
 
-  mkDeps = name: pkgs: final.runCommand name { }
-    "printf '%s\n' ${toString (collect (x: x ? outPath) pkgs)} > $out";
-
   bscPkgs = {
     bench6 = callPackage ./pkgs/bench6/default.nix { };
     bigotes = callPackage ./pkgs/bigotes/default.nix { };
@@ -49,58 +46,74 @@ let
     wxparaver = callPackage ./pkgs/paraver/default.nix { };
   };
 
+  test = rec {
+    #hwloc = callPackage ./test/bugs/hwloc.nix { }; # Broken, no /sys
+    #sigsegv = callPackage ./test/reproducers/sigsegv.nix { };
+    hello-c = callPackage ./test/compilers/hello-c.nix { };
+    hello-cpp = callPackage ./test/compilers/hello-cpp.nix { };
+    lto = callPackage ./test/compilers/lto.nix { };
+    asan = callPackage ./test/compilers/asan.nix { };
+    intel2023-icx-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv; };
+    intel2023-icc-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv-icc; };
+    intel2023-icx-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv; };
+    intel2023-icc-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv-icc; };
+    intel2023-ifort   = callPackage ./test/compilers/hello-f.nix {
+      stdenv = final.intelPackages_2023.stdenv-ifort;
+    };
+    clangOmpss2-lto   = lto.override       { stdenv = final.stdenvClangOmpss2Nanos6; };
+    clangOmpss2-asan  = asan.override      { stdenv = final.stdenvClangOmpss2Nanos6; };
+    clangOmpss2-task  = callPackage ./test/compilers/ompss2.nix {
+      stdenv = final.stdenvClangOmpss2Nanos6;
+    };
+    clangNodes-task = callPackage ./test/compilers/ompss2.nix {
+      stdenv = final.stdenvClangOmpss2Nodes;
+    };
+    clangNosvOpenmp-task = callPackage ./test/compilers/clang-openmp.nix {
+      stdenv = final.stdenvClangOmpss2Nodes;
+    };
+    clangNosvOmpv-nosv = callPackage ./test/compilers/clang-openmp-nosv.nix {
+      stdenv = final.stdenvClangOmpss2NodesOmpv;
+    };
+    clangNosvOmpv-ld = callPackage ./test/compilers/clang-openmp-ld.nix {
+      stdenv = final.stdenvClangOmpss2NodesOmpv;
+    };
+  };
+
+  pkgs = filterAttrs (_: isDerivation) bscPkgs;
+
+  crossTargets = [ "riscv64" ];
+  cross = prev.lib.genAttrs crossTargets (target:
+    final.pkgsCross.${target}.bsc-ci.pkgs
+  );
+
+  buildList = name: paths:
+    final.runCommandLocal name { } ''
+      printf '%s\n' ${toString paths} | tee $out
+    '';
+
+  buildList' = name: paths:
+    final.runCommandLocal name { } ''
+      deps="${toString paths}"
+      cat $deps
+      printf '%s\n' $deps >$out
+    '';
+
+  crossList = builtins.mapAttrs (t: v: buildList t (builtins.attrValues v)) cross;
+
+  pkgsList = buildList "ci-pkgs" (builtins.attrValues pkgs);
+  tests = buildList "ci-tests" (collect isDerivation test);
+
+  all = buildList' "ci-all" [ pkgsList tests ];
+
 in bscPkgs // {
   # Prevent accidental usage of bsc attribute
   bsc = throw "the bsc attribute is deprecated, packages are now in the root";
 
   # Internal for our CI tests
   bsc-ci = {
-    test = rec {
+    inherit pkgs pkgsList;
-      #hwloc = callPackage ./test/bugs/hwloc.nix { }; # Broken, no /sys
+    inherit test tests;
-      #sigsegv = callPackage ./test/reproducers/sigsegv.nix { };
+    inherit cross crossList;
-      hello-c = callPackage ./test/compilers/hello-c.nix { };
+    inherit all;
-      hello-cpp = callPackage ./test/compilers/hello-cpp.nix { };
-      lto = callPackage ./test/compilers/lto.nix { };
-      asan = callPackage ./test/compilers/asan.nix { };
-      intel2023-icx-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv; };
-      intel2023-icc-c   = hello-c.override   { stdenv = final.intelPackages_2023.stdenv-icc; };
-      intel2023-icx-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv; };
-      intel2023-icc-cpp = hello-cpp.override { stdenv = final.intelPackages_2023.stdenv-icc; };
-      intel2023-ifort   = callPackage ./test/compilers/hello-f.nix {
-        stdenv = final.intelPackages_2023.stdenv-ifort;
-      };
-      clangOmpss2-lto   = lto.override       { stdenv = final.stdenvClangOmpss2Nanos6; };
-      clangOmpss2-asan  = asan.override      { stdenv = final.stdenvClangOmpss2Nanos6; };
-      clangOmpss2-task  = callPackage ./test/compilers/ompss2.nix {
-        stdenv = final.stdenvClangOmpss2Nanos6;
-      };
-      clangNodes-task = callPackage ./test/compilers/ompss2.nix {
-        stdenv = final.stdenvClangOmpss2Nodes;
-      };
-      clangNosvOpenmp-task = callPackage ./test/compilers/clang-openmp.nix {
-        stdenv = final.stdenvClangOmpss2Nodes;
-      };
-      clangNosvOmpv-nosv = callPackage ./test/compilers/clang-openmp-nosv.nix {
-        stdenv = final.stdenvClangOmpss2NodesOmpv;
-      };
-      clangNosvOmpv-ld = callPackage ./test/compilers/clang-openmp-ld.nix {
-        stdenv = final.stdenvClangOmpss2NodesOmpv;
-      };
-    };
-
-    pkgs = filterAttrs (_: isDerivation) bscPkgs;
-
-    pkgsList = final.runCommand "ci-pkgs" { }
-      "printf '%s\n' ${toString (collect isDerivation bscPkgs)} > $out";
-
-    tests = final.runCommand "ci-tests" { }
-      "printf '%s\n' ${toString (collect isDerivation final.bsc-ci.test)} > $out";
-
-    all = final.runCommand "ci-all" { }
-    ''
-      deps="${toString [ final.bsc-ci.pkgsList final.bsc-ci.tests ]}"
-      cat $deps
-      printf '%s\n' $deps > $out
-    '';
   };
 }

pkgs/bench6/default.nix

@@ -27,10 +27,13 @@ stdenv.mkDerivation rec {
     rev = gitCommit;
   };
 
-  buildInputs = [
+  nativeBuildInputs = [
-    bigotes
     cmake
     clangOmpss2
+  ];
+
+  buildInputs = [
+    bigotes
     openmp
     openmpv
     nanos6

pkgs/bigotes/default.nix

@@ -4,7 +4,7 @@
 , cmake
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation {
   pname = "bigotes";
   version = "9dce13";
   src = fetchFromGitHub {
@@ -13,5 +13,5 @@ stdenv.mkDerivation rec {
     rev = "9dce13446a8da30bea552d569d260d54e0188518";
     sha256 = "sha256-ktxM3pXiL8YXSK+/IKWYadijhYXqGoLY6adLk36iigE=";
   };
-  buildInputs = [ cmake ];
+  nativeBuildInputs = [ cmake ];
 }

pkgs/intel-oneapi/2023.nix

@@ -78,7 +78,7 @@ let
   uncompressDebs = debs: name: stdenv.mkDerivation {
     name = name;
     srcs = debs;
-    buildInputs = [ dpkg ];
+    nativeBuildInputs = [ dpkg ];
     phases = [ "installPhase" ];
     installPhase = ''
       mkdir -p $out
@@ -108,14 +108,17 @@ let
       "intel-oneapi-mpi-${version}"
     ];
 
-    buildInputs = [
+    nativeBuildInputs = [
+      autoPatchelfHook
       rsync
+    ];
+
+    buildInputs = [
       libfabric
       zlib
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
     installPhase = ''
@@ -154,7 +157,6 @@ let
 
     buildInputs = [
       intel-mpi
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -162,7 +164,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
 
@@ -192,7 +197,6 @@ let
     buildInputs = [
       intel-mpi
       intel-tbb
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -201,7 +205,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     phases = [ "installPhase" "fixupPhase" ];
     dontStrip = true;
 
@@ -254,7 +261,6 @@ let
     buildInputs = [
       intel-mpi
       intel-compiler-shared
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -262,7 +268,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
 
     phases = [ "installPhase" "fixupPhase" ];
 
@@ -337,7 +346,6 @@ let
 
     buildInputs = [
       intel-compiler-shared
-      rsync
       libffi_3_3
       libelf
       libxml2
@@ -345,7 +353,10 @@ let
       stdenv.cc.cc.lib
     ];
 
-    nativeBuildInputs = [ autoPatchelfHook ];
+    nativeBuildInputs = [
+      autoPatchelfHook
+      rsync
+    ];
     autoPatchelfIgnoreMissingDeps = [ "libtbb.so.12" "libtbbmalloc.so.2" "libze_loader.so.1" ];
 
     phases = [ "installPhase" "fixupPhase" ];

pkgs/llvm-ompss2/clang.nix

@@ -43,32 +43,36 @@ let
 
   source = if (useGit) then git else release;
 
-in stdenv.mkDerivation rec {
+in stdenv.mkDerivation {
   pname = "clang-ompss2";
   inherit (source) src version;
 
   enableParallelBuilding = true;
-  isClang = true;
 
   passthru = {
     CC = "clang";
     CXX = "clang++";
+
+    isClang = true;
+    isClangWithOmpss = true;
+
+    inherit gcc zlib;
   };
 
-  isClangWithOmpss = true;
+  nativeBuildInputs = [
-
-  nativeBuildInputs = [ zlib ];
-
-  buildInputs = [
-    which
     bash
+    cmake
+    elfutils
+    llvmPackages_latest.lld
+    pkg-config
     python3
     perl
-    cmake
+    which
-    llvmPackages_latest.lld
+    zlib
-    elfutils
+  ];
+
+  buildInputs = [
     libffi
-    pkg-config
     zlib
     gcc.cc.lib # Required for libstdc++.so.6
   ];

pkgs/mcxx/default.nix

@@ -32,19 +32,22 @@ stdenv.mkDerivation rec {
 
   enableParallelBuilding = true;
 
-  buildInputs = [
+  nativeBuildInputs = [
     autoreconfHook
-    nanos6
+    bison
-    gperf
+    flex
     python3
     gfortran
     pkg-config
-    sqlite.dev
+    gperf
-    bison
-    flex
     gcc
   ];
 
+  buildInputs = [
+    nanos6
+    sqlite.dev
+  ];
+
   patches = [ ./intel.patch ];
 
   preConfigure = ''

pkgs/nanos6/default.nix

@@ -88,11 +88,19 @@ in
     dontStrip = enableDebug;
     separateDebugInfo = true;
 
-    buildInputs = [
+    nativeBuildInputs = [
       autoconf
       automake
       libtool
       pkg-config
+
+      # TODO: papi_version is needed for configure:
+      # ./configure: line 27378: papi_version: command not found
+      # This probably breaks cross-compilation
+      papi
+    ];
+
+    buildInputs = [
       boost
       numactl
       hwloc
@@ -104,7 +112,7 @@ in
     postInstall = ''
       mkdir -p $out/nix-support
       echo "export NANOS6_HOME=$out" >> $out/nix-support/setup-hook
-    ''; 
+    '';
 
     meta = with lib; {
       homepage = "https://github.com/bsc-pm/nanos6";

pkgs/osu/default.nix

@@ -24,9 +24,10 @@ stdenv.mkDerivation rec {
 
   doCheck = true;
   enableParallelBuilding = true;
+  nativeBuildInputs = [ mpiAll ];
   buildInputs = [ mpiAll ];
   hardeningDisable = [ "all" ];
-  configureFlags = [ 
+  configureFlags = [
       "CC=mpicc"
       "CXX=mpicxx"
   ];
@@ -40,7 +41,7 @@ stdenv.mkDerivation rec {
 
   meta = {
     description = "OSU Micro-Benchmarks";
-    homepage = http://mvapich.cse.ohio-state.edu/benchmarks/;
+    homepage = "http://mvapich.cse.ohio-state.edu/benchmarks/";
     maintainers = [ ];
     platforms = lib.platforms.all;
   };

pkgs/ovni/default.nix

@@ -13,8 +13,6 @@
 , useMpi ? (stdenv.buildPlatform.canExecute stdenv.hostPlatform)
 }:
 
-with lib;
-
 let
   release = rec {
     version = "1.12.0";
@@ -45,7 +43,7 @@ in
     postPatch = ''
       patchShebangs --build test/
     '';
-    nativeBuildInputs = [ cmake ];
+    nativeBuildInputs = [ cmake ] ++ lib.optionals (useMpi) [ mpi ];
     buildInputs = lib.optionals (useMpi) [ mpi ];
     cmakeBuildType = if (enableDebug) then "Debug" else "Release";
     cmakeFlags = [

pkgs/paraver/default.nix

@@ -60,17 +60,17 @@ stdenv.mkDerivation rec {
   ];
 
   nativeBuildInputs = [
+    autoconf
+    automake
+    autoreconfHook
     wrapGAppsHook
   ];
 
   buildInputs = [
-    autoreconfHook
     boost
     libxml2.dev
     xml2
     wx
-    autoconf
-    automake
     paraverKernel
     openssl.dev
   ];

pkgs/paraver/kernel.nix

@@ -44,14 +44,17 @@ stdenv.mkDerivation rec {
     "--enable-openmp"
   ];
 
-  buildInputs = [
+  nativeBuildInputs = [
     autoreconfHook
-    boost
-    libxml2.dev
-    xml2
     autoconf
     automake
     pkg-config
+  ];
+
+  buildInputs = [
+    boost
+    libxml2.dev
+    xml2
     zlib
   ];
 }

pkgs/paraver/release.nix

@@ -61,13 +61,16 @@ stdenv.mkDerivation rec {
     "--with-wx-config=${wx}/bin/wx-config"
   ];
 
+  nativeBuildInputs = [
+    autoconf
+    automake
+  ];
+
   buildInputs = [
     boost
     xml2
     libxml2.dev
     wx
-    autoconf
-    automake
     openssl.dev
   ];
 

pkgs/sonar/default.nix

@@ -18,8 +18,12 @@ stdenv.mkDerivation rec {
   hardeningDisable = [ "all" ];
   dontStrip = true;
   configureFlags = [ "--with-ovni=${ovni}" ];
-  buildInputs = [
+
+  nativeBuildInputs = [
     autoreconfHook
+  ];
+
+  buildInputs = [
     ovni
     mpi
   ];

pkgs/tampi/default.nix

@@ -8,7 +8,6 @@
 , gnumake
 , boost
 , mpi
-, gcc
 , autoreconfHook
 , enableOvni ? true
 , ovni ? null
@@ -41,20 +40,23 @@ let
     };
   };
   source = if (useGit) then git else release;
-in stdenv.mkDerivation rec {
+in stdenv.mkDerivation {
   pname = "tampi";
   inherit (source) src version;
   enableParallelBuilding = true;
   separateDebugInfo = true;
-  buildInputs = [
+
-    autoreconfHook
+  nativeBuildInputs = [
-    automake
     autoconf
-    libtool
+    automake
+    autoreconfHook
     gnumake
+    libtool
+  ];
+
+  buildInputs = [
     boost
     mpi
-    gcc
   ] ++ optional (enableOvni) ovni;
   configureFlags = optional (enableOvni) "--with-ovni=${ovni}";
   dontDisableStatic = true;