Enable ovni for cross compilation

Reviewed-by: Aleix Boné <abonerib@bsc.es>
Tested-by: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>

.gitea/workflows/ci.yaml

@@ -12,4 +12,9 @@ jobs:
     runs-on: native
     steps:
       - uses: https://gitea.com/ScMi1/checkout@v1.4
-      - run: nix build -L --no-link --print-out-paths .#bsc-ci.all
+      - run: nix build -L --no-link --print-out-paths .#bsc.ci.all
+  build:cross:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: nix build -L --no-link --print-out-paths .#bsc.ci.cross

flake.nix

@@ -42,9 +42,7 @@ in
     # full nixpkgs with our overlay applied
     legacyPackages.${system} = pkgs;
 
-    hydraJobs = {
+    hydraJobs = self.legacyPackages.${system}.bsc.hydraJobs;
-      inherit (self.legacyPackages.${system}.bsc-ci) tests pkgs cross;
-    };
 
     # propagate nixpkgs lib, so we can do bscpkgs.lib
     inherit (nixpkgs) lib;

m/common/base.nix

@@ -11,6 +11,7 @@
     ./base/hw.nix
     ./base/net.nix
     ./base/nix.nix
+    ./base/sys-devices.nix
     ./base/ntp.nix
     ./base/rev.nix
     ./base/ssh.nix

m/common/base/sys-devices.nix

@@ -0,0 +1,9 @@
+{
+  nix.settings.system-features = [ "sys-devices" ];
+
+  programs.nix-required-mounts.enable = true;
+  programs.nix-required-mounts.allowedPatterns.sys-devices.paths = [
+    "/sys/devices/system/cpu"
+    "/sys/devices/system/node"
+  ];
+}

overlay.nix

@@ -62,7 +62,7 @@ let
   };
 
   tests = rec {
-    #hwloc = callPackage ./test/bugs/hwloc.nix { }; # Broken, no /sys
+    hwloc = callPackage ./test/bugs/hwloc.nix { };
     #sigsegv = callPackage ./test/reproducers/sigsegv.nix { };
     hello-c = callPackage ./test/compilers/hello-c.nix { };
     hello-cpp = callPackage ./test/compilers/hello-cpp.nix { };
@@ -94,12 +94,18 @@ let
     };
   };
 
-  pkgs = filterAttrs (_: isDerivation) bscPkgs;
+  # For now, only build toplevel packages in CI/Hydra
+  pkgsTopLevel = filterAttrs (_: isDerivation) bscPkgs;
 
-  crossTargets = [ "riscv64" ];
+  # Native build in that platform doesn't imply cross build works
-  cross = prev.lib.genAttrs crossTargets (target:
+  canCrossCompile = platform: pkg:
-    final.pkgsCross.${target}.bsc-ci.pkgs
+    (isDerivation pkg) &&
-  );
+    # Must be defined explicitly
+    (pkg.meta.cross or false) &&
+    (meta.availableOn platform pkg);
+
+  # For now only RISC-V
+  crossSet = { riscv64 = final.pkgsCross.riscv64.bsc.pkgsTopLevel; };
 
   buildList = name: paths:
     final.runCommandLocal name { } ''
@@ -113,22 +119,31 @@ let
       printf '%s\n' $deps >$out
     '';
 
-  crossList = builtins.mapAttrs (t: v: buildList t (builtins.attrValues v)) cross;
+  pkgsList = buildList "ci-pkgs" (builtins.attrValues pkgsTopLevel);
-
+  testsList = buildList "ci-tests" (collect isDerivation tests);
-  pkgsList = buildList "ci-pkgs" (builtins.attrValues pkgs);
+  allList = buildList' "ci-all" [ pkgsList testsList ];
-  testList = buildList "ci-tests" (collect isDerivation tests);
+  # For now only RISC-V
-
+  crossList = buildList "ci-cross"
-  all = buildList' "ci-all" [ pkgsList testList ];
+    (filter
+      (canCrossCompile final.pkgsCross.riscv64.stdenv.hostPlatform)
+        (builtins.attrValues crossSet.riscv64));
 
 in bscPkgs // {
-  # Prevent accidental usage of bsc attribute
+  # Prevent accidental usage of bsc-ci attribute
-  bsc = throw "the bsc attribute is deprecated, packages are now in the root";
+  bsc-ci = throw "the bsc-ci attribute is deprecated, use bsc.ci";
 
   # Internal for our CI tests
-  bsc-ci = {
+  bsc = {
-    inherit pkgs pkgsList;
+    # CI targets for nix build
-    inherit tests testList;
+    ci = { pkgs = pkgsList; tests = testsList; all = allList; cross = crossList; };
-    inherit cross crossList;
+
-    inherit all;
+    # Direct access to package sets
+    tests = tests;
+    pkgs = bscPkgs;
+    pkgsTopLevel = pkgsTopLevel;
+    cross = crossSet;
+
+    # Hydra uses attribute sets of pkgs
+    hydraJobs = { tests = tests; pkgs = pkgsTopLevel; cross = crossSet; };
   };
 }

pkgs/extrae/default.nix

@@ -20,6 +20,7 @@
 #, python3Packages
 , installShellFiles
 , symlinkJoin
+, enablePapi ? stdenv.hostPlatform == stdenv.buildPlatform # Disabled when cross-compiling
 }:
 
 let
@@ -87,7 +88,7 @@ stdenv.mkDerivation rec {
       --enable-sampling
       --with-unwind=${libunwind.dev}
       --with-xml-prefix=${libxml2.dev}
-      --with-papi=${papi}
+      ${lib.optionalString enablePapi "--with-papi=${papi}"}
       ${if (mpi != null) then ''--with-mpi=${mpi}''
         else ''--without-mpi''}
       --without-dyninst)

pkgs/lmbench/default.nix

@@ -35,6 +35,8 @@ stdenv.mkDerivation rec {
       CFLAGS=-Wno-implicit-int
       CPPFLAGS=-I${libtirpc.dev}/include/tirpc
       LDFLAGS=-ltirpc
+      CC=$CC
+      AR=$AR
     )
   '';
 

pkgs/nanos6/default.nix

@@ -16,6 +16,7 @@
 , jemallocNanos6 ? null
 , cachelineBytes ? 64
 , enableGlibcxxDebug ? false
+, enablePapi ? stdenv.hostPlatform == stdenv.buildPlatform # Disabled when cross-compiling
 , useGit ? false
 , gitUrl ? "ssh://git@bscpm04.bsc.es/nanos6/nanos6"
 , gitBranch ? "master"
@@ -47,6 +48,8 @@ let
   };
 
   source = if (useGit) then git else release;
+
+  isCross = stdenv.hostPlatform != stdenv.buildPlatform;
 in
   stdenv.mkDerivation (source // {
     pname = "nanos6";
@@ -71,9 +74,13 @@ in
       "--disable-all-instrumentations"
       "--enable-ovni-instrumentation"
       "--with-ovni=${ovni}"
+      "--with-boost=${boost.dev}"
     ] ++
       (optional enableJemalloc "--with-jemalloc=${jemallocNanos6}") ++
-      (optional enableGlibcxxDebug "CXXFLAGS=-D_GLIBCXX_DEBUG");
+      (optional enableGlibcxxDebug "CXXFLAGS=-D_GLIBCXX_DEBUG") ++
+      # Most nanos6 api symbols are resolved at runtime, so prefer
+      # ifunc by default
+      (optional isCross "--with-symbol-resolution=ifunc");
 
     postConfigure = lib.optionalString (!enableDebug) ''
       # Disable debug
@@ -97,16 +104,14 @@ in
       # TODO: papi_version is needed for configure:
       # ./configure: line 27378: papi_version: command not found
       # This probably breaks cross-compilation
-      papi
+    ] ++ lib.optionals enablePapi [ papi ];
-    ];
 
     buildInputs = [
       boost
       numactl
       hwloc
-      papi
       ovni
-    ];
+    ] ++ lib.optionals enablePapi [ papi ];
 
     # Create a script that sets NANOS6_HOME
     postInstall = ''

pkgs/nosv/default.nix

@@ -7,7 +7,7 @@
 , numactl
 , hwloc
 , papi
-, enablePapi ? true
+, enablePapi ? stdenv.hostPlatform == stdenv.buildPlatform # Disabled when cross-compiling
 , cacheline ? 64 # bits
 , ovni ? null
 , useGit ? false

pkgs/ovni/default.nix

@@ -55,4 +55,7 @@ in
     doCheck = true;
     checkTarget = "test";
     hardeningDisable = [ "all" ];
+    meta = {
+      cross = true;
+    };
   }

test/bugs/hwloc.nix

@@ -6,6 +6,7 @@
 
 stdenv.mkDerivation {
   name = "hwloc-test";
+  requiredSystemFeatures = [ "sys-devices" ];
 
   src = ./.;
 
@@ -14,7 +15,7 @@ stdenv.mkDerivation {
   buildPhase = ''
     ls -l /sys
     gcc -lhwloc hwloc.c -o hwloc
-    strace ./hwloc
+    strace ./hwloc > $out
   '';
 
 }

test/compilers/clang-openmp-ld.nix

@@ -23,9 +23,8 @@ in stdenv.mkDerivation {
   dontUnpack = true;
   dontConfigure = true;
 
-  # nOS-V requires access to /sys/devices to request NUMA information. It will
+  # nOS-V requires access to /sys/devices to request NUMA information
-  # fail to run otherwise, so we disable the sandbox for this test.
+  requiredSystemFeatures = [ "sys-devices" ];
-  __noChroot = true;
 
   buildInputs = [ openmp ];
 

test/compilers/clang-openmp-nosv.nix

@@ -36,9 +36,8 @@ in stdenv.mkDerivation {
   dontUnpack = true;
   dontConfigure = true;
 
-  # nOS-V requires access to /sys/devices to request NUMA information. It will
+  # nOS-V requires access to /sys/devices to request NUMA information
-  # fail to run otherwise, so we disable the sandbox for this test.
+  requiredSystemFeatures = [ "sys-devices" ];
-  __noChroot = true;
 
   buildInputs = [ nosv ];
 

test/compilers/clang-openmp.nix

@@ -24,9 +24,8 @@ in stdenv.mkDerivation {
   dontUnpack = true;
   dontConfigure = true;
 
-  # nOS-V requires access to /sys/devices to request NUMA information. It will
+  # nOS-V requires access to /sys/devices to request NUMA information
-  # fail to run otherwise, so we disable the sandbox for this test.
+  requiredSystemFeatures = [ "sys-devices" ];
-  __noChroot = true;
 
   buildPhase = ''
     set -x

test/compilers/ompss2.nix

@@ -25,9 +25,10 @@ stdenv.mkDerivation rec {
   hardeningDisable = [ "all" ];
   #NIX_DEBUG = 1;
   buildInputs = [ ]; #strace gdb;
-  # NODES requires access to /sys/devices to request NUMA information. It will
+
-  # fail to run otherwise, so we disable the sandbox for this test.
+  # NODES requires access to /sys/devices to request NUMA information
-  __noChroot = true;
+  requiredSystemFeatures = [ "sys-devices" ];
+
   buildPhase = ''
     set -x
     #$CC -v