Add tarball script for CI

Allows the same script to be executed by GitLab and Gitea runners.
Use --zstd flag in tar
2024-10-22 16:21:14 +02:00 · 2024-10-21 09:38:59 +02:00 · 2024-10-21 09:25:37 +02:00 · 2024-10-21 09:17:10 +02:00 · 2024-10-21 09:09:34 +02:00 · 2024-10-21 08:41:08 +02:00
14 changed files with 85 additions and 193 deletions
--- a/.gitea/workflows/ci.yaml
+++ b/.gitea/workflows/ci.yaml
@ -0,0 +1,9 @@
+name: CI
+on: push
+
+jobs:
+  build:lagarto-ox-tarball:
+    runs-on: native
+    steps:
+      - uses: https://gitea.com/ScMi1/checkout@v1.4
+      - run: ci/make-tarball.sh
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -5,6 +5,7 @@ build:lagarto-ox:
    - nix
  script:
    - nix develop -L .#lagarto-ox --command fpga/run-remotely.sh fpgalogin1:ci
+
 build:lagarto-ox-tarball:
  stage: build
  when: manual
@ -12,20 +13,4 @@ build:lagarto-ox-tarball:
    - nix
  script:
    - nix build -L .#nixosConfigurations.lagarto-ox.pkgs.fpga-tarball
-    - cd result
-    - out=/ceph/home/gitlab-runner/public_html/nixos-riscv/
-    - fn=$(ls | head -1)
-    - cp $fn $out/$fn
-    - rm -f $out/latest.tar.zst
-    - ln -s $out/$fn $out/latest.tar.zst
-    - cd ..
-    - rm result
-    - |
-      echo "Pinned: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/$fn"
-      echo "Latest: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/latest.tar.zst"
-build:qemu-lagarto-ox:
-  stage: build
-  tags:
-    - nix
-  script:
-    - nix develop -L .#qemu-lagarto-ox --command ./boot.sh
+    - ci/make-tarball.sh result && rm result
--- a/NOTES.md
+++ b/NOTES.md
@ -26,16 +26,3 @@ To dump a ROM without the ELF header:
 $ nix build .\#nixosConfigurations.lagarto-hun.config.system.build.kernel.configfile
 $ nix build .\#nixosConfigurations.lagarto-hun.config.system.build.kernel
 ```
-
-## Interactive build of rvb
-
-```
-hut% cd .../nixos-riscv
-hur% nix develop .#nixosConfigurations.qemu-lagarto-ox.pkgs.rvb
-hut$ mkdir /tmp/foo
-hut$ cd /tmp/foo
-hut$ genericBuild
-...
-error...
-hut$ make ...
-```
--- a/README.md
+++ b/README.md
@ -45,20 +45,6 @@ $ picocom -q -b 115200 $FPGACTL_UART

 It should boot without any user interaction.

-## Lagarto Ox on QEMU
-
-To build the system and boot it on QEMU, you can run the following:
-
-```
-$ nix develop -L '.#qemu-lagarto-ox'
-```
-
-Then, simply run boot.sh to start the QEMU system.
-
-```
-$ ./boot.sh
-```
-
 ## Lagarto Hun

 WIP
--- a/boot.sh
+++ b/boot.sh
@ -27,7 +27,7 @@ fi
 # Create a directory for exchanging data with the VM.
 mkdir -p "$TMPDIR/xchg"

-system="$TOPLEVEL"
+system="$NIXOS_SYSTEM_TOPLEVEL"
 echo "Booting $system"

 cd "$TMPDIR"
@ -35,11 +35,9 @@ cd "$TMPDIR"

 #QEMU_KERNEL_PARAMS="boot.debug1devices"
 #QEMU_KERNEL_PARAMS="boot.trace boot.debug1"
-QEMU_KERNEL_PARAMS=bench2
-#QEMU_KERNEL_PARAMS=debug2

 # Trap the CPU on compressed instructions?
-#TRAP_COMPRESSED="-cpu rv64,c=false"
+TRAP_COMPRESSED="-cpu rv64,c=false"

 CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_jump.elf"
@ -47,10 +45,11 @@ CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #CUSTOM_BIOS="-bios $CDIR/bios.bin"
 #CUSTOM_BIOS="-bios opensbi-1.4-rv-bin/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #DEBUG_GDB="-s -S"
-#DEBUG_CPU="-d cpu_reset"
+DEBUG_CPU="-d cpu_reset"
 #DEBUG_CPU="-d in_asm,cpu,cpu_reset"


+# Start QEMU.
 exec qemu-system-riscv64 \
    $DEBUG_GDB \
    $DEBUG_CPU \
@ -61,16 +60,18 @@ exec qemu-system-riscv64 \
    -machine virt \
    -device virtio-rng-pci \
    $TRAP_COMPRESSED \
-    -netdev user,id=net0,hostfwd=tcp::60022-:22 \
-    -device virtio-net-device,netdev=net0 \
+    $CUSTOM_BIOS \
+    -netdev user,id=net0,hostfwd=tcp::60022-:22 -device virtio-net-device,netdev=net0 \
    -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store \
    -virtfs local,path="${SHARED_DIR:-$TMPDIR/xchg}",security_model=none,mount_tag=shared \
    -virtfs local,path="$TMPDIR"/xchg,security_model=none,mount_tag=xchg \
+    -device virtio-keyboard \
    -drive "file=$NIX_DISK_IMAGE,if=none,id=hd0" \
    -device virtio-blk-device,drive=hd0 \
+    -device loader,addr=0x84000000,file=$system/kernel \
    -kernel $system/kernel \
    -initrd $system/initrd \
-    -append "$(cat $system/kernel-params) init=$system/init console=ttyS0,115200n8 loglevel=7 $QEMU_KERNEL_PARAMS" \
+    -append "$(cat $system/kernel-params) init=$system/init console=ttyS0,115200n8 loglevel=7 $QEMU_KERNEL_PARAMS" 
    $QEMU_OPTS \
    "$@"

--- a/ci/make-tarball.sh
+++ b/ci/make-tarball.sh
@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -e
+
+nix build -L '.#nixosConfigurations.lagarto-ox.pkgs.fpga-tarball'
+
+cd result
+out=/ceph/home/gitlab-runner/public_html/nixos-riscv/
+fn=$(ls | head -1)
+cp "$fn" "$out/$fn"
+rm -f "$out/latest.tar.zst"
+ln -s "$out/$fn" "$out/latest.tar.zst"
+rm result
+
+echo "Pinned: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/$fn"
+echo "Latest: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/latest.tar.zst"
--- a/configuration.nix
+++ b/configuration.nix
@ -42,7 +42,7 @@

  #environment.systemPackages = with pkgs; [ vim gdb neofetch gcc bintools ];
  environment.systemPackages = with pkgs; [
-    rvb-clang riscv-tools stream spec-cpu-mini
+    rvb riscv-tools stream spec-cpu-mini
    config.boot.kernelPackages.perf
  ];

--- a/flake.nix
+++ b/flake.nix
@ -18,7 +18,6 @@
        system = "${system}";
        modules = [
          ./configuration.nix
-          ./lagarto-ox.nix
          ./vm.nix
        ];
      };
@ -55,18 +54,6 @@
          ./no-compressed.nix
        ];
      };
-
-      # QEMU configuration (x86 host) for a FPGA Lagarto Ox CPU virtual machine
-      qemu-lagarto-ox = nixosSystem {
-        specialArgs = { inherit self; };
-        system = "${system}";
-        modules = [
-          ./configuration.nix
-          ./lagarto-ox.nix
-          ./no-compressed.nix
-          ./vm.nix
-        ];
-      };
    };

    # A development shell with QEMU ready to boot the RISC-V system in an x86
@ -118,7 +105,6 @@
    in syspkgs.mkShell rec {
      pname = "lagarto-ox-shell";
      COMMIT = if self ? rev then self.rev else "dirty";
-      REPO = self;
      TOPLEVEL = build.toplevel;
      OPENSBI = syspkgs.opensbi;
      KERNEL = build.kernel;
@ -130,11 +116,9 @@
      GCROOT = mkRoots syspkgs [
        syspkgs.stdenv KERNEL OPENSBI syspkgs.riscv-tools
      ];
-      nativeBuildInputs = with syspkgs; [ rsync ];
      shellHook = ''
        echo "Here are the current system pieces:"
        echo " COMMIT    = $COMMIT"
-        echo " REPO      = $REPO"
        echo " TOPLEVEL  = $TOPLEVEL"
        echo " KERNEL    = $KERNEL"
        echo " OPENSBI   = $OPENSBI"
@ -147,39 +131,6 @@
      '';
    };

-    devShells.x86_64-linux.qemu-lagarto-ox =
-    let
-      nixosconf = self.nixosConfigurations.qemu-lagarto-ox;
-      syspkgs = nixosconf.pkgs;
-      build = nixosconf.config.system.build;
-    in syspkgs.mkShell rec {
-      pname = "lagarto-ox-shell";
-      nativeBuildInputs = with syspkgs; [ qemu_full ];
-      COMMIT = if self ? rev then self.rev else "dirty";
-      TOPLEVEL = build.toplevel;
-      OPENSBI = syspkgs.opensbi;
-      KERNEL = build.kernel;
-      INITRD = build.initialRamdisk;
-      UBOOT_ENV = syspkgs.uboot-env;
-      BITSTREAM = syspkgs.bitstream;
-      BOOTROM = syspkgs.bootrom;
-      GCROOT = mkRoots syspkgs [
-        syspkgs.stdenv KERNEL OPENSBI syspkgs.riscv-tools
-      ];
-      shellHook = ''
-        echo "Here are the current system pieces:"
-        echo " COMMIT    = $COMMIT"
-        echo " TOPLEVEL  = $TOPLEVEL"
-        echo " KERNEL    = $KERNEL"
-        echo " OPENSBI   = $OPENSBI"
-        echo " INITRD    = $INITRD"
-        echo " UBOOT_ENV = $UBOOT_ENV"
-        echo " BITSTREAM = $BITSTREAM"
-        echo " BOOTROM   = $BOOTROM"
-        echo " GCROOT    = $GCROOT"
-      '';
-    };
-
    devShells.x86_64-linux.lagarto-ox-rd =
    let
      nixosconf = self.nixosConfigurations.lagarto-ox;
--- a/lagarto-ox.nix
+++ b/lagarto-ox.nix
@ -401,13 +401,7 @@
    #  };
    #}));

-    blackscholesInput = final.fetchurl {
-        url = "https://gitlab.bsc.es/benchmarks/risc-v-benchmarks/-/wikis/uploads/fd1fa29bc83f24f246da304ecd8ca840/in_1024.input";
-        hash = "sha256-CCS4vMzIn0NxBASyFN57gnZDWWUaqykoTJS106sFLMo=";
-    };
-
    bench2 = final.writeShellScript "bench2" ''
-      echo --------BENCHMARKING PHASE---------
      # Performs minimal FS setup and runs the SPEC benchmark
      mkdir /tmp /bin /root
      mount -t tmpfs tmpfs /tmp
@ -416,46 +410,8 @@
      ln -s $(which sh) /bin/sh
      # Check CPU usage
      vmstat 5 5
-      #bash -x speclaunch
-      #cat /tmp/spec/time.csv
-      # hpc_benchmarks
-      set -x
-      axpy-autovect 800 1000
-      axpy-cblas 800 1000
-      axpy-omp-simd 800 1000
-      axpy-scalar 800 1000
-
-      gemm-autovect 256
-      gemm-omp-simd 256
-      gemm-scalar 256
-
-      somier-autovect 4 32
-      somier-omp-simd 4 32
-      somier-scalar 4 32
-
-      lulesh2.0 -s 4
-
-      jacobi2d_scalar_vanilla 16 2
-
-      hacckernels-autovectorisation 10
-      hacckernels-scalar 10
-
-      # desktop_benchmarks
-
-      blackscholes_scalar 1 ${final.blackscholesInput} /dev/null
-
-      streamcluster_scalar 3 10 16 16 16 10 none output.txt 1
- 
-      canneal_scalar 1 100 300 
-      canneal_vectorial_intrinsics
-
-      pathfinder_serial.exe 64 64 output_64_64.txt
-      pathfinder_vector.exe 64 64 output_64_64.txt
-
-      swaptions_scalar -ns 2 -sm 1024 -nt 1
-
-      # micro_benchmarks (NO WORKING BENCHMARKS FOR NOW...)
-
+      bash -x speclaunch
+      cat /tmp/spec/time.csv
      # Give me a shell at the end
      bash -l
    '';
--- a/overlay.nix
+++ b/overlay.nix
@ -3,7 +3,7 @@ final: prev:
 # Changes to packages from nixpkgs

 {
-  clangEpi = final.callPackage ./pkgs/llvm-epi/default.nix { };
+  clangEpi = final.callPackage ./pkgs/llvm-epi/default.nix { openmp = null; };
  clangEpiUnwrapped = final.callPackage ./pkgs/llvm-epi/clang.nix { };
  stdenvClangEpi = final.stdenv.override { cc = final.buildPackages.clangEpi; allowedRequisites = null; };

--- a/pkgs/llvm-epi/default.nix
+++ b/pkgs/llvm-epi/default.nix
@ -41,8 +41,6 @@ in wrapCCWith {

    echo "--gcc-toolchain=${gcc}" >> $out/nix-support/cc-cflags

-    # Protect against access to clang/clang++ instead of $CC/$CXX
-    wrap clang    $wrapper $ccPath/clang
    wrap clang++  $wrapper $ccPath/clang++
  '';
 }
--- a/pkgs/rvb/Makefile
+++ b/pkgs/rvb/Makefile
@ -2,62 +2,61 @@ include Makefile.in

 HPC_BENCHMARKS_DIRS=\
 	axpy \
-	gemm \
-	hacckernels \
 	jacobi-2d \
-	lulesh \
 	somier \
-	spmv-ellpack \
-	stream \
-#	Require submodules
+#	Require submodules:
+#	fft \
 #	fftp \
+#	Require vehave:
+#	lulesh \
 #	Missing compare_array_double:
+#	gemm \
 #	Broken, not found:
 #	spmv

 DESKTOP_BENCHMARKS_DIRS=\
 	blackscholes \
 	canneal \
+	particlefilter \
 	streamcluster \
-	pathfinder \
-	swaptions \
-#	particlefilter \
+	swaptions
+	#pathfinder

-MICRO_BENCHMARKS_DIRS=\
-	#BuffCopyStrided \
+MICRO_BENCHMARKS_DIRS=
 	#BuffCopyUnit \
+	#BuffCopyStrided \
+	#BuffCopyIndexed \
 	#FpuMicroKernel \
 	#InstrNopBalance \
-	#BuffCopyIndexed \
 	#MemArithBalance \
-	#LatencyVrgather \
+	#LatencyVrgather

 .PHONY: default all clean $(HPC_BENCHMARKS_DIRS) $(MICRO_BENCHMARKS_DIRS)

 all: base

 default: 
-	cd common; make; cd ..
-	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) || exit;)
-	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) || exit;)
-	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) || exit;)
+	@cd common; make; cd ..
+	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir);)
+	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir);)
+	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir);)

 all-types: 
-	cd common; make all; cd ..
-	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all || exit;)
-	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all || exit;)
-	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all || exit;)
+	@cd common; make all; cd ..
+	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all ;)
+	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all ;)
+	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all ;)

 base: 
-	cd common; make all; cd ..
-	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) base || exit;)
-	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) base || exit;)
-	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) base || exit;)
+	@cd common; make all; cd ..
+	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) base ;)
+	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) base ;)
+	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) base ;)

 install: 
-	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) install || exit;)
-	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) install || exit;)
-	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) install || exit;)
+	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) install ;)
+	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) install ;)
+	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) install ;)

 fftp:
 	${MAKE} -C third_party fftw
@ -72,9 +71,9 @@ spmv-ellpack:
 	INDEX64=1 EPI_EXT=07 PATH=${EPI_LLVM_HOME}/bin:${PATH} make

 clean:
-	cd common; make clean; cd ..
-	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;)
-	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;)
-	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;)
-	rm -rf hpc_benchmarks/spmv-ellpack/spmv/build
+	@cd common; make clean; cd ..
+	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;)
+	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;)
+	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;)
+	@rm -rf hpc_benchmarks/spmv-ellpack/spmv/build

--- a/pkgs/rvb/default.nix
+++ b/pkgs/rvb/default.nix
@ -2,11 +2,8 @@
  stdenv
 , blis
 , gitBranch ? "nix-fixes"
-#, gitURL ? "git@gitlab-internal.bsc.es:dbautist/risc-v-benchmarks.git"
-#, gitCommit ? "bfa96504976dc48f6781c2cafb1e07f8d9eb7476"
 , gitURL ? "git@gitlab-internal.bsc.es:rarias/risc-v-benchmarks.git"
 , gitCommit ? "da202d6f818421b72e06c39b5417ad2f8f6ca23c"
-
 }:

 stdenv.mkDerivation rec {
@ -14,7 +11,6 @@ stdenv.mkDerivation rec {
  version = "${src.shortRev}";

  src = builtins.fetchGit {
-    #url = "/home/Computational/dbautist/nixos_riscv_master/nix-fixes-rvb/risc-v-benchmarks"; #change pwd to the local file
    url = gitURL;
    ref = gitBranch;
    rev = gitCommit;
--- a/vm.nix
+++ b/vm.nix
@ -8,9 +8,7 @@
  ];

  # Enable ssh on boot
-  services.openssh.enable = lib.mkForce true;
-  networking.dhcpcd.enable = lib.mkForce true;
-  services.timesyncd.enable = lib.mkForce true;
+  services.openssh.enable = true;

  boot = {
    kernelParams = [
@ -49,8 +47,8 @@
        INET y
        NETWORK_FILESYSTEMS y
        OVERLAY_FS y
-        9P_FS y
-        9P_FS_POSIX_ACL y
+        #"9P_FS" y
+        #"9P_FS_POSIX_ACL" y
        PCI y
        VIRTIO_PCI y
        PCI_HOST_GENERIC y
@ -65,6 +63,16 @@

  nixpkgs.overlays = [
    (final: prev: {
+      qemu = prev.qemu.override {
+        pulseSupport = false;
+        pipewireSupport = false;
+        sdlSupport = false;
+        jackSupport = false;
+        gtkSupport = false;
+        vncSupport = false;
+        smartcardSupport = false;
+      };
+
      uboot-custom = prev.ubootQemuRiscv64Smode.override {
        # Override preboot to set 'bootcmd' directly to the kernel address in RAM
        extraConfig = ''
Author	SHA1	Message	Date
Rodrigo Arias Mallo	5da53c69b8	Add tarball script for CI Some checks failed CI / build:lagarto-ox-tarball (push) Failing after 9s Details Allows the same script to be executed by GitLab and Gitea runners.	2024-10-22 16:21:14 +02:00
Rodrigo Arias Mallo	81bd11999a	Use --zstd flag in tar	2024-10-21 09:38:59 +02:00
Rodrigo Arias Mallo	6cb22575b3	Print URL for tarball	2024-10-21 09:25:37 +02:00
Rodrigo Arias Mallo	53ab72fa20	Switch to ZST for tarball It is faster then Gzip and available in MEEP FPGA cluster.	2024-10-21 09:17:10 +02:00
Rodrigo Arias Mallo	97d477d285	Add tarball to CI and make steps manual	2024-10-21 09:09:34 +02:00
Rodrigo Arias Mallo	20fb2f9194	Add tarball for FPGA	2024-10-21 08:41:08 +02:00