Add rsync to shell dependencies

It is faster then Gzip and available in MEEP FPGA cluster.

.gitlab-ci.yml

@@ -1,6 +1,31 @@
 build:lagarto-ox:
   stage: build
+  when: manual
   tags:
     - nix
   script:
     - nix develop -L .#lagarto-ox --command fpga/run-remotely.sh fpgalogin1:ci
+build:lagarto-ox-tarball:
+  stage: build
+  when: manual
+  tags:
+    - nix
+  script:
+    - nix build -L .#nixosConfigurations.lagarto-ox.pkgs.fpga-tarball
+    - cd result
+    - out=/ceph/home/gitlab-runner/public_html/nixos-riscv/
+    - fn=$(ls | head -1)
+    - cp $fn $out/$fn
+    - rm -f $out/latest.tar.zst
+    - ln -s $out/$fn $out/latest.tar.zst
+    - cd ..
+    - rm result
+    - |
+      echo "Pinned: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/$fn"
+      echo "Latest: https://jungle.bsc.es/~gitlab-runner/nixos-riscv/latest.tar.zst"
+build:qemu-lagarto-ox:
+  stage: build
+  tags:
+    - nix
+  script:
+    - nix develop -L .#qemu-lagarto-ox --command ./boot.sh

NOTES.md

@@ -26,3 +26,16 @@ To dump a ROM without the ELF header:
 $ nix build .\#nixosConfigurations.lagarto-hun.config.system.build.kernel.configfile
 $ nix build .\#nixosConfigurations.lagarto-hun.config.system.build.kernel
 ```
+
+## Interactive build of rvb
+
+```
+hut% cd .../nixos-riscv
+hur% nix develop .#nixosConfigurations.qemu-lagarto-ox.pkgs.rvb
+hut$ mkdir /tmp/foo
+hut$ cd /tmp/foo
+hut$ genericBuild
+...
+error...
+hut$ make ...
+```

README.md

@@ -45,6 +45,20 @@ $ picocom -q -b 115200 $FPGACTL_UART
 
 It should boot without any user interaction.
 
+## Lagarto Ox on QEMU
+
+To build the system and boot it on QEMU, you can run the following:
+
+```
+$ nix develop -L '.#qemu-lagarto-ox'
+```
+
+Then, simply run boot.sh to start the QEMU system.
+
+```
+$ ./boot.sh
+```
+
 ## Lagarto Hun
 
 WIP

boot.sh

@@ -27,7 +27,7 @@ fi
 # Create a directory for exchanging data with the VM.
 mkdir -p "$TMPDIR/xchg"
 
-system="$NIXOS_SYSTEM_TOPLEVEL"
+system="$TOPLEVEL"
 echo "Booting $system"
 
 cd "$TMPDIR"
@@ -35,9 +35,11 @@ cd "$TMPDIR"
 
 #QEMU_KERNEL_PARAMS="boot.debug1devices"
 #QEMU_KERNEL_PARAMS="boot.trace boot.debug1"
+QEMU_KERNEL_PARAMS=bench2
+#QEMU_KERNEL_PARAMS=debug2
 
 # Trap the CPU on compressed instructions?
-TRAP_COMPRESSED="-cpu rv64,c=false"
+#TRAP_COMPRESSED="-cpu rv64,c=false"
 
 CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_jump.elf"
@@ -45,11 +47,10 @@ CUSTOM_BIOS="-bios $OPENSBI/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #CUSTOM_BIOS="-bios $CDIR/bios.bin"
 #CUSTOM_BIOS="-bios opensbi-1.4-rv-bin/share/opensbi/lp64/generic/firmware/fw_payload.bin"
 #DEBUG_GDB="-s -S"
-DEBUG_CPU="-d cpu_reset"
+#DEBUG_CPU="-d cpu_reset"
 #DEBUG_CPU="-d in_asm,cpu,cpu_reset"
 
 
-# Start QEMU.
 exec qemu-system-riscv64 \
     $DEBUG_GDB \
     $DEBUG_CPU \
@@ -60,18 +61,16 @@ exec qemu-system-riscv64 \
     -machine virt \
     -device virtio-rng-pci \
     $TRAP_COMPRESSED \
-    $CUSTOM_BIOS \
+    -netdev user,id=net0,hostfwd=tcp::60022-:22 \
-    -netdev user,id=net0,hostfwd=tcp::60022-:22 -device virtio-net-device,netdev=net0 \
+    -device virtio-net-device,netdev=net0 \
     -virtfs local,path=/nix/store,security_model=none,mount_tag=nix-store \
     -virtfs local,path="${SHARED_DIR:-$TMPDIR/xchg}",security_model=none,mount_tag=shared \
     -virtfs local,path="$TMPDIR"/xchg,security_model=none,mount_tag=xchg \
-    -device virtio-keyboard \
     -drive "file=$NIX_DISK_IMAGE,if=none,id=hd0" \
     -device virtio-blk-device,drive=hd0 \
-    -device loader,addr=0x84000000,file=$system/kernel \
     -kernel $system/kernel \
     -initrd $system/initrd \
-    -append "$(cat $system/kernel-params) init=$system/init console=ttyS0,115200n8 loglevel=7 $QEMU_KERNEL_PARAMS" 
+    -append "$(cat $system/kernel-params) init=$system/init console=ttyS0,115200n8 loglevel=7 $QEMU_KERNEL_PARAMS" \
     $QEMU_OPTS \
     "$@"
 

flake.nix

@@ -18,6 +18,7 @@
         system = "${system}";
         modules = [
           ./configuration.nix
+          ./lagarto-ox.nix
           ./vm.nix
         ];
       };
@@ -54,6 +55,18 @@
           ./no-compressed.nix
         ];
       };
+
+      # QEMU configuration (x86 host) for a FPGA Lagarto Ox CPU virtual machine
+      qemu-lagarto-ox = nixosSystem {
+        specialArgs = { inherit self; };
+        system = "${system}";
+        modules = [
+          ./configuration.nix
+          ./lagarto-ox.nix
+          ./no-compressed.nix
+          ./vm.nix
+        ];
+      };
     };
 
     # A development shell with QEMU ready to boot the RISC-V system in an x86
@@ -105,6 +118,7 @@
     in syspkgs.mkShell rec {
       pname = "lagarto-ox-shell";
       COMMIT = if self ? rev then self.rev else "dirty";
+      REPO = self;
       TOPLEVEL = build.toplevel;
       OPENSBI = syspkgs.opensbi;
       KERNEL = build.kernel;
@@ -116,6 +130,42 @@
       GCROOT = mkRoots syspkgs [
         syspkgs.stdenv KERNEL OPENSBI syspkgs.riscv-tools
       ];
+      nativeBuildInputs = with syspkgs; [ rsync ];
+      shellHook = ''
+        echo "Here are the current system pieces:"
+        echo " COMMIT    = $COMMIT"
+        echo " REPO      = $REPO"
+        echo " TOPLEVEL  = $TOPLEVEL"
+        echo " KERNEL    = $KERNEL"
+        echo " OPENSBI   = $OPENSBI"
+        echo " INITRD    = $INITRD"
+        echo " ROOTFS    = $ROOTFS"
+        echo " UBOOT_ENV = $UBOOT_ENV"
+        echo " BITSTREAM = $BITSTREAM"
+        echo " BOOTROM   = $BOOTROM"
+        echo " GCROOT    = $GCROOT"
+      '';
+    };
+
+    devShells.x86_64-linux.qemu-lagarto-ox =
+    let
+      nixosconf = self.nixosConfigurations.qemu-lagarto-ox;
+      syspkgs = nixosconf.pkgs;
+      build = nixosconf.config.system.build;
+    in syspkgs.mkShell rec {
+      pname = "lagarto-ox-shell";
+      nativeBuildInputs = with syspkgs; [ qemu_full ];
+      COMMIT = if self ? rev then self.rev else "dirty";
+      TOPLEVEL = build.toplevel;
+      OPENSBI = syspkgs.opensbi;
+      KERNEL = build.kernel;
+      INITRD = build.initialRamdisk;
+      UBOOT_ENV = syspkgs.uboot-env;
+      BITSTREAM = syspkgs.bitstream;
+      BOOTROM = syspkgs.bootrom;
+      GCROOT = mkRoots syspkgs [
+        syspkgs.stdenv KERNEL OPENSBI syspkgs.riscv-tools
+      ];
       shellHook = ''
         echo "Here are the current system pieces:"
         echo " COMMIT    = $COMMIT"
@@ -123,7 +173,6 @@
         echo " KERNEL    = $KERNEL"
         echo " OPENSBI   = $OPENSBI"
         echo " INITRD    = $INITRD"
-        echo " ROOTFS    = $ROOTFS"
         echo " UBOOT_ENV = $UBOOT_ENV"
         echo " BITSTREAM = $BITSTREAM"
         echo " BOOTROM   = $BOOTROM"

lagarto-ox.nix

@@ -401,7 +401,13 @@
     #  };
     #}));
 
+    blackscholesInput = final.fetchurl {
+        url = "https://gitlab.bsc.es/benchmarks/risc-v-benchmarks/-/wikis/uploads/fd1fa29bc83f24f246da304ecd8ca840/in_1024.input";
+        hash = "sha256-CCS4vMzIn0NxBASyFN57gnZDWWUaqykoTJS106sFLMo=";
+    };
+
     bench2 = final.writeShellScript "bench2" ''
+      echo --------BENCHMARKING PHASE---------
       # Performs minimal FS setup and runs the SPEC benchmark
       mkdir /tmp /bin /root
       mount -t tmpfs tmpfs /tmp
@@ -410,8 +416,38 @@
       ln -s $(which sh) /bin/sh
       # Check CPU usage
       vmstat 5 5
-      bash -x speclaunch
+      #bash -x speclaunch
-      cat /tmp/spec/time.csv
+      #cat /tmp/spec/time.csv
+      # hpc_benchmarks
+      set -x
+      axpy-autovect 800 1000
+      axpy-cblas 800 1000
+      axpy-omp-simd 800 1000
+      axpy-scalar 800 1000
+
+      gemm-autovect 256
+      gemm-omp-simd 256
+      gemm-scalar 256
+
+      somier-autovect 4 32
+      somier-omp-simd 4 32
+      somier-scalar 4 32
+
+      lulesh2.0 -s 4
+
+      jacobi2d_scalar_vanilla 16 2
+      
+      # desktop_benchmarks
+      
+      #blackscholes
+      #streamcluster
+
+      # micro_benchmarks (NO WORKING BENCHMARKS FOR NOW...)
+
+      blackscholes_scalar 1 ${final.blackscholesInput} /dev/null
+      streamcluster_scalar 3 10 16 16 16 10 none output.txt 1
+
+
       # Give me a shell at the end
       bash -l
     '';
@@ -546,6 +582,91 @@
       EOF
     '';
 
+    # A tarball with all the parts to boot NixOS in a FPGA
+    fpga-tarball = let
+      build = config.system.build;
+    in prev.stdenv.mkDerivation rec {
+      name = "nixos-riscv";
+      src = ./fpga;
+
+      COMMIT    = if self ? rev then self.rev
+        else throw ("Refusing to build tarball from a dirty Git tree!");
+
+      TOPLEVEL  = build.toplevel;
+      OPENSBI   = final.opensbi;
+      KERNEL    = build.kernel;
+      INITRD    = build.initialRamdisk;
+      ROOTFS    = build.sdImage;
+      UBOOT_ENV = final.uboot-env;
+      BITSTREAM = final.bitstream;
+      BOOTROM   = final.bootrom;
+      bitName = builtins.baseNameOf BITSTREAM;
+
+      unpackPhase = ''
+        mkdir -p src/nixos-riscv
+        ln -s $src fpga
+        fpga/upload.sh src/nixos-riscv
+        chmod +w -R src
+        sourceRoot=src
+      '';
+
+      buildPhase = ''
+        cat > nixos-riscv/README.txt <<EOF
+        -----------------------------------------
+        NixOS image for Lagarto Ox in a U55C FPGA
+        -----------------------------------------
+
+        Author: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
+
+        These files allow you to boot NixOS on Lagarto Ox. They correspond to
+        this commit:
+        
+        https://pm.bsc.es/gitlab/rarias/nixos-riscv/-/commit/${COMMIT}
+
+        Here are the pieces used by this system:
+
+          COMMIT    = ${COMMIT}
+          TOPLEVEL  = ${TOPLEVEL}
+          OPENSBI   = ${OPENSBI}
+          KERNEL    = ${KERNEL}
+          INITRD    = ${INITRD}
+          ROOTFS    = ${ROOTFS}
+          UBOOT_ENV = ${UBOOT_ENV}
+          BITSTREAM = ${BITSTREAM}
+          BOOTROM   = ${BOOTROM}
+        
+        Copy these files into a MEEP FPGA node, then allocate a FPGA node and
+        run from this directory (don't forget the dot at the end):
+        
+        $ ./run-node.sh .
+        
+        That would flash the bitstream and NixOS and boot the whole system
+        until the login, and exit after some inactivity period. You should see
+        this message near the end:
+        
+             __________________
+            < Welcome to NixOS >
+             ------------------
+                       ^__^
+                       (oo)_______
+                       (__)       )/
+                           ||----w |
+                           ||     ||
+        
+            If you can read this message then then you have
+            successfully booted NixOS into the login shell.
+
+        EOF
+      '';
+
+      installPhase = ''
+        mkdir -p $out
+        tar --zstd -cvf $out/${COMMIT}.tar.zst nixos-riscv
+      '';
+
+      nativeBuildInputs = with final.buildPackages; [ rsync zstd ];
+    };
+
     opensbi = prev.opensbi.overrideAttrs (old: rec {
       #version = "1.4";
       version = "1.5";

pkgs/llvm-epi/default.nix

@@ -41,6 +41,8 @@ in wrapCCWith {
 
     echo "--gcc-toolchain=${gcc}" >> $out/nix-support/cc-cflags
 
+    # Protect against access to clang/clang++ instead of $CC/$CXX
+    wrap clang    $wrapper $ccPath/clang
     wrap clang++  $wrapper $ccPath/clang++
   '';
 }

pkgs/rvb/Makefile

@@ -4,59 +4,60 @@ HPC_BENCHMARKS_DIRS=\
 	axpy \
 	jacobi-2d \
 	somier \
-#	Require submodules:
+	gemm \
-#	fft \
+	lulesh \
+#	hacckernels \
+#	fftv \
+#	spmv-ellpack \
+#	stream \
+#	Require submodules
 #	fftp \
-#	Require vehave:
-#	lulesh \
 #	Missing compare_array_double:
-#	gemm \
 #	Broken, not found:
 #	spmv
 
 DESKTOP_BENCHMARKS_DIRS=\
 	blackscholes \
 	canneal \
-	particlefilter \
 	streamcluster \
-	swaptions
+#	pathfinder \
-	#pathfinder
+#	particlefilter \
+#	swaptions \
 
-MICRO_BENCHMARKS_DIRS=
+MICRO_BENCHMARKS_DIRS=\
-	#BuffCopyUnit \
 	#BuffCopyStrided \
 	#BuffCopyIndexed \
 	#FpuMicroKernel \
-	#InstrNopBalance \
 	#MemArithBalance \
-	#LatencyVrgather
+	#LatencyVrgather \
+	# BuffCopyUnit 
 
 .PHONY: default all clean $(HPC_BENCHMARKS_DIRS) $(MICRO_BENCHMARKS_DIRS)
 
 all: base
 
 default: 
-	@cd common; make; cd ..
+	cd common; make; cd ..
-	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir);)
+	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir);)
-	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir);)
+	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir);)
-	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir);)
+	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir);)
 
 all-types: 
-	@cd common; make all; cd ..
+	cd common; make all; cd ..
-	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all ;)
+	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) all ;)
-	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all ;)
+	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) all ;)
-	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all ;)
+	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) all ;)
 
 base: 
-	@cd common; make all; cd ..
+	cd common; make all; cd ..
-	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) base ;)
+	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) base ;)
-	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) base ;)
+	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) base ;)
-	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) base ;)
+	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) base ;)
 
 install: 
-	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) install ;)
+	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) install ;)
-	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) install ;)
+	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) install ;)
-	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) install ;)
+	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) install ;)
 
 fftp:
 	${MAKE} -C third_party fftw
@@ -71,9 +72,9 @@ spmv-ellpack:
 	INDEX64=1 EPI_EXT=07 PATH=${EPI_LLVM_HOME}/bin:${PATH} make
 
 clean:
-	@cd common; make clean; cd ..
+	cd common; make clean; cd ..
-	@$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;)
+	$(foreach dir,$(HPC_BENCHMARKS_DIRS),${MAKE} -C hpc_benchmarks/$(dir) clean ;)
-	@$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;)
+	$(foreach dir,$(DESKTOP_BENCHMARKS_DIRS),${MAKE} -C desktop_benchmarks/$(dir) clean ;)
-	@$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;)
+	$(foreach dir,$(MICRO_BENCHMARKS_DIRS),${MAKE} -C micro_benchmarks/$(dir) clean ;)
-	@rm -rf hpc_benchmarks/spmv-ellpack/spmv/build
+	rm -rf hpc_benchmarks/spmv-ellpack/spmv/build
 

pkgs/rvb/default.nix

@@ -2,8 +2,11 @@
   stdenv
 , blis
 , gitBranch ? "nix-fixes"
+#, gitURL ? "git@gitlab-internal.bsc.es:dbautist/risc-v-benchmarks.git"
+#, gitCommit ? "bfa96504976dc48f6781c2cafb1e07f8d9eb7476"
 , gitURL ? "git@gitlab-internal.bsc.es:rarias/risc-v-benchmarks.git"
 , gitCommit ? "da202d6f818421b72e06c39b5417ad2f8f6ca23c"
+
 }:
 
 stdenv.mkDerivation rec {
@@ -11,6 +14,7 @@ stdenv.mkDerivation rec {
   version = "${src.shortRev}";
 
   src = builtins.fetchGit {
+    #url = "/home/Computational/dbautist/nixos_riscv_master/nix-fixes-rvb/risc-v-benchmarks"; #change pwd to the local file
     url = gitURL;
     ref = gitBranch;
     rev = gitCommit;

vm.nix

@@ -8,7 +8,9 @@
   ];
 
   # Enable ssh on boot
-  services.openssh.enable = true;
+  services.openssh.enable = lib.mkForce true;
+  networking.dhcpcd.enable = lib.mkForce true;
+  services.timesyncd.enable = lib.mkForce true;
 
   boot = {
     kernelParams = [
@@ -47,8 +49,8 @@
         INET y
         NETWORK_FILESYSTEMS y
         OVERLAY_FS y
-        #"9P_FS" y
+        9P_FS y
-        #"9P_FS_POSIX_ACL" y
+        9P_FS_POSIX_ACL y
         PCI y
         VIRTIO_PCI y
         PCI_HOST_GENERIC y
@@ -63,16 +65,6 @@
 
   nixpkgs.overlays = [
     (final: prev: {
-      qemu = prev.qemu.override {
-        pulseSupport = false;
-        pipewireSupport = false;
-        sdlSupport = false;
-        jackSupport = false;
-        gtkSupport = false;
-        vncSupport = false;
-        smartcardSupport = false;
-      };
-
       uboot-custom = prev.ubootQemuRiscv64Smode.override {
         # Override preboot to set 'bootcmd' directly to the kernel address in RAM
         extraConfig = ''