Prepare CI to run SPEC benchmarks

JOURNAL.md

@@ -5173,3 +5173,109 @@ We can address this problem by modifying the `-C ...` command and just use `-C
 
 I can create a symlink to the benchmark directory, so it finds it at
 `../run_base_test_nix-m64.0000`.
+
+## 2024-10-10
+
+Managed to run some:
+
+    + for srcbench in $SPEC/benchspec/CPU/*
+    ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s
+    + name=620.omnetpp_s
+    + bench=/tmp/spec/620.omnetpp_s
+    + rm -rf /tmp/spec/620.omnetpp_s
+    + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s /tmp/spec/620.omnetpp_s
+    + chmod +w -R /tmp/spec/620.omnetpp_s
+    + rundir=/tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000
+    + sed -i '/^-C/d' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.cmd
+    + echo '--- Running 620.omnetpp_s for 1 iterations ---'
+    --- Running 620.omnetpp_s for 1 iterations ---
+    + cd /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000
+    + specinvoke -i 1 -E speccmds.cmd
+    + awk '/^run [0-9]* elapsed time/{printf \
+        "%s\t%s\t%s\t%s\t%s\n", \
+        "620.omnetpp_s","test","base",$2,$7}' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.out
+    + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv
+    620.omnetpp_s   test    base    1       1080.495394000
+    + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv
+    + for srcbench in $SPEC/benchspec/CPU/*
+    ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s
+    + name=631.deepsjeng_s
+    + bench=/tmp/spec/631.deepsjeng_s
+    + rm -rf /tmp/spec/631.deepsjeng_s
+    + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s /tmp/spec/631.deepsjeng_s
+    + chmod +w -R /tmp/spec/631.deepsjeng_s
+    + rundir=/tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000
+    + sed -i '/^-C/d' /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000/speccmds.cmd
+    + echo '--- Running 631.deepsjeng_s for 1 iterations ---'
+    --- Running 631.deepsjeng_s for 1 iterations ---
+    + cd /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000
+    + specinvoke -i 1 -E speccmds.cmd
+    [12274.985482] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200002048 not enough memory for the allocation
+    [12274.998109] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200071680 not enough memory for the allocation
+    [12275.010615] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200133120 not enough memory for the allocation
+    bash-5.2# cat /tmp/spec/time.csv
+    benchmark       size    tune    iter    time_s
+    600.perlbench_s test    base    1       5326.027877000
+    602.gcc_s       test    base    1       2.515220000
+    605.mcf_s       test    base    1       1923.514976000
+    620.omnetpp_s   test    base    1       1080.495394000
+
+After commenting `631.deepsjeng_s`:
+
+    [12736.162509] 196608 pages RAM
+    [12736.165986] 0 pages HighMem/MovableOnly
+    [12736.171049] 18239 pages reserved
+    [12736.174892] 4096 pages cma reserved
+    [12736.179650] Tasks state (memory values in pages):
+    [12736.184896] [  pid  ]   uid  tgid total_vm      rss rss_anon rss_file rss_shmem pgtables_bytes swapents oom_score_adj name
+    [12736.197219] [    221]     0   221     1090      126      125        1         0    28672        0             0 bash
+    [12736.209480] [    236]     0   236     1057       84       83        1         0    28672        0             0 bash
+    [12736.221558] [    642]     0   642     1057       83       83        0         0    28672        0             0 bash
+    [12736.233637] [    643]     0   643      572       25       24        1         0    28672        0             0 specinvoke
+    [12736.246203] [    644]     0   644     1057       70       69        1         0    32768        0             0 sh
+    [12736.258037] [    645]     0   645   131277   105312   105279        1        32   872448        0             0 xz_s_base.nix-m
+    [12736.271031] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom,task_memcg=/,task=xz_s_base.nix-m,pid=645,uid=0
+    [12736.284939] Out of memory: Killed process 645 (xz_s_base.nix-m) total-vm:525108kB, anon-rss:421116kB, file-rss:4kB, shmem-rss:128kB, UID:0 pgtables:852kB oom_score_adj:0
+    [12749.985238] oom_reaper: reaped process 645 (xz_s_base.nix-m), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB
+    bash-5.2# cat /tmp/spec/time.csv
+    benchmark       size    tune    iter    time_s
+    600.perlbench_s test    base    1       5338.291831000
+    602.gcc_s       test    base    1       2.520710000
+    605.mcf_s       test    base    1       1923.825657000
+    620.omnetpp_s   test    base    1       1091.905020000
+    641.leela_s     test    base    1       767.870615000
+    648.exchange2_s test    base    1       2815.577807000
+
+Running out of memory on `657.xz_s`:
+
+    bash-5.2# free -h
+                   total        used        free      shared  buff/cache   available
+    Mem:           696Mi        19Mi       405Mi       255Mi       271Mi       411Mi
+    Swap:             0B          0B          0B
+
+    bash-5.2# df -h
+    Filesystem                   Size  Used Avail Use% Mounted on
+    devtmpfs                      35M     0   35M   0% /dev
+    tmpfs                        349M     0  349M   0% /dev/shm
+    tmpfs                        175M   56K  175M   1% /run
+    tmpfs                        349M     0  349M   0% /run/wrappers
+    /dev/disk/by-label/NIXOS_SD  2.3G  1.9G  275M  88% /
+    nodev                        349M  256M   93M  74% /tmp
+
+    bash-5.2# du -sh /tmp/spec
+    256M    /tmp/spec
+
+Let's comment it for now.
+
+We may want to start running the tests in the CI, so I can read the logs there.
+There are some operations we need to do on the FS before running the tests:
+
+    bash-5.2# mkdir /tmp
+    bash-5.2# mount -t tmpfs nodev /tmp
+    bash-5.2# mkdir /bin
+    bash-5.2# mkdir /root
+    bash-5.2# ln -s $(which sh) /bin/sh
+    bash-5.2# export  TMPDIR=/tmp
+    bash-5.2# speclaunch
+
+So, let's prepare a script that performs the mini-init.

fpga/run-node.sh

@@ -20,8 +20,13 @@ set -x
 # Then perform the boot
 ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img
 
-timeout=$((30 * 60)) # Always stop after 30 min
+# Normal timeouts
-timeout_silent=$((3 * 60)) # Stop if 3 min without output
+#timeout=$((30 * 60)) # Always stop after 30 min
+#timeout_silent=$((3 * 60)) # Stop if 3 min without output
+
+# Timeouts for SPEC benchmarks
+timeout=$((6 * 60 * 60)) # Always stop after 6 h
+timeout_silent=$((2 * 60 * 60)) # Stop if 2 h without output (some benchmarks take 1.6h)
 
 # Set dead switch
 sleep $timeout && killall picocom &

lagarto-ox.nix

@@ -66,6 +66,7 @@
       src = ./patches/stage-2-init.sh;
       shellDebug = "${pkgs.bashInteractive}/bin/bash";
       bashInteractive = "${pkgs.bashInteractive}";
+      bench2 = "${pkgs.bench2}";
       shell = "${pkgs.bash}/bin/bash";
       inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths;
       inherit (config.system.nixos) distroName;
@@ -400,6 +401,19 @@
     #  };
     #}));
 
+    bench2 = final.writeShellScript "bench2" ''
+      # Performs minimal FS setup and runs the SPEC benchmark
+      mkdir /tmp /bin /root
+      mount -t tmpfs tmpfs /tmp
+      export  TMPDIR=/tmp
+      # We need /bin/sh
+      ln -s $(which sh) /bin/sh
+      bash -x speclaunch
+      cat /tmp/spec/time.csv
+      # Give me a shell at the end
+      bash -l
+    '';
+
     ox-dtb = prev.stdenv.mkDerivation rec {
       name = "ox.dtb";
       src = ./dts;
@@ -512,6 +526,7 @@
       # NixOS interesting options:
       #    debug1                   enable debug shell in stage 1
       #    debug2                   enable debug shell in stage 2 (custom)
+      #    bench2                   run benchmark on stage 2 (custom)
       #    boot.trace               enable set -x in stage 1
       #    boot.tracedebug          enable set -x in stage 2
       # Ftrace interesting options:
@@ -523,7 +538,7 @@
     in prev.runCommand "uboot.txt" {} ''
       cat > $out <<EOF
       xtrace=yes
-      bootargs=root=/dev/ram0 loglevel=7 rw earlycon=sbi console=hvc0 debug1 debug2 init=${init}
+      bootargs=root=/dev/ram0 loglevel=7 rw earlycon=sbi console=hvc0 bench2 init=${init}
       ramdisk_size=$(stat --format %s $(readlink -f ${initrd}))
       bootcmd=fdt print; booti \''${kernel_addr_r} \''${ramdisk_addr_r}:\''${ramdisk_size} \''${fdtcontroladdr}
       EOF

patches/stage-2-init.sh

@@ -21,6 +21,10 @@ if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then
                 export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH"
                 setsid bash -c "exec bash -l </dev/hvc0 >/dev/hvc0 2>/dev/hvc0"
                 ;;
+            bench2)
+                export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH"
+                setsid bash -c "exec @bench2@ </dev/hvc0 >/dev/hvc0 2>/dev/hvc0"
+                ;;
         esac
     done
 

pkgs/spec-cpu/launcher.sh

@@ -36,6 +36,7 @@ printf 'benchmark\tsize\ttune\titer\ttime_s\n' > "$wd/time.csv"
 for srcbench in $SPEC/benchspec/CPU/*; do
   name=$(basename $srcbench)
   bench="$wd/$name"
+  bench_out="$wd/$name.csv"
   rm -rf "$bench"
   cp -r "$srcbench" "$bench"
   chmod +w -R "$bench"
@@ -53,13 +54,17 @@ for srcbench in $SPEC/benchspec/CPU/*; do
   awk '/^run [0-9]* elapsed time/{printf \
     "%s\t%s\t%s\t%s\t%s\n", \
     "'$name'","'$benchsize'","'$benchtune'",$2,$7}' \
-    "$rundir/speccmds.out" > "$rundir/time.csv"
+    "$rundir/speccmds.out" > "$bench_out"
 
-  cat "$rundir/time.csv"
+  cat "$bench_out"
 
   # Accumulate in main CSV
-  cat "$rundir/time.csv" >> "$wd/time.csv"
+  cat "$bench_out" >> "$wd/time.csv"
+
+  # Erase intermediate files as they occupy RAM.
+  rm -rf "$bench"
 done
 
 echo "--- RESULTS in $wd/time.csv ---"
 cat "$wd/time.csv"
+echo "---------------------------------------"