rarias/nixos-riscv
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Prepare CI to run SPEC benchmarks

Browse Source
This commit is contained in:
Rodrigo Arias 2024-10-10 10:28:48 +02:00
parent 9b1f6b4460
commit 78b8c7f981
5 changed files with 141 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
JOURNAL.md
View File

@ -5173,3 +5173,109 @@ We can address this problem by modifying the `-C ...` command and just use `-C
I can create a symlink to the benchmark directory, so it finds it at
`../run_base_test_nix-m64.0000`.
## 2024-10-10
Managed to run some:
+ for srcbench in $SPEC/benchspec/CPU/*
++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s
+ name=620.omnetpp_s
+ bench=/tmp/spec/620.omnetpp_s
+ rm -rf /tmp/spec/620.omnetpp_s
+ cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s /tmp/spec/620.omnetpp_s
+ chmod +w -R /tmp/spec/620.omnetpp_s
+ rundir=/tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000
+ sed -i '/^-C/d' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.cmd
+ echo '--- Running 620.omnetpp_s for 1 iterations ---'
--- Running 620.omnetpp_s for 1 iterations ---
+ cd /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000
+ specinvoke -i 1 -E speccmds.cmd
+ awk '/^run [0-9]* elapsed time/{printf \
"%s\t%s\t%s\t%s\t%s\n", \
"620.omnetpp_s","test","base",$2,$7}' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.out
+ cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv
620.omnetpp_s test base 1 1080.495394000
+ cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv
+ for srcbench in $SPEC/benchspec/CPU/*
++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s
+ name=631.deepsjeng_s
+ bench=/tmp/spec/631.deepsjeng_s
+ rm -rf /tmp/spec/631.deepsjeng_s
+ cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s /tmp/spec/631.deepsjeng_s
+ chmod +w -R /tmp/spec/631.deepsjeng_s
+ rundir=/tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000
+ sed -i '/^-C/d' /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000/speccmds.cmd
+ echo '--- Running 631.deepsjeng_s for 1 iterations ---'
--- Running 631.deepsjeng_s for 1 iterations ---
+ cd /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000
+ specinvoke -i 1 -E speccmds.cmd
[12274.985482] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200002048 not enough memory for the allocation
[12274.998109] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200071680 not enough memory for the allocation
[12275.010615] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200133120 not enough memory for the allocation
bash-5.2# cat /tmp/spec/time.csv
benchmark size tune iter time_s
600.perlbench_s test base 1 5326.027877000
602.gcc_s test base 1 2.515220000
605.mcf_s test base 1 1923.514976000
620.omnetpp_s test base 1 1080.495394000
After commenting `631.deepsjeng_s`:
[12736.162509] 196608 pages RAM
[12736.165986] 0 pages HighMem/MovableOnly
[12736.171049] 18239 pages reserved
[12736.174892] 4096 pages cma reserved
[12736.179650] Tasks state (memory values in pages):
[12736.184896] [ pid ] uid tgid total_vm rss rss_anon rss_file rss_shmem pgtables_bytes swapents oom_score_adj name
[12736.197219] [ 221] 0 221 1090 126 125 1 0 28672 0 0 bash
[12736.209480] [ 236] 0 236 1057 84 83 1 0 28672 0 0 bash
[12736.221558] [ 642] 0 642 1057 83 83 0 0 28672 0 0 bash
[12736.233637] [ 643] 0 643 572 25 24 1 0 28672 0 0 specinvoke
[12736.246203] [ 644] 0 644 1057 70 69 1 0 32768 0 0 sh
[12736.258037] [ 645] 0 645 131277 105312 105279 1 32 872448 0 0 xz_s_base.nix-m
[12736.271031] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom,task_memcg=/,task=xz_s_base.nix-m,pid=645,uid=0
[12736.284939] Out of memory: Killed process 645 (xz_s_base.nix-m) total-vm:525108kB, anon-rss:421116kB, file-rss:4kB, shmem-rss:128kB, UID:0 pgtables:852kB oom_score_adj:0
[12749.985238] oom_reaper: reaped process 645 (xz_s_base.nix-m), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB
bash-5.2# cat /tmp/spec/time.csv
benchmark size tune iter time_s
600.perlbench_s test base 1 5338.291831000
602.gcc_s test base 1 2.520710000
605.mcf_s test base 1 1923.825657000
620.omnetpp_s test base 1 1091.905020000
641.leela_s test base 1 767.870615000
648.exchange2_s test base 1 2815.577807000
Running out of memory on `657.xz_s`:
bash-5.2# free -h
total used free shared buff/cache available
Mem: 696Mi 19Mi 405Mi 255Mi 271Mi 411Mi
Swap: 0B 0B 0B
bash-5.2# df -h
Filesystem Size Used Avail Use% Mounted on
devtmpfs 35M 0 35M 0% /dev
tmpfs 349M 0 349M 0% /dev/shm
tmpfs 175M 56K 175M 1% /run
tmpfs 349M 0 349M 0% /run/wrappers
/dev/disk/by-label/NIXOS_SD 2.3G 1.9G 275M 88% /
nodev 349M 256M 93M 74% /tmp
bash-5.2# du -sh /tmp/spec
256M /tmp/spec
Let's comment it for now.
We may want to start running the tests in the CI, so I can read the logs there.
There are some operations we need to do on the FS before running the tests:
bash-5.2# mkdir /tmp
bash-5.2# mount -t tmpfs nodev /tmp
bash-5.2# mkdir /bin
bash-5.2# mkdir /root
bash-5.2# ln -s $(which sh) /bin/sh
bash-5.2# export TMPDIR=/tmp
bash-5.2# speclaunch
So, let's prepare a script that performs the mini-init.

9
fpga/run-node.sh
View File

@ -20,8 +20,13 @@ set -x
# Then perform the boot
./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img
timeout=$((30 * 60)) # Always stop after 30 min
timeout_silent=$((3 * 60)) # Stop if 3 min without output
# Normal timeouts
#timeout=$((30 * 60)) # Always stop after 30 min
#timeout_silent=$((3 * 60)) # Stop if 3 min without output
# Timeouts for SPEC benchmarks
timeout=$((6 * 60 * 60)) # Always stop after 6 h
timeout_silent=$((2 * 60 * 60)) # Stop if 2 h without output (some benchmarks take 1.6h)
# Set dead switch
sleep $timeout && killall picocom &

17
lagarto-ox.nix
View File

@ -66,6 +66,7 @@
src = ./patches/stage-2-init.sh;
shellDebug = "${pkgs.bashInteractive}/bin/bash";
bashInteractive = "${pkgs.bashInteractive}";
bench2 = "${pkgs.bench2}";
shell = "${pkgs.bash}/bin/bash";
inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths;
inherit (config.system.nixos) distroName;
@ -400,6 +401,19 @@
# };
#}));
bench2 = final.writeShellScript "bench2" ''
# Performs minimal FS setup and runs the SPEC benchmark
mkdir /tmp /bin /root
mount -t tmpfs tmpfs /tmp
export TMPDIR=/tmp
# We need /bin/sh
ln -s $(which sh) /bin/sh
bash -x speclaunch
cat /tmp/spec/time.csv
# Give me a shell at the end
bash -l
'';
ox-dtb = prev.stdenv.mkDerivation rec {
name = "ox.dtb";
src = ./dts;
@ -512,6 +526,7 @@
# NixOS interesting options:
# debug1 enable debug shell in stage 1
# debug2 enable debug shell in stage 2 (custom)
# bench2 run benchmark on stage 2 (custom)
# boot.trace enable set -x in stage 1
# boot.tracedebug enable set -x in stage 2
# Ftrace interesting options:
@ -523,7 +538,7 @@
in prev.runCommand "uboot.txt" {} ''
cat > $out <<EOF
xtrace=yes
bootargs=root=/dev/ram0 loglevel=7 rw earlycon=sbi console=hvc0 debug1 debug2 init=${init}
bootargs=root=/dev/ram0 loglevel=7 rw earlycon=sbi console=hvc0 bench2 init=${init}
ramdisk_size=$(stat --format %s $(readlink -f ${initrd}))
bootcmd=fdt print; booti \''${kernel_addr_r} \''${ramdisk_addr_r}:\''${ramdisk_size} \''${fdtcontroladdr}
EOF

4
patches/stage-2-init.sh
View File

@ -21,6 +21,10 @@ if [ "${IN_NIXOS_SYSTEMD_STAGE1:-}" != true ]; then
export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH"
setsid bash -c "exec bash -l </dev/hvc0 >/dev/hvc0 2>/dev/hvc0"
;;
bench2)
export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH"
setsid bash -c "exec @bench2@ </dev/hvc0 >/dev/hvc0 2>/dev/hvc0"
;;
esac
done

11
pkgs/spec-cpu/launcher.sh
View File

@ -36,6 +36,7 @@ printf 'benchmark\tsize\ttune\titer\ttime_s\n' > "$wd/time.csv"
for srcbench in $SPEC/benchspec/CPU/*; do
name=$(basename $srcbench)
bench="$wd/$name"
bench_out="$wd/$name.csv"
rm -rf "$bench"
cp -r "$srcbench" "$bench"
chmod +w -R "$bench"
@ -53,13 +54,17 @@ for srcbench in $SPEC/benchspec/CPU/*; do
awk '/^run [0-9]* elapsed time/{printf \
"%s\t%s\t%s\t%s\t%s\n", \
"'$name'","'$benchsize'","'$benchtune'",$2,$7}' \
"$rundir/speccmds.out" > "$rundir/time.csv"
"$rundir/speccmds.out" > "$bench_out"
cat "$rundir/time.csv"
cat "$bench_out"
# Accumulate in main CSV
cat "$rundir/time.csv" >> "$wd/time.csv"
cat "$bench_out" >> "$wd/time.csv"
# Erase intermediate files as they occupy RAM.
rm -rf "$bench"
done
echo "--- RESULTS in $wd/time.csv ---"
cat "$wd/time.csv"
echo "---------------------------------------"