diff --git a/JOURNAL.md b/JOURNAL.md index 136d903..329755a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -5173,3 +5173,109 @@ We can address this problem by modifying the `-C ...` command and just use `-C I can create a symlink to the benchmark directory, so it finds it at `../run_base_test_nix-m64.0000`. + +## 2024-10-10 + +Managed to run some: + + + for srcbench in $SPEC/benchspec/CPU/* + ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s + + name=620.omnetpp_s + + bench=/tmp/spec/620.omnetpp_s + + rm -rf /tmp/spec/620.omnetpp_s + + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/620.omnetpp_s /tmp/spec/620.omnetpp_s + + chmod +w -R /tmp/spec/620.omnetpp_s + + rundir=/tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000 + + sed -i '/^-C/d' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.cmd + + echo '--- Running 620.omnetpp_s for 1 iterations ---' + --- Running 620.omnetpp_s for 1 iterations --- + + cd /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000 + + specinvoke -i 1 -E speccmds.cmd + + awk '/^run [0-9]* elapsed time/{printf \ + "%s\t%s\t%s\t%s\t%s\n", \ + "620.omnetpp_s","test","base",$2,$7}' /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/speccmds.out + + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv + 620.omnetpp_s test base 1 1080.495394000 + + cat /tmp/spec/620.omnetpp_s/run/run_base_test_nix-m64.0000/time.csv + + for srcbench in $SPEC/benchspec/CPU/* + ++ basename /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s + + name=631.deepsjeng_s + + bench=/tmp/spec/631.deepsjeng_s + + rm -rf /tmp/spec/631.deepsjeng_s + + cp -r /nix/store/h2by3qxqpzy5b1zszz7wviphv6vy1pjs-spec-cpu-mini-riscv64-unknown-linux-gnu-1.1.7/benchspec/CPU/631.deepsjeng_s /tmp/spec/631.deepsjeng_s + + chmod +w -R /tmp/spec/631.deepsjeng_s + + rundir=/tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000 + + sed -i '/^-C/d' /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000/speccmds.cmd + + echo '--- Running 631.deepsjeng_s for 1 iterations ---' + --- Running 631.deepsjeng_s for 1 iterations --- + + cd /tmp/spec/631.deepsjeng_s/run/run_base_test_nix-m64.0000 + + specinvoke -i 1 -E speccmds.cmd + [12274.985482] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200002048 not enough memory for the allocation + [12274.998109] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200071680 not enough memory for the allocation + [12275.010615] __vm_enough_memory: pid: 661, comm: deepsjeng_s_bas, bytes: 7200133120 not enough memory for the allocation + bash-5.2# cat /tmp/spec/time.csv + benchmark size tune iter time_s + 600.perlbench_s test base 1 5326.027877000 + 602.gcc_s test base 1 2.515220000 + 605.mcf_s test base 1 1923.514976000 + 620.omnetpp_s test base 1 1080.495394000 + +After commenting `631.deepsjeng_s`: + + [12736.162509] 196608 pages RAM + [12736.165986] 0 pages HighMem/MovableOnly + [12736.171049] 18239 pages reserved + [12736.174892] 4096 pages cma reserved + [12736.179650] Tasks state (memory values in pages): + [12736.184896] [ pid ] uid tgid total_vm rss rss_anon rss_file rss_shmem pgtables_bytes swapents oom_score_adj name + [12736.197219] [ 221] 0 221 1090 126 125 1 0 28672 0 0 bash + [12736.209480] [ 236] 0 236 1057 84 83 1 0 28672 0 0 bash + [12736.221558] [ 642] 0 642 1057 83 83 0 0 28672 0 0 bash + [12736.233637] [ 643] 0 643 572 25 24 1 0 28672 0 0 specinvoke + [12736.246203] [ 644] 0 644 1057 70 69 1 0 32768 0 0 sh + [12736.258037] [ 645] 0 645 131277 105312 105279 1 32 872448 0 0 xz_s_base.nix-m + [12736.271031] oom-kill:constraint=CONSTRAINT_NONE,nodemask=(null),global_oom,task_memcg=/,task=xz_s_base.nix-m,pid=645,uid=0 + [12736.284939] Out of memory: Killed process 645 (xz_s_base.nix-m) total-vm:525108kB, anon-rss:421116kB, file-rss:4kB, shmem-rss:128kB, UID:0 pgtables:852kB oom_score_adj:0 + [12749.985238] oom_reaper: reaped process 645 (xz_s_base.nix-m), now anon-rss:0kB, file-rss:0kB, shmem-rss:0kB + bash-5.2# cat /tmp/spec/time.csv + benchmark size tune iter time_s + 600.perlbench_s test base 1 5338.291831000 + 602.gcc_s test base 1 2.520710000 + 605.mcf_s test base 1 1923.825657000 + 620.omnetpp_s test base 1 1091.905020000 + 641.leela_s test base 1 767.870615000 + 648.exchange2_s test base 1 2815.577807000 + +Running out of memory on `657.xz_s`: + + bash-5.2# free -h + total used free shared buff/cache available + Mem: 696Mi 19Mi 405Mi 255Mi 271Mi 411Mi + Swap: 0B 0B 0B + + bash-5.2# df -h + Filesystem Size Used Avail Use% Mounted on + devtmpfs 35M 0 35M 0% /dev + tmpfs 349M 0 349M 0% /dev/shm + tmpfs 175M 56K 175M 1% /run + tmpfs 349M 0 349M 0% /run/wrappers + /dev/disk/by-label/NIXOS_SD 2.3G 1.9G 275M 88% / + nodev 349M 256M 93M 74% /tmp + + bash-5.2# du -sh /tmp/spec + 256M /tmp/spec + +Let's comment it for now. + +We may want to start running the tests in the CI, so I can read the logs there. +There are some operations we need to do on the FS before running the tests: + + bash-5.2# mkdir /tmp + bash-5.2# mount -t tmpfs nodev /tmp + bash-5.2# mkdir /bin + bash-5.2# mkdir /root + bash-5.2# ln -s $(which sh) /bin/sh + bash-5.2# export TMPDIR=/tmp + bash-5.2# speclaunch + +So, let's prepare a script that performs the mini-init. diff --git a/fpga/run-node.sh b/fpga/run-node.sh index 98da578..05ba16d 100755 --- a/fpga/run-node.sh +++ b/fpga/run-node.sh @@ -20,8 +20,13 @@ set -x # Then perform the boot ./fpgactl -w bitstream.bit -b opensbi.bin -k kernel.bin -i initrd.bin -r rootfs.img -timeout=$((30 * 60)) # Always stop after 30 min -timeout_silent=$((3 * 60)) # Stop if 3 min without output +# Normal timeouts +#timeout=$((30 * 60)) # Always stop after 30 min +#timeout_silent=$((3 * 60)) # Stop if 3 min without output + +# Timeouts for SPEC benchmarks +timeout=$((6 * 60 * 60)) # Always stop after 6 h +timeout_silent=$((2 * 60 * 60)) # Stop if 2 h without output (some benchmarks take 1.6h) # Set dead switch sleep $timeout && killall picocom & diff --git a/lagarto-ox.nix b/lagarto-ox.nix index 47645f6..ca54c7c 100644 --- a/lagarto-ox.nix +++ b/lagarto-ox.nix @@ -66,6 +66,7 @@ src = ./patches/stage-2-init.sh; shellDebug = "${pkgs.bashInteractive}/bin/bash"; bashInteractive = "${pkgs.bashInteractive}"; + bench2 = "${pkgs.bench2}"; shell = "${pkgs.bash}/bin/bash"; inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths; inherit (config.system.nixos) distroName; @@ -400,6 +401,19 @@ # }; #})); + bench2 = final.writeShellScript "bench2" '' + # Performs minimal FS setup and runs the SPEC benchmark + mkdir /tmp /bin /root + mount -t tmpfs tmpfs /tmp + export TMPDIR=/tmp + # We need /bin/sh + ln -s $(which sh) /bin/sh + bash -x speclaunch + cat /tmp/spec/time.csv + # Give me a shell at the end + bash -l + ''; + ox-dtb = prev.stdenv.mkDerivation rec { name = "ox.dtb"; src = ./dts; @@ -512,6 +526,7 @@ # NixOS interesting options: # debug1 enable debug shell in stage 1 # debug2 enable debug shell in stage 2 (custom) + # bench2 run benchmark on stage 2 (custom) # boot.trace enable set -x in stage 1 # boot.tracedebug enable set -x in stage 2 # Ftrace interesting options: @@ -523,7 +538,7 @@ in prev.runCommand "uboot.txt" {} '' cat > $out </dev/hvc0 2>/dev/hvc0" ;; + bench2) + export PATH="@bashInteractive@/bin:@systemConfig@/sw/bin:$PATH" + setsid bash -c "exec @bench2@ /dev/hvc0 2>/dev/hvc0" + ;; esac done diff --git a/pkgs/spec-cpu/launcher.sh b/pkgs/spec-cpu/launcher.sh index 4aa641d..c43add8 100644 --- a/pkgs/spec-cpu/launcher.sh +++ b/pkgs/spec-cpu/launcher.sh @@ -36,6 +36,7 @@ printf 'benchmark\tsize\ttune\titer\ttime_s\n' > "$wd/time.csv" for srcbench in $SPEC/benchspec/CPU/*; do name=$(basename $srcbench) bench="$wd/$name" + bench_out="$wd/$name.csv" rm -rf "$bench" cp -r "$srcbench" "$bench" chmod +w -R "$bench" @@ -53,13 +54,17 @@ for srcbench in $SPEC/benchspec/CPU/*; do awk '/^run [0-9]* elapsed time/{printf \ "%s\t%s\t%s\t%s\t%s\n", \ "'$name'","'$benchsize'","'$benchtune'",$2,$7}' \ - "$rundir/speccmds.out" > "$rundir/time.csv" + "$rundir/speccmds.out" > "$bench_out" - cat "$rundir/time.csv" + cat "$bench_out" # Accumulate in main CSV - cat "$rundir/time.csv" >> "$wd/time.csv" + cat "$bench_out" >> "$wd/time.csv" + + # Erase intermediate files as they occupy RAM. + rm -rf "$bench" done echo "--- RESULTS in $wd/time.csv ---" cat "$wd/time.csv" +echo "---------------------------------------"