From 12237d6a79eca27058644866eb2b6390b646f4f2 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 1 Mar 2024 18:37:05 +0100 Subject: [PATCH] Add fpgactl tool --- fpga/env.sh | 22 ++++ fpga/fpgactl | 288 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 fpga/env.sh create mode 100755 fpga/fpgactl diff --git a/fpga/env.sh b/fpga/env.sh new file mode 100644 index 0000000..8ea4d83 --- /dev/null +++ b/fpga/env.sh @@ -0,0 +1,22 @@ +#!/bin/false + +# Source this file to setup the environment + +INSTALL_PATH=/home/tools +LOAD_BITSTREAM=$INSTALL_PATH/scripts +HOSTNAME=$(hostname) + +export DMA_IP_DRIVERS="$INSTALL_PATH/drivers/$HOSTNAME/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers" + +if [ ! -d $DMA_IP_DRIVERS ]; then + echo "error: DMA_IP_DRIVERS $DMA_IP_DRIVERS directory does not exist" >&2 + return +fi + +export PATH="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/:$PATH" + +if [ -x /opt/Xilinx/Vivado/2020.1/settings64.sh ]; then + source /opt/Xilinx/Vivado/2020.1/settings64.sh +elif [ -x /opt/Xilinx/Vivado/2021.2/settings64.sh ]; then + source /opt/Xilinx/Vivado/2021.2/settings64.sh +fi diff --git a/fpga/fpgactl b/fpga/fpgactl new file mode 100755 index 0000000..d5f478a --- /dev/null +++ b/fpga/fpgactl @@ -0,0 +1,288 @@ +#!/bin/bash +# +# Copyright (c) 2024, Barcelona Supercomputing Center (BSC) +# SPDX-License-Identifier: MIT +# +# Based on a script provided by Xavier Martorell +# Rewritten by Rodrigo Arias Mallo + +# Stop on first error +set -e + +function usage() # {{{ +{ + echo "Loads a bitstream CPU into an FPGA and boots an image" >&2 + echo "Usage: $0 [-s slot] [-r] [-b] [-w bitstream]" >&2 + echo "" >&2 + echo "Loads a bitstream CPU into an FPGA and boots an image" >&2 + echo "" >&2 + echo "Options" >&2 + echo " -s slot Use the given FPGA slot (default 8)" >&2 + echo " -w bitfile Write the bitstream file to the FPGA" >&2 + echo " -r Reload filesystem only" >&2 + echo " -b Boot only" >&2 + echo "Options" >&2 + exit 1 +} # }}} + +function is_module_loaded() # {{{ +{ + lsmod | grep -wq "$1" +} # }}} + +function check_environment() # {{{ +{ + # Check that all drivers are loaded and programs are available + +# if ! lsmod | grep -q qdma_pf; then +# echo 'error: driver qdma_pf not loaded' >&2 +# echo 'tip: sudo insmod /home/tools/drivers/cucu/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers/QDMA/linux-kernel/bin/qdma-pf.ko' >&2 +# exit 1 +# fi +# +# if ! lsmod | grep -q qdma_vf; then +# echo 'error: driver qdma_vf not loaded' >&2 +# echo 'tip: sudo insmod /home/tools/drivers/cucu/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers/QDMA/linux-kernel/bin/qdma-vf.ko' >&2 +# exit 1 +# fi + + if ! command -v dma-ctl &> /dev/null; then + echo "error: dma-ctl not found in PATH" >&2 + exit 1 + fi + + if ! command -v vivado &> /dev/null; then + echo "error: vivado not found in PATH" >&2 + exit 1 + fi +} # }}} + +function create_qdma_queue() # {{{ +{ + pcidir="/sys/bus/pci/devices/0000:08:00.0" + + if [ ! -d "$pcidir/qdma" ]; then + echo "missing pci directory: $pcidir" >&2 + exit 1 + fi + + if [ ! -d "$pcidir/qdma" ]; then + echo "missing qdma directory: $pcidir/qdma" >&2 + exit 1 + fi + + if [ ! -f "$pcidir/qdma/qmax" ]; then + echo "missing qmax file: $pcidir/qdma/qmax" >&2 + exit 1 + fi + + if [ ! -c "/dev/qdma08000-MM-1" ]; then + echo 2 | sudo dd of="$pcidir/qdma/qmax" + + dma-ctl qdma08000 q add mode mm idx 1 dir bi + dma-ctl qdma08000 q start idx 1 dir bi + + sudo chmod go+rw "/dev/qdma08000-MM-1" + sudo chmod go+rw "$pcidir/resource0" + sudo chmod go+rw "$pcidir/resource0_wc" + sudo chmod go+rw "$pcidir/resource2" + sudo chmod go+rw "$pcidir/resource2_wc" + fi + + if [ ! -c "/dev/qdma08000-MM-0" ]; then + dma-ctl qdma08000 q add mode mm idx 0 dir bi + dma-ctl qdma08000 q start idx 0 dir bi + sudo chmod go+rw "/dev/qdma08000-MM-0" + fi + + sleep 2 +} # }}} + +function do_system_reset() # {{{ +{ + # UartBootEn (bit2) + system reset (bit0) + dma-ctl qdma08000 reg write bar 2 0x0 0x0 + sleep 0.2 + # Release system reset, we must wait until the memory is filled with 0s + dma-ctl qdma08000 reg write bar 2 0x0 0x1 + #sleep 5 +} # }}} + +function do_system_release() # {{{ +{ + # Release Ariane's reset + dma-ctl qdma08000 reg write bar 2 0x0 0x3 +} # }}} + +function load_file_in_memory() # {{{ +{ + file="$1" + address="$2" + #strace -f dma-to-device -d /dev/qdma08000-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file" + #strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=progress conv=sync + #strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/qdma08000-MM-1 -c 1024 -s 1024 + ID=08 ./load_image.sh "$file" "$address" +} # }}} + +function do_boot_only() # {{{ +{ + do_system_reset + + ./load_image.sh ${OSBI} $((0x80000000)) && + + rm -f ${OSBI} && + + sleep 2 && + + # #Release Ariane's reset + dma-ctl qdma08000 reg write bar 2 0x0 0x3 && + + sleep 10 && + + echo mount -o nolock -o rw -o retrans=10 10.0.2.2:/media/sda2/scratch/xavim/point /root && + + echo mount -o nolock -o rw -o retrans=10 192.168.0.16:/media/sda2/scratch/xavim/point /root && + + if [ ! -c /dev/qdma08000-MM-0 ] ; then + /home/tools/drivers/`/bin/hostname`/dma_ip_drivers-onic-gamma/create-queue-qdma.sh -2 + fi && + + sleep 2 #&& + + # uncomment to enable eth-over-pcie + #sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 mtu 9000 up + sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 up +} # }}} + +function do_reload_fs() # {{{ +{ + do_system_reset + + #~xavim/LAGARTO_LINUX-4.1/./load_image.sh \ + # /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered \ + # $((0x13ff00000)) && + + load_file_in_memory /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered $((0x13ff00000)) + + sleep 1 + + # Load OpenSBI + kernel + load_file_in_memory ${OSBI} $((0x80000000)) + + rm -f ${OSBI} + + sleep 2 + + do_system_release + create_qdma_queue + +# uncomment to enable eth-over-pcie +# sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 mtu 9000 up +} # }}} + +function upload_bitstream_file() # {{{ +{ + bitfile="$1" + + fpgajtag=$(lsusb -vd 0403: 2>&1 | grep iSerial | awk ' { print $3; }') + if [ -z "$fpgajtag" ]; then + echo "error: cannot find JTAG serial" >&2 + exit 1 + fi + + vivado -nolog -nojournal -mode batch -source load-bitstream.tcl -tclargs "$bitfile" "$fpgajtag" + + echo "Bitstream $bitfile loaded." + killall hw_server +} # }}} + +function unload_modules() # {{{ +{ + drvlist="$1" + + # Unload modules + for mod in $drvlist; do + if is_module_loaded "$mod"; then + sudo rmmod $mod + fi + done + +} # }}} + +function remove_pci_devices() # {{{ +{ + for slot in $(lspci -mm -d 10ee: | awk '{printf "0000:%s\n",$1}'); do + devdir="/sys/bus/pci/devices/$slot" + if [ -d $devdir ]; then + echo 1 | sudo dd "of=$devdir/remove" + fi + done + +} # }}} + +function rescan_pci_devices() # {{{ +{ + echo 1 | sudo dd of=/sys/bus/pci/rescan +} # }}} + +function load_qdma_modules() # {{{ +{ + drv="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/qdma-pf.ko" + hw_buffers="$((0x4FFE0000))" + + if [ ! -r "$drv" ]; then + echo "error: missing $" + exit 1 + fi + + sudo insmod "$drv" "hw_buffers=$hw_buffers" + sleep 4 +} # }}} + +function load_bitstream() # {{{ +{ + bitstream="$1" + + unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable + remove_pci_devices + + upload_bitstream_file "$bitstream" + + rescan_pci_devices + unload_modules "qdma_pf xdma" # qdma_vf not removable + remove_pci_devices + load_qdma_modules + rescan_pci_devices + create_qdma_queue +} # }}} + +slotid="8" +bitstream= +bootloader= +kernelfile= +reloadfs= +resetcpu= + +while getopts "s:w:k:r:b:" opt; do + case "${opt}" in + s) slotid="${OPTARG}" ;; + w) bitstream="${OPTARG}" ;; + b) bootloader="${OPTARG}"; resetcpu=1 ;; + k) kernelfile="${OPTARG}"; resetcpu=1 ;; + r) rootfs="${OPTARG}"; resetcpu=1 ;; + *) usage ;; + esac +done + +set -x + +check_environment + +test "$bitstream" && load_bitstream "$bitstream" +test "$resetcpu" && do_system_reset +test "$bootloader" && load_file_in_memory "${bootloader}" $((0x80000000)) +test "$kernelfile" && load_file_in_memory "${kernelfile}" $((0x84000000)) +test "$rootfs" && load_file_in_memory "${rootfs}" $((0x8c300000)) +test "$resetcpu" && do_system_release + +# vim:ts=2:sw=2:ai:foldmethod=marker:foldlevel=0: