Update fpga tools for MEEP

This commit is contained in:
Rodrigo Arias 2024-07-01 09:57:05 +02:00
parent 2cefcef63d
commit 975a1c0e87
2 changed files with 242 additions and 84 deletions

View File

@ -1,21 +1,53 @@
#!/bin/bash
# Source this file to setup the environment
INSTALL_PATH=/home/tools
LOAD_BITSTREAM=$INSTALL_PATH/scripts
HOSTNAME=$(hostname)
function setup_cucu()
{
INSTALL_PATH=/home/tools
LOAD_BITSTREAM=$INSTALL_PATH/scripts
export DMA_IP_DRIVERS="$INSTALL_PATH/drivers/$hostname/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers"
if [ ! -d $DMA_IP_DRIVERS ]; then
echo "error: DMA_IP_DRIVERS $DMA_IP_DRIVERS directory does not exist" >&2
return
fi
export PATH="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/:$PATH"
if [ -x /opt/Xilinx/Vivado/2020.1/settings64.sh ]; then
source /opt/Xilinx/Vivado/2020.1/settings64.sh
elif [ -x /opt/Xilinx/Vivado/2021.2/settings64.sh ]; then
source /opt/Xilinx/Vivado/2021.2/settings64.sh
fi
}
export DMA_IP_DRIVERS="$INSTALL_PATH/drivers/$HOSTNAME/dma_ip_drivers-onic-gamma/xilinx_pcie_drivers"
function setup_meep()
{
. /nfs/apps/XILINX/xilinx_22_env.sh
export PATH="$PATH:/apps/QDMA/meep-ionic/2022.1.4.4/linux-kernel/bin/"
if [ ! -d $DMA_IP_DRIVERS ]; then
echo "error: DMA_IP_DRIVERS $DMA_IP_DRIVERS directory does not exist" >&2
return
fi
# Select the first FPGA in the node
local line=$(grep fpgan /etc/motd | head -1 | tr -d ' ')
export FPGACTL_PCIDEV=$(echo "$line" | awk -F'|' '{print $5}')
export FPGACTL_SERIAL=$(echo "$line" | awk -F'|' '{print $4}')
export FPGACTL_UART=$(echo "$line" | awk -F'|' '{print "/dev/"$7}')
export PATH="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/:$PATH"
# Setup mappings
if [ -x /opt/Xilinx/Vivado/2020.1/settings64.sh ]; then
source /opt/Xilinx/Vivado/2020.1/settings64.sh
elif [ -x /opt/Xilinx/Vivado/2021.2/settings64.sh ]; then
source /opt/Xilinx/Vivado/2021.2/settings64.sh
fi
# Delta between where we load in the dma device and RAM
local delta_addr=-0x60000000
export FPGACTL_BOOTLOADER_ADDR=$((0x80000000+$delta_addr))
export FPGACTL_KERNEL_ADDR=$((0x84000000+$delta_addr))
export FPGACTL_INITRD_ADDR=$((0x8c300000+$delta_addr))
export FPGACTL_ROOTFS_ADDR=$((0x1bff00000+$delta_addr))
}
hostname=$(hostname)
case "$hostname" in
cucu) setup_cucu ;;
fpgan*) setup_meep ;;
*) echo "ERROR: unknown host $hostname";;
esac

View File

@ -27,9 +27,7 @@ function check_environment() # {{{
} # }}}
function create_qdma_queue() # {{{
{
pcidir="/sys/bus/pci/devices/0000:08:00.0"
if [ ! -d "$pcidir/qdma" ]; then
if [ ! -d "$pcidir" ]; then
echo "missing pci directory: $pcidir" >&2
exit 1
fi
@ -44,48 +42,72 @@ function create_qdma_queue() # {{{
exit 1
fi
if [ ! -c "/dev/qdma08000-MM-1" ]; then
echo 2 | sudo dd of="$pcidir/qdma/qmax"
dma-ctl qdma08000 q add mode mm idx 1 dir bi
dma-ctl qdma08000 q start idx 1 dir bi
sudo chmod go+rw "/dev/qdma08000-MM-1"
sudo chmod go+rw "$pcidir/resource0"
sudo chmod go+rw "$pcidir/resource0_wc"
sudo chmod go+rw "$pcidir/resource2"
sudo chmod go+rw "$pcidir/resource2_wc"
if [ ! -r "$pcidir/qdma/qmax" ]; then
echo "cannot read qmax file: $pcidir/qdma/qmax" >&2
exit 1
fi
if [ ! -c "/dev/qdma08000-MM-0" ]; then
dma-ctl qdma08000 q add mode mm idx 0 dir bi
dma-ctl qdma08000 q start idx 0 dir bi
sudo chmod go+rw "/dev/qdma08000-MM-0"
# There should be two queues
local qmax=$(cat "$pcidir/qdma/qmax")
if [ "$qmax" != 2 ]; then
if [ -w "$pcidir/qdma/qmax" ]; then
echo 2 | dd of="$pcidir/qdma/qmax"
else
echo 2 | sudo dd of="$pcidir/qdma/qmax"
fi
fi
# Create the two queues if they don't exist
if [ ! -c "/dev/${qdmadev}-MM-1" ]; then
dma-ctl "${qdmadev}" q add mode mm idx 1 dir bi
dma-ctl "${qdmadev}" q start idx 1 dir bi
fi
if [ ! -c "/dev/${qdmadev}-MM-0" ]; then
dma-ctl "${qdmadev}" q add mode mm idx 0 dir bi
dma-ctl "${qdmadev}" q start idx 0 dir bi
fi
# Wait for udev to process the new devices
udevadm settle
# Ensure we have write access. On some clusters this is automatically done
# by udev rules, on others we are expect to use sudo.
for f in /dev/${qdmadev}-MM-{0,1} ${pcidir}/resource{0,0_wc,2,2_wc}; do
test -w "$f" || sudo chmod go+rw "$f"
done
sleep 2
} # }}}
function do_system_reset() # {{{
function do_cpu_reset() # {{{
{
# UartBootEn (bit2) + system reset (bit0)
dma-ctl qdma08000 reg write bar 2 0x0 0x0 > /dev/null
sleep 0.2
# Release system reset, we must wait until the memory is filled with 0s
dma-ctl qdma08000 reg write bar 2 0x0 0x1 > /dev/null
#sleep 5
if [ "$model" == "hun" ]; then
# UartBootEn (bit2) + system reset (bit0)
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x0
sleep 0.2
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x1
elif [ "$model" == "ox" ]; then
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x0
sleep 0.2
fi
} # }}}
function do_system_release() # {{{
function do_cpu_release() # {{{
{
# Release Ariane's reset
dma-ctl qdma08000 reg write bar 2 0x0 0x3 > /dev/null
if [ "$model" == "hun" ]; then
# Release Ariane's reset
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x3
elif [ "$model" == "ox" ]; then
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x1
sleep 1
fi
} # }}}
function copy_by_dma() # {{{
{
ifile="$1"
address="$2"
ofile="/dev/qdma08000-MM-1"
bs=$((8*1024*1024)) # 8 MiB
ofile="/dev/${qdmadev}-MM-1"
#bs=$((8*1024*1024)) # 8 MiB
bs=$((1*1024*1024)) # 1 MiB
total_size=$(stat --format "%s" "$ifile")
@ -101,6 +123,7 @@ function copy_by_dma() # {{{
dd if="$ifile" skip=$skip count=1 bs=$bs of="$ofile" seek=$dst oflag=seek_bytes status=none
let skip=$skip+1
done
#dma-to-device -d "$ofile" -s "$total_size" -a "$address" -f "$ifile"
} # }}}
function load_file_in_memory() # {{{
@ -112,21 +135,21 @@ function load_file_in_memory() # {{{
total_size=$(stat --format "%s" "$file")
# Previous tests...
#strace -f dma-to-device -d /dev/qdma08000-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file"
#strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=progress conv=sync
#strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/qdma08000-MM-1 -c 1024 -s 1024
#strace -f dma-to-device -d /dev/${qdmadev}-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file"
#strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/${qdmadev}-MM-1 status=progress conv=sync
#strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/${qdmadev}-MM-1 -c 1024 -s 1024
#ID=08 ./load_image.sh "$file" "$address"
# Now dd seems to work fine, but I will leave this as fallback:
copy_by_dma "$file" "$address"
#dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=none
#dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/${qdmadev}-MM-1 status=none
printf "loaded '%s' at 0x%x with size %d\n" "$file" "$address" "$total_size" >&2
} # }}}
function do_boot_only() # {{{
{
do_system_reset
do_cpu_reset
./load_image.sh ${OSBI} $((0x80000000)) &&
@ -135,7 +158,7 @@ function do_boot_only() # {{{
sleep 2 &&
# #Release Ariane's reset
dma-ctl qdma08000 reg write bar 2 0x0 0x3 &&
dma-ctl "${qdmadev}" reg write bar 2 0x0 0x3 &&
sleep 10 &&
@ -143,7 +166,7 @@ function do_boot_only() # {{{
echo mount -o nolock -o rw -o retrans=10 192.168.0.16:/media/sda2/scratch/xavim/point /root &&
if [ ! -c /dev/qdma08000-MM-0 ] ; then
if [ ! -c "/dev/${qdmadev}-MM-0" ] ; then
/home/tools/drivers/`/bin/hostname`/dma_ip_drivers-onic-gamma/create-queue-qdma.sh -2
fi &&
@ -155,7 +178,7 @@ function do_boot_only() # {{{
} # }}}
function do_reload_fs() # {{{
{
do_system_reset
do_cpu_reset
#~xavim/LAGARTO_LINUX-4.1/./load_image.sh \
# /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered \
@ -172,7 +195,7 @@ function do_reload_fs() # {{{
sleep 2
do_system_release
do_cpu_release
create_qdma_queue
# uncomment to enable eth-over-pcie
@ -180,23 +203,25 @@ function do_reload_fs() # {{{
} # }}}
function upload_bitstream_file() # {{{
{
bitfile="$1"
if [ -z "$jtagserial" ]; then
>&2 echo "JTAG serial required"
usage
fi
fpgajtag=$(lsusb -vd 0403: 2>&1 | grep iSerial | awk ' { print $3; }')
if [ -z "$fpgajtag" ]; then
echo "error: cannot find JTAG serial" >&2
exit 1
if [ -z "$bitstream" ]; then
>&2 echo "bitstream file required"
usage
fi
script=$(mktemp vivado-XXXXXXXXXX.tcl)
cat > "$script" <<EOF
open_hw_manager
connect_hw_server -url localhost:3121
current_hw_target "localhost:3121/xilinx_tcf/Xilinx/${fpgajtag}A"
current_hw_target "localhost:3121/xilinx_tcf/Xilinx/${jtagserial}A"
open_hw_target
set dev [lindex [get_hw_devices] 0]
current_hw_device \$dev
set_property PROGRAM.FILE ${bitfile} \$dev
set_property PROGRAM.FILE ${bitstream} \$dev
program_hw_devices \$dev
exit
EOF
@ -213,7 +238,7 @@ function unload_modules() # {{{
# Unload modules
for mod in $drvlist; do
if is_module_loaded "$mod"; then
sudo rmmod $mod
sudo rmmod $mod
fi
done
@ -223,14 +248,22 @@ function remove_pci_devices() # {{{
for slot in $(lspci -mm -d 10ee: | awk '{printf "0000:%s\n",$1}'); do
devdir="/sys/bus/pci/devices/$slot"
if [ -d $devdir ]; then
echo 1 | sudo dd "of=$devdir/remove"
if [ -w "$devdir/remove" ]; then
echo 1 | dd "of=$devdir/remove"
else
echo 1 | sudo dd "of=$devdir/remove"
fi
fi
done
} # }}}
function rescan_pci_devices() # {{{
{
echo 1 | sudo dd of=/sys/bus/pci/rescan
if [ -w /sys/bus/pci/rescan ]; then
echo 1 | dd of=/sys/bus/pci/rescan
else
echo 1 | sudo dd of=/sys/bus/pci/rescan
fi
} # }}}
function load_qdma_modules() # {{{
{
@ -245,21 +278,91 @@ function load_qdma_modules() # {{{
sudo insmod "$drv" "hw_buffers=$hw_buffers"
sleep 4
} # }}}
function select_pcidev() # {{{
{
if [ -z "$pcidev" ]; then
>&2 echo -e "error: missing PCI device (hint: lspci -d 10ee:902f)"
usage
fi
# Ensure it is ok
local matches=$(lspci -s "$pcidev")
if [ -z "$matches" ]; then
>&2 echo "no match for PCI device '$pcidev'"
exit 1
fi
local n="$(echo "$matches" | wc -l)"
if [ "$n" -gt 1 ]; then
>&2 echo "multiple matches for PCI device '$pcidev'"
exit 1
fi
# Fill the PCI device with the domain
local fulldev=$(lspci -s "$pcidev" -D | cut -d' ' -f1)
pcidir="/sys/bus/pci/devices/$fulldev"
if [ ! -d "$pcidir" ]; then
>&2 echo "cannot find PCI dir: $pcidir"
exit 1
fi
# Set the PCI device to the full device
pcidev="$fulldev"
# Find slot
slot=$(lspci -s "$pcidev" -vm | grep PhySlot | cut -f2)
if [ -z "$slot" ]; then
>&2 echo "cannot find physical slot for PCI '$pcidev'"
exit 1
fi
local devid=$(echo "$pcidev" | cut -d: -f2- | tr -d ':.')
qdmadev="qdma${devid}"
} # }}}
function preload_hook() #{{{
{
case "$hostname" in
cucu)
unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable
remove_pci_devices
;;
fpgan*)
;;
*)
echo "hostname $hostname not known"
exit 1
;;
esac
} #}}}
function postload_hook() #{{{
{
rescan_pci_devices
case "$hostname" in
cucu)
unload_modules "qdma_pf xdma" # qdma_vf not removable
remove_pci_devices
load_qdma_modules
rescan_pci_devices
;;
fpgan*)
;;
*)
echo "hostname $hostname not known"
exit 1
;;
esac
create_qdma_queue
} #}}}
function load_bitstream() # {{{
{
bitstream="$1"
unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable
remove_pci_devices
upload_bitstream_file "$bitstream"
rescan_pci_devices
unload_modules "qdma_pf xdma" # qdma_vf not removable
remove_pci_devices
load_qdma_modules
rescan_pci_devices
create_qdma_queue
preload_hook
upload_bitstream_file
postload_hook
} # }}}
bitstream=
@ -269,32 +372,46 @@ initrd=
rootfs=
resetcpu=
verbose=
pcidev=
model=ox
# Internal
slot=
pcidir=
qdmadev=
bootloader_addr=0x80000000
kernel_addr=0x84000000
initrd_addr=0x8c300000
rootfs_addr=0x140000000
bootloader_addr="${FPGA_BOOTLOADER_ADDR:-0x80000000}"
kernel_addr="${FPGA_KERNEL_ADDR:-0x84000000}"
initrd_addr="${FPGA_INITRD_ADDR:-0x8c300000}"
rootfs_addr="${FPGA_ROOTFS_ADDR:-0x140000000}"
hostname="${hostname:-$(hostname)}"
echo "hostname=$hostname"
function usage()
{
echo "" >&2
echo "Usage: $0 [-v] [-w bitstream] [-b bootloader] [-k kernel] [-i initrd]" >&2
echo "Usage: $0 [-p pcidev] [-v] [-w bitstream] [-j serial] [-b bootloader] [-k kernel] [-i initrd]" >&2
echo "" >&2
echo "First writes the bitstream if given. Then loads the rest of files" >&2
echo "into memory and restarts the CPU." >&2
echo "" >&2
echo "Options" >&2
echo " -p pcidev Select PCI device (same format as lspci -s)." >&2
echo " Read from \$FPGACTL_PCIDEV if not given." >&2
echo " -w bitstream Write the bitstream file to the FPGA" >&2
echo " -j serial JTAG serial (can be found by lsusb -v)" >&2
echo " Read from \$FPGACTL_SERIAL if not given." >&2
echo " -b bootloader Load the bootloader file in $bootloader_addr" >&2
echo " -k kernel Load the kernel file in $kernel_addr" >&2
echo " -i initrd Load the initrd file in $initrd_addr" >&2
echo " -r rootfs Load the rootfs file in $rootfs_addr" >&2
echo " -m model CPU model: Either 'hun' or 'ox' (default ox)" >&2
echo " -v Be verbose" >&2
echo "" >&2
exit 1
}
while getopts "hvw:b:k:i:r:" opt; do
while getopts "hvw:b:k:i:r:p:j:m:" opt; do
case "${opt}" in
v) verbose=1 ;;
w) bitstream="${OPTARG}" ;;
@ -302,21 +419,30 @@ while getopts "hvw:b:k:i:r:" opt; do
k) kernel="${OPTARG}"; resetcpu=1 ;;
i) initrd="${OPTARG}"; resetcpu=1 ;;
r) rootfs="${OPTARG}"; resetcpu=1 ;;
p) pcidev="${OPTARG}" ;;
j) jtagserial="${OPTARG}" ;;
m) model="${OPTARG}" ;;
h) usage ;;
*) usage ;;
esac
done
jtagserial="${jtagserial:-$FPGACTL_SERIAL}"
pcidev="${pcidev:-$FPGACTL_PCIDEV}"
test "$verbose" && set -x
check_environment
select_pcidev
test "$bitstream" && load_bitstream "$bitstream"
test "$resetcpu" && do_system_reset
test "$resetcpu" && do_cpu_reset
test "$bootloader" && load_file_in_memory "$bootloader" $bootloader_addr
test "$kernel" && load_file_in_memory "$kernel" $kernel_addr
test "$initrd" && load_file_in_memory "$initrd" $initrd_addr
test "$rootfs" && load_file_in_memory "$rootfs" $rootfs_addr
test "$resetcpu" && do_system_release
test "$resetcpu" && do_cpu_release
exit 0
# vim:ts=2:sw=2:ai:foldmethod=marker:foldlevel=0: