nixos-riscv/fpga/fpgactl

#!/bin/bash
#
# Copyright (c) 2024, Barcelona Supercomputing Center (BSC)
# SPDX-License-Identifier: MIT
#
# Based on a script provided by Xavier Martorell
# Rewritten by Rodrigo Arias Mallo <rodrigo.arias@bsc.es>

# Stop on first error
set -e

function is_module_loaded() # {{{
{
  lsmod | grep -wq "$1"
} # }}}
function check_environment() # {{{
{
  if ! command -v dma-ctl &> /dev/null; then
    echo "error: dma-ctl not found in PATH" >&2
    exit 1
  fi

  if ! command -v vivado &> /dev/null; then
    echo "error: vivado not found in PATH" >&2
    exit 1
  fi
} # }}}
function create_qdma_queue() # {{{
{
  pcidir="/sys/bus/pci/devices/0000:08:00.0"

  if [ ! -d "$pcidir/qdma" ]; then
    echo "missing pci directory: $pcidir" >&2
    exit 1
  fi

  if [ ! -d "$pcidir/qdma" ]; then
    echo "missing qdma directory: $pcidir/qdma" >&2
    exit 1
  fi

  if [ ! -f "$pcidir/qdma/qmax" ]; then
    echo "missing qmax file: $pcidir/qdma/qmax" >&2
    exit 1
  fi

  if [ ! -c "/dev/qdma08000-MM-1" ]; then
    echo 2 | sudo dd of="$pcidir/qdma/qmax"

    dma-ctl qdma08000 q add mode mm idx 1 dir bi
    dma-ctl qdma08000 q start idx 1 dir bi

    sudo chmod go+rw "/dev/qdma08000-MM-1"
    sudo chmod go+rw "$pcidir/resource0"
    sudo chmod go+rw "$pcidir/resource0_wc"
    sudo chmod go+rw "$pcidir/resource2"
    sudo chmod go+rw "$pcidir/resource2_wc"
  fi

  if [ ! -c "/dev/qdma08000-MM-0" ]; then
    dma-ctl qdma08000 q add mode mm idx 0 dir bi
    dma-ctl qdma08000 q start idx 0 dir bi
    sudo chmod go+rw "/dev/qdma08000-MM-0"
  fi

  sleep 2
} # }}}
function do_system_reset() # {{{
{
  # UartBootEn (bit2) + system reset (bit0)
  dma-ctl qdma08000 reg write bar 2 0x0 0x0 > /dev/null
  sleep 0.2
  # Release system reset, we must wait until the memory is filled with 0s
  dma-ctl qdma08000 reg write bar 2 0x0 0x1 > /dev/null
  #sleep 5
} # }}}
function do_system_release() # {{{
{
  # Release Ariane's reset
  dma-ctl qdma08000 reg write bar 2 0x0 0x3 > /dev/null
} # }}}
function copy_by_dma() # {{{
{
  ifile="$1"
  address="$2"

  ofile="/dev/qdma08000-MM-1"
  bs=$((8*1024*1024)) # 8 MiB

  total_size=$(stat --format "%s" "$ifile")

  nblocks=$(( ($total_size + $bs - 1) / $bs ))
  skip=0

  # Using just a single dd command doesn't seem to work. My hypothesis is that
  # the driver doesn't allow multiple writes without setting the fseek position.
  # So we keep copying blocks until the end of the file, one at a time.

  while [ "$skip" -lt "$nblocks" ]; do
    dst=$(($address + $skip * $bs))
    dd if="$ifile" skip=$skip count=1 bs=$bs of="$ofile" seek=$dst oflag=seek_bytes status=none
    let skip=$skip+1
  done

} # }}}
function load_file_in_memory() # {{{
{
  file="$1"
  address_hex="$2"
  address=$(($address_hex))

  total_size=$(stat --format "%s" "$file")

  # Previous tests...
  #strace -f dma-to-device -d /dev/qdma08000-MM-1 -a "$address" -s $((8*1024*1024)) -f "$file"
  #strace -f dd if="$file" bs=16M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=progress conv=sync
  #strace -f fpgakit/fpgadd -i "$file" -a "$address" -d /dev/qdma08000-MM-1 -c 1024 -s 1024
  #ID=08 ./load_image.sh "$file" "$address"

  # Now dd seems to work fine, but I will leave this as fallback:
  copy_by_dma "$file" "$address"

  #dd if="$file" bs=8M seek="${address}" oflag=seek_bytes of=/dev/qdma08000-MM-1 status=none

  printf "loaded '%s' at 0x%x with size %d\n" "$file" "$address" "$total_size" >&2
} # }}}
function do_boot_only() # {{{
{
  do_system_reset

  ./load_image.sh ${OSBI} $((0x80000000)) &&

  rm -f ${OSBI} &&

  sleep 2 &&

  # #Release Ariane's reset
  dma-ctl qdma08000 reg write bar 2 0x0 0x3 &&

  sleep 10 &&

  echo mount  -o nolock -o rw -o retrans=10 10.0.2.2:/media/sda2/scratch/xavim/point /root &&

  echo mount  -o nolock -o rw -o retrans=10 192.168.0.16:/media/sda2/scratch/xavim/point /root &&

  if [ ! -c /dev/qdma08000-MM-0 ] ; then
     /home/tools/drivers/`/bin/hostname`/dma_ip_drivers-onic-gamma/create-queue-qdma.sh -2
  fi &&

  sleep 2 #&&

  # uncomment to enable eth-over-pcie
  #sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 mtu 9000 up
  sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 up
} # }}}
function do_reload_fs() # {{{
{
  do_system_reset

  #~xavim/LAGARTO_LINUX-4.1/./load_image.sh \
  #  /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered \
  #  $((0x13ff00000)) &&

  load_file_in_memory /home/xavim/ARIANE_LINUX-3.0/recovery/fedora-fs-dx-java-cucu-0.108.raw.recovered $((0x13ff00000))

  sleep 1

  # Load OpenSBI + kernel
  load_file_in_memory ${OSBI} $((0x80000000))

  rm -f ${OSBI}

  sleep 2

  do_system_release
  create_qdma_queue

# uncomment to enable eth-over-pcie
#     sudo ifconfig onic${onicid}s0f0 10.0.2.2 netmask 255.255.255.0 mtu 9000 up
} # }}}
function upload_bitstream_file()  # {{{
{
  bitfile="$1"

  fpgajtag=$(lsusb -vd 0403: 2>&1 | grep iSerial | awk ' { print $3; }')
  if [ -z "$fpgajtag" ]; then
    echo "error: cannot find JTAG serial" >&2
    exit 1
  fi

  script=$(mktemp vivado-XXXXXXXXXX.tcl)
  cat > "$script" <<EOF
open_hw_manager
connect_hw_server -url localhost:3121
current_hw_target "localhost:3121/xilinx_tcf/Xilinx/${fpgajtag}A"
open_hw_target
set dev [lindex [get_hw_devices] 0]
current_hw_device \$dev
set_property PROGRAM.FILE ${bitfile} \$dev
program_hw_devices \$dev
exit
EOF

  vivado -nolog -nojournal -mode batch -source "$script"
  rm "$script"

  killall hw_server
} # }}}
function unload_modules() # {{{
{
  drvlist="$1"

  # Unload modules
  for mod in $drvlist; do
    if is_module_loaded "$mod"; then
	    sudo rmmod $mod
    fi
  done

} # }}}
function remove_pci_devices() # {{{
{
  for slot in $(lspci -mm -d 10ee: | awk '{printf "0000:%s\n",$1}'); do
    devdir="/sys/bus/pci/devices/$slot"
    if [ -d $devdir ]; then
      echo 1 | sudo dd "of=$devdir/remove"
    fi
  done

} # }}}
function rescan_pci_devices() # {{{
{
  echo 1 | sudo dd of=/sys/bus/pci/rescan
} # }}}
function load_qdma_modules() # {{{
{
  drv="$DMA_IP_DRIVERS/QDMA/linux-kernel/bin/qdma-pf.ko"
  hw_buffers="$((0x4FFE0000))"

  if [ ! -r "$drv" ]; then
    echo "error: missing $"
    exit 1
  fi

  sudo insmod "$drv" "hw_buffers=$hw_buffers"
  sleep 4
} # }}}
function load_bitstream()  # {{{
{
  bitstream="$1"

  unload_modules "xocl xclmgmt qdma_pf xdma" # qdma_vf not removable
  remove_pci_devices

  upload_bitstream_file "$bitstream"

  rescan_pci_devices
  unload_modules "qdma_pf xdma" # qdma_vf not removable
  remove_pci_devices
  load_qdma_modules
  rescan_pci_devices
  create_qdma_queue
} # }}}

bitstream=
bootloader=
kernel=
initrd=
rootfs=
resetcpu=
verbose=

bootloader_addr=0x80000000
kernel_addr=0x84000000
initrd_addr=0x8c300000
rootfs_addr=0x140000000

function usage()
{
  echo "" >&2
  echo "Usage: $0 [-v] [-w bitstream] [-b bootloader] [-k kernel] [-i initrd]" >&2
  echo "" >&2
  echo "First writes the bitstream if given. Then loads the rest of files" >&2
  echo "into memory and restarts the CPU." >&2
  echo "" >&2
  echo "Options" >&2
  echo "  -w bitstream   Write the bitstream file to the FPGA" >&2
  echo "  -b bootloader  Load the bootloader file in $bootloader_addr" >&2
  echo "  -k kernel      Load the kernel file in $kernel_addr" >&2
  echo "  -i initrd      Load the initrd file in $initrd_addr" >&2
  echo "  -r rootfs      Load the rootfs file in $rootfs_addr" >&2
  echo "  -v             Be verbose" >&2
  echo "" >&2
  exit 1
}

while getopts "hvw:b:k:i:r:" opt; do
  case "${opt}" in
    v) verbose=1 ;;
    w) bitstream="${OPTARG}" ;;
    b) bootloader="${OPTARG}"; resetcpu=1 ;;
    k) kernel="${OPTARG}"; resetcpu=1 ;;
    i) initrd="${OPTARG}"; resetcpu=1 ;;
    r) rootfs="${OPTARG}"; resetcpu=1 ;;
    h) usage ;;
    *) usage ;;
  esac
done

test "$verbose"     && set -x

check_environment

test "$bitstream"   && load_bitstream "$bitstream"
test "$resetcpu"    && do_system_reset
test "$bootloader"  && load_file_in_memory "$bootloader" $bootloader_addr
test "$kernel"      && load_file_in_memory "$kernel" $kernel_addr
test "$initrd"      && load_file_in_memory "$initrd" $initrd_addr
test "$rootfs"      && load_file_in_memory "$rootfs" $rootfs_addr
test "$resetcpu"    && do_system_release

# vim:ts=2:sw=2:ai:foldmethod=marker:foldlevel=0: