nixos-riscv/lagarto-ox.nix

{ config, lib, pkgs, modulesPath, self, ... }:

{
  imports = [
    "${modulesPath}/installer/sd-card/sd-image.nix"
  ];

  #nixpkgs.crossSystem = {
  #  system = "riscv64-linux";
  #  gcc.arch = "rv64imafd";
  #  gcc.tune = "generic";
  #};

  boot.kernelPackages = pkgs.linuxPackages_latest;
  boot = {
    kernelPatches = [
      {
        name = assert false; "sbi-early-console";
        patch = null;
        extraConfig =
        # Early console via SBI
        ''
          RISCV_SBI y
          RISCV_SBI_V01 y
          SERIAL_EARLYCON y
          SERIAL_EARLYCON_RISCV_SBI y
          HVC_DRIVER y
          HVC_RISCV_SBI y
        ''
        # Enable console driver
        +''
          SERIAL_8250 y
          SERIAL_8250_CONSOLE y
          SERIAL_OF_PLATFORM y
          CONSOLE_POLL y
        ''
        # Allows regions of persistent memory to be described in the device-tree.
        + ''
          OF_PMEM y
        ''
        # Allow you to use a contiguous range of reserved memory as one or more
        # persistent block devices (/dev/pmem0)
        + ''
          LIBNVDIMM y
          BLK_DEV_PMEM y
        ''
        # No vector extensions
        + ''
          RISCV_ISA_V n
          RISCV_ISA_V_DEFAULT_ENABLE n
        ''
        # Debugging
        + ''
          DEBUG_KERNEL y
          DEBUG_MISC y
          DEBUG_WX y
          MAGIC_SYSRQ y
          SYSRQ_SERIAL y
          DEBUG_VM y
          SOFTLOCKUP_DETECTOR y
          SOFTLOCKUP_DETECTOR_INTR_STORM y
          HARDLOCKUP_DETECTOR y
          DETECT_HUNG_TASK y
          WQ_WATCHDOG y
          WQ_CPU_INTENSIVE_REPORT y
          TRACING y
          BOOTTIME_TRACING y
          STRICT_DEVMEM n
        ''
        # Disable SMP so we don't have IPI
        + ''
          SMP n
        ''
        # Disable huge pages
        + ''
          HUGETLBFS n
          HUGETLB_PAGE n
        ''
        ;
      }
    ];

    initrd = {
      # Avoid zstd as we don't have the tools in "cucu" machine
      compressor = "gzip";
      kernelModules = [ ];

      # Custom init script
      extraFiles."/init2".source = pkgs.writeScript "init2" ''
        #!${config.system.build.extraUtils}/bin/ash

        set -x

        export PATH=${config.system.build.extraUtils}/bin

        #mkdir -p /tracing
        #mount -t tracefs nodev /tracing

        #cd /tracing
        #echo nop > current_tracer
        #echo 100 > max_graph_depth
        #echo handle_page_fault > set_graph_function
        #echo function_graph > current_tracer
        #cd /

        # Open an interactive shell
        ash

        memtool fill $((512 * 1024 * 1024))

        # Unlikely to reach this point
        exec /init
      '';

      # Add the csrtool to the initrd so we can change the
      # in-order/out-of-order, and memtool to stress the memory.
      extraUtilsCommands = ''
        cp -a ${pkgs.csrtool}/bin/csrtool $out/bin
        cp -a ${pkgs.memtool}/bin/memtool $out/bin
      '';

      # Write a counter to the DMA region, so we can check the kernel is not
      # dead. Monitor from the host with:
      # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \
      #   -l 4 /dev/qdma34000-MM-1; sleep 0.2; done
      preDeviceCommands = ''
        echo "Creating a heartbeat counter at 0x1bfff0000"
        sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' &
      ''
      +
      # Disable proactive compaction. May be better to disable CONFIG_COMPACTION.
      ''
        echo 0 > /proc/sys/vm/compaction_proactiveness
      ''
      +
      # Show stacktrace on calls to the hvc_remove function.
      ''
        echo "Mount debugfs"
        mkdir -p /sys/kernel/debug/
        mount -t debugfs none /sys/kernel/debug/
        td=/sys/kernel/debug/tracing
        echo hvc_remove > $td/set_ftrace_filter
        echo function > $td/current_tracer
        echo 1 > $td/options/func_stack_trace
      ''
#      FIXME: Disable sched_switch for now, as it still hangs the boot...
#      +
#      # Exclude the second pid, which is the kthread that will dump the trace to
#      # the console, otherwise we live lock the kernel. Then enable the
#      # sched_switch events.
#      ''
#        echo "Mount debugfs"
#        mkdir -p /sys/kernel/debug/
#        mount -t debugfs none /sys/kernel/debug/
#        echo "Exclude pid 2 from sched"
#        echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter
#        echo "Enable sched_switch events"
#        echo 1 > /sys/kernel/debug/tracing/events/sched/sched_switch/enable
#      ''
      ;
    };

    loader = {
      grub.enable = false;
      generic-extlinux-compatible.enable = true;
    };
  };

  # No network
  services.openssh.enable = false;

  # Run getty on /dev/console and restartt until it works
  systemd.services."serial-getty@console" = {
    enable = true;
    wantedBy = [ "getty.target" ]; # to start at boot
    serviceConfig.Restart = "always";
  };

  # Disable hvc0 as it is racing for the same console
  systemd.services."serial-getty@hvc0" = {
    enable = lib.mkForce false;
    wantedBy = lib.mkForce [ ];
  };

  sdImage = {
    # The image will be loaded as-is in memory, so no compression
    compressImage = false;
    imageName = "rootfs.img";
    # Not needed for now
    expandOnBoot = false;
    populateFirmwareCommands = "";
    populateRootCommands = ''
      mkdir -p ./files/boot
      ${config.boot.loader.generic-extlinux-compatible.populateCmd} \
        -c ${config.system.build.toplevel} \
        -d ./files/boot
    '';
  };

  nixpkgs.overlays = [ (final: prev: {
    #busybox = prev.busybox.overrideAttrs (old: {
    #  # Print some debug lines on switch_root to see where it hangs.
    #  patches = (old.patches or []) ++ [ ./busybox-debug.patch ];
    #});

    linuxPackages_latest = prev.linuxPackages_latest;
    #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: {
    #  kernel = ksuper.kernel.override {
    #    stdenv = prev.gcc8Stdenv;
    #  };
    #}));

    ox-dtb = prev.runCommand "ox.dtb" {
      dtsFile = ./ox-plic.dts;
      #dtsFile = ./xavi.dts;
      nativeBuildInputs = [ prev.buildPackages.dtc ];
    } "dtc -O dtb -o $out $dtsFile";

    #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit";
    bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_a234c132_two_uarts.bit";

    bootrom = "${final.bitstreams}/lagarto-3-ox/large_bootrom.bin";

    uboot = prev.ubootQemuRiscv64Smode.override {
      filesToInstall = [ "u-boot.bin" "u-boot-nodtb.bin" ];
      #version = "2023.07.02-print-cpu-probe";
      #src = builtins.fetchGit {
      #  url = "file:///home/Computational/rarias/riscv/u-boot";
      #  rev = "f80a22a480f0e4157647bacf90e663be457c72c4";
      #};
      #patches = [ ./u-boot-debug.patch ];
      #
      # CONFIG_SERIAL_PRESENT=n
      # CONFIG_SYS_NS16550=n
      extraConfig = ''
        CONFIG_RISCV_ISA_C=n
        CONFIG_REQUIRE_SERIAL_CONSOLE=n
        CONFIG_SERIAL=y
        CONFIG_SERIAL_PUTS=y
        CONFIG_SHOW_BOOT_PROGRESS=y
        CONFIG_SHOW_REGS=y
        CONFIG_LIBCOMMON_SUPPORT=y
        CONFIG_SERIAL_SEARCH_ALL=n
        CONFIG_SERIAL_PROBE_ALL=n
        CONFIG_OF_CONTROL=y
        CONFIG_OF_EMBED=y
        CONFIG_OF_HAS_PRIOR_STAGE=y
        CONFIG_BLKMAP=y
        CONFIG_CMD_BLKMAP=y
        CONFIG_SBI_V01=y
        CONFIG_DEBUG_UART=y
        CONFIG_DEBUG_UART_ANNOUNCE=y
        CONFIG_DEBUG_SBI_CONSOLE=y
        CONFIG_SMP=n
        CONFIG_TRACE_EARLY=y
        CONFIG_CMD_MEMTEST=y
      ''
#      # Enable debug logs
#      +
#      ''
#        CONFIG_LOG=y
#        CONFIG_LOGLEVEL=9
#        CONFIG_LOG_MAX_LEVEL=9
#        CONFIG_LOG_DEFAULT_LEVEL=9
#      ''
      ;
      extraMakeFlags = [
        #"KCPPFLAGS=-DLOG_DEBUG"
        #"EXT_DTB=${final.ox-dtb}"
      ];
    };

    uboot-env = let
      init = "${config.system.build.toplevel}/init";
      initrd = "${config.system.build.initialRamdisk}/initrd";
      rev = if self ? rev then self.rev
        else throw ("Refusing to build from a dirty Git tree!");
    in prev.runCommand "uboot.txt" {} ''
      cat > $out <<EOF
      # Git commit: ${rev}

      # Create pmem of 3 GiB [0x140000000, 0x200000000)
      fdt mknode / pmem@0x140000000
      fdt set /pmem@0x140000000 compatible "pmem-region"
      fdt set /pmem@0x140000000 reg <0x1 0x40000000 0x0 0xc0000000>

      # Reduce memory to 3 GiB [0x80000000, 0x140000000)
      fdt set /memory@80000000  reg <0x0 0x80000000 0x0 0xc0000000>

      # Set kernel bootcmd options.
      # NixOS interesting options:
      #    debug1                   enable debug shell in stage 1
      #    boot.trace               enable set -x in stage 1
      #    boot.tracedebug          enable set -x in stage 2
      # Ftrace interesting options:
      #   trace_event=initcall:*    trace the init function of drivers
      #   trace_options=sym-addr    display function address
      #   tp_printk                 write ftrace events to console
      #   trace_buf_size=1M         set ftrace buffer to 1M
      #
      setenv bootargs "root=/dev/ram0 loglevel=7 debug rw earlycon=sbi console=hvc0 tp_printk trace_buf_size=1M rdinit=/init2 init=${init}"

      EOF

      # Populate the bood commands from the initrd size
      echo "# Boot system" >> $out
      echo "setenv ramdisk_size $(stat --format %s $(readlink -f ${initrd}))" >> $out
      echo 'booti ''${kernel_addr_r} ''${ramdisk_addr_r}:''${ramdisk_size} ''${fdtcontroladdr}' >> $out
    '';

    opensbi = prev.opensbi.overrideAttrs (old: rec {
      version = "1.4";
      src = prev.fetchFromGitHub {
        owner = "riscv-software-src";
        repo = "opensbi";
        rev = "v${version}";
        hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU=";
      };
      #NIX_DEBUG=5;
      makeFlags = [
        "PLATFORM=fpga/ox_alveo"
        #"CONFIG_SBI_ECALL_RFENCE=n"
        #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions
        #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions
        #"PLATFORM_RISCV_ABI=lp64d"
        "FW_PAYLOAD_PATH=${final.uboot}/u-boot.bin"
        "FW_FDT_PATH=${final.ox-dtb}"
      ];
      patches = [ ./ox-alveo-platform-plic.patch ];
    });
#    opensbi = prev.opensbi.overrideAttrs (old: {
#      #NIX_DEBUG=5;
#      # Compile dts to dtb form
#      #nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ prev.buildPackages.dtc ];
#      #preBuild = ''
#      #  dtc -O dtb -o ox.dtb ${./ox.dts}
#      #'';
#      makeFlags = old.makeFlags ++ [
#        "PLATFORM=fpga/sargantana_alveo"
#        #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions
#        "PLATFORM_RISCV_ISA=rv64g" # No compressed instructions
#        #"PLATFORM_RISCV_ABI=lp64d"
#        "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin"
#        #"FW_FDT_PATH=ox.dtb"
#      ];
#      patches = [ ./alveo-uart-sbi-hack.patch ];
#    });
#    opensbi = prev.opensbi.overrideAttrs (old: {
#      makeFlags = old.makeFlags ++ [
#        "PLATFORM=fpga/openpiton"
#        "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin"
#      ];
#      patches = [ ./opensbi-lagarto-hun.patch ];
#    });
  }) ];
}