nixos-riscv/lagarto-ox.nix

{ config, lib, utils, pkgs, modulesPath, self, ... }:

{
  imports = [
    "${modulesPath}/installer/sd-card/sd-image.nix"
  ];

  #nixpkgs.crossSystem = {
  #  system = "riscv64-linux";
  #  gcc.arch = "rv64imafd";
  #  gcc.tune = "generic";
  #};

  # We don't need any firmware
  hardware.firmware = lib.mkForce [];

  # Doesn't work, it gets activated via the kernel socket.
  # # No need for udev
  # services.udev.enable = false;
  # systemd.suppressedSystemUnits = [
  #   "systemd-udev-trigger.service"
  # ];

  # Output the unit name so we can remove it
  systemd.extraConfig = ''
    StatusUnitFormat=name
  '';

  # Prevent executing the nscd program as it seems to hang the CPU
  system.activationScripts.users = lib.mkForce (
    let
      cfg = config.users;
      spec = pkgs.writeText "users-groups.json" (builtins.toJSON {
        inherit (cfg) mutableUsers;
        users = lib.mapAttrsToList (_: u:
        {
          inherit (u)
          name uid group description home homeMode createHome isSystemUser
          password hashedPasswordFile hashedPassword
          autoSubUidGidRange subUidRanges subGidRanges
          initialPassword initialHashedPassword expires;
          shell = utils.toShellPath u.shell;
        }) cfg.users;
        groups = lib.attrValues cfg.groups;
      });
    in
      if !config.systemd.sysusers.enable then {
        supportsDryActivation = true;
        text = ''
          install -m 0700 -d /root
          install -m 0755 -d /home

          ${pkgs.perl.withPackages (p: [ p.FileSlurp p.JSON ])}/bin/perl \
          -w ${./patches/update-users-groups.pl} ${spec}
        '';
      } else "" # keep around for backwards compatibility
  ); 

  # Also disable the nscd daemon
  services.nscd.enable = false;
  system.nssModules = lib.mkForce []; # Required

  system.build.bootStage2 = let
    useHostResolvConf = config.networking.resolvconf.enable && config.networking.useHostResolvConf;
    bootStage2 = pkgs.substituteAll {
      src = ./patches/stage-2-init.sh;
      shellDebug = "${pkgs.bashInteractive}/bin/bash";
      bashInteractive = "${pkgs.bashInteractive}";
      bench2 = "${pkgs.bench2}";
      shell = "${pkgs.bash}/bin/bash";
      inherit (config.boot) readOnlyNixStore systemdExecutable extraSystemdUnitPaths;
      inherit (config.system.nixos) distroName;
      isExecutable = true;
      inherit useHostResolvConf;
      inherit (config.system.build) earlyMountScript;
      path = lib.makeBinPath ([
        pkgs.coreutils
        pkgs.util-linux
        pkgs.strace
      ] ++ lib.optional useHostResolvConf pkgs.openresolv);
      postBootCommands = pkgs.writeText "local-cmds"
      ''
        ${config.boot.postBootCommands}
        ${config.powerManagement.powerUpCommands}
      '';
    };
  in lib.mkForce bootStage2;

  boot.kernelPackages = pkgs.linuxPackages_latest;
  boot = {
    extraModulePackages = [
      # Add the custom Ethernet module
      #pkgs.xilinx-axienet-carv
    ];
    kernelModules = config.boot.initrd.kernelModules;
    kernelPatches = [
      {
        name = assert false; "sbi-early-console";
        patch = null;
        extraConfig =
        # Early console via SBI
        ''
          RISCV_SBI y
          RISCV_SBI_V01 y
          SERIAL_EARLYCON y
          SERIAL_EARLYCON_RISCV_SBI y
          HVC_DRIVER y
          HVC_RISCV_SBI y
        ''
        # Enable console driver
        +''
          SERIAL_8250 y
          SERIAL_8250_CONSOLE y
          SERIAL_OF_PLATFORM y
          CONSOLE_POLL y
        ''
        # Allows regions of persistent memory to be described in the device-tree.
        + ''
          OF_PMEM y
        ''
        # Allow you to use a contiguous range of reserved memory as one or more
        # persistent block devices (/dev/pmem0)
        + ''
          LIBNVDIMM y
          BLK_DEV_PMEM y
        ''
        # No vector extensions
        + ''
          RISCV_ISA_V n
          RISCV_ISA_V_DEFAULT_ENABLE n
        ''
        # Debugging
        + ''
          DEBUG_KERNEL y
          DEBUG_MISC y
          DEBUG_WX y
          MAGIC_SYSRQ y
          SYSRQ_SERIAL y
          DEBUG_VM y
          SOFTLOCKUP_DETECTOR y
          SOFTLOCKUP_DETECTOR_INTR_STORM y
          HARDLOCKUP_DETECTOR y
          DETECT_HUNG_TASK y
          WQ_WATCHDOG y
          WQ_CPU_INTENSIVE_REPORT y
          TRACING y
          BOOTTIME_TRACING y
          STRICT_DEVMEM n
          MMIOTRACE y
        ''
        # Disable SMP so we don't have IPI
        + ''
          SMP n
        ''
        # SPI driver
        + ''
          COMPILE_TEST y
          SPI y
          SPI_DAVINCI m
        ''
        ;
      }
    ];

    initrd = {
      # Avoid zstd as we don't have the tools in "cucu" machine
      compressor = "gzip";
      kernelModules = [
        # DMA for Ethernet
        #"xilinx_dma"
        # Load the Ethernet module by default
        #"xxvnet_carv"

        # For SPI
        "spi_davinci"
        "spidev"
      ];

      # Custom init script
      extraFiles = {
        "/shell".source = pkgs.writeScript "shell" ''
          #!${config.system.build.extraUtils}/bin/ash

          set -x

          export PATH=${config.system.build.extraUtils}/bin
          ash
        '';

        "/testplic".source = pkgs.writeScript "testplic" ''
          #!${config.system.build.extraUtils}/bin/ash
          export PATH=${config.system.build.extraUtils}/bin

          set -x

          (
            echo "--- Testing threshold register init value"
            # Ensure that reading a few times the threshold value
            # always gives the same initial value 0
            t1=$(devmem 0x40a00000) # Read context 1 threshold value
            t2=$(devmem 0x40a00000) # Read context 1 threshold value
            t3=$(devmem 0x40a00000) # Read context 1 threshold value
            found="$t1 $t2 $t3"
            expected="0x00000000 0x00000000 0x00000000"
            if [ "$found" = "$expected" ]; then
              echo "--- Threshold init value: OK"
            else
              echo "found   =$found"
              echo "expected=$expected"
              echo "--- Threshold init value: FAIL"
            fi
          )

          (
            echo "--- Testing threshold register stability"
            # Write the priority register of an interrupt and ensure
            # the threshold register didn't change
            devmem 0x40a00000 32 0 # Write context 1 threshold value 0
            devmem 0x40800010 32 5 # Write source 4 priority value 5
            t1=$(devmem 0x40a00000) # Read context 1 threshold value
            t2=$(devmem 0x40a00000) # Read context 1 threshold value
            found="$t1 $t2"
            expected="0x00000000 0x00000000"
            if [ "$found" = "$expected" ]; then
              echo "--- Threshold stability: OK"
            else
              echo "found   =$found"
              echo "expected=$expected"
              echo "--- Threshold stability: FAIL"
            fi
          )

          (
            echo "--- Testing claim register"

            # Use aux timer on source 4 for this one
            pending=$(devmem 0x40801000) # Dump pending bits of sources 0-31

            # Ensure the aux timer is pending
            if [ "$pending" = "0x00000010" ]; then

              # Make sure the priority is higher than the threshold
              devmem 0x40800010 32 0x10 # Write source 4 priority value 16
              devmem 0x40802080 32 0x10 # Enable source 4 in context 1
              # Writing the threshold has to be last, otherwise it will change
              devmem 0x40a00000 32 0    # Write context 1 threshold value 0
              c1=$(devmem 0x40a01004)   # Claim context 1
              c2=$(devmem 0x40a01004)   # Claim context 1
              c3=$(devmem 0x40a01004)   # Claim context 1
              found="$c1 $c2 $c3"
              expected="0x00000004 0x00000004 0x00000004"
              if [ "$found" = "$expected" ]; then
                echo "--- Testing claim register: OK"
              else
                echo "found   =$found"
                echo "expected=$expected"
                echo "--- Testing claim register: FAIL"
              fi
            else
              echo "unknown pending bits: $pending"
              echo "--- Testing claim register: SKIP"
            fi
          )
          set +x
          #echo "all done, dropping to a shell..."
          #ash
        '';

        "/preinit".source = pkgs.writeScript "preinit" ''
          #!${config.system.build.extraUtils}/bin/ash
          export PATH=${config.system.build.extraUtils}/bin
          # csrtool all-in-order

          # ip addr
          # cat /proc/interrupts

          # modprobe xxvnet_carv

          # plictool -c2
          # plictool -c2
          # plictool -c2

          # ip addr

          exec /init
        '';
      };

      # Add riscv-tools to initrd
      extraUtilsCommands = ''
        cp -a ${pkgs.riscv-tools}/bin/* $out/bin
      '';

      # Write a counter to the DMA region, so we can check the kernel is not
      # dead. Monitor from the host with:
      # while [ 1 ]; do xxd -s $((0x1bfff0000 - 0x60000000)) \
      #   -l 4 /dev/qdma34000-MM-1; sleep 0.2; done
      preDeviceCommands = ''
        # Seed RNG
        seedrng -d /tmp || true
        mv /tmp/seed.no-credit /tmp/seed.credit || true
        seedrng -d /tmp || true
        echo "Available entropy: $(cat /proc/sys/kernel/random/entropy_avail)"

        # Last chance to enter a shell
        if read -t 3 -p 'Press enter for shell... '; then
          allowShell=1
          fail
        fi

        # echo "Running tests..."
        # sh /testplic

        # echo "Creating a heartbeat counter at 0x1bfff0000"
        # sh -c 'hb=0; while [ 1 ]; do let hb=$hb+1; devmem 0x1bfff0000 32 $hb; done' &
      ''
      +
      # Disable proactive compaction. May be better to disable CONFIG_COMPACTION.
      ''
        echo 0 > /proc/sys/vm/compaction_proactiveness
      ''
#      +
#      # Show stacktrace on calls to the hvc_remove function.
#      ''
#        echo "Mount debugfs"
#        mkdir -p /sys/kernel/debug/
#        mount -t debugfs none /sys/kernel/debug/
#        td=/sys/kernel/debug/tracing
#        echo hvc_remove > $td/set_ftrace_filter
#        echo function > $td/current_tracer
#        echo 1 > $td/options/func_stack_trace
#      ''
#      FIXME: Disable sched_switch for now, as it still hangs the boot...
#      +
#      # Exclude the second pid, which is the kthread that will dump the trace to
#      # the console, otherwise we live lock the kernel. Then enable the
#      # sched_switch events.
#      ''
#        echo "Mount debugfs"
#        mkdir -p /sys/kernel/debug/
#        mount -t debugfs none /sys/kernel/debug/
#        echo "Exclude pid 2 from sched"
#        echo '(prev_pid != 2 && next_pid != 2)' > /sys/kernel/debug/tracing/events/sched/filter
#        echo "Enable sched_switch events"
#        echo 1 > /sys/kernel/debug/tracing/events/sched/sched_switch/enable
#      ''
      ;
    };

    loader = {
      grub.enable = false;
      generic-extlinux-compatible.enable = true;
    };
  };

  # No network
  services.openssh.enable = false;
  networking.useDHCP = false;

  # Run getty on /dev/console and restartt until it works
  systemd.services."serial-getty@console" = {
    enable = true;
    after = [ "network.target" ];
    wantedBy = [ "getty.target" ]; # to start at boot
    serviceConfig.Restart = "always";
  };

  # Disable hvc0 as it is racing for the same console
  systemd.services."serial-getty@hvc0" = {
    enable = lib.mkForce false;
    wantedBy = lib.mkForce [ ];
  };

  services.getty.autologinUser = lib.mkForce "root";

  sdImage = {
    # The image will be loaded as-is in memory, so no compression
    compressImage = false;
    imageName = "rootfs.img";
    # Not needed for now
    expandOnBoot = false;
    populateFirmwareCommands = "";
    populateRootCommands = ''
      mkdir -p ./files/boot
      ${config.boot.loader.generic-extlinux-compatible.populateCmd} \
        -c ${config.system.build.toplevel} \
        -d ./files/boot
    '';
  };

  nixpkgs.overlays = [ (final: prev: {
    #busybox = prev.busybox.overrideAttrs (old: {
    #  # Print some debug lines on switch_root to see where it hangs.
    #  patches = (old.patches or []) ++ [ ./patches/busybox-debug.patch ];
    #});

    linuxPackages_latest = prev.linuxPackages_latest;
    #linuxPackages_latest = prev.linuxPackages_latest.extend (lib.const (ksuper: {
    #  kernel = ksuper.kernel.override {
    #    stdenv = prev.gcc8Stdenv;
    #  };
    #}));

    bench2 = final.writeShellScript "bench2" ''
      # Performs minimal FS setup and runs the SPEC benchmark
      mkdir /tmp /bin /root
      mount -t tmpfs tmpfs /tmp
      export  TMPDIR=/tmp
      # We need /bin/sh
      ln -s $(which sh) /bin/sh
      # Check CPU usage
      vmstat 5 5
      bash -x speclaunch
      cat /tmp/spec/time.csv
      # Give me a shell at the end
      bash -l
    '';

    ox-dtb = prev.stdenv.mkDerivation rec {
      name = "ox.dtb";
      src = ./dts;
      dontConfigure = true;
      nativeBuildInputs = [ prev.buildPackages.dtc ];
      buildPhase = ''
        make lagarto_ox.dtb
      '';
      installPhase = ''
        mkdir $out
        cp lagarto_ox.* $out
      '';
      dontFixup = true;
      hardeningDisable = [ "all" ];
    };

    #bitstream = "${final.bitstreams}/lagarto-3-ox/gold.bit";
    bitstream = "${final.bitstreams}/lagarto-3-ox/ox_u55c_87a14c32_fix_threshold.bit";

    bootrom = "${final.rbootrom}/rbootrom.bin";

    uboot = prev.ubootQemuRiscv64Smode.override {
      filesToInstall = [ "u-boot-nodtb.bin" ];
      #version = "2023.07.02-print-cpu-probe";
      #src = builtins.fetchGit {
      #  url = "file:///home/Computational/rarias/riscv/u-boot";
      #  rev = "f80a22a480f0e4157647bacf90e663be457c72c4";
      #};
      patches = [
        #./patches/u-boot-debug.patch
        ./patches/uboot-debug-ext-interrupts.patch
        ./patches/uboot-exception-extras.patch
      ];
      # Copy our environment to board/emulation/qemu-riscv/environ.env
      preConfigure = ''
        cp ${final.uboot-env} board/emulation/qemu-riscv/environ.env
      '';
      #postConfigure = ''
      #  echo --------------------------- generated config:
      #  cat .config
      #  echo ---------------------------
      #'';
      #postBuild = ''
      #  echo --------------------------- generated env starts
      #  cat include/generated/env.in
      #  echo --------------------------- generated env ends
      #'';
      #
      # CONFIG_SERIAL_PRESENT=n
      # CONFIG_SYS_NS16550=n
      extraConfig = ''
        CONFIG_RISCV_ISA_C=n
        CONFIG_REQUIRE_SERIAL_CONSOLE=n
        CONFIG_SERIAL=y
        CONFIG_SERIAL_PUTS=y
        CONFIG_SHOW_BOOT_PROGRESS=y
        CONFIG_SHOW_REGS=y
        CONFIG_LIBCOMMON_SUPPORT=y
        CONFIG_SERIAL_SEARCH_ALL=n
        CONFIG_SERIAL_PROBE_ALL=n
        CONFIG_OF_CONTROL=y
        CONFIG_OF_EMBED=y
        CONFIG_OF_HAS_PRIOR_STAGE=y
        CONFIG_BLKMAP=y
        CONFIG_CMD_BLKMAP=y
        CONFIG_SBI_V01=y
        CONFIG_DEBUG_UART=y
        CONFIG_DEBUG_UART_ANNOUNCE=y
        CONFIG_DEBUG_SBI_CONSOLE=y
        CONFIG_SMP=n
        CONFIG_TRACE_EARLY=y
        CONFIG_CMD_MEMTEST=y
        CONFIG_CMD_EXCEPTION=y
        CONFIG_CMD_TIMER=y
        CONFIG_ENV_SOURCE_FILE="environ"
      ''
#      # Enable debug logs
#      +
#      ''
#        CONFIG_LOG=y
#        CONFIG_LOGLEVEL=9
#        CONFIG_LOG_MAX_LEVEL=9
#        CONFIG_LOG_DEFAULT_LEVEL=9
#      ''
      ;
      extraMakeFlags = [
        #"V=1"
        #"KCPPFLAGS=-DLOG_DEBUG"
        #"EXT_DTB=${final.ox-dtb}/lagarto_ox.dtb"
      ];
    };

    uboot-env = let
      init = "${config.system.build.toplevel}/init";
      initrd = "${config.system.build.initialRamdisk}/initrd";
      # Create pmem of 3 GiB [0x140000000, 0x200000000)
      #fdt mknode / pmem@0x140000000
      #fdt set /pmem@0x140000000 compatible "pmem-region"
      #fdt set /pmem@0x140000000 reg <0x1 0x40000000 0x0 0xc0000000>

      # Reduce memory to 3 GiB [0x80000000, 0x140000000)
      #fdt set /memory@80000000  reg <0x0 0x80000000 0x0 0xc0000000>

      # Set kernel bootcmd options.
      #    rdinit=/preinit          boot custom preinit script
      #    console=ttyS0,115200n8   use serial driver (slow)
      # Systemd options
      #    systemd.log_level=debug
      #    systemd.log_target=console
      # NixOS interesting options:
      #    debug1                   enable debug shell in stage 1
      #    debug2                   enable debug shell in stage 2 (custom)
      #    bench2                   run benchmark on stage 2 (custom)
      #    boot.trace               enable set -x in stage 1
      #    boot.tracedebug          enable set -x in stage 2
      # Ftrace interesting options:
      #   trace_event=initcall:*    trace the init function of drivers
      #   trace_options=sym-addr    display function address
      #   tp_printk                 write ftrace events to console
      #   trace_buf_size=1M         set ftrace buffer to 1M
      #
    in prev.runCommand "uboot.txt" {} ''
      cat > $out <<EOF
      xtrace=yes
      bootargs=root=/dev/ram0 loglevel=7 rw earlycon=sbi console=hvc0 debug2 init=${init}
      ramdisk_size=$(stat --format %s $(readlink -f ${initrd}))
      bootcmd=fdt print; booti \''${kernel_addr_r} \''${ramdisk_addr_r}:\''${ramdisk_size} \''${fdtcontroladdr}
      EOF
    '';

    opensbi = prev.opensbi.overrideAttrs (old: rec {
      #version = "1.4";
      version = "1.5";
      src = prev.fetchFromGitHub {
        owner = "riscv-software-src";
        repo = "opensbi";
        rev = "v${version}";
        #hash = "sha256-T8ZeAzjM9aeTXitjE7s+m+jjGGtDo2jK1qO5EuKiVLU="; #1.4
        hash = "sha256-vK14P97FcaVz4GDr/0055Z6s/k7BPKPQGZ/MQxbOWu0="; #1.5
      };
      #NIX_DEBUG=5;
      makeFlags = [
        "PLATFORM=generic"
        #"CONFIG_SBI_ECALL_RFENCE=n"
        #"PLATFORM_RISCV_ISA=rv64imafd" # No compressed instructions
        #"PLATFORM_RISCV_ISA=rv64g" # No compressed instructions
        #"PLATFORM_RISCV_ABI=lp64d"
        "FW_PAYLOAD_PATH=${final.uboot}/u-boot-nodtb.bin"
        "FW_FDT_PATH=${final.ox-dtb}/lagarto_ox.dtb"

        # Ensure it doesn't overlap from the 0x80200000 where the kernel will be
        # placed.
        "FW_PAYLOAD_FDT_ADDR=0xc0000000"
      ];
      patches = [
        #./patches/opensbi-timer-debug.patch # Print calls to machine trap
        #./patches/opensbi-enable-meip.patch
        #./patches/opensbi-enable-seip.patch
        #./patches/opensbi-test-plic.patch # Working delegation test, disabled for now
        ./patches/opensbi-dump-mregs.patch
        #./patches/opensbi-dont-delegate.patch
        #./patches/ox-alveo-platform-plic.patch
      ];
    });

    # Custom kernel driver for the Ethernet
    xilinx-axienet-carv = let
      kernel = config.boot.kernelPackages.kernel;
    in prev.stdenv.mkDerivation rec {
      pname = "xilinx-axienet-carv";
      version = src.shortRev;
      src = builtins.fetchGit {
        url = "git@gitlab-internal.bsc.es:meep/meep-os/lagarto-openpiton-sdk.git";
        rev = "d2ae2e788bf1cc60676599184a9ec1128cc81d81";
        ref = "master";
      };
      patches = [
        ./patches/ethernet-driver-poll.patch
        ./patches/ethernet-driver-build.patch
        ./patches/ethernet-driver-kbuild.patch
      ];
      preConfigure = ''
        export sourceRoot=$PWD/drivers
        cd drivers
        #rm Makefile
      '';

      nativeBuildInputs = kernel.moduleBuildDependencies;
      makeFlags = kernel.makeFlags ++ [
        "-C"
        "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
        "M=$(PWD)"
      ];
      buildFlags = [ "modules" ];
      installFlags = [ "INSTALL_MOD_PATH=${placeholder "out"}" ];
      installTargets = [ "modules_install" ];
    };
  }) ];
}