.gitignore

@@ -1,4 +1,3 @@
-**/.gdbinit
 !*/
 difftest/
 !/nemu/*

flake.nix

@@ -93,7 +93,7 @@
         devShells.npc = pkgs.mkShell.override { stdenv = pkgs.ccacheStdenv; } {
           inherit (self.checks.${system}.pre-commit-check) shellHook;
           CHISEL_FIRTOOL_PATH = "${nixpkgs-circt162.legacyPackages.${system}.circt}/bin";
-          NPC_IMAGES_PATH = "${self.packages.${system}.rv32Cross.am-kernels-npc}/share/am-kernels";
+          NPC_IMAGES_DIR="${self.packages.${system}.am-kernels-npc}/share/am-kernels";
           packages = with pkgs; [
             clang-tools
             cmake
@@ -109,7 +109,6 @@
 
           nativeBuildInputs = with pkgs; [
             cmake
-            ninja
             sbt
             nvboard
             nixpkgs-circt162.legacyPackages.${system}.circt
@@ -118,11 +117,10 @@
             flex
             bison
             verilator
-            self.packages.${system}.rv32Cross.am-kernels-npc
+            self.packages.${system}.am-kernels-npc
           ];
 
           buildInputs = with pkgs; [
-            spdlog
             nvboard
             openssl
             libllvm
@@ -130,6 +128,10 @@
             readline
             mini-gdbstub
           ] ++ self.checks.${system}.pre-commit-check.enabledPackages;
+
+          cmakeFlags = [
+            "-DDIFFTEST_LIB:string=${self.packages.${system}.nemu-lib}/lib/riscv32-nemu-interpreter-so"
+          ];
         };
       }
     );

npc/CMakeLists.txt

@@ -41,7 +41,6 @@ if(BUILD_SIM_NVBOARD_TARGET)
   find_package(SDL2_image REQUIRED)
 endif()
 find_package(CLI11 CONFIG REQUIRED)
-find_package(spdlog REQUIRED)
 
 option(ENABLE_SDB "Enable simple debugger" OFF)
 
@@ -59,6 +58,9 @@ set(CHISEL_OUTPUT_VERILATOR_CONF ${CHISEL_OUTPUT_DIR}/conf.vlt)
 set(CHISEL_OUTPUT_TOPMODULE ${CHISEL_OUTPUT_DIR}/${TOPMODULE}.sv)
 set(CHISEL_EMIT_ARGS "--target-dir ${CHISEL_OUTPUT_DIR}")
 
+# -- Find difftest binaries
+file(GLOB_RECURSE DIFFTEST_BINARY_FILES "${DIFFTEST_RESOURCE_DIR}/*.bin")
+
 # -- Build NVBoard executable
 if(BUILD_SIM_NVBOARD_TARGET)
   add_subdirectory(csrc_nvboard)

npc/cmake/ChiselBuild.cmake

@@ -1,19 +1,20 @@
-# -- Add an always run target to generate verilog files with sbt/bloop, as we
-# don't know if the result files will be different from cmake NOTE: Must
-# reconfigure if we add new files in SCALA_CORE directory
+# -- Add an always run target to generate verilog files with sbt/bloop,
+#    as we don't know if the result files will be different from cmake
+# NOTE: Must reconfigure if we add new files in SCALA_CORE directory
 file(GLOB_RECURSE SCALA_CORE_SOURCES "${SCALA_CORE}/src/main/scala/*.scala")
 file(GLOB_RECURSE SCALA_CORE_RESOURCES "${SCALA_CORE}/src/main/resources/*")
 message(STATUS "Found scala source file: ${SCALA_CORE_SOURCES}")
-set(CHISEL_DEPENDENCY ${SCALA_CORE_SOURCES} ${SCALA_CORE_RESOURCES}
-                      ${SCALA_CORE}/build.sbt)
+set(CHISEL_DEPENDENCY ${SCALA_CORE_SOURCES} ${SCALA_CORE_RESOURCES} ${SCALA_CORE}/build.sbt)
 
 if(BUILD_USE_BLOOP)
-  message(STATUS "Building core using bloop")
   set(CHISEL_TARGET bloop_${TOPMODULE})
   set(CHISEL_TEST_TARGET bloop_${TOPMODULE}_test)
   # Export sbt build config to bloop
   if(NOT EXISTS ${SCALA_CORE}/.bloop)
-    execute_process(COMMAND sbt bloopInstall WORKING_DIRECTORY ${SCALA_CORE})
+    execute_process(
+      COMMAND sbt bloopInstall
+      WORKING_DIRECTORY ${SCALA_CORE}
+    )
   endif()
   string(REPLACE " " ";" CHISEL_EMIT_ARGS_LIST ${CHISEL_EMIT_ARGS})
   list(TRANSFORM CHISEL_EMIT_ARGS_LIST PREPEND "--args;")
@@ -23,11 +24,14 @@ if(BUILD_USE_BLOOP)
     WORKING_DIRECTORY ${SCALA_CORE}
     DEPENDS ${CHISEL_DEPENDENCY}
     COMMAND_EXPAND_LISTS
-    COMMENT "Run bloop from CMake")
-  # add_test( NAME bloop_${TOPMODULE}_test COMMAND bloop test WORKING_DIRECTORY
-  # ${SCALA_CORE} )
+    COMMENT "Run bloop from CMake"
+  )
+#   add_test(
+#     NAME bloop_${TOPMODULE}_test
+#     COMMAND bloop test
+#     WORKING_DIRECTORY ${SCALA_CORE}
+#   )
 else()
-  message(STATUS "Building core using sbt")
   set(CHISEL_TARGET sbt_${TOPMODULE})
   set(CHISEL_TEST_TARGET sbt_${TOPMODULE}_test)
   add_custom_command(
@@ -36,16 +40,19 @@ else()
     WORKING_DIRECTORY ${SCALA_CORE}
     DEPENDS ${CHISEL_DEPENDENCY}
     VERBATIM
-    COMMENT "Run sbt from CMake")
+    COMMENT "Run sbt from CMake"
+  )
   add_test(
     NAME sbt_${TOPMODULE}_test
     COMMAND sbt test
-    WORKING_DIRECTORY ${SCALA_CORE})
+    WORKING_DIRECTORY ${SCALA_CORE}
+  )
 endif()
 
 if(NOT EXISTS ${CHISEL_OUTPUT_TOPMODULE})
-  # Probably cold build, generate verilog at configure time to produce top
-  # module file
-  execute_process(COMMAND sbt "run ${CHISEL_EMIT_ARGS}"
-                  WORKING_DIRECTORY ${SCALA_CORE})
+  # Probably cold build, generate verilog at configure time to produce top module file
+  execute_process(
+    COMMAND sbt "run ${CHISEL_EMIT_ARGS}"
+    WORKING_DIRECTORY ${SCALA_CORE}
+  )
 endif()

npc/core/src/main/scala/components/ALU.scala

@@ -6,8 +6,7 @@ import shapeless.{HNil, ::}
 
 class ALUControlInterface extends Bundle {
   object OpSelect extends ChiselEnum {
-    val aOpAdd, aOpSub, aOpNot, aOpAnd, aOpOr, aOpXor, aOpSlt, aOpSltu, aOpSll,
-        aOpSrl, aOpSra = Value
+    val aOpAdd, aOpSub, aOpNot, aOpAnd, aOpOr, aOpXor, aOpSlt, aOpSltu, aOpSll, aOpSrl, aOpSra = Value
   }
   object SrcASelect extends ChiselEnum {
     val aSrcARs1, aSrcAPc, aSrcAZero = Value
@@ -55,21 +54,19 @@ class ALU[T <: UInt](tpe: T) extends Module {
 
   import control.OpSelect._
 
-  out.result := MuxLookup(control.op, 0.U)(
-    Seq(
-      aOpAdd -> add,
-      aOpSub -> sub,
-      aOpNot -> not,
-      aOpAnd -> and,
-      aOpOr -> or,
-      aOpXor -> xor,
-      aOpSlt -> slt,
-      aOpSltu -> sltu,
-      aOpSll -> sll,
-      aOpSrl -> srl,
-      aOpSra -> sra.asUInt
-    )
-  )
+  out.result := MuxLookup(control.op, 0.U)(Seq(
+    aOpAdd -> add,
+    aOpSub -> sub,
+    aOpNot -> not,
+    aOpAnd -> and,
+    aOpOr  -> or,
+    aOpXor -> xor,
+    aOpSlt -> slt,
+    aOpSltu -> sltu,
+    aOpSll -> sll,
+    aOpSrl -> srl,
+    aOpSra -> sra.asUInt
+  ))
 }
 
 object ALU {

npc/core/src/main/scala/components/Mem.scala

@@ -1,15 +1,13 @@
 package flow.components
 
 import chisel3._
-import chisel3.experimental.noPrefix
 import chisel3.util.HasBlackBoxPath
 import chisel3.util.HasBlackBoxResource
 import chisel3.util.log2Ceil
-import flow.components
-import shapeless.::
-import shapeless.HNil
-
+import chisel3.experimental.noPrefix
 import scala.collection.SeqMap
+import flow.components
+import shapeless.{HNil, ::}
 
 class RamControlInterface(addrWidth: Int) extends Bundle {
   val valid = Input(Bool())

npc/csrc/Flow/CMakeLists.txt

@@ -1,6 +1,6 @@
 include(ChiselBuild)
-add_executable(V${TOPMODULE} config.cpp gdbstub_wrapper.cpp main.cpp)
-target_link_libraries(V${TOPMODULE} PRIVATE devices gdbstub spdlog::spdlog)
+add_executable(V${TOPMODULE} config.cpp main.cpp)
+target_link_libraries(V${TOPMODULE} PRIVATE devices gdbstub)
 target_include_directories(V${TOPMODULE} PRIVATE ${CMAKE_SOURCE_DIR}/include)
 
 verilate(
@@ -28,8 +28,8 @@ foreach(DIFFTEST_BINARY_FILE IN LISTS DIFFTEST_BINARY_FILES)
   unset(FILENAME)
 endforeach()
 
-add_library(${TOPMODULE} SHARED config.cpp gdbstub_wrapper.cpp)
-target_link_libraries(${TOPMODULE} PRIVATE devices gdbstub spdlog::spdlog)
+add_library(${TOPMODULE} SHARED config.cpp main.cpp)
+target_link_libraries(${TOPMODULE} PRIVATE devices gdbstub)
 target_include_directories(${TOPMODULE} PRIVATE ${CMAKE_SOURCE_DIR}/include)
 set_property(TARGET PROPERTY POSITION_INDEPENDENT_CODE ON)
 target_link_options(${TOPMODULE} PRIVATE -Wl,-E)

npc/csrc/Flow/config.cpp

@@ -2,15 +2,21 @@
 
 void Config::cli_parse(int argc, char **argv) {
   CLI::App app;
-  app.add_option("-l,--listen", gdbsocket,
-                 "Listen to debugger at this address");
-  app.add_flag("-g", do_debug, "Listen for gdb");
   app.add_option("-m,--memory", memory_file, "Content of memory")
+      ->required()
       ->check(CLI::ExistingFile);
   app.add_flag("!--no-bin", memory_file_binary,
                "Memory file is in text format");
   app.add_option("--wav", wavefile, "output .vcd file path");
   app.add_flag("-i", interactive, "Launch sdb for interactive session");
+  app.add_option("--diff-lib", lib_ref,
+                 "Dynamic library file of difftest reference")
+      ->check(CLI::ExistingFile);
+  app.add_flag("--mtrace", do_mtrace, "Enable memory tracing");
+  app.add_option(
+         "--mtrace-range", mtrace_ranges,
+         "Specify memory tracing range (default: 0x80000000-0x8fffffff)")
+      ->delimiter(',');
 
   try {
     app.parse(argc, argv);

npc/csrc/Flow/gdbstub_wrapper.cpp

@@ -1,196 +0,0 @@
-extern "C" {
-#include <gdbstub.h>
-}
-#include "components.hpp"
-#include <VFlow.h>
-#include <config.hpp>
-#include <cstdint>
-#include <cstdlib>
-#include <devices.hpp>
-#include <types.h>
-#include <vl_wrapper.hpp>
-#include <vpi_user.h>
-#include <vpi_wrapper.hpp>
-
-using Registers = _RegistersVPI<uint32_t, 32>;
-using VlModule = VlModuleInterfaceCommon<VFlow, Registers>;
-
-// SDB::SDB<NPC::npc_interface> sdb_dut;
-bool g_skip_memcheck = false;
-VlModule *top;
-Registers *regs;
-vpiHandle pc = nullptr;
-
-const size_t PMEM_START = 0x80000000;
-const size_t PMEM_END = 0x87ffffff;
-
-extern "C" {
-/* === Memory Access === */
-using MMap = MemoryMap<Memory<128 * 1024>, Devices::DeviceMap>;
-void *pmem_get() {
-  static Devices::DeviceMap devices{
-      new Devices::Serial(0x10000000, 0x1000),
-      new Devices::RTC(0x10001000, 0x1000),
-  };
-  static auto pmem = new MemoryMap<Memory<128 * 1024>, Devices::DeviceMap>(
-      std::make_unique<Memory<128 * 1024>>(config.memory_file, true, PMEM_START,
-                                           PMEM_END),
-      std::make_unique<Devices::DeviceMap>(devices));
-  return pmem;
-}
-
-int pmem_read(int raddr) {
-  void *pmem = pmem_get();
-  auto mem = static_cast<MMap *>(pmem);
-  // TODO: Do memory difftest at memory read and write to diagnose at a finer
-  // granularity
-  mem->trace(raddr, true, regs->get_pc());
-  if (g_skip_memcheck)
-    return mem->read(PMEM_START);
-  return mem->read(raddr);
-}
-
-void pmem_write(int waddr, int wdata, char wmask) {
-  void *pmem = pmem_get();
-  auto mem = static_cast<MMap *>(pmem);
-  mem->trace((std::size_t)waddr, false, regs->get_pc(), wdata);
-  return mem->write((std::size_t)waddr, wdata, wmask);
-}
-
-/* === For gdbstub === */
-
-int npc_read_mem(void *args, size_t addr, size_t len, void *val) {
-  void *pmem = pmem_get();
-  auto mmap = static_cast<MMap *>(pmem);
-  return mmap->copy_to(addr, (uint8_t *)val, len);
-}
-
-int npc_write_mem(void *args, size_t addr, size_t len, void *val) {
-  void *pmem = pmem_get();
-  auto mmap = static_cast<MMap *>(pmem);
-  return mmap->copy_from(addr, (uint8_t *)val, len);
-}
-
-int npc_read_reg(void *args, int regno, size_t *value) {
-  if (regno == 32)
-    *value = regs->get_pc();
-  else
-    *value = (*regs)[regno];
-  return 0;
-}
-
-int npc_write_reg(void *args, int regno, size_t value) { return 1; }
-
-inline void breakpoint_to_action(const Breakpoint *bp, gdb_action_t *res) {
-  if (bp == nullptr) {
-    res->reason = gdb_action_t::ACT_NONE;
-    return;
-  }
-  switch (bp->type) {
-  case BP_SOFTWARE:
-    res->reason = gdb_action_t::ACT_BREAKPOINT;
-    break;
-  case BP_ACCESS:
-    res->reason = gdb_action_t::ACT_WATCH;
-    break;
-  case BP_WRITE:
-    res->reason = gdb_action_t::ACT_WWATCH;
-    break;
-  case BP_READ:
-    res->reason = gdb_action_t::ACT_RWATCH;
-    break;
-  }
-  res->data = bp->addr;
-}
-
-void npc_cont(void *args, gdb_action_t *res) {
-  DbgState *dbg = (DbgState *)args;
-  const Breakpoint *stopped_at = nullptr;
-  stopped_at = top->cont(*dbg->bp);
-  breakpoint_to_action(stopped_at, res);
-}
-
-void npc_stepi(void *args, gdb_action_t *res) {
-  DbgState *dbg = (DbgState *)args;
-  const Breakpoint *stopped_at = nullptr;
-  stopped_at = top->stepi(*dbg->bp);
-  breakpoint_to_action(stopped_at, res);
-}
-
-bool npc_set_bp(void *args, size_t addr, bp_type_t type) {
-  DbgState *dbg = (DbgState *)args;
-  for (const auto &bp : *dbg->bp) {
-    if (bp.addr == addr && bp.type == type) {
-      return true;
-    }
-  }
-  dbg->bp->push_back({.addr = addr, .type = type});
-  return true;
-}
-
-bool npc_del_bp(void *args, size_t addr, bp_type_t type) {
-  DbgState *dbg = (DbgState *)args;
-  for (auto it = dbg->bp->begin(); it != dbg->bp->end(); it++) {
-    if (it->addr == addr && it->type == type) {
-      std::swap(*it, *dbg->bp->rbegin());
-      dbg->bp->pop_back();
-      return true;
-    }
-  }
-  return false;
-}
-
-void npc_on_interrupt(void *args) { ; }
-
-void npc_init(void *args) {
-  DbgState *dbg = (DbgState *)args;
-  void *mem = pmem_get();
-  dbg->bp = new std::vector<Breakpoint>;
-
-  top = new VlModule;
-  regs = new Registers("TOP.Flow.reg_0.regFile_", "TOP.Flow.pc.out");
-  top->setup(config.wavefile, regs);
-  top->reset_eval(10);
-}
-
-bool npc_do_difftest = true;
-
-static gdbstub_t gdbstub_priv;
-arch_info_t npc_isa_arch_info{
-    .target_desc = strdup(TARGET_RV32), .reg_num = 32, .reg_byte = 4};
-
-size_t npc_dbg_state_size = sizeof(DbgState);
-
-} // extern "C"
-
-int gdbstub_loop() {
-  DbgState dbg;
-
-  target_ops npc_gdbstub_ops = {.cont = npc_cont,
-                                .stepi = npc_stepi,
-                                .read_reg = npc_read_reg,
-                                .write_reg = npc_write_reg,
-                                .read_mem = npc_read_mem,
-                                .write_mem = npc_write_mem,
-                                .set_bp = npc_set_bp,
-                                .del_bp = npc_del_bp,
-                                .on_interrupt = NULL};
-
-  if (!config.do_debug) {
-    gdb_action_t res;
-    npc_init(&dbg);
-    npc_cont(&dbg, &res);
-    return !(res.reason == gdb_action_t::ACT_SHUTDOWN);
-  }
-
-  if (!gdbstub_init(&gdbstub_priv, &npc_gdbstub_ops,
-                    (arch_info_t)npc_isa_arch_info, NULL,
-                    config.gdbsocket.c_str())) {
-    return EINVAL;
-  }
-  npc_init(&dbg);
-
-  bool success = gdbstub_run(&gdbstub_priv, &dbg);
-  // gdbstub_close(&gdbstub_priv);
-  return !success;
-}

npc/csrc/Flow/main.cpp

@@ -1,13 +1,180 @@
+extern "C" {
+#include <gdbstub.h>
+}
+// #include "VFlow___024root.h"
+#include "components.hpp"
+#include <VFlow.h>
 #include <config.hpp>
-#include <spdlog/cfg/env.h>
-#include <spdlog/spdlog.h>
+#include <cstdint>
+#include <cstdlib>
+#include <devices.hpp>
+#include <memory>
 #include <types.h>
+#include <vector>
+#include <vl_wrapper.hpp>
+#include <vpi_user.h>
+#include <vpi_wrapper.hpp>
 
-int gdbstub_loop();
+using Registers = _RegistersVPI<uint32_t, 32>;
+using VlModule = VlModuleInterfaceCommon<VFlow, Registers>;
+
+// SDB::SDB<NPC::npc_interface> sdb_dut;
+bool g_skip_memcheck = false;
+VlModule *top;
+Registers *regs;
+vpiHandle pc = nullptr;
+
+const size_t PMEM_START = 0x80000000;
+const size_t PMEM_END = 0x87ffffff;
+
+struct DbgState {
+  std::vector<Breakpoint> *bp;
+};
+
+extern "C" {
+
+/* === Memory Access === */
+using MMap = MemoryMap<Memory<128 * 1024>, Devices::DeviceMap>;
+void *pmem_get() {
+  static Devices::DeviceMap devices{
+      new Devices::Serial(0x10000000, 0x1000),
+      new Devices::RTC(0x10001000, 0x1000),
+  };
+  static auto pmem = new MemoryMap<Memory<128 * 1024>, Devices::DeviceMap>(
+      std::make_unique<Memory<128 * 1024>>(
+          "/nix/store/"
+          "nv2c00y24qwz1jihfbaip9n1lskbzyb3-am-kernel-riscv32-none-elf-2024-07-"
+          "10/share/am-kernels/add.bin",
+          config.memory_file_binary, PMEM_START, PMEM_END),
+      std::make_unique<Devices::DeviceMap>(devices), config.mtrace_ranges);
+  return pmem;
+}
+
+int pmem_read(int raddr) {
+  void *pmem = pmem_get();
+  auto mem = static_cast<MMap *>(pmem);
+  // TODO: Do memory difftest at memory read and write to diagnose at a finer
+  // granularity
+  if (config.do_mtrace)
+    mem->trace(raddr, true, regs->get_pc());
+  if (g_skip_memcheck)
+    return mem->read(PMEM_START);
+  return mem->read(raddr);
+}
+
+void pmem_write(int waddr, int wdata, char wmask) {
+  void *pmem = pmem_get();
+  auto mem = static_cast<MMap *>(pmem);
+  if (config.do_mtrace)
+    mem->trace((std::size_t)waddr, false, regs->get_pc(), wdata);
+  return mem->write((std::size_t)waddr, wdata, wmask);
+}
+
+/* === For gdbstub === */
+
+int npc_read_mem(void *args, size_t addr, size_t len, void *val) {
+  void *pmem = pmem_get();
+  auto mmap = static_cast<MMap *>(pmem);
+  mmap->copy_to(addr, (uint8_t *)val, len);
+  return 0;
+}
+
+int npc_write_mem(void *args, size_t addr, size_t len, void *val) {
+  void *pmem = pmem_get();
+  auto mmap = static_cast<MMap *>(pmem);
+  mmap->copy_from(addr, (uint8_t *)val, len);
+  return 0;
+}
+
+int npc_read_reg(void *args, int regno, size_t *value) {
+  if (regno == 32)
+    *value = regs->get_pc();
+  else
+    *value = (*regs)[regno];
+  return 0;
+}
+
+int npc_write_reg(void *args, int regno, size_t value) { return 1; }
+
+void npc_cont(void *args, gdb_action_t *res) {
+  DbgState *dbg = (DbgState *)args;
+  *res = top->eval(*dbg->bp);
+}
+
+void npc_stepi(void *args, gdb_action_t *res) {
+  DbgState *dbg = (DbgState *)args;
+  *res = top->eval(*dbg->bp);
+}
+
+bool npc_set_bp(void *args, size_t addr, bp_type_t type) {
+  DbgState *dbg = (DbgState *)args;
+  for (const auto &bp : *dbg->bp) {
+    if (bp.addr == addr && bp.type == type) {
+      return true;
+    }
+  }
+  dbg->bp->push_back({.addr = addr, .type = type});
+  return true;
+}
+
+bool npc_del_bp(void *args, size_t addr, bp_type_t type) {
+  DbgState *dbg = (DbgState *)args;
+  for (auto it = dbg->bp->begin(); it != dbg->bp->end(); it++) {
+    if (it->addr == addr && it->type == type) {
+      std::swap(*it, *dbg->bp->rbegin());
+      dbg->bp->pop_back();
+      return true;
+    }
+  }
+  return false;
+}
+
+void npc_on_interrupt(void *args) { ; }
+
+void npc_init(void *args) {
+  DbgState *dbg = (DbgState *)args;
+  dbg->bp = new std::vector<Breakpoint>;
+
+  top = new VlModule;
+  regs = new Registers("TOP.Flow.reg_0.regFile_", "TOP.Flow.pc.out");
+  top->setup(config.wavefile, regs);
+  top->reset_eval(10);
+}
+
+static gdbstub_t gdbstub_priv;
+arch_info_t isa_arch_info = {
+    .target_desc = strdup(TARGET_RV32), .reg_num = 32, .reg_byte = 4};
+
+size_t argsize = sizeof(DbgState);
+
+} // extern "C"
+
+int gdbstub_loop() {
+  DbgState dbg;
+
+  target_ops npc_gdbstub_ops = {.cont = npc_cont,
+                                .stepi = npc_stepi,
+                                .read_reg = npc_read_reg,
+                                .write_reg = npc_write_reg,
+                                .read_mem = npc_read_mem,
+                                .write_mem = npc_write_mem,
+                                .set_bp = npc_set_bp,
+                                .del_bp = npc_del_bp,
+                                .on_interrupt = NULL};
+
+  if (!gdbstub_init(&gdbstub_priv, &npc_gdbstub_ops, (arch_info_t)isa_arch_info,
+                    strdup("/tmp/gdbstub-npc.sock"))) {
+    return EINVAL;
+  }
+  npc_init(&dbg);
+
+  bool success = gdbstub_run(&gdbstub_priv, &dbg);
+  // gdbstub_close(&gdbstub_priv);
+  return !success;
+}
 
 int main(int argc, char **argv, char **env) {
-  spdlog::cfg::load_env_levels();
   config.cli_parse(argc, argv);
-  spdlog::debug("Configuration parsed");
+
   return gdbstub_loop();
 }

npc/csrc/devices/CMakeLists.txt

@@ -1,4 +1,3 @@
 add_library(devices serial.cpp rtc.cpp)
 
 target_include_directories(devices PUBLIC include)
-target_link_libraries(devices PRIVATE spdlog::spdlog)

npc/csrc/devices/include/devices.hpp

@@ -1,12 +1,14 @@
+#include <array>
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
 #include <initializer_list>
 #include <iostream>
+#include <iterator>
 #include <map>
 #include <memory>
-#include <spdlog/logger.h>
-#include <spdlog/sinks/stdout_color_sinks.h>
+#include <unordered_map>
 #include <utility>
 
 namespace Devices {
@@ -53,7 +55,6 @@ public:
 
 class DeviceMap {
   std::map<uint64_t, Device *> addr_to_device;
-  std::shared_ptr<spdlog::logger> logger = spdlog::stdout_color_mt("devicemap");
 
 public:
   DeviceMap(std::initializer_list<Device *> devices) {
@@ -65,8 +66,8 @@ public:
     auto it = addr_to_device.upper_bound(addr);
     if (it == addr_to_device.begin() ||
         (--it)->second->addr + it->second->len <= addr) {
-      logger->error("Accessed an unintialized device at memory addr: 0x{:x}",
-                    addr);
+      std::cerr << "Use of a unintialized device at memory addr: 0x" << std::hex
+                << addr << std::dec << std::endl;
       return false;
     }
     auto &device = it->second;

npc/csrc_nvboard/Keyboard/main.cpp

@@ -49,10 +49,11 @@ int main(int argc, char **argv, char **env) {
   while (true) {
     nvboard_update();
     cycle(top, [&] {
-      if (keycode != top->io_ps2_data) {
+      if (keycode != top->io_ps2_data){
         keycode = top->io_ps2_data;
         printf("%d\n", keycode);
-      });
+      }
+    });
   }
   delete top;
-  }
+}

npc/include/components.hpp

@@ -3,59 +3,51 @@
 #define _NPC_COMPONENTS_H_
 #include "types.h"
 #include <array>
+#include <cmath>
 #include <cstdint>
 #include <cstdlib>
+#include <exception>
 #include <filesystem>
 #include <fstream>
 #include <iostream>
-#include <spdlog/sinks/stdout_color_sinks.h>
-#include <spdlog/spdlog.h>
+#include <map>
+#include <memory>
+#include <sstream>
 #include <stdexcept>
 #include <string>
+#include <vector>
 
 template <typename T, std::size_t nr> class _RegistersBase {
-  std::shared_ptr<spdlog::logger> logger = spdlog::stdout_color_mt("registers");
-
-  virtual T fetch_pc() const = 0;
-  virtual T fetch_reg(std::size_t id) const = 0;
+  std::array<T, nr> regs;
+  T pc;
+  virtual T fetch_pc() const;
+  virtual T fetch_reg(std::size_t id) const;
 
 public:
   T operator[](size_t id) const { return fetch_reg(id); }
   T get_pc() const { return fetch_pc(); }
+  void update() {
+    for (int i = 0; i < regs.size(); i++) {
+      regs[i] = fetch_reg(i);
+    }
+  }
 };
 
+// class MemoryFile {
+//   std::filesystem::path filepath;
+//   public:
+
+// };
+
 template <std::size_t n> class Memory {
+  paddr_t pmem_start, pmem_end;
+
 public:
   std::array<uint8_t, n> mem;
-
+  // TODO: Read memory file before init and use memcpy to initialize memory.
   Memory(std::filesystem::path filepath, bool is_binary, paddr_t pmem_start,
          paddr_t pmem_end)
       : pmem_start(pmem_start), pmem_end(pmem_end) {
-    read_memory(filepath, is_binary);
-  }
-
-  const word_t &operator[](std::size_t addr) { return this->read(addr); }
-
-  void transfer(paddr_t addr, uint8_t data[], size_t len, bool is_write) {
-    if (is_write) {
-      // memcpy(guest_to_host(addr), data, len);
-      size_t offset = (addr - pmem_start);
-      std::copy(data, data + len, &mem[offset]);
-    } else {
-      // memcpy(data, guest_to_host(addr), len);
-      size_t offset = (addr - pmem_start);
-      std::copy(&mem[offset], &mem[offset + len], data);
-    }
-  }
-
-  bool in_pmem(paddr_t addr) const {
-    return addr >= pmem_start && addr <= pmem_end;
-  }
-
-private:
-  paddr_t pmem_start, pmem_end;
-
-  void read_memory(std::filesystem::path filepath, bool is_binary) {
     if (!std::filesystem::exists(filepath))
       throw std::runtime_error("Memory file not found");
     if (is_binary) {
@@ -71,15 +63,35 @@ private:
       }
     }
   }
+  const word_t &operator[](std::size_t addr) { return this->read(addr); }
+  void transfer(paddr_t addr, uint8_t data[], size_t len, bool is_write) {
+    if (is_write) {
+      // memcpy(guest_to_host(addr), data, len);
+      size_t offset = (addr - pmem_start);
+      std::copy(data, data + len, &mem[offset]);
+    } else {
+      // memcpy(data, guest_to_host(addr), len);
+      size_t offset = (addr - pmem_start);
+      std::copy(&mem[offset], &mem[offset + len], data);
+    }
+  }
+  bool in_pmem(paddr_t addr) const {
+    return addr >= pmem_start && addr <= pmem_end;
+  }
 };
 
 template <typename Mem, typename DevMap> class MemoryMap {
-public:
-  MemoryMap(std::unique_ptr<Mem> &&ram,
-            std::unique_ptr<DevMap> &&devices) noexcept
-      : ram(std::move(ram)), devices(std::move(devices)) {}
+  std::unique_ptr<Mem> ram;
+  std::unique_ptr<DevMap> devices;
+  const std::vector<std::array<uint64_t, 2>> &trace_ranges;
 
+public:
+  MemoryMap(std::unique_ptr<Mem> &&ram, std::unique_ptr<DevMap> &&devices,
+            const std::vector<std::array<uint64_t, 2>> &trace_ranges)
+      : ram(std::move(ram)), devices(std::move(devices)),
+        trace_ranges(trace_ranges) {}
   void write(paddr_t waddr, word_t wdata, char wmask) {
+    // printf("waddr: 0x%x\n", waddr);
     size_t len = (wmask & 1) + ((wmask & 2) >> 1) + ((wmask & 4) >> 2) +
                  ((wmask & 8) >> 3);
     if (ram->in_pmem(waddr)) {
@@ -87,43 +99,47 @@ public:
     } else if (devices->handle(waddr, (uint8_t *)&wdata, len, true)) {
     }
   }
-
   word_t read(paddr_t raddr) const {
     word_t res = 0;
+    // printf("raddr: 0x%x, in_pmem: %d\n", raddr, ram->in_pmem(raddr));
     if (ram->in_pmem(raddr)) {
       ram->transfer(raddr, (uint8_t *)&res, 4, false);
     } else if (devices->handle(raddr, (uint8_t *)&res, 4, false)) {
     }
     return res;
   }
-
-  int copy_to(paddr_t addr, uint8_t *buf, size_t len) const {
+  void copy_to(paddr_t addr, uint8_t *buf, size_t len) const {
     if (ram->in_pmem(addr)) {
       ram->transfer(addr, buf, len, false);
-      return 0;
     } else {
-      return EINVAL;
+      std::cerr << "0x" << std::hex << addr << " not in pmem" << std::endl;
     }
   }
-
-  int copy_from(paddr_t addr, uint8_t *buf, size_t len) {
+  void copy_from(paddr_t addr, uint8_t *buf, size_t len) {
     if (ram->in_pmem(addr)) {
       ram->transfer(addr, buf, len, true);
-      return 0;
     } else {
-      return EINVAL;
+      std::cerr << "0x" << std::hex << addr << " not in pmem" << std::endl;
     }
   }
-
   void *get_pmem() { return ram->mem.data(); }
-
   void trace(paddr_t addr, bool is_read, word_t pc = 0, word_t value = 0) {
-    logger->trace("[{}] 0x{:x}", is_read ? 'R' : 'W', this->read(addr));
+    for (auto &r : trace_ranges) {
+      if (r[0] <= addr && r[1] >= addr) {
+        std::stringstream os;
+        if (pc != 0)
+          os << "0x" << std::hex << pc << " ";
+        if (is_read)
+          os << "[R] "
+             << "0x" << addr << ": 0x" << this->read(addr);
+        else
+          os << "[W] " << value << " -> "
+             << "0x" << addr;
+        os << std::dec << std::endl;
+        std::cout << os.rdbuf();
+        break;
+      }
+    }
   }
-
-private:
-  std::unique_ptr<Mem> ram;
-  std::unique_ptr<DevMap> devices;
-  std::shared_ptr<spdlog::logger> logger = spdlog::stdout_color_mt("mmap");
 };
 #endif

npc/include/config.hpp

@@ -3,15 +3,19 @@
 #include <CLI/App.hpp>
 #include <CLI/CLI.hpp>
 #include <CLI/Validators.hpp>
+#include <cstddef>
 #include <filesystem>
+#include <utility>
 
 struct Config {
   std::filesystem::path memory_file;
-  std::string gdbsocket = "gdbstub-npc.sock";
-  bool do_debug{false};
   bool interactive{false};
   bool memory_file_binary = {true};
+  bool do_mtrace{false};
+  std::vector<std::array<std::size_t, 2>> mtrace_ranges{
+      {0x80000000, 0x8ffffffff}};
   std::filesystem::path wavefile;
+  std::filesystem::path lib_ref;
   void cli_parse(int argc, char **argv);
 };
 

npc/include/trm_difftest.hpp

@@ -0,0 +1,102 @@
+#ifndef _DIFFTEST_DIFFTEST_H_
+#define _DIFFTEST_DIFFTEST_H_
+#include "disasm.hpp"
+#include "types.h"
+#include <cassert>
+#include <components.hpp>
+#include <cstdint>
+#include <cstdlib>
+#include <dlfcn.h>
+#include <filesystem>
+#include <functional>
+#include <iostream>
+#include <map>
+#include <memory>
+#include <ostream>
+#include <stdexcept>
+#include <trm_interface.hpp>
+
+using paddr_t = uint32_t;
+struct DifftestTrmInterface : public TrmInterface {
+  TrmInterface &dut;
+  TrmInterface &ref;
+
+  DifftestTrmInterface(TrmInterface &dut, TrmInterface &ref, void *mem,
+                       size_t mem_size)
+      : dut(dut), ref(ref) {
+    init = [this, mem, mem_size](int n) {
+      this->ref.init(n);
+      this->dut.init(n);
+      paddr_t reset_vector = 0x80000000;
+      this->ref.memcpy(reset_vector, mem, mem_size, TRM_TO_MACHINE);
+      this->dut.memcpy(reset_vector, mem, mem_size, TRM_TO_MACHINE);
+      fetch_state();
+    };
+    exec = [this](uint64_t n) {
+      bool enable_disasm = true;
+      if (n > 30) {
+        enable_disasm = false;
+      }
+
+      while (n--) {
+        word_t pc = this->ref.at("pc");
+        word_t inst = this->ref.at(pc);
+        if (enable_disasm)
+          std::cout << d.disassemble(pc, (uint8_t *)&inst, WORD_BYTES)
+                    << std::endl;
+        if (inst == 1048691) {
+          // ebreak
+          throw TrmRuntimeException(TrmRuntimeException::EBREAK, "ebreak");
+        }
+        this->ref.exec(1);
+        this->dut.exec(1);
+        this->ref.fetch_state();
+        this->dut.fetch_state();
+        if (*(CPUState *)this->ref.cpu_state !=
+            *(CPUState *)this->dut.cpu_state) {
+          throw TrmRuntimeException(TrmRuntimeException::DIFFTEST_FAILED,
+                                    "Difftest failed");
+        }
+      }
+    };
+
+    // NOTE: Different from normal Trm, we copy 2 * sizeof(CPUState) to/from p,
+    // which represents ref_state and dut state
+    regcpy = [this](void *p, bool direction) {
+      // this->ref.regcpy(p, direction);
+      // this->dut.regcpy(p, direction);
+    };
+
+    memcpy = [this](paddr_t paddr, void *p, size_t n, bool direction) {
+      this->dut.memcpy(paddr, p, n, direction);
+      this->ref.memcpy(paddr, (uint8_t *)p + n, n, direction);
+    };
+  }
+
+  word_t at(std::string name) const override {
+    if (name.empty()) {
+      throw std::runtime_error("Empty register name");
+    } else if (name[0] == 'r') {
+      std::cout << name.substr(1) << std::endl;
+      this->ref.at(name.substr(1));
+    } else if (name[0] == 'd') {
+      this->dut.at(name.substr(1));
+    } else {
+      throw std::runtime_error("Register name provided to difftest interface "
+                               "must start with r or d.");
+    }
+    return 0;
+  }
+
+  word_t at(paddr_t addr) const override {
+    std::cout << ref.at(addr) << "\t" << dut.at(addr) << std::endl;
+    return dut.at(addr);
+  }
+
+  void print(std::ostream &os) const override {
+    os << "REF state:\n"
+       << *(CPUState *)ref.cpu_state << "DUT state:\n"
+       << *(CPUState *)dut.cpu_state << std::endl;
+  }
+};
+#endif

npc/include/trm_interface.hpp

@@ -0,0 +1,158 @@
+#ifndef _NPC_TRM_INTERFACE_HEADER_FILE_
+#define _NPC_TRM_INTERFACE_HEADER_FILE_
+#include <disasm.hpp>
+#include <dlfcn.h>
+#include <filesystem>
+#include <functional>
+#include <stdexcept>
+#include <string>
+#include <types.h>
+
+extern Disassembler d;
+
+template <typename R, size_t nr_reg> struct CPUStateBase {
+  R reg[nr_reg] = {0};
+  word_t pc = 0x80000000;
+
+  static const std::map<std::string, int> inline regs_by_name =
+      riscv32_regs_by_name;
+  CPUStateBase() {
+    for (int i = 0; i < nr_reg; i++)
+      reg[i] = 0;
+  }
+
+  bool operator==(const CPUStateBase &other) const {
+    if (pc != other.pc)
+      return false;
+    for (int i = 0; i < nr_reg; ++i) {
+      if (reg[i] != other.reg[i])
+        return false;
+    }
+    return true;
+  }
+
+  bool operator!=(const CPUStateBase &other) const {
+    return !(*this == other); // Reuse the == operator for != implementation
+  }
+
+  /* This does not update the register!!! */
+  R at(std::string name) {
+    // FIXME: Using this to get pc seems broken
+    return name == "pc" ? pc : reg[regs_by_name.at(name)];
+  }
+
+  uint32_t reg_str2val(const char *name, bool *success) {
+    try {
+      *success = true;
+      return this->at(name);
+    } catch (std::runtime_error) {
+      *success = false;
+      return 0;
+    }
+  }
+};
+
+template <typename R, size_t nr_reg>
+std::ostream &operator<<(std::ostream &os, const CPUStateBase<R, nr_reg> &cpu) {
+  os << "PC: " << std::hex << cpu.pc << std::endl;
+  for (int i = 0; i < nr_reg; i++) {
+    os << "reg " << std::dec << std::setw(2) << i << ":" << std::hex
+       << std::setw(10) << cpu.reg[i];
+    if (i % 4 == 3) {
+      os << std::endl;
+    } else {
+      os << " | ";
+    }
+  }
+  return os;
+}
+
+using CPUState = CPUStateBase<word_t, REG_COUNT>;
+
+enum { TRM_FROM_MACHINE, TRM_TO_MACHINE };
+
+class TrmInterface {
+protected:
+  using memcpy_t = void (*)(paddr_t, void *, size_t, bool);
+  using regcpy_t = void (*)(void *, bool);
+  using exec_t = void (*)(uint64_t);
+  using init_t = void (*)(int);
+  std::function<void(void *, bool)> regcpy;
+
+public:
+  std::function<void(uint64_t)> exec;
+  std::function<void(int)> init;
+  // TODO: paddr_t can probably changed to (void *)?
+  std::function<void(paddr_t, void *, size_t, bool)> memcpy;
+  // Managed by callee
+  void *cpu_state;
+
+  TrmInterface() {}
+  TrmInterface(memcpy_t f_memcpy, regcpy_t f_regcpy, exec_t f_exec,
+               init_t f_init, void *cpu_state)
+      : memcpy(f_memcpy), regcpy(f_regcpy), exec(f_exec), init(f_init),
+        cpu_state(cpu_state) {}
+
+  void fetch_state() { this->regcpy(cpu_state, TRM_FROM_MACHINE); }
+  void push_state() { this->regcpy(cpu_state, TRM_TO_MACHINE); }
+  virtual word_t at(std::string) const = 0;
+  virtual word_t at(word_t addr) const = 0;
+  virtual void print(std::ostream &os) const = 0;
+};
+
+class TrmRuntimeException : public std::exception {
+private:
+  const char *msg_;
+  int code_;
+
+public:
+  enum { EBREAK, DIFFTEST_FAILED };
+  TrmRuntimeException(int code, const char *message)
+      : code_(code), msg_(message) {}
+
+  virtual const char *what() const throw() { return msg_; }
+
+  int error_code() const { return code_; }
+};
+
+struct RefTrmInterface : TrmInterface {
+  RefTrmInterface(std::filesystem::path lib_file) {
+    void *handle = dlopen(lib_file.c_str(), RTLD_LAZY);
+    if (handle == nullptr) {
+      throw std::runtime_error("Failed to open diff library file");
+    };
+    memcpy = (memcpy_t)dlsym(handle, "difftest_memcpy");
+    if (handle == nullptr) {
+      throw std::runtime_error("Failed to find `difftest_memcpy`");
+    };
+    regcpy = (regcpy_t)dlsym(handle, "difftest_regcpy");
+    if (handle == nullptr) {
+      throw std::runtime_error("Failed to find `difftest_regcpy`");
+    };
+    exec = (exec_t)dlsym(handle, "difftest_exec");
+    if (handle == nullptr) {
+      throw std::runtime_error("Failed to find `difftest_exec`");
+    };
+    init = (init_t)dlsym(handle, "difftest_init");
+    if (handle == nullptr) {
+      throw std::runtime_error("Failed to find `difftest_init`");
+    };
+    cpu_state = new CPUState{};
+  }
+
+  ~RefTrmInterface() { delete (CPUState *)cpu_state; }
+
+  word_t at(std::string name) const override {
+    return ((CPUState *)cpu_state)->at(name);
+  }
+
+  word_t at(paddr_t addr) const override {
+    word_t buf;
+    this->memcpy(addr, &buf, sizeof(word_t), TRM_FROM_MACHINE);
+    return buf;
+  }
+
+  void print(std::ostream &os) const override { os << *(CPUState *)cpu_state; }
+};
+
+#endif

npc/include/types.h

@@ -1,10 +1,6 @@
 #ifndef _NPC_TYPES_H__
 #define _NPC_TYPES_H__
-#ifdef __cplusplus
-extern "C" {
-#endif
-#include <gdbstub.h>
-#include <stdint.h>
+#include <inttypes.h>
 
 typedef uint32_t word_t;
 typedef int32_t sword_t;
@@ -14,25 +10,29 @@ static const sword_t SWORD_T_MIN = INT32_MIN;
 #define WORD_BYTES 4
 #define REG_COUNT 32
 
+#define FMT_WORD "0x%08x"
 typedef uint32_t vaddr_t;
 typedef uint32_t paddr_t;
+#define FMT_ADDR "0x%08x"
 typedef uint16_t ioaddr_t;
 
+#ifdef __cplusplus
+#include <map>
+#include <string>
+
+const std::map<std::string, int> riscv32_regs_by_name{
+    {"$0", 0},  {"ra", 1},  {"sp", 2},   {"gp", 3},   {"tp", 4},  {"t0", 5},
+    {"t1", 6},  {"t2", 7},  {"s0", 8},   {"s1", 9},   {"a0", 10}, {"a1", 11},
+    {"a2", 12}, {"a3", 13}, {"a4", 14},  {"a5", 15},  {"a6", 16}, {"a7", 17},
+    {"s2", 18}, {"s3", 19}, {"s4", 20},  {"s5", 21},  {"s6", 22}, {"s7", 23},
+    {"s8", 24}, {"s9", 25}, {"s10", 26}, {"s11", 27}, {"t3", 28}, {"t4", 29},
+    {"t5", 30}, {"t6", 31}};
+#endif
+
+#include <gdbstub.h>
 struct Breakpoint {
   size_t addr;
   bp_type_t type;
 };
 
-#ifdef __cplusplus
-}
-#endif
-
-#ifdef __cplusplus
-#include <vector>
-
-struct DbgState {
-  std::vector<Breakpoint> *bp;
-};
-#endif
-
 #endif

npc/include/vl_wrapper.hpp

@@ -2,7 +2,6 @@
 #define _NPC_TRACER_H_
 #include "components.hpp"
 #include "types.h"
-#include "verilated.h"
 #include <filesystem>
 #include <gdbstub.h>
 #include <sys/types.h>
@@ -48,7 +47,7 @@ public:
     registers = r;
   }
 
-  void eval(void) {
+  void eval() {
     if (this->is_posedge()) {
       posedge_cnt++;
     }
@@ -58,34 +57,35 @@ public:
     if (tracer)
       tracer->update();
   }
-
-  const Breakpoint *stepi(const std::vector<Breakpoint> &breakpoints) {
-    this->eval();
-    this->eval();
-    size_t pc = registers->get_pc();
-    for (const auto &bp : breakpoints) {
-      if (pc == bp.addr) {
-        return &bp;
-      }
+  void eval(int n) {
+    for (int i = 0; i < n; i++) {
+      this->eval();
     }
-    return nullptr;
   }
 
-  const Breakpoint *cont(const std::vector<Breakpoint> &breakpoints) {
-    const Breakpoint *res = nullptr;
+  gdb_action_t eval(const std::vector<Breakpoint> &breakpoints) {
+    gdb_action_t res;
     do {
-      res = stepi(breakpoints);
-    } while (res == nullptr);
-    return res;
+      this->eval();
+      size_t pc = registers->get_pc();
+      for (const auto &bp : breakpoints) {
+        if (pc == bp.addr) {
+          res.data = bp.addr;
+          switch (bp.type) {
+          default:
+            res.reason = gdb_action_t::ACT_BREAKPOINT;
+          }
+          return res;
+        }
+      }
+    } while (true);
   }
 
   void reset_eval(int n) {
     extern bool g_skip_memcheck;
     g_skip_memcheck = true;
     this->reset = 1;
-    do {
-      this->eval();
-    } while (--n);
+    this->eval(n);
     this->reset = 0;
     g_skip_memcheck = false;
   }

npc/include/vpi_wrapper.hpp

@@ -1,16 +1,12 @@
 #ifndef _NPC_VPI_WRAPPER_H_
 #define _NPC_VPI_WRAPPER_H_
 #include <components.hpp>
-#include <spdlog/logger.h>
-#include <spdlog/sinks/stdout_color_sinks.h>
-#include <spdlog/spdlog.h>
 #include <vpi_user.h>
 
 template <typename T, std::size_t nr>
 class _RegistersVPI : public _RegistersBase<T, nr> {
   std::array<vpiHandle, nr> reg_handles;
   vpiHandle pc_handle;
-
   T vpi_get(vpiHandle vh) const {
     s_vpi_value v;
     v.format = vpiIntVal;
@@ -23,11 +19,6 @@ class _RegistersVPI : public _RegistersBase<T, nr> {
 public:
   _RegistersVPI<T, nr>(const std::string regs_prefix,
                        const std::string pcname) {
-    init_handlers(regs_prefix, pcname);
-  }
-
-private:
-  void init_handlers(const std::string regs_prefix, const std::string pcname) {
     for (int i = 0; i < nr; i++) {
       std::string regname = regs_prefix + std::to_string(i);
       vpiHandle vh = vpi_handle_by_name((PLI_BYTE8 *)regname.c_str(), nullptr);