feat: difftest framework

.clang-format

@@ -0,0 +1,4 @@
+---
+Language: Cpp
+BasedOnStyle: LLVM
+ReflowComments: false

nemu/.clang-format

@@ -1,3 +0,0 @@
----
-Language: Cpp
-BasedOnStyle: LLVM

nemu/Kconfig

@@ -126,6 +126,26 @@ endmenu
 
 menu "Testing and Debugging"
 
+choice
+  prompt "Choose log level"
+  default LOG_TRACE
+config LOG_TRACE
+  bool "trace"
+config LOG_INFO
+  bool "info"
+config LOG_WARNING
+  bool "warning"
+config LOG_ERROR
+  bool "error"
+endchoice
+
+config LOG_LEVEL
+  int
+  default 1 if LOG_ERROR
+  default 2 if LOG_WARNING
+  default 3 if LOG_INFO
+  default 4 if LOG_TRACE
+  default 0
 
 config TRACE
   bool "Enable tracer"

nemu/include/debug.h

@@ -16,33 +16,51 @@
 #ifndef __DEBUG_H__
 #define __DEBUG_H__
 
+#include <macro.h>
 #include <stdio.h>
 #include <utils.h>
-#include <macro.h>
 
 IFDEF(CONFIG_ITRACE, void log_itrace_print());
 
-#define Trace(format, ...) \
+#if (CONFIG_LOG_LEVEL >= 4)
-    _Log("[TRACE] " format "\n", ## __VA_ARGS__)
+#define Trace(format, ...) _Log("[TRACE] " format "\n", ##__VA_ARGS__)
+#else
+#define Trace(format, ...)
+#endif
 
+#if (CONFIG_LOG_LEVEL >= 3)
 #define Log(format, ...)                                                       \
-    _Log(ANSI_FMT("[INFO] %s:%d %s() ", ANSI_FG_BLUE) format "\n", \
+  _Log(ANSI_FMT("[INFO] %s:%d %s() ", ANSI_FG_BLUE) format "\n", __FILE__,     \
-        __FILE__, __LINE__, __func__, ## __VA_ARGS__)
+       __LINE__, __func__, ##__VA_ARGS__)
+#else
+#define Log(format, ...)
+#endif
 
+#if (CONFIG_LOG_LEVEL >= 2)
 #define Warning(format, ...)                                                   \
   _Log(ANSI_FMT("[WARNING] %s:%d %s() ", ANSI_FG_YELLOW) format "\n",          \
-        __FILE__, __LINE__, __func__, ## __VA_ARGS__)
+       __FILE__, __LINE__, __func__, ##__VA_ARGS__)
+#else
+#define Warning(format, ...)
+#endif
 
+#if (CONFIG_LOG_LEVEL >= 1)
 #define Error(format, ...)                                                     \
-    _Log(ANSI_FMT("[ERROR] %s:%d %s() ", ANSI_FG_RED) format "\n", \
+  _Log(ANSI_FMT("[ERROR] %s:%d %s() ", ANSI_FG_RED) format "\n", __FILE__,     \
-        __FILE__, __LINE__, __func__, ## __VA_ARGS__)
+       __LINE__, __func__, ##__VA_ARGS__)
+#else
+#define Error(format, ...)
+#endif
 
 #define Assert(cond, format, ...)                                              \
   do {                                                                         \
     if (!(cond)) {                                                             \
-      MUXDEF(CONFIG_TARGET_AM, printf(ANSI_FMT(format, ANSI_FG_RED) "\n", ## __VA_ARGS__), \
+      MUXDEF(                                                                  \
-        (fflush(stdout), fprintf(stderr, ANSI_FMT(format, ANSI_FG_RED) "\n", ##  __VA_ARGS__))); \
+          CONFIG_TARGET_AM,                                                    \
-      IFNDEF(CONFIG_TARGET_AM, extern FILE* log_fp; fflush(log_fp)); \
+          printf(ANSI_FMT(format, ANSI_FG_RED) "\n", ##__VA_ARGS__),           \
+          (fflush(stdout), fprintf(stderr, ANSI_FMT(format, ANSI_FG_RED) "\n", \
+                                   ##__VA_ARGS__)));                           \
+      IFNDEF(CONFIG_TARGET_AM, extern FILE *log_fp; fflush(log_fp));           \
       IFDEF(CONFIG_ITRACE, log_itrace_print());                                \
       extern void assert_fail_msg();                                           \
       assert_fail_msg();                                                       \
@@ -50,7 +68,7 @@ IFDEF(CONFIG_ITRACE, void log_itrace_print());
     }                                                                          \
   } while (0)
 
-#define panic(format, ...) Assert(0, format, ## __VA_ARGS__)
+#define panic(format, ...) Assert(0, format, ##__VA_ARGS__)
 
 #define TODO() panic("please implement me")
 

nemu/src/cpu/difftest/ref.c

@@ -13,25 +13,34 @@
 * See the Mulan PSL v2 for more details.
 ***************************************************************************************/
 
-#include <isa.h>
+#include "types.h"
 #include <cpu/cpu.h>
+#include <cpu/decode.h>
 #include <difftest-def.h>
+#include <isa.h>
 #include <memory/paddr.h>
 
-__EXPORT void difftest_memcpy(paddr_t addr, void *buf, size_t n, bool direction) {
+__EXPORT void difftest_memcpy(paddr_t addr, void *buf, size_t n,
+                              bool direction) {
+  if (direction == DIFFTEST_TO_REF) {
+    memcpy(guest_to_host(addr), buf, n);
+  } else {
     assert(0);
+  }
 }
 
 __EXPORT void difftest_regcpy(void *dut, bool direction) {
-  assert(0);
+  // assert(0);
+  if (direction == DIFFTEST_TO_DUT)
+    memcpy(dut, &cpu, sizeof(CPU_state));
+  else
+    memcpy(&cpu, dut, sizeof(CPU_state));
 }
 
-__EXPORT void difftest_exec(uint64_t n) {
+__EXPORT void difftest_exec(uint64_t n) { cpu_exec(n); }
-  assert(0);
-}
 
 __EXPORT void difftest_raise_intr(word_t NO) {
-  assert(0);
+  // assert(0);
 }
 
 __EXPORT void difftest_init(int port) {

npc/.clang-format

@@ -1,2 +0,0 @@
----
-BasedOnStyle: LLVM

npc/CMakeLists.txt

@@ -135,6 +135,8 @@ if(BUILD_SIM_NVBOARD_TARGET)
 endif()
 
 # -- Build Verilator executable and add to test
+include_directories(include)
+
 file(GLOB_RECURSE SOURCES csrc/${TOPMODULE}/*.cpp)
 add_executable(V${TOPMODULE} ${SOURCES})
 

npc/core/src/main/scala/ALU.scala

@@ -9,7 +9,7 @@ class ALUControlInterface extends Bundle {
     val aOpAdd, aOpSub, aOpNot, aOpAnd, aOpOr, aOpXor, aOpSlt, aOpEq, aOpNop = Value
   }
   object SrcSelect extends ChiselEnum {
-    val aSrcRs1, aSrcImm = Value
+    val aSrcRs2, aSrcImm = Value
   }
   val op = Input(OpSelect())
   val src = Input(SrcSelect())

npc/core/src/main/scala/FlowMain.scala

@@ -87,7 +87,6 @@ class Flow extends Module {
   ram.io.readAddr := pc.out
   val inst = ram.io.readData
 
-  Trace.traceName(reg.control.writeEnable)
   dontTouch(reg.control.writeEnable)
 
   import control.pc.SrcSelect._
@@ -107,8 +106,8 @@ class Flow extends Module {
   reg.in.writeData(rMemOut.litValue.toInt) := DontCare
 
   reg.in.writeAddr := inst(11, 7)
-  reg.in.rs(0) := inst(19, 15)
+  reg.in.rs(0) := inst(19, 15) // rs1
-  reg.in.rs(1) := inst(24, 20)
+  reg.in.rs(1) := inst(24, 20) // rs2
 
   // TODO: Memory write goes here
   ram.io.writeAddr := DontCare
@@ -118,10 +117,10 @@ class Flow extends Module {
   ram.io.valid := true.B
 
   import control.alu.SrcSelect._
-  alu.in.a(aSrcRs1.litValue.toInt) := reg.out.src(0)
+  alu.in.a(aSrcRs2.litValue.toInt) := reg.out.src(1)
   alu.in.a(aSrcImm.litValue.toInt) := inst(31, 20)
-  alu.in.b := reg.out.src(1)
+  alu.in.b := reg.out.src(0)
-
 
+  Trace.traceName(pc.out);
   dontTouch(control.out)
 }

npc/csrc/Flow/components.hpp

@@ -1,53 +1,61 @@
 
 #ifndef _NPC_COMPONENTS_H_
 #define _NPC_COMPONENTS_H_
+#include "vpi_user.h"
 #include <array>
+#include <cstdlib>
 #include <filesystem>
 #include <fstream>
+#include <iostream>
 #include <verilated_vpi.h>
 
 template <typename T, std::size_t nr> class _RegistersBase {
   std::array<T, nr> regs;
+  T pc;
+  virtual T fetch_pc();
   virtual T fetch_reg(std::size_t id);
 
 public:
+  T operator[](size_t id) { return fetch_reg(id); }
+  T get_pc() { return fetch_pc(); }
   void update() {
     for (int i = 0; i < regs.size(); i++) {
       regs[i] = fetch_reg(i);
     }
   }
-  void print_regs() {
-    for (int i = 0; i < regs.size(); i++) {
-      printf("%d: %d\t", i, regs[i]);
-      if (i % 8 == 7)
-        putchar('\n');
-    }
-    putchar('\n');
-  }
 };
 
 template <typename T, std::size_t nr>
 class _RegistersVPI : public _RegistersBase<T, nr> {
   std::array<vpiHandle, nr> reg_handles;
-  T fetch_reg(std::size_t id) {
+  vpiHandle pc_handle;
+  T vpi_get(vpiHandle vh) {
     s_vpi_value v;
     v.format = vpiIntVal;
-    vpi_get_value(reg_handles[id], &v);
+    vpi_get_value(vh, &v);
     return v.value.integer;
   }
+  T fetch_pc(void) { return vpi_get(pc_handle); }
+  T fetch_reg(std::size_t id) { return vpi_get(reg_handles[id]); }
 
 public:
-  _RegistersVPI<T, nr>(const std::string regs_prefix) {
+  _RegistersVPI<T, nr>(const std::string regs_prefix,
+                       const std::string pcname) {
     for (int i = 0; i < nr; i++) {
       std::string regname = regs_prefix + std::to_string(i);
-      vpiHandle vh = vpi_handle_by_name((PLI_BYTE8 *)regname.c_str(), NULL);
+      vpiHandle vh = vpi_handle_by_name((PLI_BYTE8 *)regname.c_str(), nullptr);
+      if (vh == nullptr) {
+        std::cerr << "vpiHandle " << regname.c_str() << " not found"
+                  << std::endl;
+        exit(EXIT_FAILURE);
+      }
       reg_handles[i] = vh;
     }
+    pc_handle = vpi_handle_by_name((PLI_BYTE8 *)pcname.c_str(), nullptr);
   }
 };
 
 template <typename T, std::size_t n> class Memory {
-  std::array<T, n> mem;
   std::size_t addr_to_index(std::size_t addr) {
     if (addr < 0x80000000) {
       return 0;
@@ -64,12 +72,13 @@ template <typename T, std::size_t n> class Memory {
   }
 
 public:
+  std::array<T, n> mem;
   Memory(std::filesystem::path filepath, bool is_binary = true) {
     assert(std::filesystem::exists(filepath));
     if (is_binary) {
       std::ifstream file(filepath, std::ios::binary);
       char *pmem = reinterpret_cast<char *>(mem.data());
-      file.read(pmem, mem.size() / sizeof(mem[0]));
+      file.read(pmem, mem.size() * sizeof(mem[0]));
     } else {
       std::string line;
       std::ifstream file(filepath);
@@ -84,7 +93,7 @@ public:
    * Always reads and returns 4 bytes from the address raddr & ~0x3u.
    */
   T read(int raddr) {
-    printf("raddr: 0x%x\n", raddr);
+    // printf("raddr: 0x%x\n", raddr);
     return mem[addr_to_index((uint32_t)raddr)];
   }
   /**
@@ -94,7 +103,7 @@ public:
    * and the other bytes in memory remain unchanged.
    */
   void write(int waddr, T wdata, char wmask) {
-    printf("waddr: 0x%x\n", waddr);
+    // printf("waddr: 0x%x\n", waddr);
     mem[addr_to_index((uint32_t)waddr)] = expand_bits(wmask) & wdata;
   }
 };

npc/csrc/Flow/config.cpp

@@ -9,13 +9,16 @@ void Config::cli_parse(int argc, char **argv) {
                "Memory file is in text format");
   app.add_flag("--trace", do_trace, "Enable tracing");
   app.add_option("--wav", wavefile, "output .vcd file path")
-      ->check([this](const std::string &) {
+      ->check([=](const std::string &) {
-        if (!this->do_trace)
+        if (!do_trace)
           throw CLI::ValidationError(
               "dependency", "You must turn on trace before specify wave file");
         return std::string();
       });
   app.add_option("-t", max_sim_time, "Max simulation timestep");
+  app.add_option("--diff-lib", lib_ref,
+                 "Dynamic library file of difftest reference")
+      ->check(CLI::ExistingFile);
 
   try {
     app.parse(argc, argv);

npc/csrc/Flow/config.hpp

@@ -6,11 +6,12 @@
 #include <filesystem>
 
 struct Config {
-  std::filesystem::path wavefile;
   std::filesystem::path memory_file;
   uint64_t max_sim_time = 1000;
   bool memory_file_binary = {true};
   bool do_trace{false};
+  std::filesystem::path wavefile;
+  std::filesystem::path lib_ref;
   void cli_parse(int argc, char **argv);
 };
 

npc/csrc/Flow/main.cpp

@@ -1,7 +1,10 @@
 #include "components.hpp"
 #include "config.hpp"
 #include "vl_wrapper.hpp"
+#include "vpi_user.h"
 #include <VFlow.h>
+#include <cstdint>
+#include <difftest.hpp>
 
 using VlModule = VlModuleInterfaceCommon<VFlow>;
 using Registers = _RegistersVPI<uint32_t, 32>;
@@ -16,6 +19,8 @@ void *pmem_get() {
 int pmem_read(int raddr) {
   void *pmem = pmem_get();
   auto mem = static_cast<Memory<int, 128 * 1024> *>(pmem);
+  // TODO: Do memory difftest at memory read and write to diagnose at a finer
+  // granularity
   return mem->read(raddr);
 }
 
@@ -26,19 +31,55 @@ void pmem_write(int waddr, int wdata, char wmask) {
 }
 }
 
-int main(int argc, char **argv, char **env) {
+VlModule *top;
-  config.cli_parse(argc, argv);
+Registers *regs;
-  auto top = std::make_shared<VlModule>(config.do_trace, config.wavefile);
+using CPUState = CPUStateBase<uint32_t, 32>;
-  Registers regs("TOP.Flow.reg_0.regFile_");
+vpiHandle pc = nullptr;
+void difftest_memcpy(paddr_t, void *, size_t, bool){};
 
-  top->reset_eval(10);
+void difftest_regcpy(void *p, bool direction) {
-  for (int i = 0; i < config.max_sim_time; i++) {
+
+  if (direction == DIFFTEST_FROM_REF) {
+    ((CPUState *)p)->pc = regs->get_pc();
+    for (int i = 0; i < 32; i++) {
+      ((CPUState *)p)->reg[i] = (*regs)[i];
+    }
+  }
+}
+
+void difftest_exec(uint64_t n) {
+  while (n--) {
+    for (int i = 0; i < 2; i++) {
       if (top->is_posedge()) {
         // Posedge
-      regs.update();
+        regs->update();
-      regs.print_regs();
       }
       top->eval();
     }
+  }
+}
+void difftest_init(int port) {
+  //   top = std::make_unique<VlModule>(config.do_trace, config.wavefile);
+  top = new VlModule{config.do_trace, config.wavefile};
+  regs = new Registers("TOP.Flow.reg_0.regFile_", "TOP.Flow.pc.out");
+  top->reset_eval(10);
+}
+
+int main(int argc, char **argv, char **env) {
+  config.cli_parse(argc, argv);
+
+  /* -- Difftest -- */
+  std::filesystem::path ref{config.lib_ref};
+  DifftestInterface dut_interface = DifftestInterface{
+      &difftest_memcpy, &difftest_regcpy, &difftest_exec, &difftest_init};
+  DifftestInterface ref_interface = DifftestInterface{ref};
+
+  Difftest<CPUStateBase<uint32_t, 32>> diff{dut_interface, ref_interface,
+                                            pmem_get(), 128};
+  int t = 8;
+  while (t--) {
+    diff.step(1);
+  }
+
   return 0;
 }

npc/include/difftest.hpp

@@ -0,0 +1,126 @@
+#ifndef _DIFFTEST_DIFFTEST_H_
+#define _DIFFTEST_DIFFTEST_H_
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <dlfcn.h>
+#include <filesystem>
+#include <functional>
+#include <iostream>
+#include <memory>
+
+using paddr_t = uint32_t;
+enum { DIFFTEST_FROM_REF, DIFFTEST_TO_REF };
+
+struct DifftestInterface {
+  using memcpy_t = void (*)(paddr_t, void *, size_t, bool);
+  using regcpy_t = void (*)(void *, bool);
+  using exec_t = void (*)(uint64_t);
+  using init_t = void (*)(int);
+  std::function<void(paddr_t, void *, size_t, bool)> memcpy;
+  std::function<void(void *, bool)> regcpy;
+  std::function<void(uint64_t)> exec;
+  std::function<void(int)> init;
+
+  DifftestInterface(memcpy_t memcpy, regcpy_t regcpy, exec_t exec, init_t init)
+      : memcpy(memcpy), regcpy(regcpy), exec(exec), init(init){};
+
+  // using fs = std::filesystem::path;
+  DifftestInterface(std::filesystem::path lib_file) {
+    void *handle = dlopen(lib_file.c_str(), RTLD_LAZY);
+    assert(handle != nullptr);
+    memcpy = (memcpy_t)dlsym(handle, "difftest_memcpy");
+    assert(memcpy);
+    regcpy = (regcpy_t)dlsym(handle, "difftest_regcpy");
+    assert(regcpy);
+    exec = (exec_t)dlsym(handle, "difftest_exec");
+    assert(exec);
+    init = (init_t)dlsym(handle, "difftest_init");
+    assert(init);
+  }
+};
+
+template <typename S> class Difftest {
+  const DifftestInterface &ref;
+  std::unique_ptr<S> ref_state;
+  const DifftestInterface &dut;
+  std::unique_ptr<S> dut_state;
+
+public:
+  Difftest(const DifftestInterface &dut, const DifftestInterface &ref,
+           void *mem, size_t n, std::unique_ptr<S> ref_state = nullptr,
+           std::unique_ptr<S> dut_state = nullptr)
+      : ref(ref), dut(dut), ref_state(std::move(ref_state)),
+        dut_state(std::move(dut_state)) {
+    if (ref_state == nullptr)
+      this->ref_state = std::make_unique<S>();
+    if (dut_state == nullptr)
+      this->dut_state = std::make_unique<S>();
+    ref.init(0);
+    dut.init(0);
+    fetch_state();
+    paddr_t reset_vector = 0x80000000;
+    ref.memcpy(reset_vector, mem, n, DIFFTEST_TO_REF);
+    dut.memcpy(reset_vector, mem, n, DIFFTEST_TO_REF);
+  };
+
+  void fetch_state() {
+    ref.regcpy(ref_state.get(), DIFFTEST_FROM_REF);
+    dut.regcpy(dut_state.get(), DIFFTEST_FROM_REF);
+  }
+
+  void step(uint64_t n) {
+    ref.exec(n);
+    dut.exec(n);
+    fetch_state();
+    if (*ref_state != *dut_state) {
+      std::cout << *this;
+      exit(EXIT_FAILURE);
+    }
+  }
+
+  friend std::ostream &operator<<(std::ostream &os, const Difftest<S> &d) {
+    os << "REF state:\n"
+       << *d.ref_state << "DUT state:\n"
+       << *d.dut_state << std::endl;
+    return os;
+  }
+};
+
+template <typename R, size_t nr_reg> struct CPUStateBase {
+  R reg[nr_reg] = {0};
+  paddr_t pc = 0x80000000;
+  CPUStateBase() {
+    for (int i = 0; i < nr_reg; i++)
+      reg[i] = 0;
+  }
+  bool operator==(const CPUStateBase &other) const {
+    if (pc != other.pc)
+      return false;
+    for (int i = 0; i < nr_reg; ++i) {
+      if (reg[i] != other.reg[i])
+        return false;
+    }
+    return true;
+  }
+  bool operator!=(const CPUStateBase &other) const {
+    return !(*this == other); // Reuse the == operator for != implementation
+  }
+};
+
+template <typename R, size_t nr_reg>
+std::ostream &operator<<(std::ostream &os, const CPUStateBase<R, nr_reg> &cpu) {
+  os << "PC: " << std::hex << cpu.pc << std::endl;
+  for (int i = 0; i < nr_reg; i++) {
+    os << "reg " << std::dec << std::setw(2) << i << ":" << std::hex
+       << std::setw(10) << cpu.reg[i];
+    if (i % 4 == 3) {
+      os << std::endl;
+    } else {
+      os << " | ";
+    }
+  }
+  return os;
+}
+
+#endif

npc/resource/addi.txt

@@ -1,10 +1,17 @@
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
-00114113
+00110113
+00110113
+00110113
+00110113
+00110113
+00110113
+00110113
+00110113