npc,fix: bugs of new arch in cpu-tests

.gitignore

@@ -6,3 +6,7 @@ difftest/
 **/result
 /.pre-commit-config.yaml
 **/.vscode/
+*.sock
+logs/
+**/.envrc
+**/.gdbinit

flake.nix

@@ -163,20 +163,22 @@
             ++ self.checks.${system}.pre-commit-check.enabledPackages;
         };
 
-        devShells.difftest = pkgs.mkShell {
+        devShells.default = pkgs.mkShell {
+          inherit (self.checks.${system}.pre-commit-check) shellHook;
+          buildInputs = self.checks.${system}.pre-commit-check.enabledPackages;
+
           packages = [
             diffu.packages.${system}.default
             am-kernels.packages.${system}.rv32Cross.am-kernels-npc
             self.packages.${system}.nemu-lib
             spike-diff.packages.${system}.default
+            pkgs.gef
+            pkgs.gtkwave
           ];
 
-          DIFFU_IMAGES_PATH = "${am-kernels.packages.${system}.rv32Cross.am-kernels-npc}/share";
+          DIFFU_IMAGES_PATH = "${am-kernels.packages.${system}.rv32Cross.am-kernels-npc}";
-        };
+          NEMU_SO = "${self.packages.${system}.nemu-lib}/lib/riscv32-nemu-interpreter-so";
-
+          NPC_SO = "/home/xin/repo/ysyx-workbench/npc/build/csrc/Flow/libFlow.so";
-        devShells.default = pkgs.mkShell {
-          inherit (self.checks.${system}.pre-commit-check) shellHook;
-          buildInputs = self.checks.${system}.pre-commit-check.enabledPackages;
         };
       }
     );

npc/.gitignore

@@ -17,3 +17,5 @@ hs_err_pid*
 compile_commands.json
 
 *.vcd
+*.gtkw
+*.sock

npc/CMakePresets.json

@@ -0,0 +1,29 @@
+{
+  "version": 3,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 19,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "default",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "ENABLE_YSYX_GIT_TRACKER": "ON",
+        "BUILD_CHISEL_EMIT_TARGET": "ON",
+        "TOPMODULE": "Flow",
+        "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
+        "BUILD_USE_BLOOP": "ON"
+      }
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "default",
+      "configurePreset": "default"
+    }
+  ]
+}

npc/core/build.sbt

@@ -1,7 +1,7 @@
 ThisBuild / scalaVersion := "2.13.12"
-ThisBuild / version := "0.1.0"
+ThisBuild / version := "0.1.1"
 
-val chiselVersion = "6.2.0"
+val chiselVersion = "6.5.0"
 val circeVersion = "0.14.1"
 
 lazy val root = (project in file("."))

npc/core/project/build.properties

@@ -1 +1 @@
-sbt.version=1.9.9
+sbt.version=1.10.1

npc/core/src/main/resources/RamDpi.v

@@ -1,4 +1,4 @@
-import "DPI-C" function int pmem_read(input int addr);
+import "DPI-C" function int pmem_read(input int addr, input byte rmask);
 import "DPI-C" function void pmem_write(input int waddr, input int wdata, input byte wmask);
 
 module RamDpi (
@@ -9,6 +9,7 @@ module RamDpi (
   input [31:0] writeAddr,
   input [31:0] writeData,
   input [3:0] writeMask,
+  input [3:0] readMask,
   input reg [31:0] readAddr,
   output reg [31:0] readData,
   input reg [31:0] pc,
@@ -16,7 +17,7 @@ module RamDpi (
 );
   always @(posedge clock) begin
     if (valid) begin // 有读写请求时
-      readData = pmem_read(readAddr);
+      readData = pmem_read(readAddr, { 4'h0, readMask });
       if (writeEnable) begin // 有写请求时
         pmem_write(writeAddr, writeData, { 4'h0, writeMask });
       end
@@ -25,5 +26,5 @@ module RamDpi (
       readData = 32'h80000000;
     end
   end
-  assign inst = pmem_read(pc);
+  assign inst = pmem_read(pc, 8'hF);
 endmodule

npc/core/src/main/scala/components/ALU.scala

@@ -94,7 +94,7 @@ class newALU(implicit p: Params) extends Module {
     val b = Input(Vec(SrcBSelect.all.length, UInt(p.XLEN)))
   })
   val out = IO(new Bundle {
-    val eq = Output(Bool())
+    val isEqual = Output(Bool())
     val result = Output(UInt(p.XLEN))
   })
 
@@ -113,7 +113,7 @@ class newALU(implicit p: Params) extends Module {
   val sll = a << b(5, 0)
   val srl = a >> b(5, 0)
   val sra = a.asSInt >> b(5, 0)
-  out.eq := a === b
+  out.isEqual := a === b
 
   import OpSelect._
 

npc/core/src/main/scala/components/Mem.scala

@@ -16,38 +16,14 @@ import chisel3.util.experimental.decode.{decoder, TruthTable}
 import flow.components.util._
 import flow.components.RV32Inst._
 
-class RamControlInterface(addrWidth: Int) extends Bundle {
+class RamDpi(implicit p: Params) extends BlackBox with HasBlackBoxResource {
-  val valid = Input(Bool())
+  val io = IO(new DpiRamInterface)
-  val writeMask = Input(UInt((addrWidth / 8).W))
-  val writeEnable = Input(Bool())
-  def ctrlBindPorts = {
-    valid :: writeMask :: writeEnable :: HNil
-  }
-
-}
-
-/* FIXME: Extends here might not be the best solution.
- *        We need a way to merge two bundles together
- */
-class RamInterface[T <: Data](tpe: T, addrWidth: Int)
-    extends RamControlInterface(addrWidth) {
-  val clock = Input(Clock())
-  val reset = Input(Reset())
-  val writeAddr = Input(UInt(addrWidth.W))
-  val writeData = Input(tpe)
-  val readAddr = Input(UInt(addrWidth.W))
-  val readData = Output(tpe)
-  val pc = Input(UInt(addrWidth.W))
-  val inst = Output(tpe)
-}
-
-class RamDpi extends BlackBox with HasBlackBoxResource {
-  val io = IO((new RamInterface(UInt(32.W), 32)))
   addResource("/RamDpi.v")
 }
 
 class DpiRamControlInterface(implicit p: Params) extends Bundle {
   val valid = Input(Bool())
+  val readMask = Input(UInt((p.XLEN.get / 8).W))
   val writeMask = Input(UInt((p.XLEN.get / 8).W))
   val writeEnable = Input(Bool())
 }
@@ -84,6 +60,15 @@ class RamController(implicit p: Params) extends Module {
     BitPat.dontCare(out.writeMask.getWidth)
   )
 
+  private val readMaskMapping = TruthTable(
+    Array(
+      lb -> BitPat("b0001"),
+      lh -> BitPat("b0011"),
+      lw -> BitPat("b1111")
+    ),
+    BitPat.dontCare(out.readMask.getWidth)
+  )
+
   private val writeEnableMapping = TruthTable(
     Array(
       sb -> BitPat("b1"),
@@ -94,6 +79,7 @@ class RamController(implicit p: Params) extends Module {
   )
 
   out.valid := decoder(in.inst, validMapping)
-  out.writeEnable := decoder(in.inst, writeMaskMapping)
+  out.writeEnable := decoder(in.inst, writeEnableMapping)
   out.writeMask := decoder(in.inst, writeMaskMapping)
+  out.readMask := decoder(in.inst, readMaskMapping)
 }

npc/core/src/main/scala/components/ProgramCounter.scala

@@ -72,7 +72,7 @@ class newProgramCounter(implicit p: Params) extends Module {
   val control = IO(newPcControlInterface())
   import newPcControlInterface.SrcSelect._
   val in = IO(new Bundle {
-    val brAddr = Input(UInt(p.XLEN))
+    val brOffset = Input(UInt(p.XLEN))
     val jAddr = Input(UInt(p.XLEN))
   })
   val out = IO(Output(UInt(p.XLEN)))
@@ -83,8 +83,8 @@ class newProgramCounter(implicit p: Params) extends Module {
   private val npc = MuxLookup(control.srcSelect, 4.U)(
     Seq(
       pStatic -> (pcReg + 4.U),
-      pJmp -> in.brAddr,
+      pJmp -> in.jAddr,
-      pBR -> in.jAddr
+      pBR -> (pcReg + in.brOffset)
     )
   )
 

npc/core/src/main/scala/components/RegisterFile.scala

@@ -58,8 +58,6 @@ class RegisterFile[T <: Data](tpe: T, regCount: Int, numReadPorts: Int)
   for (port <- 0 until numReadPorts) {
     out.src(port) := regFile(in.rs(port).asUInt)
   }
-  traceName(regFile)
-  dontTouch(regFile)
 }
 
 class newRegisterFile(implicit p: Params) extends Module {
@@ -94,6 +92,8 @@ class newRegisterFile(implicit p: Params) extends Module {
 
   out.src1 := regFile(in.rs1)
   out.src2 := regFile(in.rs2)
+  traceName(regFile)
+  dontTouch(regFile)
 }
 
 class RegisterFileController(implicit p: Params) extends Module {
@@ -124,18 +124,20 @@ class RegisterFileController(implicit p: Params) extends Module {
   // format: on
 
   private val writeSelectMapping = TruthTable(
-    memOutMapping ++ npcMapping,
+    aluOutMapping ++ memOutMapping ++ npcMapping,
     BitPat.dontCare(out.writeSelect.getWidth)
   )
 
   // enable write if instruction belongs to any mapping above
   private val writeEnableMapping = TruthTable(
     (aluOutMapping ++ memOutMapping ++ npcMapping).map(x =>
-      (x._1 -> BitPat(true.B))
+      (x._1 -> BitPat("b1"))
     ),
-    BitPat(false.B)
+    BitPat("b0")
   )
 
+  println(writeEnableMapping)
+
   out.writeSelect := RegControl.WriteSelect
     .safe(
       decoder(in.inst, writeSelectMapping)

npc/core/src/main/scala/stages/EX.scala

@@ -34,31 +34,34 @@ class EX(implicit val p: Params) extends Module {
   {
     import flow.components.ALUControlInterface.SrcASelect._
     import flow.components.util.chiselEnumAsInt
-    alu.in.a(aSrcARs1) := _in.inst.rs1
+    alu.in.a(aSrcARs1) := _in.src1
     alu.in.a(aSrcAPc) := _in.pc
     alu.in.a(aSrcAZero) := 0.U
 
     import flow.components.ALUControlInterface.SrcBSelect._
-    alu.in.b(aSrcBRs2) := _in.inst.rs2
+    alu.in.b(aSrcBRs2) := _in.src2
     alu.in.b(aSrcBImmI) := _in.inst.immI
     alu.in.b(aSrcBImmJ) := _in.inst.immJ
     alu.in.b(aSrcBImmU) := _in.inst.immU
     alu.in.b(aSrcBImmS) := _in.inst.immS
   }
 
-  _out.exeEq := alu.out.result
+  _out.exeOut := alu.out.result
-  _out.exeOut := alu.out.eq
 
   _toIF.jAddr := alu.out.result
-  _toIF.brAddr := _in.inst.immB
+  _toIF.brOffset := _in.inst.immB
+  _toIF.pc := _in.pc
 
   import flow.components.newPcControlInterface.SrcSelect._
+  val regSrcEq = Wire(Bool());
+  regSrcEq := (_in.src1 === _in.src2);
   when(_in.pcCtrl.srcSelect === pBR) {
     val branchUseSlt = _in.inst(14)
     val branchInvertResult = _in.inst(12)
-    val _branchResult = Mux(branchUseSlt, alu.out.result(0), alu.out.eq)
+    val _branchResult =
+      Mux(branchUseSlt, alu.out.result(0), regSrcEq)
     val branchResult = Mux(branchInvertResult, !_branchResult, _branchResult)
-    _toIF.pcCtrl.srcSelect := Mux(branchInvertResult, pBR, pStatic)
+    _toIF.pcCtrl.srcSelect := Mux(branchResult, pBR, pStatic)
   }.otherwise {
     _toIF.pcCtrl.srcSelect := _in.pcCtrl.srcSelect
   }

npc/core/src/main/scala/stages/ID.scala

@@ -7,6 +7,8 @@ import flow.stages.messages._
 import flow.components._
 import flow.components.RV32InstSubfields._
 import chisel3.util.DecoupledIO
+import chisel3.experimental.Trace._
+import chisel3.util.experimental.InlineInstance
 
 class ID(implicit val p: Params) extends Module {
   val io = IO(new Bundle {
@@ -25,7 +27,7 @@ class ID(implicit val p: Params) extends Module {
   val _out = msgio.out.bits
   val _fromWB = io.fromWB
 
-  val regs = Module(new newRegisterFile)
+  val regs = Module(new newRegisterFile with InlineInstance)
 
   // Controllers
   val pcController = Module(new PcController)

npc/core/src/main/scala/stages/IF.scala

@@ -34,7 +34,7 @@ class IF(implicit val p: Params) extends Module {
   private val pc = Module(new newProgramCounter)
 
   // PC update
-  pc.in.brAddr := _fromEx.brAddr
+  pc.in.brOffset := _fromEx.brOffset
   pc.in.jAddr := _fromEx.jAddr
 
   pc.control := _fromEx.pcCtrl

npc/core/src/main/scala/stages/LS.scala

@@ -30,8 +30,10 @@ class LS(implicit val p: Params) extends Module {
   ram.io.clock := clock
   ram.io.reset := reset
   ram.io.writeAddr := _in.exeOut
-  ram.io.writeData := _in.src1
+  ram.io.writeData := _in.src2
   ram.io.writeMask := _in.ramCtrl.writeMask
+  ram.io.readMask := _in.ramCtrl.readMask
+
   ram.io.writeEnable := _in.ramCtrl.writeEnable
   ram.io.valid := _in.ramCtrl.valid // TODO: change to a better name
   ram.io.readAddr := _in.exeOut
@@ -39,7 +41,7 @@ class LS(implicit val p: Params) extends Module {
   // TODO: Change to icache, and move to IF stage.
   // Change to arbitor here
   ram.io.pc := _fromIF.pc
-  _toIF.inst := ram.io.pc
+  _toIF.inst := ram.io.inst
 
   _out.memOut := ram.io.readData
   _out.exeOut := _in.exeOut

npc/core/src/main/scala/stages/Messages.scala

@@ -30,7 +30,6 @@ class EX2LS(implicit p: Params) extends Bundle {
   val src2 = UInt(p.XLEN)
 
   val exeOut = UInt(p.XLEN)
-  val exeEq = Bool()
 
   // Control
   val ramCtrl = Flipped(new DpiRamControlInterface)
@@ -47,7 +46,8 @@ class LS2WB(implicit p: Params) extends Bundle {
 }
 
 class EX2IF(implicit p: Params) extends Bundle {
-  val brAddr = UInt(p.XLEN)
+  val pc = UInt(p.XLEN)
+  val brOffset = UInt(p.XLEN)
   val jAddr = UInt(p.XLEN)
 
   // Control

npc/core/src/main/scala/stages/WB.scala

@@ -6,6 +6,7 @@ import chisel3.util.DecoupledIO
 import flow.Params
 import flow.stages.utils._
 import flow.stages.messages._
+import chisel3.experimental.Trace._
 
 class WB(implicit val p: Params) extends Module {
   // val msgio = IO(DecoupledMsgIO(in = (new LS2WB).S))
@@ -25,4 +26,7 @@ class WB(implicit val p: Params) extends Module {
   _toID.regCtrl := _in.regCtrl;
   _toID.rd := _in.rd
   _toID.npc := _in.pc + 4.U
+
+  traceName(_in.pc)
+  traceName(_in.rd)
 }

npc/core/src/main/scala/top/FlowMain.scala

@@ -13,7 +13,7 @@ import chisel3.util.{BinaryMemoryFile, HexMemoryFile}
 import chisel3.experimental.Trace
 import scala.collection.IndexedSeqView
 import shapeless.Poly1
-import flow.components.RamControlInterface
+import flow.components.DpiRamControlInterface
 import flow.components.RV32Inst
 import flow.components.RV32InstSubfields._
 import flow.components.util._
@@ -21,6 +21,11 @@ import flow.components.util._
 import flow.components.{RegControl, PcControlInterface, ALUControlInterface}
 
 import flow.stages._
+import chisel3.experimental.annotate
+import chisel3.experimental.ChiselAnnotation
+
+import chisel3.Data
+import chisel3.experimental.{annotate, requireIsAnnotatable, ChiselAnnotation}
 
 class Flow extends Module {
   implicit val p: Params = new Params(XLEN = 32.W, arch = "single")
@@ -40,448 +45,3 @@ class Flow extends Module {
   LS.io.fromIF := IF.io.toRam
   ID.io.fromWB := WB.io.toID
 }
-
-// class Control(width: Int) extends RawModule {
-//   // Helpers
-//   class WrapList[T](vl: T) { type Type = T; val v = vl }
-//   object wrap extends Poly1 {
-//     implicit def default[A] = at[A](Right(_).withLeft[Int])
-//   }
-//   def lit(x: Element) = { x.litValue.toInt }
-//   def toBits(t: dst.Type): BitPat = {
-//     val list = t.toList
-//     list
-//       .map(e =>
-//         e match {
-//           case Right(x) => BitPat(lit(x).U(x.getWidth.W))
-//           case Left(x)  => BitPat.dontCare(x)
-//         }
-//       )
-//       .reduceLeft(_ ## _)
-//   }
-//   val r = Right
-//   def l[T <: Any](x: T) = x match {
-//     case x: ChiselEnum => Left(log2Ceil(x.all.length))
-//     case x: Data       => Left(x.getWidth)
-//     case _             => throw new IllegalArgumentException
-//   }
-//
-//   val inst = IO(Input(UInt(width.W)))
-//
-//   val reg = IO(Flipped(RegControl()))
-//   val pc = IO(Flipped(PcControlInterface()))
-//   val alu = IO(Flipped(ALUControlInterface()))
-//   val ram = IO(Flipped(RamControlInterface(32)))
-//
-//   val dst = new WrapList(
-//     (reg.ctrlBindPorts ++
-//       pc.ctrlBindPorts ++
-//       alu.ctrlBindPorts ++
-//       ram.ctrlBindPorts).map(wrap)
-//   )
-//
-//   val dstList = dst.v.toList
-//   val controlWidth = dstList.map(_.toOption.get.getWidth).reduce(_ + _)
-//   val reversePrefixSum = dstList.scanLeft(0)(_ + _.toOption.get.getWidth)
-//   val sliceIndex = reversePrefixSum.map(controlWidth - _)
-//   val slices = sliceIndex.map(_ - 1).zip(sliceIndex.tail)
-//
-//   import reg.WriteSelect._
-//   import reg._
-//   import pc.SrcSelect._
-//   import pc._
-//   import alu.OpSelect._
-//   import alu.SrcASelect._
-//   import alu.SrcBSelect._
-//   import pc._
-//   import RV32Inst._
-//   // format: off
-//   val ControlMapping: Array[(BitPat, dst.Type)] = Array(
-//     //     Regs | writeEnable :: writeSelect :: HNil
-//     //     PC   | useImmB :: srcSelect :: HNil
-//     //     Exe  | op :: srcASelect :: srcBSelect :: signExt :: HNil
-//     //     Mem  | valid :: writeMask :: writeEnable :: HNil
-//
-//     (lui   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc)::
-//       r(aOpAdd)     :: r(aSrcAZero)  :: r(aSrcBImmU) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (auipc , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc)::
-//       r(aOpAdd)     :: r(aSrcAPc)  :: r(aSrcBImmU) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     // ---- Control Transfer Instructions ----
-//     (jal   , (
-//       r(true.B)     :: r(rNpc)     ::
-//       r(false.B)    :: r(pExeOut)  ::
-//       r(aOpAdd)     :: r(aSrcAPc)  :: r(aSrcBImmJ) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (jalr  , (
-//       r(true.B)     :: r(rNpc)     ::
-//       r(false.B)    :: r(pExeOut)    ::
-//       r(aOpAdd)     :: r(aSrcARs1) :: r(aSrcBImmI) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (beq   , (
-//       r(false.B)    :: l(WriteSelect) ::
-//       r(true.B)     :: r(pStaticNpc)     ::
-//       r(aOpSlt)     :: r(aSrcARs1)  :: r(aSrcBRs2) :: l(Bool())    ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (bne   , (
-//       r(false.B)    :: l(WriteSelect) ::
-//       r(true.B)     :: r(pStaticNpc)     ::
-//       r(aOpSlt)     :: r(aSrcARs1)  :: r(aSrcBRs2) :: l(Bool())    ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (blt   , (
-//       r(false.B)    :: l(WriteSelect) ::
-//       r(true.B)     :: r(pStaticNpc)     ::
-//       r(aOpSlt)     :: r(aSrcARs1)  :: r(aSrcBRs2) :: r(true.B)  ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (bge   , (
-//       r(false.B)    :: l(WriteSelect) ::
-//       r(true.B)     :: r(pStaticNpc)     ::
-//       r(aOpSlt)     :: r(aSrcARs1)  :: r(aSrcBRs2) :: r(true.B)  ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (bltu  , (
-//       r(false.B)    :: l(WriteSelect)::
-//       r(true.B)     :: r(pStaticNpc)    ::
-//       r(aOpSltu)     :: r(aSrcARs1)    :: r(aSrcBRs2) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (bgeu  , (
-//       r(false.B)    :: l(WriteSelect)::
-//       r(true.B)     :: r(pStaticNpc)    ::
-//       r(aOpSltu)     :: r(aSrcARs1)    :: r(aSrcBRs2) :: r(false.B) ::
-//       r(false.B)    :: l(UInt(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     // ---- Memory Access Instructions ----
-//
-//     (lb    , (
-//       r(true.B)     :: r(rMemOut)    ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmI) :: l(Bool()) ::
-//       r(true.B)     :: r(1.U(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (lbu   , (
-//       r(true.B)    :: r(rMemOut)    ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmI) :: l(Bool()) ::
-//       r(true.B)     :: r(0.U(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (lh     , (
-//       r(true.B)    :: r(rMemOut)    ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmI) :: l(Bool()) ::
-//       r(true.B)     :: r(3.U(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (lhu    , (
-//       r(true.B)    :: r(rMemOut)    ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmI) :: l(Bool()) ::
-//       r(true.B)     :: r(2.U(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (lw    , (
-//       r(true.B)     :: r(rMemOut)    ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmI) :: l(Bool()) ::
-//       r(true.B)     :: r(14.U(4.W))  :: r(false.B)   :: HNil
-//     )),
-//
-//     (sb    , (
-//       r(false.B)    :: l(WriteSelect)::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmS) :: l(Bool()) ::
-//       r(true.B)     :: r(1.U(4.W))   :: r(true.B)   :: HNil
-//     )),
-//
-//     (sh    , (
-//       r(false.B)    :: l(WriteSelect)::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmS) :: l(Bool()) ::
-//       r(true.B)     :: r(3.U(4.W))   :: r(true.B)   :: HNil
-//     )),
-//
-//     (sw    , (
-//       r(false.B)    :: l(WriteSelect)::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1)   :: r(aSrcBImmS) :: l(Bool()) ::
-//       r(true.B)     :: r(15.U(4.W))  :: r(true.B)   :: HNil
-//     )),
-//
-//     // ---- Integer Computational Instructions ---
-//
-//     (addi  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)    :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (slti  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSlt)     :: r(aSrcARs1) :: r(aSrcBImmI) :: r(true.B)  ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (sltiu , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSltu)     :: r(aSrcARs1) :: r(aSrcBImmI) :: r(false.B) ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (xori  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpXor)      :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (ori   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpOr)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (andi  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAnd)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (slli  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSll)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (srli  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSrl)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (srai  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSra)     :: r(aSrcARs1) :: r(aSrcBImmI) :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (add   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAdd)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (sub   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSub)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (sll   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSll)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (slt   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSlt)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: r(true.B)  ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (sltu  , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSltu)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: r(false.B) ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (xor   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpXor)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (srl   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSrl)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (sra   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpSra)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (or    , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpOr)      :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//
-//     (and   , (
-//       r(true.B)     :: r(rAluOut)  ::
-//       r(false.B)    :: r(pStaticNpc) ::
-//       r(aOpAnd)     :: r(aSrcARs1) :: r(aSrcBRs2)  :: l(Bool())    ::
-//       r(false.B)     :: l(UInt(4.W)):: r(false.B)   :: HNil
-//     )),
-//   )
-//   // format: on
-//
-//   val default = BitPat(0.U(controWidth.W))
-//
-// //   println(s"ControlMapping = ${ControlMapping.map(it => (it._1 -> toBits(it._2))).foreach(x => println(x._2))}\n")
-//   val out = decoder(
-//     inst,
-//     TruthTable(ControlMapping.map(it => (it._1 -> toBits(it._2))), default)
-//   )
-//   val srcList = slices.map(s => out(s._1, s._2))
-//
-//   assert(out != default)
-//   println(s"out = $out, default = $default\n")
-//   println(s"dstList = ${dstList}\n")
-//   println(s"srcList = ${srcList}\n")
-//   srcList
-//     .zip(dstList)
-//     .foreach({ case (src, dst) =>
-//       dst.toOption.get := src.asTypeOf(dst.toOption.get)
-//     })
-// }
-
-// class Flow extends Module {
-//   def lit(x: Data) = { x.litValue.toInt }
-//
-//   val dataType = UInt(32.W)
-//   val ram = Module(new RamDpi)
-//   val control = Module(new Control(32))
-//   val reg = Module(new RegisterFile(dataType, 32, 2))
-//   val pc = Module(new ProgramCounter(dataType))
-//   val alu = Module(new ALU(dataType))
-//
-//   // TODO: Switch to Decoupled and Arbiter later
-//   ram.io.pc := pc.out
-//   val inst = ram.io.inst
-//
-//   dontTouch(reg.control.writeEnable)
-//
-//   import control.pc.SrcSelect._
-//
-//   val npc = Wire(dataType)
-//   npc := pc.out + 4.U
-//   pc.in.exeOut := alu.out.result
-//   pc.in.immB := inst.immB
-//
-//   control.inst := inst
-//   reg.control <> control.reg
-//   // FIXME: Probably optimizable with bulk connection
-//   pc.control <> control.pc
-//   pc.control.useImmB := control.pc.useImmB
-//   alu.control <> control.alu
-//   val branchUseSlt = Wire(Bool())
-//   val branchInvertResult = Wire(Bool())
-//   branchUseSlt := inst(14)
-//   branchInvertResult := inst(12)
-//   val _branchResult = Mux(branchUseSlt, alu.out.result(0), alu.out.eq)
-//   val branchResult = Mux(branchInvertResult, !_branchResult, _branchResult)
-//   pc.control.useImmB := control.pc.useImmB && branchResult
-//   // printf(cf"_branchResult = ${_branchResult}, branchResult = ${branchResult}\n")
-//   // printf(cf"pcin.useImmB = ${pc.control.useImmB}, control.out.useImmB = ${control.pc.useImmB} \n")
-//
-//   import control.reg.WriteSelect._
-//   reg.in.writeData(rAluOut) := alu.out.result
-//   val maskedData = ram.io.readData & Cat(
-//     Fill(8, ram.io.writeMask(3)),
-//     Fill(8, ram.io.writeMask(2)),
-//     Fill(8, ram.io.writeMask(1)),
-//     "b11111111".U
-//   )
-//
-//   val doSignExt = control.ram.writeMask(0)
-//   val signExt16 = control.ram.writeMask(1)
-//   when(!doSignExt) {
-//     reg.in.writeData(rMemOut) := maskedData
-//     // printf(cf"!doSignExt\n")
-//   }.elsewhen(signExt16) {
-//     reg.in.writeData(rMemOut) := Cat(
-//       Fill(16, maskedData(15)),
-//       maskedData(15, 0)
-//     )
-//     // printf(cf"elsewhen\n")
-//   }.otherwise {
-//     reg.in
-//       .writeData(rMemOut) := Cat(Fill(24, maskedData(7)), maskedData(7, 0))
-//     // printf(cf"otherwise\n")
-//   }
-//   // printf(cf"maskedData = ${maskedData}, writeData = ${reg.in.writeData(lit(rMemOut))}\n")
-//   reg.in.writeData(rNpc) := npc
-//
-//   reg.in.writeAddr := inst.rd
-//   reg.in.rs(0) := inst.rs1
-//   reg.in.rs(1) := inst.rs2
-//
-//   // TODO: Bulk connection here
-//   ram.io.clock := clock
-//   ram.io.reset := reset
-//   ram.io.writeAddr := alu.out.result
-//   ram.io.writeData := reg.out.src(1)
-//   ram.io.writeMask := control.ram.writeMask
-//   ram.io.writeEnable := control.ram.writeEnable
-//   ram.io.valid := control.ram.valid
-//   ram.io.readAddr := alu.out.result
-//
-//   import control.alu.SrcASelect._
-//   import control.alu.SrcBSelect._
-//   alu.in.a(aSrcARs1) := reg.out.src(0)
-//   alu.in.a(aSrcAPc) := pc.out
-//   alu.in.a(aSrcAZero) := 0.U
-//
-//   alu.in.b(aSrcBRs2) := reg.out.src(1)
-//   // alu.in.b(lit(aSrcBImmI)) := inst(31, 20).pad(aSrcBImmI.getWidth)
-//   alu.in.b(aSrcBImmI) := inst.immI
-//   alu.in.b(aSrcBImmJ) := inst.immJ
-//   alu.in.b(aSrcBImmS) := inst.immS
-//   alu.in.b(aSrcBImmU) := inst.immU
-//
-//   Trace.traceName(pc.out)
-//   dontTouch(control.out)
-// }

npc/core/src/main/scala/top/Main.scala

@@ -7,10 +7,10 @@ import chisel3.experimental.Trace._
 import chisel3.stage.{ChiselGeneratorAnnotation, DesignAnnotation}
 import chisel3.util.experimental.InlineInstance
 import circt.stage.ChiselStage
-import firrtl.annotations.TargetToken.{Instance, OfModule, Ref}
 import java.io.PrintWriter
 import scala.io.Source
 import java.io.File
+import firrtl.annotations.TargetToken.{OfModule, Instance, Ref}
 
 // TODO: Generate verilator config file
 

npc/csrc/Flow/config.cpp

@@ -19,4 +19,4 @@ void Config::cli_parse(int argc, char **argv) {
   }
 }
 
-Config config;
+Config config{.wavefile = "flowwave.vcd"};

npc/csrc/Flow/gdbstub_wrapper.cpp

@@ -7,6 +7,7 @@ extern "C" {
 #include <cstdint>
 #include <cstdlib>
 #include <devices.hpp>
+#include <string>
 #include <types.h>
 #include <vl_wrapper.hpp>
 #include <vpi_user.h>
@@ -39,21 +40,19 @@ void *pmem_get() {
   return pmem;
 }
 
-int pmem_read(int raddr) {
+int pmem_read(int raddr, int rmask) {
   void *pmem = pmem_get();
   auto mem = static_cast<MMap *>(pmem);
   // TODO: Do memory difftest at memory read and write to diagnose at a finer
   // granularity
-  mem->trace(raddr, true, regs->get_pc());
   if (g_skip_memcheck)
-    return mem->read(PMEM_START);
+    return mem->read(PMEM_START, rmask);
-  return mem->read(raddr);
+  return mem->read(raddr, rmask);
 }
 
 void pmem_write(int waddr, int wdata, char wmask) {
   void *pmem = pmem_get();
   auto mem = static_cast<MMap *>(pmem);
-  mem->trace((std::size_t)waddr, false, regs->get_pc(), wdata);
   return mem->write((std::size_t)waddr, wdata, wmask);
 }
 
@@ -146,13 +145,37 @@ void npc_init(void *args) {
   DbgState *dbg = (DbgState *)args;
   void *mem = pmem_get();
   dbg->bp = new std::vector<Breakpoint>;
+  dbg->cmd_return_buf = new std::string;
 
   top = new VlModule;
-  regs = new Registers("TOP.Flow.reg_0.regFile_", "TOP.Flow.pc.out");
+  regs = new Registers("TOP.Flow.ID.regs_regFile_",
+                       "TOP.Flow.WB.msgio_in_bits_pc");
   top->setup(config.wavefile, regs);
   top->reset_eval(10);
 }
 
+char *npc_monitor(void *args, char **argv, int argc) {
+  DbgState *dbg = (DbgState *)args;
+  std::string *cmd_return_buf = dbg->cmd_return_buf;
+
+  if (strncmp(argv[0], "trace", 5) == 0) {
+    if (strncmp(argv[1], "on", 2) == 0) {
+      *cmd_return_buf = "Tracing turned on\n";
+      if (!top->start_trace())
+        *cmd_return_buf = "Failed to turn on tracing\n";
+      return cmd_return_buf->data();
+    } else if (strncmp(argv[1], "off", 3) == 0) {
+      *cmd_return_buf = "Tracing turned off\n";
+      if (!top->end_trace())
+        *cmd_return_buf = "Failed to turn on tracing\n";
+      return cmd_return_buf->data();
+    }
+  }
+
+  *cmd_return_buf = "Command not found\n";
+  return cmd_return_buf->data();
+}
+
 bool npc_do_difftest = true;
 
 static gdbstub_t gdbstub_priv;

npc/flake.nix

@@ -1,6 +1,7 @@
 {
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs-stable.url = "github:NixOS/nixpkgs/nixos-24.05";
     nixpkgs-circt162.url = "github:NixOS/nixpkgs/7995cae3ad60e3d6931283d650d7f43d31aaa5c7";
     nur-xin = {
       url = "git+https://git.xinyang.life/xin/nur.git";
@@ -11,11 +12,18 @@
       url = "github:zaninime/sbt-derivation";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    am-kernels.url = "git+https://git.xinyang.life/xin/am-kernels.git?ref=dev";
   };
 
-  outputs = { self, nixpkgs, flake-utils, nur-xin, nixpkgs-circt162, sbt-derivation }:
+  outputs = { self, nixpkgs, nixpkgs-stable, flake-utils, nur-xin, nixpkgs-circt162, sbt-derivation, am-kernels }:
     flake-utils.lib.eachDefaultSystem (system:
       let
+        stablePkgs = import nixpkgs-stable {
+          inherit system;
+          config.allowUnfree = true;
+          overlays = [
+          ];
+        };
         pkgs = import nixpkgs {
           inherit system;
           config.allowUnfree = true;
@@ -45,14 +53,15 @@
             flex
             bison
             nvboard
-            verilator
             flow
             espresso
             bloop
             coursier
             sbt
-          ];
+            gef
+          ] ++ [stablePkgs.verilator];
           CHISEL_FIRTOOL_PATH = "${nixpkgs-circt162.legacyPackages.${system}.circt}/bin";
+          NPC_IMAGES_PATH = "${am-kernels.packages.${system}.rv32Cross.am-kernels-npc}/share";
 
           buildInputs = with pkgs; [
             cli11

npc/include/components.hpp

@@ -88,14 +88,18 @@ public:
       ram->transfer(waddr, (uint8_t *)&wdata, len, true);
     } else if (devices->handle(waddr, (uint8_t *)&wdata, len, true)) {
     }
+    logger->trace("[W] 0x{:x}: 0x{:x}", waddr, wdata);
   }
 
-  word_t read(paddr_t raddr) const {
+  word_t read(paddr_t raddr, int rmask) const {
     word_t res = 0;
+    size_t len = (rmask & 1) + ((rmask & 2) >> 1) + ((rmask & 4) >> 2) +
+                 ((rmask & 8) >> 3);
     if (ram->in_pmem(raddr)) {
-      ram->transfer(raddr, (uint8_t *)&res, 4, false);
+      ram->transfer(raddr, (uint8_t *)&res, len, false);
     } else if (devices->handle(raddr, (uint8_t *)&res, 4, false)) {
     }
+    logger->trace("[R] 0x{:x}: 0x{:x}", raddr, res);
     return res;
   }
 
@@ -120,7 +124,7 @@ public:
   void *get_pmem() { return ram->mem.data(); }
 
   void trace(paddr_t addr, bool is_read, word_t pc = 0, word_t value = 0) {
-    logger->trace("[{}] 0x{:x}", is_read ? 'R' : 'W', this->read(addr));
+    logger->trace("[{}] 0x{:x}", is_read ? 'R' : 'W', this->read(addr, value));
   }
 
 private:

npc/include/types.h

@@ -1,6 +1,7 @@
 #ifndef _NPC_TYPES_H__
 #define _NPC_TYPES_H__
 #ifdef __cplusplus
+#include <string>
 extern "C" {
 #endif
 #include <gdbstub.h>
@@ -32,6 +33,7 @@ struct Breakpoint {
 
 struct DbgState {
   std::vector<Breakpoint> *bp;
+  std::string *cmd_return_buf;
 };
 #endif
 

npc/include/vl_wrapper.hpp

@@ -34,6 +34,7 @@ template <typename T, typename R> class VlModuleInterfaceCommon : public T {
   uint64_t sim_time = 0;
   uint64_t posedge_cnt = 0;
   std::unique_ptr<Tracer<T>> tracer;
+  std::filesystem::path wavefile;
 
 public:
   const R *registers;
@@ -43,8 +44,7 @@ public:
   }
 
   void setup(std::filesystem::path wavefile, const R *r) {
-    if (!wavefile.empty())
+    wavefile = "wave.vcd";
-      tracer = std::make_unique<Tracer<T>>(this, wavefile);
     registers = r;
   }
 
@@ -89,10 +89,30 @@ public:
     this->reset = 0;
     g_skip_memcheck = false;
   }
+
   bool is_posedge() {
     // Will be posedge when eval is called
     return T::clock == 0;
   }
+
+  bool start_trace() { return init_tracer(wavefile); }
+
+  bool end_trace() {
+    tracer.reset();
+    return true;
+  }
+
+private:
+  bool init_tracer(std::filesystem::path wavefile) {
+    fmt::print("wavefile: {}", wavefile.string());
+    std::filesystem::path wav = "wave.vcd";
+    if (!wav.empty()) {
+      // Creating of tracer must happen after this class fully initialized
+      tracer = std::make_unique<Tracer<T>>(this, wav);
+      return true;
+    }
+    return false;
+  }
 };
 
 #endif

npc/include/vpi_wrapper.hpp

@@ -27,18 +27,23 @@ public:
   }
 
 private:
+  static vpiHandle get_handle(const std::string name) {
+    vpiHandle hdl = vpi_handle_by_name((PLI_BYTE8 *)name.c_str(), nullptr);
+    if (hdl == nullptr) {
+      SPDLOG_ERROR("VPI Handle {} not found", name);
+      exit(EXIT_FAILURE);
+    } else {
+      SPDLOG_INFO("Found VPI handle {} at {}", name, (void *)hdl);
+    }
+    return hdl;
+  }
+
   void init_handlers(const std::string regs_prefix, const std::string pcname) {
+    pc_handle = get_handle(pcname);
     for (int i = 0; i < nr; i++) {
       std::string regname = regs_prefix + std::to_string(i);
-      vpiHandle vh = vpi_handle_by_name((PLI_BYTE8 *)regname.c_str(), nullptr);
+      reg_handles[i] = get_handle(regname);
-      if (vh == nullptr) {
-        std::cerr << "vpiHandle " << regname.c_str() << " not found"
-                  << std::endl;
-        exit(EXIT_FAILURE);
     }
-      reg_handles[i] = vh;
-    }
-    pc_handle = vpi_handle_by_name((PLI_BYTE8 *)pcname.c_str(), nullptr);
   }
 };
 

scripts/difftests.py

@@ -0,0 +1,86 @@
+#/usr/bin/env python
+
+"""
+This script is used to provide a wrapper to difftest, so that they can be easily
+deployed on ci environment.
+"""
+
+import os
+import os.path as osp
+import sys
+from multiprocessing import Pool, Process
+from functools import partial
+
+def find_all_test_images(path):
+    tests = []
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            # Get file extensions and select files with .bin
+            ext = osp.splitext(file)[1]
+            if ext == ".bin":
+                tests.append(osp.join(root, file))
+    return tests
+
+def run_test(test_image, ref, ref_prefix, dut, dut_prefix):
+    diffu = "diffu"
+    args = [
+        "--images-path", "/",
+        "-m", test_image,
+        "--ref", ref,
+        "--ref-prefix", ref_prefix,
+        "--dut", dut,
+        "--dut-prefix", dut_prefix,
+        "> logs/" + osp.basename(test_image) + ".log",
+        "2>&1"
+    ]
+
+    status = os.system(diffu + " " + " ".join(args))
+    exitcode = os.waitstatus_to_exitcode(status)
+
+    image_shortname = osp.basename(test_image)
+    print(f"{ 'FAILED' if exitcode else 'PASSED' } {image_shortname}")
+    if exitcode:
+      print(f"cmd: {diffu + ' ' + ' '.join(args)}")
+      print(f"exitcode: {exitcode}")
+    sys.exit(exitcode)
+
+def print_statistics(results):
+    pass
+
+def main():
+    DIFFU_IMAGES_PATH = os.environ["DIFFU_IMAGES_PATH"]
+    print(DIFFU_IMAGES_PATH)
+    assert(osp.isdir(DIFFU_IMAGES_PATH))
+    os.makedirs("logs", exist_ok = True)
+    # Run tests in a multiprocess pool
+    tests = find_all_test_images(DIFFU_IMAGES_PATH)
+    ref, ref_prefix, dut, dut_prefix = sys.argv[1:]
+    ref_shortname = osp.basename(ref)
+    dut_shortname = osp.basename(dut)
+    print(f"[{ref_shortname}, {dut_shortname}]")
+    procs = []
+    for test in tests:
+      image_shortname = osp.basename(test)
+      p = Process(target=run_test, args=(test, ref, ref_prefix, dut, dut_prefix), name=image_shortname, daemon=True)
+      procs.append(p)
+      p.start()
+
+    timeout = 0
+    for p in procs:
+      p.join(5)
+      if p.exitcode is None:
+        print(f"{ 'TIMEOUT' } {p.name}")
+        p.terminate()
+        p.join()
+        timeout += 1
+
+    not_success = sum((1 for p in procs if p.exitcode != 0))
+    failed = not_success - timeout
+
+    print("==========")
+    print(f"TOTAL {len(procs)}\tFAILED: {failed}\tTIMEOUT: {timeout}")
+
+    return not_success
+
+if __name__ == "__main__":
+    sys.exit(main())