ELF-based-loadmem | architectural restartable checkpoints

.github/scripts/defaults.sh

@@ -45,7 +45,7 @@ mapping["chipyard-mempress"]=" CONFIG=MempressRocketConfig"
 mapping["chipyard-digitaltop"]=" TOP=DigitalTop"
 mapping["chipyard-manymmioaccels"]=" CONFIG=ManyMMIOAcceleratorRocketConfig"
 mapping["chipyard-hetero"]=" CONFIG=LargeBoomAndRocketConfig"
-mapping["chipyard-boom"]=" CONFIG=MediumBoomCosimConfig"
+mapping["chipyard-boom"]=" CONFIG=dmiMediumBoomCosimConfig"
 mapping["chipyard-spike"]=" CONFIG=SpikeFastUARTConfig EXTRA_SIM_FLAGS='+spike-ipc=10'"
 mapping["chipyard-hwacha"]=" CONFIG=HwachaRocketConfig"
 mapping["chipyard-gemmini"]=" CONFIG=GemminiRocketConfig"

.github/scripts/run-tests.sh

@@ -33,10 +33,12 @@ case $1 in
         run_bmark ${mapping[$1]}
         ;;
     chipyard-dmirocket)
-        run_bmark ${mapping[$1]}
+        $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -c 10000
+        make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
         ;;
     chipyard-boom)
-        run_bmark ${mapping[$1]}
+        $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -c 10000
+        make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch
         ;;
     chipyard-spike)
         run_bmark ${mapping[$1]}

common.mk

@@ -318,29 +318,15 @@ run-binary-debug: $(SIM_DEBUG_PREREQ) check-binary | $(output_dir)
 run-fast: run-asm-tests-fast run-bmark-tests-fast
 
 #########################################################################################
-# helper rules to run simulator with fast loadmem via hex files
+# helper rules to run simulator with fast loadmem
+# LEGACY - use LOADMEM=1 instead
 #########################################################################################
-$(binary_hex): $(firstword $(BINARY)) | $(output_dir)
-	$(base_dir)/scripts/smartelf2hex.sh $(firstword $(BINARY)) > $(binary_hex)
-
-run-binary-hex: check-binary
-run-binary-hex: $(SIM_PREREQ) $(binary_hex) | $(output_dir)
 run-binary-hex: run-binary
-run-binary-hex: override LOADMEM_ADDR = 80000000
-run-binary-hex: override LOADMEM = $(binary_hex)
-run-binary-hex: override SIM_FLAGS += +loadmem=$(LOADMEM) +loadmem_addr=$(LOADMEM_ADDR)
-run-binary-debug-hex: check-binary
-run-binary-debug-hex: $(SIM_DEBUG_REREQ) $(binary_hex) | $(output_dir)
+run-binary-hex: override SIM_FLAGS += +loadmem=$(BINARY)
 run-binary-debug-hex: run-binary-debug
-run-binary-debug-hex: override LOADMEM_ADDR = 80000000
-run-binary-debug-hex: override LOADMEM = $(binary_hex)
-run-binary-debug-hex: override SIM_FLAGS += +loadmem=$(LOADMEM) +loadmem_addr=$(LOADMEM_ADDR)
-run-binary-fast-hex: check-binary
-run-binary-fast-hex: $(SIM_PREREQ) $(binary_hex) | $(output_dir)
+run-binary-debug-hex: override SIM_FLAGS += +loadmem=$(BINARY)
 run-binary-fast-hex: run-binary-fast
-run-binary-fast-hex: override LOADMEM_ADDR = 80000000
-run-binary-fast-hex: override LOADMEM = $(binary_hex)
-run-binary-fast-hex: override SIM_FLAGS += +loadmem=$(LOADMEM) +loadmem_addr=$(LOADMEM_ADDR)
+run-binary-fast-hex: override SIM_FLAGS += +loadmem=$(BINARY)
 
 #########################################################################################
 # run assembly/benchmarks rules

docs/Advanced-Concepts/Architectural-Checkpoints.rst

@@ -0,0 +1,39 @@
+.. _checkpointing:
+
+Architectural Checkpoints
+=========================
+
+Chipyard supports generating architectural checkpoints using Spike.
+These checkpoints contain a snapshot of the architectural state of a RISC-V SoC at some point in the execution of a program.
+The checkpoints include the contents of cacheable memory, core architectural registers, and core CSRs.
+RTL simulations of SoCs can resume execution from checkpoints after restoring the architectural state.
+
+.. note::
+   Currently, only checkpoints of single-core systems are supported
+
+Generating Checkpoints
+------------------------
+
+``scripts/generate-ckpt.sh`` is a script that runs spike with the right commands to generate an architectural checkpoint
+``scripts/generate-ckpt.sh -h`` lists options for checkpoint generation.
+
+Example: run the ``hello.riscv`` binary for 1000 instructions before generating a checkpoint.
+This should produce a directory named ``hello.riscv.0x80000000.1000.loadarch``
+
+.. code::
+
+   scripts/generate-ckpt.sh -b tests/hello.riscv -i 1000
+
+
+Loading Checkpoints in RTL Simulation
+--------------------------------------
+
+Checkpoints can be loaded in RTL simulations with the ``LOADARCH`` flag.
+The target config **MUST** use dmi-based bringup (as opposed to the default TSI-based bringup), and support fast ``LOADMEM``.
+The target config should also match the architectural configuration of however spike was configured when generating the checkpoint.
+
+.. code::
+
+   cd sims/vcs
+   make CONFIG=dmiRocketConfig run-binary LOADARCH=../../hello.riscv.0x80000000.1000.loadarch
+

docs/Advanced-Concepts/index.rst

@@ -16,3 +16,4 @@ They expect you to know about Chisel, Parameters, configs, etc.
    CDEs
    Harness-Clocks
    Managing-Published-Scala-Dependencies
+   Architectural-Checkpoints

docs/Simulation/Software-RTL-Simulation.rst

@@ -151,25 +151,17 @@ Fast Memory Loading
 -------------------
 
 The simulator loads the program binary over a simulated serial line. This can be quite slow if there is a lot of static data, so the simulator also allows data to be loaded from a file directly into the DRAM model.
+Loadmem files should be ELF files. In the most common use case, this can be the binary.
 
 .. code-block:: shell
 
-    make run-binary BINARY=test.riscv LOADMEM=testdata.hex LOADMEM_ADDR=81000000
+    make run-binary BINARY=test.riscv LOADMEM=test.riscv
 
-The ``.hex`` file should be a text file with a hexadecimal number on each line.
-
-.. code-block:: text
-
-    deadbeef
-    0123
-
-Each line uses little-endian order, so this file would produce the bytes "ef be ad de 01 23". ``LOADMEM_ADDR`` specifies which address in memory (in hexadecimal) to write the first byte to. The default is 0x81000000.
-
-A special target that facilitates automatically generating a hex file for an entire elf RISC-V exectuable and then running the simulator with the appropriate flags is also available.
+Usually the ``LOADMEM`` ELF is the same as the ``BINARY`` ELF, so ``LOADMEM=1`` can be used as a shortcut.
 
 .. code-block:: shell
 
-    make run-binary-hex BINARY=test.riscv
+   make run-binary BINARY=test.riscv LOADMEM=1
 
 Generating Waveforms
 -----------------------

docs/Software/Spike.rst

@@ -43,7 +43,7 @@ Spike-as-a-Tile can be configured with custom IPC, commit logging, and other beh
 
 ..  code-block:: shell
 
-    make CONFIG=SpikeUltraFastConfig run-binary-hex BINARY=hello.riscv EXTRA_SPIKE_FLAGS="+spike-ipc=10000 +spike-fast-clint +spike-debug"
+    make CONFIG=SpikeUltraFastConfig run-binary BINARY=hello.riscv EXTRA_SPIKE_FLAGS="+spike-ipc=10000 +spike-fast-clint +spike-debug" LOADMEM=1
 
 
 * ``+spike-ipc=``: Sets the maximum number of instructions Spike can retire in a single "tick", or cycle of the uncore simulation.

generators/chipyard/src/main/resources/csrc/cospike.cc

@@ -6,6 +6,13 @@
 #include <sstream>
 #include <set>
 
+#if __has_include ("cospike_dtm.h")
+#define COSPIKE_DTM
+#include "testchip_dtm.h"
+extern testchip_dtm_t* dtm;
+bool spike_loadarch_done = false;
+#endif
+
 #define CLINT_BASE (0x2000000)
 #define CLINT_SIZE (0x1000)
 
@@ -64,7 +71,8 @@ extern "C" void cospike_cosim(long long int cycle,
                               int raise_exception,
                               int raise_interrupt,
                               unsigned long long int cause,
-                              unsigned long long int wdata)
+                              unsigned long long int wdata,
+			      int priv)
 {
   assert(info);
   if (!sim) {
@@ -168,16 +176,76 @@ extern "C" void cospike_cosim(long long int cycle,
     printf("Fromhost: %lx\n", fromhost_addr);
   }
 
+  if (priv & 0x4) { // debug
+    return;
+  }
+
   processor_t* p = sim->get_core(hartid);
   state_t* s = p->get_state();
+#ifdef COSPIKE_DTM
+  if (dtm && dtm->loadarch_done && !spike_loadarch_done) {
+    printf("Restoring spike state from testchip_dtm loadarch\n");
+    // copy the loadarch state into the cosim
+    loadarch_state_t &ls = dtm->loadarch_state[hartid];
+    s->pc  = ls.pc;
+    s->prv = ls.prv;
+    s->csrmap[CSR_MSTATUS]->write(s->csrmap[CSR_MSTATUS]->read() | MSTATUS_VS | MSTATUS_XS | MSTATUS_FS);
+#define RESTORE(CSRID, csr) s->csrmap[CSRID]->write(ls.csr);
+    RESTORE(CSR_FCSR     , fcsr);
+    RESTORE(CSR_VSTART   , vstart);
+    RESTORE(CSR_VXSAT    , vxsat);
+    RESTORE(CSR_VXRM     , vxrm);
+    RESTORE(CSR_VCSR     , vcsr);
+    RESTORE(CSR_VTYPE    , vtype);
+    RESTORE(CSR_STVEC    , stvec);
+    RESTORE(CSR_SSCRATCH , sscratch);
+    RESTORE(CSR_SEPC     , sepc);
+    RESTORE(CSR_SCAUSE   , scause);
+    RESTORE(CSR_STVAL    , stval);
+    RESTORE(CSR_SATP     , satp);
+    RESTORE(CSR_MSTATUS  , mstatus);
+    RESTORE(CSR_MEDELEG  , medeleg);
+    RESTORE(CSR_MIDELEG  , mideleg);
+    RESTORE(CSR_MIE      , mie);
+    RESTORE(CSR_MTVEC    , mtvec);
+    RESTORE(CSR_MSCRATCH , mscratch);
+    RESTORE(CSR_MEPC     , mepc);
+    RESTORE(CSR_MCAUSE   , mcause);
+    RESTORE(CSR_MTVAL    , mtval);
+    RESTORE(CSR_MIP      , mip);
+    RESTORE(CSR_MCYCLE   , mcycle);
+    RESTORE(CSR_MINSTRET , minstret);
+    if (ls.VLEN != p->VU.VLEN) {
+      printf("VLEN mismatch loadarch: $d != spike: $d\n", ls.VLEN, p->VU.VLEN);
+      abort();
+    }
+    if (ls.ELEN != p->VU.ELEN) {
+      printf("ELEN mismatch loadarch: $d != spike: $d\n", ls.ELEN, p->VU.ELEN);
+      abort();
+    }
+    for (size_t i = 0; i < 32; i++) {
+      s->XPR.write(i, ls.XPR[i]);
+      s->FPR.write(i, { (uint64_t)ls.FPR[i], (uint64_t)-1 });
+      memcpy(p->VU.reg_file + i * ls.VLEN / 8, ls.VPR[i], ls.VLEN / 8);
+    }
+    spike_loadarch_done = true;
+    p->clear_waiting_for_interrupt();
+  }
+#endif
   uint64_t s_pc = s->pc;
+  uint64_t interrupt_cause = cause & 0x7FFFFFFFFFFFFFFF;
+  bool msip_interrupt = interrupt_cause == 0x3;
+  bool debug_interrupt = interrupt_cause == 0xe;
   if (raise_interrupt) {
     printf("%d interrupt %lx\n", cycle, cause);
-    uint64_t interrupt_cause = cause & 0x7FFFFFFFFFFFFFFF;
-    if (interrupt_cause == 3) {
+
+    if (msip_interrupt) {
       s->mip->backdoor_write_with_mask(MIP_MSIP, MIP_MSIP);
+    } else if (debug_interrupt) {
+      return;
     } else {
       printf("Unknown interrupt %lx\n", interrupt_cause);
+      abort();
     }
   }
   if (raise_exception)
@@ -189,12 +257,13 @@ extern "C" void cospike_cosim(long long int cycle,
     }
     printf("\n");
   }
-  if (valid || raise_interrupt || raise_exception)
+  if (valid || raise_interrupt || raise_exception) {
     p->step(1);
+  }
 
   if (valid) {
     if (s_pc != iaddr) {
-      printf("%d PC mismatch %lx != %lx\n", cycle, s_pc, iaddr);
+      printf("%d PC mismatch spike:%lx != dut:%lx\n", cycle, s_pc, iaddr);
       exit(1);
     }
 
@@ -247,12 +316,13 @@ extern "C" void cospike_cosim(long long int cycle,
 	    printf("Read override %lx\n", mem_read_addr);
 	    s->XPR.write(rd, wdata);
           } else if (wdata != regwrite.second.v[0]) {
-	    printf("%d wdata mismatch reg %d %lx != %lx\n", cycle, rd, regwrite.second.v[0], wdata);
+	    printf("%d wdata mismatch reg %d spike:%lx != dut:%lx addr: %lx\n",
+		   cycle, rd, regwrite.second.v[0], wdata, mem_read_addr);
 	    exit(1);
 	  }
 	}
       }
     }
   }
 }
-// }
+