Add aarch64 x86 riscv dcache manage

modules/axhal/src/arch/aarch64/cache.rs

@@ -0,0 +1,110 @@
+use core::arch::asm;
+
+use aarch64_cpu::{
+    asm::barrier::{dsb, isb, SY},
+    registers::*,
+};
+use memory_addr::VirtAddr;
+
+use crate::arch::CacheOp;
+
+#[inline(always)]
+fn cache_line_size() -> usize {
+    unsafe {
+        let mut ctr_el0: u64;
+        asm!("mrs {}, ctr_el0", out(reg) ctr_el0);
+        let log2_cache_line_size = ((ctr_el0 >> 16) & 0xF) as usize;
+        // Calculate the cache line size
+        4 << log2_cache_line_size
+    }
+}
+
+/// Performs a cache operation on all memory.
+pub fn dcache_all(op: CacheOp) {
+    let clidr = CLIDR_EL1.get();
+
+    for level in 0..8 {
+        let ty = (clidr >> (level * 3)) & 0b111;
+        if ty == 0 {
+            return;
+        }
+
+        dcache_level(op, level);
+    }
+    dsb(SY);
+    isb(SY);
+}
+
+/// Performs a cache operation on a range of memory.
+#[inline]
+pub fn dcache_range(op: CacheOp, addr: VirtAddr, size: usize) {
+    let start = addr.as_usize();
+    let end = start + size;
+    let cache_line_size = cache_line_size();
+
+    let mut aligned_addr = addr.as_usize() & !(cache_line_size - 1);
+
+    while aligned_addr < end {
+        let addr = aligned_addr;
+        aligned_addr += cache_line_size;
+
+        _dcache_line(op, addr.into());
+    }
+
+    dsb(SY);
+}
+
+/// Performs a cache operation on a cache line.
+#[inline]
+fn _dcache_line(op: CacheOp, addr: VirtAddr) {
+    unsafe {
+        match op {
+            CacheOp::Invalidate => asm!("dc ivac, {0:x}", in(reg) addr.as_usize()),
+            CacheOp::Clean => asm!("dc cvac, {0:x}", in(reg) addr.as_usize()),
+            CacheOp::CleanAndInvalidate => asm!("dc civac, {0:x}", in(reg) addr.as_usize()),
+        }
+    }
+}
+
+/// Performs a cache operation on a cache line.
+#[inline]
+pub fn dcache_line(op: CacheOp, addr: VirtAddr) {
+    _dcache_line(op, addr);
+    dsb(SY);
+    isb(SY);
+}
+
+/// Performs a cache operation on a cache level.
+/// https://developer.arm.com/documentation/ddi0601/2024-12/AArch64-Instructions/DC-CISW--Data-or-unified-Cache-line-Clean-and-Invalidate-by-Set-Way
+#[inline]
+fn dcache_level(op: CacheOp, level: u64) {
+    assert!(level < 8, "armv8 level range is 0-7");
+
+    isb(SY);
+    CSSELR_EL1.write(CSSELR_EL1::InD::Data + CSSELR_EL1::Level.val(level));
+    isb(SY);
+    let lines = CCSIDR_EL1.read(CCSIDR_EL1::LineSize) as u32;
+    let ways = CCSIDR_EL1.read(CCSIDR_EL1::AssociativityWithCCIDX) as u32;
+    let sets = CCSIDR_EL1.read(CCSIDR_EL1::NumSetsWithCCIDX) as u32;
+
+    let l = lines + 4;
+
+    // Calculate bit position of number of ways
+    let way_adjust = ways.leading_zeros();
+
+    // Loop over sets and ways
+    for set in 0..sets {
+        for way in 0..ways {
+            let set_way = (way << way_adjust) | set << l;
+
+            let cisw = (level << 1) | set_way as u64;
+            unsafe {
+                match op {
+                    CacheOp::Invalidate => asm!("dc isw, {0}", in(reg) cisw),
+                    CacheOp::Clean => asm!("dc csw, {0}", in(reg) cisw),
+                    CacheOp::CleanAndInvalidate => asm!("dc cisw, {0}", in(reg) cisw),
+                }
+            }
+        }
+    }
+}

modules/axhal/src/arch/aarch64/mod.rs

@@ -1,3 +1,4 @@
+mod cache;
 mod context;
 pub(crate) mod trap;
 
@@ -8,6 +9,7 @@ use memory_addr::{PhysAddr, VirtAddr};
 use tock_registers::interfaces::{Readable, Writeable};
 
 pub use self::context::{FpState, TaskContext, TrapFrame};
+pub use cache::*;
 
 /// Allows the current CPU to respond to interrupts.
 #[inline]
@@ -110,12 +112,6 @@ pub fn set_exception_vector_base(vbar_el1: usize) {
     VBAR_EL1.set(vbar_el1 as _);
 }
 
-/// Flushes the data cache line (64 bytes) at the given virtual address
-#[inline]
-pub fn flush_dcache_line(vaddr: VirtAddr) {
-    unsafe { asm!("dc ivac, {0:x}; dsb sy; isb", in(reg) vaddr.as_usize()) };
-}
-
 /// Reads the thread pointer of the current CPU.
 ///
 /// It is used to implement TLS (Thread Local Storage).

modules/axhal/src/arch/mod.rs

@@ -12,3 +12,14 @@ cfg_if::cfg_if! {
         pub use self::aarch64::*;
     }
 }
+
+/// Cache operations.
+#[derive(Debug, Clone, Copy)]
+pub enum CacheOp {
+    /// Write back to memory
+    Clean,
+    /// Invalidate cache
+    Invalidate,
+    /// Clean and invalidate
+    CleanAndInvalidate,
+}

modules/axhal/src/arch/riscv/cache.rs

@@ -0,0 +1,13 @@
+use memory_addr::VirtAddr;
+
+use crate::arch::CacheOp;
+
+/// Performs a cache operation on a range of memory.
+#[inline]
+pub fn dcache_range(_op: CacheOp, _addr: VirtAddr, _size: usize) {
+    // The cache coherency in x86 architectures is guaranteed by hardware.
+}
+
+/// Performs a cache operation on a cache line.
+#[inline]
+pub fn dcache_line(_op: CacheOp, _addr: VirtAddr) {}

modules/axhal/src/arch/riscv/mod.rs

@@ -1,6 +1,7 @@
 #[macro_use]
 mod macros;
 
+mod cache;
 mod context;
 mod trap;
 
@@ -9,6 +10,7 @@ use riscv::asm;
 use riscv::register::{satp, sstatus, stvec};
 
 pub use self::context::{GeneralRegisters, TaskContext, TrapFrame};
+pub use cache::*;
 
 /// Allows the current CPU to respond to interrupts.
 #[inline]

modules/axhal/src/arch/x86_64/cache.rs

@@ -0,0 +1,10 @@
+use memory_addr::VirtAddr;
+
+use crate::arch::CacheOp;
+
+/// Performs a cache operation on a range of memory.
+pub fn dcache_range(_op: CacheOp, _addr: VirtAddr, _size: usize) {}
+
+/// Performs a cache operation on a cache line.
+#[inline]
+pub fn dcache_line(_op: CacheOp, _addr: VirtAddr) {}

modules/axhal/src/arch/x86_64/mod.rs

@@ -1,3 +1,4 @@
+mod cache;
 mod context;
 mod gdt;
 mod idt;
@@ -14,6 +15,7 @@ use x86_64::instructions::interrupts;
 pub use self::context::{ExtendedState, FxsaveArea, TaskContext, TrapFrame};
 pub use self::gdt::GdtStruct;
 pub use self::idt::IdtStruct;
+pub use cache::*;
 pub use x86_64::structures::tss::TaskStateSegment;
 
 /// Allows the current CPU to respond to interrupts.

modules/axhal/src/platform/aarch64_raspi/mp.rs

@@ -32,11 +32,14 @@ pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) {
         let spintable_vaddr = phys_to_virt(CPU_SPIN_TABLE[cpu_id]);
         let release_ptr = spintable_vaddr.as_mut_ptr() as *mut usize;
         release_ptr.write_volatile(entry_paddr);
-        crate::arch::flush_dcache_line(spintable_vaddr);
+        crate::arch::dcache_line(crate::arch::CacheOp::CleanAndInvalidate, spintable_vaddr);
 
         // set the boot stack of the given secondary CPU
         SECONDARY_STACK_TOP = stack_top.as_usize();
-        crate::arch::flush_dcache_line(va!(core::ptr::addr_of!(SECONDARY_STACK_TOP) as usize));
+        crate::arch::dcache_line(
+            crate::arch::CacheOp::CleanAndInvalidate,
+            va!(core::ptr::addr_of!(SECONDARY_STACK_TOP) as usize),
+        );
     }
     aarch64_cpu::asm::sev();
 }