From 135901d3cc2c2f41090b40aaae66c83ac5564146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=91=A8=E7=9D=BF?= Date: Mon, 9 Dec 2024 09:27:04 +0800 Subject: [PATCH] Add aarch64 x86 riscv dcache manage --- modules/axhal/src/arch/aarch64/cache.rs | 110 ++++++++++++++++++ modules/axhal/src/arch/aarch64/mod.rs | 8 +- modules/axhal/src/arch/mod.rs | 11 ++ modules/axhal/src/arch/riscv/cache.rs | 13 +++ modules/axhal/src/arch/riscv/mod.rs | 2 + modules/axhal/src/arch/x86_64/cache.rs | 10 ++ modules/axhal/src/arch/x86_64/mod.rs | 2 + .../axhal/src/platform/aarch64_raspi/mp.rs | 7 +- 8 files changed, 155 insertions(+), 8 deletions(-) create mode 100644 modules/axhal/src/arch/aarch64/cache.rs create mode 100644 modules/axhal/src/arch/riscv/cache.rs create mode 100644 modules/axhal/src/arch/x86_64/cache.rs diff --git a/modules/axhal/src/arch/aarch64/cache.rs b/modules/axhal/src/arch/aarch64/cache.rs new file mode 100644 index 0000000000..2e35ca45e3 --- /dev/null +++ b/modules/axhal/src/arch/aarch64/cache.rs @@ -0,0 +1,110 @@ +use core::arch::asm; + +use aarch64_cpu::{ + asm::barrier::{dsb, isb, SY}, + registers::*, +}; +use memory_addr::VirtAddr; + +use crate::arch::CacheOp; + +#[inline(always)] +fn cache_line_size() -> usize { + unsafe { + let mut ctr_el0: u64; + asm!("mrs {}, ctr_el0", out(reg) ctr_el0); + let log2_cache_line_size = ((ctr_el0 >> 16) & 0xF) as usize; + // Calculate the cache line size + 4 << log2_cache_line_size + } +} + +/// Performs a cache operation on all memory. +pub fn dcache_all(op: CacheOp) { + let clidr = CLIDR_EL1.get(); + + for level in 0..8 { + let ty = (clidr >> (level * 3)) & 0b111; + if ty == 0 { + return; + } + + dcache_level(op, level); + } + dsb(SY); + isb(SY); +} + +/// Performs a cache operation on a range of memory. +#[inline] +pub fn dcache_range(op: CacheOp, addr: VirtAddr, size: usize) { + let start = addr.as_usize(); + let end = start + size; + let cache_line_size = cache_line_size(); + + let mut aligned_addr = addr.as_usize() & !(cache_line_size - 1); + + while aligned_addr < end { + let addr = aligned_addr; + aligned_addr += cache_line_size; + + _dcache_line(op, addr.into()); + } + + dsb(SY); +} + +/// Performs a cache operation on a cache line. +#[inline] +fn _dcache_line(op: CacheOp, addr: VirtAddr) { + unsafe { + match op { + CacheOp::Invalidate => asm!("dc ivac, {0:x}", in(reg) addr.as_usize()), + CacheOp::Clean => asm!("dc cvac, {0:x}", in(reg) addr.as_usize()), + CacheOp::CleanAndInvalidate => asm!("dc civac, {0:x}", in(reg) addr.as_usize()), + } + } +} + +/// Performs a cache operation on a cache line. +#[inline] +pub fn dcache_line(op: CacheOp, addr: VirtAddr) { + _dcache_line(op, addr); + dsb(SY); + isb(SY); +} + +/// Performs a cache operation on a cache level. +/// https://developer.arm.com/documentation/ddi0601/2024-12/AArch64-Instructions/DC-CISW--Data-or-unified-Cache-line-Clean-and-Invalidate-by-Set-Way +#[inline] +fn dcache_level(op: CacheOp, level: u64) { + assert!(level < 8, "armv8 level range is 0-7"); + + isb(SY); + CSSELR_EL1.write(CSSELR_EL1::InD::Data + CSSELR_EL1::Level.val(level)); + isb(SY); + let lines = CCSIDR_EL1.read(CCSIDR_EL1::LineSize) as u32; + let ways = CCSIDR_EL1.read(CCSIDR_EL1::AssociativityWithCCIDX) as u32; + let sets = CCSIDR_EL1.read(CCSIDR_EL1::NumSetsWithCCIDX) as u32; + + let l = lines + 4; + + // Calculate bit position of number of ways + let way_adjust = ways.leading_zeros(); + + // Loop over sets and ways + for set in 0..sets { + for way in 0..ways { + let set_way = (way << way_adjust) | set << l; + + let cisw = (level << 1) | set_way as u64; + unsafe { + match op { + CacheOp::Invalidate => asm!("dc isw, {0}", in(reg) cisw), + CacheOp::Clean => asm!("dc csw, {0}", in(reg) cisw), + CacheOp::CleanAndInvalidate => asm!("dc cisw, {0}", in(reg) cisw), + } + } + } + } +} diff --git a/modules/axhal/src/arch/aarch64/mod.rs b/modules/axhal/src/arch/aarch64/mod.rs index 160dabdd12..a109b43a63 100644 --- a/modules/axhal/src/arch/aarch64/mod.rs +++ b/modules/axhal/src/arch/aarch64/mod.rs @@ -1,3 +1,4 @@ +mod cache; mod context; pub(crate) mod trap; @@ -8,6 +9,7 @@ use memory_addr::{PhysAddr, VirtAddr}; use tock_registers::interfaces::{Readable, Writeable}; pub use self::context::{FpState, TaskContext, TrapFrame}; +pub use cache::*; /// Allows the current CPU to respond to interrupts. #[inline] @@ -110,12 +112,6 @@ pub fn set_exception_vector_base(vbar_el1: usize) { VBAR_EL1.set(vbar_el1 as _); } -/// Flushes the data cache line (64 bytes) at the given virtual address -#[inline] -pub fn flush_dcache_line(vaddr: VirtAddr) { - unsafe { asm!("dc ivac, {0:x}; dsb sy; isb", in(reg) vaddr.as_usize()) }; -} - /// Reads the thread pointer of the current CPU. /// /// It is used to implement TLS (Thread Local Storage). diff --git a/modules/axhal/src/arch/mod.rs b/modules/axhal/src/arch/mod.rs index b8bc0af3d0..9b8681d450 100644 --- a/modules/axhal/src/arch/mod.rs +++ b/modules/axhal/src/arch/mod.rs @@ -12,3 +12,14 @@ cfg_if::cfg_if! { pub use self::aarch64::*; } } + +/// Cache operations. +#[derive(Debug, Clone, Copy)] +pub enum CacheOp { + /// Write back to memory + Clean, + /// Invalidate cache + Invalidate, + /// Clean and invalidate + CleanAndInvalidate, +} diff --git a/modules/axhal/src/arch/riscv/cache.rs b/modules/axhal/src/arch/riscv/cache.rs new file mode 100644 index 0000000000..bdf87b1ac0 --- /dev/null +++ b/modules/axhal/src/arch/riscv/cache.rs @@ -0,0 +1,13 @@ +use memory_addr::VirtAddr; + +use crate::arch::CacheOp; + +/// Performs a cache operation on a range of memory. +#[inline] +pub fn dcache_range(_op: CacheOp, _addr: VirtAddr, _size: usize) { + // The cache coherency in x86 architectures is guaranteed by hardware. +} + +/// Performs a cache operation on a cache line. +#[inline] +pub fn dcache_line(_op: CacheOp, _addr: VirtAddr) {} diff --git a/modules/axhal/src/arch/riscv/mod.rs b/modules/axhal/src/arch/riscv/mod.rs index df6927dddc..fc39fd4538 100644 --- a/modules/axhal/src/arch/riscv/mod.rs +++ b/modules/axhal/src/arch/riscv/mod.rs @@ -1,6 +1,7 @@ #[macro_use] mod macros; +mod cache; mod context; mod trap; @@ -9,6 +10,7 @@ use riscv::asm; use riscv::register::{satp, sstatus, stvec}; pub use self::context::{GeneralRegisters, TaskContext, TrapFrame}; +pub use cache::*; /// Allows the current CPU to respond to interrupts. #[inline] diff --git a/modules/axhal/src/arch/x86_64/cache.rs b/modules/axhal/src/arch/x86_64/cache.rs new file mode 100644 index 0000000000..fd4a96da43 --- /dev/null +++ b/modules/axhal/src/arch/x86_64/cache.rs @@ -0,0 +1,10 @@ +use memory_addr::VirtAddr; + +use crate::arch::CacheOp; + +/// Performs a cache operation on a range of memory. +pub fn dcache_range(_op: CacheOp, _addr: VirtAddr, _size: usize) {} + +/// Performs a cache operation on a cache line. +#[inline] +pub fn dcache_line(_op: CacheOp, _addr: VirtAddr) {} diff --git a/modules/axhal/src/arch/x86_64/mod.rs b/modules/axhal/src/arch/x86_64/mod.rs index c9002a58d3..d4feac5fa7 100644 --- a/modules/axhal/src/arch/x86_64/mod.rs +++ b/modules/axhal/src/arch/x86_64/mod.rs @@ -1,3 +1,4 @@ +mod cache; mod context; mod gdt; mod idt; @@ -14,6 +15,7 @@ use x86_64::instructions::interrupts; pub use self::context::{ExtendedState, FxsaveArea, TaskContext, TrapFrame}; pub use self::gdt::GdtStruct; pub use self::idt::IdtStruct; +pub use cache::*; pub use x86_64::structures::tss::TaskStateSegment; /// Allows the current CPU to respond to interrupts. diff --git a/modules/axhal/src/platform/aarch64_raspi/mp.rs b/modules/axhal/src/platform/aarch64_raspi/mp.rs index 8447daec8d..6a5211f96d 100644 --- a/modules/axhal/src/platform/aarch64_raspi/mp.rs +++ b/modules/axhal/src/platform/aarch64_raspi/mp.rs @@ -32,11 +32,14 @@ pub fn start_secondary_cpu(cpu_id: usize, stack_top: PhysAddr) { let spintable_vaddr = phys_to_virt(CPU_SPIN_TABLE[cpu_id]); let release_ptr = spintable_vaddr.as_mut_ptr() as *mut usize; release_ptr.write_volatile(entry_paddr); - crate::arch::flush_dcache_line(spintable_vaddr); + crate::arch::dcache_line(crate::arch::CacheOp::CleanAndInvalidate, spintable_vaddr); // set the boot stack of the given secondary CPU SECONDARY_STACK_TOP = stack_top.as_usize(); - crate::arch::flush_dcache_line(va!(core::ptr::addr_of!(SECONDARY_STACK_TOP) as usize)); + crate::arch::dcache_line( + crate::arch::CacheOp::CleanAndInvalidate, + va!(core::ptr::addr_of!(SECONDARY_STACK_TOP) as usize), + ); } aarch64_cpu::asm::sev(); }