diff --git a/src/mmtk.rs b/src/mmtk.rs index bf3d87c732..6a231041db 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -119,7 +119,7 @@ impl MMTK { let scheduler = GCWorkScheduler::new(num_workers, (*options.thread_affinity).clone()); - let plan = crate::plan::create_plan( + let mut plan = crate::plan::create_plan( *options.plan, VM_MAP.as_ref(), MMAPPER.as_ref(), @@ -128,11 +128,13 @@ impl MMTK { ); // TODO: This probably does not work if we have multiple MMTk instances. - VM_MAP.boot(); // This needs to be called after we create Plan. It needs to use HeapMeta, which is gradually built when we create spaces. VM_MAP.finalize_static_space_map( plan.base().heap.get_discontig_start(), plan.base().heap.get_discontig_end(), + &mut |start| { + plan.update_discontiguous_page_resources(start); + }, ); if *options.transparent_hugepages { diff --git a/src/plan/global.rs b/src/plan/global.rs index df887488dd..2f03d76d40 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -25,7 +25,7 @@ use crate::util::metadata::side_metadata::SideMetadataSpec; use crate::util::options::Options; use crate::util::options::PlanSelector; use crate::util::statistics::stats::Stats; -use crate::util::{conversions, ObjectReference}; +use crate::util::{conversions, Address, ObjectReference}; use crate::util::{VMMutatorThread, VMWorkerThread}; use crate::vm::*; use downcast_rs::Downcast; @@ -371,6 +371,14 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { true } + fn update_discontiguous_page_resources(&mut self, start: Address) { + self.for_each_space_mut(&mut |space: &mut dyn Space| { + space.for_each_page_resource_mut(&mut |pr| { + pr.update_discontiguous_start(start); + }); + }); + } + /// Call `space.verify_side_metadata_sanity` for all spaces in this plan. fn verify_side_metadata_sanity(&self) { let mut side_metadata_sanity_checker = SideMetadataSanity::new(); diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index b377db5e96..119c085d1b 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -102,6 +102,10 @@ impl Space for CopySpace { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } + fn common(&self) -> &CommonSpace { &self.common } diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index fb578fefe8..61a4cf31b9 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -170,6 +170,9 @@ impl Space for ImmixSpace { fn get_page_resource(&self) -> &dyn PageResource { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } fn common(&self) -> &CommonSpace { &self.common } diff --git a/src/policy/immortalspace.rs b/src/policy/immortalspace.rs index 5eeebd58c9..73fcccace9 100644 --- a/src/policy/immortalspace.rs +++ b/src/policy/immortalspace.rs @@ -87,6 +87,9 @@ impl Space for ImmortalSpace { fn get_page_resource(&self) -> &dyn PageResource { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } fn common(&self) -> &CommonSpace { &self.common } diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index ec6b2f7506..b39dfefea3 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -106,6 +106,9 @@ impl Space for LargeObjectSpace { fn get_page_resource(&self) -> &dyn PageResource { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) { self.common().initialize_sft(self.as_sft(), sft_map) diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index bd2cabad40..0089b5c535 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -95,6 +95,10 @@ impl Space for LockFreeImmortalSpace { fn get_page_resource(&self) -> &dyn PageResource { unimplemented!() } + fn for_each_page_resource_mut(&mut self, _f: &mut dyn FnMut(&mut dyn PageResource)) { + // We have no page resources! + } + fn common(&self) -> &CommonSpace { unimplemented!() } diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 693218b492..777b765d38 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -109,6 +109,10 @@ impl Space for MarkCompactSpace { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } + fn common(&self) -> &CommonSpace { &self.common } diff --git a/src/policy/marksweepspace/malloc_ms/global.rs b/src/policy/marksweepspace/malloc_ms/global.rs index 7454fbb287..63d6c3d882 100644 --- a/src/policy/marksweepspace/malloc_ms/global.rs +++ b/src/policy/marksweepspace/malloc_ms/global.rs @@ -137,6 +137,10 @@ impl Space for MallocSpace { unreachable!() } + fn for_each_page_resource_mut(&mut self, _f: &mut dyn FnMut(&mut dyn PageResource)) { + // We have no page resources! + } + fn common(&self) -> &CommonSpace { unreachable!() } diff --git a/src/policy/marksweepspace/native_ms/global.rs b/src/policy/marksweepspace/native_ms/global.rs index 8d8eae7d0e..c5f6ca39f8 100644 --- a/src/policy/marksweepspace/native_ms/global.rs +++ b/src/policy/marksweepspace/native_ms/global.rs @@ -7,7 +7,7 @@ use crate::{ scheduler::{GCWorkScheduler, GCWorker}, util::{ copy::CopySemantics, - heap::FreeListPageResource, + heap::{FreeListPageResource, PageResource}, metadata::{self, side_metadata::SideMetadataSpec, MetadataSpec}, ObjectReference, }, @@ -152,10 +152,14 @@ impl Space for MarkSweepSpace { self } - fn get_page_resource(&self) -> &dyn crate::util::heap::PageResource { + fn get_page_resource(&self) -> &dyn PageResource { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr); + } + fn initialize_sft(&self, sft_map: &mut dyn crate::policy::sft_map::SFTMap) { self.common().initialize_sft(self.as_sft(), sft_map) } diff --git a/src/policy/space.rs b/src/policy/space.rs index 77e5694cc2..a67a048484 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -39,8 +39,13 @@ use downcast_rs::Downcast; pub trait Space: 'static + SFT + Sync + Downcast { fn as_space(&self) -> &dyn Space; fn as_sft(&self) -> &(dyn SFT + Sync + 'static); + + // TODO: Generate this function automatically in mmtk-macros like PlanTraceObject. fn get_page_resource(&self) -> &dyn PageResource; + // TODO: Generate this function automatically in mmtk-macros like PlanTraceObject. + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)); + /// Initialize entires in SFT map for the space. This is called when the Space object /// has a non-moving address, as we will use the address to set sft. /// Currently after we create a boxed plan, spaces in the plan have a non-moving address. diff --git a/src/policy/vmspace.rs b/src/policy/vmspace.rs index 7aa1eb9903..63a06c8741 100644 --- a/src/policy/vmspace.rs +++ b/src/policy/vmspace.rs @@ -87,6 +87,9 @@ impl Space for VMSpace { fn get_page_resource(&self) -> &dyn PageResource { &self.pr } + fn for_each_page_resource_mut(&mut self, f: &mut dyn FnMut(&mut dyn PageResource)) { + f(&mut self.pr) + } fn common(&self) -> &CommonSpace { &self.common } diff --git a/src/util/freelist.rs b/src/util/freelist.rs index 5ac4101901..21ac28b2a5 100644 --- a/src/util/freelist.rs +++ b/src/util/freelist.rs @@ -14,7 +14,12 @@ const MULTI_MASK: i32 = 1 << (TOTAL_BITS - 1); const COALESC_MASK: i32 = 1 << (TOTAL_BITS - 2); const SIZE_MASK: i32 = (1 << UNIT_BITS) - 1; -pub trait FreeList: Sync + Downcast { +// TODO: FreeList should not implement Sync. +// FreeList instances are not thread-safe. +// They need external synchronisation (e.g. using Mutex). +// On the other hand, to put FreeList into a Mutex, FreeList must implement Send. +// There is no problem sending FreeList instances between threads. +pub trait FreeList: Sync + Send + Downcast { fn head(&self) -> i32; // fn head_mut(&mut self) -> &mut i32; fn heads(&self) -> i32; diff --git a/src/util/heap/blockpageresource.rs b/src/util/heap/blockpageresource.rs index 5b3ffbc2f2..dd7279d723 100644 --- a/src/util/heap/blockpageresource.rs +++ b/src/util/heap/blockpageresource.rs @@ -36,6 +36,10 @@ impl PageResource for BlockPageResource { self.flpr.common_mut() } + fn update_discontiguous_start(&mut self, start: Address) { + self.flpr.update_discontiguous_start(start); + } + fn alloc_pages( &self, space_descriptor: SpaceDescriptor, diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index 756ccfda03..536837099f 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -1,5 +1,3 @@ -use std::cell::UnsafeCell; -use std::ops::{Deref, DerefMut}; use std::sync::{Mutex, MutexGuard}; use super::layout::vm_layout::PAGES_IN_CHUNK; @@ -13,98 +11,76 @@ use crate::util::conversions; use crate::util::freelist; use crate::util::freelist::FreeList; use crate::util::heap::layout::vm_layout::*; +use crate::util::heap::layout::CreateFreeListResult; use crate::util::heap::pageresource::CommonPageResource; use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::memory; use crate::util::opaque_pointer::*; +use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::vm::*; use std::marker::PhantomData; const UNINITIALIZED_WATER_MARK: i32 = -1; -pub struct CommonFreeListPageResource { - pub(crate) free_list: Box, - start: Address, -} - -impl CommonFreeListPageResource { - pub fn get_start(&self) -> Address { - self.start - } - - pub fn resize_freelist(&mut self, start_address: Address) { - self.start = start_address.align_up(BYTES_IN_REGION); - } -} - pub struct FreeListPageResource { - inner: UnsafeCell, + common: CommonPageResource, sync: Mutex, _p: PhantomData, /// Protect memory on release, and unprotect on re-allocate. pub(crate) protect_memory_on_release: bool, } -unsafe impl Send for FreeListPageResource {} -unsafe impl Sync for FreeListPageResource {} - -struct FreeListPageResourceInner { - common: CommonPageResource, - common_flpr: Box, -} - struct FreeListPageResourceSync { pages_currently_on_freelist: usize, highwater_mark: i32, + pub(crate) free_list: Box, + start: Address, } -impl Deref for FreeListPageResource { - type Target = CommonFreeListPageResource; - - fn deref(&self) -> &CommonFreeListPageResource { - &self.inner().common_flpr - } -} - -impl DerefMut for FreeListPageResource { - fn deref_mut(&mut self) -> &mut CommonFreeListPageResource { - &mut self.inner.get_mut().common_flpr - } -} - -impl Deref for FreeListPageResourceInner { - type Target = CommonFreeListPageResource; - - fn deref(&self) -> &CommonFreeListPageResource { - &self.common_flpr - } -} +impl FreeListPageResourceSync { + pub fn resize_freelist(&mut self, start_address: Address) { + if cfg!(target_pointer_width = "64") { + // Hack. We'll probably revamp the space initialization process, + // so that we can reset the start addresses in a safer way. + panic!("I don't need to be resized on 64-bit system!"); + } -impl DerefMut for FreeListPageResourceInner { - fn deref_mut(&mut self) -> &mut CommonFreeListPageResource { - &mut self.common_flpr + debug!( + "resizing from {} to {} (align up to {})", + self.start, + start_address, + start_address.align_up(BYTES_IN_REGION) + ); + self.start = start_address.align_up(BYTES_IN_REGION); } } impl PageResource for FreeListPageResource { fn common(&self) -> &CommonPageResource { - &self.inner().common + &self.common } fn common_mut(&mut self) -> &mut CommonPageResource { - &mut self.inner.get_mut().common + &mut self.common + } + fn update_discontiguous_start(&mut self, start: Address) { + // Only discontiguous FreeListPageResource needs adjustment. + if !self.common.contiguous { + // Note: since we have a `&mut self`, we must have exclusive access to self. + // So We don't need to hold the lock. + self.sync.get_mut().unwrap().start = start; + } } fn get_available_physical_pages(&self) -> usize { let mut rtn = self.sync.lock().unwrap().pages_currently_on_freelist; - if !self.inner().common.contiguous { + if !self.common.contiguous { let chunks: usize = self - .inner() .common .vm_map .get_available_discontiguous_chunks() - .saturating_sub(self.inner().common.vm_map.get_chunk_consumer_count()); + .saturating_sub(self.common.vm_map.get_chunk_consumer_count()); rtn += chunks * PAGES_IN_CHUNK; - } else if self.inner().common.growable && cfg!(target_pointer_width = "64") { + } else if self.common.growable && cfg!(target_pointer_width = "64") { rtn = vm_layout().pages_in_space64() - self.reserved_pages(); } @@ -118,15 +94,12 @@ impl PageResource for FreeListPageResource { required_pages: usize, tls: VMThread, ) -> Result { - // FIXME: We need a safe implementation - let self_mut = unsafe { self.inner_mut() }; let mut sync = self.sync.lock().unwrap(); let mut new_chunk = false; - let mut page_offset = self_mut.free_list.alloc(required_pages as _); - if page_offset == freelist::FAILURE && self.inner().common.growable { - page_offset = unsafe { - self.allocate_contiguous_chunks(space_descriptor, required_pages, &mut sync) - }; + let mut page_offset = sync.free_list.alloc(required_pages as _); + if page_offset == freelist::FAILURE && self.common.growable { + page_offset = + self.allocate_contiguous_chunks(space_descriptor, required_pages, &mut sync); new_chunk = true; } @@ -144,7 +117,7 @@ impl PageResource for FreeListPageResource { } } - let rtn = self.start + conversions::pages_to_bytes(page_offset as _); + let rtn = sync.start + conversions::pages_to_bytes(page_offset as _); // The meta-data portion of reserved Pages was committed above. self.commit_pages(reserved_pages, required_pages, tls); if self.protect_memory_on_release { @@ -160,13 +133,13 @@ impl PageResource for FreeListPageResource { // in Space.acquire(). We can either move it the option of 'protect_on_release' to space, or have a call to page resource // after ensure_mapped(). However, I think this is sufficient given that this option is only used for PageProtect for debugging use. while !new_chunk && !MMAPPER.is_mapped_address(rtn) {} - self.munprotect(rtn, self.free_list.size(page_offset as _) as _) + self.munprotect(rtn, sync.free_list.size(page_offset as _) as _) } else if !self.common().contiguous && new_chunk { // Don't unprotect if this is a new unmapped discontiguous chunk // For a new mapped discontiguous chunk, this should previously be released and protected by us. // We still need to unprotect it. if MMAPPER.is_mapped_address(rtn) { - self.munprotect(rtn, self.free_list.size(page_offset as _) as _) + self.munprotect(rtn, sync.free_list.size(page_offset as _) as _) } } }; @@ -180,29 +153,30 @@ impl PageResource for FreeListPageResource { impl FreeListPageResource { pub fn new_contiguous(start: Address, bytes: usize, vm_map: &'static dyn VMMap) -> Self { + debug!("new_contiguous({start}, {bytes})"); let pages = conversions::bytes_to_pages(bytes); - let common_flpr = { - let common_flpr = Box::new(CommonFreeListPageResource { - free_list: vm_map.create_parent_freelist(start, pages, PAGES_IN_REGION as _), - start, - }); - // `CommonFreeListPageResource` lives as a member in space instances. - // Since `Space` instances are always stored as global variables, so it is okay here - // to turn `&CommonFreeListPageResource` into `&'static CommonFreeListPageResource` - unsafe { - vm_map.bind_freelist(&*(&common_flpr as &CommonFreeListPageResource as *const _)); - } - common_flpr - }; + let CreateFreeListResult { + free_list, + space_displacement, + } = vm_map.create_parent_freelist(start, pages, PAGES_IN_REGION as _); + + // If it is RawMemoryFreeList, it will occupy `space_displacement` bytes at the start of + // the space. We displace the start address. + let actual_start = start + space_displacement; + + debug!( + " in new_contiguous: space_displacement = {:?}, actual_start = {}", + space_displacement, actual_start + ); + let growable = cfg!(target_pointer_width = "64"); FreeListPageResource { - inner: UnsafeCell::new(FreeListPageResourceInner { - common: CommonPageResource::new(true, growable, vm_map), - common_flpr, - }), + common: CommonPageResource::new(true, growable, vm_map), sync: Mutex::new(FreeListPageResourceSync { pages_currently_on_freelist: if growable { 0 } else { pages }, highwater_mark: UNINITIALIZED_WATER_MARK, + free_list, + start: actual_start, }), _p: PhantomData, protect_memory_on_release: false, @@ -210,42 +184,30 @@ impl FreeListPageResource { } pub fn new_discontiguous(vm_map: &'static dyn VMMap) -> Self { - let common_flpr = { - let start = vm_layout().available_start(); - let common_flpr = Box::new(CommonFreeListPageResource { - free_list: vm_map.create_freelist(start), - start, - }); - // `CommonFreeListPageResource` lives as a member in space instances. - // Since `Space` instances are always stored as global variables, so it is okay here - // to turn `&CommonFreeListPageResource` into `&'static CommonFreeListPageResource` - unsafe { - vm_map.bind_freelist(&*(&common_flpr as &CommonFreeListPageResource as *const _)); - } - common_flpr - }; + let start = vm_layout().available_start(); + let CreateFreeListResult { + free_list, + space_displacement, + } = vm_map.create_freelist(start); + + // Discontiguous free list page resources are only used on 32-bit machines, where + // IntArrayFreeList is used exclusively. It does not have space displacement. + debug_assert_eq!(space_displacement, 0); + debug!("new_discontiguous. start: {start})"); + FreeListPageResource { - inner: UnsafeCell::new(FreeListPageResourceInner { - common: CommonPageResource::new(false, true, vm_map), - common_flpr, - }), + common: CommonPageResource::new(false, true, vm_map), sync: Mutex::new(FreeListPageResourceSync { pages_currently_on_freelist: 0, highwater_mark: UNINITIALIZED_WATER_MARK, + free_list, + start, }), _p: PhantomData, protect_memory_on_release: false, } } - fn inner(&self) -> &FreeListPageResourceInner { - unsafe { &*self.inner.get() } - } - #[allow(clippy::mut_from_ref)] - unsafe fn inner_mut(&self) -> &mut FreeListPageResourceInner { - &mut *self.inner.get() - } - /// Protect the memory fn mprotect(&self, start: Address, pages: usize) { // We may fail here for ENOMEM, especially in PageProtect plan. @@ -280,11 +242,10 @@ impl FreeListPageResource { &self, space_descriptor: SpaceDescriptor, ) -> Result { - assert!(self.inner().common.growable); - // FIXME: We need a safe implementation + assert!(self.common.growable); let mut sync = self.sync.lock().unwrap(); let page_offset = - unsafe { self.allocate_contiguous_chunks(space_descriptor, PAGES_IN_CHUNK, &mut sync) }; + self.allocate_contiguous_chunks(space_descriptor, PAGES_IN_CHUNK, &mut sync); if page_offset == freelist::FAILURE { return Result::Err(PRAllocFail); @@ -295,7 +256,7 @@ impl FreeListPageResource { } } - let rtn = self.start + conversions::pages_to_bytes(page_offset as _); + let rtn = sync.start + conversions::pages_to_bytes(page_offset as _); Result::Ok(PRAllocResult { start: rtn, pages: PAGES_IN_CHUNK, @@ -303,7 +264,7 @@ impl FreeListPageResource { }) } - unsafe fn allocate_contiguous_chunks( + fn allocate_contiguous_chunks( &self, space_descriptor: SpaceDescriptor, pages: usize, @@ -311,45 +272,44 @@ impl FreeListPageResource { ) -> i32 { let mut rtn = freelist::FAILURE; let required_chunks = crate::policy::space::required_chunks(pages); - let region = self - .inner() - .common - .grow_discontiguous_space(space_descriptor, required_chunks); + let region = { + let maybe_raw_memory_freelist = sync.free_list.downcast_mut::(); + self.common.grow_discontiguous_space( + space_descriptor, + required_chunks, + maybe_raw_memory_freelist, + ) + }; if !region.is_zero() { - let region_start = conversions::bytes_to_pages(region - self.start); + dbg!(region); + dbg!(sync.start); + let region_start = conversions::bytes_to_pages(region - sync.start); + dbg!(region_start); let region_end = region_start + (required_chunks * PAGES_IN_CHUNK) - 1; - self.inner_mut() - .free_list - .set_uncoalescable(region_start as _); - self.inner_mut() - .free_list - .set_uncoalescable(region_end as i32 + 1); + sync.free_list.set_uncoalescable(region_start as _); + sync.free_list.set_uncoalescable(region_end as i32 + 1); for p in (region_start..region_end).step_by(PAGES_IN_CHUNK) { if p != region_start { - self.inner_mut().free_list.clear_uncoalescable(p as _); + sync.free_list.clear_uncoalescable(p as _); } - let liberated = self.inner_mut().free_list.free(p as _, true); // add chunk to our free list - debug_assert!(liberated as usize == PAGES_IN_CHUNK + (p - region_start)); + let liberated = sync.free_list.free(p as _, true); // add chunk to our free list + debug_assert_eq!(liberated as usize, PAGES_IN_CHUNK + (p - region_start)); sync.pages_currently_on_freelist += PAGES_IN_CHUNK; } - rtn = self.inner_mut().free_list.alloc(pages as _); // re-do the request which triggered this call + rtn = sync.free_list.alloc(pages as _); // re-do the request which triggered this call } - rtn } - unsafe fn free_contiguous_chunk(&self, chunk: Address, sync: &mut FreeListPageResourceSync) { + fn free_contiguous_chunk(&self, chunk: Address, sync: &mut FreeListPageResourceSync) { let num_chunks = self.vm_map().get_contiguous_region_chunks(chunk); /* nail down all pages associated with the chunk, so it is no longer on our free list */ - let mut chunk_start = conversions::bytes_to_pages(chunk - self.start); + let mut chunk_start = conversions::bytes_to_pages(chunk - sync.start); let chunk_end = chunk_start + (num_chunks * PAGES_IN_CHUNK); while chunk_start < chunk_end { - self.inner_mut() - .free_list - .set_uncoalescable(chunk_start as _); - let tmp = self - .inner_mut() + sync.free_list.set_uncoalescable(chunk_start as _); + let tmp = sync .free_list .alloc_from_unit(PAGES_IN_CHUNK as _, chunk_start as _) as usize; // then alloc the entire chunk @@ -358,29 +318,26 @@ impl FreeListPageResource { sync.pages_currently_on_freelist -= PAGES_IN_CHUNK; } /* now return the address space associated with the chunk for global reuse */ - - self.inner_mut().common.release_discontiguous_chunks(chunk); + self.common.release_discontiguous_chunks(chunk); } pub fn release_pages(&self, first: Address) { debug_assert!(conversions::is_page_aligned(first)); - let page_offset = conversions::bytes_to_pages(first - self.start); - let pages = self.free_list.size(page_offset as _); + let mut sync = self.sync.lock().unwrap(); + let page_offset = conversions::bytes_to_pages(first - sync.start); + let pages = sync.free_list.size(page_offset as _); // if (VM.config.ZERO_PAGES_ON_RELEASE) // VM.memory.zero(false, first, Conversions.pagesToBytes(pages)); - debug_assert!(pages as usize <= self.inner().common.accounting.get_committed_pages()); + debug_assert!(pages as usize <= self.common.accounting.get_committed_pages()); if self.protect_memory_on_release { self.mprotect(first, pages as _); } - let mut sync = self.sync.lock().unwrap(); - // FIXME: We need a safe implementation - let me = unsafe { self.inner_mut() }; - self.inner().common.accounting.release(pages as _); - let freed = me.free_list.free(page_offset as _, true); + self.common.accounting.release(pages as _); + let freed = sync.free_list.free(page_offset as _, true); sync.pages_currently_on_freelist += pages as usize; - if !self.inner().common.contiguous { + if !self.common.contiguous { // only discontiguous spaces use chunks self.release_free_chunks(first, freed as _, &mut sync); } @@ -392,7 +349,7 @@ impl FreeListPageResource { pages_freed: usize, sync: &mut FreeListPageResourceSync, ) { - let page_offset = conversions::bytes_to_pages(freed_page - self.start); + let page_offset = conversions::bytes_to_pages(freed_page - sync.start); // may be multiple chunks if pages_freed % PAGES_IN_CHUNK == 0 { @@ -401,24 +358,19 @@ impl FreeListPageResource { let mut region_start = page_offset & !(PAGES_IN_CHUNK - 1); let mut next_region_start = region_start + PAGES_IN_CHUNK; /* now try to grow (end point pages are marked as non-coalescing) */ - while self.free_list.is_coalescable(region_start as _) { + while sync.free_list.is_coalescable(region_start as _) { // region_start is guaranteed to be positive. Otherwise this line will fail due to subtraction overflow. region_start -= PAGES_IN_CHUNK; } while next_region_start < freelist::MAX_UNITS as usize - && self.free_list.is_coalescable(next_region_start as _) + && sync.free_list.is_coalescable(next_region_start as _) { next_region_start += PAGES_IN_CHUNK; } debug_assert!(next_region_start < freelist::MAX_UNITS as usize); if pages_freed == next_region_start - region_start { - let start = self.start; - unsafe { - self.free_contiguous_chunk( - start + conversions::pages_to_bytes(region_start), - sync, - ); - } + let start = sync.start; + self.free_contiguous_chunk(start + conversions::pages_to_bytes(region_start), sync); } } } diff --git a/src/util/heap/layout/map.rs b/src/util/heap/layout/map.rs index 889800f59c..cad553cdbe 100644 --- a/src/util/heap/layout/map.rs +++ b/src/util/heap/layout/map.rs @@ -1,28 +1,33 @@ use crate::util::freelist::FreeList; -use crate::util::heap::freelistpageresource::CommonFreeListPageResource; use crate::util::heap::space_descriptor::SpaceDescriptor; +use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::util::Address; +/// The result of creating free lists +pub struct CreateFreeListResult { + // The created free list. + pub free_list: Box, + // Some free lists (notably `RawMemoryFreeList`) will occupy a portion of the space at the + // start of the space. `space_displacement` is the number of bytes (aligned up to chunk) that + // the free list occupies. The actual starting address of the space should be displaced by + // this amount. If the free list does not occupy address space of the `Space`, this field will + // be zero. + pub space_displacement: usize, +} + pub trait VMMap: Sync { fn insert(&self, start: Address, extent: usize, descriptor: SpaceDescriptor); /// Create a free-list for a discontiguous space. Must only be called at boot time. - /// bind_freelist() must be called by the caller after this method. - fn create_freelist(&self, start: Address) -> Box; + fn create_freelist(&self, start: Address) -> CreateFreeListResult; /// Create a free-list for a contiguous space. Must only be called at boot time. - /// bind_freelist() must be called by the caller after this method. - fn create_parent_freelist(&self, start: Address, units: usize, grain: i32) - -> Box; - - /// Bind a created freelist with the page resource. - /// This must called after create_freelist() or create_parent_freelist(). - /// - /// # Safety - /// - /// * `pr` must be a valid pointer to a CommonFreeListPageResource and be alive - /// for the duration of the VMMap. - unsafe fn bind_freelist(&self, pr: *const CommonFreeListPageResource); + fn create_parent_freelist( + &self, + start: Address, + units: usize, + grain: i32, + ) -> CreateFreeListResult; /// # Safety /// @@ -32,6 +37,7 @@ pub trait VMMap: Sync { descriptor: SpaceDescriptor, chunks: usize, head: Address, + maybe_raw_memory_freelist: Option<&mut RawMemoryFreeList>, ) -> Address; fn get_next_contiguous_region(&self, start: Address) -> Address; @@ -55,9 +61,12 @@ pub trait VMMap: Sync { /// Caller must ensure that only one thread is calling this method. unsafe fn free_contiguous_chunks(&self, start: Address) -> usize; - fn boot(&self) {} - - fn finalize_static_space_map(&self, from: Address, to: Address); + fn finalize_static_space_map( + &self, + from: Address, + to: Address, + update_starts: &mut dyn FnMut(Address), + ); fn is_finalized(&self) -> bool; diff --git a/src/util/heap/layout/map32.rs b/src/util/heap/layout/map32.rs index c4aa08f52f..61d1913a60 100644 --- a/src/util/heap/layout/map32.rs +++ b/src/util/heap/layout/map32.rs @@ -1,31 +1,28 @@ +use super::map::CreateFreeListResult; use super::map::VMMap; use crate::mmtk::SFT_MAP; use crate::util::conversions; use crate::util::freelist::FreeList; -use crate::util::heap::freelistpageresource::CommonFreeListPageResource; use crate::util::heap::layout::heap_parameters::*; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::space_descriptor::SpaceDescriptor; use crate::util::int_array_freelist::IntArrayFreeList; +use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::util::Address; -use std::cell::UnsafeCell; -use std::ptr::NonNull; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, MutexGuard}; pub struct Map32 { - sync: Mutex<()>, - inner: UnsafeCell, + sync: Mutex, } #[doc(hidden)] -pub struct Map32Inner { +struct Map32Sync { prev_link: Vec, next_link: Vec, region_map: IntArrayFreeList, global_page_map: IntArrayFreeList, shared_discontig_fl_count: usize, - shared_fl_map: Vec>>, total_available_discontiguous_chunks: usize, finalized: bool, descriptor_map: Vec, @@ -44,58 +41,38 @@ impl Map32 { pub fn new() -> Self { let max_chunks = vm_layout().max_chunks(); Map32 { - inner: UnsafeCell::new(Map32Inner { + sync: Mutex::new(Map32Sync { prev_link: vec![0; max_chunks], next_link: vec![0; max_chunks], region_map: IntArrayFreeList::new(max_chunks, max_chunks as _, 1), global_page_map: IntArrayFreeList::new(1, 1, MAX_SPACES), shared_discontig_fl_count: 0, - shared_fl_map: vec![None; MAX_SPACES], total_available_discontiguous_chunks: 0, finalized: false, descriptor_map: vec![SpaceDescriptor::UNINITIALIZED; max_chunks], cumulative_committed_pages: AtomicUsize::new(0), }), - sync: Mutex::new(()), } } } -impl std::ops::Deref for Map32 { - type Target = Map32Inner; - fn deref(&self) -> &Self::Target { - unsafe { &*self.inner.get() } - } -} - impl VMMap for Map32 { fn insert(&self, start: Address, extent: usize, descriptor: SpaceDescriptor) { - // Each space will call this on exclusive address ranges. It is fine to mutate the descriptor map, - // as each space will update different indices. - let self_mut: &mut Map32Inner = unsafe { self.mut_self() }; - let mut e = 0; - while e < extent { - let index = (start + e).chunk_index(); - assert!( - self.descriptor_map[index].is_empty(), - "Conflicting virtual address request" - ); - debug!( - "Set descriptor {:?} for Chunk {}", - descriptor, - conversions::chunk_index_to_address(index) - ); - self_mut.descriptor_map[index] = descriptor; - // VM.barriers.objectArrayStoreNoGCBarrier(spaceMap, index, space); - e += BYTES_IN_CHUNK; - } + let mut sync = self.sync(); + sync.insert_no_lock(start, extent, descriptor) } - fn create_freelist(&self, _start: Address) -> Box { - Box::new(IntArrayFreeList::from_parent( - &self.global_page_map, - self.get_discontig_freelist_pr_ordinal() as _, - )) + fn create_freelist(&self, _start: Address) -> CreateFreeListResult { + let mut sync = self.sync(); + let ordinal = sync.get_discontig_freelist_pr_ordinal() as i32; + let free_list = Box::new(IntArrayFreeList::from_parent( + &mut sync.global_page_map, + ordinal, + )); + CreateFreeListResult { + free_list, + space_displacement: 0, + } } fn create_parent_freelist( @@ -103,18 +80,12 @@ impl VMMap for Map32 { _start: Address, units: usize, grain: i32, - ) -> Box { - Box::new(IntArrayFreeList::new(units, grain, 1)) - } - - unsafe fn bind_freelist(&self, pr: *const CommonFreeListPageResource) { - let ordinal: usize = (*pr) - .free_list - .downcast_ref::() - .unwrap() - .get_ordinal() as usize; - let self_mut: &mut Map32Inner = self.mut_self(); - self_mut.shared_fl_map[ordinal] = Some(NonNull::new_unchecked(pr as *mut _)); + ) -> CreateFreeListResult { + let free_list = Box::new(IntArrayFreeList::new(units, grain, 1)); + CreateFreeListResult { + free_list, + space_displacement: 0, + } } unsafe fn allocate_contiguous_chunks( @@ -122,33 +93,35 @@ impl VMMap for Map32 { descriptor: SpaceDescriptor, chunks: usize, head: Address, + _maybe_raw_memory_freelist: Option<&mut RawMemoryFreeList>, ) -> Address { - let (_sync, self_mut) = self.mut_self_with_sync(); - let chunk = self_mut.region_map.alloc(chunks as _); + let mut sync = self.sync(); + let chunk = sync.region_map.alloc(chunks as _); debug_assert!(chunk != 0); if chunk == -1 { return Address::zero(); } - self_mut.total_available_discontiguous_chunks -= chunks; + sync.total_available_discontiguous_chunks -= chunks; let rtn = conversions::chunk_index_to_address(chunk as _); - self.insert(rtn, chunks << LOG_BYTES_IN_CHUNK, descriptor); + sync.insert_no_lock(rtn, chunks << LOG_BYTES_IN_CHUNK, descriptor); if head.is_zero() { - debug_assert!(self.next_link[chunk as usize] == 0); + debug_assert!(sync.next_link[chunk as usize] == 0); } else { - self_mut.next_link[chunk as usize] = head.chunk_index() as _; - self_mut.prev_link[head.chunk_index()] = chunk; + sync.next_link[chunk as usize] = head.chunk_index() as _; + sync.prev_link[head.chunk_index()] = chunk; } - debug_assert!(self.prev_link[chunk as usize] == 0); + debug_assert!(sync.prev_link[chunk as usize] == 0); rtn } fn get_next_contiguous_region(&self, start: Address) -> Address { debug_assert!(start == conversions::chunk_align_down(start)); let chunk = start.chunk_index(); - if chunk == 0 || self.next_link[chunk] == 0 { + let sync = self.sync(); + if chunk == 0 || sync.next_link[chunk] == 0 { unsafe { Address::zero() } } else { - let a = self.next_link[chunk]; + let a = sync.next_link[chunk]; conversions::chunk_index_to_address(a as _) } } @@ -156,7 +129,8 @@ impl VMMap for Map32 { fn get_contiguous_region_chunks(&self, start: Address) -> usize { debug_assert!(start == conversions::chunk_align_down(start)); let chunk = start.chunk_index(); - self.region_map.size(chunk as i32) as _ + let sync = self.sync(); + sync.region_map.size(chunk as i32) as _ } fn get_contiguous_region_size(&self, start: Address) -> usize { @@ -164,43 +138,49 @@ impl VMMap for Map32 { } fn get_available_discontiguous_chunks(&self) -> usize { - self.total_available_discontiguous_chunks + let sync = self.sync(); + sync.total_available_discontiguous_chunks } fn get_chunk_consumer_count(&self) -> usize { - self.shared_discontig_fl_count + let sync = self.sync(); + sync.shared_discontig_fl_count } + #[allow(clippy::while_immutable_condition)] fn free_all_chunks(&self, any_chunk: Address) { debug!("free_all_chunks: {}", any_chunk); - let (_sync, self_mut) = self.mut_self_with_sync(); + let mut sync = self.sync(); debug_assert!(any_chunk == conversions::chunk_align_down(any_chunk)); if !any_chunk.is_zero() { let chunk = any_chunk.chunk_index(); - while self_mut.next_link[chunk] != 0 { - let x = self_mut.next_link[chunk]; - self.free_contiguous_chunks_no_lock(x); + while sync.next_link[chunk] != 0 { + let x = sync.next_link[chunk]; + sync.free_contiguous_chunks_no_lock(x); } - while self_mut.prev_link[chunk] != 0 { - let x = self_mut.prev_link[chunk]; - self.free_contiguous_chunks_no_lock(x); + while sync.prev_link[chunk] != 0 { + let x = sync.prev_link[chunk]; + sync.free_contiguous_chunks_no_lock(x); } - self.free_contiguous_chunks_no_lock(chunk as _); + sync.free_contiguous_chunks_no_lock(chunk as _); } } unsafe fn free_contiguous_chunks(&self, start: Address) -> usize { debug!("free_contiguous_chunks: {}", start); - let (_sync, _) = self.mut_self_with_sync(); debug_assert!(start == conversions::chunk_align_down(start)); let chunk = start.chunk_index(); - self.free_contiguous_chunks_no_lock(chunk as _) + let mut sync = self.sync(); + sync.free_contiguous_chunks_no_lock(chunk as _) } - fn finalize_static_space_map(&self, from: Address, to: Address) { - // This is only called during boot process by a single thread. - // It is fine to get a mutable reference. - let self_mut: &mut Map32Inner = unsafe { self.mut_self() }; + fn finalize_static_space_map( + &self, + from: Address, + to: Address, + update_starts: &mut dyn FnMut(Address), + ) { + let mut sync = self.sync(); /* establish bounds of discontiguous space */ let start_address = from; let first_chunk = start_address.chunk_index(); @@ -210,18 +190,10 @@ impl VMMap for Map32 { let pages = (1 + last_chunk - first_chunk) * PAGES_IN_CHUNK; // start_address=0xb0000000, first_chunk=704, last_chunk=703, unavail_start_chunk=704, trailing_chunks=320, pages=0 // startAddress=0x68000000 firstChunk=416 lastChunk=703 unavailStartChunk=704 trailingChunks=320 pages=294912 - self_mut.global_page_map.resize_freelist(pages, pages as _); - // TODO: Clippy favors using iter().flatten() rather than iter() with if-let. - // https://rust-lang.github.io/rust-clippy/master/index.html#manual_flatten - // Yi: I am not doing this refactoring right now, as I am not familiar with flatten() and - // there is no test to ensure the refactoring will be correct. - #[allow(clippy::manual_flatten)] - for fl in self_mut.shared_fl_map.iter().copied() { - if let Some(mut fl) = fl { - let fl_mut = unsafe { fl.as_mut() }; - fl_mut.resize_freelist(start_address); - } - } + sync.global_page_map.resize_freelist(pages, pages as _); + + update_starts(start_address); + // [ // 2: -1073741825 // 3: -1073741825 @@ -232,11 +204,11 @@ impl VMMap for Map32 { // 2051: 1024 // ] /* set up the region map free list */ - self_mut.region_map.alloc(first_chunk as _); // block out entire bottom of address range + sync.region_map.alloc(first_chunk as _); // block out entire bottom of address range for _ in first_chunk..=last_chunk { - self_mut.region_map.alloc(1); + sync.region_map.alloc(1); } - let alloced_chunk = self_mut.region_map.alloc(trailing_chunks as _); + let alloced_chunk = sync.region_map.alloc(trailing_chunks as _); debug_assert!( alloced_chunk == unavail_start_chunk as i32, "{} != {}", @@ -246,76 +218,86 @@ impl VMMap for Map32 { /* set up the global page map and place chunks on free list */ let mut first_page = 0; for chunk_index in first_chunk..=last_chunk { - self_mut.total_available_discontiguous_chunks += 1; - self_mut.region_map.free(chunk_index as _, false); // put this chunk on the free list - self_mut.global_page_map.set_uncoalescable(first_page); - let alloced_pages = self_mut.global_page_map.alloc(PAGES_IN_CHUNK as _); // populate the global page map + sync.total_available_discontiguous_chunks += 1; + sync.region_map.free(chunk_index as _, false); // put this chunk on the free list + sync.global_page_map.set_uncoalescable(first_page); + let alloced_pages = sync.global_page_map.alloc(PAGES_IN_CHUNK as _); // populate the global page map debug_assert!(alloced_pages == first_page); first_page += PAGES_IN_CHUNK as i32; } - self_mut.finalized = true; + sync.finalized = true; } fn is_finalized(&self) -> bool { - self.finalized + let sync = self.sync(); + sync.finalized } fn get_descriptor_for_address(&self, address: Address) -> SpaceDescriptor { let index = address.chunk_index(); - self.descriptor_map[index] + let sync = self.sync(); + sync.descriptor_map[index] } fn add_to_cumulative_committed_pages(&self, pages: usize) { - self.cumulative_committed_pages + let sync = self.sync(); + sync.cumulative_committed_pages .fetch_add(pages, Ordering::Relaxed); } } impl Map32 { - /// # Safety - /// - /// The caller needs to guarantee there is no race condition. Either only one single thread - /// is using this method, or multiple threads are accessing mutally exclusive data (e.g. different indices in arrays). - /// In other cases, use mut_self_with_sync(). - #[allow(clippy::mut_from_ref)] - unsafe fn mut_self(&self) -> &mut Map32Inner { - &mut *self.inner.get() + fn sync(&self) -> MutexGuard { + self.sync.lock().unwrap() } +} - fn mut_self_with_sync(&self) -> (MutexGuard<()>, &mut Map32Inner) { - let guard = self.sync.lock().unwrap(); - (guard, unsafe { self.mut_self() }) +impl Map32Sync { + fn insert_no_lock(&mut self, start: Address, extent: usize, descriptor: SpaceDescriptor) { + let mut e = 0; + while e < extent { + let index = (start + e).chunk_index(); + assert!( + self.descriptor_map[index].is_empty(), + "Conflicting virtual address request" + ); + debug!( + "Set descriptor {:?} for Chunk {}", + descriptor, + conversions::chunk_index_to_address(index) + ); + self.descriptor_map[index] = descriptor; + // VM.barriers.objectArrayStoreNoGCBarrier(spaceMap, index, space); + e += BYTES_IN_CHUNK; + } } - - fn free_contiguous_chunks_no_lock(&self, chunk: i32) -> usize { + fn free_contiguous_chunks_no_lock(&mut self, chunk: i32) -> usize { unsafe { - let chunks = self.mut_self().region_map.free(chunk, false); - self.mut_self().total_available_discontiguous_chunks += chunks as usize; + let chunks = self.region_map.free(chunk, false); + self.total_available_discontiguous_chunks += chunks as usize; let next = self.next_link[chunk as usize]; let prev = self.prev_link[chunk as usize]; if next != 0 { - self.mut_self().prev_link[next as usize] = prev + self.prev_link[next as usize] = prev }; if prev != 0 { - self.mut_self().next_link[prev as usize] = next + self.next_link[prev as usize] = next }; - self.mut_self().prev_link[chunk as usize] = 0; - self.mut_self().next_link[chunk as usize] = 0; + self.prev_link[chunk as usize] = 0; + self.next_link[chunk as usize] = 0; for offset in 0..chunks { let index = (chunk + offset) as usize; let chunk_start = conversions::chunk_index_to_address(index); debug!("Clear descriptor for Chunk {}", chunk_start); - self.mut_self().descriptor_map[index] = SpaceDescriptor::UNINITIALIZED; + self.descriptor_map[index] = SpaceDescriptor::UNINITIALIZED; SFT_MAP.clear(chunk_start); } chunks as _ } } - fn get_discontig_freelist_pr_ordinal(&self) -> usize { - // This is only called during creating a page resource/space/plan/mmtk instance, which is single threaded. - let self_mut: &mut Map32Inner = unsafe { self.mut_self() }; - self_mut.shared_discontig_fl_count += 1; + fn get_discontig_freelist_pr_ordinal(&mut self) -> usize { + self.shared_discontig_fl_count += 1; self.shared_discontig_fl_count } } diff --git a/src/util/heap/layout/map64.rs b/src/util/heap/layout/map64.rs index ec4383d010..50a251372d 100644 --- a/src/util/heap/layout/map64.rs +++ b/src/util/heap/layout/map64.rs @@ -1,8 +1,8 @@ +use super::map::CreateFreeListResult; use super::map::VMMap; use crate::util::constants::*; use crate::util::conversions; use crate::util::freelist::FreeList; -use crate::util::heap::freelistpageresource::CommonFreeListPageResource; use crate::util::heap::layout::heap_parameters::*; use crate::util::heap::layout::vm_layout::*; use crate::util::heap::space_descriptor::SpaceDescriptor; @@ -11,7 +11,6 @@ use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::util::rust_util::zeroed_alloc::new_zeroed_vec; use crate::util::Address; use std::cell::UnsafeCell; -use std::ptr::NonNull; use std::sync::atomic::{AtomicUsize, Ordering}; const NON_MAP_FRACTION: f64 = 1.0 - 8.0 / 4096.0; @@ -21,8 +20,6 @@ pub struct Map64 { } struct Map64Inner { - fl_page_resources: Vec>>, - fl_map: Vec>>, finalized: bool, descriptor_map: Vec, base_address: Vec
, @@ -61,8 +58,6 @@ impl Map64 { }, high_water, base_address, - fl_page_resources: vec![None; MAX_SPACES], - fl_map: vec![None; MAX_SPACES], finalized: false, cumulative_committed_pages: AtomicUsize::new(0), }), @@ -81,7 +76,7 @@ impl VMMap for Map64 { self_mut.descriptor_map[index] = descriptor; } - fn create_freelist(&self, start: Address) -> Box { + fn create_freelist(&self, start: Address) -> CreateFreeListResult { let units = vm_layout().space_size_64() >> LOG_BYTES_IN_PAGE; self.create_parent_freelist(start, units, units as _) } @@ -91,7 +86,9 @@ impl VMMap for Map64 { start: Address, mut units: usize, grain: i32, - ) -> Box { + ) -> CreateFreeListResult { + debug_assert!(start.is_aligned_to(BYTES_IN_CHUNK)); + // This is only called during creating a page resource/space/plan/mmtk instance, which is single threaded. let self_mut = unsafe { self.mut_self() }; let index = Self::space_index(start).unwrap(); @@ -102,7 +99,7 @@ impl VMMap for Map64 { let heads = 1; let pages_per_block = RawMemoryFreeList::default_block_size(units as _, heads); - let mut list = Box::new(RawMemoryFreeList::new( + let list = Box::new(RawMemoryFreeList::new( start, start + list_extent, pages_per_block, @@ -112,22 +109,17 @@ impl VMMap for Map64 { MmapStrategy::Normal, )); - /*self_mut.fl_map[index] = - Some(unsafe { &*(&list as &RawMemoryFreeList as *const RawMemoryFreeList) }); - */ - self_mut.fl_map[index] = unsafe { Some(NonNull::new_unchecked(&mut *list)) }; /* Adjust the base address and highwater to account for the allocated chunks for the map */ let base = conversions::chunk_align_up(start + list_extent); self_mut.high_water[index] = base; self_mut.base_address[index] = base; - list - } - unsafe fn bind_freelist(&self, pr: *const CommonFreeListPageResource) { - let index = Self::space_index((*pr).get_start()).unwrap(); - let self_mut = self.mut_self(); - self_mut.fl_page_resources[index] = Some(NonNull::new_unchecked(pr as _)); + let space_displacement = base - start; + CreateFreeListResult { + free_list: list, + space_displacement, + } } /// # Safety @@ -138,6 +130,7 @@ impl VMMap for Map64 { descriptor: SpaceDescriptor, chunks: usize, _head: Address, + maybe_raw_memory_freelist: Option<&mut RawMemoryFreeList>, ) -> Address { debug_assert!(Self::space_index(descriptor.get_start()).unwrap() == descriptor.get_index()); // Each space will call this on exclusive address ranges. It is fine to mutate the descriptor map, @@ -150,9 +143,9 @@ impl VMMap for Map64 { self_mut.high_water[index] = rtn + extent; /* Grow the free list to accommodate the new chunks */ - let free_list = self.inner().fl_map[Self::space_index(descriptor.get_start()).unwrap()]; - if let Some(mut free_list) = free_list { - let free_list = free_list.as_mut(); + // TODO: Maybe the following should be moved to Space or PageResource. + // Map64 currently knows too much about spaces! + if let Some(free_list) = maybe_raw_memory_freelist { free_list.grow_freelist(conversions::bytes_to_pages(extent) as _); let base_page = conversions::bytes_to_pages(rtn - self.inner().base_address[index]); for offset in (0..(chunks * PAGES_IN_CHUNK)).step_by(PAGES_IN_CHUNK) { @@ -192,31 +185,12 @@ impl VMMap for Map64 { unreachable!() } - fn boot(&self) { - // This is only called during boot process by a single thread. - // It is fine to get a mutable reference. - let self_mut: &mut Map64Inner = unsafe { self.mut_self() }; - for pr in 0..MAX_SPACES { - if let Some(mut fl) = self_mut.fl_map[pr] { - let fl_mut: &mut RawMemoryFreeList = unsafe { fl.as_mut() }; - fl_mut.grow_freelist(0); - } - } - } - - fn finalize_static_space_map(&self, _from: Address, _to: Address) { - // This is only called during boot process by a single thread. - // It is fine to get a mutable reference. - let self_mut: &mut Map64Inner = unsafe { self.mut_self() }; - for pr in 0..MAX_SPACES { - if let Some(mut fl) = self_mut.fl_page_resources[pr] { - let fl_mut = unsafe { fl.as_mut() }; - fl_mut.resize_freelist(conversions::chunk_align_up(unsafe { - self.inner().fl_map[pr].unwrap().as_ref().get_limit() - })); - } - } - self_mut.finalized = true; + fn finalize_static_space_map( + &self, + _from: Address, + _to: Address, + _update_starts: &mut dyn FnMut(Address), + ) { } fn is_finalized(&self) -> bool { diff --git a/src/util/heap/layout/mod.rs b/src/util/heap/layout/mod.rs index 41ce58ee94..c4eea74582 100644 --- a/src/util/heap/layout/mod.rs +++ b/src/util/heap/layout/mod.rs @@ -8,6 +8,7 @@ mod byte_map_mmapper; mod fragmented_mapper; mod map; +pub(crate) use self::map::CreateFreeListResult; pub use self::map::VMMap; use self::vm_layout::vm_layout; mod map32; diff --git a/src/util/heap/monotonepageresource.rs b/src/util/heap/monotonepageresource.rs index a0c27a6a80..fd34f04af8 100644 --- a/src/util/heap/monotonepageresource.rs +++ b/src/util/heap/monotonepageresource.rs @@ -113,9 +113,9 @@ impl PageResource for MonotonePageResource { if !self.common().contiguous && tmp > sync.sentinel { /* we're out of virtual memory within our discontiguous region, so ask for more */ let required_chunks = required_chunks(required_pages); - sync.current_chunk = self - .common - .grow_discontiguous_space(space_descriptor, required_chunks); // Returns zero on failure + sync.current_chunk = + self.common + .grow_discontiguous_space(space_descriptor, required_chunks, None); // Returns zero on failure sync.cursor = sync.current_chunk; sync.sentinel = sync.cursor + if sync.current_chunk.is_zero() { diff --git a/src/util/heap/pageresource.rs b/src/util/heap/pageresource.rs index 9154620e0d..9da33410c1 100644 --- a/src/util/heap/pageresource.rs +++ b/src/util/heap/pageresource.rs @@ -1,6 +1,7 @@ use crate::util::address::Address; use crate::util::conversions; use crate::util::opaque_pointer::*; +use crate::util::raw_memory_freelist::RawMemoryFreeList; use crate::vm::ActivePlan; use std::sync::Mutex; @@ -106,6 +107,13 @@ pub trait PageResource: 'static { fn vm_map(&self) -> &'static dyn VMMap { self.common().vm_map } + + // Some page resources need to record the start address. + // This method will be called after the start address of the discontigous region is determined. + // `start` is the computed start address. By default, this does nothing. + fn update_discontiguous_start(&mut self, _start: Address) { + // Do nothing. + } } pub struct PRAllocResult { @@ -145,6 +153,7 @@ impl CommonPageResource { &self, space_descriptor: SpaceDescriptor, chunks: usize, + maybe_raw_memory_freelist: Option<&mut RawMemoryFreeList>, ) -> Address { let mut head_discontiguous_region = self.head_discontiguous_region.lock().unwrap(); @@ -153,6 +162,7 @@ impl CommonPageResource { space_descriptor, chunks, *head_discontiguous_region, + maybe_raw_memory_freelist, ) }; if new_head.is_zero() { diff --git a/src/util/int_array_freelist.rs b/src/util/int_array_freelist.rs index e8a6e5b7cf..9d6fbe81fc 100644 --- a/src/util/int_array_freelist.rs +++ b/src/util/int_array_freelist.rs @@ -41,7 +41,7 @@ impl IntArrayFreeList { iafl.initialize_heap(units as _, grain); iafl } - pub fn from_parent(parent: &IntArrayFreeList, ordinal: i32) -> Self { + pub fn from_parent(parent: &mut IntArrayFreeList, ordinal: i32) -> Self { let iafl = IntArrayFreeList { head: -(1 + ordinal), heads: parent.heads, @@ -356,9 +356,9 @@ mod tests { #[test] fn multi_heads_alloc_free() { - let parent = IntArrayFreeList::new(LIST_SIZE, 1, 2); - let mut child1 = IntArrayFreeList::from_parent(&parent, 0); - let child2 = IntArrayFreeList::from_parent(&parent, 1); + let mut parent = IntArrayFreeList::new(LIST_SIZE, 1, 2); + let mut child1 = IntArrayFreeList::from_parent(&mut parent, 0); + let child2 = IntArrayFreeList::from_parent(&mut parent, 1); // child1 alloc let res = child1.alloc(1); @@ -377,9 +377,9 @@ mod tests { #[test] #[should_panic] fn multi_heads_exceed_heads() { - let parent = IntArrayFreeList::new(LIST_SIZE, 1, 2); - let _child1 = IntArrayFreeList::from_parent(&parent, 0); - let _child2 = IntArrayFreeList::from_parent(&parent, 1); - let _child3 = IntArrayFreeList::from_parent(&parent, 2); + let mut parent = IntArrayFreeList::new(LIST_SIZE, 1, 2); + let _child1 = IntArrayFreeList::from_parent(&mut parent, 0); + let _child2 = IntArrayFreeList::from_parent(&mut parent, 1); + let _child3 = IntArrayFreeList::from_parent(&mut parent, 2); } } diff --git a/src/util/raw_memory_freelist.rs b/src/util/raw_memory_freelist.rs index 3c87148d6a..8c0819a9e5 100644 --- a/src/util/raw_memory_freelist.rs +++ b/src/util/raw_memory_freelist.rs @@ -35,8 +35,16 @@ impl FreeList for RawMemoryFreeList { self.heads } fn get_entry(&self, index: i32) -> i32 { + debug!("get_entry({index})"); let offset = (index << LOG_BYTES_IN_ENTRY) as usize; - debug_assert!(self.base + offset >= self.base && self.base + offset < self.high_water); + debug_assert!( + self.base + offset >= self.base && self.base + offset < self.high_water, + "Base: {}, offset: {}, sum: {}, high_water: {}", + self.base, + offset, + self.base + offset, + self.high_water + ); unsafe { (self.base + offset).load() } } fn set_entry(&mut self, index: i32, value: i32) { @@ -49,8 +57,16 @@ impl FreeList for RawMemoryFreeList { self.base + offset, self.high_water ); + debug!( + "Storing 0x{:x} at {} + 0x{:x} = {}", + value, + self.base, + offset, + self.base + offset + ); unsafe { (self.base + offset).store(value) } } + fn alloc(&mut self, size: i32) -> i32 { if self.current_units == 0 { return FAILURE; @@ -96,6 +112,7 @@ impl RawMemoryFreeList { heads: i32, strategy: MmapStrategy, ) -> Self { + dbg!(base, limit, pages_per_block, units, grain, heads, strategy); debug_assert!(units <= MAX_UNITS && heads <= MAX_HEADS); debug_assert!( base + conversions::pages_to_bytes(Self::size_in_pages(units, heads) as _) <= limit