Internal pointer support

benches/main.rs

@@ -27,6 +27,7 @@ pub fn bench_main(_c: &mut Criterion) {
     match std::env::var("MMTK_BENCH") {
         Ok(bench) => match bench.as_str() {
             "alloc" => mock_bench::alloc::bench(_c),
+            "internal_pointer" => mock_bench::internal_pointer::bench(_c),
             "sft" => mock_bench::sft::bench(_c),
             _ => panic!("Unknown benchmark {:?}", bench),
         },

benches/mock_bench/internal_pointer.rs

@@ -0,0 +1,97 @@
+use criterion::Criterion;
+
+#[cfg(feature = "is_mmtk_object")]
+use mmtk::util::test_util::fixtures::*;
+use mmtk::util::test_util::mock_method::*;
+use mmtk::util::test_util::mock_vm::{write_mockvm, MockVM};
+
+pub fn bench(c: &mut Criterion) {
+    // Setting a larger heap, although the GC should be disabled in the MockVM
+    #[cfg(feature = "is_mmtk_object")]
+    let mut fixture = MutatorFixture::create_with_heapsize(1 << 30);
+
+    // Normal objects
+    // 16KB object -- we want to make sure the object can fit into any normal space (e.g. immix space or mark sweep space)
+    const NORMAL_OBJECT_SIZE: usize = 16 * 1024;
+    write_mockvm(|mock| {
+        *mock = MockVM {
+            get_object_size: MockMethod::new_fixed(Box::new(|_| NORMAL_OBJECT_SIZE)),
+            is_collection_enabled: MockMethod::new_fixed(Box::new(|_| false)),
+            ..MockVM::default()
+        }
+    });
+
+    c.bench_function("internal pointer - normal objects", |_b| {
+        #[cfg(feature = "is_mmtk_object")]
+        {
+            use mmtk::memory_manager;
+            use mmtk::vm::ObjectModel;
+            use mmtk::AllocationSemantics;
+            let addr = memory_manager::alloc(
+                &mut fixture.mutator,
+                NORMAL_OBJECT_SIZE,
+                8,
+                0,
+                AllocationSemantics::Default,
+            );
+            let obj_ref = MockVM::address_to_ref(addr);
+            memory_manager::post_alloc(
+                &mut fixture.mutator,
+                obj_ref,
+                NORMAL_OBJECT_SIZE,
+                AllocationSemantics::Default,
+            );
+            let obj_end = addr + NORMAL_OBJECT_SIZE;
+            _b.iter(|| {
+                memory_manager::find_object_from_internal_pointer::<MockVM>(
+                    obj_end - 1,
+                    NORMAL_OBJECT_SIZE,
+                );
+            })
+        }
+        #[cfg(not(feature = "is_mmtk_object"))]
+        panic!("The benchmark requires is_mmtk_object feature to run");
+    });
+
+    // Large objects
+    // 16KB object
+    const LARGE_OBJECT_SIZE: usize = 16 * 1024;
+    write_mockvm(|mock| {
+        *mock = MockVM {
+            get_object_size: MockMethod::new_fixed(Box::new(|_| LARGE_OBJECT_SIZE)),
+            is_collection_enabled: MockMethod::new_fixed(Box::new(|_| false)),
+            ..MockVM::default()
+        }
+    });
+    c.bench_function("internal pointer - large objects", |_b| {
+        #[cfg(feature = "is_mmtk_object")]
+        {
+            use mmtk::memory_manager;
+            use mmtk::vm::ObjectModel;
+            use mmtk::AllocationSemantics;
+            let addr = memory_manager::alloc(
+                &mut fixture.mutator,
+                LARGE_OBJECT_SIZE,
+                8,
+                0,
+                AllocationSemantics::Los,
+            );
+            let obj_ref = MockVM::address_to_ref(addr);
+            memory_manager::post_alloc(
+                &mut fixture.mutator,
+                obj_ref,
+                LARGE_OBJECT_SIZE,
+                AllocationSemantics::Los,
+            );
+            let obj_end = addr + LARGE_OBJECT_SIZE;
+            _b.iter(|| {
+                memory_manager::find_object_from_internal_pointer::<MockVM>(
+                    obj_end - 1,
+                    LARGE_OBJECT_SIZE,
+                );
+            })
+        }
+        #[cfg(not(feature = "is_mmtk_object"))]
+        panic!("The benchmark requires is_mmtk_object feature to run");
+    });
+}

benches/mock_bench/mod.rs

@@ -1,2 +1,3 @@
 pub mod alloc;
+pub mod internal_pointer;
 pub mod sft;

src/memory_manager.rs

@@ -630,6 +630,50 @@ pub fn is_mmtk_object(addr: Address) -> bool {
     SFT_MAP.get_checked(addr).is_mmtk_object(addr)
 }
 
+/// Find if there is an object with VO bit set for the given address range.
+/// This should be used instead of [`crate::memory_manager::is_mmtk_object`] for conservative stack scanning if
+/// the binding may have internal pointers on the stack.
+///
+/// The function cannot directly return an object reference. Instead, it returns
+/// an address range and guarantees the object ference is in the range.
+/// The reason behind this is that we use the VO
+/// bit to represent a valid object for every 8 bytes ([`crate::util::is_mmtk_object::VO_BIT_REGION_SIZE`]).
+/// We use VO bits to find the object for an internal pointer.
+/// When we find a set VO bit, we only know that in the 8 bytes there is an object, and we cannot know where
+/// exactly the object is. The binding needs to use their knowledge about the alignment
+/// and offset for object references to find out the object reference from the return value.
+/// See Case 2 in [`crate::memory_manager::is_mmtk_object`] for more explanation.
+///
+/// Note that, we only consider pointers that points to an address that is equal or after the in-object addresss
+/// (i.e. the address returned from [`crate::vm::ObjectModel::ref_to_address`]), and within the allocation
+/// as 'internal pointers'. To be precise, for each object ref `obj_ref`, internal pointers are in the range
+/// `[ObjectModel::ref_to_address(obj_ref), ObjectModel::ref_to_object_start(obj_ref) + ObjectModel::get_current_size(obj_ref))`.
+/// If a binding defines internal pointers differently, calling this method is undefined behavior.
+/// If this is the case for you, please submit an issue or engage us on Zulip to discuss more.
+///
+/// Note that, in the similar situation as [`crate::memory_manager::is_mmtk_object`], the binding should filter
+/// out obvious non-pointers (e.g. alignment check, bound check, etc) before calling this function to avoid unnecessary
+/// cost. This method is not cheap.
+///
+/// To minimize the cost, the user should also use a small `max_search_bytes`.
+///
+/// Argument:
+/// * `internal_ptr`: The internal pointer to start. We search backwards from this internal pointer to find the base reference.
+/// * `max_search_bytes`: The maximum number of bytes we may search for an object with VO bit set.
+#[cfg(feature = "is_mmtk_object")]
+pub fn find_object_from_internal_pointer<VM: VMBinding>(
+    internal_ptr: Address,
+    max_search_bytes: usize,
+) -> Option<ObjectReference> {
+    use crate::mmtk::SFT_MAP;
+    let ret = SFT_MAP
+        .get_checked(internal_ptr)
+        .find_object_from_internal_pointer(internal_ptr, max_search_bytes);
+    #[cfg(debug_assertions)]
+    ret.inspect(|obj| debug_assert!(is_mmtk_object(obj.to_raw_address())));
+    ret
+}
+
 /// Return true if the `object` lies in a region of memory where
 /// -   only MMTk can allocate into, or
 /// -   only MMTk's delegated memory allocator (such as a malloc implementation) can allocate into

src/policy/copyspace.rs

@@ -78,6 +78,18 @@ impl<VM: VMBinding> SFT for CopySpace<VM> {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
 
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
+
     fn sft_trace_object(
         &self,
         queue: &mut VectorObjectQueue,

src/policy/immix/immixspace.rs

@@ -144,6 +144,16 @@ impl<VM: VMBinding> SFT for ImmixSpace<VM> {
     fn is_mmtk_object(&self, addr: Address) -> bool {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        // We don't need to search more than the max object size in the immix space.
+        let search_bytes = usize::min(super::MAX_IMMIX_OBJECT_SIZE, max_search_bytes);
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(ptr, search_bytes)
+    }
     fn sft_trace_object(
         &self,
         _queue: &mut VectorObjectQueue,

src/policy/immortalspace.rs

@@ -67,6 +67,17 @@ impl<VM: VMBinding> SFT for ImmortalSpace<VM> {
     fn is_mmtk_object(&self, addr: Address) -> bool {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
     fn sft_trace_object(
         &self,
         queue: &mut VectorObjectQueue,

src/policy/largeobjectspace.rs

@@ -86,6 +86,45 @@ impl<VM: VMBinding> SFT for LargeObjectSpace<VM> {
     fn is_mmtk_object(&self, addr: Address) -> bool {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        use crate::util::metadata::vo_bit;
+        // For large object space, it is a bit special. We only need to check VO bit for each page.
+        let mut cur_page = ptr.align_down(BYTES_IN_PAGE);
+        let low_page = ptr
+            .saturating_sub(max_search_bytes)
+            .align_down(BYTES_IN_PAGE);
+        while cur_page > low_page {
+            // If the page start is not mapped, there can't be an object in it.
+            if !cur_page.is_mapped() {
+                return None;
+            }
+            if vo_bit::get_raw_vo_bit_word(cur_page) != 0 {
+                // Find the exact address that has vo bit set
+                for offset in 0..vo_bit::VO_BIT_WORD_TO_REGION {
+                    let addr = cur_page + offset;
+                    if unsafe { vo_bit::is_vo_addr(addr) } {
+                        let obj = vo_bit::is_internal_ptr_from_vo_bit::<VM>(addr, ptr);
+                        if obj.is_some() {
+                            return obj;
+                        } else {
+                            return None;
+                        }
+                    }
+                }
+                unreachable!(
+                    "We found vo bit in the raw word, but we cannot find the exact address"
+                );
+            }
+
+            cur_page -= BYTES_IN_PAGE;
+        }
+        None
+    }
     fn sft_trace_object(
         &self,
         queue: &mut VectorObjectQueue,

src/policy/lockfreeimmortalspace.rs

@@ -77,6 +77,17 @@ impl<VM: VMBinding> SFT for LockFreeImmortalSpace<VM> {
     fn is_mmtk_object(&self, addr: Address) -> bool {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
     fn sft_trace_object(
         &self,
         _queue: &mut VectorObjectQueue,

src/policy/markcompactspace.rs

@@ -79,6 +79,18 @@ impl<VM: VMBinding> SFT for MarkCompactSpace<VM> {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
 
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
+
     fn sft_trace_object(
         &self,
         _queue: &mut VectorObjectQueue,

src/policy/marksweepspace/malloc_ms/global.rs

@@ -109,6 +109,18 @@ impl<VM: VMBinding> SFT for MallocSpace<VM> {
         has_object_alloced_by_malloc::<VM>(addr).is_some()
     }
 
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
+
     fn initialize_object_metadata(&self, object: ObjectReference, _alloc: bool) {
         trace!("initialize_object_metadata for object {}", object);
         set_vo_bit::<VM>(object);

src/policy/marksweepspace/native_ms/global.rs

@@ -188,6 +188,17 @@ impl<VM: VMBinding> SFT for MarkSweepSpace<VM> {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
 
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        // We don't need to search more than the max object size in the mark sweep space.
+        let search_bytes = usize::min(MAX_OBJECT_SIZE, max_search_bytes);
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(ptr, search_bytes)
+    }
+
     fn sft_trace_object(
         &self,
         queue: &mut VectorObjectQueue,

src/policy/sft.rs

@@ -78,6 +78,13 @@ pub trait SFT {
     #[cfg(feature = "is_mmtk_object")]
     fn is_mmtk_object(&self, addr: Address) -> bool;
 
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference>;
+
     /// Initialize object metadata (in the header, or in the side metadata).
     fn initialize_object_metadata(&self, object: ObjectReference, alloc: bool);
 
@@ -157,6 +164,14 @@ impl SFT for EmptySpaceSFT {
     fn is_mmtk_object(&self, _addr: Address) -> bool {
         false
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        _ptr: Address,
+        _max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        None
+    }
 
     fn initialize_object_metadata(&self, object: ObjectReference, _alloc: bool) {
         panic!(

src/policy/vmspace.rs

@@ -67,6 +67,17 @@ impl<VM: VMBinding> SFT for VMSpace<VM> {
     fn is_mmtk_object(&self, addr: Address) -> bool {
         crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
     }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
+    }
     fn sft_trace_object(
         &self,
         queue: &mut VectorObjectQueue,