Internal pointer support (#1165)

This PR adds internal pointer support. It supersedes
#1155 which provides a simple but
inefficient implementation for internal pointers. This PR is based on
#1159 which adds requirements for
object reference alignment.

This PR
* adds `memory_manager::find_object_from_internal_pointer`
  * The call is dispatched using SFT to each space.
* Large object space only checks the first word in VO bit for every
page.
* Mark sweep and immix space only searches for the max object size for
those spaces.
* Allow iterating side metadata bits.
* Allow loading raw byte/word in side metadata.

benches/main.rs

@@ -27,6 +27,7 @@ pub fn bench_main(_c: &mut Criterion) {
     match std::env::var("MMTK_BENCH") {
         Ok(bench) => match bench.as_str() {
             "alloc" => mock_bench::alloc::bench(_c),
+            "internal_pointer" => mock_bench::internal_pointer::bench(_c),
             "sft" => mock_bench::sft::bench(_c),
             _ => panic!("Unknown benchmark {:?}", bench),
         },

benches/mock_bench/internal_pointer.rs

@@ -0,0 +1,95 @@
+use criterion::Criterion;
+
+#[cfg(feature = "is_mmtk_object")]
+use mmtk::util::test_util::fixtures::*;
+use mmtk::util::test_util::mock_method::*;
+use mmtk::util::test_util::mock_vm::{write_mockvm, MockVM};
+
+pub fn bench(c: &mut Criterion) {
+    // Setting a larger heap, although the GC should be disabled in the MockVM
+    #[cfg(feature = "is_mmtk_object")]
+    let mut fixture = MutatorFixture::create_with_heapsize(1 << 30);
+
+    // Normal objects
+    // 16KB object -- we want to make sure the object can fit into any normal space (e.g. immix space or mark sweep space)
+    const NORMAL_OBJECT_SIZE: usize = 16 * 1024;
+    write_mockvm(|mock| {
+        *mock = MockVM {
+            get_object_size: MockMethod::new_fixed(Box::new(|_| NORMAL_OBJECT_SIZE)),
+            is_collection_enabled: MockMethod::new_fixed(Box::new(|_| false)),
+            ..MockVM::default()
+        }
+    });
+
+    c.bench_function("internal pointer - normal objects", |_b| {
+        #[cfg(feature = "is_mmtk_object")]
+        {
+            use mmtk::memory_manager;
+            use mmtk::AllocationSemantics;
+            let addr = memory_manager::alloc(
+                &mut fixture.mutator,
+                NORMAL_OBJECT_SIZE,
+                8,
+                0,
+                AllocationSemantics::Default,
+            );
+            let obj_ref = MockVM::object_start_to_ref(addr);
+            memory_manager::post_alloc(
+                &mut fixture.mutator,
+                obj_ref,
+                NORMAL_OBJECT_SIZE,
+                AllocationSemantics::Default,
+            );
+            let obj_end = addr + NORMAL_OBJECT_SIZE;
+            _b.iter(|| {
+                memory_manager::find_object_from_internal_pointer::<MockVM>(
+                    obj_end - 1,
+                    NORMAL_OBJECT_SIZE,
+                );
+            })
+        }
+        #[cfg(not(feature = "is_mmtk_object"))]
+        panic!("The benchmark requires is_mmtk_object feature to run");
+    });
+
+    // Large objects
+    // 16KB object
+    const LARGE_OBJECT_SIZE: usize = 16 * 1024;
+    write_mockvm(|mock| {
+        *mock = MockVM {
+            get_object_size: MockMethod::new_fixed(Box::new(|_| LARGE_OBJECT_SIZE)),
+            is_collection_enabled: MockMethod::new_fixed(Box::new(|_| false)),
+            ..MockVM::default()
+        }
+    });
+    c.bench_function("internal pointer - large objects", |_b| {
+        #[cfg(feature = "is_mmtk_object")]
+        {
+            use mmtk::memory_manager;
+            use mmtk::AllocationSemantics;
+            let addr = memory_manager::alloc(
+                &mut fixture.mutator,
+                LARGE_OBJECT_SIZE,
+                8,
+                0,
+                AllocationSemantics::Los,
+            );
+            let obj_ref = MockVM::object_start_to_ref(addr);
+            memory_manager::post_alloc(
+                &mut fixture.mutator,
+                obj_ref,
+                LARGE_OBJECT_SIZE,
+                AllocationSemantics::Los,
+            );
+            let obj_end = addr + LARGE_OBJECT_SIZE;
+            _b.iter(|| {
+                memory_manager::find_object_from_internal_pointer::<MockVM>(
+                    obj_end - 1,
+                    LARGE_OBJECT_SIZE,
+                );
+            })
+        }
+        #[cfg(not(feature = "is_mmtk_object"))]
+        panic!("The benchmark requires is_mmtk_object feature to run");
+    });
+}

benches/mock_bench/mod.rs

@@ -1,2 +1,3 @@
 pub mod alloc;
+pub mod internal_pointer;
 pub mod sft;

docs/userguide/src/migration/prefix.md

@@ -32,6 +32,22 @@ Notes for the mmtk-core developers:
 
 ## 0.27.0
 
+### `is_mmtk_object` returns `Option<ObjectReference>
+
+```admonish tldr
+`memory_manager::is_mmtk_object` now returns `Option<ObjectReference>` instead of `bool`.
+Bindings can use the returned object reference instead of computing the object reference at the binding side.
+```
+
+API changes:
+* module `memory_manager`
+  - `is_mmtk_object` now returns `Option<ObjectReference>`.
+
+See also:
+
+-   PR: <https://github.com/mmtk/mmtk-core/pull/1165>
+-   Example: <https://github.com/mmtk/mmtk-ruby/pull/86>
+
 ### Introduce `ObjectModel::IN_OBJECT_ADDRESS_OFFSET`
 
 ```admonish tldr
@@ -49,6 +65,11 @@ API changes:
   - Add a constant `ALIGNMENT` which equals to the word size. All object references should be at least aligned
     to the word size. This is checked in debug builds when an `ObjectReference` is constructed.
 
+See also:
+
+-   PR: <https://github.com/mmtk/mmtk-core/pull/1159>
+-   Example: <https://github.com/mmtk/mmtk-openjdk/pull/283>
+
 ## 0.26.0
 
 ### Rename "edge" to "slot"

src/memory_manager.rs

@@ -588,40 +588,56 @@ pub fn is_live_object<VM: VMBinding>(object: ObjectReference) -> bool {
 /// Concretely:
 /// 1.  Return true if `ObjectReference::from_raw_address(addr)` is a valid object reference to an
 ///     object in any space in MMTk.
-/// 2.  Also return true if there exists an `objref: ObjectReference` such that
-///     -   `objref` is a valid object reference to an object in any space in MMTk, and
-///     -   `lo <= objref.to_address() < hi`, where
-///         -   `lo = addr.align_down(VO_BIT_REGION_SIZE)` and
-///         -   `hi = lo + VO_BIT_REGION_SIZE` and
-///         -   `VO_BIT_REGION_SIZE` is [`crate::util::is_mmtk_object::VO_BIT_REGION_SIZE`].
-///             It is the byte granularity of the valid object (VO) bit.
-/// 3.  Return false otherwise.  This function never panics.
-///
-/// This function uses the "valid object (VO) bits" side metadata, i.e. a bitmap.
-/// For space efficiency, each bit of the bitmap governs a small region of memory.
-/// The size of a region is currently defined as the [minimum object size](crate::util::constants::MIN_OBJECT_SIZE),
-/// which is currently defined as the [word size](crate::util::constants::BYTES_IN_WORD),
-/// which is 4 bytes on 32-bit systems or 8 bytes on 64-bit systems.
-/// The alignment of a region is also the region size.
-/// If a VO bit is `1`, the bitmap cannot tell which address within the 4-byte or 8-byte region
-/// is the valid object reference.
-/// Therefore, if this method returns true, the binding can compute the object reference by
-/// aligning the address to [`crate::util::ObjectReference::ALIGNMENT`].
+/// 2.  Return false otherwise.
 ///
 /// This function is useful for conservative root scanning.  The VM can iterate through all words in
 /// a stack, filter out zeros, misaligned words, obviously out-of-range words (such as addresses
 /// greater than `0x0000_7fff_ffff_ffff` on Linux on x86_64), and use this function to deside if the
 /// word is really a reference.
 ///
+/// This function does not handle internal pointers. If a binding may have internal pointers on
+/// the stack, and requires identifying the base reference for an internal pointer, they should use
+/// [`find_object_from_internal_pointer`] instead.
+///
 /// Note: This function has special behaviors if the VM space (enabled by the `vm_space` feature)
 /// is present.  See `crate::plan::global::BasePlan::vm_space`.
 ///
 /// Argument:
 /// * `addr`: An arbitrary address.
 #[cfg(feature = "is_mmtk_object")]
-pub fn is_mmtk_object(addr: Address) -> bool {
-    use crate::mmtk::SFT_MAP;
-    SFT_MAP.get_checked(addr).is_mmtk_object(addr)
+pub fn is_mmtk_object(addr: Address) -> Option<ObjectReference> {
+    crate::util::is_mmtk_object::check_object_reference(addr)
+}
+
+/// Find if there is an object with VO bit set for the given address range.
+/// This should be used instead of [`crate::memory_manager::is_mmtk_object`] for conservative stack scanning if
+/// the binding may have internal pointers on the stack.
+///
+/// Note that, we only consider pointers that point to addresses that are equal or greater than the in-object addresss
+/// (i.e. [`crate::util::ObjectReference::to_address()`] which is the same as `object_ref.to_raw_address() + ObjectModel::IN_OBJECT_ADDRESS_OFFSET`),
+/// and within the allocation as 'internal pointers'. To be precise, for each object ref `obj_ref`, internal pointers are in the range
+/// `[obj_ref + ObjectModel::IN_OBJECT_ADDRESS_OFFSET, ObjectModel::ref_to_object_start(obj_ref) + ObjectModel::get_current_size(obj_ref))`.
+/// If a binding defines internal pointers differently, calling this method is undefined behavior.
+/// If this is the case for you, please submit an issue or engage us on Zulip to discuss more.
+///
+/// Note that, in the similar situation as [`crate::memory_manager::is_mmtk_object`], the binding should filter
+/// out obvious non-pointers (e.g. alignment check, bound check, etc) before calling this function to avoid unnecessary
+/// cost. This method is not cheap.
+///
+/// To minimize the cost, the user should also use a small `max_search_bytes`.
+///
+/// Note: This function has special behaviors if the VM space (enabled by the `vm_space` feature)
+/// is present.  See `crate::plan::global::BasePlan::vm_space`.
+///
+/// Argument:
+/// * `internal_ptr`: The address to start searching. We search backwards from this address (including this address) to find the base reference.
+/// * `max_search_bytes`: The maximum number of bytes we may search for an object with VO bit set. `internal_ptr - max_search_bytes` is not included.
+#[cfg(feature = "is_mmtk_object")]
+pub fn find_object_from_internal_pointer<VM: VMBinding>(
+    internal_ptr: Address,
+    max_search_bytes: usize,
+) -> Option<ObjectReference> {
+    crate::util::is_mmtk_object::check_internal_reference(internal_ptr, max_search_bytes)
 }
 
 /// Return true if the `object` lies in a region of memory where

src/plan/global.rs

@@ -355,9 +355,10 @@ pub struct BasePlan<VM: VMBinding> {
     /// If VM space is present, it has some special interaction with the
     /// `memory_manager::is_mmtk_object` and the `memory_manager::is_in_mmtk_spaces` functions.
     ///
-    /// -   The `is_mmtk_object` funciton requires the valid object (VO) bit side metadata to identify objects,
-    ///     but currently we do not require the boot image to provide it, so it will not work if the
-    ///     address argument is in the VM space.
+    /// -   The functions `is_mmtk_object` and `find_object_from_internal_pointer` require
+    ///     the valid object (VO) bit side metadata to identify objects.
+    ///     If the binding maintains the VO bit for objects in VM spaces, those functions will work accordingly.
+    ///     Otherwise, calling them is undefined behavior.
     ///
     /// -   The `is_in_mmtk_spaces` currently returns `true` if the given object reference is in
     ///     the VM space.

src/policy/copyspace.rs

@@ -74,8 +74,20 @@ impl<VM: VMBinding> SFT for CopySpace<VM> {
     }
 
     #[cfg(feature = "is_mmtk_object")]
-    fn is_mmtk_object(&self, addr: Address) -> bool {
-        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
+    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr)
+    }
+
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
     }
 
     fn sft_trace_object(

src/policy/immix/immixspace.rs

@@ -141,8 +141,18 @@ impl<VM: VMBinding> SFT for ImmixSpace<VM> {
         crate::util::metadata::vo_bit::set_vo_bit::<VM>(_object);
     }
     #[cfg(feature = "is_mmtk_object")]
-    fn is_mmtk_object(&self, addr: Address) -> bool {
-        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
+    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr)
+    }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        // We don't need to search more than the max object size in the immix space.
+        let search_bytes = usize::min(super::MAX_IMMIX_OBJECT_SIZE, max_search_bytes);
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(ptr, search_bytes)
     }
     fn sft_trace_object(
         &self,

src/policy/immortalspace.rs

@@ -64,8 +64,19 @@ impl<VM: VMBinding> SFT for ImmortalSpace<VM> {
         crate::util::metadata::vo_bit::set_vo_bit::<VM>(object);
     }
     #[cfg(feature = "is_mmtk_object")]
-    fn is_mmtk_object(&self, addr: Address) -> bool {
-        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
+    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr)
+    }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
     }
     fn sft_trace_object(
         &self,

src/policy/largeobjectspace.rs

@@ -80,11 +80,65 @@ impl<VM: VMBinding> SFT for LargeObjectSpace<VM> {
 
         #[cfg(feature = "vo_bit")]
         crate::util::metadata::vo_bit::set_vo_bit::<VM>(object);
+        #[cfg(all(feature = "is_mmtk_object", debug_assertions))]
+        {
+            use crate::util::constants::LOG_BYTES_IN_PAGE;
+            let vo_addr = object.to_address::<VM>();
+            let offset_from_page_start = vo_addr & ((1 << LOG_BYTES_IN_PAGE) - 1) as usize;
+            debug_assert!(
+                offset_from_page_start < crate::util::metadata::vo_bit::VO_BIT_WORD_TO_REGION,
+                "The in-object address is not in the first 512 bytes of a page. The internal pointer searching for LOS won't work."
+            );
+        }
+
         self.treadmill.add_to_treadmill(object, alloc);
     }
     #[cfg(feature = "is_mmtk_object")]
-    fn is_mmtk_object(&self, addr: Address) -> bool {
-        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
+    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr)
+    }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        use crate::util::metadata::vo_bit;
+        // For large object space, it is a bit special. We only need to check VO bit for each page.
+        let mut cur_page = ptr.align_down(BYTES_IN_PAGE);
+        let low_page = ptr
+            .saturating_sub(max_search_bytes)
+            .align_down(BYTES_IN_PAGE);
+        while cur_page >= low_page {
+            // If the page start is not mapped, there can't be an object in it.
+            if !cur_page.is_mapped() {
+                return None;
+            }
+            // For performance, we only check the first word which maps to the first 512 bytes in the page.
+            // In almost all the cases, it should be sufficient.
+            // However, if the in-object address is not in the first 512 bytes, this won't work.
+            // We assert this when we set VO bit for LOS.
+            if vo_bit::get_raw_vo_bit_word(cur_page) != 0 {
+                // Find the exact address that has vo bit set
+                for offset in 0..vo_bit::VO_BIT_WORD_TO_REGION {
+                    let addr = cur_page + offset;
+                    if unsafe { vo_bit::is_vo_addr(addr) } {
+                        let obj = vo_bit::is_internal_ptr_from_vo_bit::<VM>(addr, ptr);
+                        if obj.is_some() {
+                            return obj;
+                        } else {
+                            return None;
+                        }
+                    }
+                }
+                unreachable!(
+                    "We found vo bit in the raw word, but we cannot find the exact address"
+                );
+            }
+
+            cur_page -= BYTES_IN_PAGE;
+        }
+        None
     }
     fn sft_trace_object(
         &self,

src/policy/lockfreeimmortalspace.rs

@@ -74,8 +74,19 @@ impl<VM: VMBinding> SFT for LockFreeImmortalSpace<VM> {
         crate::util::metadata::vo_bit::set_vo_bit::<VM>(_object);
     }
     #[cfg(feature = "is_mmtk_object")]
-    fn is_mmtk_object(&self, addr: Address) -> bool {
-        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr).is_some()
+    fn is_mmtk_object(&self, addr: Address) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::<VM>(addr)
+    }
+    #[cfg(feature = "is_mmtk_object")]
+    fn find_object_from_internal_pointer(
+        &self,
+        ptr: Address,
+        max_search_bytes: usize,
+    ) -> Option<ObjectReference> {
+        crate::util::metadata::vo_bit::find_object_from_internal_pointer::<VM>(
+            ptr,
+            max_search_bytes,
+        )
     }
     fn sft_trace_object(
         &self,