From 35914f193c347390c4d8ed80115f48fe1278b8fc Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 12 Oct 2023 12:57:49 +0800 Subject: [PATCH 1/8] WIP: Encapsulate object forwarding idioms --- src/policy/copyspace.rs | 39 ++++------ src/policy/immix/immixspace.rs | 119 ++++++++++++++-------------- src/util/object_forwarding.rs | 138 +++++++++++++++++++++++++++++---- src/vm/object_model.rs | 6 ++ 4 files changed, 206 insertions(+), 96 deletions(-) diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index b377db5e96..971b02fdf0 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -233,30 +233,25 @@ impl CopySpace { ); trace!("attempting to forward"); - let forwarding_status = object_forwarding::attempt_to_forward::(object); - - trace!("checking if object is being forwarded"); - if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) { - trace!("... yes it is"); - let new_object = - object_forwarding::spin_and_get_forwarded_object::(object, forwarding_status); - trace!("Returning"); - new_object - } else { - trace!("... no it isn't. Copying"); - let new_object = object_forwarding::forward_object::( - object, - semantics.unwrap(), - worker.get_copy_context_mut(), - ); + match object_forwarding::ForwardingAttempt::::attempt(object) { + object_forwarding::ForwardingAttempt::Lost(lost) => { + trace!("... Lost the forwarding race. Another thread is forwarding the object"); + let new_object = lost.spin_and_get_forwarded_object(); + trace!("... {} is already forwarded to {}", object, new_object); + new_object + } + object_forwarding::ForwardingAttempt::Won(won) => { + trace!("... Won the forwarding race. Forwarding..."); + let new_object = + won.forward_object(semantics.unwrap(), worker.get_copy_context_mut()); - #[cfg(feature = "vo_bit")] - crate::util::metadata::vo_bit::set_vo_bit::(new_object); + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::set_vo_bit::(new_object); - trace!("Forwarding pointer"); - queue.enqueue(new_object); - trace!("Copied [{:?} -> {:?}]", object, new_object); - new_object + queue.enqueue(new_object); + trace!("Copied [{:?} -> {:?}]", object, new_object); + new_object + } } } diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index fb578fefe8..6637d27c58 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -16,7 +16,7 @@ use crate::util::metadata::side_metadata::SideMetadataSpec; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit; use crate::util::metadata::{self, MetadataSpec}; -use crate::util::object_forwarding as ForwardingWord; +use crate::util::object_forwarding; use crate::util::{Address, ObjectReference}; use crate::vm::*; use crate::{ @@ -90,8 +90,8 @@ impl SFT for ImmixSpace { return None; } - if ForwardingWord::is_forwarded::(object) { - Some(ForwardingWord::read_forwarding_pointer::(object)) + if object_forwarding::is_forwarded::(object) { + Some(object_forwarding::read_forwarding_pointer::(object)) } else { None } @@ -120,7 +120,7 @@ impl SFT for ImmixSpace { } // If the object is forwarded, it is live, too. - ForwardingWord::is_forwarded::(object) + object_forwarding::is_forwarded::(object) } #[cfg(feature = "object_pinning")] fn pin_object(&self, object: ObjectReference) -> bool { @@ -591,75 +591,78 @@ impl ImmixSpace { #[cfg(feature = "vo_bit")] vo_bit::helper::on_trace_object::(object); - let forwarding_status = ForwardingWord::attempt_to_forward::(object); - if ForwardingWord::state_is_forwarded_or_being_forwarded(forwarding_status) { - // We lost the forwarding race as some other thread has set the forwarding word; wait - // until the object has been forwarded by the winner. Note that the object may not - // necessarily get forwarded since Immix opportunistically moves objects. - #[allow(clippy::let_and_return)] - let new_object = - ForwardingWord::spin_and_get_forwarded_object::(object, forwarding_status); - #[cfg(debug_assertions)] - { - if new_object == object { - debug_assert!( + match object_forwarding::ForwardingAttempt::::attempt(object) { + object_forwarding::ForwardingAttempt::Lost(lost) => { + // We lost the forwarding race as some other thread has set the forwarding word; wait + // until the object has been forwarded by the winner. Note that the object may not + // necessarily get forwarded since Immix opportunistically moves objects. + #[allow(clippy::let_and_return)] + let new_object = lost.spin_and_get_forwarded_object(); + + #[cfg(debug_assertions)] + { + if new_object == object { + debug_assert!( self.is_marked(object) || self.defrag.space_exhausted() || self.is_pinned(object), "Forwarded object is the same as original object {} even though it should have been copied", object, ); - } else { - // new_object != object - debug_assert!( + } else { + // new_object != object + debug_assert!( !Block::containing::(new_object).is_defrag_source(), "Block {:?} containing forwarded object {} should not be a defragmentation source", Block::containing::(new_object), new_object, ); + } } + new_object } - new_object - } else if self.is_marked(object) { - // We won the forwarding race but the object is already marked so we clear the - // forwarding status and return the unmoved object - ForwardingWord::clear_forwarding_bits::(object); - object - } else { - // We won the forwarding race; actually forward and copy the object if it is not pinned - // and we have sufficient space in our copy allocator - let new_object = if self.is_pinned(object) - || (!nursery_collection && self.defrag.space_exhausted()) - { - self.attempt_mark(object, self.mark_state); - ForwardingWord::clear_forwarding_bits::(object); - Block::containing::(object).set_state(BlockState::Marked); - - #[cfg(feature = "vo_bit")] - vo_bit::helper::on_object_marked::(object); - - object - } else { - // We are forwarding objects. When the copy allocator allocates the block, it should - // mark the block. So we do not need to explicitly mark it here. + object_forwarding::ForwardingAttempt::Won(won) => { + if self.is_marked(object) { + // We won the forwarding race but the object is already marked so we clear the + // forwarding status and return the unmoved object + won.revert(); + object + } else { + // We won the forwarding race; actually forward and copy the object if it is not pinned + // and we have sufficient space in our copy allocator + let new_object = if self.is_pinned(object) + || (!nursery_collection && self.defrag.space_exhausted()) + { + self.attempt_mark(object, self.mark_state); + won.revert(); + Block::containing::(object).set_state(BlockState::Marked); + + #[cfg(feature = "vo_bit")] + vo_bit::helper::on_object_marked::(object); + + object + } else { + // We are forwarding objects. When the copy allocator allocates the block, it should + // mark the block. So we do not need to explicitly mark it here. - // Clippy complains if the "vo_bit" feature is not enabled. - #[allow(clippy::let_and_return)] - let new_object = - ForwardingWord::forward_object::(object, semantics, copy_context); + // Clippy complains if the "vo_bit" feature is not enabled. + #[allow(clippy::let_and_return)] + let new_object = won.forward_object(semantics, copy_context); - #[cfg(feature = "vo_bit")] - vo_bit::helper::on_object_forwarded::(new_object); + #[cfg(feature = "vo_bit")] + vo_bit::helper::on_object_forwarded::(new_object); - new_object - }; - debug_assert_eq!( - Block::containing::(new_object).get_state(), - BlockState::Marked - ); + new_object + }; + debug_assert_eq!( + Block::containing::(new_object).get_state(), + BlockState::Marked + ); - queue.enqueue(new_object); - debug_assert!(new_object.is_live()); - self.unlog_object_if_needed(new_object); - new_object + queue.enqueue(new_object); + debug_assert!(new_object.is_live()); + self.unlog_object_if_needed(new_object); + new_object + } + } } } diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index f7a4bea239..2206d90404 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -1,9 +1,9 @@ use crate::util::copy::*; use crate::util::metadata::MetadataSpec; -/// https://github.com/JikesRVM/JikesRVM/blob/master/MMTk/src/org/mmtk/utility/ForwardingWord.java use crate::util::{constants, ObjectReference}; use crate::vm::ObjectModel; use crate::vm::VMBinding; +use std::marker::PhantomData; use std::sync::atomic::Ordering; const FORWARDING_NOT_TRIGGERED_YET: u8 = 0b00; @@ -19,9 +19,118 @@ const FORWARDING_POINTER_MASK: usize = 0x00ff_ffff_ffff_fff8; #[cfg(target_pointer_width = "32")] const FORWARDING_POINTER_MASK: usize = 0xffff_fffc; +/// This type, together with `WonForwardingAttempt` and `LostForwardingAttempt`, represents an +/// attempt to forward an object, and provides methods for manipulating the forwarding state and +/// forwarding pointer of an object in an idiomatic way. +/// +/// A GC worker thread initiates object forwarding by calling `ForwardingAttempt::attempt(object)`. +/// It will try to atomically transition the forwarding state from `FORWARDING_NOT_TRIGGERED_YET` +/// to `BEING_FORWARDED`. +/// +/// - If the transition is successful (i.e. we "won" the race), the GC worker gains exclusive +/// right to further transition its forwarding state. +/// +/// - The GC worker can finish the forwarding by calling +/// `WonForwardingAttempt::forward_object`. It will actually copy the object and set the +/// forwarding bits and forwarding pointers. +/// +/// - The GC worker can abort the forwarding by calling `WonForwardingAttempt::revert`. It +/// will revert the forwarding state to the state before calling +/// `ForwardingAttempt::attempt`. +/// +/// - If the transition failed (i.e. we "lost" the race), it means another GC worker is +/// forwarding or has forwarded the object. +/// +/// - The current GC worker can call `LostForwardingAttempt::spin_and_get_forwarded_object` +/// to wait until the other GC worker finished forwarding, and read the forwarding pointer. +#[must_use] +pub enum ForwardingAttempt { + /// The old state before `Self::attempt` was `FORWARDING_NOT_TRIGGERED_YET`. + /// It means the current thread transitioned the forwarding state from + /// `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. + Won(WonForwardingAttempt), + /// The old state before `Self::attempt` was `BEING_FORWARDED` or `FORWARDED`. + /// It means another thread is forwarding or has already forwarded the object. + Lost(LostForwardingAttempt), +} + +/// Provide states and methods for a won forwarding attempt. +/// +/// See [`ForwardingAttempt`] +#[must_use] +pub struct WonForwardingAttempt { + /// The object to forward. This field holds the from-space address. + object: ObjectReference, + /// VM-specific temporary data used for reverting the forwarding states. + vm_data: usize, + phantom_data: PhantomData, +} + +/// Provide states and methods for a lost forwarding attempt. +/// +/// See [`ForwardingAttempt`] +#[must_use] +pub struct LostForwardingAttempt { + /// The object to forward. This field holds the from-space address. + object: ObjectReference, + old_state: u8, + phantom_data: PhantomData, +} + +impl ForwardingAttempt { + /// Attempt to forward an object by atomically transitioning the forwarding state of `object` + /// from `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. + pub fn attempt(object: ObjectReference) -> Self { + let old_value = attempt_to_forward::(object); + + if is_forwarded_or_being_forwarded::(object) { + Self::Lost(LostForwardingAttempt { + object, + old_state: old_value, + phantom_data: PhantomData, + }) + } else { + Self::Won(WonForwardingAttempt { + object, + vm_data: 0, + phantom_data: PhantomData, + }) + } + } +} + +impl WonForwardingAttempt { + /// Call this to forward the object. + /// + /// This method will also set the forwarding state to `FORWARDED` and store the forwarding + /// pointer at the appropriate location. + pub fn forward_object( + self, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext, + ) -> ObjectReference { + let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); + write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); + new_object + } + + /// Call this to revert the forwarding state to the state before calling `attempt` + pub fn revert(self) { + clear_forwarding_bits::(self.object); + } +} + +impl LostForwardingAttempt { + /// Spin-wait for the object's forwarding to become complete and then read the forwarding + /// pointer to the new object. + pub fn spin_and_get_forwarded_object(self) -> ObjectReference { + spin_and_get_forwarded_object::(self.object, self.old_state) + } +} + /// Attempt to become the worker thread who will forward the object. /// The successful worker will set the object forwarding bits to BEING_FORWARDED, preventing other workers from forwarding the same object. -pub fn attempt_to_forward(object: ObjectReference) -> u8 { +fn attempt_to_forward(object: ObjectReference) -> u8 { loop { let old_value = get_forwarding_status::(object); if old_value != FORWARDING_NOT_TRIGGERED_YET @@ -50,7 +159,7 @@ pub fn attempt_to_forward(object: ObjectReference) -> u8 { /// /// Returns a reference to the new object. /// -pub fn spin_and_get_forwarded_object( +fn spin_and_get_forwarded_object( object: ObjectReference, forwarding_bits: u8, ) -> ObjectReference { @@ -75,12 +184,10 @@ pub fn spin_and_get_forwarded_object( } } -pub fn forward_object( +fn write_forwarding_bits_and_forwarding_pointer( object: ObjectReference, - semantics: CopySemantics, - copy_context: &mut GCWorkerCopyContext, -) -> ObjectReference { - let new_object = VM::VMObjectModel::copy(object, semantics, copy_context); + new_object: ObjectReference, +) { if let Some(shift) = forwarding_bits_offset_in_forwarding_pointer::() { VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.store_atomic::( object, @@ -97,11 +204,10 @@ pub fn forward_object( Ordering::SeqCst, ); } - new_object } /// Return the forwarding bits for a given `ObjectReference`. -pub fn get_forwarding_status(object: ObjectReference) -> u8 { +fn get_forwarding_status(object: ObjectReference) -> u8 { VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.load_atomic::( object, None, @@ -117,15 +223,15 @@ fn is_being_forwarded(object: ObjectReference) -> bool { get_forwarding_status::(object) == BEING_FORWARDED } -pub fn is_forwarded_or_being_forwarded(object: ObjectReference) -> bool { +fn is_forwarded_or_being_forwarded(object: ObjectReference) -> bool { get_forwarding_status::(object) != FORWARDING_NOT_TRIGGERED_YET } -pub fn state_is_forwarded_or_being_forwarded(forwarding_bits: u8) -> bool { +fn state_is_forwarded_or_being_forwarded(forwarding_bits: u8) -> bool { forwarding_bits != FORWARDING_NOT_TRIGGERED_YET } -pub fn state_is_being_forwarded(forwarding_bits: u8) -> bool { +fn state_is_being_forwarded(forwarding_bits: u8) -> bool { forwarding_bits == BEING_FORWARDED } @@ -163,7 +269,7 @@ pub fn read_forwarding_pointer(object: ObjectReference) -> Object /// Write the forwarding pointer of an object. /// This function is called on being_forwarded objects. -pub fn write_forwarding_pointer( +fn write_forwarding_pointer( object: ObjectReference, new_object: ObjectReference, ) { @@ -191,7 +297,7 @@ pub fn write_forwarding_pointer( /// Otherwise, returns `Some(shift)`, where `shift` is the left shift needed on forwarding bits. /// #[cfg(target_endian = "little")] -pub(super) fn forwarding_bits_offset_in_forwarding_pointer() -> Option { +fn forwarding_bits_offset_in_forwarding_pointer() -> Option { use std::ops::Deref; // if both forwarding bits and forwarding pointer are in-header match ( @@ -211,6 +317,6 @@ pub(super) fn forwarding_bits_offset_in_forwarding_pointer() -> O } #[cfg(target_endian = "big")] -pub(super) fn forwarding_bits_offset_in_forwarding_pointer() -> Option { +fn forwarding_bits_offset_in_forwarding_pointer() -> Option { unimplemented!() } diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index e19415cfc1..3b73caee2a 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -117,6 +117,12 @@ pub trait ObjectModel { #[cfg(feature = "vo_bit")] const NEED_VO_BITS_DURING_TRACING: bool = false; + /// When true, mmtk-core will use the metadata `LOCAL_FORWARDING_BITS_SPEC` and + /// `LOCAL_FORWARDING_POINTER_SPEC` defined above to implement object forwarding. + /// + /// When false, + const VM_IMPLEMENTED_FORWARDING: bool = false; + /// A function to non-atomically load the specified per-object metadata's content. /// The default implementation assumes the bits defined by the spec are always avilable for MMTk to use. If that is not the case, a binding should override this method, and provide their implementation. /// Returns the metadata value. From 5df3790b3ea6de4ea5295f6ac7441567ab474ee2 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Thu, 19 Oct 2023 13:37:33 +0800 Subject: [PATCH 2/8] Minor fix --- src/util/object_forwarding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index e3447fc8bc..a11ee0ef37 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -83,7 +83,7 @@ impl ForwardingAttempt { pub fn attempt(object: ObjectReference) -> Self { let old_value = attempt_to_forward::(object); - if is_forwarded_or_being_forwarded::(object) { + if state_is_forwarded_or_being_forwarded(old_value) { Self::Lost(LostForwardingAttempt { object, old_state: old_value, From 8693a85256f6d11f6ccf2edcb35ccc92743cecc7 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 12:54:40 +0800 Subject: [PATCH 3/8] WIP: Guard vm_forwarding behind a Cargo feature --- Cargo.toml | 5 ++ src/util/object_forwarding.rs | 85 ++++++++++++++++++++++--------- src/vm/object_model.rs | 96 ++++++++++++++++++++++++++++++++--- 3 files changed, 155 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b5d5dfc621..d7b92aba5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,6 +143,11 @@ malloc_counted_size = [] # Count the size of all live objects in GC count_live_bytes_in_gc = [] +# Let the VM implement forwarding states (forwarding bits) and forwarding pointers. +# Useful for VMs that have to use the type tag to encode forwarding states due to the lack of spare +# bits in the header. +vm_forwarding = [] + # Do not modify the following line - ci-common.sh matches it # -- Mutally exclusive features -- # Only one feature from each group can be provided. Otherwise build will fail. diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index a11ee0ef37..b5f49c3385 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -62,7 +62,8 @@ pub struct WonForwardingAttempt { /// The object to forward. This field holds the from-space address. object: ObjectReference, /// VM-specific temporary data used for reverting the forwarding states. - vm_data: usize, + #[cfg(feature = "vm_forwarding")] + vm_data: VM::VMObjectModel::VMForwardingDataType, phantom_data: PhantomData, } @@ -73,6 +74,7 @@ pub struct WonForwardingAttempt { pub struct LostForwardingAttempt { /// The object to forward. This field holds the from-space address. object: ObjectReference, + #[cfg(not(feature = "vm_forwarding"))] old_state: u8, phantom_data: PhantomData, } @@ -81,20 +83,26 @@ impl ForwardingAttempt { /// Attempt to forward an object by atomically transitioning the forwarding state of `object` /// from `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. pub fn attempt(object: ObjectReference) -> Self { - let old_value = attempt_to_forward::(object); + #[cfg(not(feature = "vm_forwarding"))] + { + let old_state = attempt_to_forward::(object); + if state_is_forwarded_or_being_forwarded(old_state) { + Self::Lost(LostForwardingAttempt { + object, + old_state, + phantom_data: PhantomData, + }) + } else { + Self::Won(WonForwardingAttempt { + object, + phantom_data: PhantomData, + }) + } + } - if state_is_forwarded_or_being_forwarded(old_value) { - Self::Lost(LostForwardingAttempt { - object, - old_state: old_value, - phantom_data: PhantomData, - }) - } else { - Self::Won(WonForwardingAttempt { - object, - vm_data: 0, - phantom_data: PhantomData, - }) + #[cfg(feature = "vm_forwarding")] + { + match VM::VMObjectModel::attempt_to_forward(object) {} } } } @@ -109,22 +117,51 @@ impl WonForwardingAttempt { semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, ) -> ObjectReference { - let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); - write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); - new_object + #[cfg(not(feature = "vm_forwarding"))] + { + let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); + write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); + new_object + } + + #[cfg(feature = "vm_forwarding")] + { + let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context, self.vm_data); + VM::VMObjectModel::write_forwarding_state_and_forwarding_pointer( + self.object, + new_object, + ); + new_object + } } /// Call this to revert the forwarding state to the state before calling `attempt` pub fn revert(self) { + #[cfg(not(feature = "vm_forwarding"))] clear_forwarding_bits::(self.object); + + #[cfg(feature = "vm_forwarding")] + VM::VMObjectModel::revert_forwarding_state(self.object, self.vm_data); } } impl LostForwardingAttempt { /// Spin-wait for the object's forwarding to become complete and then read the forwarding - /// pointer to the new object. + /// pointer to the new object. If the forwarding state is reverted, this function will simply + /// return `self.object`. pub fn spin_and_get_forwarded_object(self) -> ObjectReference { - spin_and_get_forwarded_object::(self.object, self.old_state) + // About the curly braces: The Rust compiler seems not to know that + // `not(feature = "vm_forwarding")` and `feature = "vm_forwarding"` are mutually exclusive. + // It will suggest adding semicolon (which is wrong) if we remove the curly braces. + #[cfg(not(feature = "vm_forwarding"))] + { + spin_and_get_forwarded_object::(self.object, self.old_state) + } + + #[cfg(feature = "vm_forwarding")] + { + VM::VMObjectModel::spin_and_get_forwarded_object(self.object) + } } } @@ -157,8 +194,9 @@ fn attempt_to_forward(object: ObjectReference) -> u8 { /// * `object`: the forwarded/being_forwarded object. /// * `forwarding_bits`: the last state of the forwarding bits before calling this function. /// -/// Returns a reference to the new object. -/// +/// Returns a reference to the new object. But if the GC algorithm decides that the object should +/// be pinned and should not move, it will revert the forwarding bits to +/// `FORWARDING_NOT_TRIGGERED_YET`. In that case, this function will simply return `object`. fn spin_and_get_forwarded_object( object: ObjectReference, forwarding_bits: u8, @@ -269,10 +307,7 @@ pub fn read_forwarding_pointer(object: ObjectReference) -> Object /// Write the forwarding pointer of an object. /// This function is called on being_forwarded objects. -fn write_forwarding_pointer( - object: ObjectReference, - new_object: ObjectReference, -) { +fn write_forwarding_pointer(object: ObjectReference, new_object: ObjectReference) { debug_assert!( is_being_forwarded::(object), "write_forwarding_pointer called for object {:?} that is not being forwarded! Forwarding state = 0x{:x}", diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index 3b73caee2a..8d9a8454ec 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -64,6 +64,13 @@ pub trait ObjectModel { // Any side metadata offset calculation must consider these to prevent overlaps. A binding should start their // side metadata from GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS or LOCAL_SIDE_METADATA_VM_BASE_ADDRESS. + /// The temporary VM-specific data generated by `attempt_to_forward` which is used when + /// finishing or reverting the forwarding attempt. The VM binding can customize the type as + /// it needs, and usually uses the data to hold the old value of type tag which can be + /// overwritten to represent forwarding states. + #[cfg(feature = "vm_forwarding")] + type VMForwardingDataType; + /// A global 1-bit metadata used by generational plans to track cross-generational pointers. It is generally /// located in side metadata. /// @@ -117,12 +124,6 @@ pub trait ObjectModel { #[cfg(feature = "vo_bit")] const NEED_VO_BITS_DURING_TRACING: bool = false; - /// When true, mmtk-core will use the metadata `LOCAL_FORWARDING_BITS_SPEC` and - /// `LOCAL_FORWARDING_POINTER_SPEC` defined above to implement object forwarding. - /// - /// When false, - const VM_IMPLEMENTED_FORWARDING: bool = false; - /// A function to non-atomically load the specified per-object metadata's content. /// The default implementation assumes the bits defined by the spec are always avilable for MMTk to use. If that is not the case, a binding should override this method, and provide their implementation. /// Returns the metadata value. @@ -347,6 +348,8 @@ pub trait ObjectModel { from: ObjectReference, semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, + #[cfg(feature = "vm_forwarding")] + vm_data: usize, ) -> ObjectReference; /// Copy an object. This is required @@ -478,6 +481,87 @@ pub trait ObjectModel { fn is_object_sane(_object: ObjectReference) -> bool { true } + + /// Initiate the attempt to forward an object. This function should atomically transition the + /// state of the `object` from "forwarding not triggered yet" to "being forwarded". + /// + /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. + /// + /// This function is called when an evacuating GC attempts to forward an object. It should be + /// semantically equivalent to `crate::util::object_forwarding::attempt_to_forward`, albeit + /// this function is allowed to use any VM-specific encoding of the states, not limited to the + /// `FORWARDING_BITS` metadata. + /// + /// Arguments: + /// * `object`: The object to forward. + /// + /// Returns `Ok(word)` if this function successfully transitioned the state, and `word` can be + /// any VM-specific `usize` value that can be used to revert the state change. For VMs that + /// implement the forwarding states by overwriting the type tag in the header, this word can be + /// the original type tag before calling this function. + /// + /// Returns `Err(())` if the transition failed (i.e. another GC worker is forwarding or has + /// forwarded the object). + #[cfg(feature = "vm_forwarding")] + fn attempt_to_forward(_object: ObjectReference) -> Result; + + /// Change the the forwarding state of `object` to represent "forwarded" and write the + /// forwarding pointer to its proper location. + /// + /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. + /// + /// This function is called after `object` has been copied to the new location `new_object`. It + /// should be semantically equivalent to + /// `crate::util::object_forwarding::write_forwarding_bits_and_forwarding_pointer`, albeit this + /// function is allowed to use any VM-specific encoding of the states, not limited to the + /// `FORWARDING_BITS` metadata. + /// + /// Argiments: + /// * `object`: The old address of the object. + /// * `new_object`: The new address of the object. + #[cfg(feature = "vm_forwarding")] + fn write_forwarding_state_and_forwarding_pointer( + object: ObjectReference, + new_object: ObjectReference, + ); + + /// Revert the forwarding state of `object` to the state before `attempt_to_forward` was + /// called. After this function is called, the state should represents "object forwarding is + /// not triggered yet". + /// + /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. + /// + /// This function is called after `object` has been copied to the new location `new_object`. + /// + /// Note that mmtk-core's intrinsic implementation of this operation clears the forwarding bits + /// to `0b00` which represents "object forwarding is not triggered yet". If the VM binding + /// represents the forwarding states by overwriting the type tag, it should restore the type + /// tag to the value before `attempt_to_forward` is called. The VM binding can use the + /// Ok(word)` value returned from `attempt_to_forward` to remember the old type tag. + /// + /// Arguments: + /// * `object`: The object to restore state. + /// * `vm_data`: The `word` of the `Ok(word)` returned from the `attempt_to_forward` call. + #[cfg(feature = "vm_forwarding")] + fn revert_forwarding_state(object: ObjectReference, vm_data: usize); + + /// Wait until the object is no longer in the "being forwarded state" and return the forwarding + /// pointer if the object is forwarded. + /// + /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. + /// + /// The function is called after `attempt_to_forward` failed. The failing GC worker calls this + /// function to wait for another GC thread to finish forwarding the object or reverting the + /// forwarding state. It should be semantically equivalent to + /// `crate::util::object_forwarding::spin_and_get_forwarded_object`. + /// + /// Arguments: + /// * `object`: The object to get forwarding pointer from. + /// + /// If the object is forwarded, return the forwarding pointer; if the state is reverted back to + /// the "object forwarding is not triggered yet" state, simply return `object`. + #[cfg(feature = "vm_forwarding")] + fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference; } pub mod specs { From d31f9aefdb91adc7596e867d5f955bfe18b462b8 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 14:35:58 +0800 Subject: [PATCH 4/8] WIP: Split traditional implementation --- src/util/copy/mod.rs | 4 + src/util/object_forwarding/mod.rs | 198 ++++++++++++++++++ .../traditional.rs} | 178 +++------------- src/vm/object_model.rs | 24 ++- 4 files changed, 241 insertions(+), 163 deletions(-) create mode 100644 src/util/object_forwarding/mod.rs rename src/util/{object_forwarding.rs => object_forwarding/traditional.rs} (53%) diff --git a/src/util/copy/mod.rs b/src/util/copy/mod.rs index 255fc47469..a1e799f435 100644 --- a/src/util/copy/mod.rs +++ b/src/util/copy/mod.rs @@ -8,6 +8,7 @@ use crate::policy::copyspace::CopySpaceCopyContext; use crate::policy::immix::ImmixSpace; use crate::policy::immix::{ImmixCopyContext, ImmixHybridCopyContext}; use crate::policy::space::Space; +#[cfg(not(feature = "vm_forwarding"))] use crate::util::object_forwarding; use crate::util::opaque_pointer::VMWorkerThread; use crate::util::{Address, ObjectReference}; @@ -108,6 +109,9 @@ impl GCWorkerCopyContext { /// * `semantics`: The copy semantic used for the copying. pub fn post_copy(&mut self, object: ObjectReference, bytes: usize, semantics: CopySemantics) { // Clear forwarding bits. + // Not used when using VM-side forwarding implementation because the VM binding is supposed + // to ensure the copied object is not in the "being forwarded" or "forwarded" state. + #[cfg(not(feature = "vm_forwarding"))] object_forwarding::clear_forwarding_bits::(object); // If we are copying objects in mature space, we would need to mark the object as mature. if semantics.is_mature() && self.config.constraints.needs_log_bit { diff --git a/src/util/object_forwarding/mod.rs b/src/util/object_forwarding/mod.rs new file mode 100644 index 0000000000..f979d88424 --- /dev/null +++ b/src/util/object_forwarding/mod.rs @@ -0,0 +1,198 @@ +use crate::util::copy::*; +use crate::util::ObjectReference; +use crate::vm::ObjectModel; +use crate::vm::VMBinding; +use std::marker::PhantomData; + +mod traditional; + +/// This type, together with `WonForwardingAttempt` and `LostForwardingAttempt`, represents an +/// attempt to forward an object, and provides methods for manipulating the forwarding state and +/// forwarding pointer of an object in an idiomatic way. +/// +/// A GC worker thread initiates object forwarding by calling `ForwardingAttempt::attempt(object)`. +/// It will try to atomically transition the forwarding state from `FORWARDING_NOT_TRIGGERED_YET` +/// to `BEING_FORWARDED`. +/// +/// - If the transition is successful (i.e. we "won" the race), the GC worker gains exclusive +/// right to further transition its forwarding state. +/// +/// - The GC worker can finish the forwarding by calling +/// `WonForwardingAttempt::forward_object`. It will actually copy the object and set the +/// forwarding bits and forwarding pointers. +/// +/// - The GC worker can abort the forwarding by calling `WonForwardingAttempt::revert`. It +/// will revert the forwarding state to the state before calling +/// `ForwardingAttempt::attempt`. +/// +/// - If the transition failed (i.e. we "lost" the race), it means another GC worker is +/// forwarding or has forwarded the object. +/// +/// - The current GC worker can call `LostForwardingAttempt::spin_and_get_forwarded_object` +/// to wait until the other GC worker finished forwarding, and read the forwarding pointer. +#[must_use] +pub enum ForwardingAttempt { + /// The old state before `Self::attempt` was `FORWARDING_NOT_TRIGGERED_YET`. + /// It means the current thread transitioned the forwarding state from + /// `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. + Won(WonForwardingAttempt), + /// The old state before `Self::attempt` was `BEING_FORWARDED` or `FORWARDED`. + /// It means another thread is forwarding or has already forwarded the object. + Lost(LostForwardingAttempt), +} + +/// Provide states and methods for a won forwarding attempt. +/// +/// See [`ForwardingAttempt`] +#[must_use] +pub struct WonForwardingAttempt { + /// The object to forward. This field holds the from-space address. + object: ObjectReference, + /// VM-specific temporary data used for reverting the forwarding states. + #[cfg(feature = "vm_forwarding")] + vm_data: >::VMForwardingDataType, + phantom_data: PhantomData, +} + +/// Provide states and methods for a lost forwarding attempt. +/// +/// See [`ForwardingAttempt`] +#[must_use] +pub struct LostForwardingAttempt { + /// The object to forward. This field holds the from-space address. + object: ObjectReference, + #[cfg(not(feature = "vm_forwarding"))] + old_state: u8, + phantom_data: PhantomData, +} + +impl ForwardingAttempt { + /// Attempt to forward an object by atomically transitioning the forwarding state of `object` + /// from `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. + pub fn attempt(object: ObjectReference) -> Self { + #[cfg(not(feature = "vm_forwarding"))] + { + let old_state = traditional::attempt_to_forward::(object); + if traditional::state_is_forwarded_or_being_forwarded(old_state) { + Self::Lost(LostForwardingAttempt { + object, + old_state, + phantom_data: PhantomData, + }) + } else { + Self::Won(WonForwardingAttempt { + object, + phantom_data: PhantomData, + }) + } + } + + #[cfg(feature = "vm_forwarding")] + { + match VM::VMObjectModel::attempt_to_forward(object) { + Ok(vm_data) => Self::Won(WonForwardingAttempt { + object, + vm_data, + phantom_data: PhantomData, + }), + Err(_) => Self::Lost(LostForwardingAttempt { + object, + phantom_data: PhantomData, + }), + } + } + } +} + +impl WonForwardingAttempt { + /// Call this to forward the object. + /// + /// This method will also set the forwarding state to `FORWARDED` and store the forwarding + /// pointer at the appropriate location. + pub fn forward_object( + self, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext, + ) -> ObjectReference { + #[cfg(not(feature = "vm_forwarding"))] + { + let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); + traditional::write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); + new_object + } + + #[cfg(feature = "vm_forwarding")] + { + let new_object = + VM::VMObjectModel::copy(self.object, semantics, copy_context, self.vm_data); + VM::VMObjectModel::write_forwarding_state_and_forwarding_pointer( + self.object, + new_object, + ); + new_object + } + } + + /// Call this to revert the forwarding state to the state before calling `attempt` + pub fn revert(self) { + #[cfg(not(feature = "vm_forwarding"))] + traditional::clear_forwarding_bits::(self.object); + + #[cfg(feature = "vm_forwarding")] + VM::VMObjectModel::revert_forwarding_state(self.object, self.vm_data); + } +} + +impl LostForwardingAttempt { + /// Spin-wait for the object's forwarding to become complete and then read the forwarding + /// pointer to the new object. If the forwarding state is reverted, this function will simply + /// return `self.object`. + pub fn spin_and_get_forwarded_object(self) -> ObjectReference { + // About the curly braces: The Rust compiler seems not to know that + // `not(feature = "vm_forwarding")` and `feature = "vm_forwarding"` are mutually exclusive. + // It will suggest adding semicolon (which is wrong) if we remove the curly braces. + #[cfg(not(feature = "vm_forwarding"))] + { + traditional::spin_and_get_forwarded_object::(self.object, self.old_state) + } + + #[cfg(feature = "vm_forwarding")] + { + VM::VMObjectModel::spin_and_get_forwarded_object(self.object) + } + } +} + +pub fn is_forwarded(object: ObjectReference) -> bool { + #[cfg(not(feature = "vm_forwarding"))] + { + traditional::is_forwarded::(object) + } + + #[cfg(feature = "vm_forwarding")] + { + VM::VMObjectModel::is_forwarded(object) + } +} + + +/// Read the forwarding pointer of an object. +/// This function is called on forwarded/being_forwarded objects. +pub fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference { + #[cfg(not(feature = "vm_forwarding"))] + { + traditional::read_forwarding_pointer::(object) + } + + #[cfg(feature = "vm_forwarding")] + { + VM::VMObjectModel::read_forwarding_pointer(object) + } +} + +/// Clear the forwarding bits. +/// Not needed when using VM-side forwarding implementation. +#[cfg(not(feature = "vm_forwarding"))] +pub fn clear_forwarding_bits(object: ObjectReference) { + traditional::clear_forwarding_bits::(object); +} diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding/traditional.rs similarity index 53% rename from src/util/object_forwarding.rs rename to src/util/object_forwarding/traditional.rs index b5f49c3385..1fde717c62 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding/traditional.rs @@ -1,9 +1,22 @@ -use crate::util::copy::*; +//! This module contains the traditional implementation of object forwarding from JikesRVM. +//! +//! In this implementaion, +//! +//! - The forwarding states are encoded by the two-bit-per-object +//! `ObjectModel::LOCAL_FORWARDING_BITS_SPEC` metadata. +//! - The forwarding pointer is encoded by the one-word-per-object +//! `ObjectModel::LOCAL_FORWARDING_POINTER_SPEC` metadata. +//! +//! It also supports putting the forwarding bits and forwarding pointers into one single word to +//! conserve space. Simply specify both `ObjectModel::LOCAL_FORWARDING_BITS_SPEC` and +//! `ObjectModel::LOCAL_FORWARDING_POINTER_SPEC` as in-header, and let them overlap in the same +//! word, and the `write_forwarding_bits_and_forwarding_pointer` will handle this use case with +//! proper bit operations. + use crate::util::metadata::MetadataSpec; use crate::util::{constants, ObjectReference}; use crate::vm::ObjectModel; use crate::vm::VMBinding; -use std::marker::PhantomData; use std::sync::atomic::Ordering; const FORWARDING_NOT_TRIGGERED_YET: u8 = 0b00; @@ -19,155 +32,9 @@ const FORWARDING_POINTER_MASK: usize = 0x00ff_ffff_ffff_fff8; #[cfg(target_pointer_width = "32")] const FORWARDING_POINTER_MASK: usize = 0xffff_fffc; -/// This type, together with `WonForwardingAttempt` and `LostForwardingAttempt`, represents an -/// attempt to forward an object, and provides methods for manipulating the forwarding state and -/// forwarding pointer of an object in an idiomatic way. -/// -/// A GC worker thread initiates object forwarding by calling `ForwardingAttempt::attempt(object)`. -/// It will try to atomically transition the forwarding state from `FORWARDING_NOT_TRIGGERED_YET` -/// to `BEING_FORWARDED`. -/// -/// - If the transition is successful (i.e. we "won" the race), the GC worker gains exclusive -/// right to further transition its forwarding state. -/// -/// - The GC worker can finish the forwarding by calling -/// `WonForwardingAttempt::forward_object`. It will actually copy the object and set the -/// forwarding bits and forwarding pointers. -/// -/// - The GC worker can abort the forwarding by calling `WonForwardingAttempt::revert`. It -/// will revert the forwarding state to the state before calling -/// `ForwardingAttempt::attempt`. -/// -/// - If the transition failed (i.e. we "lost" the race), it means another GC worker is -/// forwarding or has forwarded the object. -/// -/// - The current GC worker can call `LostForwardingAttempt::spin_and_get_forwarded_object` -/// to wait until the other GC worker finished forwarding, and read the forwarding pointer. -#[must_use] -pub enum ForwardingAttempt { - /// The old state before `Self::attempt` was `FORWARDING_NOT_TRIGGERED_YET`. - /// It means the current thread transitioned the forwarding state from - /// `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. - Won(WonForwardingAttempt), - /// The old state before `Self::attempt` was `BEING_FORWARDED` or `FORWARDED`. - /// It means another thread is forwarding or has already forwarded the object. - Lost(LostForwardingAttempt), -} - -/// Provide states and methods for a won forwarding attempt. -/// -/// See [`ForwardingAttempt`] -#[must_use] -pub struct WonForwardingAttempt { - /// The object to forward. This field holds the from-space address. - object: ObjectReference, - /// VM-specific temporary data used for reverting the forwarding states. - #[cfg(feature = "vm_forwarding")] - vm_data: VM::VMObjectModel::VMForwardingDataType, - phantom_data: PhantomData, -} - -/// Provide states and methods for a lost forwarding attempt. -/// -/// See [`ForwardingAttempt`] -#[must_use] -pub struct LostForwardingAttempt { - /// The object to forward. This field holds the from-space address. - object: ObjectReference, - #[cfg(not(feature = "vm_forwarding"))] - old_state: u8, - phantom_data: PhantomData, -} - -impl ForwardingAttempt { - /// Attempt to forward an object by atomically transitioning the forwarding state of `object` - /// from `FORWARDING_NOT_TRIGGERED_YET` to `BEING_FORWARDED`. - pub fn attempt(object: ObjectReference) -> Self { - #[cfg(not(feature = "vm_forwarding"))] - { - let old_state = attempt_to_forward::(object); - if state_is_forwarded_or_being_forwarded(old_state) { - Self::Lost(LostForwardingAttempt { - object, - old_state, - phantom_data: PhantomData, - }) - } else { - Self::Won(WonForwardingAttempt { - object, - phantom_data: PhantomData, - }) - } - } - - #[cfg(feature = "vm_forwarding")] - { - match VM::VMObjectModel::attempt_to_forward(object) {} - } - } -} - -impl WonForwardingAttempt { - /// Call this to forward the object. - /// - /// This method will also set the forwarding state to `FORWARDED` and store the forwarding - /// pointer at the appropriate location. - pub fn forward_object( - self, - semantics: CopySemantics, - copy_context: &mut GCWorkerCopyContext, - ) -> ObjectReference { - #[cfg(not(feature = "vm_forwarding"))] - { - let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); - write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); - new_object - } - - #[cfg(feature = "vm_forwarding")] - { - let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context, self.vm_data); - VM::VMObjectModel::write_forwarding_state_and_forwarding_pointer( - self.object, - new_object, - ); - new_object - } - } - - /// Call this to revert the forwarding state to the state before calling `attempt` - pub fn revert(self) { - #[cfg(not(feature = "vm_forwarding"))] - clear_forwarding_bits::(self.object); - - #[cfg(feature = "vm_forwarding")] - VM::VMObjectModel::revert_forwarding_state(self.object, self.vm_data); - } -} - -impl LostForwardingAttempt { - /// Spin-wait for the object's forwarding to become complete and then read the forwarding - /// pointer to the new object. If the forwarding state is reverted, this function will simply - /// return `self.object`. - pub fn spin_and_get_forwarded_object(self) -> ObjectReference { - // About the curly braces: The Rust compiler seems not to know that - // `not(feature = "vm_forwarding")` and `feature = "vm_forwarding"` are mutually exclusive. - // It will suggest adding semicolon (which is wrong) if we remove the curly braces. - #[cfg(not(feature = "vm_forwarding"))] - { - spin_and_get_forwarded_object::(self.object, self.old_state) - } - - #[cfg(feature = "vm_forwarding")] - { - VM::VMObjectModel::spin_and_get_forwarded_object(self.object) - } - } -} - /// Attempt to become the worker thread who will forward the object. /// The successful worker will set the object forwarding bits to BEING_FORWARDED, preventing other workers from forwarding the same object. -fn attempt_to_forward(object: ObjectReference) -> u8 { +pub fn attempt_to_forward(object: ObjectReference) -> u8 { loop { let old_value = get_forwarding_status::(object); if old_value != FORWARDING_NOT_TRIGGERED_YET @@ -197,7 +64,7 @@ fn attempt_to_forward(object: ObjectReference) -> u8 { /// Returns a reference to the new object. But if the GC algorithm decides that the object should /// be pinned and should not move, it will revert the forwarding bits to /// `FORWARDING_NOT_TRIGGERED_YET`. In that case, this function will simply return `object`. -fn spin_and_get_forwarded_object( +pub fn spin_and_get_forwarded_object( object: ObjectReference, forwarding_bits: u8, ) -> ObjectReference { @@ -222,7 +89,7 @@ fn spin_and_get_forwarded_object( } } -fn write_forwarding_bits_and_forwarding_pointer( +pub fn write_forwarding_bits_and_forwarding_pointer( object: ObjectReference, new_object: ObjectReference, ) { @@ -265,7 +132,7 @@ fn is_forwarded_or_being_forwarded(object: ObjectReference) -> bo get_forwarding_status::(object) != FORWARDING_NOT_TRIGGERED_YET } -fn state_is_forwarded_or_being_forwarded(forwarding_bits: u8) -> bool { +pub fn state_is_forwarded_or_being_forwarded(forwarding_bits: u8) -> bool { forwarding_bits != FORWARDING_NOT_TRIGGERED_YET } @@ -307,7 +174,10 @@ pub fn read_forwarding_pointer(object: ObjectReference) -> Object /// Write the forwarding pointer of an object. /// This function is called on being_forwarded objects. -fn write_forwarding_pointer(object: ObjectReference, new_object: ObjectReference) { +fn write_forwarding_pointer( + object: ObjectReference, + new_object: ObjectReference, +) { debug_assert!( is_being_forwarded::(object), "write_forwarding_pointer called for object {:?} that is not being forwarded! Forwarding state = 0x{:x}", @@ -354,4 +224,4 @@ fn forwarding_bits_offset_in_forwarding_pointer() -> Option() -> Option { unimplemented!() -} +} \ No newline at end of file diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index 8d9a8454ec..513f64be0e 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -344,12 +344,13 @@ pub trait ObjectModel { /// * `from`: The address of the object to be copied. /// * `semantics`: The copy semantic to use. /// * `copy_context`: The `GCWorkerCopyContext` for the GC thread. + /// * `vm_data`: The VM-specific data from the `attempt_to_forward` call. This argument only + /// exists if the Cargo feature `"vm_forwarding"` is enabled. fn copy( from: ObjectReference, semantics: CopySemantics, copy_context: &mut GCWorkerCopyContext, - #[cfg(feature = "vm_forwarding")] - vm_data: usize, + #[cfg(feature = "vm_forwarding")] vm_data: Self::VMForwardingDataType, ) -> ObjectReference; /// Copy an object. This is required @@ -495,15 +496,15 @@ pub trait ObjectModel { /// Arguments: /// * `object`: The object to forward. /// - /// Returns `Ok(word)` if this function successfully transitioned the state, and `word` can be - /// any VM-specific `usize` value that can be used to revert the state change. For VMs that - /// implement the forwarding states by overwriting the type tag in the header, this word can be - /// the original type tag before calling this function. + /// Returns `Ok(value)` if this function successfully transitioned the state, and `value` can + /// be any VM-specific value that can be used to revert the state change. For VMs that + /// implement the forwarding states by overwriting the type tag in the header, this value can + /// be the original type tag before calling this function. /// /// Returns `Err(())` if the transition failed (i.e. another GC worker is forwarding or has /// forwarded the object). #[cfg(feature = "vm_forwarding")] - fn attempt_to_forward(_object: ObjectReference) -> Result; + fn attempt_to_forward(_object: ObjectReference) -> Result; /// Change the the forwarding state of `object` to represent "forwarded" and write the /// forwarding pointer to its proper location. @@ -541,9 +542,9 @@ pub trait ObjectModel { /// /// Arguments: /// * `object`: The object to restore state. - /// * `vm_data`: The `word` of the `Ok(word)` returned from the `attempt_to_forward` call. + /// * `vm_data`: The `value` of the `Ok(value)` returned from the `attempt_to_forward` call. #[cfg(feature = "vm_forwarding")] - fn revert_forwarding_state(object: ObjectReference, vm_data: usize); + fn revert_forwarding_state(object: ObjectReference, vm_data: Self::VMForwardingDataType); /// Wait until the object is no longer in the "being forwarded state" and return the forwarding /// pointer if the object is forwarded. @@ -562,6 +563,11 @@ pub trait ObjectModel { /// the "object forwarding is not triggered yet" state, simply return `object`. #[cfg(feature = "vm_forwarding")] fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference; + + #[cfg(feature = "vm_forwarding")] + fn is_forwarded(object: ObjectReference) -> bool; + #[cfg(feature = "vm_forwarding")] + fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference; } pub mod specs { From 6112020542396d2db1160bc1ee92a5e0f42cadc1 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 14:36:33 +0800 Subject: [PATCH 5/8] Some trace logs in immix space --- src/policy/immix/immixspace.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index ae7f380442..66bc6a369f 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -583,6 +583,7 @@ impl ImmixSpace { match object_forwarding::ForwardingAttempt::::attempt(object) { object_forwarding::ForwardingAttempt::Lost(lost) => { + trace!("Lost. object: {}", object); // We lost the forwarding race as some other thread has set the forwarding word; wait // until the object has been forwarded by the winner. Note that the object may not // necessarily get forwarded since Immix opportunistically moves objects. @@ -610,17 +611,21 @@ impl ImmixSpace { new_object } object_forwarding::ForwardingAttempt::Won(won) => { + trace!("Won. object: {}", object); if self.is_marked(object) { + trace!("Is marked."); // We won the forwarding race but the object is already marked so we clear the // forwarding status and return the unmoved object won.revert(); object } else { + trace!("Not marked."); // We won the forwarding race; actually forward and copy the object if it is not pinned // and we have sufficient space in our copy allocator let new_object = if self.is_pinned(object) || (!nursery_collection && self.defrag.space_exhausted()) { + trace!("Is pinned or space exhausted."); self.attempt_mark(object, self.mark_state); won.revert(); Block::containing::(object).set_state(BlockState::Marked); @@ -630,6 +635,7 @@ impl ImmixSpace { object } else { + trace!("We should forward."); // We are forwarding objects. When the copy allocator allocates the block, it should // mark the block. So we do not need to explicitly mark it here. @@ -647,6 +653,7 @@ impl ImmixSpace { BlockState::Marked ); + trace!("Enqeuing {}", new_object); queue.enqueue(new_object); debug_assert!(new_object.is_live()); self.unlog_object_if_needed(new_object); From 4a17a5353b0b81fe236aa72ca300b9f4af06be7e Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 15:21:19 +0800 Subject: [PATCH 6/8] Fixes for whe "vm_forwarding" is enabled --- src/util/object_forwarding/mod.rs | 4 +- src/vm/object_model.rs | 61 +++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/util/object_forwarding/mod.rs b/src/util/object_forwarding/mod.rs index f979d88424..1ed70bad84 100644 --- a/src/util/object_forwarding/mod.rs +++ b/src/util/object_forwarding/mod.rs @@ -90,12 +90,12 @@ impl ForwardingAttempt { #[cfg(feature = "vm_forwarding")] { match VM::VMObjectModel::attempt_to_forward(object) { - Ok(vm_data) => Self::Won(WonForwardingAttempt { + Some(vm_data) => Self::Won(WonForwardingAttempt { object, vm_data, phantom_data: PhantomData, }), - Err(_) => Self::Lost(LostForwardingAttempt { + None => Self::Lost(LostForwardingAttempt { object, phantom_data: PhantomData, }), diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index 513f64be0e..5219a728b4 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -486,36 +486,31 @@ pub trait ObjectModel { /// Initiate the attempt to forward an object. This function should atomically transition the /// state of the `object` from "forwarding not triggered yet" to "being forwarded". /// - /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. - /// /// This function is called when an evacuating GC attempts to forward an object. It should be - /// semantically equivalent to `crate::util::object_forwarding::attempt_to_forward`, albeit - /// this function is allowed to use any VM-specific encoding of the states, not limited to the - /// `FORWARDING_BITS` metadata. + /// semantically equivalent to + /// [`crate::util::object_forwarding::traditional::attempt_to_forward`], albeit this function + /// uses VM-specific encoding of forwarding states. /// /// Arguments: /// * `object`: The object to forward. /// - /// Returns `Ok(value)` if this function successfully transitioned the state, and `value` can + /// Returns `Some(value)` if this function successfully transitioned the state, and `value` can /// be any VM-specific value that can be used to revert the state change. For VMs that /// implement the forwarding states by overwriting the type tag in the header, this value can /// be the original type tag before calling this function. /// - /// Returns `Err(())` if the transition failed (i.e. another GC worker is forwarding or has + /// Returns `None`` if the transition failed (i.e. another GC worker is forwarding or has /// forwarded the object). #[cfg(feature = "vm_forwarding")] - fn attempt_to_forward(_object: ObjectReference) -> Result; + fn attempt_to_forward(_object: ObjectReference) -> Option; /// Change the the forwarding state of `object` to represent "forwarded" and write the /// forwarding pointer to its proper location. /// - /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. - /// /// This function is called after `object` has been copied to the new location `new_object`. It /// should be semantically equivalent to - /// `crate::util::object_forwarding::write_forwarding_bits_and_forwarding_pointer`, albeit this - /// function is allowed to use any VM-specific encoding of the states, not limited to the - /// `FORWARDING_BITS` metadata. + /// [`crate::util::object_forwarding::traditional::write_forwarding_bits_and_forwarding_pointer`], + /// albeit this function uses VM-specific encoding of forwarding states. /// /// Argiments: /// * `object`: The old address of the object. @@ -530,31 +525,28 @@ pub trait ObjectModel { /// called. After this function is called, the state should represents "object forwarding is /// not triggered yet". /// - /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. - /// /// This function is called after `object` has been copied to the new location `new_object`. /// /// Note that mmtk-core's intrinsic implementation of this operation clears the forwarding bits /// to `0b00` which represents "object forwarding is not triggered yet". If the VM binding /// represents the forwarding states by overwriting the type tag, it should restore the type /// tag to the value before `attempt_to_forward` is called. The VM binding can use the - /// Ok(word)` value returned from `attempt_to_forward` to remember the old type tag. + /// `Some(value)` value returned from `attempt_to_forward` to remember the old type tag. /// /// Arguments: /// * `object`: The object to restore state. - /// * `vm_data`: The `value` of the `Ok(value)` returned from the `attempt_to_forward` call. + /// * `vm_data`: The `value` of the `Some(value)` returned from the `attempt_to_forward` call. #[cfg(feature = "vm_forwarding")] fn revert_forwarding_state(object: ObjectReference, vm_data: Self::VMForwardingDataType); /// Wait until the object is no longer in the "being forwarded state" and return the forwarding /// pointer if the object is forwarded. /// - /// This function only needs to be implemented if `VM_IMPLEMENTED_FORWARDING` is true. - /// /// The function is called after `attempt_to_forward` failed. The failing GC worker calls this /// function to wait for another GC thread to finish forwarding the object or reverting the /// forwarding state. It should be semantically equivalent to - /// `crate::util::object_forwarding::spin_and_get_forwarded_object`. + /// [`crate::util::object_forwarding::traditional::spin_and_get_forwarded_object`], albeit this + /// function uses VM-specific encoding of forwarding states. /// /// Arguments: /// * `object`: The object to get forwarding pointer from. @@ -564,8 +556,37 @@ pub trait ObjectModel { #[cfg(feature = "vm_forwarding")] fn spin_and_get_forwarded_object(object: ObjectReference) -> ObjectReference; + /// Returns true if `object` is already forwarded. + /// + /// This function may be called during tracing and weak reference processing. This test must + /// be atomic with respect to other GC workers which may attempt ot forward the same `object`. + /// + /// For example, during weak reference processing, in `Scanning::process_weak_refs`, the VM + /// binding may call `ObjectReference::is_reachable` to test if an object is reached, and call + /// `ObjectTracer::trace_object` to resurrect dead finalizable objects. For some spaces, such + /// as `CopySpace` and `ImmixSpace`, `is_reachable` may call `is_forwarded`, and `trace_object` + /// may forward the object. The implementer of this function must be aware that while one GC + /// worker is executing `is_forwarding`, another GC worker may be forwarding the same object. + /// The implementor of weak reference process must also be aware that even if + /// `is_forwarded(object)` returns `false`, the object may be concurrently forwarded by another + /// GC worker. The good news is, `trace_object(object)` is thread-safe. If two GC workers + /// call `trace_object` on the same `object`, one will forward it. The other will not forward + /// it, but will get its forwarded address. + /// + /// Argument: + /// * `object`: The object to test if it is forwarded. #[cfg(feature = "vm_forwarding")] fn is_forwarded(object: ObjectReference) -> bool; + + /// Read the forwarding pointer of `object` which is already forwarded. + /// + /// This function may be called during weak reference processing. Because the `object` must + /// have already been forwarded before calling this function, it is OK if this function is not + /// atomic because other GC workers will not revert the forwarding state. + /// + /// Argument: + /// * `object`: The forwarding pointer of this object will be read. This object must be + /// already forwarded before calling this function. #[cfg(feature = "vm_forwarding")] fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference; } From e6a081986bd693c522240a1b7ef738c7b83a5b74 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 15:22:10 +0800 Subject: [PATCH 7/8] Revert "Some trace logs in immix space" This reverts commit 6112020542396d2db1160bc1ee92a5e0f42cadc1. --- src/policy/immix/immixspace.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 66bc6a369f..ae7f380442 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -583,7 +583,6 @@ impl ImmixSpace { match object_forwarding::ForwardingAttempt::::attempt(object) { object_forwarding::ForwardingAttempt::Lost(lost) => { - trace!("Lost. object: {}", object); // We lost the forwarding race as some other thread has set the forwarding word; wait // until the object has been forwarded by the winner. Note that the object may not // necessarily get forwarded since Immix opportunistically moves objects. @@ -611,21 +610,17 @@ impl ImmixSpace { new_object } object_forwarding::ForwardingAttempt::Won(won) => { - trace!("Won. object: {}", object); if self.is_marked(object) { - trace!("Is marked."); // We won the forwarding race but the object is already marked so we clear the // forwarding status and return the unmoved object won.revert(); object } else { - trace!("Not marked."); // We won the forwarding race; actually forward and copy the object if it is not pinned // and we have sufficient space in our copy allocator let new_object = if self.is_pinned(object) || (!nursery_collection && self.defrag.space_exhausted()) { - trace!("Is pinned or space exhausted."); self.attempt_mark(object, self.mark_state); won.revert(); Block::containing::(object).set_state(BlockState::Marked); @@ -635,7 +630,6 @@ impl ImmixSpace { object } else { - trace!("We should forward."); // We are forwarding objects. When the copy allocator allocates the block, it should // mark the block. So we do not need to explicitly mark it here. @@ -653,7 +647,6 @@ impl ImmixSpace { BlockState::Marked ); - trace!("Enqeuing {}", new_object); queue.enqueue(new_object); debug_assert!(new_object.is_live()); self.unlog_object_if_needed(new_object); From 06e0287763b01ab2caf19bf33e37af2e9c26e216 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 20 Oct 2023 18:02:15 +0800 Subject: [PATCH 8/8] Style fixes Formatting. Don't link to private items. --- src/util/object_forwarding/mod.rs | 6 ++++-- src/util/object_forwarding/traditional.rs | 7 ++----- src/vm/object_model.rs | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/util/object_forwarding/mod.rs b/src/util/object_forwarding/mod.rs index 1ed70bad84..ea250eeefc 100644 --- a/src/util/object_forwarding/mod.rs +++ b/src/util/object_forwarding/mod.rs @@ -117,7 +117,10 @@ impl WonForwardingAttempt { #[cfg(not(feature = "vm_forwarding"))] { let new_object = VM::VMObjectModel::copy(self.object, semantics, copy_context); - traditional::write_forwarding_bits_and_forwarding_pointer::(self.object, new_object); + traditional::write_forwarding_bits_and_forwarding_pointer::( + self.object, + new_object, + ); new_object } @@ -175,7 +178,6 @@ pub fn is_forwarded(object: ObjectReference) -> bool { } } - /// Read the forwarding pointer of an object. /// This function is called on forwarded/being_forwarded objects. pub fn read_forwarding_pointer(object: ObjectReference) -> ObjectReference { diff --git a/src/util/object_forwarding/traditional.rs b/src/util/object_forwarding/traditional.rs index 1fde717c62..e8391ebbb0 100644 --- a/src/util/object_forwarding/traditional.rs +++ b/src/util/object_forwarding/traditional.rs @@ -174,10 +174,7 @@ pub fn read_forwarding_pointer(object: ObjectReference) -> Object /// Write the forwarding pointer of an object. /// This function is called on being_forwarded objects. -fn write_forwarding_pointer( - object: ObjectReference, - new_object: ObjectReference, -) { +fn write_forwarding_pointer(object: ObjectReference, new_object: ObjectReference) { debug_assert!( is_being_forwarded::(object), "write_forwarding_pointer called for object {:?} that is not being forwarded! Forwarding state = 0x{:x}", @@ -224,4 +221,4 @@ fn forwarding_bits_offset_in_forwarding_pointer() -> Option() -> Option { unimplemented!() -} \ No newline at end of file +} diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index 5219a728b4..d8843c3dde 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -488,7 +488,7 @@ pub trait ObjectModel { /// /// This function is called when an evacuating GC attempts to forward an object. It should be /// semantically equivalent to - /// [`crate::util::object_forwarding::traditional::attempt_to_forward`], albeit this function + /// `crate::util::object_forwarding::traditional::attempt_to_forward`, albeit this function /// uses VM-specific encoding of forwarding states. /// /// Arguments: @@ -509,7 +509,7 @@ pub trait ObjectModel { /// /// This function is called after `object` has been copied to the new location `new_object`. It /// should be semantically equivalent to - /// [`crate::util::object_forwarding::traditional::write_forwarding_bits_and_forwarding_pointer`], + /// `crate::util::object_forwarding::traditional::write_forwarding_bits_and_forwarding_pointer`, /// albeit this function uses VM-specific encoding of forwarding states. /// /// Argiments: @@ -545,7 +545,7 @@ pub trait ObjectModel { /// The function is called after `attempt_to_forward` failed. The failing GC worker calls this /// function to wait for another GC thread to finish forwarding the object or reverting the /// forwarding state. It should be semantically equivalent to - /// [`crate::util::object_forwarding::traditional::spin_and_get_forwarded_object`], albeit this + /// `crate::util::object_forwarding::traditional::spin_and_get_forwarded_object`, albeit this /// function uses VM-specific encoding of forwarding states. /// /// Arguments: