diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 875981151c..8383a7aa72 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -86,14 +86,13 @@ pub fn mmtk_init(builder: &MMTKBuilder) -> Box> { Box::new(mmtk) } +/// Add an externally mmapped region to the VM space. A VM space can be set through MMTk options (`vm_space_start` and `vm_space_size`), +/// and can also be set through this function call. A VM space can be discontiguous. This function can be called multiple times, +/// and all the address ranges passed as arguments in the function will be considered as part of the VM space. +/// Currently we do not allow removing regions from VM space. #[cfg(feature = "vm_space")] -pub fn lazy_init_vm_space(mmtk: &'static mut MMTK, start: Address, size: usize) { - unsafe { - mmtk.get_plan_mut() - .base_mut() - .vm_space - .lazy_initialize(start, size); - } +pub fn set_vm_space(mmtk: &'static mut MMTK, start: Address, size: usize) { + unsafe { mmtk.get_plan_mut() }.base_mut().vm_space.set_vm_region(start, size); } /// Request MMTk to create a mutator for the given thread. The ownership diff --git a/src/plan/global.rs b/src/plan/global.rs index c718c0c816..941089430f 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -529,7 +529,11 @@ impl BasePlan { VMRequest::discontiguous(), )), #[cfg(feature = "vm_space")] - vm_space: VMSpace::new(&mut args), + vm_space: VMSpace::new(args.get_space_args( + "vm_space", + false, + VMRequest::discontiguous(), + )), initialized: AtomicBool::new(false), trigger_gc_when_heap_is_full: AtomicBool::new(true), diff --git a/src/plan/mod.rs b/src/plan/mod.rs index 15c16182fa..b2c415464d 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -26,8 +26,6 @@ pub use global::AllocationSemantics; pub(crate) use global::GcStatus; pub use global::Plan; pub(crate) use global::PlanTraceObject; -#[cfg(feature = "vm_space")] // This is used for creating VM space -pub(crate) use global::{CreateGeneralPlanArgs, CreateSpecificPlanArgs}; mod mutator_context; pub use mutator_context::Mutator; diff --git a/src/policy/vmspace.rs b/src/policy/vmspace.rs index 4d26460966..e2964c52c9 100644 --- a/src/policy/vmspace.rs +++ b/src/policy/vmspace.rs @@ -1,52 +1,79 @@ -use crate::plan::{CreateGeneralPlanArgs, CreateSpecificPlanArgs}; +use crate::mmtk::SFT_MAP; use crate::plan::{ObjectQueue, VectorObjectQueue}; -use crate::policy::immortalspace::ImmortalSpace; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; use crate::util::address::Address; -use crate::util::heap::HeapMeta; +use crate::util::constants::BYTES_IN_PAGE; +use crate::util::heap::externalpageresource::{ExternalPageResource, ExternalPages}; +use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK; use crate::util::heap::PageResource; -use crate::util::heap::VMRequest; -use crate::util::metadata::side_metadata::SideMetadataContext; -use crate::util::metadata::side_metadata::SideMetadataSanity; +use crate::util::metadata::mark_bit::MarkState; +use crate::util::opaque_pointer::*; use crate::util::ObjectReference; -use crate::vm::VMBinding; +use crate::vm::{ObjectModel, VMBinding}; -use delegate::delegate; +use std::sync::atomic::Ordering; +/// A special space for VM/Runtime managed memory. The implementation is similar to [`crate::policy::immortalspace::ImmortalSpace`], +/// except that VM space does not allocate. Instead, the runtime can add regions that are externally managed +/// and mmapped to the space, and allow objects in those regions to be traced in the same way +/// as other MMTk objects allocated by MMTk. pub struct VMSpace { - inner: Option>, - // Save it - args: CreateSpecificPlanArgs, + mark_state: MarkState, + common: CommonSpace, + pr: ExternalPageResource, } impl SFT for VMSpace { - delegate! { - // Delegate every call to the inner space. Given that we have acquired SFT, we can assume there are objects in the space and the space is initialized. - to self.space() { - fn name(&self) -> &str; - fn is_live(&self, object: ObjectReference) -> bool; - fn is_reachable(&self, object: ObjectReference) -> bool; - #[cfg(feature = "object_pinning")] - fn pin_object(&self, object: ObjectReference) -> bool; - #[cfg(feature = "object_pinning")] - fn unpin_object(&self, object: ObjectReference) -> bool; - #[cfg(feature = "object_pinning")] - fn is_object_pinned(&self, object: ObjectReference) -> bool; - fn is_movable(&self) -> bool; - #[cfg(feature = "sanity")] - fn is_sane(&self) -> bool; - fn initialize_object_metadata(&self, object: ObjectReference, alloc: bool); - #[cfg(feature = "is_mmtk_object")] - fn is_mmtk_object(&self, addr: Address) -> bool; - fn sft_trace_object( - &self, - queue: &mut VectorObjectQueue, - object: ObjectReference, - worker: GCWorkerMutRef, - ) -> ObjectReference; + fn name(&self) -> &str { + self.common.name + } + fn is_live(&self, _object: ObjectReference) -> bool { + true + } + fn is_reachable(&self, object: ObjectReference) -> bool { + self.mark_state.is_marked::(object) + } + #[cfg(feature = "object_pinning")] + fn pin_object(&self, _object: ObjectReference) -> bool { + false + } + #[cfg(feature = "object_pinning")] + fn unpin_object(&self, _object: ObjectReference) -> bool { + false + } + #[cfg(feature = "object_pinning")] + fn is_object_pinned(&self, _object: ObjectReference) -> bool { + true + } + fn is_movable(&self) -> bool { + false + } + #[cfg(feature = "sanity")] + fn is_sane(&self) -> bool { + true + } + fn initialize_object_metadata(&self, object: ObjectReference, _alloc: bool) { + self.mark_state + .on_object_metadata_initialization::(object); + if self.common.needs_log_bit { + VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.mark_as_unlogged::(object, Ordering::SeqCst); } + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::set_vo_bit::(object); + } + #[cfg(feature = "is_mmtk_object")] + fn is_mmtk_object(&self, addr: Address) -> bool { + crate::util::metadata::vo_bit::is_vo_bit_set_for_addr::(addr).is_some() + } + fn sft_trace_object( + &self, + queue: &mut VectorObjectQueue, + object: ObjectReference, + _worker: GCWorkerMutRef, + ) -> ObjectReference { + self.trace_object(queue, object) } } @@ -58,38 +85,56 @@ impl Space for VMSpace { self } fn get_page_resource(&self) -> &dyn PageResource { - self.space().get_page_resource() + &self.pr } fn common(&self) -> &CommonSpace { - self.space().common() + &self.common } fn initialize_sft(&self) { - if self.inner.is_some() { - self.common().initialize_sft(self.as_sft()) + // Initialize sft for current external pages. This method is called at the end of plan creation. + // So we only set SFT for VM regions that are set by options (we skipped sft initialization for them earlier). + for external_pages in self.pr.get_external_pages().iter() { + // Chunk align things. + let start = external_pages.start.align_down(BYTES_IN_CHUNK); + let size = external_pages.end.align_up(BYTES_IN_CHUNK) - start; + // The region should be empty in SFT map -- if they were set before this point, there could be invalid SFT pointers. + debug_assert_eq!( + SFT_MAP.get_checked(start).name(), + crate::policy::sft::EMPTY_SFT_NAME + ); + // Set SFT + self.set_sft_for_vm_region(start, size); } } fn release_multiple_pages(&mut self, _start: Address) { - panic!("immortalspace only releases pages enmasse") + unreachable!() } - fn verify_side_metadata_sanity(&self, side_metadata_sanity_checker: &mut SideMetadataSanity) { - side_metadata_sanity_checker.verify_metadata_context( - std::any::type_name::(), - &SideMetadataContext { - global: self.args.global_side_metadata_specs.clone(), - local: vec![], - }, - ) + fn acquire(&self, _tls: VMThread, _pages: usize) -> Address { + unreachable!() } fn address_in_space(&self, start: Address) -> bool { - if let Some(space) = self.space_maybe() { - space.address_in_space(start) - } else { - false - } + // The default implementation checks with vm map. But vm map has some assumptions about + // the address range for spaces and the VM space may break those assumptions (as the space is + // mmapped by the runtime rather than us). So we we use SFT here. + + // However, SFT map may not be an ideal solution either for 64 bits. The default + // implementation of SFT map on 64 bits is `SFTSpaceMap`, which maps the entire address + // space into an index between 0 and 31, and assumes any address with the same index + // is in the same space (with the same SFT). MMTk spaces uses 1-16. We guarantee that + // VM space does not overlap with the address range that MMTk spaces may use. So + // any region used as VM space will have an index of 0, or 17-31, and all the addresses + // that are mapped to the same index will be considered as in the VM space. That means, + // after we map a region as VM space, the nearby addresses will also be considered + // as in the VM space if we use the default `SFTSpaceMap`. We can guarantee the nearby + // addresses are not MMTk spaces, but we cannot tell whether they really in the VM space + // or not. + // A solution to this is to use `SFTDenseChunkMap` if `vm_space` is enabled on 64 bits. + // `SFTDenseChunkMap` has an overhead of a few percentages (~3%) compared to `SFTSpaceMap`. + SFT_MAP.get_checked(start).name() == self.name() } } @@ -112,107 +157,100 @@ impl crate::policy::gc_work::PolicyTraceObject for VMSpace VMSpace { - pub fn new(args: &mut CreateSpecificPlanArgs) -> Self { - let args_clone = CreateSpecificPlanArgs { - global_args: CreateGeneralPlanArgs { - vm_map: args.global_args.vm_map, - mmapper: args.global_args.mmapper, - heap: HeapMeta::new(), // we do not use this - options: args.global_args.options.clone(), - gc_trigger: args.global_args.gc_trigger.clone(), - scheduler: args.global_args.scheduler.clone(), - }, - constraints: args.constraints, - global_side_metadata_specs: args.global_side_metadata_specs.clone(), + pub fn new(args: crate::policy::space::PlanCreateSpaceArgs) -> Self { + let (vm_space_start, vm_space_size) = + (*args.options.vm_space_start, *args.options.vm_space_size); + let space = Self { + mark_state: MarkState::new(), + pr: ExternalPageResource::new(args.vm_map), + common: CommonSpace::new(args.into_policy_args( + false, + true, + crate::util::metadata::extract_side_metadata(&[ + *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + ]), + )), }; - // Create the space if the VM space start/size is set. Otherwise, use None. - let inner = (!args.global_args.options.vm_space_start.is_zero()) - .then(|| Self::create_space(args, None)); - Self { - inner, - args: args_clone, + + if !vm_space_start.is_zero() { + // Do not set sft here, as the space may be moved. We do so for those regions in `initialize_sft`. + space.set_vm_region_inner(vm_space_start, vm_space_size, false); } - } - pub fn lazy_initialize(&mut self, start: Address, size: usize) { - assert!(self.inner.is_none(), "VM space has been initialized"); - self.inner = Some(Self::create_space(&mut self.args, Some((start, size)))); + space + } - self.common().initialize_sft(self.as_sft()); + pub fn set_vm_region(&mut self, start: Address, size: usize) { + self.set_vm_region_inner(start, size, true); } - fn create_space( - args: &mut CreateSpecificPlanArgs, - location: Option<(Address, usize)>, - ) -> ImmortalSpace { - use crate::util::heap::layout::vm_layout::BYTES_IN_CHUNK; + fn set_vm_region_inner(&self, start: Address, size: usize, set_sft: bool) { + assert!(size > 0); + assert!(!start.is_zero()); - // If the location of the VM space is not supplied, find them in the options. - let (vm_space_start, vm_space_bytes) = location.unwrap_or(( - *args.global_args.options.vm_space_start, - *args.global_args.options.vm_space_size, - )); - // Verify the start and the size is valid - assert!(!vm_space_start.is_zero()); - assert!(vm_space_bytes > 0); + let end = start + size; - // We only map on chunk granularity. Align them. - let vm_space_start_aligned = vm_space_start.align_down(BYTES_IN_CHUNK); - let vm_space_end_aligned = (vm_space_start + vm_space_bytes).align_up(BYTES_IN_CHUNK); - let vm_space_bytes_aligned = vm_space_end_aligned - vm_space_start_aligned; + let chunk_start = start.align_down(BYTES_IN_CHUNK); + let chunk_end = end.align_up(BYTES_IN_CHUNK); + let chunk_size = chunk_end - chunk_start; // For simplicity, VMSpace has to be outside our available heap range. // TODO: Allow VMSpace in our available heap range. assert!(Address::range_intersection( - &(vm_space_start_aligned..vm_space_end_aligned), + &(chunk_start..chunk_end), &crate::util::heap::layout::available_range() ) .is_empty()); debug!( "Align VM space ({}, {}) to chunk ({}, {})", - vm_space_start, - vm_space_start + vm_space_bytes, - vm_space_start_aligned, - vm_space_end_aligned - ); - - let space_args = args.get_space_args( - "vm_space", - false, - VMRequest::fixed(vm_space_start_aligned, vm_space_bytes_aligned), + start, end, chunk_start, chunk_end ); - let space = - ImmortalSpace::new_vm_space(space_args, vm_space_start_aligned, vm_space_bytes_aligned); - // The space is mapped externally by the VM. We need to update our mmapper to mark the range as mapped. - space.ensure_mapped(); - - space - } + // Mark as mapped in mmapper + self.common.mmapper.mark_as_mapped(chunk_start, chunk_size); + // Map side metadata + self.common + .metadata + .try_map_metadata_space(chunk_start, chunk_size) + .unwrap(); + // Insert to vm map: it would be good if we can make VM map aware of the region. However, the region may be outside what we can map in our VM map implementation. + // self.common.vm_map.insert(chunk_start, chunk_size, self.common.descriptor); + // Set SFT if we should + if set_sft { + self.set_sft_for_vm_region(chunk_start, chunk_size); + } - fn space_maybe(&self) -> Option<&ImmortalSpace> { - self.inner.as_ref() + self.pr.add_new_external_pages(ExternalPages { + start: start.align_down(BYTES_IN_PAGE), + end: end.align_up(BYTES_IN_PAGE), + }); } - fn space(&self) -> &ImmortalSpace { - self.inner.as_ref().unwrap() + fn set_sft_for_vm_region(&self, chunk_start: Address, chunk_size: usize) { + assert!(chunk_start.is_aligned_to(BYTES_IN_CHUNK)); + assert!(crate::util::conversions::raw_is_aligned( + chunk_size, + BYTES_IN_CHUNK + )); + assert!(SFT_MAP.has_sft_entry(chunk_start), "The VM space start (aligned to {}) does not have a valid SFT entry. Possibly the address range is not in the address range we use.", chunk_start); + unsafe { + SFT_MAP.eager_initialize(self.as_sft(), chunk_start, chunk_size); + } } - // fn space_mut(&mut self) -> &mut ImmortalSpace { - // self.inner.as_mut().unwrap() - // } - pub fn prepare(&mut self) { - if let Some(ref mut space) = &mut self.inner { - space.prepare() + self.mark_state.on_global_prepare::(); + for external_pages in self.pr.get_external_pages().iter() { + self.mark_state.on_block_reset::( + external_pages.start, + external_pages.end - external_pages.start, + ); } } pub fn release(&mut self) { - if let Some(ref mut space) = &mut self.inner { - space.release() - } + self.mark_state.on_global_release::(); } pub fn trace_object( @@ -220,10 +258,16 @@ impl VMSpace { queue: &mut Q, object: ObjectReference, ) -> ObjectReference { - if let Some(ref space) = &self.inner { - space.trace_object(queue, object) - } else { - panic!("We haven't initialized vm space, but we tried to trace the object {} and thought it was in vm space?", object) + #[cfg(feature = "vo_bit")] + debug_assert!( + crate::util::metadata::vo_bit::is_vo_bit_set::(object), + "{:x}: VO bit not set", + object + ); + debug_assert!(self.in_space(object)); + if self.mark_state.test_and_mark::(object) { + queue.enqueue(object); } + object } } diff --git a/src/util/alloc/allocators.rs b/src/util/alloc/allocators.rs index 2c47ff1675..ffcd3f1950 100644 --- a/src/util/alloc/allocators.rs +++ b/src/util/alloc/allocators.rs @@ -180,7 +180,9 @@ pub enum AllocatorSelector { #[repr(C, u8)] #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] pub enum AllocatorInfo { - BumpPointer { bump_pointer_offset: usize }, + BumpPointer { + bump_pointer_offset: usize, + }, // FIXME: Add free-list fast-path Unimplemented, #[default] diff --git a/src/util/heap/externalpageresource.rs b/src/util/heap/externalpageresource.rs new file mode 100644 index 0000000000..613042b8ba --- /dev/null +++ b/src/util/heap/externalpageresource.rs @@ -0,0 +1,83 @@ +use super::layout::VMMap; +use super::pageresource::{PRAllocFail, PRAllocResult}; +use super::PageResource; +use crate::util::address::Address; +use crate::util::constants::BYTES_IN_PAGE; +use crate::util::constants::LOG_BYTES_IN_PAGE; +use crate::util::heap::pageresource::CommonPageResource; +use crate::util::heap::space_descriptor::SpaceDescriptor; +use crate::util::opaque_pointer::*; +use crate::vm::VMBinding; + +use std::marker::PhantomData; +use std::sync::{Mutex, MutexGuard}; + +/// A special page resource that records some external pages that are not mmapped by us, +/// but are used by our space (namely VM space). Unlike other page resources, we cannot +/// allocate from this page resource. +pub struct ExternalPageResource { + common: CommonPageResource, + ranges: Mutex>, + _p: PhantomData, +} + +#[derive(Copy, Clone, Debug)] +pub struct ExternalPages { + pub start: Address, + pub end: Address, +} + +impl PageResource for ExternalPageResource { + fn common(&self) -> &CommonPageResource { + &self.common + } + + fn common_mut(&mut self) -> &mut CommonPageResource { + &mut self.common + } + + fn reserve_pages(&self, _pages: usize) -> usize { + unreachable!() + } + fn commit_pages(&self, _reserved_pages: usize, _actual_pages: usize, _tls: VMThread) { + unreachable!() + } + + fn get_available_physical_pages(&self) -> usize { + 0 + } + + fn alloc_pages( + &self, + _space_descriptor: SpaceDescriptor, + _reserved_pages: usize, + _required_pages: usize, + _tls: VMThread, + ) -> Result { + panic!("Cannot allocate from ExternalPageResource") + } +} + +impl ExternalPageResource { + pub fn new(vm_map: &'static dyn VMMap) -> Self { + Self { + common: CommonPageResource::new(false, false, vm_map), + ranges: Mutex::new(vec![]), + _p: PhantomData, + } + } + + pub fn add_new_external_pages(&self, pages: ExternalPages) { + assert!(pages.start.is_aligned_to(BYTES_IN_PAGE)); + assert!(pages.end.is_aligned_to(BYTES_IN_PAGE)); + + let mut lock = self.ranges.lock().unwrap(); + let n_pages = (pages.end - pages.start) >> LOG_BYTES_IN_PAGE; + self.common.accounting.reserve_and_commit(n_pages); + lock.push(pages); + } + + pub fn get_external_pages(&self) -> MutexGuard> { + self.ranges.lock().unwrap() + } +} diff --git a/src/util/heap/mod.rs b/src/util/heap/mod.rs index a41a5a53e3..e980b5adff 100644 --- a/src/util/heap/mod.rs +++ b/src/util/heap/mod.rs @@ -3,6 +3,7 @@ mod accounting; pub(crate) mod layout; pub(crate) mod blockpageresource; pub(crate) mod chunk_map; +pub(crate) mod externalpageresource; pub(crate) mod freelistpageresource; pub(crate) mod gc_trigger; mod heap_meta;