diff --git a/.github/workflows/cargo-publish.yml b/.github/workflows/cargo-publish.yml index 5b7e9f8956..5a10621747 100644 --- a/.github/workflows/cargo-publish.yml +++ b/.github/workflows/cargo-publish.yml @@ -19,5 +19,9 @@ jobs: override: true - name: Cargo login run: cargo login ${{ secrets.CI_CARGO_LOGIN }} - - name: Cargo publish - run: cargo publish + - name: Publish sub crates + run: | + cargo publish --manifest-path=macros/Cargo.toml + - name: Public mmtk-core + run: | + cargo publish diff --git a/Cargo.toml b/Cargo.toml index 51cb2cc69a..ee9d6d5b56 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,9 @@ crate-type = ["rlib"] doctest = false [dependencies] +# MMTk macros +mmtk-macros = { version = "0.11.0", path = "macros/" } + custom_derive = "0.1" enum_derive = "0.1" libc = "0.2" diff --git a/docs/tutorial/code/mygc_semispace/gc_work.rs b/docs/tutorial/code/mygc_semispace/gc_work.rs index 02b122e29e..0a36583bf4 100644 --- a/docs/tutorial/code/mygc_semispace/gc_work.rs +++ b/docs/tutorial/code/mygc_semispace/gc_work.rs @@ -5,11 +5,95 @@ use crate::vm::VMBinding; use std::ops::{Deref, DerefMut}; // ANCHOR_END: imports -// ANCHOR: workcontext +// ANCHOR: workcontext_sft pub struct MyGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for MyGCWorkContext { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = SFTProcessEdges; } -// ANCHOR_END: workcontext +// ANCHOR_END: workcontext_sft + +// ANCHOR: workcontext_plan +use crate::scheduler::gc_work::PlanProcessEdges; +use crate::policy::gc_work::DEFAULT_TRACE; +pub struct MyGCWorkContext2(std::marker::PhantomData); +impl crate::scheduler::GCWorkContext for MyGCWorkContext2 { + type VM = VM; + type PlanType = MyGC; + type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; +} +// ANCHOR: workcontext_plan + +use crate::util::{Address, ObjectReference}; +use crate::util::copy::CopySemantics; +use crate::MMTK; +use crate::policy::space::Space; + +// ANCHOR: mygc_process_edges +pub struct MyGCProcessEdges { + plan: &'static MyGC, + base: ProcessEdgesBase, +} +// ANCHOR_END: mygc_process_edges + +// ANCHOR: mygc_process_edges_impl +impl ProcessEdgesWork for MyGCProcessEdges { + type VM = VM; + fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk); + let plan = base.plan().downcast_ref::>().unwrap(); + Self { base, plan } + } + + #[inline] + fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { + if object.is_null() { + return object; + } + if self.plan.tospace().in_space(object) { + self.plan.tospace().trace_object::( + self, + object, + Some(CopySemantics::DefaultCopy), + self.worker(), + ) + } else if self.plan.fromspace().in_space(object) { + self.plan.fromspace().trace_object::( + self, + object, + Some(CopySemantics::DefaultCopy), + self.worker(), + ) + } else { + self.plan.common.trace_object::(self, object) + } + } +} +// ANCHOR_END: mygc_process_edges_impl + +// ANCHOR: mygc_process_edges_deref +impl Deref for MyGCProcessEdges { + type Target = ProcessEdgesBase; + #[inline] + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl DerefMut for MyGCProcessEdges { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} +// ANCHOR_END: mygc_process_edges_deref + +// ANCHOR: workcontext_mygc +pub struct MyGCWorkContext3(std::marker::PhantomData); +impl crate::scheduler::GCWorkContext for MyGCWorkContext3 { + type VM = VM; + type PlanType = MyGC; + type ProcessEdgesWorkType = MyGCProcessEdges; +} +// ANCHOR: workcontext_mygc diff --git a/docs/tutorial/code/mygc_semispace/global.rs b/docs/tutorial/code/mygc_semispace/global.rs index bbb0dccd64..c0c09d0b00 100644 --- a/docs/tutorial/code/mygc_semispace/global.rs +++ b/docs/tutorial/code/mygc_semispace/global.rs @@ -29,12 +29,18 @@ use std::sync::Arc; // Remove #[allow(unused_imports)]. // Remove handle_user_collection_request(). +use mmtk_macros::PlanTraceObject; + // Modify // ANCHOR: plan_def +#[derive(PlanTraceObject)] pub struct MyGC { pub hi: AtomicBool, + #[trace(CopySemantics::DefaultCopy)] pub copyspace0: CopySpace, + #[trace(CopySemantics::DefaultCopy)] pub copyspace1: CopySpace, + #[fallback_trace] pub common: CommonPlan, } // ANCHOR_END: plan_def diff --git a/docs/tutorial/src/mygc/ss/collection.md b/docs/tutorial/src/mygc/ss/collection.md index 7a1a70b246..985e5e1a9c 100644 --- a/docs/tutorial/src/mygc/ss/collection.md +++ b/docs/tutorial/src/mygc/ss/collection.md @@ -43,14 +43,15 @@ method `schedule_common_work()` that will add common work packets for you. To use `schedule_common_work()`, first we need to create a type `MyGCWorkContext` and implement the trait `GCWorkContext` for it. We create `gc_work.rs` and add the -following implementation. Note that we do not set a specific `ProcessEdgesWorkType` -and we will use the default [`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html), +following implementation. Note that we will use the default +[`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html), which is a general work packet that a plan can use to trace objects. For plans like semispace, `SFTProcessEdges` is sufficient. For more complex GC plans, one can create and write their own work packet that implements the `ProcessEdgesWork` trait. +We will discuss about this later, and discuss the alternatives. ```rust -{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext}} +{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_sft}} ``` Then we implement `schedule_collection()` using `MyGCWorkContext` and `schedule_common_work()`. @@ -111,35 +112,6 @@ there aren't any preparation steps for the mutator in this GC. In `create_mygc_mutator()`, find the field `prep_func` and change it from `mygc_mutator_noop()` to `mygc_mutator_prepare()`. - -## Scan objects - -Next, we'll add the code to allow the plan to collect garbage - filling out -functions for work packets. - -In `gc_work.rs`, add a new method to `ProcessEdgesWork for MyGCProcessEdges`, -`trace_object(&mut self, object: ObjectReference)`. -This method should return an ObjectReference, and use the -inline attribute. -Check if the object passed into the function is null -(`object.is_null()`). If it is, return the object. -Otherwise, check which space the object is in, and forward the call to the -policy-specific object tracing code. If it is in neither space, forward the -call to the common space and let the common space to handle object tracing in -its spaces (e.g. immortal or large object space): - -```rust -{{#include ../../../code/mygc_semispace/gc_work.rs:trace_object}} -``` - -Add two new implementation blocks, `Deref` and `DerefMut` for -`MyGCProcessEdges`. These allow `MyGCProcessEdges` to be dereferenced to -`ProcessEdgesBase`, and allows easy access to fields in `ProcessEdgesBase`. - -```rust -{{#include ../../../code/mygc_semispace/gc_work.rs:deref}} -``` - ## Release Finally, we need to fill out the functions that are, roughly speaking, @@ -178,6 +150,91 @@ will then go to the new tospace. Delete `mygc_mutator_noop()`. It was a placeholder for the prepare and release functions that you have now added, so it is now dead code. +## ProcessEdgesWork for MyGC + +[`ProcessEdgesWork`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/trait.ProcessEdgesWork.html) +is the key work packet for tracing objects in a GC. A `ProcessEdgesWork` implementation +defines how to trace objects, and how to generate more work packets based on the current tracing +to finish the object closure. + +`GCWorkContext` specifies a type +that implements `ProcessEdgesWork`, and we used `SFTProcessEdges` earlier. In +this section, we discuss what `SFTProcessEdges` does, and what the alternatives +are. + +### Approach 1: Use `SFTProcessEdges` + +[`SFTProcessEdges`](https://www.mmtk.io/mmtk-core/mmtk/scheduler/gc_work/struct.SFTProcessEdges.html) dispatches +the tracing of objects to their respective spaces through [Space Function Table (SFT)](https://www.mmtk.io/mmtk-core/mmtk/policy/space/trait.SFT.html). +As long as all the policies in a plan provide an implementation of `sft_trace_object()` in their SFT implementations, +the plan can use `SFTProcessEdges`. Currently most policies provide an implementation for `sft_trace_object()`, except +mark compact and immix. Those two policies use multiple GC traces, and due to the limitation of SFT, SFT does not allow +multiple `sft_trace_object()` for a policy. + +`SFTProcessEdges` is the simplest approach when all the policies support it. Fortunately, we can use it for our GC, semispace. + +### Approach 2: Derive `PlanTraceObject` and use `PlanProcessEdges` + +`PlanProcessEdges` is another general `ProcessEdgesWork` implementation that can be used by most plans. When a plan +implements the [`PlanTraceObject`](https://www.mmtk.io/mmtk-core/mmtk/plan/transitive_closure/trait.PlanTraceObject.html), +it can use `PlanProcessEdges`. + +You can manually provide an implementation of `PlanTraceObject` for `MyGC`. But you can also use the derive macro MMTK provides, +and the macro will generate an implementation of `PlanTraceObject`: +* add `#[derive(PlanTraceObject)]` for `MyGC` (import the macro properly: `use mmtk_macros::PlanTraceObject`) +* add `#[trace(CopySemantics::Default)]` to both copy space fields, `copyspace0` and `copyspace1`. This tells the macro to generate + trace code for both spaces, and for any copying in the spaces, use `CopySemantics::DefaultCopy` that we have configured early. +* add `#[fallback_trace]` to `common`. This tells the macro that if an object is not found in any space with `#[trace]` in ths plan, + try find the space for the object in the 'parent' plan. In our case, we fall back to the `CommonPlan`, as the object may be + in large object space or immortal space in the common plan. `CommonPlan` also implements `PlanTraceObject`, so it knows how to + find a space for the object and trace it in the same way. + +With the derive macro, your `MyGC` struct should look like this: +```rust +{{#include ../../../code/mygc_semispace/global.rs:plan_def}} +``` + +Once this is done, you can specify `PlanProcessEdges` as the `ProcessEdgesWorkType` in your GC work context: +```rust +{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_plan}} +``` + +### Approach 3: Implement your own `ProcessEdgesWork` + +Apart from the two approaches above, you can always implement your own `ProcessEdgesWork`. This is +an overkill for simple plans like semi space, but might be necessary for more complex plans. +We discuss how to implement it for `MyGC`. + +Create a struct `MyGCProcessEdges` in the `gc_work` module. It includes a reference +back to the plan, and a `ProcessEdgesBase` field: +```rust +{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges}} +``` + +Implement `ProcessEdgesWork` for `MyGCProcessEdges`. As most methods in the trait have a default +implemetation, we only need to implement `new()` and `trace_object()` for our plan. However, this +may not be true when you implement it for other GC plans. It would be better to check the default +implementation of `ProcessEdgesWork`. + +For `trace_object()`, what we do is similar to the approach above (except that we need to write the code +ourselves rather than letting the macro to generate it for us). We try to figure out +which space the object is in, and invoke `trace_object()` for the object on that space. If the +object is not in any of the semi spaces in the plan, we forward the call to `CommonPlan`. +```rust +{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges_impl}} +``` + +We would also need to implement `Deref` and `DerefMut` to our `ProcessEdgesWork` impl to be +dereferenced as `ProcessEdgesBase`. +```rust +{{#include ../../../code/mygc_semispace/gc_work.rs:mygc_process_edges_deref}} +``` + +In the end, use `MyGCProcessEdges` as `ProcessEdgesWorkType` in the `GCWorkContext`: +```rust +{{#include ../../../code/mygc_semispace/gc_work.rs:workcontext_mygc}} +``` + ## Summary You should now have MyGC working and able to collect garbage. All three diff --git a/macros/Cargo.toml b/macros/Cargo.toml new file mode 100644 index 0000000000..efb0839006 --- /dev/null +++ b/macros/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "mmtk-macros" +# the macro crate uses the same version as mmtk-core +version = "0.11.0" +edition = "2021" +license = "MIT OR Apache-2.0" +description = "MMTk macros provides procedural macros used by mmtk-core." +homepage = "https://www.mmtk.io" +repository = "https://github.com/mmtk/mmtk-core/tree/master/macros" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.37" +syn = { version = "1.0.91", features = ["extra-traits"] } +quote = "1.0.18" +proc-macro-error = "1.0.4" diff --git a/macros/src/lib.rs b/macros/src/lib.rs new file mode 100644 index 0000000000..40ee8f9b01 --- /dev/null +++ b/macros/src/lib.rs @@ -0,0 +1,68 @@ +extern crate proc_macro; +extern crate syn; +extern crate proc_macro_error; +extern crate quote; + +use proc_macro::TokenStream; +use proc_macro_error::proc_macro_error; +use syn::{parse_macro_input}; +use proc_macro_error::abort_call_site; +use quote::quote; +use syn::DeriveInput; + +mod util; +mod plan_trace_object_impl; + +const DEBUG_MACRO_OUTPUT: bool = false; + +/// Generally a plan needs to add these attributes in order for the macro to work. The macro will +/// generate an implementation of `PlanTraceObject` for the plan. With `PlanTraceObject`, the plan use +/// `PlanProcessEdges` for GC tracing. The attributes only affects code generation in the macro, thus +/// only affects the generated `PlanTraceObject` implementation. +/// * add `#[derive(PlanTraceObject)]` to the plan struct. +/// * add `#[trace]` to each space field the plan struct has. If the policy is a copying policy, +/// it needs to further specify the copy semantic (`#[trace(CopySemantics::X)]`) +/// * add `#[fallback_trace]` to the parent plan if the plan is composed with other plans (or parent plans). +/// For example, `GenImmix` is composed with `Gen`, `Gen` is composed with `CommonPlan`, `CommonPlan` is composed +/// with `BasePlan`. +/// * add `#[post_scan]` to any space field that has some policy-specific post_scan_object(). For objects in those spaces, +/// `post_scan_object()` in the policy will be called after `VM::VMScanning::scan_object()`. +#[proc_macro_error] +#[proc_macro_derive(PlanTraceObject, attributes(trace, post_scan, fallback_trace))] +pub fn derive_plan_trace_object(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + let ident = input.ident; + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + + let output = if let syn::Data::Struct(syn::DataStruct { + fields: syn::Fields::Named(ref fields), + .. + }) = input.data { + let spaces = util::get_fields_with_attribute(fields, "trace"); + let post_scan_spaces = util::get_fields_with_attribute(fields, "post_scan"); + let fallback = util::get_unique_field_with_attribute(fields, "fallback_trace"); + + let trace_object_function = plan_trace_object_impl::generate_trace_object(&spaces, &fallback, &ty_generics); + let post_scan_object_function = plan_trace_object_impl::generate_post_scan_object(&post_scan_spaces, &ty_generics); + let may_move_objects_function = plan_trace_object_impl::generate_may_move_objects(&spaces, &fallback, &ty_generics); + quote!{ + impl #impl_generics crate::plan::PlanTraceObject #ty_generics for #ident #ty_generics #where_clause { + #trace_object_function + + #post_scan_object_function + + #may_move_objects_function + } + } + } else { + abort_call_site!("`#[derive(PlanTraceObject)]` only supports structs with named fields.") + }; + + // Debug the output - use the following code to debug the generated code (when cargo exapand is not working) + if DEBUG_MACRO_OUTPUT { + use quote::ToTokens; + println!("{}", output.to_token_stream()); + } + + output.into() +} diff --git a/macros/src/plan_trace_object_impl.rs b/macros/src/plan_trace_object_impl.rs new file mode 100644 index 0000000000..13914efd0e --- /dev/null +++ b/macros/src/plan_trace_object_impl.rs @@ -0,0 +1,127 @@ +use quote::quote; +use syn::{Field, TypeGenerics}; +use proc_macro2::TokenStream as TokenStream2; + +use crate::util; + +pub(crate) fn generate_trace_object<'a>( + space_fields: &[&'a Field], + parent_field: &Option<&'a Field>, + ty_generics: &TypeGenerics, +) -> TokenStream2 { + // Generate a check with early return for each space + let space_field_handler = space_fields.iter().map(|f| { + let f_ident = f.ident.as_ref().unwrap(); + let ref f_ty = f.ty; + + // Figure out copy + let trace_attr = util::get_field_attribute(f, "trace").unwrap(); + let copy = if !trace_attr.tokens.is_empty() { + use syn::Token; + use syn::NestedMeta; + use syn::punctuated::Punctuated; + + let args = trace_attr.parse_args_with(Punctuated::::parse_terminated).unwrap(); + // CopySemantics::X is a path. + if let Some(NestedMeta::Meta(syn::Meta::Path(p))) = args.first() { + quote!{ Some(#p) } + } else { + quote!{ None } + } + } else { + quote!{ None } + }; + + quote! { + if self.#f_ident.in_space(__mmtk_objref) { + return <#f_ty as PolicyTraceObject #ty_generics>::trace_object::(&self.#f_ident, __mmtk_trace, __mmtk_objref, #copy, __mmtk_worker); + } + } + }); + + // Generate a fallback to the parent plan + let parent_field_delegator = if let Some(f) = parent_field { + let f_ident = f.ident.as_ref().unwrap(); + let ref f_ty = f.ty; + quote! { + <#f_ty as PlanTraceObject #ty_generics>::trace_object::(&self.#f_ident, __mmtk_trace, __mmtk_objref, __mmtk_worker) + } + } else { + quote! { + panic!("No more spaces to try") + } + }; + + quote! { + #[inline(always)] + fn trace_object(&self, __mmtk_trace: &mut T, __mmtk_objref: crate::util::ObjectReference, __mmtk_worker: &mut crate::scheduler::GCWorker) -> crate::util::ObjectReference { + use crate::policy::space::Space; + use crate::policy::gc_work::PolicyTraceObject; + use crate::plan::PlanTraceObject; + #(#space_field_handler)* + #parent_field_delegator + } + } +} + +pub(crate) fn generate_post_scan_object<'a>( + post_scan_object_fields: &[&'a Field], + ty_generics: &TypeGenerics, +) -> TokenStream2 { + let scan_field_handler = post_scan_object_fields.iter().map(|f| { + let f_ident = f.ident.as_ref().unwrap(); + let ref f_ty = f.ty; + + quote! { + if self.#f_ident.in_space(__mmtk_objref) { + use crate::policy::gc_work::PolicyTraceObject; + <#f_ty as PolicyTraceObject #ty_generics>::post_scan_object(&self.#f_ident, __mmtk_objref); + return; + } + } + }); + + quote! { + #[inline(always)] + fn post_scan_object(&self, __mmtk_objref: crate::util::ObjectReference) { + #(#scan_field_handler)* + } + } +} + +// The generated function needs to be inlined and constant folded. Otherwise, there will be a huge +// performance penalty. +pub(crate) fn generate_may_move_objects<'a>( + space_fields: &[&'a Field], + parent_field: &Option<&'a Field>, + ty_generics: &TypeGenerics, +) -> TokenStream2 { + // If any space or the parent may move objects, the plan may move objects + let space_handlers = space_fields.iter().map(|f| { + let ref f_ty = f.ty; + + quote! { + || <#f_ty as PolicyTraceObject #ty_generics>::may_move_objects::() + } + }); + + let parent_handler = if let Some(p) = parent_field { + let ref p_ty = p.ty; + + quote! { + || <#p_ty as PlanTraceObject #ty_generics>::may_move_objects::() + } + } else { + TokenStream2::new() + }; + + quote! { + #[inline(always)] + fn may_move_objects() -> bool { + use crate::policy::gc_work::PolicyTraceObject; + use crate::plan::PlanTraceObject; + + false #(#space_handlers)* #parent_handler + } + } +} diff --git a/macros/src/util.rs b/macros/src/util.rs new file mode 100644 index 0000000000..ed6f53d880 --- /dev/null +++ b/macros/src/util.rs @@ -0,0 +1,45 @@ +use proc_macro_error::abort; +use syn::{spanned::Spanned, Attribute, Field, FieldsNamed}; + +pub fn get_field_attribute<'f>(field: &'f Field, attr_name: &str) -> Option<&'f Attribute> { + let attrs = field + .attrs + .iter() + .filter(|a| a.path.is_ident(attr_name)) + .collect::>(); + if attrs.len() > 1 { + let second_attr = attrs.get(1).unwrap(); + abort! { second_attr.path.span(), "Duplicated attribute: #[{}]", attr_name } + }; + + attrs.get(0).cloned() +} + +pub fn get_fields_with_attribute<'f>(fields: &'f FieldsNamed, attr_name: &str) -> Vec<&'f Field> { + fields + .named + .iter() + .filter(|f| get_field_attribute(f, attr_name).is_some()) + .collect::>() +} + +pub fn get_unique_field_with_attribute<'f>( + fields: &'f FieldsNamed, + attr_name: &str, +) -> Option<&'f Field> { + let mut result = None; + + for field in fields.named.iter() { + if let Some(attr) = get_field_attribute(field, attr_name) { + if result.is_none() { + result = Some(field); + continue; + } else { + let span = attr.path.span(); + abort! { span, "At most one field in a struct can have the #[{}] attribute.", attr_name }; + } + } + } + + result +} diff --git a/src/plan/generational/copying/gc_work.rs b/src/plan/generational/copying/gc_work.rs index 867b09b1fc..d8f245d3fd 100644 --- a/src/plan/generational/copying/gc_work.rs +++ b/src/plan/generational/copying/gc_work.rs @@ -1,10 +1,20 @@ use super::global::GenCopy; -use crate::scheduler::gc_work::SFTProcessEdges; +use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::vm::*; +use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; + +pub struct GenCopyNurseryGCWorkContext(std::marker::PhantomData); +impl crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkContext { + type VM = VM; + type PlanType = GenCopy; + type ProcessEdgesWorkType = GenNurseryProcessEdges; +} + pub struct GenCopyGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for GenCopyGCWorkContext { type VM = VM; type PlanType = GenCopy; - type ProcessEdgesWorkType = SFTProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/generational/copying/global.rs b/src/plan/generational/copying/global.rs index 9ce19e2d79..3384760827 100644 --- a/src/plan/generational/copying/global.rs +++ b/src/plan/generational/copying/global.rs @@ -1,4 +1,5 @@ use super::gc_work::GenCopyGCWorkContext; +use super::gc_work::GenCopyNurseryGCWorkContext; use super::mutator::ALLOCATOR_MAPPING; use crate::plan::generational::global::Gen; use crate::plan::global::BasePlan; @@ -25,10 +26,16 @@ use enum_map::EnumMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + +#[derive(PlanTraceObject)] pub struct GenCopy { + #[fallback_trace] pub gen: Gen, pub hi: AtomicBool, + #[trace(CopySemantics::Mature)] pub copyspace0: CopySpace, + #[trace(CopySemantics::Mature)] pub copyspace1: CopySpace, } @@ -79,10 +86,14 @@ impl Plan for GenCopy { } fn schedule_collection(&'static self, scheduler: &GCWorkScheduler) { - let _ = self.request_full_heap_collection(); + let is_full_heap = self.request_full_heap_collection(); self.base().set_collection_kind::(self); self.base().set_gc_status(GcStatus::GcPrepare); - scheduler.schedule_common_work::>(self); + if is_full_heap { + scheduler.schedule_common_work::>(self); + } else { + scheduler.schedule_common_work::>(self); + } } fn get_allocator_mapping(&self) -> &'static EnumMap { diff --git a/src/plan/generational/global.rs b/src/plan/generational/global.rs index cbd15b9cc6..e2ffe4a3b1 100644 --- a/src/plan/generational/global.rs +++ b/src/plan/generational/global.rs @@ -21,12 +21,17 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + /// Common implementation for generational plans. Each generational plan /// should include this type, and forward calls to it where possible. +#[derive(PlanTraceObject)] pub struct Gen { - /// The nursery space. Its type depends on the actual plan. + /// The nursery space. + #[trace(CopySemantics::PromoteToMature)] pub nursery: CopySpace, /// The common plan. + #[fallback_trace] pub common: CommonPlan, /// Is this GC full heap? pub gc_full_heap: AtomicBool, @@ -160,6 +165,15 @@ impl Gen { self.gc_full_heap.store(is_full_heap, Ordering::SeqCst); + info!( + "{}", + if is_full_heap { + "Full heap GC" + } else { + "nursery GC" + } + ); + is_full_heap } diff --git a/src/plan/generational/immix/gc_work.rs b/src/plan/generational/immix/gc_work.rs index 1ed773c328..1b4af605e0 100644 --- a/src/plan/generational/immix/gc_work.rs +++ b/src/plan/generational/immix/gc_work.rs @@ -1,89 +1,8 @@ use super::global::GenImmix; use crate::plan::generational::gc_work::GenNurseryProcessEdges; -use crate::policy::space::Space; -use crate::scheduler::gc_work::*; -use crate::util::{Address, ObjectReference}; -use crate::vm::*; -use crate::MMTK; -use std::ops::{Deref, DerefMut}; - -use crate::plan::immix::gc_work::{TraceKind, TRACE_KIND_FAST}; - -/// ProcessEdges for a full heap GC for generational immix. The const type parameter -/// defines whether there is copying in the GC. -/// Note that even with TraceKind::Fast, there is no defragmentation, we are still -/// copying from nursery to immix space. So we always need to write new object -/// references in process_edge() (i.e. we do not need to overwrite the default implementation -/// of process_edge() as the immix plan does). -pub(super) struct GenImmixMatureProcessEdges { - plan: &'static GenImmix, - base: ProcessEdgesBase, -} - -impl ProcessEdgesWork - for GenImmixMatureProcessEdges -{ - type VM = VM; - - fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); - let plan = base.plan().downcast_ref::>().unwrap(); - Self { plan, base } - } - - #[cold] - fn flush(&mut self) { - if self.nodes.is_empty() { - return; - } - - let scan_objects_work = crate::policy::immix::ScanObjectsAndMarkLines::::new( - self.pop_nodes(), - false, - &self.plan.immix, - ); - self.new_scan_work(scan_objects_work); - } - - #[inline] - fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { - if object.is_null() { - return object; - } - - if self.plan.immix.in_space(object) { - if KIND == TRACE_KIND_FAST { - return self.plan.immix.fast_trace_object(self, object); - } else { - return self.plan.immix.trace_object( - self, - object, - crate::util::copy::CopySemantics::Mature, - self.worker(), - ); - } - } - - self.plan - .gen - .trace_object_full_heap::(self, object, self.worker()) - } -} - -impl Deref for GenImmixMatureProcessEdges { - type Target = ProcessEdgesBase; - #[inline] - fn deref(&self) -> &Self::Target { - &self.base - } -} - -impl DerefMut for GenImmixMatureProcessEdges { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.base - } -} +use crate::policy::gc_work::TraceKind; +use crate::scheduler::gc_work::PlanProcessEdges; +use crate::vm::VMBinding; pub struct GenImmixNurseryGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkContext { @@ -100,5 +19,5 @@ impl crate::scheduler::GCWorkContext { type VM = VM; type PlanType = GenImmix; - type ProcessEdgesWorkType = GenImmixMatureProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, KIND>; } diff --git a/src/plan/generational/immix/global.rs b/src/plan/generational/immix/global.rs index 04c5fb63fe..eb0714adf3 100644 --- a/src/plan/generational/immix/global.rs +++ b/src/plan/generational/immix/global.rs @@ -4,11 +4,11 @@ use crate::plan::generational::global::Gen; use crate::plan::global::BasePlan; use crate::plan::global::CommonPlan; use crate::plan::global::GcStatus; -use crate::plan::immix::gc_work::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST}; use crate::plan::AllocationSemantics; use crate::plan::Plan; use crate::plan::PlanConstraints; use crate::policy::immix::ImmixSpace; +use crate::policy::immix::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST}; use crate::policy::space::Space; use crate::scheduler::GCWorkScheduler; use crate::util::alloc::allocators::AllocatorSelector; @@ -26,14 +26,20 @@ use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + /// Generational immix. This implements the functionality of a two-generation copying /// collector where the higher generation is an immix space. /// See the PLDI'08 paper by Blackburn and McKinley for a description /// of the algorithm: http://doi.acm.org/10.1145/1375581.137558. +#[derive(PlanTraceObject)] pub struct GenImmix { /// Generational plan, which includes a nursery space and operations related with nursery. + #[fallback_trace] pub gen: Gen, /// An immix space as the mature space. + #[post_scan] + #[trace(CopySemantics::Mature)] pub immix: ImmixSpace, /// Whether the last GC was a defrag GC for the immix space. pub last_gc_was_defrag: AtomicBool, diff --git a/src/plan/global.rs b/src/plan/global.rs index 2c990cd115..515023d576 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -33,6 +33,8 @@ use enum_map::EnumMap; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use mmtk_macros::PlanTraceObject; + pub fn create_mutator( tls: VMMutatorThread, mmtk: &'static MMTK, @@ -338,6 +340,7 @@ pub enum GcStatus { /** BasePlan should contain all plan-related state and functions that are _fundamental_ to _all_ plans. These include VM-specific (but not plan-specific) features such as a code space or vm space, which are fundamental to all plans for a given VM. Features that are common to _many_ (but not intrinsically _all_) plans should instead be included in CommonPlan. */ +#[derive(PlanTraceObject)] pub struct BasePlan { /// Whether MMTk is now ready for collection. This is set to true when initialize_collection() is called. pub initialized: AtomicBool, @@ -377,10 +380,13 @@ pub struct BasePlan { // Spaces in base plan #[cfg(feature = "code_space")] + #[trace] pub code_space: ImmortalSpace, #[cfg(feature = "code_space")] + #[trace] pub code_lo_space: ImmortalSpace, #[cfg(feature = "ro_space")] + #[trace] pub ro_space: ImmortalSpace, /// A VM space is a space allocated and populated by the VM. Currently it is used by JikesRVM @@ -396,6 +402,7 @@ pub struct BasePlan { /// - The `is_in_mmtk_spaces` currently returns `true` if the given object reference is in /// the VM space. #[cfg(feature = "vm_space")] + #[trace] pub vm_space: ImmortalSpace, } @@ -834,9 +841,13 @@ impl BasePlan { /** CommonPlan is for representing state and features used by _many_ plans, but that are not fundamental to _all_ plans. Examples include the Large Object Space and an Immortal space. Features that are fundamental to _all_ plans must be included in BasePlan. */ +#[derive(PlanTraceObject)] pub struct CommonPlan { + #[trace] pub immortal: ImmortalSpace, + #[trace] pub los: LargeObjectSpace, + #[fallback_trace] pub base: BasePlan, } @@ -945,6 +956,43 @@ impl CommonPlan { } } +use crate::policy::gc_work::TraceKind; +use crate::vm::VMBinding; + +/// A plan that uses `PlanProcessEdges` needs to provide an implementation for this trait. +/// Generally a plan does not need to manually implement this trait. Instead, we provide +/// a procedural macro that helps generate an implementation. Please check `macros/trace_object`. +/// +/// A plan could also manually implement this trait. For the sake of performance, the implementation +/// of this trait should mark methods as `[inline(always)]`. +pub trait PlanTraceObject { + /// Trace objects in the plan. Generally one needs to figure out + /// which space an object resides in, and invokes the corresponding policy + /// trace object method. + /// + /// Arguments: + /// * `trace`: the current transitive closure + /// * `object`: the object to trace. This is a non-nullable object reference. + /// * `worker`: the GC worker that is tracing this object. + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + worker: &mut GCWorker, + ) -> ObjectReference; + + /// Post-scan objects in the plan. Each object is scanned by `VM::VMScanning::scan_object()`, and this function + /// will be called after the `VM::VMScanning::scan_object()` as a hook to invoke possible policy post scan method. + /// If a plan does not have any policy that needs post scan, this method can be implemented as empty. + /// If a plan has a policy that has some policy specific behaviors for scanning (e.g. mark lines in Immix), + /// this method should also invoke those policy specific methods for objects in that space. + fn post_scan_object(&self, object: ObjectReference); + + /// Whether objects in this plan may move. If any of the spaces used by the plan may move objects, this should + /// return true. + fn may_move_objects() -> bool; +} + use enum_map::Enum; /// Allocation semantics that MMTk provides. /// Each allocation request requires a desired semantic for the object to allocate. diff --git a/src/plan/immix/gc_work.rs b/src/plan/immix/gc_work.rs index 903b579ff1..0696086fc1 100644 --- a/src/plan/immix/gc_work.rs +++ b/src/plan/immix/gc_work.rs @@ -1,108 +1,7 @@ use super::global::Immix; -use crate::policy::space::Space; -use crate::scheduler::gc_work::*; -use crate::util::copy::CopySemantics; -use crate::util::{Address, ObjectReference}; +use crate::policy::gc_work::TraceKind; +use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; -use crate::MMTK; -use std::ops::{Deref, DerefMut}; - -// It would be better if we use an enum for this. However, we use this as -// a constant type parameter, and Rust only accepts integer and bool for -// constant type parameters for now. We need to wait until `adt_const_params` is -// stablized. -pub(in crate::plan) type TraceKind = u8; -pub(in crate::plan) const TRACE_KIND_FAST: TraceKind = 0; -pub(in crate::plan) const TRACE_KIND_DEFRAG: TraceKind = 1; - -/// Object tracing for Immix. -/// Note that it is possible to use [`SFTProcessEdges`](mmtk/scheduler/gc_work/SFTProcessEdges) for immix. -/// We need to: 1. add a plan-specific method to create scan work packets, as most plans use `ScanObjects` while -/// immix uses `ScanObjectsAndMarkLines`, 2. use `ImmixSpace.trace_object()` which has an overhead of checking -/// which trace method to use (with ImmixProcessEdges, we can know which trace method to use by statically checking `TraceKind`). -pub(super) struct ImmixProcessEdges { - // Use a static ref to the specific plan to avoid overhead from dynamic dispatch or - // downcast for each traced object. - plan: &'static Immix, - base: ProcessEdgesBase, -} - -impl ImmixProcessEdges { - fn immix(&self) -> &'static Immix { - self.plan - } -} - -impl ProcessEdgesWork for ImmixProcessEdges { - type VM = VM; - - const OVERWRITE_REFERENCE: bool = crate::policy::immix::DEFRAG; - - fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); - let plan = base.plan().downcast_ref::>().unwrap(); - Self { plan, base } - } - - #[cold] - fn flush(&mut self) { - if self.nodes.is_empty() { - return; - } - let scan_objects_work = crate::policy::immix::ScanObjectsAndMarkLines::::new( - self.pop_nodes(), - false, - &self.immix().immix_space, - ); - self.new_scan_work(scan_objects_work); - } - - /// Trace and evacuate objects. - #[inline(always)] - fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { - if object.is_null() { - return object; - } - if self.immix().immix_space.in_space(object) { - if KIND == TRACE_KIND_FAST { - self.immix().immix_space.fast_trace_object(self, object) - } else { - self.immix().immix_space.trace_object( - self, - object, - CopySemantics::DefaultCopy, - self.worker(), - ) - } - } else { - self.immix().common.trace_object::(self, object) - } - } - - #[inline] - fn process_edge(&mut self, slot: Address) { - let object = unsafe { slot.load::() }; - let new_object = self.trace_object(object); - if KIND == TRACE_KIND_DEFRAG && Self::OVERWRITE_REFERENCE { - unsafe { slot.store(new_object) }; - } - } -} - -impl Deref for ImmixProcessEdges { - type Target = ProcessEdgesBase; - #[inline] - fn deref(&self) -> &Self::Target { - &self.base - } -} - -impl DerefMut for ImmixProcessEdges { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.base - } -} pub(super) struct ImmixGCWorkContext( std::marker::PhantomData, @@ -112,5 +11,5 @@ impl crate::scheduler::GCWorkContext { type VM = VM; type PlanType = Immix; - type ProcessEdgesWorkType = ImmixProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, KIND>; } diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index 78008cf3bc..2d40d32a8b 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -1,5 +1,4 @@ use super::gc_work::ImmixGCWorkContext; -use super::gc_work::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST}; use super::mutator::ALLOCATOR_MAPPING; use crate::plan::global::BasePlan; use crate::plan::global::CommonPlan; @@ -7,6 +6,7 @@ use crate::plan::global::GcStatus; use crate::plan::AllocationSemantics; use crate::plan::Plan; use crate::plan::PlanConstraints; +use crate::policy::immix::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST}; use crate::policy::space::Space; use crate::scheduler::*; use crate::util::alloc::allocators::AllocatorSelector; @@ -26,8 +26,14 @@ use std::sync::Arc; use atomic::Ordering; use enum_map::EnumMap; +use mmtk_macros::PlanTraceObject; + +#[derive(PlanTraceObject)] pub struct Immix { + #[post_scan] + #[trace(CopySemantics::DefaultCopy)] pub immix_space: ImmixSpace, + #[fallback_trace] pub common: CommonPlan, last_gc_was_defrag: AtomicBool, } diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index 848fdd2d8e..d7f76df34e 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -1,17 +1,16 @@ use super::global::MarkCompact; use crate::policy::markcompactspace::MarkCompactSpace; -use crate::policy::space::Space; +use crate::policy::markcompactspace::{TRACE_KIND_FORWARD, TRACE_KIND_MARK}; +use crate::scheduler::gc_work::PlanProcessEdges; use crate::scheduler::gc_work::*; use crate::scheduler::GCWork; use crate::scheduler::GCWorker; use crate::scheduler::WorkBucketStage; -use crate::util::{Address, ObjectReference}; use crate::vm::ActivePlan; use crate::vm::Scanning; use crate::vm::VMBinding; use crate::MMTK; use std::marker::PhantomData; -use std::ops::{Deref, DerefMut}; /// iterate through the heap and calculate the new location of live objects pub struct CalculateForwardingAddress { @@ -83,105 +82,10 @@ impl Compact { } } -// Transitive closure to mark live objects -pub struct MarkingProcessEdges { - plan: &'static MarkCompact, - base: ProcessEdgesBase, -} - -impl MarkingProcessEdges { - fn markcompact(&self) -> &'static MarkCompact { - self.plan - } -} - -impl ProcessEdgesWork for MarkingProcessEdges { - type VM = VM; - fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); - let plan = base.plan().downcast_ref::>().unwrap(); - Self { base, plan } - } - - #[inline] - fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { - if object.is_null() { - return object; - } - if self.markcompact().mc_space().in_space(object) { - self.markcompact() - .mc_space() - .trace_mark_object::(self, object) - } else { - self.markcompact().common.trace_object::(self, object) - } - } -} - -impl Deref for MarkingProcessEdges { - type Target = ProcessEdgesBase; - #[inline] - fn deref(&self) -> &Self::Target { - &self.base - } -} - -impl DerefMut for MarkingProcessEdges { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.base - } -} - -/// Transitive closure to update object references -pub struct ForwardingProcessEdges { - plan: &'static MarkCompact, - base: ProcessEdgesBase, -} - -impl ForwardingProcessEdges { - fn markcompact(&self) -> &'static MarkCompact { - self.plan - } -} - -impl ProcessEdgesWork for ForwardingProcessEdges { - type VM = VM; - fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); - let plan = base.plan().downcast_ref::>().unwrap(); - Self { base, plan } - } - - #[inline] - fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { - if object.is_null() { - return object; - } - if self.markcompact().mc_space().in_space(object) { - self.markcompact() - .mc_space() - .trace_forward_object::(self, object) - } else { - self.markcompact().common.trace_object::(self, object) - } - } -} - -impl Deref for ForwardingProcessEdges { - type Target = ProcessEdgesBase; - #[inline] - fn deref(&self) -> &Self::Target { - &self.base - } -} - -impl DerefMut for ForwardingProcessEdges { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.base - } -} +/// Marking trace +pub type MarkingProcessEdges = PlanProcessEdges, TRACE_KIND_MARK>; +/// Forwarding trace +pub type ForwardingProcessEdges = PlanProcessEdges, TRACE_KIND_FORWARD>; pub struct MarkCompactGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for MarkCompactGCWorkContext { diff --git a/src/plan/markcompact/global.rs b/src/plan/markcompact/global.rs index fe521cdb42..c7e3f451d8 100644 --- a/src/plan/markcompact/global.rs +++ b/src/plan/markcompact/global.rs @@ -17,6 +17,7 @@ use crate::scheduler::*; use crate::util::alloc::allocators::AllocatorSelector; #[cfg(not(feature = "global_alloc_bit"))] use crate::util::alloc_bit::ALLOC_SIDE_METADATA_SPEC; +use crate::util::copy::CopySemantics; use crate::util::heap::layout::heap_layout::Mmapper; use crate::util::heap::layout::heap_layout::VMMap; use crate::util::heap::layout::vm_layout_constants::{HEAP_END, HEAP_START}; @@ -26,11 +27,17 @@ use crate::util::metadata::side_metadata::{SideMetadataContext, SideMetadataSani use crate::util::opaque_pointer::*; use crate::util::options::UnsafeOptionsWrapper; use crate::vm::VMBinding; + use enum_map::EnumMap; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + +#[derive(PlanTraceObject)] pub struct MarkCompact { + #[trace(CopySemantics::DefaultCopy)] pub mc_space: MarkCompactSpace, + #[fallback_trace] pub common: CommonPlan, } diff --git a/src/plan/marksweep/gc_work.rs b/src/plan/marksweep/gc_work.rs index 6665d1046b..01933edebd 100644 --- a/src/plan/marksweep/gc_work.rs +++ b/src/plan/marksweep/gc_work.rs @@ -68,10 +68,12 @@ impl GCWork for MSSweepChunks { } } -use crate::scheduler::gc_work::SFTProcessEdges; pub struct MSGCWorkContext(std::marker::PhantomData); +use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; + impl crate::scheduler::GCWorkContext for MSGCWorkContext { type VM = VM; type PlanType = MarkSweep; - type ProcessEdgesWorkType = SFTProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/marksweep/global.rs b/src/plan/marksweep/global.rs index 1bf8ef0611..c06ac5f5c8 100644 --- a/src/plan/marksweep/global.rs +++ b/src/plan/marksweep/global.rs @@ -25,8 +25,13 @@ use std::sync::Arc; use enum_map::EnumMap; +use mmtk_macros::PlanTraceObject; + +#[derive(PlanTraceObject)] pub struct MarkSweep { + #[fallback_trace] common: CommonPlan, + #[trace] ms: MallocSpace, } diff --git a/src/plan/mod.rs b/src/plan/mod.rs index 3abc62916e..cd667c7c99 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -25,6 +25,7 @@ pub(crate) use global::create_plan; pub use global::AllocationSemantics; pub(crate) use global::GcStatus; pub use global::Plan; +pub(crate) use global::PlanTraceObject; mod mutator_context; pub use mutator_context::Mutator; diff --git a/src/plan/pageprotect/gc_work.rs b/src/plan/pageprotect/gc_work.rs index c95d9e6f21..c962eae190 100644 --- a/src/plan/pageprotect/gc_work.rs +++ b/src/plan/pageprotect/gc_work.rs @@ -1,10 +1,11 @@ use super::global::PageProtect; -use crate::scheduler::gc_work::SFTProcessEdges; +use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; pub struct PPGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for PPGCWorkContext { type VM = VM; type PlanType = PageProtect; - type ProcessEdgesWorkType = SFTProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/pageprotect/global.rs b/src/plan/pageprotect/global.rs index 08ecff151e..6f5b8b47e2 100644 --- a/src/plan/pageprotect/global.rs +++ b/src/plan/pageprotect/global.rs @@ -22,7 +22,11 @@ use crate::{ use enum_map::EnumMap; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + +#[derive(PlanTraceObject)] pub struct PageProtect { + #[trace] pub space: LargeObjectSpace, pub common: CommonPlan, } diff --git a/src/plan/semispace/gc_work.rs b/src/plan/semispace/gc_work.rs index 0868e61b7f..9f02c73eea 100644 --- a/src/plan/semispace/gc_work.rs +++ b/src/plan/semispace/gc_work.rs @@ -1,10 +1,11 @@ use super::global::SemiSpace; -use crate::scheduler::gc_work::SFTProcessEdges; +use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; pub struct SSGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for SSGCWorkContext { type VM = VM; type PlanType = SemiSpace; - type ProcessEdgesWorkType = SFTProcessEdges; + type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/semispace/global.rs b/src/plan/semispace/global.rs index 4de2526874..6b8db36328 100644 --- a/src/plan/semispace/global.rs +++ b/src/plan/semispace/global.rs @@ -22,12 +22,18 @@ use crate::{plan::global::BasePlan, vm::VMBinding}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use mmtk_macros::PlanTraceObject; + use enum_map::EnumMap; +#[derive(PlanTraceObject)] pub struct SemiSpace { pub hi: AtomicBool, + #[trace(CopySemantics::DefaultCopy)] pub copyspace0: CopySpace, + #[trace(CopySemantics::DefaultCopy)] pub copyspace1: CopySpace, + #[fallback_trace] pub common: CommonPlan, } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 7bdec29a17..006aac523b 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -108,6 +108,24 @@ impl Space for CopySpace { } } +impl crate::policy::gc_work::PolicyTraceObject for CopySpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + copy: Option, + worker: &mut GCWorker, + ) -> ObjectReference { + self.trace_object(trace, object, copy, worker) + } + + #[inline(always)] + fn may_move_objects() -> bool { + true + } +} + impl CopySpace { #[allow(clippy::too_many_arguments)] pub fn new( @@ -213,7 +231,6 @@ impl CopySpace { // If this is not from space, we do not need to trace it (the object has been copied to the tosapce) if !self.is_from_space() { // The copy semantics for tospace should be none. - debug_assert!(semantics.is_none()); return object; } diff --git a/src/policy/gc_work.rs b/src/policy/gc_work.rs new file mode 100644 index 0000000000..8520c437bd --- /dev/null +++ b/src/policy/gc_work.rs @@ -0,0 +1,40 @@ +/// Used to identify the trace if a policy has different kinds of traces. For example, defrag vs fast trace for Immix, +/// mark vs forward trace for mark compact. +pub(crate) type TraceKind = u8; + +pub const DEFAULT_TRACE: u8 = u8::MAX; + +use crate::plan::TransitiveClosure; +use crate::scheduler::GCWorker; +use crate::util::copy::CopySemantics; + +use crate::util::ObjectReference; + +use crate::vm::VMBinding; + +/// This trait defines policy-specific behavior for tracing objects. +/// The procedural macro #[derive(PlanTraceObject)] will generate code +/// that uses this trait. We expect any policy to implement this trait. +/// For the sake of performance, the implementation +/// of this trait should mark methods as `[inline(always)]`. +pub trait PolicyTraceObject { + /// Trace object in the policy. If the policy copies objects, we should + /// expect `copy` to be a `Some` value. + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + copy: Option, + worker: &mut GCWorker, + ) -> ObjectReference; + + /// Policy-specific post-scan-object hook. It is called after scanning + /// each object in this space. + #[inline(always)] + fn post_scan_object(&self, _object: ObjectReference) { + // Do nothing. + } + + /// Return whether the policy moves objects. + fn may_move_objects() -> bool; +} diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index b550a60a0b..a40df63cb5 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -4,7 +4,7 @@ use super::{ chunk::{Chunk, ChunkMap, ChunkState}, defrag::Defrag, }; -use crate::plan::ObjectsClosure; +use crate::policy::gc_work::TraceKind; use crate::policy::space::SpaceOptions; use crate::policy::space::*; use crate::policy::space::{CommonSpace, Space, SFT}; @@ -23,7 +23,7 @@ use crate::util::{Address, ObjectReference}; use crate::vm::*; use crate::{ plan::TransitiveClosure, - scheduler::{gc_work::ProcessEdgesWork, GCWork, GCWorkScheduler, GCWorker, WorkBucketStage}, + scheduler::{GCWork, GCWorkScheduler, GCWorker, WorkBucketStage}, util::{ heap::FreeListPageResource, opaque_pointer::{VMThread, VMWorkerThread}, @@ -33,6 +33,9 @@ use crate::{ use atomic::Ordering; use std::sync::{atomic::AtomicU8, Arc}; +pub(crate) const TRACE_KIND_FAST: TraceKind = 0; +pub(crate) const TRACE_KIND_DEFRAG: TraceKind = 1; + pub struct ImmixSpace { common: CommonSpace, pr: FreeListPageResource, @@ -113,6 +116,44 @@ impl Space for ImmixSpace { } } +impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + copy: Option, + worker: &mut GCWorker, + ) -> ObjectReference { + if KIND == TRACE_KIND_DEFRAG { + self.trace_object(trace, object, copy.unwrap(), worker) + } else if KIND == TRACE_KIND_FAST { + self.fast_trace_object(trace, object) + } else { + unreachable!() + } + } + + #[inline(always)] + fn post_scan_object(&self, object: ObjectReference) { + if super::MARK_LINE_AT_SCAN_TIME && !super::BLOCK_ONLY { + debug_assert!(self.in_space(object)); + self.mark_lines(object); + } + } + + #[inline(always)] + fn may_move_objects() -> bool { + if KIND == TRACE_KIND_DEFRAG { + true + } else if KIND == TRACE_KIND_FAST { + false + } else { + unreachable!() + } + } +} + impl ImmixSpace { const UNMARKED_STATE: u8 = 0; const MARKED_STATE: u8 = 1; @@ -602,45 +643,6 @@ impl GCWork for PrepareBlockState { } } -/// A work packet to scan the fields of each objects and mark lines. -pub struct ScanObjectsAndMarkLines { - buffer: Vec, - #[allow(unused)] - concurrent: bool, - immix_space: &'static ImmixSpace, -} - -impl ScanObjectsAndMarkLines { - pub fn new( - buffer: Vec, - concurrent: bool, - immix_space: &'static ImmixSpace, - ) -> Self { - Self { - buffer, - concurrent, - immix_space, - } - } -} - -impl GCWork for ScanObjectsAndMarkLines { - fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { - trace!("ScanObjectsAndMarkLines"); - let tls = worker.tls; - let mut closure = ObjectsClosure::::new(worker); - for object in &self.buffer { - ::VMScanning::scan_object(tls, *object, &mut closure); - if super::MARK_LINE_AT_SCAN_TIME - && !super::BLOCK_ONLY - && self.immix_space.in_space(*object) - { - self.immix_space.mark_lines(*object); - } - } - } -} - use crate::plan::Plan; use crate::policy::copy_context::PolicyCopyContext; use crate::util::alloc::Allocator; diff --git a/src/policy/immortalspace.rs b/src/policy/immortalspace.rs index fbc344ff70..214358b597 100644 --- a/src/policy/immortalspace.rs +++ b/src/policy/immortalspace.rs @@ -111,6 +111,26 @@ impl Space for ImmortalSpace { } } +use crate::scheduler::GCWorker; +use crate::util::copy::CopySemantics; + +impl crate::policy::gc_work::PolicyTraceObject for ImmortalSpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + _copy: Option, + _worker: &mut GCWorker, + ) -> ObjectReference { + self.trace_object(trace, object) + } + #[inline(always)] + fn may_move_objects() -> bool { + false + } +} + impl ImmortalSpace { #[allow(clippy::too_many_arguments)] pub fn new( diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index 0884efeb97..dc016d036c 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -115,6 +115,26 @@ impl Space for LargeObjectSpace { } } +use crate::scheduler::GCWorker; +use crate::util::copy::CopySemantics; + +impl crate::policy::gc_work::PolicyTraceObject for LargeObjectSpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + _copy: Option, + _worker: &mut GCWorker, + ) -> ObjectReference { + self.trace_object(trace, object) + } + #[inline(always)] + fn may_move_objects() -> bool { + false + } +} + impl LargeObjectSpace { #[allow(clippy::too_many_arguments)] pub fn new( diff --git a/src/policy/lockfreeimmortalspace.rs b/src/policy/lockfreeimmortalspace.rs index 0c6ce2d2b1..924be8a294 100644 --- a/src/policy/lockfreeimmortalspace.rs +++ b/src/policy/lockfreeimmortalspace.rs @@ -149,6 +149,27 @@ impl Space for LockFreeImmortalSpace { } } +use crate::plan::TransitiveClosure; +use crate::scheduler::GCWorker; +use crate::util::copy::CopySemantics; + +impl crate::policy::gc_work::PolicyTraceObject for LockFreeImmortalSpace { + #[inline(always)] + fn trace_object( + &self, + _trace: &mut T, + _object: ObjectReference, + _copy: Option, + _worker: &mut GCWorker, + ) -> ObjectReference { + unreachable!() + } + #[inline(always)] + fn may_move_objects() -> bool { + unreachable!() + } +} + impl LockFreeImmortalSpace { #[allow(dead_code)] // Only used with certain features. pub fn new( diff --git a/src/policy/mallocspace/global.rs b/src/policy/mallocspace/global.rs index 136a212ed8..e36318a46a 100644 --- a/src/policy/mallocspace/global.rs +++ b/src/policy/mallocspace/global.rs @@ -189,6 +189,27 @@ impl Space for MallocSpace { } } +use crate::scheduler::GCWorker; +use crate::util::copy::CopySemantics; + +impl crate::policy::gc_work::PolicyTraceObject for MallocSpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + _copy: Option, + _worker: &mut GCWorker, + ) -> ObjectReference { + self.trace_object(trace, object) + } + + #[inline(always)] + fn may_move_objects() -> bool { + false + } +} + impl MallocSpace { pub fn new(global_side_metadata_specs: Vec) -> Self { MallocSpace { diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index 82bdf269a8..b7c8ccec3f 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -1,7 +1,10 @@ use super::space::{CommonSpace, Space, SpaceOptions, SFT}; +use crate::policy::gc_work::TraceKind; use crate::policy::space::*; +use crate::scheduler::GCWorker; use crate::util::alloc::allocator::align_allocation_no_fill; use crate::util::constants::LOG_BYTES_IN_WORD; +use crate::util::copy::CopySemantics; use crate::util::heap::layout::heap_layout::{Mmapper, VMMap}; use crate::util::heap::{HeapMeta, MonotonePageResource, PageResource, VMRequest}; use crate::util::metadata::load_metadata; @@ -11,6 +14,9 @@ use crate::util::{alloc_bit, Address, ObjectReference}; use crate::{vm::*, TransitiveClosure}; use atomic::Ordering; +pub(crate) const TRACE_KIND_MARK: TraceKind = 0; +pub(crate) const TRACE_KIND_FORWARD: TraceKind = 1; + pub struct MarkCompactSpace { common: CommonSpace, pr: MonotonePageResource, @@ -97,6 +103,35 @@ impl Space for MarkCompactSpace { } } +impl crate::policy::gc_work::PolicyTraceObject for MarkCompactSpace { + #[inline(always)] + fn trace_object( + &self, + trace: &mut T, + object: ObjectReference, + _copy: Option, + _worker: &mut GCWorker, + ) -> ObjectReference { + if KIND == TRACE_KIND_MARK { + self.trace_mark_object(trace, object) + } else if KIND == TRACE_KIND_FORWARD { + self.trace_forward_object(trace, object) + } else { + unreachable!() + } + } + #[inline(always)] + fn may_move_objects() -> bool { + if KIND == TRACE_KIND_MARK { + false + } else if KIND == TRACE_KIND_FORWARD { + true + } else { + unreachable!() + } + } +} + impl MarkCompactSpace { /// We need one extra header word for each object. Considering the alignment requirement, this is /// the actual bytes we need to reserve for each allocation. diff --git a/src/policy/mod.rs b/src/policy/mod.rs index 1b6d5f6f5b..3675a294ac 100644 --- a/src/policy/mod.rs +++ b/src/policy/mod.rs @@ -15,6 +15,8 @@ pub mod space; /// Copy context defines the thread local copy allocator for copying policies. pub mod copy_context; +/// Policy specific GC work +pub mod gc_work; pub mod copyspace; pub mod immix; diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index e20a8c6c62..71b137faa0 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -421,22 +421,31 @@ pub trait ProcessEdgesWork: // So maximum 1 `ScanObjects` work can be created from `nodes` buffer } - /// Create a new scan work packet. If SCAN_OBJECTS_IMMEDIATELY, the work packet will be executed immediately, in this method. + /// Start the a scan work packet. If SCAN_OBJECTS_IMMEDIATELY, the work packet will be executed immediately, in this method. /// Otherwise, the work packet will be added the Closure work bucket and will be dispatched later by the scheduler. #[inline] - fn new_scan_work(&mut self, work_packet: impl GCWork) { + fn start_or_dispatch_scan_work(&mut self, work_packet: Box>) { if Self::SCAN_OBJECTS_IMMEDIATELY { // We execute this `scan_objects_work` immediately. // This is expected to be a useful optimization because, // say for _pmd_ with 200M heap, we're likely to have 50000~60000 `ScanObjects` work packets // being dispatched (similar amount to `ProcessEdgesWork`). // Executing these work packets now can remarkably reduce the global synchronization time. - self.worker().do_work(work_packet); + self.worker().do_boxed_work(work_packet); } else { - self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(work_packet); + self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add_boxed(work_packet); } } + /// Create scan work for the policy. By default, we use [`ScanObjects`](crate::scheduler::gc_work::ScanObjects). + /// If a policy has its own scan object work packet, they can override this method. + #[inline(always)] + fn create_scan_work(&self, nodes: Vec) -> Box> { + Box::new(crate::scheduler::gc_work::ScanObjects::::new( + nodes, false, + )) + } + /// Flush the nodes in ProcessEdgesBase, and create a ScanObjects work packet for it. If the node set is empty, /// this method will simply return with no work packet created. #[cold] @@ -444,8 +453,8 @@ pub trait ProcessEdgesWork: if self.nodes.is_empty() { return; } - let scan_objects_work = ScanObjects::::new(self.pop_nodes(), false); - self.new_scan_work(scan_objects_work); + let nodes = self.pop_nodes(); + self.start_or_dispatch_scan_work(self.create_scan_work(nodes)); } #[inline] @@ -488,6 +497,7 @@ impl GCWork for E { /// (such as `Space.set_copy_for_sft_trace()`, `SFT.sft_trace_object()`). /// Some plans are not using this type, mostly due to more complex tracing. Either it is impossible to use this type, or /// there is performance overheads for using this general trace type. In such cases, they implement their specific process edges. +// TODO: This is not used any more. Should we remove it? pub struct SFTProcessEdges { pub base: ProcessEdgesBase, } @@ -536,7 +546,10 @@ impl DerefMut for SFTProcessEdges { } } -/// Scan & update a list of object slots +/// Scan & update a list of object slots. +/// Note that this work packet does not do any policy-specific scan +/// object work (it won't call `scan_object()` in [`policy::gc_work::PolicytraceObject`]). +/// It should be used only for policies that do not have policy-specific scan_object(). pub struct ScanObjects { buffer: Vec, #[allow(unused)] @@ -601,3 +614,112 @@ impl GCWork for ProcessModBuf { } } } + +use crate::mmtk::MMTK; +use crate::plan::Plan; +use crate::plan::PlanTraceObject; +use crate::policy::gc_work::TraceKind; + +/// This provides an implementation of [`ProcessEdgesWork`](scheduler/gc_work/ProcessEdgesWork). A plan that implements +/// `PlanTraceObject` can use this work packet for tracing objects. +pub struct PlanProcessEdges< + VM: VMBinding, + P: Plan + PlanTraceObject, + const KIND: TraceKind, +> { + plan: &'static P, + base: ProcessEdgesBase, +} + +impl + Plan, const KIND: TraceKind> ProcessEdgesWork + for PlanProcessEdges +{ + type VM = VM; + + fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk); + let plan = base.plan().downcast_ref::

().unwrap(); + Self { plan, base } + } + + #[inline(always)] + fn create_scan_work(&self, nodes: Vec) -> Box> { + Box::new(PlanScanObjects::::new(self.plan, nodes, false)) + } + + #[inline(always)] + fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { + if object.is_null() { + return object; + } + self.plan + .trace_object::(self, object, self.worker()) + } + + #[inline] + fn process_edge(&mut self, slot: Address) { + let object = unsafe { slot.load::() }; + let new_object = self.trace_object(object); + if P::may_move_objects::() { + unsafe { slot.store(new_object) }; + } + } +} + +// Impl Deref/DerefMut to ProcessEdgesBase for PlanProcessEdges +impl + Plan, const KIND: TraceKind> Deref + for PlanProcessEdges +{ + type Target = ProcessEdgesBase; + #[inline] + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl + Plan, const KIND: TraceKind> DerefMut + for PlanProcessEdges +{ + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} + +/// This provides an implementation of scanning objects work. Each object will be scanned by calling `scan_object()` +/// in `PlanTraceObject`. +pub struct PlanScanObjects + PlanTraceObject> { + plan: &'static P, + buffer: Vec, + #[allow(dead_code)] + concurrent: bool, + phantom: PhantomData, +} + +impl + PlanTraceObject> PlanScanObjects { + pub fn new(plan: &'static P, buffer: Vec, concurrent: bool) -> Self { + Self { + plan, + buffer, + concurrent, + phantom: PhantomData, + } + } +} + +impl + PlanTraceObject> GCWork + for PlanScanObjects +{ + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + trace!("PlanScanObjects"); + { + let tls = worker.tls; + let mut closure = ObjectsClosure::::new(worker); + for object in &self.buffer { + ::VMScanning::scan_object(tls, *object, &mut closure); + self.plan.post_scan_object(*object); + } + } + trace!("PlanScanObjects End"); + } +} diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index 269d2f4f07..9694708eac 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -113,6 +113,10 @@ impl WorkBucket { pub fn add>(&self, work: W) { self.add_with_priority(Self::DEFAULT_PRIORITY, Box::new(work)); } + /// Add a boxed work packet to this bucket, with a default priority (1000) + pub fn add_boxed(&self, work: Box>) { + self.add_with_priority(Self::DEFAULT_PRIORITY, work); + } pub fn bulk_add_with_priority(&self, priority: usize, work_vec: Vec>>) { { let mut queue = self.queue.write(); diff --git a/src/scheduler/worker.rs b/src/scheduler/worker.rs index ce7eb02f61..6b2eb1e721 100644 --- a/src/scheduler/worker.rs +++ b/src/scheduler/worker.rs @@ -140,6 +140,10 @@ impl GCWorker { work.do_work(self, self.mmtk); } + pub fn do_boxed_work(&'static mut self, mut work: Box>) { + work.do_work(self, self.mmtk); + } + pub fn run(&mut self, tls: VMWorkerThread, mmtk: &'static MMTK) { self.tls = tls; self.copy = crate::plan::create_gc_worker_context(tls, mmtk);