From 57716dfb739c07a56dc3756a99f81d5ab7e1d42f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 6 Jul 2022 15:57:32 +0800 Subject: [PATCH] Node-enqueuing tracing. This commit implements node-enqueuing tracing. This is mainly for supporting VMs that does not support edge-enqueuing for some or all objects. --- src/plan/generational/gc_work.rs | 6 + src/scheduler/gc_work.rs | 254 +++++++++++++++++++++++++----- src/util/sanity/sanity_checker.rs | 9 ++ src/vm/mod.rs | 1 + src/vm/scanning.rs | 87 +++++++--- 5 files changed, 297 insertions(+), 60 deletions(-) diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index 1d386da72e..081cdaf5ab 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -16,6 +16,7 @@ pub struct GenNurseryProcessEdges { impl ProcessEdgesWork for GenNurseryProcessEdges { type VM = VM; + type ScanObjectsWorkType = ScanObjects; fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { let base = ProcessEdgesBase::new(edges, roots, mmtk); @@ -39,6 +40,11 @@ impl ProcessEdgesWork for GenNurseryProcessEdges { debug_assert!(!self.gen.nursery.in_space(new_object)); unsafe { slot.store(new_object) }; } + + #[inline(always)] + fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { + ScanObjects::::new(nodes, false, roots) + } } impl Deref for GenNurseryProcessEdges { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index badaa2999b..d917826cc5 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -401,6 +401,9 @@ pub trait ProcessEdgesWork: { type VM: VMBinding; + /// The work packet type for scanning objects when using this ProcessEdgesWork. + type ScanObjectsWorkType: AbstractScanObjects; + const CAPACITY: usize = 4096; const OVERWRITE_REFERENCE: bool = true; const SCAN_OBJECTS_IMMEDIATELY: bool = true; @@ -425,27 +428,28 @@ pub trait ProcessEdgesWork: /// Start the a scan work packet. If SCAN_OBJECTS_IMMEDIATELY, the work packet will be executed immediately, in this method. /// Otherwise, the work packet will be added the Closure work bucket and will be dispatched later by the scheduler. #[inline] - fn start_or_dispatch_scan_work(&mut self, work_packet: Box>) { + fn start_or_dispatch_scan_work(&mut self, work_packet: impl GCWork) { if Self::SCAN_OBJECTS_IMMEDIATELY { // We execute this `scan_objects_work` immediately. // This is expected to be a useful optimization because, // say for _pmd_ with 200M heap, we're likely to have 50000~60000 `ScanObjects` work packets // being dispatched (similar amount to `ProcessEdgesWork`). // Executing these work packets now can remarkably reduce the global synchronization time. - self.worker().do_boxed_work(work_packet); + self.worker().do_work(work_packet); } else { - self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add_boxed(work_packet); + self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(work_packet); } } - /// Create scan work for the policy. By default, we use [`ScanObjects`](crate::scheduler::gc_work::ScanObjects). - /// If a policy has its own scan object work packet, they can override this method. - #[inline(always)] - fn create_scan_work(&self, nodes: Vec) -> Box> { - Box::new(crate::scheduler::gc_work::ScanObjects::::new( - nodes, false, - )) - } + /// Create an object-scanning work packet to be used for this ProcessEdgesWork. + /// + /// `roots` indicates if we are creating a packet for root scanning. It is only true when this + /// method is called to handle `RootsWorkFactory::create_process_node_roots_work`. + fn create_scan_work( + &self, + nodes: Vec, + roots: bool, + ) -> Self::ScanObjectsWorkType; /// Flush the nodes in ProcessEdgesBase, and create a ScanObjects work packet for it. If the node set is empty, /// this method will simply return with no work packet created. @@ -455,7 +459,7 @@ pub trait ProcessEdgesWork: return; } let nodes = self.pop_nodes(); - self.start_or_dispatch_scan_work(self.create_scan_work(nodes)); + self.start_or_dispatch_scan_work(self.create_scan_work(nodes, false)); } #[inline] @@ -505,6 +509,8 @@ pub struct SFTProcessEdges { impl ProcessEdgesWork for SFTProcessEdges { type VM = VM; + type ScanObjectsWorkType = ScanObjects; + fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { let base = ProcessEdgesBase::new(edges, roots, mmtk); Self { base } @@ -529,6 +535,11 @@ impl ProcessEdgesWork for SFTProcessEdges { let sft = crate::mmtk::SFT_MAP.get(object.to_address()); sft.sft_trace_object(&mut self.base.nodes, object, worker) } + + #[inline(always)] + fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { + ScanObjects::::new(nodes, false, roots) + } } struct ProcessEdgesWorkRootsWorkFactory { @@ -550,8 +561,11 @@ impl RootsWorkFactory for ProcessEdgesWorkRootsWorkFactory< ); } - fn create_process_node_roots_work(&mut self, _nodes: Vec) { - todo!() + fn create_process_node_roots_work(&mut self, nodes: Vec) { + // We want to use E::create_scan_work. + let process_edges_work = E::new(vec![], true, self.mmtk); + let work = process_edges_work.create_scan_work(nodes, true); + crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, work); } } @@ -576,35 +590,168 @@ impl DerefMut for SFTProcessEdges { } } -/// Scan & update a list of object slots. -/// Note that this work packet does not do any policy-specific scan -/// object work (it won't call `scan_object()` in [`policy::gc_work::PolicytraceObject`]). -/// It should be used only for policies that do not have policy-specific scan_object(). +/// Trait for a work packet that scans objects +pub trait AbstractScanObjects: GCWork + Sized { + /// The associated ProcessEdgesWork for processing the edges of the objects in this packet. + type E: ProcessEdgesWork; + + /// Return true if the objects in this packet are pointed by roots, in which case we need to + /// call trace_object on them. + fn roots(&self) -> bool; + + /// Called after each object is scanned. + fn post_scan_object(&self, object: ObjectReference); + + /// Create another object-scanning work packet of the same kind, to scan adjacent objects of + /// the objects in this packet. + fn make_another(&self, buffer: Vec) -> Self; + + /// The common code for ScanObjects and PlanScanObjects. + fn do_work_common( + &self, + buffer: &[ObjectReference], + worker: &mut GCWorker<::VM>, + mmtk: &'static MMTK<::VM>, + ) { + let tls = worker.tls; + + // If this is a root packet, the objects in this packet will have not been traced, yet. + // + // This step conceptually traces the edges from root slots to the objects they point to. + // However, VMs that deliver root objects instead of root edges are incapable of updating + // root slots. Like processing an edge, we call `trace_object` on those objects, and + // assert the GC doesn't move those objects because we cannot store back to the slots. + // + // If this is a root packet, the `scanned_root_objects` variable will hold those root + // objects which are traced for the first time. + let scanned_root_objects = self.roots().then(|| { + // We create an instance of E to use its `trace_object` method and its object queue. + let mut process_edges_work = Self::E::new(vec![], false, mmtk); + + for object in buffer.iter().copied() { + let new_object = process_edges_work.trace_object(object); + debug_assert_eq!( + object, new_object, + "Object moved while tracing root unmovable root object: {} -> {}", + object, new_object + ); + } + + // This contains root objects that are visited the first time. + // It is sufficient to only scan these objects. + process_edges_work.nodes.take() + }); + + // If it is a root packet, scan the nodes that are first scanned; + // otherwise, scan the nodes in the buffer. + let objects_to_scan = scanned_root_objects.as_deref().unwrap_or(buffer); + + // Then scan those objects for edges. + let mut scan_later = vec![]; + { + let mut closure = ObjectsClosure::::new(worker); + for object in objects_to_scan.iter().copied() { + if ::VMScanning::support_edge_enqueuing(tls, object) { + // If an object supports edge-enqueuing, we enqueue its edges. + ::VMScanning::scan_object(tls, object, &mut closure); + self.post_scan_object(object); + } else { + // If an object does not support edge-enqueuing, we have to use + // `Scanning::scan_object_and_trace_edges` and offload the job of updating the + // reference field to the VM. + // + // However, at this point, `closure` is borrowing `worker`. + // So we postpone the processing of objects that needs object enqueuing + scan_later.push(object); + } + } + } + + // If any object does not support edge-enqueuing, we process them now. + if !scan_later.is_empty() { + // We create an instance of E to use its `trace_object` method and its object queue. + let mut process_edges_work = Self::E::new(vec![], false, mmtk); + let mut closure = |object| process_edges_work.trace_object(object); + + // Scan objects and trace their edges at the same time. + for object in scan_later.iter().copied() { + ::VMScanning::scan_object_and_trace_edges( + tls, + object, + &mut closure, + ); + self.post_scan_object(object); + } + + // Create work packets to scan adjacent objects. We skip ProcessEdgesWork and create + // object-scanning packets directly, because the edges are already traced. + if !process_edges_work.nodes.is_empty() { + let next_nodes = process_edges_work.nodes.take(); + let make_packet = |nodes| { + let work_packet = self.make_another(nodes); + memory_manager::add_work_packet(mmtk, WorkBucketStage::Closure, work_packet); + }; + + // Divide the resulting nodes into appropriately sized packets. + if next_nodes.len() <= Self::E::CAPACITY { + make_packet(next_nodes); + } else { + for chunk in next_nodes.chunks(Self::E::CAPACITY) { + make_packet(chunk.into()); + } + } + } + } + } +} + +/// Scan objects and enqueue the edges of the objects. For objects that do not support +/// edge-enqueuing, this work packet also processes the edges. +/// +/// This work packet does not execute policy-specific post-scanning hooks +/// (it won't call `post_scan_object()` in [`policy::gc_work::PolicyTraceObject`]). +/// It should be used only for policies that do not perform policy-specific actions when scanning +/// an object. pub struct ScanObjects { buffer: Vec, #[allow(unused)] concurrent: bool, + roots: bool, phantom: PhantomData, } impl ScanObjects { - pub fn new(buffer: Vec, concurrent: bool) -> Self { + pub fn new(buffer: Vec, concurrent: bool, roots: bool) -> Self { Self { buffer, concurrent, + roots, phantom: PhantomData, } } } +impl> AbstractScanObjects for ScanObjects { + type E = E; + + fn roots(&self) -> bool { + self.roots + } + + #[inline(always)] + fn post_scan_object(&self, _object: ObjectReference) { + // Do nothing. + } + + fn make_another(&self, buffer: Vec) -> Self { + Self::new(buffer, self.concurrent, false) + } +} + impl GCWork for ScanObjects { - fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanObjects"); - { - let tls = worker.tls; - let mut closure = ObjectsClosure::::new(worker); - ::VMScanning::scan_objects(tls, &self.buffer, &mut closure); - } + self.do_work_common(&self.buffer, worker, mmtk); trace!("ScanObjects End"); } } @@ -637,7 +784,11 @@ impl GCWork for ProcessModBuf { if !self.modbuf.is_empty() { let mut modbuf = vec![]; ::std::mem::swap(&mut modbuf, &mut self.modbuf); - GCWork::do_work(&mut ScanObjects::::new(modbuf, false), worker, mmtk) + GCWork::do_work( + &mut ScanObjects::::new(modbuf, false, false), + worker, + mmtk, + ) } } else { // Do nothing @@ -665,6 +816,7 @@ impl + Plan, const KIND: TraceKin for PlanProcessEdges { type VM = VM; + type ScanObjectsWorkType = PlanScanObjects; fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { let base = ProcessEdgesBase::new(edges, roots, mmtk); @@ -673,8 +825,12 @@ impl + Plan, const KIND: TraceKin } #[inline(always)] - fn create_scan_work(&self, nodes: Vec) -> Box> { - Box::new(PlanScanObjects::::new(self.plan, nodes, false)) + fn create_scan_work( + &self, + nodes: Vec, + roots: bool, + ) -> Self::ScanObjectsWorkType { + PlanScanObjects::::new(self.plan, nodes, false, roots) } #[inline(always)] @@ -718,40 +874,58 @@ impl + Plan, const KIND: TraceKin } } -/// This provides an implementation of scanning objects work. Each object will be scanned by calling `scan_object()` -/// in `PlanTraceObject`. +/// This is an alternative to `ScanObjects` that calls the `post_scan_object` of the policy +/// selected by the plan. It is applicable to plans that derive `PlanTraceObject`. pub struct PlanScanObjects + PlanTraceObject> { plan: &'static P, buffer: Vec, #[allow(dead_code)] concurrent: bool, + roots: bool, phantom: PhantomData, } impl + PlanTraceObject> PlanScanObjects { - pub fn new(plan: &'static P, buffer: Vec, concurrent: bool) -> Self { + pub fn new( + plan: &'static P, + buffer: Vec, + concurrent: bool, + roots: bool, + ) -> Self { Self { plan, buffer, concurrent, + roots, phantom: PhantomData, } } } +impl + PlanTraceObject> AbstractScanObjects + for PlanScanObjects +{ + type E = E; + + fn roots(&self) -> bool { + self.roots + } + + fn post_scan_object(&self, object: ObjectReference) { + self.plan.post_scan_object(object); + } + + fn make_another(&self, buffer: Vec) -> Self { + Self::new(self.plan, buffer, self.concurrent, false) + } +} + impl + PlanTraceObject> GCWork for PlanScanObjects { - fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("PlanScanObjects"); - { - let tls = worker.tls; - let mut closure = ObjectsClosure::::new(worker); - for object in &self.buffer { - ::VMScanning::scan_object(tls, *object, &mut closure); - self.plan.post_scan_object(*object); - } - } + self.do_work_common(&self.buffer, worker, mmtk); trace!("PlanScanObjects End"); } } diff --git a/src/util/sanity/sanity_checker.rs b/src/util/sanity/sanity_checker.rs index 78fb3130f1..78093dd3cb 100644 --- a/src/util/sanity/sanity_checker.rs +++ b/src/util/sanity/sanity_checker.rs @@ -161,6 +161,7 @@ impl DerefMut for SanityGCProcessEdges { impl ProcessEdgesWork for SanityGCProcessEdges { type VM = VM; + type ScanObjectsWorkType = ScanObjects; const OVERWRITE_REFERENCE: bool = false; fn new(edges: Vec
, roots: bool, mmtk: &'static MMTK) -> Self { @@ -185,4 +186,12 @@ impl ProcessEdgesWork for SanityGCProcessEdges { } object } + + fn create_scan_work( + &self, + nodes: Vec, + roots: bool, + ) -> Self::ScanObjectsWorkType { + ScanObjects::::new(nodes, false, roots) + } } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index 23ef735478..8b4d5935ca 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -30,6 +30,7 @@ pub use self::object_model::ObjectModel; pub use self::reference_glue::Finalizable; pub use self::reference_glue::ReferenceGlue; pub use self::scanning::EdgeVisitor; +pub use self::scanning::ObjectTracer; pub use self::scanning::RootsWorkFactory; pub use self::scanning::Scanning; diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index fd55e8e6e5..56841df299 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -3,11 +3,31 @@ use crate::util::VMWorkerThread; use crate::util::{Address, ObjectReference}; use crate::vm::VMBinding; -// Callback trait of scanning functions that report edges. +/// Callback trait of scanning functions that report edges. pub trait EdgeVisitor { /// Call this function for each edge. fn visit_edge(&mut self, edge: Address); - // TODO: Add visit_soft_edge, visit_weak_edge, ... here. +} + +/// This lets us use closures as EdgeVisitor. +impl EdgeVisitor for F { + fn visit_edge(&mut self, edge: Address) { + self(edge) + } +} + +/// Callback trait of scanning functions that directly trace through edges. +pub trait ObjectTracer { + /// Call this function for the content of each edge, + /// and assign the returned value back to the edge. + fn trace_object(&mut self, object: ObjectReference) -> ObjectReference; +} + +/// This lets us use closures as ObjectTracer. +impl ObjectReference> ObjectTracer for F { + fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { + self(object) + } } /// Root-scanning methods use this trait to create work packets for processing roots. @@ -60,8 +80,28 @@ pub trait Scanning { /// `SCAN_MUTATORS_IN_SAFEPOINT` should also be enabled const SINGLE_THREAD_MUTATOR_SCANNING: bool = true; - /// Delegated scanning of a object, visiting each pointer field - /// encountered. + /// Return true if the given object supports edge enqueuing. + /// + /// - If this returns true, MMTk core will call `scan_object` on the object. + /// - Otherwise, MMTk core will call `scan_object_and_trace_edges` on the object. + /// + /// For maximum performance, the VM should support edge-enqueuing for as many objects as + /// practical. + /// + /// Arguments: + /// * `tls`: The VM-specific thread-local storage for the current worker. + /// * `object`: The object to be scanned. + #[inline(always)] + fn support_edge_enqueuing(_tls: VMWorkerThread, _object: ObjectReference) -> bool { + true + } + + /// Delegated scanning of a object, visiting each reference field encountered. + /// + /// The VM shall call `edge_visitor.visit_edge` on each reference field. + /// + /// The VM may skip a reference field if it holds a null reference. If the VM supports tagged + /// references, it must skip tagged reference fields which are not holding references. /// /// Arguments: /// * `tls`: The VM-specific thread-local storage for the current worker. @@ -73,6 +113,29 @@ pub trait Scanning { edge_visitor: &mut EV, ); + /// Delegated scanning of a object, visiting each reference field encountered, and trace the + /// objects pointed by each field. + /// + /// The VM shall call `object_tracer.trace_object` on the value held in each reference field, + /// and assign the returned value back to the field. If the VM uses tagged references, the + /// value passed to `object_tracer.trace_object` shall be the `ObjectReference` to the object + /// without any tag bits. + /// + /// The VM may skip a reference field if it holds a null reference. If the VM supports tagged + /// references, it must skip tagged reference fields which are not holding references. + /// + /// Arguments: + /// * `tls`: The VM-specific thread-local storage for the current worker. + /// * `object`: The object to be scanned. + /// * `object_tracer`: Called back for the content of each edge. + fn scan_object_and_trace_edges( + _tls: VMWorkerThread, + _object: ObjectReference, + _object_tracer: &mut OT, + ) { + unimplemented!(); + } + /// MMTk calls this method at the first time during a collection that thread's stacks /// have been scanned. This can be used (for example) to clean up /// obsolete compiled methods that are no longer being executed. @@ -82,22 +145,6 @@ pub trait Scanning { /// * `tls`: The GC thread that is performing the thread scan. fn notify_initial_thread_scan_complete(partial_scan: bool, tls: VMWorkerThread); - /// Bulk scanning of objects, processing each pointer field for each object. - /// - /// Arguments: - /// * `tls`: The VM-specific thread-local storage for the current worker. - /// * `objects`: The slice of object references to be scanned. - /// * `edge_visitor`: Called back for each edge in each object in `objects`. - fn scan_objects( - tls: VMWorkerThread, - objects: &[ObjectReference], - edge_visitor: &mut EV, - ) { - for object in objects.iter() { - Self::scan_object(tls, *object, edge_visitor); - } - } - /// Scan all the mutators for roots. /// /// Arguments: