From 622ac80af297ea7322d7a7a7846a3995368bf450 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 7 Mar 2023 06:10:45 +0000 Subject: [PATCH 1/4] Collect live bytes during GC. --- Cargo.toml | 3 ++ src/memory_manager.rs | 20 +++++++++++-- src/plan/global.rs | 5 ++++ src/plan/markcompact/gc_work.rs | 6 ++++ src/plan/tracing.rs | 2 +- src/scheduler/gc_work.rs | 51 +++++++++++++++++++++++++++++++-- src/scheduler/worker.rs | 25 ++++++++++++++++ src/vm/scanning.rs | 2 +- 8 files changed, 107 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c5fbe14541..0378ab309d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,6 +125,9 @@ work_packet_stats = [] # Count the malloc'd memory into the heap size malloc_counted_size = [] +# Count the size of all live objects in GC +count_live_bytes_in_gc = [] + # Do not modify the following line - ci-common.sh matches it # -- Mutally exclusive features -- # Only one feature from each group can be provided. Otherwise build will fail. diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 1a28645343..5587c8ebec 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -521,7 +521,8 @@ pub fn process_bulk(builder: &mut MMTKBuilder, options: &str) -> bool { builder.set_options_bulk_by_str(options) } -/// Return used memory in bytes. +/// Return used memory in bytes. MMTk accounts for memory in pages, thus this method always returns a value in +/// page granularity. /// /// Arguments: /// * `mmtk`: A reference to an MMTk instance. @@ -529,7 +530,8 @@ pub fn used_bytes(mmtk: &MMTK) -> usize { mmtk.plan.get_used_pages() << LOG_BYTES_IN_PAGE } -/// Return free memory in bytes. +/// Return free memory in bytes. MMTk accounts for memory in pages, thus this method always returns a value in +/// page granularity. /// /// Arguments: /// * `mmtk`: A reference to an MMTk instance. @@ -537,6 +539,20 @@ pub fn free_bytes(mmtk: &MMTK) -> usize { mmtk.plan.get_free_pages() << LOG_BYTES_IN_PAGE } +/// Return the size of all the live objects in bytes in the last GC. MMTk usually accounts for memory in pages. +/// This is a special method that we count the size of every live object in a GC, and sum up the total bytes. +/// We provide this method so users can compare with `used_bytes` (which does page accounting), and know if +/// the heap is fragmented. +/// The value returned by this method is only updated when we finish tracing in a GC. A recommended timing +/// to call this method is at the end of a GC (e.g. when the runtime is about to resume threads). +#[cfg(feature = "count_live_bytes_in_gc")] +pub fn live_bytes_in_last_gc(mmtk: &MMTK) -> usize { + mmtk.plan + .base() + .live_bytes_in_last_gc + .load(Ordering::SeqCst) +} + /// Return the starting address of the heap. *Note that currently MMTk uses /// a fixed address range as heap.* pub fn starting_heap_address() -> Address { diff --git a/src/plan/global.rs b/src/plan/global.rs index 7e5d730f4f..313e566d1d 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -398,6 +398,9 @@ pub struct BasePlan { /// A counteer that keeps tracks of the number of bytes allocated by malloc #[cfg(feature = "malloc_counted_size")] malloc_bytes: AtomicUsize, + /// This stores the size in bytes for all the live objects in last GC. This counter is only updated in the GC release phase. + #[cfg(feature = "count_live_bytes_in_gc")] + pub live_bytes_in_last_gc: AtomicUsize, /// Wrapper around analysis counters #[cfg(feature = "analysis")] pub analysis_manager: AnalysisManager, @@ -552,6 +555,8 @@ impl BasePlan { allocation_bytes: AtomicUsize::new(0), #[cfg(feature = "malloc_counted_size")] malloc_bytes: AtomicUsize::new(0), + #[cfg(feature = "count_live_bytes_in_gc")] + live_bytes_in_last_gc: AtomicUsize::new(0), #[cfg(feature = "analysis")] analysis_manager, } diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index 9f86b7acc2..22c615589b 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -43,6 +43,12 @@ impl GCWork for UpdateReferences { #[cfg(feature = "extreme_assertions")] mmtk.edge_logger.reset(); + // We do two passes of transitive closures. We clear the live bytes from the first pass. + #[cfg(feature = "count_live_bytes_in_gc")] + mmtk.scheduler + .worker_group + .get_and_clear_worker_live_bytes(); + // TODO investigate why the following will create duplicate edges // scheduler.work_buckets[WorkBucketStage::RefForwarding] // .add(ScanStackRoots::>::new()); diff --git a/src/plan/tracing.rs b/src/plan/tracing.rs index 1f3b4e6dc3..e2011f9b38 100644 --- a/src/plan/tracing.rs +++ b/src/plan/tracing.rs @@ -74,7 +74,7 @@ impl ObjectQueue for VectorQueue { /// A transitive closure visitor to collect all the edges of an object. pub struct ObjectsClosure<'a, E: ProcessEdgesWork> { buffer: VectorQueue>, - worker: &'a mut GCWorker, + pub(crate) worker: &'a mut GCWorker, } impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 1b440833e7..9372ca7336 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -124,6 +124,18 @@ impl GCWork for Release { let result = w.designated_work.push(Box::new(ReleaseCollector)); debug_assert!(result.is_ok()); } + + #[cfg(feature = "count_live_bytes_in_gc")] + { + let live_bytes = mmtk + .scheduler + .worker_group + .get_and_clear_worker_live_bytes(); + self.plan + .base() + .live_bytes_in_last_gc + .store(live_bytes, std::sync::atomic::Ordering::SeqCst); + } } } @@ -232,6 +244,28 @@ impl GCWork for EndOfGC { self.elapsed.as_millis() ); + #[cfg(feature = "count_live_bytes_in_gc")] + { + let live_bytes = mmtk + .plan + .base() + .live_bytes_in_last_gc + .load(std::sync::atomic::Ordering::SeqCst); + let used_bytes = + mmtk.plan.get_used_pages() << crate::util::constants::LOG_BYTES_IN_PAGE; + debug_assert!( + live_bytes <= used_bytes, + "Live bytes of all live objects ({} bytes) is larger than used pages ({} bytes), something is wrong.", + live_bytes, used_bytes + ); + info!( + "Live objects = {} bytes ({:04.1}% of {} used pages)", + live_bytes, + live_bytes as f64 * 100.0 / used_bytes as f64, + mmtk.plan.get_used_pages() + ); + } + // We assume this is the only running work packet that accesses plan at the point of execution #[allow(clippy::cast_ref_to_mut)] let plan_mut: &mut dyn Plan = unsafe { &mut *(&*mmtk.plan as *const _ as *mut _) }; @@ -322,7 +356,7 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC fn with_tracer(&self, worker: &mut GCWorker, func: F) -> R where - F: FnOnce(&mut Self::TracerType) -> R, + F: FnOnce(&mut Self::TracerType, &mut GCWorker) -> R, { let mmtk = worker.mmtk; @@ -339,7 +373,7 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC }; // The caller can use the tracer here. - let result = func(&mut tracer); + let result = func(&mut tracer, worker); // Flush the queued nodes. tracer.flush_if_not_empty(); @@ -826,6 +860,12 @@ pub trait ScanObjectsWork: GCWork + Sized { // If an object supports edge-enqueuing, we enqueue its edges. ::VMScanning::scan_object(tls, object, &mut closure); self.post_scan_object(object); + + #[cfg(feature = "count_live_bytes_in_gc")] + closure + .worker + .shared + .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); } else { // If an object does not support edge-enqueuing, we have to use // `Scanning::scan_object_and_trace_edges` and offload the job of updating the @@ -845,7 +885,7 @@ pub trait ScanObjectsWork: GCWork + Sized { phantom_data: PhantomData, }; - object_tracer_context.with_tracer(worker, |object_tracer| { + object_tracer_context.with_tracer(worker, |object_tracer, _worker| { // Scan objects and trace their edges at the same time. for object in scan_later.iter().copied() { trace!("Scan object (node) {}", object); @@ -855,6 +895,11 @@ pub trait ScanObjectsWork: GCWork + Sized { object_tracer, ); self.post_scan_object(object); + + #[cfg(feature = "count_live_bytes_in_gc")] + _worker + .shared + .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); } }); } diff --git a/src/scheduler/worker.rs b/src/scheduler/worker.rs index 2a0f3611dd..6fb578b607 100644 --- a/src/scheduler/worker.rs +++ b/src/scheduler/worker.rs @@ -31,6 +31,11 @@ pub fn current_worker_ordinal() -> Option { pub struct GCWorkerShared { /// Worker-local statistics data. stat: AtomicRefCell>, + /// Accumulated bytes for live objects in this GC. When each worker scans + /// objects, we increase the live bytes. We get this value from each worker + /// at the end of a GC, and reset this counter. + #[cfg(feature = "count_live_bytes_in_gc")] + live_bytes: AtomicUsize, /// A queue of GCWork that can only be processed by the owned thread. /// /// Note: Currently, designated work cannot be added from the GC controller thread, or @@ -45,10 +50,22 @@ impl GCWorkerShared { pub fn new(stealer: Option>>>) -> Self { Self { stat: Default::default(), + #[cfg(feature = "count_live_bytes_in_gc")] + live_bytes: AtomicUsize::new(0), designated_work: ArrayQueue::new(16), stealer, } } + + #[cfg(feature = "count_live_bytes_in_gc")] + pub(crate) fn increase_live_bytes(&self, bytes: usize) { + self.live_bytes.fetch_add(bytes, Ordering::Relaxed); + } + + #[cfg(feature = "count_live_bytes_in_gc")] + pub(crate) fn get_and_clear_live_bytes(&self) -> usize { + self.live_bytes.swap(0, Ordering::SeqCst) + } } /// A GC worker. This part is privately owned by a worker thread. @@ -285,6 +302,14 @@ impl WorkerGroup { .iter() .any(|w| !w.designated_work.is_empty()) } + + #[cfg(feature = "count_live_bytes_in_gc")] + pub fn get_and_clear_worker_live_bytes(&self) -> usize { + self.workers_shared + .iter() + .map(|w| w.get_and_clear_live_bytes()) + .sum() + } } /// This ensures the worker always decrements the parked worker count on all control flow paths. diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 81fef2f0d4..60b0880871 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -76,7 +76,7 @@ pub trait ObjectTracerContext: Clone + Send + 'static { /// Returns: The return value of `func`. fn with_tracer(&self, worker: &mut GCWorker, func: F) -> R where - F: FnOnce(&mut Self::TracerType) -> R; + F: FnOnce(&mut Self::TracerType, &mut GCWorker) -> R; } /// Root-scanning methods use this trait to create work packets for processing roots. From 18f677fb9b00edd83301e20359fb227bb6b994a7 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 7 Mar 2023 06:26:50 +0000 Subject: [PATCH 2/4] Avoid changing ObjectTracerContext --- src/scheduler/gc_work.rs | 14 ++++++++++++-- src/vm/scanning.rs | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 9372ca7336..4dd58bb05a 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -356,7 +356,17 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC fn with_tracer(&self, worker: &mut GCWorker, func: F) -> R where - F: FnOnce(&mut Self::TracerType, &mut GCWorker) -> R, + F: FnOnce(&mut Self::TracerType) -> R, + { + self.with_tracer_and_worker(worker, |tracer, _| func(tracer)) + } +} + +impl ProcessEdgesWorkTracerContext { + // This Also exposes worker to the callback function. This is not a public method. + fn with_tracer_and_worker(&self, worker: &mut GCWorker, func: F) -> R + where + F: FnOnce(&mut ProcessEdgesWorkTracer, &mut GCWorker) -> R, { let mmtk = worker.mmtk; @@ -885,7 +895,7 @@ pub trait ScanObjectsWork: GCWork + Sized { phantom_data: PhantomData, }; - object_tracer_context.with_tracer(worker, |object_tracer, _worker| { + object_tracer_context.with_tracer_and_worker(worker, |object_tracer, _worker| { // Scan objects and trace their edges at the same time. for object in scan_later.iter().copied() { trace!("Scan object (node) {}", object); diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 60b0880871..81fef2f0d4 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -76,7 +76,7 @@ pub trait ObjectTracerContext: Clone + Send + 'static { /// Returns: The return value of `func`. fn with_tracer(&self, worker: &mut GCWorker, func: F) -> R where - F: FnOnce(&mut Self::TracerType, &mut GCWorker) -> R; + F: FnOnce(&mut Self::TracerType) -> R; } /// Root-scanning methods use this trait to create work packets for processing roots. From 845a8e781159551400826d79fb8e86088762aa54 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 8 Mar 2023 04:21:59 +0000 Subject: [PATCH 3/4] Count live bytes at the beginning of the loop. Revert changes about object tracer --- src/scheduler/gc_work.rs | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 4dd58bb05a..66329d46bd 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -357,16 +357,6 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC fn with_tracer(&self, worker: &mut GCWorker, func: F) -> R where F: FnOnce(&mut Self::TracerType) -> R, - { - self.with_tracer_and_worker(worker, |tracer, _| func(tracer)) - } -} - -impl ProcessEdgesWorkTracerContext { - // This Also exposes worker to the callback function. This is not a public method. - fn with_tracer_and_worker(&self, worker: &mut GCWorker, func: F) -> R - where - F: FnOnce(&mut ProcessEdgesWorkTracer, &mut GCWorker) -> R, { let mmtk = worker.mmtk; @@ -383,7 +373,7 @@ impl ProcessEdgesWorkTracerContext { }; // The caller can use the tracer here. - let result = func(&mut tracer, worker); + let result = func(&mut tracer); // Flush the queued nodes. tracer.flush_if_not_empty(); @@ -865,17 +855,18 @@ pub trait ScanObjectsWork: GCWork + Sized { { let mut closure = ObjectsClosure::::new(worker); for object in objects_to_scan.iter().copied() { + // For any object we need to scan, we count its liv bytes + #[cfg(feature = "count_live_bytes_in_gc")] + closure + .worker + .shared + .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); + if ::VMScanning::support_edge_enqueuing(tls, object) { trace!("Scan object (edge) {}", object); // If an object supports edge-enqueuing, we enqueue its edges. ::VMScanning::scan_object(tls, object, &mut closure); self.post_scan_object(object); - - #[cfg(feature = "count_live_bytes_in_gc")] - closure - .worker - .shared - .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); } else { // If an object does not support edge-enqueuing, we have to use // `Scanning::scan_object_and_trace_edges` and offload the job of updating the @@ -895,7 +886,7 @@ pub trait ScanObjectsWork: GCWork + Sized { phantom_data: PhantomData, }; - object_tracer_context.with_tracer_and_worker(worker, |object_tracer, _worker| { + object_tracer_context.with_tracer(worker, |object_tracer| { // Scan objects and trace their edges at the same time. for object in scan_later.iter().copied() { trace!("Scan object (node) {}", object); @@ -905,11 +896,6 @@ pub trait ScanObjectsWork: GCWork + Sized { object_tracer, ); self.post_scan_object(object); - - #[cfg(feature = "count_live_bytes_in_gc")] - _worker - .shared - .increase_live_bytes(VM::VMObjectModel::get_current_size(object)); } }); } From 76dc0fc2b11a83dc3452f47235ed7aa699141fb4 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 8 Aug 2023 01:15:27 +0000 Subject: [PATCH 4/4] Fix build issue --- src/scheduler/worker.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scheduler/worker.rs b/src/scheduler/worker.rs index 8836b772a5..9bb9ca6443 100644 --- a/src/scheduler/worker.rs +++ b/src/scheduler/worker.rs @@ -9,6 +9,8 @@ use atomic::Atomic; use atomic_refcell::{AtomicRef, AtomicRefCell, AtomicRefMut}; use crossbeam::deque::{self, Stealer}; use crossbeam::queue::ArrayQueue; +#[cfg(feature = "count_live_bytes_in_gc")] +use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering; use std::sync::{Arc, Condvar, Mutex};