diff --git a/src/heap/concurrent-marking.cc b/src/heap/concurrent-marking.cc index 3decc5788242..34ae98c59781 100644 --- a/src/heap/concurrent-marking.cc +++ b/src/heap/concurrent-marking.cc @@ -434,7 +434,7 @@ void ConcurrentMarking::Run(JobDelegate* delegate, isolate->PrintWithTimestamp("Starting concurrent marking task %d\n", task_id); } - bool ephemeron_marked = false; + bool another_ephemeron_iteration = false; { TimedScope scope(&time_ms); @@ -444,7 +444,7 @@ void ConcurrentMarking::Run(JobDelegate* delegate, while (weak_objects_->current_ephemerons.Pop(task_id, &ephemeron)) { if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) { - ephemeron_marked = true; + another_ephemeron_iteration = true; } } } @@ -497,6 +497,7 @@ void ConcurrentMarking::Run(JobDelegate* delegate, current_marked_bytes += visited_size; } } + if (objects_processed > 0) another_ephemeron_iteration = true; marked_bytes += current_marked_bytes; base::AsAtomicWord::Relaxed_Store(&task_state->marked_bytes, marked_bytes); @@ -512,7 +513,7 @@ void ConcurrentMarking::Run(JobDelegate* delegate, while (weak_objects_->discovered_ephemerons.Pop(task_id, &ephemeron)) { if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) { - ephemeron_marked = true; + another_ephemeron_iteration = true; } } } @@ -532,8 +533,8 @@ void ConcurrentMarking::Run(JobDelegate* delegate, base::AsAtomicWord::Relaxed_Store(&task_state->marked_bytes, 0); total_marked_bytes_ += marked_bytes; - if (ephemeron_marked) { - set_ephemeron_marked(true); + if (another_ephemeron_iteration) { + set_another_ephemeron_iteration(true); } } if (FLAG_trace_concurrent_marking) { diff --git a/src/heap/concurrent-marking.h b/src/heap/concurrent-marking.h index c685f5cca6de..54f6057f58b1 100644 --- a/src/heap/concurrent-marking.h +++ b/src/heap/concurrent-marking.h @@ -91,10 +91,12 @@ class V8_EXPORT_PRIVATE ConcurrentMarking { size_t TotalMarkedBytes(); - void set_ephemeron_marked(bool ephemeron_marked) { - ephemeron_marked_.store(ephemeron_marked); + void set_another_ephemeron_iteration(bool another_ephemeron_iteration) { + another_ephemeron_iteration_.store(another_ephemeron_iteration); + } + bool another_ephemeron_iteration() { + return another_ephemeron_iteration_.load(); } - bool ephemeron_marked() { return ephemeron_marked_.load(); } private: struct TaskState { @@ -115,7 +117,7 @@ class V8_EXPORT_PRIVATE ConcurrentMarking { WeakObjects* const weak_objects_; TaskState task_state_[kMaxTasks + 1]; std::atomic total_marked_bytes_{0}; - std::atomic ephemeron_marked_{false}; + std::atomic another_ephemeron_iteration_{false}; }; } // namespace internal diff --git a/src/heap/incremental-marking.cc b/src/heap/incremental-marking.cc index 29b2a84d68b4..e6fa23072f35 100644 --- a/src/heap/incremental-marking.cc +++ b/src/heap/incremental-marking.cc @@ -944,7 +944,8 @@ StepResult IncrementalMarking::Step(double max_step_size_in_ms, // This ignores that case where the embedder finds new V8-side objects. The // assumption is that large graphs are well connected and can mostly be // processed on their own. For small graphs, helping is not necessary. - v8_bytes_processed = collector_->ProcessMarkingWorklist(bytes_to_process); + std::tie(v8_bytes_processed, std::ignore) = + collector_->ProcessMarkingWorklist(bytes_to_process); StepResult v8_result = local_marking_worklists()->IsEmpty() ? StepResult::kNoImmediateWork : StepResult::kMoreWorkRemaining; diff --git a/src/heap/mark-compact.cc b/src/heap/mark-compact.cc index d6c644696dc8..2d29cd123935 100644 --- a/src/heap/mark-compact.cc +++ b/src/heap/mark-compact.cc @@ -1667,24 +1667,24 @@ void MarkCompactCollector::MarkDescriptorArrayFromWriteBarrier( descriptors, number_of_own_descriptors); } -void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() { - bool work_to_do = true; +bool MarkCompactCollector::ProcessEphemeronsUntilFixpoint() { int iterations = 0; int max_iterations = FLAG_ephemeron_fixpoint_iterations; - while (work_to_do) { + bool another_ephemeron_iteration_main_thread; + + do { PerformWrapperTracing(); if (iterations >= max_iterations) { // Give up fixpoint iteration and switch to linear algorithm. - ProcessEphemeronsLinear(); - break; + return false; } // Move ephemerons from next_ephemerons into current_ephemerons to // drain them in this iteration. weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons); - heap()->concurrent_marking()->set_ephemeron_marked(false); + heap()->concurrent_marking()->set_another_ephemeron_iteration(false); { TRACE_GC(heap()->tracer(), @@ -1695,47 +1695,54 @@ void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() { TaskPriority::kUserBlocking); } - work_to_do = ProcessEphemerons(); + another_ephemeron_iteration_main_thread = ProcessEphemerons(); FinishConcurrentMarking(); } CHECK(weak_objects_.current_ephemerons.IsEmpty()); CHECK(weak_objects_.discovered_ephemerons.IsEmpty()); - work_to_do = work_to_do || !local_marking_worklists()->IsEmpty() || - heap()->concurrent_marking()->ephemeron_marked() || - !local_marking_worklists()->IsEmbedderEmpty() || - !heap()->local_embedder_heap_tracer()->IsRemoteTracingDone(); ++iterations; - } + } while (another_ephemeron_iteration_main_thread || + heap()->concurrent_marking()->another_ephemeron_iteration() || + !local_marking_worklists()->IsEmpty() || + !local_marking_worklists()->IsEmbedderEmpty() || + !heap()->local_embedder_heap_tracer()->IsRemoteTracingDone()); CHECK(local_marking_worklists()->IsEmpty()); CHECK(weak_objects_.current_ephemerons.IsEmpty()); CHECK(weak_objects_.discovered_ephemerons.IsEmpty()); + return true; } bool MarkCompactCollector::ProcessEphemerons() { Ephemeron ephemeron; - bool ephemeron_marked = false; + bool another_ephemeron_iteration = false; // Drain current_ephemerons and push ephemerons where key and value are still // unreachable into next_ephemerons. while (weak_objects_.current_ephemerons.Pop(kMainThreadTask, &ephemeron)) { if (ProcessEphemeron(ephemeron.key, ephemeron.value)) { - ephemeron_marked = true; + another_ephemeron_iteration = true; } } // Drain marking worklist and push discovered ephemerons into // discovered_ephemerons. - DrainMarkingWorklist(); + size_t objects_processed; + std::tie(std::ignore, objects_processed) = ProcessMarkingWorklist(0); + + // As soon as a single object was processed and potentially marked another + // object we need another iteration. Otherwise we might miss to apply + // ephemeron semantics on it. + if (objects_processed > 0) another_ephemeron_iteration = true; // Drain discovered_ephemerons (filled in the drain MarkingWorklist-phase // before) and push ephemerons where key and value are still unreachable into // next_ephemerons. while (weak_objects_.discovered_ephemerons.Pop(kMainThreadTask, &ephemeron)) { if (ProcessEphemeron(ephemeron.key, ephemeron.value)) { - ephemeron_marked = true; + another_ephemeron_iteration = true; } } @@ -1743,7 +1750,7 @@ bool MarkCompactCollector::ProcessEphemerons() { weak_objects_.ephemeron_hash_tables.FlushToGlobal(kMainThreadTask); weak_objects_.next_ephemerons.FlushToGlobal(kMainThreadTask); - return ephemeron_marked; + return another_ephemeron_iteration; } void MarkCompactCollector::ProcessEphemeronsLinear() { @@ -1829,6 +1836,12 @@ void MarkCompactCollector::ProcessEphemeronsLinear() { ephemeron_marking_.newly_discovered.shrink_to_fit(); CHECK(local_marking_worklists()->IsEmpty()); + CHECK(weak_objects_.current_ephemerons.IsEmpty()); + CHECK(weak_objects_.discovered_ephemerons.IsEmpty()); + + // Flush local ephemerons for main task to global pool. + weak_objects_.ephemeron_hash_tables.FlushToGlobal(kMainThreadTask); + weak_objects_.next_ephemerons.FlushToGlobal(kMainThreadTask); } void MarkCompactCollector::PerformWrapperTracing() { @@ -1850,9 +1863,11 @@ void MarkCompactCollector::PerformWrapperTracing() { void MarkCompactCollector::DrainMarkingWorklist() { ProcessMarkingWorklist(0); } template -size_t MarkCompactCollector::ProcessMarkingWorklist(size_t bytes_to_process) { +std::pair MarkCompactCollector::ProcessMarkingWorklist( + size_t bytes_to_process) { HeapObject object; size_t bytes_processed = 0; + size_t objects_processed = 0; bool is_per_context_mode = local_marking_worklists()->IsPerContextMode(); Isolate* isolate = heap()->isolate(); while (local_marking_worklists()->Pop(&object) || @@ -1892,18 +1907,19 @@ size_t MarkCompactCollector::ProcessMarkingWorklist(size_t bytes_to_process) { map, object, visited_size); } bytes_processed += visited_size; + objects_processed++; if (bytes_to_process && bytes_processed >= bytes_to_process) { break; } } - return bytes_processed; + return std::make_pair(bytes_processed, objects_processed); } // Generate definitions for use in other files. -template size_t MarkCompactCollector::ProcessMarkingWorklist< +template std::pair MarkCompactCollector::ProcessMarkingWorklist< MarkCompactCollector::MarkingWorklistProcessingMode::kDefault>( size_t bytes_to_process); -template size_t MarkCompactCollector::ProcessMarkingWorklist< +template std::pair MarkCompactCollector::ProcessMarkingWorklist< MarkCompactCollector::MarkingWorklistProcessingMode:: kTrackNewlyDiscoveredObjects>(size_t bytes_to_process); @@ -1928,7 +1944,23 @@ void MarkCompactCollector::ProcessEphemeronMarking() { // buffer, flush it into global pool. weak_objects_.next_ephemerons.FlushToGlobal(kMainThreadTask); - ProcessEphemeronsUntilFixpoint(); + if (!ProcessEphemeronsUntilFixpoint()) { + // Fixpoint iteration needed too many iterations and was cancelled. Use the + // guaranteed linear algorithm. + ProcessEphemeronsLinear(); + } + +#ifdef VERIFY_HEAP + if (FLAG_verify_heap) { + Ephemeron ephemeron; + + weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons); + + while (weak_objects_.current_ephemerons.Pop(kMainThreadTask, &ephemeron)) { + CHECK(!ProcessEphemeron(ephemeron.key, ephemeron.value)); + } + } +#endif CHECK(local_marking_worklists()->IsEmpty()); CHECK(heap()->local_embedder_heap_tracer()->IsRemoteTracingDone()); diff --git a/src/heap/mark-compact.h b/src/heap/mark-compact.h index b077522213b4..4a6656db5970 100644 --- a/src/heap/mark-compact.h +++ b/src/heap/mark-compact.h @@ -591,7 +591,7 @@ class MarkCompactCollector final : public MarkCompactCollectorBase { // is drained until it is empty. template - size_t ProcessMarkingWorklist(size_t bytes_to_process); + std::pair ProcessMarkingWorklist(size_t bytes_to_process); private: void ComputeEvacuationHeuristics(size_t area_size, @@ -637,8 +637,9 @@ class MarkCompactCollector final : public MarkCompactCollectorBase { bool ProcessEphemeron(HeapObject key, HeapObject value); // Marks ephemerons and drains marking worklist iteratively - // until a fixpoint is reached. - void ProcessEphemeronsUntilFixpoint(); + // until a fixpoint is reached. Returns false if too many iterations have been + // tried and the linear approach should be used. + bool ProcessEphemeronsUntilFixpoint(); // Drains ephemeron and marking worklists. Single iteration of the // fixpoint iteration.