Create a GcThrashingDetector.

Unlike `RetainedHeapLimiter`, does not trigger manual GC. Can be configured with multiple limits, i.e. "OOM if there are X consecutive full GCs over threshold within Y seconds" for various values of X and Y.

Maintains a sliding window for each limit. This may have some waste for overlapping windows, but in practice the limits are expected to be small and there won't be many of them.

Not wired in via flags yet - will need to write a custom option converter.

PiperOrigin-RevId: 519825944
Change-Id: Ifbc4391ebe088bfca9b9841baccc5bc0de992022

src/main/java/com/google/devtools/build/lib/clock/Clock.java

@@ -13,21 +13,26 @@
 // limitations under the License.
 package com.google.devtools.build.lib.clock;
 
-/**
- * This class provides an interface for a pluggable clock.
- */
+import java.time.Instant;
+
+/** Provides an interface for a pluggable clock. */
 public interface Clock {
 
   /**
-   * Returns the current time in milliseconds. The milliseconds are counted from midnight
-   * Jan 1, 1970.
+   * Returns the current time in milliseconds. The milliseconds are counted from midnight Jan 1,
+   * 1970.
    */
   long currentTimeMillis();
 
   /**
-   * Returns the current time in nanoseconds. The nanoseconds are measured relative to some
-   * unknown, but fixed event. Unfortunately, a sequence of calls to this method is *not*
-   * guaranteed to return non-decreasing values, so callers should be tolerant to this behavior.
+   * Returns the current time in nanoseconds. The nanoseconds are measured relative to some unknown,
+   * but fixed event. Unfortunately, a sequence of calls to this method is *not* guaranteed to
+   * return non-decreasing values, so callers should be tolerant to this behavior.
    */
   long nanoTime();
+
+  /** Returns {@link #currentTimeMillis} as an {@link Instant}. */
+  default Instant now() {
+    return Instant.ofEpochMilli(currentTimeMillis());
+  }
 }

src/main/java/com/google/devtools/build/lib/runtime/GcThrashingDetector.java

@@ -0,0 +1,128 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.runtime;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.common.flogger.GoogleLogger;
+import com.google.devtools.build.lib.bugreport.BugReporter;
+import com.google.devtools.build.lib.bugreport.Crash;
+import com.google.devtools.build.lib.bugreport.CrashContext;
+import com.google.devtools.build.lib.clock.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayDeque;
+import java.util.List;
+import java.util.Queue;
+
+/**
+ * Listens for {@link MemoryPressureEvent} to detect GC thrashing.
+ *
+ * <p>For each {@link Limit}, maintains a sliding window of the timestamps of consecutive full GCs
+ * within {@link Limit#period} where {@link MemoryPressureEvent#percentTenuredSpaceUsed} was more
+ * than {@link #threshold}. If {@link Limit#count} consecutive over-threshold full GCs within {@link
+ * Limit#period} are observed, calls {@link BugReporter#handleCrash} with an {@link
+ * OutOfMemoryError}.
+ *
+ * <p>Manual GCs do not contribute to the limit. This is to avoid OOMing on GCs manually triggered
+ * for memory metrics.
+ */
+final class GcThrashingDetector {
+
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+
+  @AutoValue
+  abstract static class Limit {
+    abstract Duration period();
+
+    abstract int count();
+
+    static Limit of(Duration period, int count) {
+      checkArgument(
+          !period.isNegative() && !period.isZero(), "period must be positive: %s", period);
+      checkArgument(count > 0, "count must be positive: %s", count);
+      return new AutoValue_GcThrashingDetector_Limit(period, count);
+    }
+  }
+
+  private final int threshold;
+  private final ImmutableList<SingleLimitTracker> trackers;
+  private final Clock clock;
+  private final BugReporter bugReporter;
+
+  GcThrashingDetector(int threshold, List<Limit> limits, Clock clock, BugReporter bugReporter) {
+    this.threshold = threshold;
+    this.trackers = limits.stream().map(SingleLimitTracker::new).collect(toImmutableList());
+    this.clock = clock;
+    this.bugReporter = bugReporter;
+  }
+
+  void handle(MemoryPressureEvent event) {
+    if (event.percentTenuredSpaceUsed() < threshold) {
+      for (var tracker : trackers) {
+        tracker.underThresholdGc();
+      }
+      return;
+    }
+
+    if (!event.wasFullGc() || event.wasManualGc()) {
+      return;
+    }
+
+    Instant now = clock.now();
+    for (var tracker : trackers) {
+      tracker.overThresholdGc(now);
+    }
+  }
+
+  /** Tracks GC history for a single {@link Limit}. */
+  private final class SingleLimitTracker {
+    private final Duration period;
+    private final int count;
+    private final Queue<Instant> window;
+
+    SingleLimitTracker(Limit limit) {
+      this.period = limit.period();
+      this.count = limit.count();
+      this.window = new ArrayDeque<>(count);
+    }
+
+    void underThresholdGc() {
+      window.clear();
+    }
+
+    void overThresholdGc(Instant now) {
+      Instant periodStart = now.minus(period);
+      while (!window.isEmpty() && window.element().isBefore(periodStart)) {
+        window.remove();
+      }
+      window.add(now);
+
+      if (window.size() == count) {
+        OutOfMemoryError oom =
+            new OutOfMemoryError(
+                String.format(
+                    "GcThrashingDetector forcing exit: the tenured space has been more than %s%%"
+                        + " occupied after %s consecutive full GCs within the past %s seconds.",
+                    threshold, count, period.toSeconds()));
+        logger.atInfo().log("Calling handleCrash");
+        bugReporter.handleCrash(Crash.from(oom), CrashContext.halt());
+      }
+    }
+  }
+}

src/main/java/com/google/devtools/build/lib/runtime/MemoryPressureEvent.java

@@ -30,6 +30,10 @@ public abstract class MemoryPressureEvent {
 
   public abstract long tenuredSpaceMaxBytes();
 
+  public final int percentTenuredSpaceUsed() {
+    return (int) ((tenuredSpaceUsedBytes() * 100L) / tenuredSpaceMaxBytes());
+  }
+
   @VisibleForTesting
   public static Builder newBuilder() {
     return new AutoValue_MemoryPressureEvent.Builder()

src/main/java/com/google/devtools/build/lib/runtime/RetainedHeapLimiter.java

@@ -130,7 +130,7 @@ public void handle(MemoryPressureEvent event) {
       return; // Inactive.
     }
 
-    int actual = (int) ((event.tenuredSpaceUsedBytes() * 100L) / event.tenuredSpaceMaxBytes());
+    int actual = event.percentTenuredSpaceUsed();
     if (actual < threshold) {
       if (wasHeapLimiterTriggeredGc || wasGcLockerDeferredHeapLimiterTriggeredGc) {
         logger.atInfo().log("Back under threshold (%s%% of tenured space)", actual);