From c77d944c63ee6e4e975b308c609dc71e8c45c0fe Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Thu, 17 Aug 2023 10:24:58 +0200 Subject: [PATCH] Intern repository mapping entries All repositories generated by a module extension have the same repository mapping entries. Without interning, if a module extension generates N repositories, each such repository would have its own copy with (more than) N entries, resulting in memory usage that is quadratic in N. We intern the entries, not the RepositoryMapping itself, because the latter also includes the owner repo, which differs between extension repos. Output of `bazel info used-heap-size-after-gc` after running a build with a synthetic module extension generating N + 1 repos: before N=100: 32MB N=1000: 77MB N=3000: 371MB N=5000: 961MB N=10000: 3614MB after N=100: 32MB N=1000: 44MB N=3000: 71MB N=5000: 91MB N=10000: 158MB --- .../build/lib/cmdline/RepositoryMapping.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java index 0246c930171776..d1e5f35cc7c53f 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java @@ -17,6 +17,8 @@ import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Interner; +import com.google.devtools.build.lib.concurrent.BlazeInterners; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -34,6 +36,15 @@ @AutoValue public abstract class RepositoryMapping { + // All repositories generated by a module extension have the same repository mapping + // entries. Without interning, if a module extension generates N repositories, each such + // repository would have its own copy with (more than) N entries, resulting in memory usage that + // is quadratic in N. + // Note: We intern the entries, not the RepositoryMapping itself, because the latter also includes + // the owner repo, which differs between extension repos. + private static final Interner> ENTRIES_INTERNER = + BlazeInterners.newWeakInterner(); + // Always fallback to the requested name public static final RepositoryMapping ALWAYS_FALLBACK = createAllowingFallback(ImmutableMap.of()); @@ -60,7 +71,8 @@ public static RepositoryMapping createAllowingFallback(Map entries, RepositoryName ownerRepo) { - return new AutoValue_RepositoryMapping(ImmutableMap.copyOf(entries), ownerRepo); + return new AutoValue_RepositoryMapping( + ENTRIES_INTERNER.intern(ImmutableMap.copyOf(entries)), ownerRepo); } /**