Optimize representation of runfiles in compact execution log

Runfiles trees are now represented with a custom `RunfilesTree` message in the compact execution log. This allows using `InputSet`s to representing all artifacts staged at canonical locations, with only symlinks and root symlinks stored flattened and with explicit runfiles paths.

Since runfile paths can collide, this change makes it necessary to preserve the order of elements in an `InputSet`. The previous representation as repeated ID fields for each type (file, symlink, directory) made this impossible, so the representation has been modified to reference all direct entry IDs in a single repeated field. Since this also reduces the potential for type mismatches between the ID field type and the referenced message type, all other typed IDs are replaced with untyped ID fields. By slightly tweaking the way IDs are generated for nested entries and not emitting IDs for entries that are never referenced (e.g. `Spawn`s), IDs are now consecutive, which simplifies the (possibly concurrent) bookkeeping for consumers by allowing them to use an array to store the entries.

Progress on bazelbuild#18643.

RELNOTES: The compact execution log now stores runfiles in a more compact representation that should reduce the memory overhead and log output size, in particular for test spawns. This change required breaking changes to the (experimental) log format.

Closes bazelbuild#23321.

PiperOrigin-RevId: 676773599
Change-Id: I010653681ffa44557142bf25009e9178b5d68515
(cherry picked from commit c2f539c)

src/main/java/com/google/devtools/build/lib/actions/BUILD

@@ -157,6 +157,7 @@ java_library(
  ":thread_state_receiver",
  "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
  "//src/main/java/com/google/devtools/build/lib/analysis:config/core_options",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
  "//src/main/java/com/google/devtools/build/lib/analysis/platform",
  "//src/main/java/com/google/devtools/build/lib/bugreport",
  "//src/main/java/com/google/devtools/build/lib/buildeventstream",
@@ -498,6 +499,7 @@ java_library(
  deps = [
  ":artifacts",
  "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
  "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
  "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
  "//src/main/java/net/starlark/java/eval",

src/main/java/com/google/devtools/build/lib/actions/RunfilesSupplier.java

@@ -16,6 +16,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.SymlinkEntry;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue.RunfileSymlinksMode;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -66,4 +67,32 @@ public interface RunfilesSupplier extends StarlarkValue {
  * #getRunfilesDirs} returns a set of size 1.
  */
  RunfilesSupplier withOverriddenRunfilesDir(PathFragment newRunfilesDir);
+
+ /**
+ * Returns artifacts the runfiles tree contain symlinks to at their canonical locations.
+ *
+ * <p>This does <b>not</b> include artifacts that only the symlinks and root symlinks point to.
+ */
+ NestedSet<Artifact> getArtifactsAtCanonicalLocationsForLogging();
+
+ /**
+ * Returns the set of names of implicit empty files to materialize.
+ *
+ * <p>If this runfiles tree does not implicitly add empty files, implementations should have a
+ * dedicated fast path that returns an empty set without traversing the tree.
+ */
+ Iterable<PathFragment> getEmptyFilenamesForLogging();
+
+ /** Returns the set of custom symlink entries. */
+ NestedSet<SymlinkEntry> getSymlinksForLogging();
+
+ /** Returns the set of root symlinks. */
+ NestedSet<SymlinkEntry> getRootSymlinksForLogging();
+
+ /** Returns the repo mapping manifest if it exists. */
+ @Nullable
+ Artifact getRepoMappingManifestForLogging();
+
+ /** Whether this runfiles tree materializes external runfiles also at their legacy locations. */
+ boolean isLegacyExternalRunfiles();
 }

src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java

@@ -250,10 +250,17 @@ public NestedSet<SymlinkEntry> getSymlinks() {
 
  @Override
  public Depset /*<String>*/ getEmptyFilenamesForStarlark() {
- return Depset.of(String.class, getEmptyFilenames());
+ return Depset.of(
+ String.class,
+ NestedSetBuilder.wrap(
+ Order.STABLE_ORDER,
+ Iterables.transform(getEmptyFilenames(), PathFragment::getPathString)));
  }
 
- public NestedSet<String> getEmptyFilenames() {
+ public Iterable<PathFragment> getEmptyFilenames() {
+ if (emptyFilesSupplier == DUMMY_EMPTY_FILES_SUPPLIER) {
+ return ImmutableList.of();
+ }
  Set<PathFragment> manifestKeys =
  Streams.concat(
  symlinks.toList().stream().map(SymlinkEntry::getPath),
@@ -264,13 +271,7 @@ public NestedSet<String> getEmptyFilenames() {
  ? artifact.getOutputDirRelativePath(false)
  : artifact.getRunfilesPath()))
  .collect(ImmutableSet.toImmutableSet());
- Iterable<PathFragment> emptyKeys = emptyFilesSupplier.getExtraPaths(manifestKeys);
- return NestedSetBuilder.<String>stableOrder()
- .addAll(
- Streams.stream(emptyKeys)
- .map(PathFragment::toString)
- .collect(ImmutableList.toImmutableList()))
- .build();
+ return emptyFilesSupplier.getExtraPaths(manifestKeys);
  }
 
  /**
@@ -387,6 +388,10 @@ public Map<PathFragment, Artifact> getRunfilesInputs(
  return builder.build();
  }
 
+ public boolean isLegacyExternalRunfiles() {
+ return legacyExternalRunfiles;
+ }
+
  /**
  * Helper class to handle munging the paths of external artifacts.
  */

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -136,6 +136,7 @@ java_library(
  "//src/main/java/com/google/devtools/build/lib/exec:executor_builder",
  "//src/main/java/com/google/devtools/build/lib/exec:module_action_context_registry",
  "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context",
+ "//src/main/java/com/google/devtools/build/lib/packages/semantics",
  "//src/main/java/com/google/devtools/build/lib/remote/options",
  "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
  "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",

src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java

@@ -27,6 +27,7 @@
 import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext.Encoding;
 import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
 import com.google.devtools.build.lib.exec.SpawnLogContext;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
 import com.google.devtools.build.lib.remote.options.RemoteOptions;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
@@ -104,6 +105,10 @@ private void initOutputs(CommandEnvironment env) throws IOException {
  new CompactSpawnLogContext(
  outputPath,
  env.getExecRoot().asFragment(),
+ env.getWorkspaceName(),
+ env.getOptions()
+ .getOptions(BuildLanguageOptions.class)
+ .experimentalSiblingRepositoryLayout,
  env.getOptions().getOptions(RemoteOptions.class),
  env.getRuntime().getFileSystem().getDigestFunction(),
  env.getXattrProvider());

src/main/java/com/google/devtools/build/lib/exec/BUILD

@@ -269,6 +269,7 @@ java_library(
  "//src/main/java/com/google/devtools/build/lib/actions:artifacts",
  "//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
  "//src/main/java/com/google/devtools/build/lib/actions:runfiles_supplier",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
  "//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
  "//src/main/java/com/google/devtools/build/lib/cmdline",
  "//src/main/java/com/google/devtools/build/lib/collect/nestedset",
@@ -297,8 +298,8 @@ java_library(
  ],
  deps = [
  "//src/main/java/com/google/devtools/build/lib/profiler",
- "//src/main/java/com/google/devtools/build/lib/util/io",
  "//src/main/java/com/google/devtools/build/lib/util:pair",
+ "//src/main/java/com/google/devtools/build/lib/util/io",
  "//src/main/protobuf:spawn_java_proto",
  "//third_party:guava",
  "//third_party:jsr305",