diff --git a/src/main/java/com/google/devtools/build/lib/actions/BUILD b/src/main/java/com/google/devtools/build/lib/actions/BUILD
index 4419d2f3effb81..25b3698457a432 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/actions/BUILD
@@ -157,6 +157,7 @@ java_library(
":thread_state_receiver",
"//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
"//src/main/java/com/google/devtools/build/lib/analysis:config/core_options",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
"//src/main/java/com/google/devtools/build/lib/analysis/platform",
"//src/main/java/com/google/devtools/build/lib/bugreport",
"//src/main/java/com/google/devtools/build/lib/buildeventstream",
@@ -498,6 +499,7 @@ java_library(
deps = [
":artifacts",
"//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/java/net/starlark/java/eval",
diff --git a/src/main/java/com/google/devtools/build/lib/actions/RunfilesSupplier.java b/src/main/java/com/google/devtools/build/lib/actions/RunfilesSupplier.java
index a57e1452365326..b966ddb14bad97 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/RunfilesSupplier.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/RunfilesSupplier.java
@@ -16,6 +16,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.analysis.SymlinkEntry;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue.RunfileSymlinksMode;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.vfs.PathFragment;
@@ -66,4 +67,32 @@ public interface RunfilesSupplier extends StarlarkValue {
* #getRunfilesDirs} returns a set of size 1.
*/
RunfilesSupplier withOverriddenRunfilesDir(PathFragment newRunfilesDir);
+
+ /**
+ * Returns artifacts the runfiles tree contain symlinks to at their canonical locations.
+ *
+ *
This does not include artifacts that only the symlinks and root symlinks point to.
+ */
+ NestedSet getArtifactsAtCanonicalLocationsForLogging();
+
+ /**
+ * Returns the set of names of implicit empty files to materialize.
+ *
+ *
If this runfiles tree does not implicitly add empty files, implementations should have a
+ * dedicated fast path that returns an empty set without traversing the tree.
+ */
+ Iterable getEmptyFilenamesForLogging();
+
+ /** Returns the set of custom symlink entries. */
+ NestedSet getSymlinksForLogging();
+
+ /** Returns the set of root symlinks. */
+ NestedSet getRootSymlinksForLogging();
+
+ /** Returns the repo mapping manifest if it exists. */
+ @Nullable
+ Artifact getRepoMappingManifestForLogging();
+
+ /** Whether this runfiles tree materializes external runfiles also at their legacy locations. */
+ boolean isLegacyExternalRunfiles();
}
diff --git a/src/main/java/com/google/devtools/build/lib/actions/RunfilesTree.java b/src/main/java/com/google/devtools/build/lib/actions/RunfilesTree.java
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
index 04854ed8317c0c..c724792fe93f43 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java
@@ -250,10 +250,17 @@ public NestedSet getSymlinks() {
@Override
public Depset /**/ getEmptyFilenamesForStarlark() {
- return Depset.of(String.class, getEmptyFilenames());
+ return Depset.of(
+ String.class,
+ NestedSetBuilder.wrap(
+ Order.STABLE_ORDER,
+ Iterables.transform(getEmptyFilenames(), PathFragment::getPathString)));
}
- public NestedSet getEmptyFilenames() {
+ public Iterable getEmptyFilenames() {
+ if (emptyFilesSupplier == DUMMY_EMPTY_FILES_SUPPLIER) {
+ return ImmutableList.of();
+ }
Set manifestKeys =
Streams.concat(
symlinks.toList().stream().map(SymlinkEntry::getPath),
@@ -264,13 +271,7 @@ public NestedSet getEmptyFilenames() {
? artifact.getOutputDirRelativePath(false)
: artifact.getRunfilesPath()))
.collect(ImmutableSet.toImmutableSet());
- Iterable emptyKeys = emptyFilesSupplier.getExtraPaths(manifestKeys);
- return NestedSetBuilder.stableOrder()
- .addAll(
- Streams.stream(emptyKeys)
- .map(PathFragment::toString)
- .collect(ImmutableList.toImmutableList()))
- .build();
+ return emptyFilesSupplier.getExtraPaths(manifestKeys);
}
/**
@@ -387,6 +388,10 @@ public Map getRunfilesInputs(
return builder.build();
}
+ public boolean isLegacyExternalRunfiles() {
+ return legacyExternalRunfiles;
+ }
+
/**
* Helper class to handle munging the paths of external artifacts.
*/
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
index ae5d5a454476d0..0fe03e3831053c 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD
@@ -136,6 +136,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/exec:executor_builder",
"//src/main/java/com/google/devtools/build/lib/exec:module_action_context_registry",
"//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context",
+ "//src/main/java/com/google/devtools/build/lib/packages/semantics",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
index 5be63fb76b23d0..19710e2d1b93ef 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
@@ -27,6 +27,7 @@
import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext.Encoding;
import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
import com.google.devtools.build.lib.exec.SpawnLogContext;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.runtime.BlazeModule;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
@@ -104,6 +105,10 @@ private void initOutputs(CommandEnvironment env) throws IOException {
new CompactSpawnLogContext(
outputPath,
env.getExecRoot().asFragment(),
+ env.getWorkspaceName(),
+ env.getOptions()
+ .getOptions(BuildLanguageOptions.class)
+ .experimentalSiblingRepositoryLayout,
env.getOptions().getOptions(RemoteOptions.class),
env.getRuntime().getFileSystem().getDigestFunction(),
env.getXattrProvider());
diff --git a/src/main/java/com/google/devtools/build/lib/exec/BUILD b/src/main/java/com/google/devtools/build/lib/exec/BUILD
index cfd94913148ad0..1dc4fed16b7f64 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/exec/BUILD
@@ -269,6 +269,7 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/actions:runfiles_supplier",
+ "//src/main/java/com/google/devtools/build/lib/analysis:symlink_entry",
"//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
@@ -297,8 +298,8 @@ java_library(
],
deps = [
"//src/main/java/com/google/devtools/build/lib/profiler",
- "//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/util:pair",
+ "//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/protobuf:spawn_java_proto",
"//third_party:guava",
"//third_party:jsr305",
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
index 1b2273d6b527c0..5562b5e9e2c231 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
@@ -17,7 +17,9 @@
import static com.google.common.base.Preconditions.checkState;
import com.github.luben.zstd.ZstdOutputStream;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import com.google.devtools.build.lib.actions.AbstractAction;
import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.Artifact;
@@ -28,6 +30,7 @@
import com.google.devtools.build.lib.actions.SpawnResult;
import com.google.devtools.build.lib.actions.Spawns;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
+import com.google.devtools.build.lib.analysis.SymlinkEntry;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.concurrent.AbstractQueueVisitor;
@@ -49,14 +52,13 @@
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.Symlinks;
import com.google.devtools.build.lib.vfs.XattrProvider;
-import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import com.google.errorprone.annotations.CheckReturnValue;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -136,12 +138,15 @@ private interface ExecLogEntrySupplier {
}
private final PathFragment execRoot;
+ private final String workspaceName;
+ private final boolean siblingRepositoryLayout;
@Nullable private final RemoteOptions remoteOptions;
private final DigestHashFunction digestHashFunction;
private final XattrProvider xattrProvider;
// Maps a key identifying an entry into its ID.
- // Each key is either a NestedSet.Node or the String path of a file, directory or symlink.
+ // Each key is either a NestedSet.Node or the String path of a file, directory, symlink or
+ // runfiles tree.
// Only entries that are likely to be referenced by future entries are stored.
// Use a specialized map for minimal memory footprint.
@GuardedBy("this")
@@ -157,11 +162,15 @@ private interface ExecLogEntrySupplier {
public CompactSpawnLogContext(
Path outputPath,
PathFragment execRoot,
+ String workspaceName,
+ boolean siblingRepositoryLayout,
@Nullable RemoteOptions remoteOptions,
DigestHashFunction digestHashFunction,
XattrProvider xattrProvider)
throws IOException, InterruptedException {
this.execRoot = execRoot;
+ this.workspaceName = workspaceName;
+ this.siblingRepositoryLayout = siblingRepositoryLayout;
this.remoteOptions = remoteOptions;
this.digestHashFunction = digestHashFunction;
this.xattrProvider = xattrProvider;
@@ -178,13 +187,14 @@ private static MessageOutputStream getOutputStream(Path path) thro
}
private void logInvocation() throws IOException, InterruptedException {
- logEntry(
- null,
+ logEntryWithoutId(
() ->
ExecLogEntry.newBuilder()
.setInvocation(
ExecLogEntry.Invocation.newBuilder()
- .setHashFunctionName(digestHashFunction.toString())));
+ .setHashFunctionName(digestHashFunction.toString())
+ .setWorkspaceRunfilesDirectory(workspaceName)
+ .setSiblingRepositoryLayout(siblingRepositoryLayout)));
}
@Override
@@ -223,21 +233,16 @@ public void logSpawn(
for (ActionInput output : spawn.getOutputFiles()) {
Path path = fileSystem.getPath(execRoot.getRelative(output.getExecPath()));
if (!output.isDirectory() && !output.isSymlink() && path.isFile()) {
- builder.addOutputs(
- ExecLogEntry.Output.newBuilder()
- .setFileId(logFile(output, path, inputMetadataProvider)));
+ builder.addOutputsBuilder().setOutputId(logFile(output, path, inputMetadataProvider));
} else if (!output.isSymlink() && path.isDirectory()) {
// TODO(tjgq): Tighten once --incompatible_disallow_unsound_directory_outputs is gone.
- builder.addOutputs(
- ExecLogEntry.Output.newBuilder()
- .setDirectoryId(logDirectory(output, path, inputMetadataProvider)));
+ builder
+ .addOutputsBuilder()
+ .setOutputId(logDirectory(output, path, inputMetadataProvider));
} else if (output.isSymlink() && path.isSymbolicLink()) {
- builder.addOutputs(
- ExecLogEntry.Output.newBuilder()
- .setUnresolvedSymlinkId(logUnresolvedSymlink(output, path)));
+ builder.addOutputsBuilder().setOutputId(logUnresolvedSymlink(output, path));
} else {
- builder.addOutputs(
- ExecLogEntry.Output.newBuilder().setInvalidOutputPath(output.getExecPathString()));
+ builder.addOutputsBuilder().setInvalidOutputPath(output.getExecPathString());
}
}
@@ -259,7 +264,7 @@ public void logSpawn(
builder.setMetrics(getSpawnMetricsProto(result));
try (SilentCloseable c1 = Profiler.instance().profile("logEntry")) {
- logEntry(null, () -> ExecLogEntry.newBuilder().setSpawn(builder));
+ logEntryWithoutId(() -> ExecLogEntry.newBuilder().setSpawn(builder));
}
}
}
@@ -285,7 +290,7 @@ public void logSymlinkAction(AbstractAction action) throws IOException, Interrup
builder.setMnemonic(action.getMnemonic());
try (SilentCloseable c1 = Profiler.instance().profile("logEntry")) {
- logEntry(null, () -> ExecLogEntry.newBuilder().setSymlinkAction(builder));
+ logEntryWithoutId(() -> ExecLogEntry.newBuilder().setSymlinkAction(builder));
}
}
}
@@ -293,8 +298,8 @@ public void logSymlinkAction(AbstractAction action) throws IOException, Interrup
/**
* Logs the inputs.
*
- * @return the entry ID of the {@link ExecLogEntry.Set} describing the inputs, or 0 if there are
- * no inputs.
+ * @return the entry ID of the {@link ExecLogEntry.InputSet} describing the inputs, or 0 if there
+ * are no inputs.
*/
private int logInputs(
Spawn spawn, InputMetadataProvider inputMetadataProvider, FileSystem fileSystem)
@@ -324,7 +329,7 @@ private int logInputs(
inputMetadataProvider));
}
- return logNestedSet(
+ return logInputSet(
spawn.getInputFiles(),
additionalDirectoryIds.build(),
inputMetadataProvider,
@@ -335,13 +340,13 @@ private int logInputs(
/**
* Logs the tool inputs.
*
- * @return the entry ID of the {@link ExecLogEntry.Set} describing the tool inputs, or 0 if there
- * are no tool inputs.
+ * @return the entry ID of the {@link ExecLogEntry.InputSet} describing the tool inputs, or 0 if
+ * there are no tool inputs.
*/
private int logTools(
Spawn spawn, InputMetadataProvider inputMetadataProvider, FileSystem fileSystem)
throws IOException, InterruptedException {
- return logNestedSet(
+ return logInputSet(
spawn.getToolFiles(),
ImmutableList.of(),
inputMetadataProvider,
@@ -359,7 +364,7 @@ private int logTools(
* @return the entry ID of the {@link ExecLogEntry.InputSet} describing the nested set, or 0 if
* the nested set is empty.
*/
- private int logNestedSet(
+ private int logInputSet(
NestedSet extends ActionInput> set,
Collection additionalDirectoryIds,
InputMetadataProvider inputMetadataProvider,
@@ -374,12 +379,12 @@ private int logNestedSet(
shared ? set.toNode() : null,
() -> {
ExecLogEntry.InputSet.Builder builder =
- ExecLogEntry.InputSet.newBuilder().addAllDirectoryIds(additionalDirectoryIds);
+ ExecLogEntry.InputSet.newBuilder().addAllInputIds(additionalDirectoryIds);
for (NestedSet extends ActionInput> transitive : set.getNonLeaves()) {
checkState(!transitive.isEmpty());
builder.addTransitiveSetIds(
- logNestedSet(
+ logInputSet(
transitive,
/* additionalDirectoryIds= */ ImmutableList.of(),
inputMetadataProvider,
@@ -388,28 +393,87 @@ private int logNestedSet(
}
for (ActionInput input : set.getLeaves()) {
- // Runfiles are logged separately.
- if (input instanceof Artifact && ((Artifact) input).isMiddlemanArtifact()) {
+ if (input instanceof Artifact artifact && artifact.isMiddlemanArtifact()) {
+ RunfilesTree runfilesTree =
+ inputMetadataProvider.getRunfilesMetadata(input).getRunfilesTree();
+ builder.addInputIds(
+ logRunfilesTree(
+ runfilesTree,
+ inputMetadataProvider,
+ fileSystem,
+ // If the nested set containing the runfiles tree isn't shared (i.e., it
+ // contains inputs, not tools), the runfiles are also likely not shared. This
+ // avoids storing the runfiles tree of a test.
+ shared));
continue;
}
// Filesets are logged separately.
- if (input instanceof Artifact && ((Artifact) input).isFileset()) {
+ if (input instanceof Artifact artifact && artifact.isFileset()) {
continue;
}
- Path path = fileSystem.getPath(execRoot.getRelative(input.getExecPath()));
- if (isInputDirectory(input, path, inputMetadataProvider)) {
- builder.addDirectoryIds(logDirectory(input, path, inputMetadataProvider));
- } else if (input.isSymlink()) {
- builder.addUnresolvedSymlinkIds(logUnresolvedSymlink(input, path));
- } else {
- builder.addFileIds(logFile(input, path, inputMetadataProvider));
- }
+
+ builder.addInputIds(logInput(input, inputMetadataProvider, fileSystem));
}
return ExecLogEntry.newBuilder().setInputSet(builder);
});
}
+ /**
+ * Logs a nested set of {@link SymlinkEntry}.
+ *
+ * @return the entry ID of the {@link ExecLogEntry.SymlinkEntrySet} describing the nested set, or
+ * 0 if the nested set is empty.
+ */
+ private int logSymlinkEntries(
+ NestedSet symlinks,
+ InputMetadataProvider inputMetadataProvider,
+ FileSystem fileSystem)
+ throws IOException, InterruptedException {
+ if (symlinks.isEmpty()) {
+ return 0;
+ }
+
+ return logEntry(
+ symlinks.toNode(),
+ () -> {
+ ExecLogEntry.SymlinkEntrySet.Builder builder = ExecLogEntry.SymlinkEntrySet.newBuilder();
+
+ for (NestedSet transitive : symlinks.getNonLeaves()) {
+ checkState(!transitive.isEmpty());
+ builder.addTransitiveSetIds(
+ logSymlinkEntries(transitive, inputMetadataProvider, fileSystem));
+ }
+
+ for (SymlinkEntry input : symlinks.getLeaves()) {
+ builder.putDirectEntries(
+ input.getPathString(),
+ logInput(input.getArtifact(), inputMetadataProvider, fileSystem));
+ }
+
+ return ExecLogEntry.newBuilder().setSymlinkEntries(builder);
+ });
+ }
+
+ /**
+ * Logs a single input that is either a file, a directory or a symlink.
+ *
+ * @return the entry ID of the {@link ExecLogEntry} describing the input.
+ */
+ private int logInput(
+ ActionInput input, InputMetadataProvider inputMetadataProvider, FileSystem fileSystem)
+ throws IOException, InterruptedException {
+ Path path = fileSystem.getPath(execRoot.getRelative(input.getExecPath()));
+ // TODO(tjgq): Tighten once --incompatible_disallow_unsound_directory_outputs is gone.
+ if (isInputDirectory(input, path, inputMetadataProvider)) {
+ return logDirectory(input, path, inputMetadataProvider);
+ } else if (input.isSymlink()) {
+ return logUnresolvedSymlink(input, path);
+ } else {
+ return logFile(input, path, inputMetadataProvider);
+ }
+ }
+
/**
* Logs a file.
*
@@ -449,7 +513,7 @@ private int logFile(ActionInput input, Path path, InputMetadataProvider inputMet
* Logs a directory.
*
*
This may be either a source directory, a fileset or an output directory. For runfiles,
- * {@link #logRunfilesDirectory} must be used instead.
+ * {@link #logRunfilesTree} must be used instead.
*
* @param input the input representing the directory.
* @param root the path to the directory, which must have already been verified to be of the
@@ -466,64 +530,69 @@ private int logDirectory(
.setDirectory(
ExecLogEntry.Directory.newBuilder()
.setPath(input.getExecPathString())
- .addAllFiles(
- expandDirectory(root, /* pathPrefix= */ null, inputMetadataProvider))));
+ .addAllFiles(expandDirectory(root, inputMetadataProvider))));
}
/**
- * Logs a runfiles directory.
+ * Logs a runfiles directory by storing the information in its {@link RunfilesTree}.
*
- *
We can't use {@link #logDirectory} because the runfiles symlink tree might not have been
- * materialized on disk. We must follow the mappings to the actual location of the artifacts.
+ *
Since runfiles trees can be very large and, for tests, are only used by a single spawn, we
+ * store them in the log as a special entry that references the nested set of artifacts instead of
+ * as a flat directory.
*
- * @param root the path to the runfiles directory
- * @param mapping a map from runfiles-relative path to the underlying artifact, or null for an
- * empty file
- * @return the entry ID of the {@link ExecLogEntry.Directory} describing the directory.
+ * @param shared whether this runfiles tree is likely to be contained in more than one Spawn's
+ * inputs
+ * @return the entry ID of the {@link ExecLogEntry.RunfilesTree} describing the directory.
*/
- private int logRunfilesDirectory(
- PathFragment root,
- Map mapping,
+ private int logRunfilesTree(
+ RunfilesTree runfilesTree,
InputMetadataProvider inputMetadataProvider,
- FileSystem fileSystem)
+ FileSystem fileSystem,
+ boolean shared)
throws IOException, InterruptedException {
return logEntry(
- root.getPathString(),
+ shared ? runfilesTree.getExecPath().getPathString() : null,
() -> {
- ExecLogEntry.Directory.Builder builder =
- ExecLogEntry.Directory.newBuilder().setPath(root.getPathString());
-
- for (Map.Entry entry : mapping.entrySet()) {
- String runfilesPath = entry.getKey().getPathString();
- Artifact input = entry.getValue();
-
- if (input == null) {
- // Empty file.
- builder.addFiles(ExecLogEntry.File.newBuilder().setPath(runfilesPath));
- continue;
- }
-
- Path path = fileSystem.getPath(execRoot.getRelative(input.getExecPath()));
+ Preconditions.checkState(workspaceName.equals(runfilesTree.getWorkspaceName()));
- if (isInputDirectory(input, path, inputMetadataProvider)) {
- builder.addAllFiles(expandDirectory(path, runfilesPath, inputMetadataProvider));
- continue;
- }
-
- Digest digest =
- computeDigest(
- input,
- path,
- inputMetadataProvider,
- xattrProvider,
- digestHashFunction,
- /* includeHashFunctionName= */ false);
+ ExecLogEntry.RunfilesTree.Builder builder =
+ ExecLogEntry.RunfilesTree.newBuilder()
+ .setPath(runfilesTree.getExecPath().getPathString())
+ .setLegacyExternalRunfiles(runfilesTree.isLegacyExternalRunfiles());
- builder.addFiles(
- ExecLogEntry.File.newBuilder().setPath(runfilesPath).setDigest(digest));
+ builder.setInputSetId(
+ logInputSet(
+ runfilesTree.getArtifactsAtCanonicalLocationsForLogging(),
+ /* additionalDirectoryIds= */ ImmutableList.of(),
+ inputMetadataProvider,
+ fileSystem,
+ // The runfiles tree itself is shared, but the nested set is unique to the tree as
+ // it contains the executable.
+ /* shared= */ false));
+ builder.setSymlinksId(
+ logSymlinkEntries(
+ runfilesTree.getSymlinksForLogging(), inputMetadataProvider, fileSystem));
+ builder.setRootSymlinksId(
+ logSymlinkEntries(
+ runfilesTree.getRootSymlinksForLogging(), inputMetadataProvider, fileSystem));
+ builder.addAllEmptyFiles(
+ Iterables.transform(
+ runfilesTree.getEmptyFilenamesForLogging(), PathFragment::getPathString));
+ Artifact repoMappingManifest = runfilesTree.getRepoMappingManifestForLogging();
+ if (repoMappingManifest != null) {
+ builder.setRepoMappingManifest(
+ ExecLogEntry.File.newBuilder()
+ .setDigest(
+ computeDigest(
+ repoMappingManifest,
+ repoMappingManifest.getPath(),
+ inputMetadataProvider,
+ xattrProvider,
+ digestHashFunction,
+ /* includeHashFunctionName= */ false)));
}
- return ExecLogEntry.newBuilder().setDirectory(builder);
+ return ExecLogEntry.newBuilder().setRunfilesTree(builder);
});
}
@@ -531,19 +600,15 @@ private int logRunfilesDirectory(
* Expands a directory.
*
* @param root the path to the directory
- * @param pathPrefix a prefix to prepend to each child path
* @return the list of files transitively contained in the directory
*/
private List expandDirectory(
- Path root, @Nullable String pathPrefix, InputMetadataProvider inputMetadataProvider)
+ Path root, InputMetadataProvider inputMetadataProvider)
throws IOException, InterruptedException {
ArrayList files = new ArrayList<>();
visitDirectory(
root,
(child) -> {
- String childPath = pathPrefix != null ? pathPrefix + "/" : "";
- childPath += child.relativeTo(root).getPathString();
-
Digest digest =
computeDigest(
/* input= */ null,
@@ -554,14 +619,17 @@ private List expandDirectory(
/* includeHashFunctionName= */ false);
ExecLogEntry.File file =
- ExecLogEntry.File.newBuilder().setPath(childPath).setDigest(digest).build();
+ ExecLogEntry.File.newBuilder()
+ .setPath(child.relativeTo(root).getPathString())
+ .setDigest(digest)
+ .build();
synchronized (files) {
files.add(file);
}
});
- Collections.sort(files, EXEC_LOG_ENTRY_FILE_COMPARATOR);
+ files.sort(EXEC_LOG_ENTRY_FILE_COMPARATOR);
return files;
}
@@ -587,6 +655,16 @@ private int logUnresolvedSymlink(ActionInput input, Path path)
.setTargetPath(path.readSymbolicLink().getPathString())));
}
+ /**
+ * Ensures an entry is written to the log without an ID.
+ *
+ * @param supplier called to compute the entry; may cause other entries to be logged
+ */
+ private synchronized void logEntryWithoutId(ExecLogEntrySupplier supplier)
+ throws IOException, InterruptedException {
+ outputStream.write(supplier.get().build());
+ }
+
/**
* Ensures an entry is written to the log and returns its assigned ID.
*
@@ -597,14 +675,15 @@ private int logUnresolvedSymlink(ActionInput input, Path path)
* @param supplier called to compute the entry; may cause other entries to be logged
* @return the entry ID
*/
- @CanIgnoreReturnValue
+ @CheckReturnValue
private synchronized int logEntry(@Nullable Object key, ExecLogEntrySupplier supplier)
throws IOException, InterruptedException {
try (SilentCloseable c = Profiler.instance().profile("logEntry/synchronized")) {
if (key == null) {
// No need to check for a previously added entry.
+ ExecLogEntry.Builder entry = supplier.get();
int id = nextEntryId++;
- outputStream.write(supplier.get().setId(id).build());
+ outputStream.write(entry.setId(id).build());
return id;
}
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
index 24e4a67357d037..17eb9c3948bb4c 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
@@ -122,7 +122,7 @@ private static void addMapping(
/** Adds runfiles inputs from runfilesSupplier to inputMappings. */
@VisibleForTesting
- void addRunfilesToInputs(
+ public void addRunfilesToInputs(
Map inputMap,
RunfilesSupplier runfilesSupplier,
InputMetadataProvider inputMetadataProvider,
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java
index 0b495b775348ea..cb095588b342e1 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java
@@ -13,38 +13,110 @@
// limitations under the License.
package com.google.devtools.build.lib.exec;
+import static com.google.common.base.Preconditions.checkState;
+import static com.google.common.collect.ImmutableList.toImmutableList;
+
import com.github.luben.zstd.ZstdInputStream;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.exec.Protos.ExecLogEntry;
import com.google.devtools.build.lib.exec.Protos.File;
import com.google.devtools.build.lib.exec.Protos.SpawnExec;
-import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.io.MessageInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayDeque;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
import javax.annotation.Nullable;
/** Reconstructs an execution log in expanded format from the compact format representation. */
public final class SpawnLogReconstructor implements MessageInputStream {
+ // The path of the repo mapping manifest file under the runfiles tree.
+ private static final String REPO_MAPPING_MANIFEST = "_repo_mapping";
+
+ // Examples:
+ // * bazel-out/k8-fastbuild/bin/pkg/file.txt (repo: null, path: "pkg/file.txt")
+ // * bazel-out/k8-fastbuild/bin/external/some_repo/pkg/file.txt (repo: "some_repo", path:
+ // "pkg/file.txt")
+ private static final Pattern DEFAULT_GENERATED_FILE_RUNFILES_PATH_PATTERN =
+ Pattern.compile("(?:bazel|blaze)-out/[^/]+/[^/]+/(?:external/(?[^/]+)/)?(?.+)");
+
+ // Examples:
+ // * pkg/file.txt (repo: null, path: "pkg/file.txt")
+ // * external/some_repo/pkg/file.txt (repo: "some_repo", path: "pkg/file.txt")
+ private static final Pattern DEFAULT_SOURCE_FILE_RUNFILES_PATH_PATTERN =
+ Pattern.compile("(?:external/(?[^/]+)/)?(?.+)");
+
+ // Examples:
+ // * bazel-out/k8-fastbuild/bin/pkg/file.txt (repo: null, path: "pkg/file.txt")
+ // * bazel-out/some_repo/k8-fastbuild/bin/pkg/file.txt (repo: "some_repo", path: "pkg/file.txt")
+ // * bazel-out/k8-fastbuild/k8-fastbuild/bin/pkg/file.txt (repo: "k8-fastbuild", path:
+ // "pkg/file.txt")
+ //
+ // Repo names are distinguished from mnemonics via a positive lookahead on the following segment,
+ // which in the case of a repo name is a mnemonic and thus contains a hyphen, whereas a mnemonic
+ // is followed by an output directory name, which does not contain a hyphen unless it is
+ // "coverage-metadata" (which in turn is not likely to be a mnemonic).
+ private static final Pattern SIBLING_LAYOUT_GENERATED_FILE_RUNFILES_PATH_PATTERN =
+ Pattern.compile(
+ "(?:bazel|blaze)-out/(?:(?[^/]+(?=/[^/]+-[^/]+/)(?!/coverage-metadata/))/)?[^/]+/[^/]+/(?.+)");
+
+ // Examples:
+ // * pkg/file.txt (repo: null, path: "pkg/file.txt")
+ // * ../some_repo/pkg/file.txt (repo: "some_repo", path: "pkg/file.txt")
+ private static final Pattern SIBLING_LAYOUT_SOURCE_FILE_RUNFILES_PATH_PATTERN =
+ Pattern.compile("(?:\\.\\./(?[^/]+)/)?(?.+)");
+
private final ZstdInputStream in;
- private final HashMap fileMap = new HashMap<>();
- private final HashMap>> dirMap = new HashMap<>();
- private final HashMap symlinkMap = new HashMap<>();
- private final HashMap setMap = new HashMap<>();
+ /** Represents a reconstructed input file, symlink, or directory. */
+ private sealed interface Input {
+ String path();
+
+ record File(Protos.File file) implements Input {
+ @Override
+ public String path() {
+ return file.getPath();
+ }
+ }
+
+ record Symlink(Protos.File symlink) implements Input {
+ @Override
+ public String path() {
+ return symlink.getPath();
+ }
+ }
+
+ record Directory(String path, Collection files) implements Input {}
+ }
+
+ // Stores both Inputs and InputSets. Bazel uses consecutive IDs starting from 1, so we can use
+ // an ArrayList to store them together efficiently.
+ private final ArrayList