[7.1.0] Teach ExecLogConverter to read the compact format.

src/main/java/com/google/devtools/build/lib/exec/BUILD

@@ -288,12 +288,18 @@ java_library(
 
 java_library(
  name = "spawn_log_context_utils",
- srcs = ["StableSort.java"],
+ srcs = [
+ "SpawnLogReconstructor.java",
+ "StableSort.java",
+ ],
  deps = [
  "//src/main/java/com/google/devtools/build/lib/profiler",
  "//src/main/java/com/google/devtools/build/lib/util/io",
+ "//src/main/java/com/google/devtools/build/lib/util:pair",
  "//src/main/protobuf:spawn_java_proto",
  "//third_party:guava",
+ "//third_party:jsr305",
+ "@zstd-jni",
  ],
 )
 

src/main/java/com/google/devtools/build/lib/exec/SpawnLogReconstructor.java

@@ -0,0 +1,234 @@
+// Copyright 2024 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.exec;
+
+import com.github.luben.zstd.ZstdInputStream;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.exec.Protos.ExecLogEntry;
+import com.google.devtools.build.lib.exec.Protos.File;
+import com.google.devtools.build.lib.exec.Protos.SpawnExec;
+import com.google.devtools.build.lib.util.Pair;
+import com.google.devtools.build.lib.util.io.MessageInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayDeque;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import javax.annotation.Nullable;
+
+/** Reconstructs an execution log in expanded format from the compact format representation. */
+public final class SpawnLogReconstructor implements MessageInputStream<SpawnExec> {
+ private final ZstdInputStream in;
+
+ private final HashMap<Integer, File> fileMap = new HashMap<>();
+ private final HashMap<Integer, Pair<String, Collection<File>>> dirMap = new HashMap<>();
+ private final HashMap<Integer, File> symlinkMap = new HashMap<>();
+ private final HashMap<Integer, ExecLogEntry.InputSet> setMap = new HashMap<>();
+ private String hashFunctionName = "";
+
+ public SpawnLogReconstructor(InputStream in) throws IOException {
+ this.in = new ZstdInputStream(in);
+ }
+
+ @Override
+ @Nullable
+ public SpawnExec read() throws IOException {
+ ExecLogEntry entry;
+ while ((entry = ExecLogEntry.parseDelimitedFrom(in)) != null) {
+ switch (entry.getTypeCase()) {
+ case INVOCATION:
+ hashFunctionName = entry.getInvocation().getHashFunctionName();
+ break;
+ case FILE:
+ fileMap.put(entry.getId(), reconstructFile(entry.getFile()));
+ break;
+ case DIRECTORY:
+ dirMap.put(entry.getId(), reconstructDir(entry.getDirectory()));
+ break;
+ case UNRESOLVED_SYMLINK:
+ symlinkMap.put(entry.getId(), reconstructSymlink(entry.getUnresolvedSymlink()));
+ break;
+ case INPUT_SET:
+ setMap.put(entry.getId(), entry.getInputSet());
+ break;
+ case SPAWN:
+ return reconstructSpawnExec(entry.getSpawn());
+ default:
+ throw new IOException(
+ String.format("unknown entry type %d", entry.getTypeCase().getNumber()));
+ }
+ }
+ return null;
+ }
+
+ private SpawnExec reconstructSpawnExec(ExecLogEntry.Spawn entry) throws IOException {
+ SpawnExec.Builder builder =
+ SpawnExec.newBuilder()
+ .addAllCommandArgs(entry.getArgsList())
+ .addAllEnvironmentVariables(entry.getEnvVarsList())
+ .setTargetLabel(entry.getTargetLabel())
+ .setMnemonic(entry.getMnemonic())
+ .setExitCode(entry.getExitCode())
+ .setStatus(entry.getStatus())
+ .setRunner(entry.getRunner())
+ .setCacheHit(entry.getCacheHit())
+ .setRemotable(entry.getRemotable())
+ .setCacheable(entry.getCacheable())
+ .setRemoteCacheable(entry.getRemoteCacheable())
+ .setTimeoutMillis(entry.getTimeoutMillis())
+ .setMetrics(entry.getMetrics());
+
+ if (entry.hasPlatform()) {
+ builder.setPlatform(entry.getPlatform());
+ }
+
+ SortedMap<String, File> inputs = reconstructInputs(entry.getInputSetId());
+ SortedMap<String, File> toolInputs = reconstructInputs(entry.getToolSetId());
+
+ for (Map.Entry<String, File> e : inputs.entrySet()) {
+ File file = e.getValue();
+ if (toolInputs.containsKey(e.getKey())) {
+ file = file.toBuilder().setIsTool(true).build();
+ }
+ builder.addInputs(file);
+ }
+
+ SortedSet<String> listedOutputs = new TreeSet<>();
+
+ for (ExecLogEntry.Output output : entry.getOutputsList()) {
+ switch (output.getTypeCase()) {
+ case FILE_ID:
+ File file = getFromMap(fileMap, output.getFileId());
+ listedOutputs.add(file.getPath());
+ builder.addActualOutputs(file);
+ break;
+ case DIRECTORY_ID:
+ Pair<String, Collection<File>> dir = getFromMap(dirMap, output.getDirectoryId());
+ listedOutputs.add(dir.getFirst());
+ for (File dirFile : dir.getSecond()) {
+ builder.addActualOutputs(dirFile);
+ }
+ break;
+ case UNRESOLVED_SYMLINK_ID:
+ File symlink = getFromMap(symlinkMap, output.getUnresolvedSymlinkId());
+ listedOutputs.add(symlink.getPath());
+ builder.addActualOutputs(symlink);
+ break;
+ case INVALID_OUTPUT_PATH:
+ listedOutputs.add(output.getInvalidOutputPath());
+ break;
+ default:
+ throw new IOException(
+ String.format("unknown output type %d", output.getTypeCase().getNumber()));
+ }
+ }
+
+ builder.addAllListedOutputs(listedOutputs);
+
+ if (entry.hasDigest()) {
+ builder.setDigest(entry.getDigest().toBuilder().setHashFunctionName(hashFunctionName));
+ }
+
+ return builder.build();
+ }
+
+ private SortedMap<String, File> reconstructInputs(int setId) throws IOException {
+ TreeMap<String, File> inputs = new TreeMap<>();
+ ArrayDeque<Integer> setsToVisit = new ArrayDeque<>();
+ HashSet<Integer> visited = new HashSet<>();
+ if (setId != 0) {
+ setsToVisit.addLast(setId);
+ visited.add(setId);
+ }
+ while (!setsToVisit.isEmpty()) {
+ ExecLogEntry.InputSet set = getFromMap(setMap, setsToVisit.removeFirst());
+ for (int fileId : set.getFileIdsList()) {
+ if (visited.add(fileId)) {
+ File file = getFromMap(fileMap, fileId);
+ inputs.put(file.getPath(), file);
+ }
+ }
+ for (int dirId : set.getDirectoryIdsList()) {
+ if (visited.add(dirId)) {
+ Pair<String, Collection<File>> dir = getFromMap(dirMap, dirId);
+ for (File dirFile : dir.getSecond()) {
+ inputs.put(dirFile.getPath(), dirFile);
+ }
+ }
+ }
+ for (int symlinkId : set.getUnresolvedSymlinkIdsList()) {
+ if (visited.add(symlinkId)) {
+ File symlink = getFromMap(symlinkMap, symlinkId);
+ inputs.put(symlink.getPath(), symlink);
+ }
+ }
+ for (int transitiveSetId : set.getTransitiveSetIdsList()) {
+ if (visited.add(transitiveSetId)) {
+ setsToVisit.addLast(transitiveSetId);
+ }
+ }
+ }
+ return inputs;
+ }
+
+ private Pair<String, Collection<File>> reconstructDir(ExecLogEntry.Directory dir) {
+ ImmutableList.Builder<File> builder =
+ ImmutableList.builderWithExpectedSize(dir.getFilesCount());
+ for (ExecLogEntry.File dirFile : dir.getFilesList()) {
+ builder.add(reconstructFile(dir, dirFile));
+ }
+ return Pair.of(dir.getPath(), builder.build());
+ }
+
+ private File reconstructFile(ExecLogEntry.File entry) {
+ return reconstructFile(null, entry);
+ }
+
+ private File reconstructFile(
+ @Nullable ExecLogEntry.Directory parentDir, ExecLogEntry.File entry) {
+ File.Builder builder = File.newBuilder();
+ builder.setPath(
+ parentDir != null ? parentDir.getPath() + "/" + entry.getPath() : entry.getPath());
+ if (entry.hasDigest()) {
+ builder.setDigest(entry.getDigest().toBuilder().setHashFunctionName(hashFunctionName));
+ }
+ return builder.build();
+ }
+
+ private static File reconstructSymlink(ExecLogEntry.UnresolvedSymlink entry) {
+ return File.newBuilder()
+ .setPath(entry.getPath())
+ .setSymlinkTargetPath(entry.getTargetPath())
+ .build();
+ }
+
+ private static <T> T getFromMap(Map<Integer, T> map, int id) throws IOException {
+ T value = map.get(id);
+ if (value == null) {
+ throw new IOException(String.format("referenced entry %d is missing or has wrong type", id));
+ }
+ return value;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+}