Add RBE support for generated unresolved symlinks

Also adds support for such symlinks to the remote worker implementation.

src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTree.java

@@ -15,6 +15,7 @@
 
 import build.bazel.remote.execution.v2.Digest;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -33,7 +34,8 @@
 final class DirectoryTree {
 
  interface Visitor {
- void visitDirectory(PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs);
+ void visitDirectory(PathFragment dirname, List<FileNode> files, List<SymlinkNode> symlinks,
+ List<DirectoryNode> dirs);
  }
 
  abstract static class Node implements Comparable<Node> {
@@ -138,6 +140,44 @@ public boolean equals(Object o) {
  }
  return false;
  }
+
+ @Override
+ public String toString() {
+ return String.format("%s (hash: %s, size: %d)",
+ getPathSegment(), digest.getHash(), digest.getSizeBytes());
+ }
+ }
+
+ static class SymlinkNode extends Node {
+ private final String target;
+
+ SymlinkNode(String pathSegment, String target) {
+ super(pathSegment);
+ this.target = target;
+ }
+
+ public String getTarget() {
+ return target;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(super.hashCode(), target);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof SymlinkNode) {
+ SymlinkNode other = (SymlinkNode) o;
+ return super.equals(other) && Objects.equals(target, other.target);
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s --> %s", getPathSegment(), getTarget());
+ }
  }
 
  static class DirectoryNode extends Node {
@@ -203,10 +243,13 @@ private void visit(Visitor visitor, PathFragment dirname) {
  }
 
  List<FileNode> files = new ArrayList<>(dir.children.size());
+ List<SymlinkNode> symlinks = new ArrayList<>();
  List<DirectoryNode> dirs = new ArrayList<>();
  for (Node child : dir.children) {
  if (child instanceof FileNode) {
  files.add((FileNode) child);
+ } else if (child instanceof SymlinkNode) {
+ symlinks.add((SymlinkNode) child);
  } else if (child instanceof DirectoryNode) {
  dirs.add((DirectoryNode) child);
  visit(visitor, dirname.getRelative(child.pathSegment));
@@ -215,14 +258,14 @@ private void visit(Visitor visitor, PathFragment dirname) {
  String.format("Node type '%s' is not supported", child.getClass().getSimpleName()));
  }
  }
- visitor.visitDirectory(dirname, files, dirs);
+ visitor.visitDirectory(dirname, files, symlinks, dirs);
  }
 
  @Override
  public String toString() {
  Map<PathFragment, StringBuilder> m = new HashMap<>();
  visit(
- (dirname, files, dirs) -> {
+ (dirname, files, symlinks, dirs) -> {
  int depth = dirname.segmentCount() - 1;
  StringBuilder sb = new StringBuilder();
 
@@ -231,12 +274,10 @@ public String toString() {
  sb.append(dirname.getBaseName());
  sb.append("\n");
  }
- if (!files.isEmpty()) {
- for (FileNode file : files) {
- sb.append(" ".repeat(2 * (depth + 1)));
- sb.append(formatFile(file));
- sb.append("\n");
- }
+ for (Node fileOrSymlink : Iterables.concat(files, symlinks)) {
+ sb.append(" ".repeat(2 * (depth + 1)));
+ sb.append(fileOrSymlink);
+ sb.append("\n");
  }
  if (!dirs.isEmpty()) {
  for (DirectoryNode dir : dirs) {
@@ -264,10 +305,4 @@ public boolean equals(Object o) {
  DirectoryTree other = (DirectoryTree) o;
  return tree.equals(other.tree);
  }
-
- private static String formatFile(FileNode file) {
- return String.format(
- "%s (hash: %s, size: %d)",
- file.getPathSegment(), file.digest.getHash(), file.digest.getSizeBytes());
- }
 }

src/main/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeBuilder.java

@@ -18,11 +18,13 @@
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.actions.Artifact.DerivedArtifact;
+import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact;
 import com.google.devtools.build.lib.actions.FileArtifactValue;
 import com.google.devtools.build.lib.actions.MetadataProvider;
 import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
 import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode;
 import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
+import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.SymlinkNode;
 import com.google.devtools.build.lib.remote.util.DigestUtil;
 import com.google.devtools.build.lib.vfs.Dirent;
 import com.google.devtools.build.lib.vfs.Path;
@@ -142,25 +144,30 @@ private static int buildFromActionInputs(
  "missing metadata for '%s'",
  input.getExecPathString());
  switch (metadata.getType()) {
- case REGULAR_FILE:
+ case REGULAR_FILE: {
  Digest d = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
  Path inputPath = ActionInputHelper.toInputPath(input, execRoot);
  boolean childAdded =
  currDir.addChild(FileNode.createExecutable(path.getBaseName(), inputPath, d));
  return childAdded ? 1 : 0;
+ }
 
  case DIRECTORY:
  SortedMap<PathFragment, ActionInput> directoryInputs =
  explodeDirectory(input.getExecPath(), execRoot);
  return buildFromActionInputs(
  directoryInputs, metadataProvider, execRoot, digestUtil, tree);
 
- case SYMLINK:
- throw new IllegalStateException(
- String.format(
- "Encountered symlink input '%s', but all"
- + " symlinks should have been resolved by SkyFrame. This is a bug.",
- path));
+ case SYMLINK: {
+ Preconditions.checkState(input instanceof SpecialArtifact && input.isSymlink(),
+ "Encountered symlink input '%s', but all source symlinks should have been"
+ + " resolved by SkyFrame. This is a bug.",
+ path);
+ Path inputPath = ActionInputHelper.toInputPath(input, execRoot);
+ boolean childAdded = currDir.addChild(new SymlinkNode(path.getBaseName(),
+ inputPath.readSymbolicLink()));
+ return childAdded ? 1 : 0;
+ }
 
  case SPECIAL_FILE:
  throw new IOException(

src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java

@@ -19,6 +19,7 @@
 import build.bazel.remote.execution.v2.Directory;
 import build.bazel.remote.execution.v2.DirectoryNode;
 import build.bazel.remote.execution.v2.FileNode;
+import build.bazel.remote.execution.v2.SymlinkNode;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
@@ -94,6 +95,7 @@ private interface MerkleTreeDirectoryVisitor {
  @Nullable private final Directory rootProto;
  private final Digest rootDigest;
  private final SortedSet<DirectoryTree.FileNode> files;
+ private final SortedSet<DirectoryTree.SymlinkNode> symlinks;
  private final SortedMap<String, MerkleTree> directories;
  private final long inputFiles;
  private final long inputBytes;
@@ -102,6 +104,7 @@ private MerkleTree(
  @Nullable Directory rootProto,
  Digest rootDigest,
  SortedSet<DirectoryTree.FileNode> files,
+ SortedSet<DirectoryTree.SymlinkNode> symlinks,
  SortedMap<String, MerkleTree> directories,
  long inputFiles,
  long inputBytes) {
@@ -110,6 +113,7 @@ private MerkleTree(
  this.rootProto = rootProto;
  this.rootDigest = Preconditions.checkNotNull(rootDigest, "rootDigest");
  this.files = Preconditions.checkNotNull(files, "files");
+ this.symlinks = Preconditions.checkNotNull(symlinks, "symlinks");
  this.directories = Preconditions.checkNotNull(directories, "directories");
  this.inputFiles = inputFiles;
  this.inputBytes = inputBytes;
@@ -121,7 +125,9 @@ public Directory getRootProto() {
  return rootProto;
  }
 
- /** Returns the protobuf representation of the Merkle tree's root. */
+ /**
+ * Returns the protobuf representation of the Merkle tree's root.
+ */
  public Digest getRootDigest() {
  return rootDigest;
  }
@@ -130,6 +136,10 @@ private SortedSet<DirectoryTree.FileNode> getFiles() {
  return files;
  }
 
+ private SortedSet<DirectoryTree.SymlinkNode> getSymlinks() {
+ return symlinks;
+ }
+
  private SortedMap<String, MerkleTree> getDirectories() {
  return directories;
  }
@@ -243,13 +253,14 @@ private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) {
  null,
  digestUtil.compute(new byte[0]),
  ImmutableSortedSet.of(),
+ ImmutableSortedSet.of(),
  ImmutableSortedMap.of(),
  0,
  0);
  }
  Map<PathFragment, MerkleTree> m = new HashMap<>();
  tree.visit(
- (dirname, files, dirs) -> {
+ (dirname, files, symlinks, dirs) -> {
  SortedMap<String, MerkleTree> subDirs = new TreeMap<>();
  for (DirectoryTree.DirectoryNode dir : dirs) {
  PathFragment subDirname = dirname.getRelative(dir.getPathSegment());
@@ -258,7 +269,8 @@ private static MerkleTree build(DirectoryTree tree, DigestUtil digestUtil) {
  m.remove(subDirname), "subMerkleTree at '%s' was null", subDirname);
  subDirs.put(dir.getPathSegment(), subMerkleTree);
  }
- MerkleTree mt = buildMerkleTree(new TreeSet<>(files), subDirs, digestUtil);
+ MerkleTree mt = buildMerkleTree(new TreeSet<>(files), new TreeSet<>(symlinks), subDirs,
+ digestUtil);
  m.put(dirname, mt);
  });
  MerkleTree rootMerkleTree = m.get(PathFragment.EMPTY_FRAGMENT);
@@ -288,6 +300,10 @@ public static MerkleTree merge(Collection<MerkleTree> merkleTrees, DigestUtil di
  for (MerkleTree merkleTree : merkleTrees) {
  files.addAll(merkleTree.getFiles());
  }
+ SortedSet<DirectoryTree.SymlinkNode> symlinks = Sets.newTreeSet();
+ for (MerkleTree merkleTree : merkleTrees) {
+ symlinks.addAll(merkleTree.getSymlinks());
+ }
 
  // Group all Merkle trees per path.
  Multimap<String, MerkleTree> allDirsToMerge = ArrayListMultimap.create();
@@ -301,24 +317,28 @@ public static MerkleTree merge(Collection<MerkleTree> merkleTrees, DigestUtil di
  .forEach(
  (baseName, dirsToMerge) -> directories.put(baseName, merge(dirsToMerge, digestUtil)));
 
- return buildMerkleTree(files, directories, digestUtil);
+ return buildMerkleTree(files, symlinks, directories, digestUtil);
  }
 
  private static MerkleTree buildMerkleTree(
  SortedSet<DirectoryTree.FileNode> files,
+ SortedSet<DirectoryTree.SymlinkNode> symlinks,
  SortedMap<String, MerkleTree> directories,
  DigestUtil digestUtil) {
  Directory.Builder b = Directory.newBuilder();
  for (DirectoryTree.FileNode file : files) {
  b.addFiles(buildProto(file));
  }
+ for (DirectoryTree.SymlinkNode symlink : symlinks) {
+ b.addSymlinks(buildProto(symlink));
+ }
  for (Map.Entry<String, MerkleTree> nameAndDir : directories.entrySet()) {
  b.addDirectories(buildProto(nameAndDir.getKey(), nameAndDir.getValue()));
  }
  Directory protoDir = b.build();
  Digest protoDirDigest = digestUtil.compute(protoDir);
 
- long inputFiles = files.size();
+ long inputFiles = files.size() + symlinks.size();
  for (MerkleTree dir : directories.values()) {
  inputFiles += dir.getInputFiles();
  }
@@ -331,7 +351,8 @@ private static MerkleTree buildMerkleTree(
  inputBytes += dir.getInputBytes();
  }
 
- return new MerkleTree(protoDir, protoDirDigest, files, directories, inputFiles, inputBytes);
+ return new MerkleTree(protoDir, protoDirDigest, files, symlinks, directories, inputFiles,
+ inputBytes);
  }
 
  private static FileNode buildProto(DirectoryTree.FileNode file) {
@@ -349,6 +370,13 @@ private static DirectoryNode buildProto(String baseName, MerkleTree dir) {
  .build();
  }
 
+ private static SymlinkNode buildProto(DirectoryTree.SymlinkNode symlink) {
+ return SymlinkNode.newBuilder()
+ .setName(decodeBytestringUtf8(symlink.getPathSegment()))
+ .setTarget(decodeBytestringUtf8(symlink.getTarget()))
+ .build();
+ }
+
  private static PathOrBytes toPathOrBytes(DirectoryTree.FileNode file) {
  return file.getPath() != null
  ? new PathOrBytes(file.getPath())

src/test/java/com/google/devtools/build/lib/remote/merkletree/DirectoryTreeTest.java

@@ -21,6 +21,7 @@
 import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.DirectoryNode;
 import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.FileNode;
 import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.Node;
+import com.google.devtools.build.lib.remote.merkletree.DirectoryTree.SymlinkNode;
 import com.google.devtools.build.lib.remote.util.DigestUtil;
 import com.google.devtools.build.lib.vfs.DigestHashFunction;
 import com.google.devtools.build.lib.vfs.FileSystem;
@@ -119,7 +120,7 @@ protected Path createFile(String path, String content) throws IOException {
  static void assertLexicographicalOrder(DirectoryTree tree) {
  // Assert the lexicographical order as defined by the remote execution protocol
  tree.visit(
- (PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
+ (PathFragment dirname, List<FileNode> files, List<SymlinkNode> symlinks, List<DirectoryNode> dirs) -> {
  assertThat(files).isInStrictOrder();
  assertThat(dirs).isInStrictOrder();
  });
@@ -136,7 +137,7 @@ private static List<String> asPathSegments(List<? extends Node> nodes) {
  private static List<DirectoryNode> directoryNodesAtDepth(DirectoryTree tree, int depth) {
  List<DirectoryNode> directoryNodes = new ArrayList<>();
  tree.visit(
- (PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
+ (PathFragment dirname, List<FileNode> files, List<SymlinkNode> symlinks, List<DirectoryNode> dirs) -> {
  int currDepth = dirname.segmentCount();
  if (currDepth == depth) {
  directoryNodes.addAll(dirs);
@@ -148,7 +149,7 @@ private static List<DirectoryNode> directoryNodesAtDepth(DirectoryTree tree, int
  static List<FileNode> fileNodesAtDepth(DirectoryTree tree, int depth) {
  List<FileNode> fileNodes = new ArrayList<>();
  tree.visit(
- (PathFragment dirname, List<FileNode> files, List<DirectoryNode> dirs) -> {
+ (PathFragment dirname, List<FileNode> files, List<SymlinkNode> symlinks, List<DirectoryNode> dirs) -> {
  int currDepth = dirname.segmentCount();
  if (currDepth == depth) {
  fileNodes.addAll(files);