Add experimental_remote_cache_key_ignore_stamping to skip volatile …

…stamping files on compute shared cache key
bazelbuild · Mar 27, 2023 · 1525844 · 1525844
1 parent 1a90fe5
commit 1525844
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 3 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java
@@ -126,6 +126,7 @@
 import io.reactivex.rxjava3.core.SingleObserver;
 import io.reactivex.rxjava3.disposables.Disposable;
 import io.reactivex.rxjava3.schedulers.Schedulers;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -337,7 +338,8 @@ public CachePolicy getWriteCachePolicy(Spawn spawn) {
   public boolean mayBeExecutedRemotely(Spawn spawn) {
     return remoteCache instanceof RemoteExecutionCache
         && remoteExecutor != null
-        && Spawns.mayBeExecutedRemotely(spawn);
+        && Spawns.mayBeExecutedRemotely(spawn)
+        && !(remoteOptions.remoteCacheKeyIgnoreStamping && hasVolatileArtifacts(spawn));
   }
 
   private SortedMap<PathFragment, ActionInput> buildOutputDirMap(Spawn spawn) {
@@ -395,7 +397,7 @@ private MerkleTree buildInputMerkleTree(
         inputMap = newInputMap;
       }
       return MerkleTree.build(
-          inputMap,
+          filterInputs(inputMap),
           toolSignature == null ? ImmutableSet.of() : toolSignature.toolInputs,
           context.getMetadataProvider(),
           execRoot,
@@ -427,7 +429,7 @@ public MerkleTree uncachedBuildMerkleTreeVisitor(
     ConcurrentLinkedQueue<MerkleTree> subMerkleTrees = new ConcurrentLinkedQueue<>();
     subMerkleTrees.add(
         MerkleTree.build(
-            walker.getLeavesInputMapping(),
+            filterInputs(walker.getLeavesInputMapping()),
             metadataProvider,
             execRoot,
             artifactPathResolver,
@@ -441,6 +443,20 @@ public MerkleTree uncachedBuildMerkleTreeVisitor(
     return MerkleTree.merge(subMerkleTrees, digestUtil);
   }
 
+  private SortedMap<PathFragment, ActionInput> filterInputs(SortedMap<PathFragment, ActionInput> inputs) {
+    if (!remoteOptions.remoteCacheKeyIgnoreStamping) {
+      return inputs;
+    }
+    SortedMap<PathFragment, ActionInput> result = new TreeMap<>();
+    for (Entry<PathFragment, ActionInput> entry : inputs.entrySet()) {
+      ActionInput input = entry.getValue();
+      if (!isConstantMetadata(input)) {
+        result.put(entry.getKey(), input);
+      }
+    }
+    return result;
+  }
+
   @Nullable
   private static ByteString buildSalt(Spawn spawn) {
     String workspace =
@@ -1537,6 +1553,23 @@ void report(Event evt) {
     }
   }
 
+  private static boolean hasVolatileArtifacts(Spawn spawn) {
+    var inputFiles = spawn.getInputFiles();
+    for (ActionInput inputFile : inputFiles.getLeaves()) {
+      if (isConstantMetadata(inputFile)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean isConstantMetadata(ActionInput input) {
+    if (input instanceof Artifact) {
+      return ((Artifact) input).isConstantMetadata();
+    }
+    return false;
+  }
+
   /**
    * A simple value class combining a hash of the tool inputs (and their digests) as well as a set
    * of the relative paths of all tool inputs.

diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -83,6 +83,14 @@ public final class RemoteOptions extends CommonRemoteOptions {
               + "disable TLS.")
   public String remoteExecutor;
 
+  @Option(
+    name = "experimental_remote_cache_key_ignore_stamping",
+    defaultValue = "false",
+    documentationCategory = OptionDocumentationCategory.REMOTE,
+    effectTags = {OptionEffectTag.UNKNOWN},
+    help = "Don't use volatile stamping data in shared cache key. Also disable remote execution for stamping actions.")
+  public boolean remoteCacheKeyIgnoreStamping;
+
   @Option(
       name = "experimental_remote_execution_keepalive",
       defaultValue = "false",