Apply eager fetch to local cache.

[ci skip-build-wheels]

src/rust/engine/process_execution/src/cache.rs

@@ -33,6 +33,7 @@ pub struct CommandRunner {
   inner: Arc<dyn crate::CommandRunner>,
   cache: PersistentCache,
   file_store: Store,
+  eager_fetch: bool,
   metadata: ProcessMetadata,
 }
 
@@ -41,12 +42,14 @@ impl CommandRunner {
     inner: Arc<dyn crate::CommandRunner>,
     cache: PersistentCache,
     file_store: Store,
+    eager_fetch: bool,
     metadata: ProcessMetadata,
   ) -> CommandRunner {
     CommandRunner {
       inner,
       cache,
       file_store,
+      eager_fetch,
       metadata,
     }
   }
@@ -196,23 +199,26 @@ impl CommandRunner {
       return Ok(None);
     };
 
-    // Ensure that all digests in the result are loadable, erroring if any are not.
-    // TODO: Make conditional on `eager_fetch`, since backtracking makes this unnecessary as well.
-    let _ = future::try_join_all(vec![
-      self
-        .file_store
-        .ensure_local_has_file(result.stdout_digest)
-        .boxed(),
-      self
-        .file_store
-        .ensure_local_has_file(result.stderr_digest)
-        .boxed(),
-      self
-        .file_store
-        .ensure_local_has_recursive_directory(result.output_directory.clone())
-        .boxed(),
-    ])
-    .await?;
+    // If eager_fetch is enabled, ensure that all digests in the result are loadable, erroring
+    // if any are not. If eager_fetch is disabled, a Digest which is discovered to be missing later
+    // on during execution will cause backtracking.
+    if self.eager_fetch {
+      let _ = future::try_join_all(vec![
+        self
+          .file_store
+          .ensure_local_has_file(result.stdout_digest)
+          .boxed(),
+        self
+          .file_store
+          .ensure_local_has_file(result.stderr_digest)
+          .boxed(),
+        self
+          .file_store
+          .ensure_local_has_recursive_directory(result.output_directory.clone())
+          .boxed(),
+      ])
+      .await?;
+    }
 
     Ok(Some(result))
   }

src/rust/engine/process_execution/src/cache_tests.rs

@@ -58,6 +58,7 @@ fn create_cached_runner(
     local.into(),
     cache,
     store,
+    true,
     ProcessMetadata::default(),
   ));
 

src/rust/engine/src/context.rs

@@ -266,12 +266,14 @@ impl Core {
     inner_runner: Arc<dyn CommandRunner>,
     full_store: &Store,
     local_cache: &PersistentCache,
+    eager_fetch: bool,
     process_execution_metadata: &ProcessMetadata,
   ) -> Arc<dyn CommandRunner> {
     Arc::new(process_execution::cache::CommandRunner::new(
       inner_runner,
       local_cache.clone(),
       full_store.clone(),
+      eager_fetch,
       process_execution_metadata.clone(),
     ))
   }
@@ -363,23 +365,19 @@ impl Core {
       None
     };
 
-    // TODO: The local cache eagerly fetches outputs independent of the `eager_fetch` flag. Once
-    // `eager_fetch` backtracks via https://github.com/pantsbuild/pants/issues/11331, the local
-    // cache will be able to obey `eager_fetch` as well, and can efficiently be used with remote
-    // execution.
-    let maybe_local_cached_runner =
-      if exec_strategy_opts.local_cache && !remoting_opts.execution_enable {
-        Some(Self::make_local_cached_runner(
-          maybe_remote_cached_runner
-            .clone()
-            .unwrap_or_else(|| leaf_runner.clone()),
-          full_store,
-          local_cache,
-          process_execution_metadata,
-        ))
-      } else {
-        None
-      };
+    let maybe_local_cached_runner = if exec_strategy_opts.local_cache {
+      Some(Self::make_local_cached_runner(
+        maybe_remote_cached_runner
+          .clone()
+          .unwrap_or_else(|| leaf_runner.clone()),
+        full_store,
+        local_cache,
+        remoting_opts.cache_eager_fetch,
+        process_execution_metadata,
+      ))
+    } else {
+      None
+    };
 
     Ok(
       vec![