diff --git a/src/rust/engine/process_execution/src/cache.rs b/src/rust/engine/process_execution/src/cache.rs index 132486da46f3..baaf7bc73bef 100644 --- a/src/rust/engine/process_execution/src/cache.rs +++ b/src/rust/engine/process_execution/src/cache.rs @@ -33,6 +33,7 @@ pub struct CommandRunner { inner: Arc, cache: PersistentCache, file_store: Store, + eager_fetch: bool, metadata: ProcessMetadata, } @@ -41,12 +42,14 @@ impl CommandRunner { inner: Arc, cache: PersistentCache, file_store: Store, + eager_fetch: bool, metadata: ProcessMetadata, ) -> CommandRunner { CommandRunner { inner, cache, file_store, + eager_fetch, metadata, } } @@ -196,23 +199,26 @@ impl CommandRunner { return Ok(None); }; - // Ensure that all digests in the result are loadable, erroring if any are not. - // TODO: Make conditional on `eager_fetch`, since backtracking makes this unnecessary as well. - let _ = future::try_join_all(vec![ - self - .file_store - .ensure_local_has_file(result.stdout_digest) - .boxed(), - self - .file_store - .ensure_local_has_file(result.stderr_digest) - .boxed(), - self - .file_store - .ensure_local_has_recursive_directory(result.output_directory.clone()) - .boxed(), - ]) - .await?; + // If eager_fetch is enabled, ensure that all digests in the result are loadable, erroring + // if any are not. If eager_fetch is disabled, a Digest which is discovered to be missing later + // on during execution will cause backtracking. + if self.eager_fetch { + let _ = future::try_join_all(vec![ + self + .file_store + .ensure_local_has_file(result.stdout_digest) + .boxed(), + self + .file_store + .ensure_local_has_file(result.stderr_digest) + .boxed(), + self + .file_store + .ensure_local_has_recursive_directory(result.output_directory.clone()) + .boxed(), + ]) + .await?; + } Ok(Some(result)) } diff --git a/src/rust/engine/process_execution/src/cache_tests.rs b/src/rust/engine/process_execution/src/cache_tests.rs index eeb185b1ed46..05f410590021 100644 --- a/src/rust/engine/process_execution/src/cache_tests.rs +++ b/src/rust/engine/process_execution/src/cache_tests.rs @@ -58,6 +58,7 @@ fn create_cached_runner( local.into(), cache, store, + true, ProcessMetadata::default(), )); diff --git a/src/rust/engine/src/context.rs b/src/rust/engine/src/context.rs index d4c0cdc8ff73..f10f4a2bd834 100644 --- a/src/rust/engine/src/context.rs +++ b/src/rust/engine/src/context.rs @@ -266,12 +266,14 @@ impl Core { inner_runner: Arc, full_store: &Store, local_cache: &PersistentCache, + eager_fetch: bool, process_execution_metadata: &ProcessMetadata, ) -> Arc { Arc::new(process_execution::cache::CommandRunner::new( inner_runner, local_cache.clone(), full_store.clone(), + eager_fetch, process_execution_metadata.clone(), )) } @@ -363,23 +365,19 @@ impl Core { None }; - // TODO: The local cache eagerly fetches outputs independent of the `eager_fetch` flag. Once - // `eager_fetch` backtracks via https://github.com/pantsbuild/pants/issues/11331, the local - // cache will be able to obey `eager_fetch` as well, and can efficiently be used with remote - // execution. - let maybe_local_cached_runner = - if exec_strategy_opts.local_cache && !remoting_opts.execution_enable { - Some(Self::make_local_cached_runner( - maybe_remote_cached_runner - .clone() - .unwrap_or_else(|| leaf_runner.clone()), - full_store, - local_cache, - process_execution_metadata, - )) - } else { - None - }; + let maybe_local_cached_runner = if exec_strategy_opts.local_cache { + Some(Self::make_local_cached_runner( + maybe_remote_cached_runner + .clone() + .unwrap_or_else(|| leaf_runner.clone()), + full_store, + local_cache, + remoting_opts.cache_eager_fetch, + process_execution_metadata, + )) + } else { + None + }; Ok( vec![