Add debug output to help to differentiate retry cases (pantsbuild#16277)

This is debug output (and labels itself as such) to assist with differentiating the cases of pantsbuild#16096.

[ci skip-rust]
[ci skip-build-wheels]

pants.toml

@@ -71,8 +71,9 @@ unmatched_build_file_globs = "error"
 remote_store_address = "grpcs://cache.toolchain.com:443"
 remote_instance_name = "main"
 remote_auth_plugin = "toolchain.pants.auth.plugin:toolchain_auth_plugin"
-# TODO: See https://github.com/pantsbuild/pants/issues/16096.
-remote_cache_eager_fetch = true
+# TODO: May cause tests which experience missing digests to hang.
+# See https://github.com/pantsbuild/pants/issues/16096.
+remote_cache_eager_fetch = false
 
 [anonymous-telemetry]
 enabled = true

src/rust/engine/src/context.rs

@@ -751,6 +751,8 @@ impl Context {
         if let Some(new_level) = maybe_new_level {
           workunit.increment_counter(Metric::BacktrackAttempts, 1);
           let description = &root.process.description;
+          // TODO: This message should likely be at `info`, or eventually, debug.
+          //   see https://github.com/pantsbuild/pants/issues/15867
           log::warn!(
             "Making attempt {new_level} to backtrack and retry `{description}`, due to \
               missing digest {digest:?}."

src/rust/engine/src/nodes.rs

@@ -468,6 +468,19 @@ impl ExecuteProcess {
       }
     }
 
+    if backtrack_level > 0 {
+      // TODO: This message is symmetrical to the "Making attempt {} to backtrack and retry {}"
+      // message in `context.rs`, but both of them are effectively debug output. They should be
+      // quieted down as part of https://github.com/pantsbuild/pants/issues/15867 once all bugs
+      // have been shaken out.
+      log::info!(
+        "On backtrack attempt {} for `{}`, produced: {:?}",
+        backtrack_level,
+        request.description,
+        res.output_directory.digests()
+      );
+    }
+
     Ok(ProcessResult {
       result: res,
       backtrack_level,