[jvm] Use immutable_inputs to provide the compiletime classpath (#1…

…3862)

As described in #13435, we expect large sandboxes for JVM compiles (for Scala in particular). #13848 added support for symlinking immutable inputs into sandboxes, and used it for tools. This change uses it for JVM classpaths as well.

In a test repo, this reduces the total amount of time taken to create sandboxes for compilation by ~30% (from 14s to 10s), with an average sandbox creation time of 40ms (including amortized materialization of the symlink destinations).

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/java/compile/javac.py

@@ -14,19 +14,12 @@
 from pants.backend.java.target_types import JavaFieldSet, JavaGeneratorFieldSet, JavaSourceField
 from pants.core.util_rules.archive import ZipBinary
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
-from pants.engine.fs import (
-    EMPTY_DIGEST,
-    AddPrefix,
-    CreateDigest,
-    Digest,
-    Directory,
-    MergeDigests,
-    Snapshot,
-)
+from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, Directory, MergeDigests, Snapshot
 from pants.engine.process import BashBinary, FallibleProcessResult, Process, ProcessResult
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import SourcesField
 from pants.engine.unions import UnionMembership, UnionRule
+from pants.jvm.classpath import Classpath
 from pants.jvm.compile import (
     ClasspathEntry,
     ClasspathEntryRequest,
@@ -135,28 +128,14 @@ async def compile_java_source(
         )
 
     dest_dir = "classfiles"
-    (merged_direct_dependency_classpath_digest, dest_dir_digest) = await MultiGet(
-        Get(
-            Digest,
-            MergeDigests(classfiles.digest for classfiles in direct_dependency_classpath_entries),
-        ),
-        Get(
-            Digest,
-            CreateDigest([Directory(dest_dir)]),
-        ),
-    )
-
-    usercp = "__cp"
-    prefixed_direct_dependency_classpath_digest = await Get(
-        Digest, AddPrefix(merged_direct_dependency_classpath_digest, usercp)
+    dest_dir_digest = await Get(
+        Digest,
+        CreateDigest([Directory(dest_dir)]),
     )
-    classpath_arg = ClasspathEntry.arg(direct_dependency_classpath_entries, prefix=usercp)
-
     merged_digest = await Get(
         Digest,
         MergeDigests(
             (
-                prefixed_direct_dependency_classpath_digest,
                 dest_dir_digest,
                 *(
                     sources.snapshot.digest
@@ -166,6 +145,14 @@ async def compile_java_source(
         ),
     )
 
+    usercp = "__cp"
+    user_classpath = Classpath(direct_dependency_classpath_entries)
+    classpath_arg = ":".join(user_classpath.root_immutable_inputs_args(prefix=usercp))
+    immutable_input_digests = {
+        **jdk_setup.immutable_input_digests,
+        **dict(user_classpath.root_immutable_inputs(prefix=usercp)),
+    }
+
     # Compile.
     compile_result = await Get(
         FallibleProcessResult,
@@ -184,7 +171,7 @@ async def compile_java_source(
                 ),
             ],
             input_digest=merged_digest,
-            immutable_input_digests=jdk_setup.immutable_input_digests,
+            immutable_input_digests=immutable_input_digests,
             use_nailgun=jdk_setup.immutable_input_digests.keys(),
             append_only_caches=jdk_setup.append_only_caches,
             env=jdk_setup.env,
@@ -206,6 +193,7 @@ async def compile_java_source(
     # the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim.
     output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest)
     output_file = f"{request.component.representative.address.path_safe_spec}.javac.jar"
+    output_files: tuple[str, ...] = (output_file,)
     if output_snapshot.files:
         jar_result = await Get(
             ProcessResult,
@@ -218,7 +206,7 @@ async def compile_java_source(
                     ),
                 ],
                 input_digest=compile_result.output_digest,
-                output_files=(output_file,),
+                output_files=output_files,
                 description=f"Capture outputs of {request.component} for javac",
                 level=LogLevel.TRACE,
             ),
@@ -227,10 +215,11 @@ async def compile_java_source(
     else:
         # If there was no output, then do not create a jar file. This may occur, for example, when compiling
         # a `package-info.java` in a single partition.
+        output_files = ()
         jar_output_digest = EMPTY_DIGEST
 
     output_classpath = ClasspathEntry(
-        jar_output_digest, (output_file,), direct_dependency_classpath_entries
+        jar_output_digest, output_files, direct_dependency_classpath_entries
     )
 
     if export_classpath_entries:

src/python/pants/backend/java/compile/javac_test.py

@@ -67,9 +67,10 @@ def rule_runner() -> RuleRunner:
             *source_files.rules(),
             *testutil.rules(),
             QueryRule(CheckResults, (JavacCheckRequest,)),
-            QueryRule(FallibleClasspathEntry, (CompileJavaSourceRequest,)),
             QueryRule(ClasspathEntry, (CompileJavaSourceRequest,)),
             QueryRule(CoarsenedTargets, (Addresses,)),
+            QueryRule(FallibleClasspathEntry, (CompileJavaSourceRequest,)),
+            QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)),
         ],
         target_types=[JavaSourcesGeneratorTarget, JvmArtifact],
         bootstrap_args=[
@@ -132,12 +133,10 @@ def test_compile_no_deps(rule_runner: RuleRunner) -> None:
         rule_runner, Address(spec_path="", target_name="lib")
     )
 
-    compiled_classfiles = rule_runner.request(
-        ClasspathEntry,
+    classpath = rule_runner.request(
+        RenderedClasspath,
         [CompileJavaSourceRequest(component=coarsened_target, resolve=make_resolve(rule_runner))],
     )
-
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
     assert classpath.content == {
         ".ExampleLib.java.lib.javac.jar": {"org/pantsbuild/example/lib/ExampleLib.class"}
     }
@@ -183,8 +182,7 @@ def test_compile_jdk_versions(rule_runner: RuleRunner) -> None:
         ["--javac-jdk=zulu:8.0.312", NAMED_RESOLVE_OPTIONS, DEFAULT_RESOLVE_OPTION],
         env_inherit=PYTHON_BOOTSTRAP_ENV,
     )
-    compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
+    classpath = rule_runner.request(RenderedClasspath, [request])
     assert classpath.content == {
         ".ExampleLib.java.lib.javac.jar": {"org/pantsbuild/example/lib/ExampleLib.class"}
     }
@@ -249,8 +247,7 @@ def test_compile_multiple_source_files(rule_runner: RuleRunner) -> None:
     request0 = CompileJavaSourceRequest(
         component=coarsened_targets_sorted[0], resolve=make_resolve(rule_runner)
     )
-    compiled_classfiles0 = rule_runner.request(ClasspathEntry, [request0])
-    classpath0 = rule_runner.request(RenderedClasspath, [compiled_classfiles0.digest])
+    classpath0 = rule_runner.request(RenderedClasspath, [request0])
     assert classpath0.content == {
         ".ExampleLib.java.lib.javac.jar": {
             "org/pantsbuild/example/lib/ExampleLib.class",
@@ -260,8 +257,7 @@ def test_compile_multiple_source_files(rule_runner: RuleRunner) -> None:
     request1 = CompileJavaSourceRequest(
         component=coarsened_targets_sorted[1], resolve=make_resolve(rule_runner)
     )
-    compiled_classfiles1 = rule_runner.request(ClasspathEntry, [request1])
-    classpath1 = rule_runner.request(RenderedClasspath, [compiled_classfiles1.digest])
+    classpath1 = rule_runner.request(RenderedClasspath, [request1])
     assert classpath1.content == {
         ".OtherLib.java.lib.javac.jar": {
             "org/pantsbuild/example/lib/OtherLib.class",
@@ -344,8 +340,7 @@ class C implements A {}
         component=coarsened_target, resolve=make_resolve(rule_runner)
     )
 
-    compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
+    classpath = rule_runner.request(RenderedClasspath, [request])
     assert classpath.content == {
         "a.A.java.javac.jar": {
             "org/pantsbuild/a/A.class",
@@ -425,8 +420,8 @@ class C implements A {}
         }
     )
 
-    compiled_classfiles = rule_runner.request(
-        ClasspathEntry,
+    classpath = rule_runner.request(
+        RenderedClasspath,
         [
             CompileJavaSourceRequest(
                 component=expect_single_expanded_coarsened_target(
@@ -436,7 +431,6 @@ class C implements A {}
             )
         ],
     )
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
     assert classpath.content == {".Main.java.main.javac.jar": {"org/pantsbuild/main/Main.class"}}
 
 
@@ -502,11 +496,10 @@ class C implements A {}
         rule_runner, Address(spec_path="", target_name="main")
     )
 
-    compiled_classfiles = rule_runner.request(
-        ClasspathEntry,
+    classpath = rule_runner.request(
+        RenderedClasspath,
         [CompileJavaSourceRequest(component=ctgt, resolve=make_resolve(rule_runner))],
     )
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
     assert classpath.content == {
         ".Main.java.main.javac.jar": {
             "org/pantsbuild/main/Main.class",
@@ -545,8 +538,8 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None:
             "lib/ExampleLib.java": JAVA_LIB_SOURCE,
         }
     )
-    compiled_classfiles = rule_runner.request(
-        ClasspathEntry,
+    classpath = rule_runner.request(
+        RenderedClasspath,
         [
             CompileJavaSourceRequest(
                 component=expect_single_expanded_coarsened_target(
@@ -556,7 +549,6 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None:
             )
         ],
     )
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
     assert classpath.content == {
         ".Example.java.main.javac.jar": {"org/pantsbuild/example/Example.class"}
     }
@@ -588,8 +580,8 @@ def test_compile_of_package_info(rule_runner: RuleRunner) -> None:
             ),
         }
     )
-    compiled_classfiles = rule_runner.request(
-        ClasspathEntry,
+    classpath = rule_runner.request(
+        RenderedClasspath,
         [
             CompileJavaSourceRequest(
                 component=expect_single_expanded_coarsened_target(
@@ -599,7 +591,6 @@ def test_compile_of_package_info(rule_runner: RuleRunner) -> None:
             )
         ],
     )
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
     assert classpath.content == {}
 
 
@@ -691,8 +682,7 @@ def test_compile_with_maven_deps(rule_runner: RuleRunner) -> None:
         ),
         resolve=make_resolve(rule_runner),
     )
-    compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
-    classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
+    classpath = rule_runner.request(RenderedClasspath, [request])
     assert classpath.content == {
         ".Example.java.main.javac.jar": {"org/pantsbuild/example/Example.class"}
     }

src/python/pants/backend/java/package/deploy_jar.py

@@ -15,7 +15,6 @@
     PackageFieldSet,
 )
 from pants.core.util_rules.archive import ZipBinary
-from pants.engine.addresses import Addresses
 from pants.engine.fs import AddPrefix, CreateDigest, Digest, FileContent, MergeDigests
 from pants.engine.process import BashBinary, Process, ProcessResult
 from pants.engine.rules import Get, collect_rules, rule
@@ -66,8 +65,7 @@ async def package_deploy_jar(
     # 1. Produce a thin JAR containing our first-party sources and other runtime dependencies
     #
 
-    dependencies = await Get(Addresses, DependenciesRequest(field_set.dependencies))
-    classpath = await Get(Classpath, Addresses, dependencies)
+    classpath = await Get(Classpath, DependenciesRequest(field_set.dependencies))
 
     #
     # 2. Produce JAR manifest, and output to a ZIP file that can be included with the JARs
@@ -122,7 +120,7 @@ async def package_deploy_jar(
     # behaviour will be non-deterministic. Sorry!  --chrisjrn
 
     output_filename = PurePath(field_set.output_path.value_or_default(file_ending="jar"))
-    input_filenames = " ".join(shlex.quote(i) for i in classpath.classpath_entries())
+    input_filenames = " ".join(shlex.quote(i) for i in classpath.args())
     _PANTS_BROKEN_DEPLOY_JAR = "pants_broken_deploy_jar.notajar"
     cat_and_repair_script = FileContent(
         _PANTS_CAT_AND_REPAIR_ZIP_FILENAME,
@@ -139,7 +137,7 @@ async def package_deploy_jar(
     cat_and_repair_script_digest = await Get(Digest, CreateDigest([cat_and_repair_script]))
     broken_deploy_jar_inputs_digest = await Get(
         Digest,
-        MergeDigests([classpath.content.digest, cat_and_repair_script_digest, manifest_jar]),
+        MergeDigests([*classpath.digests(), cat_and_repair_script_digest, manifest_jar]),
     )
 
     cat_and_repair = await Get(

src/python/pants/backend/java/test/junit.py

@@ -8,7 +8,7 @@
 from pants.backend.java.target_types import JavaTestSourceField
 from pants.core.goals.test import TestDebugRequest, TestFieldSet, TestResult, TestSubsystem
 from pants.engine.addresses import Addresses
-from pants.engine.fs import Digest, DigestSubset, PathGlobs, RemovePrefix, Snapshot
+from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, RemovePrefix, Snapshot
 from pants.engine.process import BashBinary, FallibleProcessResult, Process
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.unions import UnionRule
@@ -51,6 +51,8 @@ async def run_junit_test(
         ),
     )
 
+    merged_classpath_digest = await Get(Digest, MergeDigests(classpath.digests()))
+
     toolcp_relpath = "__toolcp"
     immutable_input_digests = {
         **jdk_setup.immutable_input_digests,
@@ -60,7 +62,8 @@ async def run_junit_test(
     reports_dir_prefix = "__reports_dir"
     reports_dir = f"{reports_dir_prefix}/{field_set.address.path_safe_spec}"
 
-    user_classpath_arg = ":".join(classpath.user_classpath_entries())
+    # Classfiles produced by the root `junit_test` targets are the only ones which should run.
+    user_classpath_arg = ":".join(classpath.root_args())
 
     process_result = await Get(
         FallibleProcessResult,
@@ -69,7 +72,7 @@ async def run_junit_test(
                 *jdk_setup.args(
                     bash,
                     [
-                        *classpath.classpath_entries(),
+                        *classpath.args(),
                         *junit_classpath.classpath_entries(toolcp_relpath),
                     ],
                 ),
@@ -80,7 +83,7 @@ async def run_junit_test(
                 reports_dir,
                 *junit.options.args,
             ],
-            input_digest=classpath.content.digest,
+            input_digest=merged_classpath_digest,
             immutable_input_digests=immutable_input_digests,
             output_directories=(reports_dir,),
             append_only_caches=jdk_setup.append_only_caches,