Skip to content

Commit

Permalink
[jvm] Use immutable_inputs to provide the compiletime classpath (#1…
Browse files Browse the repository at this point in the history
…3862)

As described in #13435, we expect large sandboxes for JVM compiles (for Scala in particular). #13848 added support for symlinking immutable inputs into sandboxes, and used it for tools. This change uses it for JVM classpaths as well.

In a test repo, this reduces the total amount of time taken to create sandboxes for compilation by ~30% (from 14s to 10s), with an average sandbox creation time of 40ms (including amortized materialization of the symlink destinations).

[ci skip-rust]
[ci skip-build-wheels]
  • Loading branch information
stuhood authored Dec 13, 2021
1 parent 189c933 commit 9762ff6
Show file tree
Hide file tree
Showing 11 changed files with 167 additions and 167 deletions.
47 changes: 18 additions & 29 deletions src/python/pants/backend/java/compile/javac.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,12 @@
from pants.backend.java.target_types import JavaFieldSet, JavaGeneratorFieldSet, JavaSourceField
from pants.core.util_rules.archive import ZipBinary
from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
from pants.engine.fs import (
EMPTY_DIGEST,
AddPrefix,
CreateDigest,
Digest,
Directory,
MergeDigests,
Snapshot,
)
from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, Directory, MergeDigests, Snapshot
from pants.engine.process import BashBinary, FallibleProcessResult, Process, ProcessResult
from pants.engine.rules import Get, MultiGet, collect_rules, rule
from pants.engine.target import SourcesField
from pants.engine.unions import UnionMembership, UnionRule
from pants.jvm.classpath import Classpath
from pants.jvm.compile import (
ClasspathEntry,
ClasspathEntryRequest,
Expand Down Expand Up @@ -135,28 +128,14 @@ async def compile_java_source(
)

dest_dir = "classfiles"
(merged_direct_dependency_classpath_digest, dest_dir_digest) = await MultiGet(
Get(
Digest,
MergeDigests(classfiles.digest for classfiles in direct_dependency_classpath_entries),
),
Get(
Digest,
CreateDigest([Directory(dest_dir)]),
),
)

usercp = "__cp"
prefixed_direct_dependency_classpath_digest = await Get(
Digest, AddPrefix(merged_direct_dependency_classpath_digest, usercp)
dest_dir_digest = await Get(
Digest,
CreateDigest([Directory(dest_dir)]),
)
classpath_arg = ClasspathEntry.arg(direct_dependency_classpath_entries, prefix=usercp)

merged_digest = await Get(
Digest,
MergeDigests(
(
prefixed_direct_dependency_classpath_digest,
dest_dir_digest,
*(
sources.snapshot.digest
Expand All @@ -166,6 +145,14 @@ async def compile_java_source(
),
)

usercp = "__cp"
user_classpath = Classpath(direct_dependency_classpath_entries)
classpath_arg = ":".join(user_classpath.root_immutable_inputs_args(prefix=usercp))
immutable_input_digests = {
**jdk_setup.immutable_input_digests,
**dict(user_classpath.root_immutable_inputs(prefix=usercp)),
}

# Compile.
compile_result = await Get(
FallibleProcessResult,
Expand All @@ -184,7 +171,7 @@ async def compile_java_source(
),
],
input_digest=merged_digest,
immutable_input_digests=jdk_setup.immutable_input_digests,
immutable_input_digests=immutable_input_digests,
use_nailgun=jdk_setup.immutable_input_digests.keys(),
append_only_caches=jdk_setup.append_only_caches,
env=jdk_setup.env,
Expand All @@ -206,6 +193,7 @@ async def compile_java_source(
# the nailgun server). We might be able to resolve this in the future via a Javac wrapper shim.
output_snapshot = await Get(Snapshot, Digest, compile_result.output_digest)
output_file = f"{request.component.representative.address.path_safe_spec}.javac.jar"
output_files: tuple[str, ...] = (output_file,)
if output_snapshot.files:
jar_result = await Get(
ProcessResult,
Expand All @@ -218,7 +206,7 @@ async def compile_java_source(
),
],
input_digest=compile_result.output_digest,
output_files=(output_file,),
output_files=output_files,
description=f"Capture outputs of {request.component} for javac",
level=LogLevel.TRACE,
),
Expand All @@ -227,10 +215,11 @@ async def compile_java_source(
else:
# If there was no output, then do not create a jar file. This may occur, for example, when compiling
# a `package-info.java` in a single partition.
output_files = ()
jar_output_digest = EMPTY_DIGEST

output_classpath = ClasspathEntry(
jar_output_digest, (output_file,), direct_dependency_classpath_entries
jar_output_digest, output_files, direct_dependency_classpath_entries
)

if export_classpath_entries:
Expand Down
44 changes: 17 additions & 27 deletions src/python/pants/backend/java/compile/javac_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,10 @@ def rule_runner() -> RuleRunner:
*source_files.rules(),
*testutil.rules(),
QueryRule(CheckResults, (JavacCheckRequest,)),
QueryRule(FallibleClasspathEntry, (CompileJavaSourceRequest,)),
QueryRule(ClasspathEntry, (CompileJavaSourceRequest,)),
QueryRule(CoarsenedTargets, (Addresses,)),
QueryRule(FallibleClasspathEntry, (CompileJavaSourceRequest,)),
QueryRule(RenderedClasspath, (CompileJavaSourceRequest,)),
],
target_types=[JavaSourcesGeneratorTarget, JvmArtifact],
bootstrap_args=[
Expand Down Expand Up @@ -132,12 +133,10 @@ def test_compile_no_deps(rule_runner: RuleRunner) -> None:
rule_runner, Address(spec_path="", target_name="lib")
)

compiled_classfiles = rule_runner.request(
ClasspathEntry,
classpath = rule_runner.request(
RenderedClasspath,
[CompileJavaSourceRequest(component=coarsened_target, resolve=make_resolve(rule_runner))],
)

classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
assert classpath.content == {
".ExampleLib.java.lib.javac.jar": {"org/pantsbuild/example/lib/ExampleLib.class"}
}
Expand Down Expand Up @@ -183,8 +182,7 @@ def test_compile_jdk_versions(rule_runner: RuleRunner) -> None:
["--javac-jdk=zulu:8.0.312", NAMED_RESOLVE_OPTIONS, DEFAULT_RESOLVE_OPTION],
env_inherit=PYTHON_BOOTSTRAP_ENV,
)
compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
classpath = rule_runner.request(RenderedClasspath, [request])
assert classpath.content == {
".ExampleLib.java.lib.javac.jar": {"org/pantsbuild/example/lib/ExampleLib.class"}
}
Expand Down Expand Up @@ -249,8 +247,7 @@ def test_compile_multiple_source_files(rule_runner: RuleRunner) -> None:
request0 = CompileJavaSourceRequest(
component=coarsened_targets_sorted[0], resolve=make_resolve(rule_runner)
)
compiled_classfiles0 = rule_runner.request(ClasspathEntry, [request0])
classpath0 = rule_runner.request(RenderedClasspath, [compiled_classfiles0.digest])
classpath0 = rule_runner.request(RenderedClasspath, [request0])
assert classpath0.content == {
".ExampleLib.java.lib.javac.jar": {
"org/pantsbuild/example/lib/ExampleLib.class",
Expand All @@ -260,8 +257,7 @@ def test_compile_multiple_source_files(rule_runner: RuleRunner) -> None:
request1 = CompileJavaSourceRequest(
component=coarsened_targets_sorted[1], resolve=make_resolve(rule_runner)
)
compiled_classfiles1 = rule_runner.request(ClasspathEntry, [request1])
classpath1 = rule_runner.request(RenderedClasspath, [compiled_classfiles1.digest])
classpath1 = rule_runner.request(RenderedClasspath, [request1])
assert classpath1.content == {
".OtherLib.java.lib.javac.jar": {
"org/pantsbuild/example/lib/OtherLib.class",
Expand Down Expand Up @@ -344,8 +340,7 @@ class C implements A {}
component=coarsened_target, resolve=make_resolve(rule_runner)
)

compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
classpath = rule_runner.request(RenderedClasspath, [request])
assert classpath.content == {
"a.A.java.javac.jar": {
"org/pantsbuild/a/A.class",
Expand Down Expand Up @@ -425,8 +420,8 @@ class C implements A {}
}
)

compiled_classfiles = rule_runner.request(
ClasspathEntry,
classpath = rule_runner.request(
RenderedClasspath,
[
CompileJavaSourceRequest(
component=expect_single_expanded_coarsened_target(
Expand All @@ -436,7 +431,6 @@ class C implements A {}
)
],
)
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
assert classpath.content == {".Main.java.main.javac.jar": {"org/pantsbuild/main/Main.class"}}


Expand Down Expand Up @@ -502,11 +496,10 @@ class C implements A {}
rule_runner, Address(spec_path="", target_name="main")
)

compiled_classfiles = rule_runner.request(
ClasspathEntry,
classpath = rule_runner.request(
RenderedClasspath,
[CompileJavaSourceRequest(component=ctgt, resolve=make_resolve(rule_runner))],
)
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
assert classpath.content == {
".Main.java.main.javac.jar": {
"org/pantsbuild/main/Main.class",
Expand Down Expand Up @@ -545,8 +538,8 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None:
"lib/ExampleLib.java": JAVA_LIB_SOURCE,
}
)
compiled_classfiles = rule_runner.request(
ClasspathEntry,
classpath = rule_runner.request(
RenderedClasspath,
[
CompileJavaSourceRequest(
component=expect_single_expanded_coarsened_target(
Expand All @@ -556,7 +549,6 @@ def test_compile_with_deps(rule_runner: RuleRunner) -> None:
)
],
)
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
assert classpath.content == {
".Example.java.main.javac.jar": {"org/pantsbuild/example/Example.class"}
}
Expand Down Expand Up @@ -588,8 +580,8 @@ def test_compile_of_package_info(rule_runner: RuleRunner) -> None:
),
}
)
compiled_classfiles = rule_runner.request(
ClasspathEntry,
classpath = rule_runner.request(
RenderedClasspath,
[
CompileJavaSourceRequest(
component=expect_single_expanded_coarsened_target(
Expand All @@ -599,7 +591,6 @@ def test_compile_of_package_info(rule_runner: RuleRunner) -> None:
)
],
)
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
assert classpath.content == {}


Expand Down Expand Up @@ -691,8 +682,7 @@ def test_compile_with_maven_deps(rule_runner: RuleRunner) -> None:
),
resolve=make_resolve(rule_runner),
)
compiled_classfiles = rule_runner.request(ClasspathEntry, [request])
classpath = rule_runner.request(RenderedClasspath, [compiled_classfiles.digest])
classpath = rule_runner.request(RenderedClasspath, [request])
assert classpath.content == {
".Example.java.main.javac.jar": {"org/pantsbuild/example/Example.class"}
}
Expand Down
8 changes: 3 additions & 5 deletions src/python/pants/backend/java/package/deploy_jar.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
PackageFieldSet,
)
from pants.core.util_rules.archive import ZipBinary
from pants.engine.addresses import Addresses
from pants.engine.fs import AddPrefix, CreateDigest, Digest, FileContent, MergeDigests
from pants.engine.process import BashBinary, Process, ProcessResult
from pants.engine.rules import Get, collect_rules, rule
Expand Down Expand Up @@ -66,8 +65,7 @@ async def package_deploy_jar(
# 1. Produce a thin JAR containing our first-party sources and other runtime dependencies
#

dependencies = await Get(Addresses, DependenciesRequest(field_set.dependencies))
classpath = await Get(Classpath, Addresses, dependencies)
classpath = await Get(Classpath, DependenciesRequest(field_set.dependencies))

#
# 2. Produce JAR manifest, and output to a ZIP file that can be included with the JARs
Expand Down Expand Up @@ -122,7 +120,7 @@ async def package_deploy_jar(
# behaviour will be non-deterministic. Sorry! --chrisjrn

output_filename = PurePath(field_set.output_path.value_or_default(file_ending="jar"))
input_filenames = " ".join(shlex.quote(i) for i in classpath.classpath_entries())
input_filenames = " ".join(shlex.quote(i) for i in classpath.args())
_PANTS_BROKEN_DEPLOY_JAR = "pants_broken_deploy_jar.notajar"
cat_and_repair_script = FileContent(
_PANTS_CAT_AND_REPAIR_ZIP_FILENAME,
Expand All @@ -139,7 +137,7 @@ async def package_deploy_jar(
cat_and_repair_script_digest = await Get(Digest, CreateDigest([cat_and_repair_script]))
broken_deploy_jar_inputs_digest = await Get(
Digest,
MergeDigests([classpath.content.digest, cat_and_repair_script_digest, manifest_jar]),
MergeDigests([*classpath.digests(), cat_and_repair_script_digest, manifest_jar]),
)

cat_and_repair = await Get(
Expand Down
11 changes: 7 additions & 4 deletions src/python/pants/backend/java/test/junit.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from pants.backend.java.target_types import JavaTestSourceField
from pants.core.goals.test import TestDebugRequest, TestFieldSet, TestResult, TestSubsystem
from pants.engine.addresses import Addresses
from pants.engine.fs import Digest, DigestSubset, PathGlobs, RemovePrefix, Snapshot
from pants.engine.fs import Digest, DigestSubset, MergeDigests, PathGlobs, RemovePrefix, Snapshot
from pants.engine.process import BashBinary, FallibleProcessResult, Process
from pants.engine.rules import Get, MultiGet, collect_rules, rule
from pants.engine.unions import UnionRule
Expand Down Expand Up @@ -51,6 +51,8 @@ async def run_junit_test(
),
)

merged_classpath_digest = await Get(Digest, MergeDigests(classpath.digests()))

toolcp_relpath = "__toolcp"
immutable_input_digests = {
**jdk_setup.immutable_input_digests,
Expand All @@ -60,7 +62,8 @@ async def run_junit_test(
reports_dir_prefix = "__reports_dir"
reports_dir = f"{reports_dir_prefix}/{field_set.address.path_safe_spec}"

user_classpath_arg = ":".join(classpath.user_classpath_entries())
# Classfiles produced by the root `junit_test` targets are the only ones which should run.
user_classpath_arg = ":".join(classpath.root_args())

process_result = await Get(
FallibleProcessResult,
Expand All @@ -69,7 +72,7 @@ async def run_junit_test(
*jdk_setup.args(
bash,
[
*classpath.classpath_entries(),
*classpath.args(),
*junit_classpath.classpath_entries(toolcp_relpath),
],
),
Expand All @@ -80,7 +83,7 @@ async def run_junit_test(
reports_dir,
*junit.options.args,
],
input_digest=classpath.content.digest,
input_digest=merged_classpath_digest,
immutable_input_digests=immutable_input_digests,
output_directories=(reports_dir,),
append_only_caches=jdk_setup.append_only_caches,
Expand Down
Loading

0 comments on commit 9762ff6

Please sign in to comment.