diff --git a/BUILD.tools b/BUILD.tools index ed7ddc13158..2055cee64c2 100644 --- a/BUILD.tools +++ b/BUILD.tools @@ -77,12 +77,12 @@ jar_library( # Running Kythe on JDK8 requires a standalone javac 9 on the bootclasspath. # This jar is published to the same custom repo as the Kythe jars. -jar_library( - name='javac9', - jars = [ - jar(org='java', name='javac', rev='9+181-r4173-1'), - ], -) +# jar_library( +# name='javac9', +# jars = [ +# jar(org='java', name='javac', rev='9+181'), +# ], +# ) # Runtime dependencies for the Avro Java generated code. avro_rev = '1.8.2' @@ -94,3 +94,19 @@ jar_library( jar(org='org.apache.avro', name='avro-ipc', rev=avro_rev), ], ) + +BOOTSTRAP_SCROOGE_VERSION='19.11.0' +jar_library( + name = 'v2-thrift-gen', + jars = [ + # jar(org='java', name='javac', rev='9+181-r4173-1'), + scala_jar(org='com.twitter', name='scrooge-generator', rev=BOOTSTRAP_SCROOGE_VERSION), + ], +) + +jar_library( + name = 'scalac', + jars = [ + jar(org='org.scala-lang', name='scala-compiler', rev='2.12.8'), + ], +) diff --git a/build-support/bin/BUILD b/build-support/bin/BUILD index 9658566fc99..2e302a693bd 100644 --- a/build-support/bin/BUILD +++ b/build-support/bin/BUILD @@ -73,7 +73,7 @@ python_binary( python_binary( name = 'generate_travis_yml', - sources = 'generate_travis_yml.py', + source = 'generate_travis_yml.py', dependencies = [ '3rdparty/python:PyYAML', ], @@ -82,7 +82,7 @@ python_binary( python_binary( name = 'get_rbe_token', - sources = 'get_rbe_token.py', + source = 'get_rbe_token.py', dependencies = [ '3rdparty/python:ansicolors', '3rdparty/python:requests', diff --git a/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/targets/python_awslambda.py b/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/targets/python_awslambda.py index d05e27bd6d9..4c8b06459bf 100644 --- a/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/targets/python_awslambda.py +++ b/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/targets/python_awslambda.py @@ -6,6 +6,7 @@ from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField from pants.build_graph.target import Target +from pants.util.memo import memoized_classproperty class PythonAWSLambda(Target): @@ -14,6 +15,10 @@ class PythonAWSLambda(Target): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['binary']) + def __init__(self, binary=None, handler=None, diff --git a/contrib/node/src/python/pants/contrib/node/targets/node_bundle.py b/contrib/node/src/python/pants/contrib/node/targets/node_bundle.py index c6519add6a5..4a5327896d1 100644 --- a/contrib/node/src/python/pants/contrib/node/targets/node_bundle.py +++ b/contrib/node/src/python/pants/contrib/node/targets/node_bundle.py @@ -5,6 +5,7 @@ from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField from pants.fs import archive as archive_lib +from pants.util.memo import memoized_classproperty from pants.contrib.node.targets.node_package import NodePackage @@ -12,6 +13,10 @@ class NodeBundle(NodePackage): """A bundle of node modules.""" + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['node_module']) + def __init__(self, node_module=None, archive='tgz', address=None, payload=None, **kwargs): """ :param dependencies: a list of node_modules diff --git a/contrib/node/src/python/pants/contrib/node/targets/node_module.py b/contrib/node/src/python/pants/contrib/node/targets/node_module.py index 850ee98f7e3..a2b35657852 100644 --- a/contrib/node/src/python/pants/contrib/node/targets/node_module.py +++ b/contrib/node/src/python/pants/contrib/node/targets/node_module.py @@ -6,6 +6,7 @@ from pants.base.exceptions import TargetDefinitionException from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField +from pants.util.memo import memoized_classproperty from pants.contrib.node.targets.node_package import NodePackage @@ -16,6 +17,10 @@ class NodeModule(NodePackage): """A Node module.""" + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['bin_executables']) + def __init__( self, package_manager=None, sources=None, build_script=None, output_dir='dist', dev_dependency=False, style_ignore_path='.eslintignore', address=None, payload=None, diff --git a/pants b/pants index eef61cd0420..98942a4952d 100755 --- a/pants +++ b/pants @@ -2,6 +2,10 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +# set -x + +export LEVEL=debug + # This bootstrap script runs pants from the live sources in this repo. # # Further support is added for projects wrapping pants with custom external extensions. In the @@ -128,4 +132,13 @@ if [[ "${test_goal_used}" == 'true' && "${no_regen_pex}" != 'true' ]]; then "$HERE/build-support/bin/bootstrap_pants_pex.sh" fi -exec_pants_bare "$@" +new_argv=() +for arg in "$@"; do + if [[ "$arg" == 'export' ]]; then + new_argv=("${new_argv[@]}" --enable-pantsd --v2 --no-v1 --tag='-impossible_to_import' export-v2 --output-dir=./wowzer --print-exception-stacktrace -ldebug) + elif [[ "$arg" != '--no-colors' && "$arg" != '--formatted' ]]; then + new_argv=("${new_argv[@]}" "$(echo "$arg" | sed -e 's#export-output-file#output-file#g')") + fi +done + +exec_pants_bare "${new_argv[@]}" diff --git a/pants.ini b/pants.ini index 6efc0fb99c9..4440d97a8a4 100644 --- a/pants.ini +++ b/pants.ini @@ -400,7 +400,7 @@ pytest_plugins: +[ "pytest-rerunfailures", # TODO(#8651): We need this until we switch to implicit namespace packages so that pytest-cov # understands our __init__ files. NB: this version matches 3rdparty/python/requirements.txt. - "setuptools==40.6.3", + "setuptools==41.0.0", ] @@ -465,6 +465,8 @@ repos: [ # Custom repo for Kythe jars, as Google doesn't currently publish them anywhere. 'https://raw.githubusercontent.com/toolchainlabs/binhost/master/' ] +fingerprint: 24945c529eaa32a16a70256ac357108edc1b51a4dd45b656a1808c0cbf00617e +size_bytes: 27573328 [cache.resolve.coursier] # Only use local artifact for coursier because the cache content contains abs path diff --git a/src/python/pants/backend/codegen/thrift/java/java_thrift_library.py b/src/python/pants/backend/codegen/thrift/java/java_thrift_library.py index f42b5569468..350720f4399 100644 --- a/src/python/pants/backend/codegen/thrift/java/java_thrift_library.py +++ b/src/python/pants/backend/codegen/thrift/java/java_thrift_library.py @@ -54,7 +54,7 @@ def check_value_for_arg(arg, value, values): self._compiler = check_value_for_arg('compiler', compiler, self._COMPILERS) self._language = language - self.namespace_map = namespace_map + self._namespace_map = namespace_map self.thrift_linter_strict = thrift_linter_strict self._default_java_namespace = default_java_namespace self._include_paths = include_paths @@ -68,6 +68,10 @@ def compiler(self): def language(self): return self._language + @property + def namespace_map(self): + return self._namespace_map + @property def compiler_args(self): return self._compiler_args diff --git a/src/python/pants/backend/codegen/thrift/java/register.py b/src/python/pants/backend/codegen/thrift/java/register.py index a5082f8140e..82891dd02ad 100644 --- a/src/python/pants/backend/codegen/thrift/java/register.py +++ b/src/python/pants/backend/codegen/thrift/java/register.py @@ -3,6 +3,7 @@ from pants.backend.codegen.thrift.java.apache_thrift_java_gen import ApacheThriftJavaGen from pants.backend.codegen.thrift.java.java_thrift_library import JavaThriftLibrary +from pants.backend.codegen.thrift.java.rules.thrift_gen import rules as thrift_rules from pants.build_graph.build_file_aliases import BuildFileAliases from pants.goal.task_registrar import TaskRegistrar as task @@ -17,3 +18,7 @@ def build_file_aliases(): def register_goals(): task(name='thrift-java', action=ApacheThriftJavaGen).install('gen') + + +def rules(): + return thrift_rules() diff --git a/src/python/pants/backend/codegen/thrift/java/rules/BUILD b/src/python/pants/backend/codegen/thrift/java/rules/BUILD new file mode 100644 index 00000000000..f1a116bd57b --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/java/rules/BUILD @@ -0,0 +1,13 @@ +python_library( + dependencies=[ + '3rdparty/python:dataclasses', + 'src/python/pants/backend/codegen/thrift/java', + 'src/python/pants/backend/jvm/rules', + 'src/python/pants/backend/jvm/subsystems:jvm_tool_mixin', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:rules', + 'src/python/pants/engine/legacy:graph', + 'src/python/pants/rules/core', + 'src/python/pants/util:memo', + ], +) diff --git a/src/python/pants/backend/codegen/thrift/java/rules/__init__.py b/src/python/pants/backend/codegen/thrift/java/rules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/thrift/java/rules/thrift_gen.py b/src/python/pants/backend/codegen/thrift/java/rules/thrift_gen.py new file mode 100644 index 00000000000..f0ca5497c55 --- /dev/null +++ b/src/python/pants/backend/codegen/thrift/java/rules/thrift_gen.py @@ -0,0 +1,228 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import os +from dataclasses import dataclass +from typing import FrozenSet, Tuple + +from pants.backend.codegen.thrift.java.thrift_defaults import ThriftDefaults +from pants.backend.jvm.rules.hermetic_dist import HermeticDist +from pants.backend.jvm.rules.jvm_tool import JvmToolClasspathResult +from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolMixin, JvmToolRequest +from pants.engine.console import Console +from pants.engine.fs import Digest, DirectoriesToMerge, Snapshot +from pants.engine.goal import Goal +from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult +from pants.engine.legacy.graph import HydratedTarget, TransitiveHydratedTargets +from pants.engine.objects import Collection +from pants.engine.rules import RootRule, console_rule, optionable_rule, rule +from pants.engine.selectors import Get, MultiGet +from pants.rules.core.strip_source_root import SourceRootStrippedSources +from pants.subsystem.subsystem import Subsystem +from pants.util.memo import memoized_classproperty + + +class ThriftGenTool(Subsystem, JvmToolMixin): + options_scope = 'v2-thrift-gen-tool' + + @classmethod + def register_options(cls, register): + super().register_options(register) + cls.register_jvm_tool(register, 'v2-thrift-gen') + + +class TestThriftGenClasspath(Goal): + name = 'test-thrift-gen-classpath' + + +@console_rule +async def test_thrift_gen_tool_classpath( + console: Console, + thrift_gen_tool: ThriftGenTool, +) -> TestThriftGenClasspath: + classpath_result = await Get[JvmToolClasspathResult](JvmToolRequest, + thrift_gen_tool.create_tool_request('v2-thrift-gen')) + console.print_stdout(f'classpath_result: {classpath_result}') + return TestThriftGenClasspath(exit_code=0) + + +@dataclass(frozen=True) +class ThriftableTarget: + underlying: HydratedTarget + + @memoized_classproperty + def known_build_file_aliases(cls) -> FrozenSet[str]: + # FIXME: copied from register.py! + return frozenset([ + 'java_antlr_library', + 'java_protobuf_library', + 'java_ragel_library', + 'java_thrift_library', + 'java_wire_library', + 'python_antlr_library', + 'python_thrift_library', + 'python_grpcio_library', + 'jaxb_library', + ]) + + +class ThriftableTargets(Collection[ThriftableTarget]): pass + + +@rule +def collect_thriftable_targets(thts: TransitiveHydratedTargets) -> ThriftableTargets: + return ThriftableTargets( + ThriftableTarget(hydrated_target) + for hydrated_target in thts.closure + if hydrated_target.adaptor.type_alias in ThriftableTarget.known_build_file_aliases + ) + + +@dataclass(frozen=True) +class ThriftGenRequest: + exe_req: ExecuteProcessRequest + + +@rule +async def create_thrift_gen_request( + thriftable_target: ThriftableTarget, + thrift_defaults: ThriftDefaults, + hermetic_dist: HermeticDist, + thrift_gen_tool: ThriftGenTool, +) -> ThriftGenRequest: + + # Extract the underlying HydratedTarget, then the v2 TargetAdaptor, then the v1 Target. + hydrated_target = thriftable_target.underlying + v2_target = hydrated_target.adaptor + v1_target = v2_target.v1_target + + args = list(thrift_defaults.compiler_args(v1_target)) + + default_java_namespace = thrift_defaults.default_java_namespace(v1_target) + if default_java_namespace: + args.extend(['--default-java-namespace', default_java_namespace]) + + # NB: No need to set import paths here, because we can materialize all the target's dependencies + # relative to their source root! + + output_language = thrift_defaults.language(v1_target) + args.extend(['--language', output_language]) + + namespace_map = thrift_defaults.namespace_map(v1_target) + namespace_map = tuple(sorted(namespace_map.items())) if namespace_map else () + for lhs, rhs in namespace_map: + args.extend(['--namespace-map', f'{lhs}={rhs}']) + + # TODO: do we need to account for `v1_target.include_paths`? + + args.append('--verbose') + + # TODO: do we need to support --gen-file-map? + + cur_target_source_root_stripped_sources = await Get[SourceRootStrippedSources](HydratedTarget, + hydrated_target) + dependency_source_root_stripped_sources = await MultiGet( + Get[SourceRootStrippedSources](HydratedTarget, dep) + for dep in hydrated_target.dependencies + ) + + thrift_gen_classpath_result = await Get[JvmToolClasspathResult]( + JvmToolRequest, thrift_gen_tool.create_tool_request('v2-thrift-gen')) + + merged_input_files = await Get[Digest](DirectoriesToMerge(tuple( + [ + thrift_gen_classpath_result.snapshot.directory_digest, + cur_target_source_root_stripped_sources.snapshot.directory_digest + ] + [ + dep_sources.snapshot.directory_digest + for dep_sources in dependency_source_root_stripped_sources + ]))) + + # NB: To calculate the output files to capture, we replace the .thrift file extension with + # ., where is the value computed for the --language argument above! + expected_output_dirs = frozenset( + os.path.join(os.path.dirname(file_name), f'thrift{output_language}') + for file_name in cur_target_source_root_stripped_sources.snapshot.files + ) + + exe_req = ExecuteProcessRequest( + argv=tuple([ + os.path.join(hermetic_dist.symbolic_home, 'bin/java'), + '-cp', ':'.join( + thrift_gen_classpath_result.snapshot.files + + thrift_gen_classpath_result.snapshot.dirs + ), + 'com.twitter.scrooge.Main', + ] + args + [ + *cur_target_source_root_stripped_sources.snapshot.files + ]), + input_files=merged_input_files, + description=f'Invoke scrooge for the sources of the target {hydrated_target.address}', + output_directories=tuple(expected_output_dirs), + jdk_home=hermetic_dist.underlying_home, + is_nailgunnable=True, + ) + return ThriftGenRequest(exe_req) + + +@dataclass(frozen=True) +class ThriftGenResult: + snapshot: Snapshot + + +@rule +async def execute_thrift(req: ThriftGenRequest) -> ThriftGenResult: + exe_result = await Get[ExecuteProcessResult](ExecuteProcessRequest, req.exe_req) + snapshot = await Get[Snapshot](Digest, exe_result.output_directory_digest) + return ThriftGenResult(snapshot) + + +@dataclass(frozen=True) +class ThriftedTarget: + original: HydratedTarget + output: Snapshot + + +@dataclass(frozen=True) +class ThriftResults: + thrifted_targets: Tuple[ThriftedTarget, ...] + + +@rule +async def fast_thrift_gen(thriftable_targets: ThriftableTargets) -> ThriftResults: + thrift_gen_results = zip( + thriftable_targets, + await MultiGet(Get[ThriftGenResult](ThriftableTarget, t) for t in thriftable_targets)) + return ThriftResults(tuple( + ThriftedTarget(original=target.underlying, output=result.snapshot) + for target, result in thrift_gen_results + )) + + +class TestThriftGen(Goal): + name = 'test-thrift-gen-request-creation' + + +@console_rule +async def test_thrift_gen( + console: Console, + thts: TransitiveHydratedTargets, +) -> TestThriftGen: + console.print_stdout(f'thts: {thts}') + results = await Get[ThriftResults](TransitiveHydratedTargets, thts) + console.print_stdout(f'results: {results}') + return TestThriftGen(exit_code=0) + + +def rules(): + return [ + optionable_rule(ThriftGenTool), + test_thrift_gen_tool_classpath, + collect_thriftable_targets, + optionable_rule(ThriftDefaults), + RootRule(ThriftableTarget), + create_thrift_gen_request, + execute_thrift, + fast_thrift_gen, + test_thrift_gen, + ] diff --git a/src/python/pants/backend/docgen/targets/doc.py b/src/python/pants/backend/docgen/targets/doc.py index 4938eec2255..a90ebb6fbb7 100644 --- a/src/python/pants/backend/docgen/targets/doc.py +++ b/src/python/pants/backend/docgen/targets/doc.py @@ -93,7 +93,7 @@ def __init__(self, """ payload = payload or Payload() if not format: - if sources.files[0].lower().endswith('.rst'): + if sources.files and sources.files[0].lower().endswith('.rst'): format = 'rst' else: format = 'md' @@ -113,7 +113,8 @@ def __init__(self, @property def source(self): """The first (and only) source listed by this Page.""" - return list(self.payload.sources.source_paths)[0] + paths = list(self.payload.sources.source_paths) + return paths[0] if paths else None @classmethod def compute_injectable_address_specs(cls, kwargs=None, payload=None): diff --git a/src/python/pants/backend/jvm/register.py b/src/python/pants/backend/jvm/register.py index 543b00d168d..185e66ba7b8 100644 --- a/src/python/pants/backend/jvm/register.py +++ b/src/python/pants/backend/jvm/register.py @@ -9,6 +9,11 @@ Scm, ) from pants.backend.jvm.repository import Repository as repo +from pants.backend.jvm.rules.coursier import rules as coursier_rules +from pants.backend.jvm.rules.export_classpath import rules as export_classpath_rules +from pants.backend.jvm.rules.hermetic_dist import rules as hermetic_dist_rules +from pants.backend.jvm.rules.jvm_tool import rules as jvm_tool_rules +from pants.backend.jvm.rules.zinc_compile import rules as zinc_compile_rules from pants.backend.jvm.scala_artifact import ScalaArtifact from pants.backend.jvm.subsystems.jar_dependency_management import JarDependencyManagementSetup from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform @@ -228,3 +233,13 @@ def register_goals(): task(name='test-jvm-prep-command', action=RunTestJvmPrepCommand).install('test', first=True) task(name='binary-jvm-prep-command', action=RunBinaryJvmPrepCommand).install('binary', first=True) task(name='compile-jvm-prep-command', action=RunCompileJvmPrepCommand).install('compile', first=True) + + +def rules(): + return [ + *export_classpath_rules(), + *zinc_compile_rules(), + *coursier_rules(), + *hermetic_dist_rules(), + *jvm_tool_rules(), + ] diff --git a/src/python/pants/backend/jvm/rules/BUILD b/src/python/pants/backend/jvm/rules/BUILD new file mode 100644 index 00000000000..a1657ee5ff0 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/BUILD @@ -0,0 +1,17 @@ +python_library( + dependencies=[ + '3rdparty/python:dataclasses', + 'src/python/pants/backend/jvm/subsystems:jar_dependency_management', + 'src/python/pants/backend/jvm/tasks:coursier_resolve', + 'src/python/pants/backend/jvm/tasks/coursier', + 'src/python/pants/backend/jvm/tasks/jvm_compile', + 'src/python/pants/binaries', + 'src/python/pants/engine:console', + 'src/python/pants/engine:fs', + 'src/python/pants/engine:goal', + 'src/python/pants/engine:isolated_process', + 'src/python/pants/engine:rules', + 'src/python/pants/java/jar', + 'src/python/pants/util:contextutil', + ], +) diff --git a/src/python/pants/backend/jvm/rules/__init__.py b/src/python/pants/backend/jvm/rules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/jvm/rules/coursier.py b/src/python/pants/backend/jvm/rules/coursier.py new file mode 100644 index 00000000000..99b73d16180 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/coursier.py @@ -0,0 +1,357 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import json +import logging +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple + +from pants.backend.jvm.rules.hermetic_dist import HermeticDist +from pants.backend.jvm.rules.jvm_options import JvmOptions +from pants.backend.jvm.subsystems.jar_dependency_management import JarDependencyManagement +from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.backend.jvm.tasks.coursier.coursier_subsystem import CoursierSubsystem +from pants.backend.jvm.tasks.coursier_resolve import CoursierMixin +from pants.base.build_root import BuildRoot +from pants.binaries.binary_tool import BinaryToolFetchRequest +from pants.build_graph.target import Target +from pants.engine.console import Console +from pants.engine.fs import (Digest, DirectoriesToMerge, FileContent, FilesContent, + InputFilesContent, PathGlobs, Snapshot) +from pants.engine.goal import Goal +from pants.engine.interactive_runner import InteractiveProcessRequest, InteractiveRunner +from pants.engine.isolated_process import ExecuteProcessRequest, ExecuteProcessResult +from pants.engine.legacy.graph import HydratedTarget, TransitiveHydratedTargets +from pants.engine.rules import RootRule, console_rule, optionable_rule, rule +from pants.engine.selectors import Get, MultiGet +from pants.java.jar.jar_dependency import JarDependency +from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar +from pants.util.collections import assert_single_element +from pants.util.contextutil import temporary_dir +from pants.util.dirutil import fast_relpath +from pants.util.memo import memoized_method, memoized_property +from pants.util.meta import frozen_after_init + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ResolvedCoursier: + snapshot: Snapshot + + +@rule +async def snapshot_coursier(coursier: CoursierSubsystem) -> ResolvedCoursier: + snapshot = await Get[Snapshot](BinaryToolFetchRequest(tool=coursier)) + return ResolvedCoursier(snapshot) + + +# TODO: we currently expect the jar dependencies to be provided directly in a JarResolveRequest, +# *without* using JarDependencyManagement#targets_by_artifact_set(), which implies that +# global_excludes is always empty and we don't make use of IvyUtils.calculate_classpath()! See +# CoursierMixin#resolve() for the missing pieces. +@dataclass(frozen=True) +class JarResolveRequest: + thts: TransitiveHydratedTargets + + @memoized_property + def hydrated_targets(self) -> Tuple[HydratedTarget, ...]: + return [ + t for t in self.thts.closure + if isinstance(t.adaptor.v1_target, JarLibrary) + ] + + @memoized_property + def jar_deps(self) -> Tuple[JarDependency, ...]: + return [ + jar_dep + for jar_lib_tht in self.hydrated_targets + for jar_dep in jar_lib_tht.adaptor.v1_target.jar_dependencies + ] + + +@dataclass(frozen=True) +class ResolveConfiguration: + conf: str = 'default' + + +@dataclass(frozen=True) +class CoursierRequest: + jar_resolution: JarResolveRequest + jvm_options: JvmOptions = JvmOptions(()) + jar_path_base: Optional[Path] = None + conf: ResolveConfiguration = ResolveConfiguration() + + @property + def source_targets(self): + return self.jar_resolution.hydrated_targets + + +@dataclass(frozen=True) +class CoursierExecutionRequest: + orig_req: CoursierRequest + exe_req: ExecuteProcessRequest + + _json_output_path = 'out.json' + + def __getattr__(self, key, **kwargs): + """Use the prototype pattern with the original abstract coursier request.""" + if hasattr(self.orig_req, key): + return getattr(self.orig_req, key) + return super().__getattr__(key, **kwargs) + + +_LOCAL_EXCLUDES_FILENAME = 'excludes.txt' + + +@rule +async def generate_coursier_execution_request( + coursier: CoursierSubsystem, + resolved_coursier: ResolvedCoursier, + manager: JarDependencyManagement, + hermetic_dist: HermeticDist, + req: CoursierRequest, +) -> CoursierExecutionRequest: + jars_to_resolve, pinned_coords = CoursierMixin._compute_jars_to_resolve_and_pin( + raw_jars=req.jar_resolution.jar_deps, + artifact_set=None, + manager=manager, + ) + common_args = coursier.common_args() + + # TODO: figure out a cleaner way for CoursierMixin._construct_cmd_args() to make use of the + # `coursier_workdir` arg! + cmd_args, local_exclude_args = CoursierMixin._construct_cmd_args( + jars=jars_to_resolve, + common_args=common_args, + global_excludes=[], + pinned_coords=pinned_coords, + coursier_workdir=None, + json_output_path=CoursierExecutionRequest._json_output_path, + strict_jar_revision_checking=False, + affecting_the_local_filesystem=False, + ) + + if local_exclude_args: + excludes_digest = await Get[Digest](InputFilesContent(tuple([FileContent( + path=_LOCAL_EXCLUDES_FILENAME, + content='\n'.join(local_exclude_args).encode(), + )]))) + merged_digest = await Get[Digest](DirectoriesToMerge(tuple([ + resolved_coursier.snapshot.directory_digest, + excludes_digest, + ]))) + exclude_argv = '--local-exclude-file', _LOCAL_EXCLUDES_FILENAME, + else: + merged_digest = resolved_coursier.snapshot.directory_digest + exclude_argv = [] + + exe_req = ExecuteProcessRequest( + argv=tuple([ + os.path.join(hermetic_dist.symbolic_home, 'bin/java'), + '-cp', ':'.join( + resolved_coursier.snapshot.files + + resolved_coursier.snapshot.dirs + ), + *req.jvm_options.options, + 'coursier.cli.Coursier', + *cmd_args, + *exclude_argv, + ]), + input_files=merged_digest, + description=f'Call coursier to resolve jars {req}.', + output_files=tuple([CoursierExecutionRequest._json_output_path]), + jdk_home=hermetic_dist.underlying_home, + is_nailgunnable=True, + ) + + return CoursierExecutionRequest(orig_req=req, exe_req=exe_req) + + +@dataclass(frozen=True) +class CoursierResolveResult: + base_conf: ResolveConfiguration + jars_per_target: Tuple[Tuple[Target, Tuple[ResolvedJar, ...]], ...] + + @memoized_property + def target_jars_mapping(self) -> Dict[Target, Tuple[ResolvedJar, ...]]: + return {t: jars for t, jars in self.jars_per_target} + + @memoized_property + def resolved_jars(self) -> List[ResolvedJar]: + return [j for _target, jars in self.jars_per_target for j in jars] + + +@rule +async def execute_coursier( + req: CoursierExecutionRequest, + coursier: CoursierSubsystem, + build_root: BuildRoot, + interactive_runner: InteractiveRunner, +) -> CoursierResolveResult: + + logger.info(f'req: {req}') + # import pdb; pdb.set_trace() + + output_digest = None + # TODO: stream stdout/stderr to console somehow! + if False: + interactive_result = interactive_runner.run_local_interactive_process(InteractiveProcessRequest( + argv=req.exe_req.argv, + env=req.exe_req.env, + hermetic_input=req.exe_req.input_files, + run_in_workspace=False, + output_file_path=CoursierExecutionRequest._json_output_path, + jdk_home=req.exe_req.jdk_home, + )) + if interactive_result.process_exit_code == 0: + output_digest = interactive_result.output_snapshot.directory_digest + if not output_digest: + exe_result = await Get[ExecuteProcessResult](ExecuteProcessRequest, req.exe_req) + output_digest = exe_result.output_directory_digest + + output_file = assert_single_element( + await Get[FilesContent](Digest, output_digest)) + assert output_file.path == CoursierExecutionRequest._json_output_path + parsed_output = json.loads(output_file.content) + + flattened_resolution = CoursierMixin.extract_dependencies_by_root(parsed_output) + + # TODO: convert the file operations performed here into v2! + coord_to_resolved_jars = CoursierMixin.map_coord_to_resolved_jars( + result=parsed_output, + coursier_cache_path=coursier.get_options().cache_dir, + pants_jar_path_base=(req.jar_path_base or build_root.path)) + + # Construct a map from org:name to the reconciled org:name:version coordinate + # This is used when there is won't be a conflict_resolution entry because the conflict + # was resolved in pants. + org_name_to_org_name_rev = {} + for coord in coord_to_resolved_jars.keys(): + org_name_to_org_name_rev[f'{coord.org}:{coord.name}'] = coord + + jars_per_target = [] + + override_classifiers = CoursierMixin.override_classifiers_for_conf(req.conf.conf) + + for ht in req.source_targets: + t = ht.adaptor + jars_to_digest = [] + if isinstance(t.v1_target, JarLibrary): + def get_transitive_resolved_jars(my_coord, resolved_jars): + transitive_jar_path_for_coord = [] + coord_str = str(my_coord) + if coord_str in flattened_resolution and my_coord in resolved_jars: + transitive_jar_path_for_coord.append(resolved_jars[my_coord]) + + for c in flattened_resolution[coord_str]: + j = resolved_jars.get(CoursierMixin.to_m2_coord(c)) + if j: + transitive_jar_path_for_coord.append(j) + + return transitive_jar_path_for_coord + + for jar in t.v1_target.jar_dependencies: + # if there are override classifiers, then force use of those. + coord_candidates = [] + if override_classifiers: + coord_candidates = [jar.coordinate.copy(classifier=c) for c in override_classifiers] + else: + coord_candidates = [jar.coordinate] + + # If there are conflict resolution entries, then update versions to the resolved ones. + jar_spec = f'{jar.coordinate.org}:{jar.coordinate.name}' + if jar.coordinate.simple_coord in parsed_output['conflict_resolution']: + parsed_conflict = CoursierMixin.to_m2_coord( + parsed_output['conflict_resolution'][jar.coordinate.simple_coord]) + coord_candidates = [c.copy(rev=parsed_conflict.rev) for c in coord_candidates] + elif jar_spec in org_name_to_org_name_rev: + parsed_conflict = org_name_to_org_name_rev[jar_spec] + coord_candidates = [c.copy(rev=parsed_conflict.rev) for c in coord_candidates] + + for coord in coord_candidates: + transitive_resolved_jars = get_transitive_resolved_jars(coord, coord_to_resolved_jars) + if transitive_resolved_jars: + for jar in transitive_resolved_jars: + jars_to_digest.append(jar) + + jars_per_target.append((t.v1_target, tuple(jars_to_digest))) + + + return CoursierResolveResult( + base_conf=req.conf, + jars_per_target=tuple(jars_per_target), + ) + + +@dataclass(frozen=True) +class SnapshottedResolveResult: + resolution: CoursierResolveResult + merged_snapshot: Snapshot + + def __getattr__(self, key, **kwargs): + """Use the prototype pattern with the original coursier resolution resuslt.""" + if hasattr(self.resolution, key): + return getattr(self.resolution, key) + return super().__getattr__(key, **kwargs) + + +@rule +async def snapshotted_coursier_result( + res: CoursierResolveResult, + coursier: CoursierSubsystem, + build_root: BuildRoot, +) -> SnapshottedResolveResult: + # TODO: figure out whether the arbitrary_root kwarg is the right way to snapshot things outside of + # the buildroot (which also makes use of the coursier cache)!! + cache_root = os.path.realpath(coursier.get_options().cache_dir) + all_jar_paths = [os.path.realpath(j.cache_path) for j in res.resolved_jars] + + jars_in_cache_dir = [] + jars_in_build_root = [] + for p in all_jar_paths: + if p.startswith(cache_root): + jars_in_cache_dir.append(fast_relpath(p, cache_root)) + else: + fast_relpath(p, build_root.path) + + cache_dir_cp, build_root_cp = tuple(await MultiGet([ + Get[Snapshot](PathGlobs(include=jars_in_cache_dir, arbitrary_root=cache_root)), + Get[Snapshot](PathGlobs(include=jars_in_build_root)), + ])) + merged_snapshot = await Get[Snapshot](DirectoriesToMerge(tuple([ + cache_dir_cp.directory_digest, + build_root_cp.directory_digest, + ]))) + return SnapshottedResolveResult(resolution=res, merged_snapshot=merged_snapshot) + + +class TestCoursierResolve(Goal): + name = 'test-coursier-resolve' + + +@console_rule +async def test_coursier_resolve(console: Console) -> TestCoursierResolve: + coursier_result = await Get[SnapshottedResolveResult](CoursierRequest( + jar_resolution=JarResolveRequest(tuple([ + JarDependency(org='org.pantsbuild', name='zinc-compiler_2.12', rev='0.0.17'), + ])), + )) + console.print_stdout(f'coursier_result: {coursier_result}') + return TestCoursierResolve(exit_code=0) + + +def rules(): + return [ + optionable_rule(CoursierSubsystem), + snapshot_coursier, + optionable_rule(JarDependencyManagement), + RootRule(JarResolveRequest), + generate_coursier_execution_request, + execute_coursier, + snapshotted_coursier_result, + test_coursier_resolve, + ] diff --git a/src/python/pants/backend/jvm/rules/export_classpath.py b/src/python/pants/backend/jvm/rules/export_classpath.py new file mode 100644 index 00000000000..8fb516501a3 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/export_classpath.py @@ -0,0 +1,23 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from pants.engine.console import Console +from pants.engine.goal import Goal +from pants.engine.rules import console_rule + + +class ExportClasspath(Goal): + """???""" + name = 'fast-export-classpath' + + +@console_rule +def fast_export_classpath(console: Console) -> ExportClasspath: + console.print_stdout('wow!') + return ExportClasspath(exit_code=0) + + +def rules(): + return [ + fast_export_classpath, + ] diff --git a/src/python/pants/backend/jvm/rules/hermetic_dist.py b/src/python/pants/backend/jvm/rules/hermetic_dist.py new file mode 100644 index 00000000000..292c5d638db --- /dev/null +++ b/src/python/pants/backend/jvm/rules/hermetic_dist.py @@ -0,0 +1,54 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import Tuple + +from pants.backend.jvm.subsystems.jvm_platform import JvmPlatform +from pants.engine.rules import optionable_rule, rule +from pants.engine.selectors import Get +from pants.java.distribution.distribution import Distribution, DistributionLocator + + +@dataclass(frozen=True) +class JvmDistributionSearchSettings: + args: Tuple[str, ...] = () + + +@rule +def non_strict_select_jvm_distribution( + settings: JvmDistributionSearchSettings, +) -> Distribution: + """General utility method to select a jvm distribution, falling back to non-strict selection.""" + try: + local_distribution = JvmPlatform.preferred_jvm_distribution(settings.args, strict=True) + except DistributionLocator.Error: + local_distribution = JvmPlatform.preferred_jvm_distribution(settings.args, strict=False) + return local_distribution + + +@dataclass(frozen=True) +class HermeticDist: + home: str + underlying: Distribution + + @property + def symbolic_home(self) -> str: + return self.home + + @property + def underlying_home(self) -> str: + return self.underlying.home + + +@rule +async def hermetic_dist() -> HermeticDist: + local_dist = await Get[Distribution](JvmDistributionSearchSettings()) + return HermeticDist('.jdk', local_dist) + + +def rules(): + return [ + non_strict_select_jvm_distribution, + hermetic_dist, + ] diff --git a/src/python/pants/backend/jvm/rules/jvm_options.py b/src/python/pants/backend/jvm/rules/jvm_options.py new file mode 100644 index 00000000000..d3e0c102e15 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/jvm_options.py @@ -0,0 +1,11 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import Tuple + + +@dataclass(frozen=True) +class JvmOptions: + """VM Options for executing a JVM process.""" + options: Tuple[str, ...] diff --git a/src/python/pants/backend/jvm/rules/jvm_tool.py b/src/python/pants/backend/jvm/rules/jvm_tool.py new file mode 100644 index 00000000000..b4e0f3447e7 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/jvm_tool.py @@ -0,0 +1,71 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from dataclasses import dataclass +from typing import Tuple + +from pants.backend.jvm.rules.coursier import (CoursierRequest, JarResolveRequest, + SnapshottedResolveResult) +from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolRequest +from pants.build_graph.address import Address +from pants.engine.fs import Snapshot +from pants.engine.legacy.graph import HydratedTarget, TransitiveHydratedTargets +from pants.engine.rules import UnionRule, rule, union +from pants.engine.selectors import Get +from pants.util.collections import Enum + + +class JvmToolBootstrapError(Exception): pass + + +class JvmToolBootstrapTargetTypes(Enum): + jar_library = 'jar_library' + + +@union +class ClasspathRequest: pass + + +@dataclass(frozen=True) +class JarLibraryClasspathRequest: + jar_req: JarResolveRequest + + +@dataclass(frozen=True) +class JvmToolClasspathResult: + snapshot: Snapshot + + +@rule +async def snapshot_jar_library_classpath(req: JarLibraryClasspathRequest) -> JvmToolClasspathResult: + result = await Get[SnapshottedResolveResult](CoursierRequest(jar_resolution=req.jar_req)) + return JvmToolClasspathResult(result.merged_snapshot) + + +@rule +async def obtain_jvm_tool_classpath(jvm_tool_request: JvmToolRequest) -> JvmToolClasspathResult: + jvm_tool_target = await Get[HydratedTarget](Address, jvm_tool_request.address) + thts = await Get[TransitiveHydratedTargets](HydratedTarget, jvm_tool_target) + + # Do an enum "pattern match" to obtain the appropriate strategy for resolving a classpath from the + # target! + try: + jvm_tool_classpath_req = JvmToolBootstrapTargetTypes(target_adaptor.type_alias).match({ + JvmToolBootstrapTargetTypes.jar_library: lambda: JarLibraryClasspathRequest( + JarResolveRequest(thts)) + })() + except ValueError as e: + raise JvmToolBootstrapError(f'unrecognized target type: {e} for' + f'jvm tool request: {jvm_tool_request}! ' + f'recognized target types are: {list(JvmToolBootstrapTargetTypes)}!') + + # Yield back to the engine using the ClasspathRequest @union to select the appropriate strategy! + return await Get[JvmToolClasspathResult](ClasspathRequest, jvm_tool_classpath_req) + + +def rules(): + return [ + UnionRule(ClasspathRequest, JarLibraryClasspathRequest), + snapshot_jar_library_classpath, + obtain_jvm_tool_classpath, + ] diff --git a/src/python/pants/backend/jvm/rules/zinc_compile.py b/src/python/pants/backend/jvm/rules/zinc_compile.py new file mode 100644 index 00000000000..c2248e46330 --- /dev/null +++ b/src/python/pants/backend/jvm/rules/zinc_compile.py @@ -0,0 +1,6 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +def rules(): + return [] diff --git a/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py b/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py index 3d2e939bd3e..d5b5fd37f45 100644 --- a/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py +++ b/src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py @@ -2,14 +2,21 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from collections import namedtuple +from dataclasses import dataclass from textwrap import dedent from typing import List from pants.base.exceptions import TaskError +from pants.build_graph.address import Address from pants.java.distribution.distribution import DistributionLocator from pants.option.custom_types import target_option +@dataclass(frozen=True) +class JvmToolRequest: + address: Address + + class JvmToolMixin: """A mixin for registering and accessing JVM-based tools. @@ -223,3 +230,9 @@ def tool_classpath_entries_from_products(products, key, scope): if not callback: raise TaskError(f'No bootstrap callback registered for {key} in {scope}') return callback() + + def create_tool_request(self, key: str) -> JvmToolRequest: + # NB: This assumes that the JvmToolMixin is used on an Optionable to get self.get_options()! + pointed_to_target = getattr(self.get_options(), key.replace('-', '_')) # type: ignore[attr-defined] + parsed_address = Address.parse(pointed_to_target) + return JvmToolRequest(parsed_address) diff --git a/src/python/pants/backend/jvm/targets/jar_library.py b/src/python/pants/backend/jvm/targets/jar_library.py index 46f95df04e1..31ae71bddd9 100644 --- a/src/python/pants/backend/jvm/targets/jar_library.py +++ b/src/python/pants/backend/jvm/targets/jar_library.py @@ -7,6 +7,7 @@ from pants.build_graph.address import Address from pants.build_graph.target import Target from pants.java.jar.jar_dependency import JarDependency +from pants.util.memo import memoized_classproperty class JarLibrary(Target): @@ -15,6 +16,10 @@ class JarLibrary(Target): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['excludes', 'strict_deps', 'managed_dependencies']) + def __init__(self, payload=None, jars=None, managed_dependencies=None, **kwargs): """ :param jars: List of `jar <#jar>`_\\s to depend upon. diff --git a/src/python/pants/backend/jvm/targets/junit_tests.py b/src/python/pants/backend/jvm/targets/junit_tests.py index 06c81ae411e..d411806d0cb 100644 --- a/src/python/pants/backend/jvm/targets/junit_tests.py +++ b/src/python/pants/backend/jvm/targets/junit_tests.py @@ -8,6 +8,7 @@ from pants.base.exceptions import TargetDefinitionException from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField +from pants.util.memo import memoized_classproperty class JUnitTests(JvmTarget): @@ -35,6 +36,10 @@ class JUnitTests(JvmTarget): def subsystems(cls): return super().subsystems() + (JUnit,) + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['test_platform']) + def __init__(self, cwd=None, test_platform=None, payload=None, timeout=None, extra_jvm_options=None, extra_env_vars=None, concurrency=None, threads=None, **kwargs): diff --git a/src/python/pants/backend/jvm/targets/jvm_app.py b/src/python/pants/backend/jvm/targets/jvm_app.py index a5a4f779127..1bda6a9d813 100644 --- a/src/python/pants/backend/jvm/targets/jvm_app.py +++ b/src/python/pants/backend/jvm/targets/jvm_app.py @@ -5,6 +5,7 @@ from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField from pants.build_graph.app_base import AppBase +from pants.util.memo import memoized_classproperty class JvmApp(AppBase): @@ -18,6 +19,10 @@ class JvmApp(AppBase): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['jar_dependencies']) + def __init__(self, payload=None, deployjar=None, **kwargs): """ :param boolean deployjar: If True, pack all 3rdparty and internal jar classfiles into diff --git a/src/python/pants/backend/jvm/targets/jvm_binary.py b/src/python/pants/backend/jvm/targets/jvm_binary.py index 0aa89cbaaab..649a7817932 100644 --- a/src/python/pants/backend/jvm/targets/jvm_binary.py +++ b/src/python/pants/backend/jvm/targets/jvm_binary.py @@ -16,6 +16,7 @@ ) from pants.base.validation import assert_list from pants.java.jar.exclude import Exclude +from pants.util.memo import memoized_classproperty class JarRule(FingerprintedMixin, metaclass=ABCMeta): @@ -293,6 +294,10 @@ class JvmBinary(JvmTarget): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['deploy_excludes', 'manifest_entries']) + def __init__(self, name=None, address=None, diff --git a/src/python/pants/backend/jvm/targets/jvm_target.py b/src/python/pants/backend/jvm/targets/jvm_target.py index 9a4fe2663f6..732ca12adde 100644 --- a/src/python/pants/backend/jvm/targets/jvm_target.py +++ b/src/python/pants/backend/jvm/targets/jvm_target.py @@ -12,7 +12,7 @@ from pants.build_graph.resources import Resources from pants.build_graph.target import Target from pants.java.jar.exclude import Exclude -from pants.util.memo import memoized_property +from pants.util.memo import memoized_classproperty, memoized_property class JvmTarget(Target, Jarable): @@ -21,6 +21,10 @@ class JvmTarget(Target, Jarable): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['excludes']) + @classmethod def subsystems(cls): return super().subsystems() + (Java, JvmPlatform) diff --git a/src/python/pants/backend/jvm/targets/scala_library.py b/src/python/pants/backend/jvm/targets/scala_library.py index c9b9a0aa892..0fd7b53768c 100644 --- a/src/python/pants/backend/jvm/targets/scala_library.py +++ b/src/python/pants/backend/jvm/targets/scala_library.py @@ -10,6 +10,7 @@ from pants.base.payload_field import PrimitiveField from pants.build_graph.address import Address from pants.build_graph.target import Target +from pants.util.memo import memoized_classproperty SCOVERAGE = "scoverage" @@ -27,6 +28,10 @@ class ScalaLibrary(ExportableJvmLibrary): :API: public """ + @memoized_classproperty + def _non_v2_target_kwargs(cls): + return super()._non_v2_target_kwargs | frozenset(['java_sources']) + default_sources_globs = '*.scala' default_sources_exclude_globs = JUnitTests.scala_test_globs @@ -40,7 +45,8 @@ def skip_instrumentation(**kwargs): ScoveragePlatform.global_instance().is_blacklisted(kwargs['address'].spec)) def __init__(self, java_sources=None, scalac_plugins=None, scalac_plugin_args=None, - compiler_option_sets=None, payload=None, **kwargs): + compiler_option_sets=None, payload=None, + **kwargs): """ :param java_sources: Java libraries this library has a *circular* dependency on. diff --git a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py index 6a9f40f0a90..97c59aac442 100644 --- a/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py +++ b/src/python/pants/backend/jvm/tasks/coursier/coursier_subsystem.py @@ -1,7 +1,9 @@ # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import itertools import os +from typing import List from pants.base.build_environment import get_pants_cachedir from pants.binaries.binary_tool import Script @@ -53,6 +55,20 @@ def register_options(cls, register): def get_external_url_generator(self): return CoursierUrlGenerator(list(self.get_options().bootstrap_jar_urls)) + def common_args(self) -> List[str]: + repos = self.get_options().repos + # make [repoX, repoY] -> ['-r', repoX, '-r', repoY] + repo_args = list(itertools.chain(*list(zip(['-r'] * len(repos), repos)))) + artifact_types_arg = ['-A', ','.join(self.get_options().artifact_types)] + advanced_options = self.get_options().fetch_options + common_args: List[str] = [ + 'fetch', + # Print the resolution tree + '-t', + '--cache', self.get_options().cache_dir, + ] + repo_args + artifact_types_arg + advanced_options + return common_args + class CoursierUrlGenerator(BinaryToolUrlGenerator): diff --git a/src/python/pants/backend/jvm/tasks/coursier_resolve.py b/src/python/pants/backend/jvm/tasks/coursier_resolve.py index bf7f4a6f2bc..49a781a3274 100644 --- a/src/python/pants/backend/jvm/tasks/coursier_resolve.py +++ b/src/python/pants/backend/jvm/tasks/coursier_resolve.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import hashlib -import itertools import json import os from collections import defaultdict @@ -182,7 +181,7 @@ def resolve(self, targets, compile_classpath, sources, javadoc, executor): for conf, result_list in results.items(): for result in result_list: self._load_json_result(conf, compile_classpath, coursier_cache_dir, invalidation_check, - pants_jar_base_dir, result, self._override_classifiers_for_conf(conf)) + pants_jar_base_dir, result, self.override_classifiers_for_conf(conf)) self._populate_results_dir(vt_set_results_dir, results) resolve_vts.update() @@ -190,7 +189,8 @@ def resolve(self, targets, compile_classpath, sources, javadoc, executor): if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(resolve_vts, [vt_set_results_dir])]) - def _override_classifiers_for_conf(self, conf): + @classmethod + def override_classifiers_for_conf(cls, conf): # TODO Encapsulate this in the result from coursier instead of here. # https://github.com/coursier/coursier/issues/803 if conf == 'src_doc': @@ -264,16 +264,7 @@ def _get_result_from_coursier(self, jars_to_resolve, global_excludes, pinned_coo coursier_subsystem_instance = CoursierSubsystem.global_instance() coursier_jar = coursier_subsystem_instance.select() - repos = coursier_subsystem_instance.get_options().repos - # make [repoX, repoY] -> ['-r', repoX, '-r', repoY] - repo_args = list(itertools.chain(*list(zip(['-r'] * len(repos), repos)))) - artifact_types_arg = ['-A', ','.join(coursier_subsystem_instance.get_options().artifact_types)] - advanced_options = coursier_subsystem_instance.get_options().fetch_options - common_args = ['fetch', - # Print the resolution tree - '-t', - '--cache', coursier_cache_path - ] + repo_args + artifact_types_arg + advanced_options + common_args = coursier_subsystem_instance.common_args() coursier_work_temp_dir = os.path.join(self.versioned_workdir, 'tmp') safe_mkdir(coursier_work_temp_dir) @@ -369,7 +360,9 @@ def _call_coursier(self, cmd_args, coursier_jar, output_fn, executor): @staticmethod def _construct_cmd_args(jars, common_args, global_excludes, - pinned_coords, coursier_workdir, json_output_path): + pinned_coords, coursier_workdir, json_output_path, + strict_jar_revision_checking=True, + affecting_the_local_filesystem=True): # Make a copy, so there is no side effect or others using `common_args` cmd_args = list(common_args) @@ -378,7 +371,7 @@ def _construct_cmd_args(jars, common_args, global_excludes, # Dealing with intransitivity and forced versions. for j in jars: - if not j.rev: + if (not j.rev) and strict_jar_revision_checking: raise TaskError('Undefined revs for jars unsupported by Coursier. "{}"'.format(repr(j.coordinate).replace('M2Coordinate', 'jar'))) module = j.coordinate.simple_coord @@ -414,19 +407,22 @@ def _construct_cmd_args(jars, common_args, global_excludes, local_exclude_args.append(ex_arg) if local_exclude_args: - with temporary_file(coursier_workdir, cleanup=False) as f: - exclude_file = f.name - with open(exclude_file, 'w') as ex_f: - ex_f.write('\n'.join(local_exclude_args)) + if affecting_the_local_filesystem: + with temporary_file(coursier_workdir, cleanup=False) as f: + exclude_file = relative_local_excludes_path or f.name + with open(exclude_file, 'w') as ex_f: + ex_f.write('\n'.join(local_exclude_args)) - cmd_args.append('--local-exclude-file') - cmd_args.append(exclude_file) + cmd_args.append('--local-exclude-file') + cmd_args.append(exclude_file) for ex in global_excludes: cmd_args.append('-E') cmd_args.append(f"{ex.org}:{ex.name or '*'}") - return cmd_args + if affecting_the_local_filesystem: + return cmd_args + return cmd_args, local_exclude_args def _load_json_result(self, conf, compile_classpath, coursier_cache_path, invalidation_check, pants_jar_path_base, result, override_classifiers=None): @@ -441,9 +437,9 @@ def _load_json_result(self, conf, compile_classpath, coursier_cache_path, invali :return: n/a """ # Parse the coursier result - flattened_resolution = self._extract_dependencies_by_root(result) + flattened_resolution = self.extract_dependencies_by_root(result) - coord_to_resolved_jars = self._map_coord_to_resolved_jars(result, coursier_cache_path, pants_jar_path_base) + coord_to_resolved_jars = self.map_coord_to_resolved_jars(result, coursier_cache_path, pants_jar_path_base) # Construct a map from org:name to the reconciled org:name:version coordinate # This is used when there is won't be a conflict_resolution entry because the conflict @@ -525,14 +521,14 @@ def _load_from_results_dir(self, compile_classpath, vts_results_dir, invalidation_check, pants_jar_path_base, result, - self._override_classifiers_for_conf(conf)) + self.override_classifiers_for_conf(conf)) except CoursierResultNotFound: return False return True @classmethod - def _extract_dependencies_by_root(cls, result): + def extract_dependencies_by_root(cls, result): """ Only extracts the transitive dependencies for the given coursier resolve. Note the "dependencies" field is already transitive. @@ -572,7 +568,7 @@ def _extract_dependencies_by_root(cls, result): return flat_result @classmethod - def _map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): + def map_coord_to_resolved_jars(cls, result, coursier_cache_path, pants_jar_path_base): """ Map resolved files to each org:name:version diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py index 43a3a6b6824..41201d6d9eb 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile.py @@ -189,10 +189,10 @@ def prepare(cls, options, round_manager): @classmethod def subsystem_dependencies(cls): return super().subsystem_dependencies() + (DependencyContext, - Java, - JvmPlatform, - ScalaPlatform, - Zinc.Factory) + Java, + JvmPlatform, + ScalaPlatform, + Zinc.Factory) @classmethod def name(cls): @@ -922,7 +922,8 @@ def _plugin_targets(self, compiler): return {t.plugin: t.closure() for t in plugin_tgts} @staticmethod - def _local_jvm_distribution(settings=None): + def local_jvm_distribution(settings=None): + """General utility method to select a jvm distribution, falling back to non-strict selection.""" settings_args = [settings] if settings else [] try: local_distribution = JvmPlatform.preferred_jvm_distribution(settings_args, strict=True) @@ -974,7 +975,7 @@ def _get_jvm_distribution(self): # java version the target expects to be compiled against. # See: https://github.com/pantsbuild/pants/issues/6416 for covering using # different jdks in remote builds. - local_distribution = self._local_jvm_distribution() + local_distribution = self.local_jvm_distribution() return match(self.execution_strategy, { self.ExecutionStrategy.subprocess: lambda: local_distribution, self.ExecutionStrategy.nailgun: lambda: local_distribution, diff --git a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py index aea1983e24a..4f1a44cbd16 100644 --- a/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py +++ b/src/python/pants/backend/jvm/tasks/jvm_compile/rsc/rsc_compile.py @@ -33,7 +33,8 @@ from pants.task.scm_publish_mixin import Semver from pants.util.collections import assert_single_element from pants.util.contextutil import Timer -from pants.util.dirutil import fast_relpath, fast_relpath_optional, safe_mkdir +from pants.util.dirutil import (fast_relpath, fast_relpath_collection, fast_relpath_optional, + safe_mkdir) from pants.util.enums import match from pants.util.memo import memoized_method, memoized_property from pants.util.strutil import safe_shlex_join @@ -49,11 +50,6 @@ logger = logging.getLogger(__name__) -def fast_relpath_collection(collection): - buildroot = get_buildroot() - return [fast_relpath_optional(c, buildroot) or c for c in collection] - - def stdout_contents(wu): if isinstance(wu, FallibleExecuteProcessResult): return wu.stdout.rstrip() @@ -729,7 +725,7 @@ def _runtool_hermetic(self, main, tool_name, distribution, input_digest, ctx): use_youtline = tool_name == 'scalac-outliner' tool_classpath_abs = self._scalac_classpath if use_youtline else self._rsc_classpath - tool_classpath = fast_relpath_collection(tool_classpath_abs) + tool_classpath = fast_relpath_collection(tool_classpath_abs, root=get_buildroot()) rsc_jvm_options = Rsc.global_instance().get_options().jvm_options diff --git a/src/python/pants/backend/project_info/register.py b/src/python/pants/backend/project_info/register.py index 61e431e19dc..28aff3771c1 100644 --- a/src/python/pants/backend/project_info/register.py +++ b/src/python/pants/backend/project_info/register.py @@ -1,7 +1,7 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.project_info.rules import dependencies, source_file_validator +from pants.backend.project_info.rules import dependencies, export, source_file_validator from pants.backend.project_info.tasks.dependencies import Dependencies from pants.backend.project_info.tasks.depmap import Depmap from pants.backend.project_info.tasks.export import Export @@ -24,7 +24,8 @@ def register_goals(): def rules(): - return ( + return [ *source_file_validator.rules(), *dependencies.rules(), - ) + *export.rules(), + ] diff --git a/src/python/pants/backend/project_info/rules/BUILD b/src/python/pants/backend/project_info/rules/BUILD index 9c3f4bc7d54..ba8309b5a35 100644 --- a/src/python/pants/backend/project_info/rules/BUILD +++ b/src/python/pants/backend/project_info/rules/BUILD @@ -1,6 +1,12 @@ python_library( dependencies=[ '3rdparty/python:dataclasses', + 'src/python/pants/backend/codegen/thrift/java/rules', + 'src/python/pants/backend/jvm/rules', + 'src/python/pants/backend/jvm/targets:all', + 'src/python/pants/backend/project_info/tasks:export', + 'src/python/pants/backend/python/rules', + 'src/python/pants/base:hash_utils', 'src/python/pants/engine:console', 'src/python/pants/engine:fs', 'src/python/pants/engine:objects', diff --git a/src/python/pants/backend/project_info/rules/export.py b/src/python/pants/backend/project_info/rules/export.py new file mode 100644 index 00000000000..120d44e8013 --- /dev/null +++ b/src/python/pants/backend/project_info/rules/export.py @@ -0,0 +1,544 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import json +import os +from abc import ABC, abstractproperty +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Optional, Set, Tuple, Type, cast + +from pex.interpreter import PythonInterpreter +from twitter.common.collections import OrderedSet + +from pants.backend.codegen.thrift.java.rules.thrift_gen import ThriftedTarget +from pants.backend.jvm.rules.coursier import (CoursierRequest, JarResolveRequest, + ResolveConfiguration, SnapshottedResolveResult) +from pants.backend.jvm.rules.jvm_options import JvmOptions +from pants.backend.jvm.rules.jvm_tool import JvmToolClasspathResult +from pants.backend.jvm.subsystems.jvm_tool_mixin import JvmToolRequest +from pants.backend.jvm.subsystems.jvm_platform import JvmPlatform +from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform +from pants.backend.jvm.targets.jar_library import JarLibrary +from pants.backend.jvm.targets.junit_tests import JUnitTests +from pants.backend.jvm.targets.jvm_target import JvmTarget +from pants.backend.jvm.targets.scala_library import ScalaLibrary +from pants.backend.project_info.tasks.export import (ExportTask, SourceRootTypes, + DEFAULT_EXPORT_VERSION) +from pants.backend.python.interpreter_cache import PythonInterpreterCache +from pants.backend.python.python_requirement import PythonRequirement +from pants.backend.python.rules.pex import (CreatePex, Pex, PexInterpreterConstraints, + PexRequirements) +from pants.backend.python.subsystems.pex_build_util import has_python_requirements +from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary +from pants.backend.python.targets.python_target import PythonTarget +from pants.backend.python.targets.python_tests import PythonTests +from pants.base.build_root import BuildRoot +from pants.base.hash_utils import stable_json_sha1 +from pants.base.specs import Specs +from pants.build_graph.address import Address +from pants.build_graph.build_configuration import BuildConfiguration +from pants.build_graph.resources import Resources +from pants.build_graph.target import Target +from pants.engine.addressable import BuildFileAddress, BuildFileAddresses +from pants.engine.console import Console +from pants.engine.fs import Digest, DirectoriesToMerge, DirectoryToMaterialize, Workspace +from pants.engine.goal import Goal +from pants.engine.legacy.graph import HydratedTarget, TransitiveHydratedTargets +from pants.engine.objects import Collection +from pants.engine.rules import console_rule, optionable_rule, rule +from pants.engine.selectors import Get, MultiGet +from pants.help.build_dictionary_info_extracter import BuildDictionaryInfoExtracter +from pants.java.distribution.distribution import Distribution, DistributionLocator +from pants.java.jar.jar_dependency import JarDependency +from pants.java.jar.jar_dependency_utils import M2Coordinate, ResolvedJar +from pants.option.custom_types import dir_option +from pants.source.source_root import SourceRoot, SourceRootConfig +from pants.util.collections import Enum +from pants.util.dirutil import fast_relpath, fast_relpath_collection +from pants.util.memo import memoized_property + + +class ExportTarget(ABC): + + @abstractproperty + def hydrated_target(self) -> HydratedTarget: ... + + @abstractproperty + def is_synthetic(self) -> bool: ... + + +@dataclass(frozen=True) +class SourceTarget(ExportTarget): + _hydrated_target: HydratedTarget + + @property + def hydrated_target(self): + return self._hydrated_target + + is_synthetic = False + + +@dataclass(frozen=True) +class SyntheticTarget(ExportTarget): + thrifted_target: ThriftedTarget + + @property + def hydrated_target(self): + return self.thrifted_target.original + + is_synthetic = True + + +@dataclass(frozen=True) +class TargetAliasesMap: + aliases: Tuple[Tuple[Type[Target], str], ...] + all_symbols: Tuple[str, ...] + + @memoized_property + def aliases_mapping(self) -> Dict[Type[Target], str]: + # If a target class is registered under multiple aliases, we return the last one. + return { + ty: alias + for ty, alias in self.aliases + } + + +@rule +def make_target_aliases_map(build_config: BuildConfiguration) -> TargetAliasesMap: + registered_aliases = build_config.registered_aliases() + + aliases = [ + (ty, alias) + for alias, target_types in registered_aliases.target_types_by_alias.items() + for ty in target_types + ] + + extracter = BuildDictionaryInfoExtracter(registered_aliases) + + return TargetAliasesMap( + aliases=tuple(aliases), + all_symbols=tuple(x.symbol for x in extracter.get_target_type_info())) + + +@dataclass(frozen=True) +class SourceRootedSourceFile: + path: Path + package_prefix: str + + +class SourceRootedSourcesForTarget(Collection[SourceRootedSourceFile]): pass + + +@rule +def source_root_for_target( + source_root_config: SourceRootConfig, + hydrated_target: HydratedTarget, +) -> SourceRoot: + all_source_roots = source_root_config.get_source_roots() + return all_source_roots.find_by_path(hydrated_target.address.spec_path) + + +@rule +def rooted_sources_for_target( + source_root_config: SourceRootConfig, + hydrated_target: HydratedTarget, + build_root: BuildRoot, + source_root: SourceRoot, +) -> SourceRootedSourcesForTarget: + sources = getattr(hydrated_target.adaptor, 'sources', None) + if not sources: + return SourceRootedSourcesForTarget(()) + + relative_source_paths = zip( + sources.files_relative_to_buildroot, + fast_relpath_collection(sources.files_relative_to_buildroot, root=source_root.path)) + return SourceRootedSourcesForTarget(tuple( + SourceRootedSourceFile( + path=build_root.new_path.resolve(build_root_rel_path), + package_prefix=source_root_rel_path.replace(os.sep, '.'), + ) + for build_root_rel_path, source_root_rel_path in relative_source_paths + )) + + +@dataclass(frozen=True) +class PythonTargetSet: + interpreter: PythonInterpreter + targets: Tuple[Target, ...] + + +@dataclass(frozen=True) +class JustRequirementsPex: + interpreter: PythonInterpreter + pex: Pex + + +@rule +async def find_pex_for_target_set(target_set: PythonTargetSet) -> JustRequirementsPex: + interpreter = target_set.interpreter + bfas = [ + BuildFileAddress( + target_name=t.address.target_name, + # TODO: this is a hack so that the BuildFileAddress's spec_path will be set to the dirname of + # this path! + rel_path=os.path.join(t.address.spec_path, 'BUILD')) + for t in target_set.targets + ] + thts = await Get[TransitiveHydratedTargets](BuildFileAddresses(tuple(bfas))) + req_libs = [target.adaptor + for target in thts.closure + if has_python_requirements(target.adaptor.v1_target)] + pex_filename_base = stable_json_sha1(tuple( + adaptor._key() + for adaptor in req_libs + )) + + pex = await Get[Pex](CreatePex( + output_filename=f'{pex_filename_base}.pex', + requirements=PexRequirements.create_from_adaptors(req_libs), + interpreter_constraints=PexInterpreterConstraints.create_from_interpreter(interpreter), + )) + + return JustRequirementsPex(interpreter, pex) + + interpreters_info[str(interpreter.identity)] = { + 'binary': interpreter.binary, + 'pex': pex + } + + + +class Export(Goal): + """Generates a dictionary containing all pertinent information about the target graph. + + The return dictionary is suitable for serialization by json.dumps. + """ + name = 'export-v2' + + @classmethod + def register_options(cls, register): + super().register_options(register) + register('--libraries', default=True, type=bool, + help='Causes libraries to be output.') + register('--libraries-sources', type=bool, + help='Causes libraries with sources to be output.') + register('--libraries-javadocs', type=bool, + help='Causes libraries with javadocs to be output.') + register('--available-target-types', type=bool, + default=False, + help='Causes a list of available target types to be output.') + register('--sources', type=bool, + help='Causes sources to be output.') + register('--formatted', type=bool, implicit_value=False, + help='Causes output to be a single line of JSON.') + register('--jvm-options', type=list, metavar='