From b1120138beeb0d1a6df9fa6f8c33f35bbeefc42a Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Mon, 14 Mar 2022 16:25:26 -0400 Subject: [PATCH 1/8] checkpoint [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/java/bsp/rules.py | 8 +- src/python/pants/backend/scala/bsp/rules.py | 8 +- src/python/pants/bsp/goal.py | 13 +- src/python/pants/bsp/util_rules/targets.py | 171 ++++++++++++++++---- 4 files changed, 160 insertions(+), 40 deletions(-) diff --git a/src/python/pants/backend/java/bsp/rules.py b/src/python/pants/backend/java/bsp/rules.py index 58487d88ca5..7abc573c50b 100644 --- a/src/python/pants/backend/java/bsp/rules.py +++ b/src/python/pants/backend/java/bsp/rules.py @@ -20,7 +20,7 @@ ) from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsRequest +from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet from pants.build_graph.address import Address, AddressInput from pants.engine.addresses import Addresses from pants.engine.fs import CreateDigest, DigestEntries @@ -53,7 +53,7 @@ class JavaBSPLanguageSupport(BSPLanguageSupport): can_compile = True -class JavaBSPBuildTargetsRequest(BSPBuildTargetsRequest): +class JavaBSPBuildTargetsFieldSet(BSPBuildTargetsFieldSet): pass @@ -99,7 +99,7 @@ async def bsp_resolve_one_java_build_target( @rule async def bsp_resolve_all_java_build_targets( - _: JavaBSPBuildTargetsRequest, + _: JavaBSPBuildTargetsFieldSet, all_java_targets: AllJavaTargets, bsp_context: BSPContext, ) -> BSPBuildTargets: @@ -216,7 +216,7 @@ def rules(): return ( *collect_rules(), UnionRule(BSPLanguageSupport, JavaBSPLanguageSupport), - UnionRule(BSPBuildTargetsRequest, JavaBSPBuildTargetsRequest), + UnionRule(BSPBuildTargetsFieldSet, JavaBSPBuildTargetsFieldSet), UnionRule(BSPHandlerMapping, JavacOptionsHandlerMapping), UnionRule(BSPCompileFieldSet, JavaBSPCompileFieldSet), ) diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 760992e34be..3ac8b148055 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -27,7 +27,7 @@ ) from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsRequest +from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet from pants.build_graph.address import Address, AddressInput from pants.engine.addresses import Addresses from pants.engine.fs import EMPTY_DIGEST, AddPrefix, CreateDigest, Digest, DigestEntries @@ -63,7 +63,7 @@ class ScalaBSPLanguageSupport(BSPLanguageSupport): can_compile = True -class ScalaBSPBuildTargetsRequest(BSPBuildTargetsRequest): +class ScalaBSPBuildTargetsFieldSet(BSPBuildTargetsFieldSet): pass @@ -177,7 +177,7 @@ async def bsp_resolve_one_scala_build_target( @rule async def bsp_resolve_all_scala_build_targets( - _: ScalaBSPBuildTargetsRequest, + _: ScalaBSPBuildTargetsFieldSet, all_scala_targets: AllScalaTargets, bsp_context: BSPContext, ) -> BSPBuildTargets: @@ -299,7 +299,7 @@ def rules(): return ( *collect_rules(), UnionRule(BSPLanguageSupport, ScalaBSPLanguageSupport), - UnionRule(BSPBuildTargetsRequest, ScalaBSPBuildTargetsRequest), + UnionRule(BSPBuildTargetsFieldSet, ScalaBSPBuildTargetsFieldSet), UnionRule(BSPHandlerMapping, ScalacOptionsHandlerMapping), UnionRule(BSPCompileFieldSet, ScalaBSPCompileFieldSet), ) diff --git a/src/python/pants/bsp/goal.py b/src/python/pants/bsp/goal.py index aef659b55bd..28e2d6c43bb 100644 --- a/src/python/pants/bsp/goal.py +++ b/src/python/pants/bsp/goal.py @@ -8,7 +8,7 @@ import shlex import sys import textwrap -from typing import Mapping +from typing import Any, Mapping from pants.base.build_root import BuildRoot from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode @@ -22,7 +22,7 @@ from pants.engine.unions import UnionMembership from pants.goal.builtin_goal import BuiltinGoal from pants.init.engine_initializer import GraphSession -from pants.option.option_types import BoolOption, StrListOption +from pants.option.option_types import BoolOption, DictOption, StrListOption from pants.option.option_value_container import OptionValueContainer from pants.option.options import Options from pants.util.docutil import bin_name @@ -66,6 +66,15 @@ def activated(cls, union_membership: UnionMembership) -> bool: advanced=True, ) + target_mapping = DictOption[Any]( + "--target-mapping", + help=( + 'Defines how Pants maps targets to "build targets" that are exposed to IDEs via the ' + 'Build Server Protocol ("BSP"). The key for each entry is the ID to be used in the BSP protocol. ' + 'The value of each entry is a list of Pants addresses specs; for example, `["src/python::"]`.' + ), + ) + def run( self, *, diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index e6b90b52eb4..28e72fe45fc 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -2,11 +2,17 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations +import logging +import os.path from dataclasses import dataclass +from typing import ClassVar from pants.base.build_root import BuildRoot +from pants.base.specs import AddressSpecs, Specs +from pants.base.specs_parser import SpecsParser +from pants.bsp.goal import BSPGoal from pants.bsp.protocol import BSPHandlerMapping -from pants.bsp.spec.base import BuildTarget, BuildTargetIdentifier +from pants.bsp.spec.base import BuildTarget, BuildTargetCapabilities, BuildTargetIdentifier from pants.bsp.spec.targets import ( DependencyModule, DependencyModulesItem, @@ -33,15 +39,25 @@ SourcesField, SourcesPaths, SourcesPathsRequest, + Targets, WrappedTarget, ) from pants.engine.unions import UnionMembership, UnionRule, union +from pants.util.frozendict import FrozenDict + + +_logger = logging.getLogger(__name__) @union -class BSPBuildTargetsRequest: - """Request language backends to provide BSP `BuildTarget` instances for their managed target - types.""" +@dataclass(frozen=True) +class BSPBuildTargetsFieldSet: + language_id: ClassVar[str] + + +@dataclass(frozen=True) +class BSPBuildTargetsNew: + targets_mapping: FrozenDict[str, Specs] @dataclass(frozen=True) @@ -52,6 +68,30 @@ class BSPBuildTargets: digest: Digest = EMPTY_DIGEST +@dataclass(frozen=True) +class _ParseOneBSPMappingRequest: + raw_specs: tuple[str, ...] + + +@rule +async def parse_one_bsp_mapping(request: _ParseOneBSPMappingRequest) -> Specs: + specs_parser = SpecsParser() + specs = specs_parser.parse_specs(request.raw_specs) + return specs + + +@rule +async def materialize_bsp_build_targets(bsp_goal: BSPGoal) -> BSPBuildTargetsNew: + specs_for_keys = await MultiGet( + Get(Specs, _ParseOneBSPMappingRequest(tuple(value))) for value in bsp_goal.target_mapping.values() + ) + addr_specs = { + key: specs_for_key + for key, specs_for_key in zip(bsp_goal.target_mapping.keys(), specs_for_keys) + } + return BSPBuildTargetsNew(FrozenDict(addr_specs)) + + # ----------------------------------------------------------------------------------------------- # Workspace Build Targets Request # See https://build-server-protocol.github.io/docs/specification.html#workspace-build-targets-request @@ -66,19 +106,64 @@ class WorkspaceBuildTargetsHandlerMapping(BSPHandlerMapping): @_uncacheable_rule async def bsp_workspace_build_targets( - _: WorkspaceBuildTargetsParams, union_membership: UnionMembership, workspace: Workspace + _: WorkspaceBuildTargetsParams, + bsp_build_targets: BSPBuildTargetsNew, + union_membership: UnionMembership, + workspace: Workspace, + build_root: BuildRoot, ) -> WorkspaceBuildTargetsResult: - request_types = union_membership.get(BSPBuildTargetsRequest) - responses = await MultiGet( - Get(BSPBuildTargets, BSPBuildTargetsRequest, request_type()) - for request_type in request_types - ) + # metadata_field_set_types = union_membership.get(BSPBuildTargetsFieldSet) + result: list[BuildTarget] = [] - for response in responses: - result.extend(response.targets) - result.sort(key=lambda btgt: btgt.id.uri) - output_digest = await Get(Digest, MergeDigests([r.digest for r in responses])) - workspace.write_digest(output_digest, path_prefix=".pants.d/bsp") + for bsp_target_name, specs in bsp_build_targets.targets_mapping.items(): + targets = await Get(Targets, AddressSpecs, specs.address_specs) + targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] + # TODO: What about literal specs? + + # applicable_field_sets: dict[Target, list[Type[BSPBuildTargetsFieldSet]]] = defaultdict(list) + # for tgt in targets: + # for field_set_type in metadata_field_set_types: + # if field_set_type.is_applicable(tgt): + # applicable_field_sets[tgt].append(field_set_type) + + sources_paths = await MultiGet( + Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources + ) + merged_sources_dirs: set[str] = set() + for sp in sources_paths: + merged_sources_dirs.update(sp.dirs) + + base_dir = build_root.pathlib_path + if merged_sources_dirs: + common_path = os.path.commonpath(list(merged_sources_dirs)) + if common_path: + base_dir = base_dir.joinpath(common_path) + + result.append( + BuildTarget( + id=BuildTargetIdentifier(f"pants:{bsp_target_name}"), + display_name=bsp_target_name, + base_directory=base_dir.as_uri(), + tags=(), + capabilities=BuildTargetCapabilities( + can_compile=True, + ), + language_ids=("java", "scala"), + dependencies=(), + data_kind=None, + data=None, + ) + ) + # responses = await MultiGet( + # Get(BSPBuildTargets, BSPBuildTargetsFieldSet, request_type()) + # for request_type in request_types + # ) + # for response in responses: + # result.extend(response.targets) + # result.sort(key=lambda btgt: btgt.id.uri) + # output_digest = await Get(Digest, MergeDigests([r.digest for r in responses])) + # workspace.write_digest(output_digest, path_prefix=".pants.d/bsp") + return WorkspaceBuildTargetsResult( targets=tuple(result), ) @@ -110,31 +195,57 @@ class MaterializeBuildTargetSourcesResult: async def materialize_bsp_build_target_sources( request: MaterializeBuildTargetSourcesRequest, build_root: BuildRoot, + bsp_build_targets: BSPBuildTargetsNew, ) -> MaterializeBuildTargetSourcesResult: - wrapped_target = await Get(WrappedTarget, AddressInput, request.bsp_target_id.address_input) - target = wrapped_target.target - - if not target.has_field(SourcesField): - raise AssertionError( - f"BSP only handles targets with sources: uri={request.bsp_target_id.uri}" - ) - - sources_paths = await Get(SourcesPaths, SourcesPathsRequest(target[SourcesField])) - sources_full_paths = [ - build_root.pathlib_path.joinpath(src_path) for src_path in sources_paths.files - ] + bsp_target_name = request.bsp_target_id.uri[len("pants:") :] + if bsp_target_name not in bsp_build_targets.targets_mapping: + raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") + targets = await Get( + Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + ) + targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] + + # wrapped_target = await Get(WrappedTarget, AddressInput, request.bsp_target_id.address_input) + # target = wrapped_target.target + # + # if not target.has_field(SourcesField): + # raise AssertionError( + # f"BSP only handles targets with sources: uri={request.bsp_target_id.uri}" + # ) + # + # sources_paths = await Get(SourcesPaths, SourcesPathsRequest(target[SourcesField])) + # sources_full_paths = [ + # build_root.pathlib_path.joinpath(src_path) for src_path in sources_paths.files + # ] + + sources_paths = await MultiGet( + Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources + ) + merged_sources_dirs: set[str] = set() + merged_sources_files: set[str] = set() + for sp in sources_paths: + merged_sources_dirs.update(sp.dirs) + merged_sources_files.update(sp.files) + _logger.info(f"merged_sources_dirs={merged_sources_dirs}") + _logger.info(f"merged_sources_files={merged_sources_files}") + + base_dir = build_root.pathlib_path + if merged_sources_dirs: + common_path = os.path.commonpath(list(merged_sources_dirs)) + if common_path: + base_dir = base_dir.joinpath(common_path) sources_item = SourcesItem( target=request.bsp_target_id, sources=tuple( SourceItem( - uri=src_full_path.as_uri(), + uri=build_root.pathlib_path.joinpath(filename).as_uri(), kind=SourceItemKind.FILE, generated=False, ) - for src_full_path in sources_full_paths + for filename in sorted(merged_sources_files) ), - roots=(), + roots=(base_dir.as_uri(),), ) return MaterializeBuildTargetSourcesResult(sources_item) From c4065f035b4fb4641f4125d42f0f72df9834e733 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Tue, 15 Mar 2022 22:23:16 -0400 Subject: [PATCH 2/8] checkpoint [ci skip-rust] [ci skip-build-wheels] --- .../backend/experimental/scala/register.py | 2 + src/python/pants/backend/java/bsp/rules.py | 35 ++++-- src/python/pants/backend/scala/bsp/rules.py | 113 +++++++----------- .../pants/backend/scala/bsp/util_rules.py | 92 ++++++++++++++ src/python/pants/bsp/util_rules/targets.py | 15 ++- 5 files changed, 169 insertions(+), 88 deletions(-) create mode 100644 src/python/pants/backend/scala/bsp/util_rules.py diff --git a/src/python/pants/backend/experimental/scala/register.py b/src/python/pants/backend/experimental/scala/register.py index c87d8e25011..cee73a19ed8 100644 --- a/src/python/pants/backend/experimental/scala/register.py +++ b/src/python/pants/backend/experimental/scala/register.py @@ -1,5 +1,6 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from pants.backend.scala.bsp import util_rules from pants.backend.scala.bsp.rules import rules as bsp_rules from pants.backend.scala.compile import scalac from pants.backend.scala.dependency_inference import rules as dep_inf_rules @@ -63,5 +64,6 @@ def rules(): *run_deploy_jar.rules(), *scala_lockfile_rules(), *bsp_rules(), + *util_rules.rules(), *war_rules(), ] diff --git a/src/python/pants/backend/java/bsp/rules.py b/src/python/pants/backend/java/bsp/rules.py index 7abc573c50b..f268685d296 100644 --- a/src/python/pants/backend/java/bsp/rules.py +++ b/src/python/pants/backend/java/bsp/rules.py @@ -10,6 +10,7 @@ from pants.backend.java.dependency_inference.symbol_mapper import AllJavaTargets from pants.backend.java.target_types import JavaSourceField from pants.base.build_root import BuildRoot +from pants.base.specs import AddressSpecs from pants.bsp.context import BSPContext from pants.bsp.protocol import BSPHandlerMapping from pants.bsp.spec.base import ( @@ -20,7 +21,7 @@ ) from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet +from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet, BSPBuildTargetsNew from pants.build_graph.address import Address, AddressInput from pants.engine.addresses import Addresses from pants.engine.fs import CreateDigest, DigestEntries @@ -32,7 +33,7 @@ Dependencies, DependenciesRequest, Target, - WrappedTarget, + WrappedTarget, Targets, ) from pants.engine.unions import UnionMembership, UnionRule from pants.jvm.bsp.spec import JvmBuildTarget @@ -137,23 +138,31 @@ class HandleJavacOptionsResult: async def handle_bsp_java_options_request( request: HandleJavacOptionsRequest, build_root: BuildRoot, + bsp_build_targets: BSPBuildTargetsNew, ) -> HandleJavacOptionsResult: - wrapped_target = await Get(WrappedTarget, AddressInput, request.bsp_target_id.address_input) - coarsened_targets = await Get(CoarsenedTargets, Addresses([wrapped_target.target.address])) - assert len(coarsened_targets) == 1 - coarsened_target = coarsened_targets[0] - resolve = await Get(CoursierResolveKey, CoarsenedTargets([coarsened_target])) - output_file = compute_output_jar_filename(coarsened_target) + bsp_target_name = request.bsp_target_id.uri[len("pants:") :] + if bsp_target_name not in bsp_build_targets.targets_mapping: + raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") + targets = await Get( + Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + ) + + coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) + # assert len(coarsened_targets) == 1 + # coarsened_target = coarsened_targets[0] + resolve = await Get(CoursierResolveKey, CoarsenedTargets, coarsened_targets) + # output_file = compute_output_jar_filename(coarsened_target) return HandleJavacOptionsResult( JavacOptionsItem( target=request.bsp_target_id, options=(), - classpath=( - build_root.pathlib_path.joinpath( - f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" - ).as_uri(), - ), + # classpath=( + # build_root.pathlib_path.joinpath( + # f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" + # ).as_uri(), + # ), + classpath=(), class_directory=build_root.pathlib_path.joinpath( f".pants.d/bsp/jvm/resolves/{resolve.name}/classes" ).as_uri(), diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 3ac8b148055..71d0bf8f2ae 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -5,6 +5,7 @@ import os from dataclasses import dataclass +from pants.backend.scala.bsp import util_rules from pants.backend.scala.bsp.spec import ( ScalaBuildTarget, ScalacOptionsItem, @@ -12,11 +13,13 @@ ScalacOptionsResult, ScalaPlatform, ) +from pants.backend.scala.bsp.util_rules import MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest from pants.backend.scala.compile.scalac import compute_output_jar_filename from pants.backend.scala.dependency_inference.symbol_mapper import AllScalaTargets from pants.backend.scala.subsystems.scala import ScalaSubsystem from pants.backend.scala.target_types import ScalaSourceField from pants.base.build_root import BuildRoot +from pants.base.specs import AddressSpecs from pants.bsp.context import BSPContext from pants.bsp.protocol import BSPHandlerMapping from pants.bsp.spec.base import ( @@ -27,7 +30,7 @@ ) from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet +from pants.bsp.util_rules.targets import BSPBuildTargetsFieldSet, BSPBuildTargetsNew from pants.build_graph.address import Address, AddressInput from pants.engine.addresses import Addresses from pants.engine.fs import EMPTY_DIGEST, AddPrefix, CreateDigest, Digest, DigestEntries @@ -39,7 +42,7 @@ Dependencies, DependenciesRequest, Target, - WrappedTarget, + WrappedTarget, Targets, ) from pants.engine.unions import UnionMembership, UnionRule from pants.jvm.compile import ( @@ -78,48 +81,6 @@ class ResolveScalaBSPBuildTargetResult: scala_runtime: Snapshot -@dataclass(frozen=True) -class MaterializeScalaRuntimeJarsRequest: - scala_version: str - - -@dataclass(frozen=True) -class MaterializeScalaRuntimeJarsResult: - content: Snapshot - - -@rule -async def materialize_scala_runtime_jars( - request: MaterializeScalaRuntimeJarsRequest, -) -> MaterializeScalaRuntimeJarsResult: - tool_classpath = await Get( - ToolClasspath, - ToolClasspathRequest( - artifact_requirements=ArtifactRequirements.from_coordinates( - [ - Coordinate( - group="org.scala-lang", - artifact="scala-compiler", - version=request.scala_version, - ), - Coordinate( - group="org.scala-lang", - artifact="scala-library", - version=request.scala_version, - ), - ] - ), - ), - ) - - materialized_classpath_digest = await Get( - Digest, - AddPrefix(tool_classpath.content.digest, f"jvm/scala-runtime/{request.scala_version}"), - ) - materialized_classpath = await Get(Snapshot, Digest, materialized_classpath_digest) - return MaterializeScalaRuntimeJarsResult(materialized_classpath) - - @rule async def bsp_resolve_one_scala_build_target( request: ResolveScalaBSPBuildTargetRequest, @@ -175,22 +136,22 @@ async def bsp_resolve_one_scala_build_target( return ResolveScalaBSPBuildTargetResult(bsp_target, scala_runtime=scala_runtime.content) -@rule -async def bsp_resolve_all_scala_build_targets( - _: ScalaBSPBuildTargetsFieldSet, - all_scala_targets: AllScalaTargets, - bsp_context: BSPContext, -) -> BSPBuildTargets: - if LANGUAGE_ID not in bsp_context.client_params.capabilities.language_ids: - return BSPBuildTargets() - build_targets = await MultiGet( - Get(ResolveScalaBSPBuildTargetResult, ResolveScalaBSPBuildTargetRequest(tgt)) - for tgt in all_scala_targets - ) - output_digest = await Get(Digest, MergeDigests([d.scala_runtime.digest for d in build_targets])) - return BSPBuildTargets( - targets=tuple(btgt.build_target for btgt in build_targets), digest=output_digest - ) +# @rule +# async def bsp_resolve_all_scala_build_targets( +# _: ScalaBSPBuildTargetsFieldSet, +# all_scala_targets: AllScalaTargets, +# bsp_context: BSPContext, +# ) -> BSPBuildTargets: +# if LANGUAGE_ID not in bsp_context.client_params.capabilities.language_ids: +# return BSPBuildTargets() +# build_targets = await MultiGet( +# Get(ResolveScalaBSPBuildTargetResult, ResolveScalaBSPBuildTargetRequest(tgt)) +# for tgt in all_scala_targets +# ) +# output_digest = await Get(Digest, MergeDigests([d.scala_runtime.digest for d in build_targets])) +# return BSPBuildTargets( +# targets=tuple(btgt.build_target for btgt in build_targets), digest=output_digest +# ) # ----------------------------------------------------------------------------------------------- @@ -219,23 +180,30 @@ class HandleScalacOptionsResult: async def handle_bsp_scalac_options_request( request: HandleScalacOptionsRequest, build_root: BuildRoot, + bsp_build_targets: BSPBuildTargetsNew, ) -> HandleScalacOptionsResult: - wrapped_target = await Get(WrappedTarget, AddressInput, request.bsp_target_id.address_input) - coarsened_targets = await Get(CoarsenedTargets, Addresses([wrapped_target.target.address])) - assert len(coarsened_targets) == 1 - coarsened_target = coarsened_targets[0] - resolve = await Get(CoursierResolveKey, CoarsenedTargets([coarsened_target])) - output_file = compute_output_jar_filename(coarsened_target) + bsp_target_name = request.bsp_target_id.uri[len("pants:") :] + if bsp_target_name not in bsp_build_targets.targets_mapping: + raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") + targets = await Get( + Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + ) + coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) + # assert len(coarsened_targets) == 1 + # coarsened_target = coarsened_targets[0] + resolve = await Get(CoursierResolveKey, CoarsenedTargets, coarsened_targets) + # output_file = compute_output_jar_filename(coarsened_target) return HandleScalacOptionsResult( ScalacOptionsItem( target=request.bsp_target_id, options=(), - classpath=( - build_root.pathlib_path.joinpath( - f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" - ).as_uri(), - ), + # classpath=( + # build_root.pathlib_path.joinpath( + # f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" + # ).as_uri(), + # ), + classpath=(), class_directory=build_root.pathlib_path.joinpath( f".pants.d/bsp/jvm/resolves/{resolve.name}/classes" ).as_uri(), @@ -298,8 +266,9 @@ async def bsp_scala_compile_request( def rules(): return ( *collect_rules(), + *util_rules.rules(), UnionRule(BSPLanguageSupport, ScalaBSPLanguageSupport), - UnionRule(BSPBuildTargetsFieldSet, ScalaBSPBuildTargetsFieldSet), + # UnionRule(BSPBuildTargetsFieldSet, ScalaBSPBuildTargetsFieldSet), UnionRule(BSPHandlerMapping, ScalacOptionsHandlerMapping), UnionRule(BSPCompileFieldSet, ScalaBSPCompileFieldSet), ) diff --git a/src/python/pants/backend/scala/bsp/util_rules.py b/src/python/pants/backend/scala/bsp/util_rules.py new file mode 100644 index 00000000000..8ebdee48c0b --- /dev/null +++ b/src/python/pants/backend/scala/bsp/util_rules.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from pants.backend.scala.bsp.spec import ScalaBuildTarget, ScalaPlatform +from pants.backend.scala.subsystems.scala import ScalaSubsystem +from pants.base.build_root import BuildRoot +from pants.engine.internals.native_engine import Digest, AddPrefix, Snapshot +from pants.engine.internals.selectors import Get +from pants.engine.rules import rule, collect_rules +from pants.jvm.resolve.common import ArtifactRequirements, Coordinate +from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest +from pants.jvm.subsystems import JvmSubsystem +from pants.jvm.target_types import JvmResolveField + + +@dataclass(frozen=True) +class MaterializeScalaRuntimeJarsRequest: + scala_version: str + + +@dataclass(frozen=True) +class MaterializeScalaRuntimeJarsResult: + content: Snapshot + + +@rule +async def materialize_scala_runtime_jars( + request: MaterializeScalaRuntimeJarsRequest, +) -> MaterializeScalaRuntimeJarsResult: + tool_classpath = await Get( + ToolClasspath, + ToolClasspathRequest( + artifact_requirements=ArtifactRequirements.from_coordinates( + [ + Coordinate( + group="org.scala-lang", + artifact="scala-compiler", + version=request.scala_version, + ), + Coordinate( + group="org.scala-lang", + artifact="scala-library", + version=request.scala_version, + ), + ] + ), + ), + ) + + materialized_classpath_digest = await Get( + Digest, + AddPrefix(tool_classpath.content.digest, f"jvm/scala-runtime/{request.scala_version}"), + ) + materialized_classpath = await Get(Snapshot, Digest, materialized_classpath_digest) + return MaterializeScalaRuntimeJarsResult(materialized_classpath) + + +@dataclass(frozen=True) +class ScalaBuildTargetInfo: + btgt: ScalaBuildTarget + digest: Digest + + +@rule +async def make_scala_build_target( + resolve_field: JvmResolveField, + jvm: JvmSubsystem, + scala: ScalaSubsystem, + build_root: BuildRoot, +) -> ScalaBuildTargetInfo: + resolve = resolve_field.normalized_value(jvm) + scala_version = scala.version_for_resolve(resolve) + scala_runtime = await Get(MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest(scala_version)) + scala_jar_uris = tuple( + build_root.pathlib_path.joinpath(".pants.d/bsp").joinpath(p).as_uri() + for p in scala_runtime.content.files + ) + return ScalaBuildTargetInfo( + btgt=ScalaBuildTarget( + scala_organization="org.scala-lang", + scala_version=scala_version, + scala_binary_version=".".join(scala_version.split(".")[0:2]), + platform=ScalaPlatform.JVM, + jars=scala_jar_uris, + ), + digest=scala_runtime.content.digest, + ) + + +def rules(): + return collect_rules() \ No newline at end of file diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index 28e72fe45fc..f9ac7d4333c 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import ClassVar +from pants.backend.scala.bsp.util_rules import ScalaBuildTargetInfo from pants.base.build_root import BuildRoot from pants.base.specs import AddressSpecs, Specs from pants.base.specs_parser import SpecsParser @@ -43,6 +44,7 @@ WrappedTarget, ) from pants.engine.unions import UnionMembership, UnionRule, union +from pants.jvm.target_types import JvmResolveField from pants.util.frozendict import FrozenDict @@ -113,13 +115,17 @@ async def bsp_workspace_build_targets( build_root: BuildRoot, ) -> WorkspaceBuildTargetsResult: # metadata_field_set_types = union_membership.get(BSPBuildTargetsFieldSet) - + digest: Digest = EMPTY_DIGEST result: list[BuildTarget] = [] for bsp_target_name, specs in bsp_build_targets.targets_mapping.items(): targets = await Get(Targets, AddressSpecs, specs.address_specs) targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] # TODO: What about literal specs? + # TODO: Check for conflicting resolves. + resolve_field = targets[0][JvmResolveField] + scala_info = await Get(ScalaBuildTargetInfo, JvmResolveField, resolve_field) + digest = await Get(Digest, MergeDigests([digest, scala_info.digest])) # applicable_field_sets: dict[Target, list[Type[BSPBuildTargetsFieldSet]]] = defaultdict(list) # for tgt in targets: # for field_set_type in metadata_field_set_types: @@ -150,8 +156,8 @@ async def bsp_workspace_build_targets( ), language_ids=("java", "scala"), dependencies=(), - data_kind=None, - data=None, + data_kind="scala", + data=scala_info.btgt, ) ) # responses = await MultiGet( @@ -164,6 +170,9 @@ async def bsp_workspace_build_targets( # output_digest = await Get(Digest, MergeDigests([r.digest for r in responses])) # workspace.write_digest(output_digest, path_prefix=".pants.d/bsp") + if digest != EMPTY_DIGEST: + workspace.write_digest(digest, path_prefix=".pants.d/bsp") + return WorkspaceBuildTargetsResult( targets=tuple(result), ) From 364177637f86ea1c03b04ee38adb7f627d0b5a14 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 16 Mar 2022 12:12:17 -0400 Subject: [PATCH 3/8] checkpoint [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/scala/bsp/rules.py | 8 +- src/python/pants/bsp/spec/base.py | 16 +- src/python/pants/bsp/util_rules/targets.py | 214 +++++++++++++------- 3 files changed, 153 insertions(+), 85 deletions(-) diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 71d0bf8f2ae..6cf6edde61a 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -30,7 +30,7 @@ ) from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargetsFieldSet, BSPBuildTargetsNew +from pants.bsp.util_rules.targets import BSPBuildTargetsFieldSet, BSPBuildTargetsNew, BSPBuildTargetsMetadataRequest from pants.build_graph.address import Address, AddressInput from pants.engine.addresses import Addresses from pants.engine.fs import EMPTY_DIGEST, AddPrefix, CreateDigest, Digest, DigestEntries @@ -66,8 +66,10 @@ class ScalaBSPLanguageSupport(BSPLanguageSupport): can_compile = True -class ScalaBSPBuildTargetsFieldSet(BSPBuildTargetsFieldSet): - pass +class ScalaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest): + language_id = LANGUAGE_ID + compatible_language_ids = ("java",) + field_set_type = ScalaSourceField @dataclass(frozen=True) diff --git a/src/python/pants/bsp/spec/base.py b/src/python/pants/bsp/spec/base.py index dde93e3efd6..3bc9c267cb1 100644 --- a/src/python/pants/bsp/spec/base.py +++ b/src/python/pants/bsp/spec/base.py @@ -121,13 +121,9 @@ class BuildTarget: # The direct upstream build target dependencies of this build target dependencies: tuple[BuildTargetIdentifier, ...] - # Kind of data to expect in the `data` field. If this field is not set, the kind of data is not specified. - data_kind: str | None - # Language-specific metadata about this target. # See ScalaBuildTarget as an example. - # TODO: Figure out generic decode/encode of this field. Maybe use UnionRule to allow language backends to hook? - data: Any | None + data: BSPData | None @classmethod def from_json_dict(cls, d): @@ -141,7 +137,7 @@ def from_json_dict(cls, d): dependencies=tuple( BuildTargetIdentifier.from_json_dict(x) for x in d.get("dependencies", []) ), - data_kind=d.get("dataKind"), + #data_kind=d.get("dataKind"), # TODO: figure out generic decode, this is only used in tests! data=d.get("data"), ) @@ -157,13 +153,9 @@ def to_json_dict(self): result["displayName"] = self.display_name if self.base_directory is not None: result["baseDirectory"] = self.base_directory - if self.data_kind is not None: - result["dataKind"] = self.data_kind if self.data is not None: - if hasattr(self.data, "to_json_dict"): - result["data"] = self.data.to_json_dict() - else: - result["data"] = self.data + result["dataKind"] = self.data.DATA_KIND + result["data"] = self.data.to_json_dict() return result diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index f9ac7d4333c..232359202f6 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -2,10 +2,12 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations +import functools import logging import os.path +from collections import defaultdict from dataclasses import dataclass -from typing import ClassVar +from typing import ClassVar, TypeVar from pants.backend.scala.bsp.util_rules import ScalaBuildTargetInfo from pants.base.build_root import BuildRoot @@ -13,7 +15,7 @@ from pants.base.specs_parser import SpecsParser from pants.bsp.goal import BSPGoal from pants.bsp.protocol import BSPHandlerMapping -from pants.bsp.spec.base import BuildTarget, BuildTargetCapabilities, BuildTargetIdentifier +from pants.bsp.spec.base import BuildTarget, BuildTargetCapabilities, BuildTargetIdentifier, BSPData from pants.bsp.spec.targets import ( DependencyModule, DependencyModulesItem, @@ -41,35 +43,63 @@ SourcesPaths, SourcesPathsRequest, Targets, - WrappedTarget, + WrappedTarget, Target, ) from pants.engine.unions import UnionMembership, UnionRule, union from pants.jvm.target_types import JvmResolveField from pants.util.frozendict import FrozenDict - +from pants.util.ordered_set import OrderedSet _logger = logging.getLogger(__name__) +_FS = TypeVar("_FS", bound=FieldSet) + @union @dataclass(frozen=True) -class BSPBuildTargetsFieldSet: +class BSPBuildTargetsMetadataRequest: + """Hook to allow language backends to provide metadata for BSP build targets.""" language_id: ClassVar[str] + compatible_language_ids: ClassVar[tuple[str, ...]] + field_set_type: ClassVar[type[_FS]] + field_sets: tuple[_FS, ...] -@dataclass(frozen=True) -class BSPBuildTargetsNew: - targets_mapping: FrozenDict[str, Specs] + +class BSPBuildTargetMetadata(BSPData): + # Merge this `BuildTargetMetadata` with a `BuildTargetMetadata` from same or compatible language backend. + def merge(self, other: BSPBuildTargetMetadata) -> BSPBuildTargetMetadata: + raise NotImplementedError @dataclass(frozen=True) -class BSPBuildTargets: - """Response type for a BSPBuildTargetsRequest.""" +class BSPBuildTargetsMetadataResult: + """Response type for a BSPBuildTargetsMetadataRequest.""" + + # Metadata for the `data` field of the final `BuildTarget`. + metadata: BSPBuildTargetMetadata + + # Build capabilities + can_compile: bool = False + can_test: bool = False + can_run: bool = False + can_debug: bool = False - targets: tuple[BuildTarget, ...] = () + # Output to write into `.pants.d/bsp` for access by IDE. digest: Digest = EMPTY_DIGEST +@dataclass(frozen=True) +class BSPBuildTargetInternal: + name: str + specs: Specs + + +@dataclass(frozen=True) +class BSPBuildTargetsNew: + targets_mapping: FrozenDict[str, BSPBuildTargetInternal] + + @dataclass(frozen=True) class _ParseOneBSPMappingRequest: raw_specs: tuple[str, ...] @@ -106,75 +136,119 @@ class WorkspaceBuildTargetsHandlerMapping(BSPHandlerMapping): response_type = WorkspaceBuildTargetsResult +@dataclass(frozen=True) +class GenerateOneBSPBuildTargetRequest: + bsp_target: BSPBuildTargetInternal + + +@dataclass(frozen=True) +class GenerateOneBSPBuildTargetResult: + build_target: BuildTarget + digest: Digest = EMPTY_DIGEST + + +@rule +async def generate_one_bsp_build_target_request( + request: GenerateOneBSPBuildTargetRequest, + union_membership: UnionMembership, + build_root: BuildRoot, +) -> GenerateOneBSPBuildTargetResult: + # Find all Pants targets that are part of this BSP build target. + targets = await Get(Targets, AddressSpecs, request.bsp_target.specs.address_specs) + + # Classify the targets by the language backends that claim them to provide metadata for them. + field_sets_by_lang_id: dict[str, OrderedSet[FieldSet]] = defaultdict(OrderedSet) + lang_ids_by_field_set: dict[FieldSet, set[str]] = defaultdict(set) + metadata_request_types = union_membership.get(BSPBuildTargetsMetadataRequest) + metadata_request_types_by_lang_id = {metadata_request_type.language_id: metadata_request_type for metadata_request_type in metadata_request_types} + for tgt in targets: + for metadata_request_type in metadata_request_types: + if metadata_request_type.field_set_type.is_applicable(tgt): + field_sets_by_lang_id[metadata_request_type.language_id].add(metadata_request_type.field_set_type.create(tgt)) + lang_ids_by_field_set[tgt].add(metadata_request_type.language_id) + + # Ensure that metadata is being provided only by languages compatible with each other. This guarantees metadata + # can be merged to produce the final metadata for this BSP build target. + for current_lang_id in lang_ids_by_field_set.keys(): + for other_lang_id, other_lang in metadata_request_types_by_lang_id.items(): + if current_lang_id == other_lang_id: + continue + if current_lang_id not in other_lang.compatible_language_ids: + raise ValueError( + f"BSP build target `{request.bsp_target.name}` resolves to an incompatible set of languages. " + f"Language `{current_lang_id}` is not compatible with `{other_lang_id}`." + ) + + # TODO: Provide a way to ensure that other compatibility criteria met. For example, JVM resolve. + + # Request each language backend to provide metadata. + metadata_results = await MultiGet( + Get( + BSPBuildTargetsMetadataResult, + BSPBuildTargetsMetadataRequest, + metadata_request_types_by_lang_id[lang_id]( + field_sets=tuple(field_sets) + ) + ) + for lang_id, field_sets in field_sets_by_lang_id.items() + ) + + # Merge the metadata into a single piece of metadata. + metadata: BSPData = functools.reduce(lambda a, b: a.metadata.merge(b.metadata), metadata_results) + digest = await Get(Digest, MergeDigests([r.digest for r in metadata_results])) + + # Determine base directory for this build target. + # TODO: Use a source root? + targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] + sources_paths = await MultiGet( + Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources + ) + merged_sources_dirs: set[str] = set() + for sp in sources_paths: + merged_sources_dirs.update(sp.dirs) + + base_dir = build_root.pathlib_path + if merged_sources_dirs: + common_path = os.path.commonpath(list(merged_sources_dirs)) + if common_path: + base_dir = base_dir.joinpath(common_path) + + return GenerateOneBSPBuildTargetResult( + build_target=BuildTarget( + id=BuildTargetIdentifier(f"pants:{request.bsp_target.name}"), + display_name=request.bsp_target.name, + base_directory=base_dir.as_uri(), + tags=(), + capabilities=BuildTargetCapabilities( + can_compile=any(r.can_compile for r in metadata_results), + can_test=any(r.can_test for r in metadata_results), + can_run=any(r.can_run for r in metadata_results), + can_debug=any(r.can_debug for r in metadata_results), + ), + language_ids=tuple(sorted(field_sets_by_lang_id.keys())), + dependencies=(), + data=metadata, + ), + digest=digest, + ) + + @_uncacheable_rule async def bsp_workspace_build_targets( _: WorkspaceBuildTargetsParams, bsp_build_targets: BSPBuildTargetsNew, - union_membership: UnionMembership, workspace: Workspace, - build_root: BuildRoot, ) -> WorkspaceBuildTargetsResult: - # metadata_field_set_types = union_membership.get(BSPBuildTargetsFieldSet) - digest: Digest = EMPTY_DIGEST - result: list[BuildTarget] = [] - for bsp_target_name, specs in bsp_build_targets.targets_mapping.items(): - targets = await Get(Targets, AddressSpecs, specs.address_specs) - targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] - # TODO: What about literal specs? - - # TODO: Check for conflicting resolves. - resolve_field = targets[0][JvmResolveField] - scala_info = await Get(ScalaBuildTargetInfo, JvmResolveField, resolve_field) - digest = await Get(Digest, MergeDigests([digest, scala_info.digest])) - # applicable_field_sets: dict[Target, list[Type[BSPBuildTargetsFieldSet]]] = defaultdict(list) - # for tgt in targets: - # for field_set_type in metadata_field_set_types: - # if field_set_type.is_applicable(tgt): - # applicable_field_sets[tgt].append(field_set_type) - - sources_paths = await MultiGet( - Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources - ) - merged_sources_dirs: set[str] = set() - for sp in sources_paths: - merged_sources_dirs.update(sp.dirs) - - base_dir = build_root.pathlib_path - if merged_sources_dirs: - common_path = os.path.commonpath(list(merged_sources_dirs)) - if common_path: - base_dir = base_dir.joinpath(common_path) - - result.append( - BuildTarget( - id=BuildTargetIdentifier(f"pants:{bsp_target_name}"), - display_name=bsp_target_name, - base_directory=base_dir.as_uri(), - tags=(), - capabilities=BuildTargetCapabilities( - can_compile=True, - ), - language_ids=("java", "scala"), - dependencies=(), - data_kind="scala", - data=scala_info.btgt, - ) - ) - # responses = await MultiGet( - # Get(BSPBuildTargets, BSPBuildTargetsFieldSet, request_type()) - # for request_type in request_types - # ) - # for response in responses: - # result.extend(response.targets) - # result.sort(key=lambda btgt: btgt.id.uri) - # output_digest = await Get(Digest, MergeDigests([r.digest for r in responses])) - # workspace.write_digest(output_digest, path_prefix=".pants.d/bsp") - + bsp_target_results = await MultiGet( + Get(GenerateOneBSPBuildTargetResult, GenerateOneBSPBuildTargetRequest(target_internal)) + for target_internal in bsp_build_targets.targets_mapping.values() + ) + digest = await Get(Digest, MergeDigests([r.digest for r in bsp_target_results])) if digest != EMPTY_DIGEST: workspace.write_digest(digest, path_prefix=".pants.d/bsp") return WorkspaceBuildTargetsResult( - targets=tuple(result), + targets=tuple(r.build_target for r in bsp_target_results), ) From d86a7f41b57b0e40cff4f8089dfa52d68eed0b9e Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 16 Mar 2022 16:11:15 -0400 Subject: [PATCH 4/8] checkpoint [ci skip-rust] [ci skip-build-wheels] --- .../backend/experimental/scala/register.py | 2 - src/python/pants/backend/java/bsp/rules.py | 98 +++-------- src/python/pants/backend/scala/bsp/rules.py | 160 +++++++++--------- src/python/pants/backend/scala/bsp/spec.py | 6 +- .../pants/backend/scala/bsp/util_rules.py | 92 ---------- src/python/pants/bsp/spec/base.py | 2 +- src/python/pants/bsp/util_rules/targets.py | 122 ++++++------- 7 files changed, 178 insertions(+), 304 deletions(-) delete mode 100644 src/python/pants/backend/scala/bsp/util_rules.py diff --git a/src/python/pants/backend/experimental/scala/register.py b/src/python/pants/backend/experimental/scala/register.py index cee73a19ed8..c87d8e25011 100644 --- a/src/python/pants/backend/experimental/scala/register.py +++ b/src/python/pants/backend/experimental/scala/register.py @@ -1,6 +1,5 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from pants.backend.scala.bsp import util_rules from pants.backend.scala.bsp.rules import rules as bsp_rules from pants.backend.scala.compile import scalac from pants.backend.scala.dependency_inference import rules as dep_inf_rules @@ -64,6 +63,5 @@ def rules(): *run_deploy_jar.rules(), *scala_lockfile_rules(), *bsp_rules(), - *util_rules.rules(), *war_rules(), ] diff --git a/src/python/pants/backend/java/bsp/rules.py b/src/python/pants/backend/java/bsp/rules.py index f268685d296..923e524d9ed 100644 --- a/src/python/pants/backend/java/bsp/rules.py +++ b/src/python/pants/backend/java/bsp/rules.py @@ -6,43 +6,32 @@ from dataclasses import dataclass from pants.backend.java.bsp.spec import JavacOptionsItem, JavacOptionsParams, JavacOptionsResult -from pants.backend.java.compile.javac import compute_output_jar_filename -from pants.backend.java.dependency_inference.symbol_mapper import AllJavaTargets from pants.backend.java.target_types import JavaSourceField from pants.base.build_root import BuildRoot from pants.base.specs import AddressSpecs -from pants.bsp.context import BSPContext from pants.bsp.protocol import BSPHandlerMapping -from pants.bsp.spec.base import ( - BuildTarget, - BuildTargetCapabilities, - BuildTargetIdentifier, - StatusCode, -) +from pants.bsp.spec.base import BuildTargetIdentifier, StatusCode from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargets, BSPBuildTargetsFieldSet, BSPBuildTargetsNew -from pants.build_graph.address import Address, AddressInput +from pants.bsp.util_rules.targets import ( + BSPBuildTargetsMetadataRequest, + BSPBuildTargetsMetadataResult, + BSPBuildTargetsNew, +) from pants.engine.addresses import Addresses from pants.engine.fs import CreateDigest, DigestEntries from pants.engine.internals.native_engine import EMPTY_DIGEST, AddPrefix, Digest from pants.engine.internals.selectors import Get, MultiGet from pants.engine.rules import collect_rules, rule -from pants.engine.target import ( - CoarsenedTargets, - Dependencies, - DependenciesRequest, - Target, - WrappedTarget, Targets, -) -from pants.engine.unions import UnionMembership, UnionRule -from pants.jvm.bsp.spec import JvmBuildTarget +from pants.engine.target import CoarsenedTargets, FieldSet, Targets +from pants.engine.unions import UnionRule from pants.jvm.compile import ( ClasspathEntryRequest, ClasspathEntryRequestFactory, FallibleClasspathEntry, ) from pants.jvm.resolve.key import CoursierResolveKey +from pants.jvm.target_types import JvmResolveField LANGUAGE_ID = "java" @@ -54,62 +43,27 @@ class JavaBSPLanguageSupport(BSPLanguageSupport): can_compile = True -class JavaBSPBuildTargetsFieldSet(BSPBuildTargetsFieldSet): - pass - - @dataclass(frozen=True) -class ResolveJavaBSPBuildTargetRequest: - target: Target +class JavaMetadataFieldSet(FieldSet): + required_fields = (JavaSourceField, JvmResolveField) + source: JavaSourceField + resolve: JvmResolveField -@rule -async def bsp_resolve_one_java_build_target( - request: ResolveJavaBSPBuildTargetRequest, - union_membership: UnionMembership, -) -> BuildTarget: - dep_addrs = await Get(Addresses, DependenciesRequest(request.target[Dependencies])) - impls = union_membership.get(BSPCompileFieldSet) - - reported_deps = [] - for dep_addr in dep_addrs: - if dep_addr == request.target.address: - continue - - wrapped_dep_tgt = await Get(WrappedTarget, Address, dep_addr) - dep_tgt = wrapped_dep_tgt.target - for impl in impls: - if impl.is_applicable(dep_tgt): - reported_deps.append(BuildTargetIdentifier.from_address(dep_tgt.address)) - break - - return BuildTarget( - id=BuildTargetIdentifier.from_address(request.target.address), - display_name=str(request.target.address), - base_directory=None, - tags=(), - capabilities=BuildTargetCapabilities( - can_compile=True, - ), - language_ids=(LANGUAGE_ID,), - dependencies=tuple(reported_deps), - data_kind="jvm", - data=JvmBuildTarget(), - ) + +class JavaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest): + language_id = LANGUAGE_ID + can_merge_metadata_from = () + field_set_type = JavaMetadataFieldSet @rule -async def bsp_resolve_all_java_build_targets( - _: JavaBSPBuildTargetsFieldSet, - all_java_targets: AllJavaTargets, - bsp_context: BSPContext, -) -> BSPBuildTargets: - if LANGUAGE_ID not in bsp_context.client_params.capabilities.language_ids: - return BSPBuildTargets() - build_targets = await MultiGet( - Get(BuildTarget, ResolveJavaBSPBuildTargetRequest(tgt)) for tgt in all_java_targets +async def bsp_resolve_java_metadata( + _: JavaBSPBuildTargetsMetadataRequest, +) -> BSPBuildTargetsMetadataResult: + return BSPBuildTargetsMetadataResult( + can_compile=True, ) - return BSPBuildTargets(targets=tuple(build_targets)) # ----------------------------------------------------------------------------------------------- @@ -144,7 +98,9 @@ async def handle_bsp_java_options_request( if bsp_target_name not in bsp_build_targets.targets_mapping: raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") targets = await Get( - Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + Targets, + AddressSpecs, + bsp_build_targets.targets_mapping[bsp_target_name].specs.address_specs, ) coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) @@ -225,7 +181,7 @@ def rules(): return ( *collect_rules(), UnionRule(BSPLanguageSupport, JavaBSPLanguageSupport), - UnionRule(BSPBuildTargetsFieldSet, JavaBSPBuildTargetsFieldSet), + UnionRule(BSPBuildTargetsMetadataRequest, JavaBSPBuildTargetsMetadataRequest), UnionRule(BSPHandlerMapping, JavacOptionsHandlerMapping), UnionRule(BSPCompileFieldSet, JavaBSPCompileFieldSet), ) diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 6cf6edde61a..46ff892f878 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -1,11 +1,12 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +from __future__ import annotations + import dataclasses import logging import os from dataclasses import dataclass -from pants.backend.scala.bsp import util_rules from pants.backend.scala.bsp.spec import ( ScalaBuildTarget, ScalacOptionsItem, @@ -13,38 +14,26 @@ ScalacOptionsResult, ScalaPlatform, ) -from pants.backend.scala.bsp.util_rules import MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest -from pants.backend.scala.compile.scalac import compute_output_jar_filename -from pants.backend.scala.dependency_inference.symbol_mapper import AllScalaTargets from pants.backend.scala.subsystems.scala import ScalaSubsystem from pants.backend.scala.target_types import ScalaSourceField from pants.base.build_root import BuildRoot from pants.base.specs import AddressSpecs -from pants.bsp.context import BSPContext from pants.bsp.protocol import BSPHandlerMapping -from pants.bsp.spec.base import ( - BuildTarget, - BuildTargetCapabilities, - BuildTargetIdentifier, - StatusCode, -) +from pants.bsp.spec.base import BuildTarget, BuildTargetIdentifier, StatusCode from pants.bsp.util_rules.compile import BSPCompileFieldSet, BSPCompileResult from pants.bsp.util_rules.lifecycle import BSPLanguageSupport -from pants.bsp.util_rules.targets import BSPBuildTargetsFieldSet, BSPBuildTargetsNew, BSPBuildTargetsMetadataRequest -from pants.build_graph.address import Address, AddressInput +from pants.bsp.util_rules.targets import ( + BSPBuildTargetsMetadataRequest, + BSPBuildTargetsMetadataResult, + BSPBuildTargetsNew, +) from pants.engine.addresses import Addresses from pants.engine.fs import EMPTY_DIGEST, AddPrefix, CreateDigest, Digest, DigestEntries -from pants.engine.internals.native_engine import MergeDigests, Snapshot +from pants.engine.internals.native_engine import Snapshot from pants.engine.internals.selectors import Get, MultiGet from pants.engine.rules import collect_rules, rule -from pants.engine.target import ( - CoarsenedTargets, - Dependencies, - DependenciesRequest, - Target, - WrappedTarget, Targets, -) -from pants.engine.unions import UnionMembership, UnionRule +from pants.engine.target import CoarsenedTargets, FieldSet, Target, Targets +from pants.engine.unions import UnionRule from pants.jvm.compile import ( ClasspathEntryRequest, ClasspathEntryRequestFactory, @@ -66,9 +55,17 @@ class ScalaBSPLanguageSupport(BSPLanguageSupport): can_compile = True +@dataclass(frozen=True) +class ScalaMetadataFieldSet(FieldSet): + required_fields = (ScalaSourceField, JvmResolveField) + + source: ScalaSourceField + resolve: JvmResolveField + + class ScalaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest): language_id = LANGUAGE_ID - compatible_language_ids = ("java",) + can_merge_metadata_from = ("java",) field_set_type = ScalaSourceField @@ -83,77 +80,81 @@ class ResolveScalaBSPBuildTargetResult: scala_runtime: Snapshot +@dataclass(frozen=True) +class MaterializeScalaRuntimeJarsRequest: + scala_version: str + + +@dataclass(frozen=True) +class MaterializeScalaRuntimeJarsResult: + content: Snapshot + + @rule -async def bsp_resolve_one_scala_build_target( - request: ResolveScalaBSPBuildTargetRequest, +async def materialize_scala_runtime_jars( + request: MaterializeScalaRuntimeJarsRequest, +) -> MaterializeScalaRuntimeJarsResult: + tool_classpath = await Get( + ToolClasspath, + ToolClasspathRequest( + artifact_requirements=ArtifactRequirements.from_coordinates( + [ + Coordinate( + group="org.scala-lang", + artifact="scala-compiler", + version=request.scala_version, + ), + Coordinate( + group="org.scala-lang", + artifact="scala-library", + version=request.scala_version, + ), + ] + ), + ), + ) + + materialized_classpath_digest = await Get( + Digest, + AddPrefix(tool_classpath.content.digest, f"jvm/scala-runtime/{request.scala_version}"), + ) + materialized_classpath = await Get(Snapshot, Digest, materialized_classpath_digest) + return MaterializeScalaRuntimeJarsResult(materialized_classpath) + + +@rule +async def bsp_resolve_scala_metadata( + request: ScalaBSPBuildTargetsMetadataRequest, jvm: JvmSubsystem, scala: ScalaSubsystem, - union_membership: UnionMembership, build_root: BuildRoot, -) -> ResolveScalaBSPBuildTargetResult: - resolve = request.target[JvmResolveField].normalized_value(jvm) +) -> BSPBuildTargetsMetadataResult: + resolves = {fs.resolve.normalized_value(jvm) for fs in request.field_sets} + if len(resolves) > 1: + raise ValueError("Cannot provide Scala metadata for multiple resolves.") + resolve = list(resolves)[0] scala_version = scala.version_for_resolve(resolve) - dep_addrs, scala_runtime = await MultiGet( - Get(Addresses, DependenciesRequest(request.target[Dependencies])), - Get(MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest(scala_version)), + scala_runtime = await Get( + MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest(scala_version) ) - impls = union_membership.get(BSPCompileFieldSet) - - reported_deps = [] - for dep_addr in dep_addrs: - if dep_addr == request.target.address: - continue - - wrapped_dep_tgt = await Get(WrappedTarget, Address, dep_addr) - dep_tgt = wrapped_dep_tgt.target - for impl in impls: - if impl.is_applicable(dep_tgt): - reported_deps.append(BuildTargetIdentifier.from_address(dep_tgt.address)) - break scala_jar_uris = tuple( build_root.pathlib_path.joinpath(".pants.d/bsp").joinpath(p).as_uri() for p in scala_runtime.content.files ) - bsp_target = BuildTarget( - id=BuildTargetIdentifier.from_address(request.target.address), - display_name=str(request.target.address), - base_directory=None, - tags=(), - capabilities=BuildTargetCapabilities( - can_compile=True, - ), - language_ids=(LANGUAGE_ID,), - dependencies=tuple(reported_deps), - data_kind="scala", - data=ScalaBuildTarget( - scala_organization="unknown", + + return BSPBuildTargetsMetadataResult( + metadata=ScalaBuildTarget( + scala_organization="org.scala-lang", scala_version=scala_version, scala_binary_version=".".join(scala_version.split(".")[0:2]), platform=ScalaPlatform.JVM, jars=scala_jar_uris, ), + can_compile=True, + digest=scala_runtime.content.digest, ) - return ResolveScalaBSPBuildTargetResult(bsp_target, scala_runtime=scala_runtime.content) - - -# @rule -# async def bsp_resolve_all_scala_build_targets( -# _: ScalaBSPBuildTargetsFieldSet, -# all_scala_targets: AllScalaTargets, -# bsp_context: BSPContext, -# ) -> BSPBuildTargets: -# if LANGUAGE_ID not in bsp_context.client_params.capabilities.language_ids: -# return BSPBuildTargets() -# build_targets = await MultiGet( -# Get(ResolveScalaBSPBuildTargetResult, ResolveScalaBSPBuildTargetRequest(tgt)) -# for tgt in all_scala_targets -# ) -# output_digest = await Get(Digest, MergeDigests([d.scala_runtime.digest for d in build_targets])) -# return BSPBuildTargets( -# targets=tuple(btgt.build_target for btgt in build_targets), digest=output_digest -# ) # ----------------------------------------------------------------------------------------------- @@ -188,7 +189,9 @@ async def handle_bsp_scalac_options_request( if bsp_target_name not in bsp_build_targets.targets_mapping: raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") targets = await Get( - Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + Targets, + AddressSpecs, + bsp_build_targets.targets_mapping[bsp_target_name].specs.address_specs, ) coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) # assert len(coarsened_targets) == 1 @@ -268,9 +271,8 @@ async def bsp_scala_compile_request( def rules(): return ( *collect_rules(), - *util_rules.rules(), UnionRule(BSPLanguageSupport, ScalaBSPLanguageSupport), - # UnionRule(BSPBuildTargetsFieldSet, ScalaBSPBuildTargetsFieldSet), + UnionRule(BSPBuildTargetsMetadataRequest, ScalaBSPBuildTargetsMetadataRequest), UnionRule(BSPHandlerMapping, ScalacOptionsHandlerMapping), UnionRule(BSPCompileFieldSet, ScalaBSPCompileFieldSet), ) diff --git a/src/python/pants/backend/scala/bsp/spec.py b/src/python/pants/backend/scala/bsp/spec.py index 75d82c3d650..977c923fd9d 100644 --- a/src/python/pants/backend/scala/bsp/spec.py +++ b/src/python/pants/backend/scala/bsp/spec.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from typing import Any -from pants.bsp.spec.base import BuildTargetIdentifier, Uri +from pants.bsp.spec.base import BSPData, BuildTargetIdentifier, Uri from pants.jvm.bsp.spec import JvmBuildTarget # ----------------------------------------------------------------------------------------------- @@ -21,7 +21,9 @@ class ScalaPlatform: @dataclass(frozen=True) -class ScalaBuildTarget: +class ScalaBuildTarget(BSPData): + DATA_KIND = "scala" + # The Scala organization that is used for a target. scala_organization: str diff --git a/src/python/pants/backend/scala/bsp/util_rules.py b/src/python/pants/backend/scala/bsp/util_rules.py deleted file mode 100644 index 8ebdee48c0b..00000000000 --- a/src/python/pants/backend/scala/bsp/util_rules.py +++ /dev/null @@ -1,92 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from pants.backend.scala.bsp.spec import ScalaBuildTarget, ScalaPlatform -from pants.backend.scala.subsystems.scala import ScalaSubsystem -from pants.base.build_root import BuildRoot -from pants.engine.internals.native_engine import Digest, AddPrefix, Snapshot -from pants.engine.internals.selectors import Get -from pants.engine.rules import rule, collect_rules -from pants.jvm.resolve.common import ArtifactRequirements, Coordinate -from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest -from pants.jvm.subsystems import JvmSubsystem -from pants.jvm.target_types import JvmResolveField - - -@dataclass(frozen=True) -class MaterializeScalaRuntimeJarsRequest: - scala_version: str - - -@dataclass(frozen=True) -class MaterializeScalaRuntimeJarsResult: - content: Snapshot - - -@rule -async def materialize_scala_runtime_jars( - request: MaterializeScalaRuntimeJarsRequest, -) -> MaterializeScalaRuntimeJarsResult: - tool_classpath = await Get( - ToolClasspath, - ToolClasspathRequest( - artifact_requirements=ArtifactRequirements.from_coordinates( - [ - Coordinate( - group="org.scala-lang", - artifact="scala-compiler", - version=request.scala_version, - ), - Coordinate( - group="org.scala-lang", - artifact="scala-library", - version=request.scala_version, - ), - ] - ), - ), - ) - - materialized_classpath_digest = await Get( - Digest, - AddPrefix(tool_classpath.content.digest, f"jvm/scala-runtime/{request.scala_version}"), - ) - materialized_classpath = await Get(Snapshot, Digest, materialized_classpath_digest) - return MaterializeScalaRuntimeJarsResult(materialized_classpath) - - -@dataclass(frozen=True) -class ScalaBuildTargetInfo: - btgt: ScalaBuildTarget - digest: Digest - - -@rule -async def make_scala_build_target( - resolve_field: JvmResolveField, - jvm: JvmSubsystem, - scala: ScalaSubsystem, - build_root: BuildRoot, -) -> ScalaBuildTargetInfo: - resolve = resolve_field.normalized_value(jvm) - scala_version = scala.version_for_resolve(resolve) - scala_runtime = await Get(MaterializeScalaRuntimeJarsResult, MaterializeScalaRuntimeJarsRequest(scala_version)) - scala_jar_uris = tuple( - build_root.pathlib_path.joinpath(".pants.d/bsp").joinpath(p).as_uri() - for p in scala_runtime.content.files - ) - return ScalaBuildTargetInfo( - btgt=ScalaBuildTarget( - scala_organization="org.scala-lang", - scala_version=scala_version, - scala_binary_version=".".join(scala_version.split(".")[0:2]), - platform=ScalaPlatform.JVM, - jars=scala_jar_uris, - ), - digest=scala_runtime.content.digest, - ) - - -def rules(): - return collect_rules() \ No newline at end of file diff --git a/src/python/pants/bsp/spec/base.py b/src/python/pants/bsp/spec/base.py index 3bc9c267cb1..d9ff6901d7a 100644 --- a/src/python/pants/bsp/spec/base.py +++ b/src/python/pants/bsp/spec/base.py @@ -137,7 +137,7 @@ def from_json_dict(cls, d): dependencies=tuple( BuildTargetIdentifier.from_json_dict(x) for x in d.get("dependencies", []) ), - #data_kind=d.get("dataKind"), # TODO: figure out generic decode, this is only used in tests! + # data_kind=d.get("dataKind"), # TODO: figure out generic decode, this is only used in tests! data=d.get("data"), ) diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index 232359202f6..a6ba2b7d367 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -2,20 +2,18 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from __future__ import annotations -import functools import logging import os.path from collections import defaultdict from dataclasses import dataclass -from typing import ClassVar, TypeVar +from typing import ClassVar, Generic, Sequence, Type, TypeVar -from pants.backend.scala.bsp.util_rules import ScalaBuildTargetInfo from pants.base.build_root import BuildRoot from pants.base.specs import AddressSpecs, Specs from pants.base.specs_parser import SpecsParser from pants.bsp.goal import BSPGoal from pants.bsp.protocol import BSPHandlerMapping -from pants.bsp.spec.base import BuildTarget, BuildTargetCapabilities, BuildTargetIdentifier, BSPData +from pants.bsp.spec.base import BSPData, BuildTarget, BuildTargetCapabilities, BuildTargetIdentifier from pants.bsp.spec.targets import ( DependencyModule, DependencyModulesItem, @@ -43,12 +41,11 @@ SourcesPaths, SourcesPathsRequest, Targets, - WrappedTarget, Target, + WrappedTarget, ) from pants.engine.unions import UnionMembership, UnionRule, union -from pants.jvm.target_types import JvmResolveField from pants.util.frozendict import FrozenDict -from pants.util.ordered_set import OrderedSet +from pants.util.ordered_set import FrozenOrderedSet, OrderedSet _logger = logging.getLogger(__name__) @@ -57,27 +54,22 @@ @union @dataclass(frozen=True) -class BSPBuildTargetsMetadataRequest: +class BSPBuildTargetsMetadataRequest(Generic[_FS]): """Hook to allow language backends to provide metadata for BSP build targets.""" + language_id: ClassVar[str] - compatible_language_ids: ClassVar[tuple[str, ...]] - field_set_type: ClassVar[type[_FS]] + can_merge_metadata_from: ClassVar[tuple[str, ...]] + field_set_type: ClassVar[Type[_FS]] field_sets: tuple[_FS, ...] -class BSPBuildTargetMetadata(BSPData): - # Merge this `BuildTargetMetadata` with a `BuildTargetMetadata` from same or compatible language backend. - def merge(self, other: BSPBuildTargetMetadata) -> BSPBuildTargetMetadata: - raise NotImplementedError - - @dataclass(frozen=True) class BSPBuildTargetsMetadataResult: """Response type for a BSPBuildTargetsMetadataRequest.""" # Metadata for the `data` field of the final `BuildTarget`. - metadata: BSPBuildTargetMetadata + metadata: BSPData | None = None # Build capabilities can_compile: bool = False @@ -115,10 +107,11 @@ async def parse_one_bsp_mapping(request: _ParseOneBSPMappingRequest) -> Specs: @rule async def materialize_bsp_build_targets(bsp_goal: BSPGoal) -> BSPBuildTargetsNew: specs_for_keys = await MultiGet( - Get(Specs, _ParseOneBSPMappingRequest(tuple(value))) for value in bsp_goal.target_mapping.values() + Get(Specs, _ParseOneBSPMappingRequest(tuple(value))) + for value in bsp_goal.target_mapping.values() ) addr_specs = { - key: specs_for_key + key: BSPBuildTargetInternal(key, specs_for_key) for key, specs_for_key in zip(bsp_goal.target_mapping.keys(), specs_for_keys) } return BSPBuildTargetsNew(FrozenDict(addr_specs)) @@ -147,6 +140,28 @@ class GenerateOneBSPBuildTargetResult: digest: Digest = EMPTY_DIGEST +def find_metadata_merge_order( + metadata_request_types: Sequence[Type[BSPBuildTargetsMetadataRequest]], +) -> Sequence[Type[BSPBuildTargetsMetadataRequest]]: + if len(metadata_request_types) <= 1: + return metadata_request_types + + # Naive algorithm (since we only support Java and Scala backends), find the metadata request type that cannot + # merge from another and put that first. + if len(metadata_request_types) != 2: + raise AssertionError( + "BSP core rules only support naive ordering of language-backend metadata. Contact Pants developers." + ) + if not metadata_request_types[0].can_merge_metadata_from: + return metadata_request_types + elif not metadata_request_types[1].can_merge_metadata_from: + return list(reversed(metadata_request_types)) + else: + raise AssertionError( + "BSP core rules only support naive ordering of language-backend metadata. Contact Pants developers." + ) + + @rule async def generate_one_bsp_build_target_request( request: GenerateOneBSPBuildTargetRequest, @@ -158,43 +173,47 @@ async def generate_one_bsp_build_target_request( # Classify the targets by the language backends that claim them to provide metadata for them. field_sets_by_lang_id: dict[str, OrderedSet[FieldSet]] = defaultdict(OrderedSet) - lang_ids_by_field_set: dict[FieldSet, set[str]] = defaultdict(set) - metadata_request_types = union_membership.get(BSPBuildTargetsMetadataRequest) - metadata_request_types_by_lang_id = {metadata_request_type.language_id: metadata_request_type for metadata_request_type in metadata_request_types} + # lang_ids_by_field_set: dict[Type[FieldSet], set[str]] = defaultdict(set) + metadata_request_types: FrozenOrderedSet[ + Type[BSPBuildTargetsMetadataRequest] + ] = union_membership.get(BSPBuildTargetsMetadataRequest) + metadata_request_types_by_lang_id = { + metadata_request_type.language_id: metadata_request_type + for metadata_request_type in metadata_request_types + } for tgt in targets: for metadata_request_type in metadata_request_types: - if metadata_request_type.field_set_type.is_applicable(tgt): - field_sets_by_lang_id[metadata_request_type.language_id].add(metadata_request_type.field_set_type.create(tgt)) - lang_ids_by_field_set[tgt].add(metadata_request_type.language_id) - - # Ensure that metadata is being provided only by languages compatible with each other. This guarantees metadata - # can be merged to produce the final metadata for this BSP build target. - for current_lang_id in lang_ids_by_field_set.keys(): - for other_lang_id, other_lang in metadata_request_types_by_lang_id.items(): - if current_lang_id == other_lang_id: - continue - if current_lang_id not in other_lang.compatible_language_ids: - raise ValueError( - f"BSP build target `{request.bsp_target.name}` resolves to an incompatible set of languages. " - f"Language `{current_lang_id}` is not compatible with `{other_lang_id}`." + field_set_type: Type[FieldSet] = metadata_request_type.field_set_type + if field_set_type.is_applicable(tgt): + field_sets_by_lang_id[metadata_request_type.language_id].add( + field_set_type.create(tgt) ) + # lang_ids_by_field_set[field_set_type].add(metadata_request_type.language_id) - # TODO: Provide a way to ensure that other compatibility criteria met. For example, JVM resolve. + # TODO: Consider how to check whether the provided languages are compatible or whether compatible resolves + # selected. - # Request each language backend to provide metadata. + # Request each language backend to provide metadata for the BuildTarget. metadata_results = await MultiGet( Get( BSPBuildTargetsMetadataResult, BSPBuildTargetsMetadataRequest, - metadata_request_types_by_lang_id[lang_id]( - field_sets=tuple(field_sets) - ) + metadata_request_types_by_lang_id[lang_id](field_sets=tuple(field_sets)), ) for lang_id, field_sets in field_sets_by_lang_id.items() ) + metadata_results_by_lang_id = { + lang_id: metadata_result + for lang_id, metadata_result in zip(field_sets_by_lang_id.keys(), metadata_results) + } - # Merge the metadata into a single piece of metadata. - metadata: BSPData = functools.reduce(lambda a, b: a.metadata.merge(b.metadata), metadata_results) + # Pretend to merge the metadata into a single piece of metadata, but really just choose the metadata + # from the last provider. + metadata_merge_order = find_metadata_merge_order( + [metadata_request_types_by_lang_id[lang_id] for lang_id in field_sets_by_lang_id.keys()] + ) + # TODO: None if no metadata obtained. + metadata = metadata_results_by_lang_id[metadata_merge_order[-1].language_id].metadata digest = await Get(Digest, MergeDigests([r.digest for r in metadata_results])) # Determine base directory for this build target. @@ -284,23 +303,12 @@ async def materialize_bsp_build_target_sources( if bsp_target_name not in bsp_build_targets.targets_mapping: raise ValueError(f"Invalid BSP target name: {request.bsp_target_id}") targets = await Get( - Targets, AddressSpecs, bsp_build_targets.targets_mapping[bsp_target_name].address_specs + Targets, + AddressSpecs, + bsp_build_targets.targets_mapping[bsp_target_name].specs.address_specs, ) targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] - # wrapped_target = await Get(WrappedTarget, AddressInput, request.bsp_target_id.address_input) - # target = wrapped_target.target - # - # if not target.has_field(SourcesField): - # raise AssertionError( - # f"BSP only handles targets with sources: uri={request.bsp_target_id.uri}" - # ) - # - # sources_paths = await Get(SourcesPaths, SourcesPathsRequest(target[SourcesField])) - # sources_full_paths = [ - # build_root.pathlib_path.joinpath(src_path) for src_path in sources_paths.files - # ] - sources_paths = await MultiGet( Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources ) From 526ac101bcd5b13bca2eb3c6c4ca66238198cc17 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 16 Mar 2022 16:15:47 -0400 Subject: [PATCH 5/8] bug fix [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/scala/bsp/rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 46ff892f878..8ca76bc1a0d 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -66,7 +66,7 @@ class ScalaMetadataFieldSet(FieldSet): class ScalaBSPBuildTargetsMetadataRequest(BSPBuildTargetsMetadataRequest): language_id = LANGUAGE_ID can_merge_metadata_from = ("java",) - field_set_type = ScalaSourceField + field_set_type = ScalaMetadataFieldSet @dataclass(frozen=True) From 96fb0d15cdfd54fe3624741f0f976a3528132c67 Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Wed, 16 Mar 2022 21:30:15 -0400 Subject: [PATCH 6/8] checkpoint [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/bsp/util_rules/targets.py | 30 ++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index a6ba2b7d367..7f2cec58220 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -44,6 +44,7 @@ WrappedTarget, ) from pants.engine.unions import UnionMembership, UnionRule, union +from pants.source.source_root import SourceRootsRequest, SourceRootsResult from pants.util.frozendict import FrozenDict from pants.util.ordered_set import FrozenOrderedSet, OrderedSet @@ -222,15 +223,28 @@ async def generate_one_bsp_build_target_request( sources_paths = await MultiGet( Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources ) - merged_sources_dirs: set[str] = set() + _logger.info(f"sources_paths = {sources_paths}") + merged_source_files: set[str] = set() for sp in sources_paths: - merged_sources_dirs.update(sp.dirs) - - base_dir = build_root.pathlib_path - if merged_sources_dirs: - common_path = os.path.commonpath(list(merged_sources_dirs)) - if common_path: - base_dir = base_dir.joinpath(common_path) + merged_source_files.update(sp.files) + + source_roots_result = await Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(merged_source_files)) + source_root_paths = {x.path for x in source_roots_result.path_to_root.values()} + if len(source_root_paths) == 0: + base_dir = build_root.pathlib_path + elif len(source_root_paths) == 1: + base_dir = build_root.pathlib_path.joinpath(list(source_root_paths)[0]) + else: + raise ValueError("Multiple source roots not supported for BSP build target.") + + # base_dir = build_root.pathlib_path + # if merged_sources_dirs: + # _logger.info(f"merged_sources_dirs = {merged_sources_dirs}") + # common_path = os.path.commonpath(list(merged_sources_dirs)) + # _logger.info(f"common_path = {common_path}") + # if common_path: + # base_dir = base_dir.joinpath(common_path) + # _logger.info(f"base_dir = {base_dir}") return GenerateOneBSPBuildTargetResult( build_target=BuildTarget( From 5e6a24412e3dcc32efdd6450b11f79c0021e151f Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 17 Mar 2022 13:20:02 -0400 Subject: [PATCH 7/8] use source root for `roots` in Build Target Sources Request [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/bsp/util_rules/targets.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index 7f2cec58220..208177828d6 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os.path from collections import defaultdict from dataclasses import dataclass from typing import ClassVar, Generic, Sequence, Type, TypeVar @@ -228,7 +227,9 @@ async def generate_one_bsp_build_target_request( for sp in sources_paths: merged_source_files.update(sp.files) - source_roots_result = await Get(SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(merged_source_files)) + source_roots_result = await Get( + SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(merged_source_files) + ) source_root_paths = {x.path for x in source_roots_result.path_to_root.values()} if len(source_root_paths) == 0: base_dir = build_root.pathlib_path @@ -334,11 +335,16 @@ async def materialize_bsp_build_target_sources( _logger.info(f"merged_sources_dirs={merged_sources_dirs}") _logger.info(f"merged_sources_files={merged_sources_files}") - base_dir = build_root.pathlib_path - if merged_sources_dirs: - common_path = os.path.commonpath(list(merged_sources_dirs)) - if common_path: - base_dir = base_dir.joinpath(common_path) + source_roots_result = await Get( + SourceRootsResult, SourceRootsRequest, SourceRootsRequest.for_files(merged_sources_files) + ) + source_root_paths = {x.path for x in source_roots_result.path_to_root.values()} + if len(source_root_paths) == 0: + base_dir = build_root.pathlib_path + elif len(source_root_paths) == 1: + base_dir = build_root.pathlib_path.joinpath(list(source_root_paths)[0]) + else: + raise ValueError("Multiple source roots not supported for BSP build target.") sources_item = SourcesItem( target=request.bsp_target_id, From cbcd62036fefe3f2d9d5163c0f5b0625661f5c7a Mon Sep 17 00:00:00 2001 From: Tom Dyas Date: Thu, 17 Mar 2022 19:41:05 -0400 Subject: [PATCH 8/8] remove commented-out code [ci skip-rust] [ci skip-build-wheels] --- src/python/pants/backend/java/bsp/rules.py | 8 -------- src/python/pants/backend/scala/bsp/rules.py | 8 -------- src/python/pants/bsp/util_rules/targets.py | 12 +----------- 3 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/python/pants/backend/java/bsp/rules.py b/src/python/pants/backend/java/bsp/rules.py index 923e524d9ed..3c6f648cb1f 100644 --- a/src/python/pants/backend/java/bsp/rules.py +++ b/src/python/pants/backend/java/bsp/rules.py @@ -104,20 +104,12 @@ async def handle_bsp_java_options_request( ) coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) - # assert len(coarsened_targets) == 1 - # coarsened_target = coarsened_targets[0] resolve = await Get(CoursierResolveKey, CoarsenedTargets, coarsened_targets) - # output_file = compute_output_jar_filename(coarsened_target) return HandleJavacOptionsResult( JavacOptionsItem( target=request.bsp_target_id, options=(), - # classpath=( - # build_root.pathlib_path.joinpath( - # f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" - # ).as_uri(), - # ), classpath=(), class_directory=build_root.pathlib_path.joinpath( f".pants.d/bsp/jvm/resolves/{resolve.name}/classes" diff --git a/src/python/pants/backend/scala/bsp/rules.py b/src/python/pants/backend/scala/bsp/rules.py index 8ca76bc1a0d..acb85cce04d 100644 --- a/src/python/pants/backend/scala/bsp/rules.py +++ b/src/python/pants/backend/scala/bsp/rules.py @@ -194,20 +194,12 @@ async def handle_bsp_scalac_options_request( bsp_build_targets.targets_mapping[bsp_target_name].specs.address_specs, ) coarsened_targets = await Get(CoarsenedTargets, Addresses(tgt.address for tgt in targets)) - # assert len(coarsened_targets) == 1 - # coarsened_target = coarsened_targets[0] resolve = await Get(CoursierResolveKey, CoarsenedTargets, coarsened_targets) - # output_file = compute_output_jar_filename(coarsened_target) return HandleScalacOptionsResult( ScalacOptionsItem( target=request.bsp_target_id, options=(), - # classpath=( - # build_root.pathlib_path.joinpath( - # f".pants.d/bsp/jvm/resolves/{resolve.name}/lib/{output_file}" - # ).as_uri(), - # ), classpath=(), class_directory=build_root.pathlib_path.joinpath( f".pants.d/bsp/jvm/resolves/{resolve.name}/classes" diff --git a/src/python/pants/bsp/util_rules/targets.py b/src/python/pants/bsp/util_rules/targets.py index 208177828d6..a0aa578afa1 100644 --- a/src/python/pants/bsp/util_rules/targets.py +++ b/src/python/pants/bsp/util_rules/targets.py @@ -216,8 +216,7 @@ async def generate_one_bsp_build_target_request( metadata = metadata_results_by_lang_id[metadata_merge_order[-1].language_id].metadata digest = await Get(Digest, MergeDigests([r.digest for r in metadata_results])) - # Determine base directory for this build target. - # TODO: Use a source root? + # Determine base directory for this build target using source roots. targets_with_sources = [tgt for tgt in targets if tgt.has_field(SourcesField)] sources_paths = await MultiGet( Get(SourcesPaths, SourcesPathsRequest(tgt[SourcesField])) for tgt in targets_with_sources @@ -238,15 +237,6 @@ async def generate_one_bsp_build_target_request( else: raise ValueError("Multiple source roots not supported for BSP build target.") - # base_dir = build_root.pathlib_path - # if merged_sources_dirs: - # _logger.info(f"merged_sources_dirs = {merged_sources_dirs}") - # common_path = os.path.commonpath(list(merged_sources_dirs)) - # _logger.info(f"common_path = {common_path}") - # if common_path: - # base_dir = base_dir.joinpath(common_path) - # _logger.info(f"base_dir = {base_dir}") - return GenerateOneBSPBuildTargetResult( build_target=BuildTarget( id=BuildTargetIdentifier(f"pants:{request.bsp_target.name}"),