allow plugins to provide "auxiliary" goals (and refactor BSP into backends)

docs/notes/2.23.x.md

@@ -16,6 +16,8 @@ We offer [formal sponsorship tiers for companies](https://www.pantsbuild.org/spo
 
 The deprecations for the `--changed-dependees` option and the `dependees` goal have expired. Use the equivalent [`--changed-dependents` option](https://www.pantsbuild.org/2.23/reference/subsystems/changed#dependents) or [`dependents` goal](https://www.pantsbuild.org/2.23/reference/goals/dependents) instead.
 
+BSP support has been moved out of core rules into separate core and language-specific backends.
+
 ### Test goal
 
 A new option `--experimental-report-test-result-info` is added to the `[test]` config section. Enabling this option will
@@ -69,6 +71,15 @@ The deprecation for `crossversion="partial"` on `scala_artifact` has expired. Us
 
 The Scala dependency inference now understand usages of the `_root_` package name as a marker for disambiguating between colliding dependencies and will try to resolve those symbols as absolute. For instance, `import _root_.io.circe.syntax` will now be understood as an import of `io.circie.syntax`.
 
+##### BSP (Build Server Protocol)
+
+The BSP (Build Server Protocol) support has been moved out of the Pants core into several new backends to faciliate disabling this support if it is not needed. The new backends are:
+
+- `pants.backend.experimental.bsp` (core)
+- `pants.backend.experimental.java.bsp` (Java support)
+- `pants.backend.experimental.scala.bsp` (Scala support)
+
+Enable the core `pants.backend.experimental.bsp` backend and one or more of the language-specific backends to enable BSP support.
 Scala dependency inference now also understands types refered to only by pattern matching cases such as `case MyType() =>`. These used to require manually adding a dependency if the type was defined in a separate file even if it was in the same package. This is now inferred.
 
 #### NEW: Trufflehog
@@ -197,6 +208,8 @@ Fixed bug where files larger than 512KB were being materialized to a process's s
 
 Fixed bug where using `RuleRunner` in plugin tests caused `ValueError` that complains about not finding build root sentinel files. Note that you may have to adjust your tests to account for the new `BUILDROOT` file that `RuleRunner` now injects in the sandbox it creates for each test. For example, a test that uses a `**` glob might have to add `!BUILDROOT` to exclude the `BUILDROOT` file, or otherwise account for its presence when inspecting a sandbox or its digest.
 
+Plugins may now provide "auxillary" goals by implememting the `auxillary_goals` function in their plugin registration module and returning one or more subclasses of `pants.goal.auxillary_goal.AuxillaryGoal`. The BSP (Build Server Protocol) support now uses this mechanism to move the `experimental-bsp` goal out of the Pants core rules.
+
 ### Other minor tweaks
 
 The "Provided by" information in the documentation now correctly reflects the proper backend to enable to activate a certain feature.

src/python/pants/backend/experimental/bsp/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/bsp/register.py

@@ -0,0 +1,13 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.bsp.goal import BSPGoal
+from pants.bsp.rules import rules as bsp_rules
+
+
+def auxillary_goals():
+    return (BSPGoal,)
+
+
+def rules():
+    return (*bsp_rules(),)

src/python/pants/backend/experimental/java/bsp/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/java/bsp/register.py

@@ -0,0 +1,29 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.backend.experimental.bsp.register import auxillary_goals as bsp_auxillary_goals
+from pants.backend.experimental.bsp.register import rules as bsp_rules
+from pants.backend.experimental.java.register import build_file_aliases as java_build_file_aliases
+from pants.backend.experimental.java.register import rules as java_rules
+from pants.backend.experimental.java.register import target_types as java_target_types
+from pants.backend.java.bsp.rules import rules as java_bsp_rules
+
+
+def auxillary_goals():
+    return bsp_auxillary_goals()
+
+
+def target_types():
+    return java_target_types()
+
+
+def rules():
+    return (
+        *java_rules(),
+        *bsp_rules(),
+        *java_bsp_rules(),
+    )
+
+
+def build_file_aliases():
+    return java_build_file_aliases()

src/python/pants/backend/experimental/java/register.py

@@ -1,7 +1,6 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-from pants.backend.java.bsp import rules as java_bsp_rules
 from pants.backend.java.compile import javac
 from pants.backend.java.dependency_inference import java_parser
 from pants.backend.java.dependency_inference import rules as dependency_inference_rules
@@ -39,7 +38,6 @@ def rules():
         *java_parser.rules(),
         *dependency_inference_rules.rules(),
         *tailor.rules(),
-        *java_bsp_rules.rules(),
         *archive.rules(),
         *target_types_rules(),
         *jvm_common.rules(),

src/python/pants/backend/experimental/scala/bsp/BUILD

@@ -0,0 +1,4 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_sources()

src/python/pants/backend/experimental/scala/bsp/register.py

@@ -0,0 +1,28 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+from pants.backend.experimental.bsp.register import auxillary_goals as bsp_auxillary_goals
+from pants.backend.experimental.bsp.register import rules as bsp_rules
+from pants.backend.experimental.scala.register import build_file_aliases as scala_build_file_aliases
+from pants.backend.experimental.scala.register import rules as scala_rules
+from pants.backend.experimental.scala.register import target_types as scala_target_types
+from pants.backend.scala.bsp.rules import rules as scala_bsp_rules
+
+
+def auxillary_goals():
+    return bsp_auxillary_goals()
+
+
+def target_types():
+    return scala_target_types()
+
+
+def rules():
+    return (
+        *scala_rules(),
+        *bsp_rules(),
+        *scala_bsp_rules(),
+    )
+
+
+def build_file_aliases():
+    return scala_build_file_aliases()

src/python/pants/backend/experimental/scala/register.py

@@ -1,6 +1,5 @@
 # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-from pants.backend.scala.bsp.rules import rules as bsp_rules
 from pants.backend.scala.compile import scalac
 from pants.backend.scala.dependency_inference import rules as dep_inf_rules
 from pants.backend.scala.goals import check, repl, tailor
@@ -50,7 +49,6 @@ def rules():
         *dep_inf_rules.rules(),
         *target_types_rules(),
         *scala_lockfile_rules(),
-        *bsp_rules(),
         *jvm_common.rules(),
         *wrap_scala.rules,
     ]

src/python/pants/bsp/goal.py

@@ -12,27 +12,24 @@
 
 from pants.base.build_root import BuildRoot
 from pants.base.exiter import PANTS_FAILED_EXIT_CODE, PANTS_SUCCEEDED_EXIT_CODE, ExitCode
-from pants.base.specs import Specs
 from pants.bsp.context import BSPContext
 from pants.bsp.protocol import BSPConnection
 from pants.bsp.util_rules.lifecycle import BSP_VERSION, BSPLanguageSupport
-from pants.build_graph.build_configuration import BuildConfiguration
 from pants.engine.env_vars import CompleteEnvironmentVars
 from pants.engine.internals.session import SessionValues
 from pants.engine.unions import UnionMembership
-from pants.goal.builtin_goal import BuiltinGoal
+from pants.goal.auxillary_goal import AuxillaryGoal, AuxillaryGoalContext
 from pants.init.engine_initializer import GraphSession
 from pants.option.option_types import BoolOption, FileListOption, StrListOption
 from pants.option.option_value_container import OptionValueContainer
-from pants.option.options import Options
 from pants.util.docutil import bin_name
 from pants.util.strutil import softwrap
 from pants.version import VERSION
 
 _logger = logging.getLogger(__name__)
 
 
-class BSPGoal(BuiltinGoal):
+class BSPGoal(AuxillaryGoal):
     name = "experimental-bsp"
     help = "Setup repository for Build Server Protocol (https://build-server-protocol.github.io/)."
 
@@ -102,23 +99,18 @@ class BSPGoal(BuiltinGoal):
 
     def run(
         self,
-        *,
-        build_config: BuildConfiguration,
-        graph_session: GraphSession,
-        options: Options,
-        specs: Specs,
-        union_membership: UnionMembership,
+        context: AuxillaryGoalContext,
     ) -> ExitCode:
-        goal_options = options.for_scope(self.name)
+        goal_options = context.options.for_scope(self.name)
         if goal_options.server:
             return self._run_server(
-                graph_session=graph_session,
-                union_membership=union_membership,
+                graph_session=context.graph_session,
+                union_membership=context.union_membership,
             )
-        current_session_values = graph_session.scheduler_session.py_session.session_values
+        current_session_values = context.graph_session.scheduler_session.py_session.session_values
         env = current_session_values[CompleteEnvironmentVars]
         return self._setup_bsp_connection(
-            union_membership=union_membership, env=env, options=goal_options
+            union_membership=context.union_membership, env=env, options=goal_options
         )
 
     def _setup_bsp_connection(

src/python/pants/build_graph/build_configuration.py

@@ -269,6 +269,24 @@ def register_target_types(
         def register_remote_auth_plugin(self, remote_auth_plugin: Callable) -> None:
             self._remote_auth_plugin = remote_auth_plugin
 
+        def register_auxillary_goals(self, plugin_or_backend: str, auxillary_goals: Iterable[type]):
+            """Registers the given auxillary goals."""
+            if not isinstance(auxillary_goals, Iterable):
+                raise TypeError(
+                    f"The entrypoint `auxillary_goals` must return an iterable. "
+                    f"Given {repr(auxillary_goals)}"
+                )
+            # Import `AuxillaryGoal` here to avoid import cycle.
+            from pants.goal.auxillary_goal import AuxillaryGoal
+
+            bad_elements = [goal for goal in auxillary_goals if not issubclass(goal, AuxillaryGoal)]
+            if bad_elements:
+                raise TypeError(
+                    "Every element of the entrypoint `auxillary_goals` must be a subclass of "
+                    f"{AuxillaryGoal.__name__}. Bad elements: {bad_elements}."
+                )
+            self.register_subsystems(plugin_or_backend, auxillary_goals)
+
         def allow_unknown_options(self, allow: bool = True) -> None:
             """Allows overriding whether Options parsing will fail for unrecognized Options.
 

src/python/pants/core/register.py

@@ -6,7 +6,6 @@
 These are always activated and cannot be disabled.
 """
 from pants.backend.codegen import export_codegen_goal
-from pants.bsp.rules import rules as bsp_rules
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.core.goals import (
     check,
@@ -86,7 +85,6 @@ def rules():
         *run.rules(),
         *tailor.rules(),
         *test.rules(),
-        *bsp_rules(),
         # util_rules
         *adhoc_binaries.rules(),
         *anonymous_telemetry.rules(),

src/python/pants/goal/auxillary_goal.py

@@ -0,0 +1,58 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import ClassVar
+
+from pants.base.exiter import ExitCode
+from pants.base.specs import Specs
+from pants.build_graph.build_configuration import BuildConfiguration
+from pants.engine.goal import GoalSubsystem
+from pants.engine.unions import UnionMembership
+from pants.init.engine_initializer import GraphSession
+from pants.option.options import Options
+from pants.option.scope import ScopeInfo
+
+
+@dataclass
+class AuxillaryGoalContext:
+    """Context passed to a `AuxillaryGoal.run` implementation."""
+
+    build_config: BuildConfiguration
+    graph_session: GraphSession
+    options: Options
+    specs: Specs
+    union_membership: UnionMembership
+
+
+class AuxillaryGoal(ABC, GoalSubsystem):
+    """Configure a "auxillary" goal which allows rules to "take over" Pants client execution in lieu
+    of executing an ordnary goal.
+
+    Only a single auxillary goal is executed per run, any remaining goals/arguments are passed
+    unaltered to the auxillary goal. Auxillary goals have precedence over regular goals.
+
+    When multiple auxillary goals are presented, the first auxillary goal will be used unless there is a
+    auxillary goal that begin with a hyphen (`-`), in which case the last such "option goal" will be
+    prioritized. This is to support things like `./pants some-builtin-goal --help`.
+
+    The intended use for this API is rule code which runs a server (for example, a BSP server)
+    which provides an alternate interface to the Pants rule engine, or other kinds of goals
+    which must run "outside" of the usual engine processing to function.
+    """
+
+    # Used by `pants.option.arg_splitter.ArgSplitter()` to optionally allow aliasing daemon goals.
+    aliases: ClassVar[tuple[str, ...]] = ()
+
+    @classmethod
+    def create_scope_info(cls, **scope_info_kwargs) -> ScopeInfo:
+        return super().create_scope_info(is_auxillary=True, **scope_info_kwargs)
+
+    @abstractmethod
+    def run(
+        self,
+        context: AuxillaryGoalContext,
+    ) -> ExitCode:
+        pass

src/python/pants/goal/builtins.py

@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-from pants.bsp.goal import BSPGoal
 from pants.build_graph.build_configuration import BuildConfiguration
 from pants.goal import help
 from pants.goal.builtin_goal import BuiltinGoal
@@ -18,7 +17,6 @@ def register_builtin_goals(build_configuration: BuildConfiguration.Builder) -> N
 
 def builtin_goals() -> tuple[type[BuiltinGoal], ...]:
     return (
-        BSPGoal,
         CompletionBuiltinGoal,
         ExplorerBuiltinGoal,
         MigrateCallByNameBuiltinGoal,

src/python/pants/init/extension_loader.py

@@ -107,6 +107,9 @@ def load_plugins(
                 f"register remote auth function {remote_auth_func.__module__}.{remote_auth_func.__name__} from plugin: {plugin}"
             )
             build_configuration.register_remote_auth_plugin(remote_auth_func)
+        if "auxillary_goals" in entries:
+            auxillary_goals = entries["auxillary_goals"].load()()
+            build_configuration.register_auxillary_goals(req.key, auxillary_goals)
 
         loaded[dist.as_requirement().key] = dist
 
@@ -168,3 +171,6 @@ def invoke_entrypoint(name: str):
             f"register remote auth function {remote_auth_func.__module__}.{remote_auth_func.__name__} from backend: {backend_package}"
         )
         build_configuration.register_remote_auth_plugin(remote_auth_func)
+    auxillary_goals = invoke_entrypoint("auxillary_goals")
+    if auxillary_goals:
+        build_configuration.register_auxillary_goals(backend_package, auxillary_goals)