Add --debug-adapter flag to run (#15829)

run's counterpart to #15799. Support is added for pex_binary (which will be spiritually lifted to the python_source with #15849)

[ci skip-rust]
[ci skip-build-wheels]

docs/markdown/Python/python-goals/python-run-goal.md

@@ -29,11 +29,11 @@ You may only run one target at a time.
 The program will have access to the same environment used by the parent `./pants` process, so you can set environment variables in the external environment, e.g. `FOO=bar ./pants run project/app.py`. (Pants will auto-set some values like `$PATH`).
 
 > 📘 Tip: check the return code
-> 
+>
 > Pants will propagate the return code from the underlying executable. Run `echo $?` after the Pants run to see the return code.
 
 > 🚧 Issues finding files?
-> 
+>
 > Run `./pants dependencies --transitive path/to/binary.py` to ensure that all the files you need are showing up, including for any [assets](doc:assets) you intend to use.
 
 Watching the filesystem
@@ -46,29 +46,36 @@ On the other hand, if your app is short lived (like a script) and you'd like to
 Debugging
 ---------
 
+> 📘 Tip: using the VS Code (or any [DAP](https://microsoft.github.io/debug-adapter-protocol/)-compliant editor) remote debugger
+>
+>
+> 1. In your editor, set your breakpoints and any other debug settings (like break-on-exception).
+> 2. Run your code with `./pants run --debug-adapter`.
+> 3. Connect your editor to the server. The server host and port are logged by Pants when executing `run --debug-adaptor`. (They can also be configured using the `[debug-adapter]` subsystem).
+
 > 📘 Tip: Using the IntelliJ/PyCharm remote debugger
-> 
+>
 > First, add the following target in some BUILD file (e.g., the one containing your other 3rd-party dependencies):
-> 
+>
 > ```
 > python_requirement(
 >   name = "pydevd-pycharm",
 >   requirements=["pydevd-pycharm==203.5419.8"],  # Or whatever version you choose.
 > )
 > ```
-> 
+>
 > You can check this into your repo, for convenience.
-> 
+>
 > Now, use the remote debugger as usual:
-> 
+>
 > 1. Start a Python remote debugging session in PyCharm, say on port 5000.
 > 2. Add the following code at the point where you want execution to pause and connect to the debugger:
-> 
+>
 > ```
 > import pydevd_pycharm
 > pydevd_pycharm.settrace('localhost', port=5000, stdoutToServer=True, stderrToServer=True)
 > ```
-> 
-> Run your executable with `./pants run` as usual. 
-> 
+>
+> Run your executable with `./pants run` as usual.
+>
 > Note: The first time you do so you may see some extra dependency resolution work, as `pydevd-pycharm` has now been added to the binary's dependencies, via inference. If you have dependency inference turned off in your repo, you will have to manually add a temporary explicit dependency in your binary target on the `pydevd-pycharm` target.

docs/markdown/Python/python-goals/python-test-goal.md

@@ -229,13 +229,12 @@ If you use multiple files with `test --debug`, they will run sequentially rather
 > ❯ ./pants test --debug  <target>   -- -s
 > ```
 
-> 📘 Tip: using the VS Code (or any debug adatper-compliant editor) remote debugger in tests
+> 📘 Tip: using the VS Code (or any [DAP](https://microsoft.github.io/debug-adapter-protocol/)-compliant editor) remote debugger in tests
 >
 >
-> 1. Configure your editor's breakpoints and exception settings
-> 2. Run your test with `./pants test --debug-adapter`
-> 3. Connect your editor to the server (the server host and port are logged, and can be configured
->    using the `[debug-adapter]` subsystem).
+> 1. In your editor, set your breakpoints and any other debug settings (like break-on-exception).
+> 2. Run your test with `./pants test --debug-adapter`.
+> 3. Connect your editor to the server. The server host and port are logged by Pants when executing `test --debug-adaptor`. (They can also be configured using the `[debug-adapter]` subsystem).
 
 > Run your test with `./pants test --debug` as usual.
 

src/python/pants/backend/docker/goals/run_image.py

@@ -9,7 +9,7 @@
 from pants.backend.docker.subsystems.docker_options import DockerOptions
 from pants.backend.docker.util_rules.docker_binary import DockerBinary
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
-from pants.core.goals.run import RunRequest
+from pants.core.goals.run import RunDebugAdapterRequest, RunRequest
 from pants.engine.environment import Environment, EnvironmentRequest
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 
@@ -28,5 +28,14 @@ async def docker_image_run_request(
     return RunRequest(args=run.argv, digest=image.digest, extra_env=run.env)
 
 
+@rule
+async def docker_image_run_debug_adapter_request(
+    field_set: DockerFieldSet,
+) -> RunDebugAdapterRequest:
+    raise NotImplementedError(
+        "Debugging a Docker image using a debug adapter has not yet been implemented."
+    )
+
+
 def rules():
     return collect_rules()

src/python/pants/backend/go/goals/run_binary.py

@@ -5,7 +5,7 @@
 
 from pants.backend.go.goals.package_binary import GoBinaryFieldSet
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest
 from pants.engine.internals.selectors import Get
 from pants.engine.rules import collect_rules, rule
 from pants.engine.unions import UnionRule
@@ -19,5 +19,14 @@ async def create_go_binary_run_request(field_set: GoBinaryFieldSet) -> RunReques
     return RunRequest(digest=binary.digest, args=(os.path.join("{chroot}", artifact_relpath),))
 
 
+@rule
+async def go_binary_run_debug_adapter_request(
+    field_set: GoBinaryFieldSet,
+) -> RunDebugAdapterRequest:
+    raise NotImplementedError(
+        "Debugging a Go binary using a debug adapter has not yet been implemented."
+    )
+
+
 def rules():
     return [*collect_rules(), UnionRule(RunFieldSet, GoBinaryFieldSet)]

src/python/pants/backend/python/goals/run_pex_binary.py

@@ -1,17 +1,19 @@
 # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+import logging
 import os
 
 from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet
+from pants.backend.python.subsystems.debugpy import DebugPy
 from pants.backend.python.target_types import (
     PexBinaryDefaults,
     ResolvedPexEntryPoint,
     ResolvePexEntryPointRequest,
 )
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest
-from pants.backend.python.util_rules.pex import Pex
+from pants.backend.python.util_rules.pex import Pex, PexRequest
 from pants.backend.python.util_rules.pex_environment import PexEnvironment
 from pants.backend.python.util_rules.pex_from_targets import (
     InterpreterConstraintsRequest,
@@ -21,12 +23,20 @@
     PythonSourceFiles,
     PythonSourceFilesRequest,
 )
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest
+from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem
 from pants.engine.fs import Digest, MergeDigests
 from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest
 from pants.engine.unions import UnionRule
 from pants.util.logging import LogLevel
+from pants.util.strutil import softwrap
+
+logger = logging.getLogger(__name__)
+
+
+def _in_chroot(relpath: str) -> str:
+    return os.path.join("{chroot}", relpath)
 
 
 @rule(level=LogLevel.DEBUG)
@@ -99,13 +109,12 @@ async def create_pex_binary_run_request(
     ]
     merged_digest = await Get(Digest, MergeDigests(input_digests))
 
-    def in_chroot(relpath: str) -> str:
-        return os.path.join("{chroot}", relpath)
-
     complete_pex_env = pex_env.in_workspace()
-    args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python)
+    # NB. If this changes, please consider how it affects the `DebugRequest` below
+    # (which is not easy to write automated tests for)
+    args = complete_pex_env.create_argv(_in_chroot(pex.name), python=pex.python)
 
-    chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots]
+    chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots]
     # The order here is important: we want the in-repo sources to take precedence over their
     # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode).
     source_roots = [
@@ -114,12 +123,62 @@ def in_chroot(relpath: str) -> str:
     ]
     extra_env = {
         **complete_pex_env.environment_dict(python_configured=pex.python is not None),
-        "PEX_PATH": in_chroot(local_dists.pex.name),
+        "PEX_PATH": _in_chroot(local_dists.pex.name),
         "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots),
     }
 
     return RunRequest(digest=merged_digest, args=args, extra_env=extra_env)
 
 
+@rule
+async def run_pex_debug_adapter_binary(
+    field_set: PexBinaryFieldSet,
+    debugpy: DebugPy,
+    debug_adapter: DebugAdapterSubsystem,
+) -> RunDebugAdapterRequest:
+    if field_set.run_in_sandbox:
+        logger.warning(
+            softwrap(
+                f"""
+                Using `run --debug-adapter` on the target {field_set.address}, which sets the field
+                `run_in_sandbox` to `True`. This will likely cause your breakpoints to not be hit,
+                as your code will be run under the sandbox's path.
+                """
+            )
+        )
+
+    entry_point, regular_run_request, debugpy_pex = await MultiGet(
+        Get(
+            ResolvedPexEntryPoint,
+            ResolvePexEntryPointRequest(field_set.entry_point),
+        ),
+        Get(RunRequest, PexBinaryFieldSet, field_set),
+        Get(Pex, PexRequest, debugpy.to_pex_request()),
+    )
+
+    entry_point_or_script = entry_point.val or field_set.script.value
+    assert entry_point_or_script is not None
+    merged_digest = await Get(
+        Digest, MergeDigests([regular_run_request.digest, debugpy_pex.digest])
+    )
+    extra_env = dict(regular_run_request.extra_env)
+    extra_env["PEX_PATH"] = os.pathsep.join(
+        [
+            extra_env["PEX_PATH"],
+            # For debugpy to work properly, we need to have just one "environment" for our
+            # command to run in. Therefore, we cobble one together by exeucting debugpy's PEX, and
+            # shoehorning in the original PEX through PEX_PATH.
+            _in_chroot(os.path.basename(regular_run_request.args[1])),
+        ]
+    )
+    args = [
+        regular_run_request.args[0],  # python executable
+        _in_chroot(debugpy_pex.name),
+        *debugpy.get_args(debug_adapter, entry_point_or_script),
+    ]
+
+    return RunDebugAdapterRequest(digest=merged_digest, args=args, extra_env=extra_env)
+
+
 def rules():
     return [*collect_rules(), UnionRule(RunFieldSet, PexBinaryFieldSet)]

src/python/pants/backend/python/packaging/pyoxidizer/rules.py

@@ -23,7 +23,7 @@
 from pants.backend.python.target_types import GenerateSetupField, WheelField
 from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest
 from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest
 from pants.core.util_rules.system_binaries import BashBinary
 from pants.engine.fs import (
     AddPrefix,
@@ -232,6 +232,15 @@ def is_executable_binary(artifact_relpath: str | None) -> bool:
     return RunRequest(digest=binary.digest, args=(os.path.join("{chroot}", artifact.relpath),))
 
 
+@rule
+async def run_pyoxidizer_debug_adapter_binary(
+    field_set: PyOxidizerFieldSet,
+) -> RunDebugAdapterRequest:
+    raise NotImplementedError(
+        "Debugging a PyOxidizer binary using a debug adapter has not yet been implemented."
+    )
+
+
 def rules():
     return (
         *collect_rules(),

src/python/pants/backend/shell/shell_command.py

@@ -23,7 +23,7 @@
     ShellCommandToolsField,
 )
 from pants.core.goals.package import BuiltPackage, PackageFieldSet
-from pants.core.goals.run import RunFieldSet, RunRequest
+from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest
 from pants.core.target_types import FileSourceField
 from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest
 from pants.core.util_rules.system_binaries import (
@@ -248,6 +248,15 @@ async def run_shell_command_request(shell_command: RunShellCommand) -> RunReques
     )
 
 
+@rule
+async def run_shell_debug_adapter_binary(
+    field_set: RunShellCommand,
+) -> RunDebugAdapterRequest:
+    raise NotImplementedError(
+        "Debugging a shell command using a debug adapter has not yet been implemented."
+    )
+
+
 def rules():
     return [
         *collect_rules(),

src/python/pants/core/goals/run.py

@@ -7,6 +7,7 @@
 from typing import Iterable, Mapping, Optional, Tuple
 
 from pants.base.build_root import BuildRoot
+from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem
 from pants.engine.environment import CompleteEnvironment
 from pants.engine.fs import Digest, Workspace
 from pants.engine.goal import Goal, GoalSubsystem
@@ -69,6 +70,13 @@ def __init__(
         self.extra_env = FrozenDict(extra_env or {})
 
 
+class RunDebugAdapterRequest(RunRequest):
+    """Like RunRequest, but launches the process using the relevant Debug Adapter server.
+
+    The process should be launched waiting for the client to connect.
+    """
+
+
 class RunSubsystem(GoalSubsystem):
     name = "run"
     help = softwrap(
@@ -107,6 +115,20 @@ def activated(cls, union_membership: UnionMembership) -> bool:
             """
         ),
     )
+    # See also `test.py`'s same option
+    debug_adapter = BoolOption(
+        "--debug-adapter",
+        default=False,
+        help=softwrap(
+            """
+            Run the interactive process using a Debug Adapter
+            (https://microsoft.github.io/debug-adapter-protocol/) for the language if supported.
+
+            The interactive process used will be immediately blocked waiting for a client before
+            continuing.
+            """
+        ),
+    )
 
 
 class Run(Goal):
@@ -116,6 +138,7 @@ class Run(Goal):
 @goal_rule
 async def run(
     run_subsystem: RunSubsystem,
+    debug_adapter: DebugAdapterSubsystem,
     global_options: GlobalOptions,
     workspace: Workspace,
     build_root: BuildRoot,
@@ -131,7 +154,11 @@ async def run(
         ),
     )
     field_set = targets_to_valid_field_sets.field_sets[0]
-    request = await Get(RunRequest, RunFieldSet, field_set)
+    request = await (
+        Get(RunRequest, RunFieldSet, field_set)
+        if not run_subsystem.debug_adapter
+        else Get(RunDebugAdapterRequest, RunFieldSet, field_set)
+    )
     wrapped_target = await Get(
         WrappedTarget, WrappedTargetRequest(field_set.address, description_of_origin="<infallible>")
     )
@@ -152,6 +179,16 @@ async def run(
 
         args = (arg.format(chroot=tmpdir) for arg in request.args)
         env = {**complete_env, **{k: v.format(chroot=tmpdir) for k, v in request.extra_env.items()}}
+        if run_subsystem.debug_adapter:
+            logger.info(
+                softwrap(
+                    f"""
+                    Launching debug adapter at '{debug_adapter.host}:{debug_adapter.port}',
+                    which will wait for a client connection...
+                    """
+                )
+            )
+
         result = await Effect(
             InteractiveProcessResult,
             InteractiveProcess(