From bf25e129288e80116220292786f7e6018184173e Mon Sep 17 00:00:00 2001 From: Joshua Cannon Date: Mon, 27 Jun 2022 16:36:48 -0500 Subject: [PATCH] Add `--debug-adapter` flag to `run` (#15829) run's counterpart to #15799. Support is added for pex_binary (which will be spiritually lifted to the python_source with #15849) [ci skip-rust] [ci skip-build-wheels] --- .../Python/python-goals/python-run-goal.md | 29 ++++--- .../Python/python-goals/python-test-goal.md | 9 +-- .../pants/backend/docker/goals/run_image.py | 11 ++- .../pants/backend/go/goals/run_binary.py | 11 ++- .../backend/python/goals/run_pex_binary.py | 75 +++++++++++++++++-- .../python/packaging/pyoxidizer/rules.py | 11 ++- .../pants/backend/shell/shell_command.py | 11 ++- src/python/pants/core/goals/run.py | 39 +++++++++- src/python/pants/core/goals/run_test.py | 28 ++++++- src/python/pants/core/goals/test.py | 1 + src/python/pants/jvm/run_deploy_jar.py | 11 ++- 11 files changed, 204 insertions(+), 32 deletions(-) diff --git a/docs/markdown/Python/python-goals/python-run-goal.md b/docs/markdown/Python/python-goals/python-run-goal.md index becd4716fa7..4f107954b5f 100644 --- a/docs/markdown/Python/python-goals/python-run-goal.md +++ b/docs/markdown/Python/python-goals/python-run-goal.md @@ -29,11 +29,11 @@ You may only run one target at a time. The program will have access to the same environment used by the parent `./pants` process, so you can set environment variables in the external environment, e.g. `FOO=bar ./pants run project/app.py`. (Pants will auto-set some values like `$PATH`). > 📘 Tip: check the return code -> +> > Pants will propagate the return code from the underlying executable. Run `echo $?` after the Pants run to see the return code. > 🚧 Issues finding files? -> +> > Run `./pants dependencies --transitive path/to/binary.py` to ensure that all the files you need are showing up, including for any [assets](doc:assets) you intend to use. Watching the filesystem @@ -46,29 +46,36 @@ On the other hand, if your app is short lived (like a script) and you'd like to Debugging --------- +> 📘 Tip: using the VS Code (or any [DAP](https://microsoft.github.io/debug-adapter-protocol/)-compliant editor) remote debugger +> +> +> 1. In your editor, set your breakpoints and any other debug settings (like break-on-exception). +> 2. Run your code with `./pants run --debug-adapter`. +> 3. Connect your editor to the server. The server host and port are logged by Pants when executing `run --debug-adaptor`. (They can also be configured using the `[debug-adapter]` subsystem). + > 📘 Tip: Using the IntelliJ/PyCharm remote debugger -> +> > First, add the following target in some BUILD file (e.g., the one containing your other 3rd-party dependencies): -> +> > ``` > python_requirement( > name = "pydevd-pycharm", > requirements=["pydevd-pycharm==203.5419.8"], # Or whatever version you choose. > ) > ``` -> +> > You can check this into your repo, for convenience. -> +> > Now, use the remote debugger as usual: -> +> > 1. Start a Python remote debugging session in PyCharm, say on port 5000. > 2. Add the following code at the point where you want execution to pause and connect to the debugger: -> +> > ``` > import pydevd_pycharm > pydevd_pycharm.settrace('localhost', port=5000, stdoutToServer=True, stderrToServer=True) > ``` -> -> Run your executable with `./pants run` as usual. -> +> +> Run your executable with `./pants run` as usual. +> > Note: The first time you do so you may see some extra dependency resolution work, as `pydevd-pycharm` has now been added to the binary's dependencies, via inference. If you have dependency inference turned off in your repo, you will have to manually add a temporary explicit dependency in your binary target on the `pydevd-pycharm` target. diff --git a/docs/markdown/Python/python-goals/python-test-goal.md b/docs/markdown/Python/python-goals/python-test-goal.md index ae6147d82fc..88f9768136d 100644 --- a/docs/markdown/Python/python-goals/python-test-goal.md +++ b/docs/markdown/Python/python-goals/python-test-goal.md @@ -229,13 +229,12 @@ If you use multiple files with `test --debug`, they will run sequentially rather > ❯ ./pants test --debug -- -s > ``` -> 📘 Tip: using the VS Code (or any debug adatper-compliant editor) remote debugger in tests +> 📘 Tip: using the VS Code (or any [DAP](https://microsoft.github.io/debug-adapter-protocol/)-compliant editor) remote debugger in tests > > -> 1. Configure your editor's breakpoints and exception settings -> 2. Run your test with `./pants test --debug-adapter` -> 3. Connect your editor to the server (the server host and port are logged, and can be configured -> using the `[debug-adapter]` subsystem). +> 1. In your editor, set your breakpoints and any other debug settings (like break-on-exception). +> 2. Run your test with `./pants test --debug-adapter`. +> 3. Connect your editor to the server. The server host and port are logged by Pants when executing `test --debug-adaptor`. (They can also be configured using the `[debug-adapter]` subsystem). > Run your test with `./pants test --debug` as usual. diff --git a/src/python/pants/backend/docker/goals/run_image.py b/src/python/pants/backend/docker/goals/run_image.py index 9182d1175a6..dfa5da63e79 100644 --- a/src/python/pants/backend/docker/goals/run_image.py +++ b/src/python/pants/backend/docker/goals/run_image.py @@ -9,7 +9,7 @@ from pants.backend.docker.subsystems.docker_options import DockerOptions from pants.backend.docker.util_rules.docker_binary import DockerBinary from pants.core.goals.package import BuiltPackage, PackageFieldSet -from pants.core.goals.run import RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunRequest from pants.engine.environment import Environment, EnvironmentRequest from pants.engine.rules import Get, MultiGet, collect_rules, rule @@ -28,5 +28,14 @@ async def docker_image_run_request( return RunRequest(args=run.argv, digest=image.digest, extra_env=run.env) +@rule +async def docker_image_run_debug_adapter_request( + field_set: DockerFieldSet, +) -> RunDebugAdapterRequest: + raise NotImplementedError( + "Debugging a Docker image using a debug adapter has not yet been implemented." + ) + + def rules(): return collect_rules() diff --git a/src/python/pants/backend/go/goals/run_binary.py b/src/python/pants/backend/go/goals/run_binary.py index b931f54ddcd..e0552b44b4f 100644 --- a/src/python/pants/backend/go/goals/run_binary.py +++ b/src/python/pants/backend/go/goals/run_binary.py @@ -5,7 +5,7 @@ from pants.backend.go.goals.package_binary import GoBinaryFieldSet from pants.core.goals.package import BuiltPackage, PackageFieldSet -from pants.core.goals.run import RunFieldSet, RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest from pants.engine.internals.selectors import Get from pants.engine.rules import collect_rules, rule from pants.engine.unions import UnionRule @@ -19,5 +19,14 @@ async def create_go_binary_run_request(field_set: GoBinaryFieldSet) -> RunReques return RunRequest(digest=binary.digest, args=(os.path.join("{chroot}", artifact_relpath),)) +@rule +async def go_binary_run_debug_adapter_request( + field_set: GoBinaryFieldSet, +) -> RunDebugAdapterRequest: + raise NotImplementedError( + "Debugging a Go binary using a debug adapter has not yet been implemented." + ) + + def rules(): return [*collect_rules(), UnionRule(RunFieldSet, GoBinaryFieldSet)] diff --git a/src/python/pants/backend/python/goals/run_pex_binary.py b/src/python/pants/backend/python/goals/run_pex_binary.py index aa9f3c6f85b..454c72edc4d 100644 --- a/src/python/pants/backend/python/goals/run_pex_binary.py +++ b/src/python/pants/backend/python/goals/run_pex_binary.py @@ -1,9 +1,11 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging import os from pants.backend.python.goals.package_pex_binary import PexBinaryFieldSet +from pants.backend.python.subsystems.debugpy import DebugPy from pants.backend.python.target_types import ( PexBinaryDefaults, ResolvedPexEntryPoint, @@ -11,7 +13,7 @@ ) from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints from pants.backend.python.util_rules.local_dists import LocalDistsPex, LocalDistsPexRequest -from pants.backend.python.util_rules.pex import Pex +from pants.backend.python.util_rules.pex import Pex, PexRequest from pants.backend.python.util_rules.pex_environment import PexEnvironment from pants.backend.python.util_rules.pex_from_targets import ( InterpreterConstraintsRequest, @@ -21,12 +23,20 @@ PythonSourceFiles, PythonSourceFilesRequest, ) -from pants.core.goals.run import RunFieldSet, RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.engine.fs import Digest, MergeDigests from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import TransitiveTargets, TransitiveTargetsRequest from pants.engine.unions import UnionRule from pants.util.logging import LogLevel +from pants.util.strutil import softwrap + +logger = logging.getLogger(__name__) + + +def _in_chroot(relpath: str) -> str: + return os.path.join("{chroot}", relpath) @rule(level=LogLevel.DEBUG) @@ -99,13 +109,12 @@ async def create_pex_binary_run_request( ] merged_digest = await Get(Digest, MergeDigests(input_digests)) - def in_chroot(relpath: str) -> str: - return os.path.join("{chroot}", relpath) - complete_pex_env = pex_env.in_workspace() - args = complete_pex_env.create_argv(in_chroot(pex.name), python=pex.python) + # NB. If this changes, please consider how it affects the `DebugRequest` below + # (which is not easy to write automated tests for) + args = complete_pex_env.create_argv(_in_chroot(pex.name), python=pex.python) - chrooted_source_roots = [in_chroot(sr) for sr in sources.source_roots] + chrooted_source_roots = [_in_chroot(sr) for sr in sources.source_roots] # The order here is important: we want the in-repo sources to take precedence over their # copies in the sandbox (see above for why those copies exist even in non-sandboxed mode). source_roots = [ @@ -114,12 +123,62 @@ def in_chroot(relpath: str) -> str: ] extra_env = { **complete_pex_env.environment_dict(python_configured=pex.python is not None), - "PEX_PATH": in_chroot(local_dists.pex.name), + "PEX_PATH": _in_chroot(local_dists.pex.name), "PEX_EXTRA_SYS_PATH": os.pathsep.join(source_roots), } return RunRequest(digest=merged_digest, args=args, extra_env=extra_env) +@rule +async def run_pex_debug_adapter_binary( + field_set: PexBinaryFieldSet, + debugpy: DebugPy, + debug_adapter: DebugAdapterSubsystem, +) -> RunDebugAdapterRequest: + if field_set.run_in_sandbox: + logger.warning( + softwrap( + f""" + Using `run --debug-adapter` on the target {field_set.address}, which sets the field + `run_in_sandbox` to `True`. This will likely cause your breakpoints to not be hit, + as your code will be run under the sandbox's path. + """ + ) + ) + + entry_point, regular_run_request, debugpy_pex = await MultiGet( + Get( + ResolvedPexEntryPoint, + ResolvePexEntryPointRequest(field_set.entry_point), + ), + Get(RunRequest, PexBinaryFieldSet, field_set), + Get(Pex, PexRequest, debugpy.to_pex_request()), + ) + + entry_point_or_script = entry_point.val or field_set.script.value + assert entry_point_or_script is not None + merged_digest = await Get( + Digest, MergeDigests([regular_run_request.digest, debugpy_pex.digest]) + ) + extra_env = dict(regular_run_request.extra_env) + extra_env["PEX_PATH"] = os.pathsep.join( + [ + extra_env["PEX_PATH"], + # For debugpy to work properly, we need to have just one "environment" for our + # command to run in. Therefore, we cobble one together by exeucting debugpy's PEX, and + # shoehorning in the original PEX through PEX_PATH. + _in_chroot(os.path.basename(regular_run_request.args[1])), + ] + ) + args = [ + regular_run_request.args[0], # python executable + _in_chroot(debugpy_pex.name), + *debugpy.get_args(debug_adapter, entry_point_or_script), + ] + + return RunDebugAdapterRequest(digest=merged_digest, args=args, extra_env=extra_env) + + def rules(): return [*collect_rules(), UnionRule(RunFieldSet, PexBinaryFieldSet)] diff --git a/src/python/pants/backend/python/packaging/pyoxidizer/rules.py b/src/python/pants/backend/python/packaging/pyoxidizer/rules.py index a4a88efb14b..213c44ea57d 100644 --- a/src/python/pants/backend/python/packaging/pyoxidizer/rules.py +++ b/src/python/pants/backend/python/packaging/pyoxidizer/rules.py @@ -23,7 +23,7 @@ from pants.backend.python.target_types import GenerateSetupField, WheelField from pants.backend.python.util_rules.pex import Pex, PexProcess, PexRequest from pants.core.goals.package import BuiltPackage, BuiltPackageArtifact, PackageFieldSet -from pants.core.goals.run import RunFieldSet, RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest from pants.core.util_rules.system_binaries import BashBinary from pants.engine.fs import ( AddPrefix, @@ -232,6 +232,15 @@ def is_executable_binary(artifact_relpath: str | None) -> bool: return RunRequest(digest=binary.digest, args=(os.path.join("{chroot}", artifact.relpath),)) +@rule +async def run_pyoxidizer_debug_adapter_binary( + field_set: PyOxidizerFieldSet, +) -> RunDebugAdapterRequest: + raise NotImplementedError( + "Debugging a PyOxidizer binary using a debug adapter has not yet been implemented." + ) + + def rules(): return ( *collect_rules(), diff --git a/src/python/pants/backend/shell/shell_command.py b/src/python/pants/backend/shell/shell_command.py index 9fc9ac0d0e3..f74af012777 100644 --- a/src/python/pants/backend/shell/shell_command.py +++ b/src/python/pants/backend/shell/shell_command.py @@ -23,7 +23,7 @@ ShellCommandToolsField, ) from pants.core.goals.package import BuiltPackage, PackageFieldSet -from pants.core.goals.run import RunFieldSet, RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest from pants.core.target_types import FileSourceField from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.core.util_rules.system_binaries import ( @@ -248,6 +248,15 @@ async def run_shell_command_request(shell_command: RunShellCommand) -> RunReques ) +@rule +async def run_shell_debug_adapter_binary( + field_set: RunShellCommand, +) -> RunDebugAdapterRequest: + raise NotImplementedError( + "Debugging a shell command using a debug adapter has not yet been implemented." + ) + + def rules(): return [ *collect_rules(), diff --git a/src/python/pants/core/goals/run.py b/src/python/pants/core/goals/run.py index 708554daf35..daf2ac065e3 100644 --- a/src/python/pants/core/goals/run.py +++ b/src/python/pants/core/goals/run.py @@ -7,6 +7,7 @@ from typing import Iterable, Mapping, Optional, Tuple from pants.base.build_root import BuildRoot +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.engine.environment import CompleteEnvironment from pants.engine.fs import Digest, Workspace from pants.engine.goal import Goal, GoalSubsystem @@ -69,6 +70,13 @@ def __init__( self.extra_env = FrozenDict(extra_env or {}) +class RunDebugAdapterRequest(RunRequest): + """Like RunRequest, but launches the process using the relevant Debug Adapter server. + + The process should be launched waiting for the client to connect. + """ + + class RunSubsystem(GoalSubsystem): name = "run" help = softwrap( @@ -107,6 +115,20 @@ def activated(cls, union_membership: UnionMembership) -> bool: """ ), ) + # See also `test.py`'s same option + debug_adapter = BoolOption( + "--debug-adapter", + default=False, + help=softwrap( + """ + Run the interactive process using a Debug Adapter + (https://microsoft.github.io/debug-adapter-protocol/) for the language if supported. + + The interactive process used will be immediately blocked waiting for a client before + continuing. + """ + ), + ) class Run(Goal): @@ -116,6 +138,7 @@ class Run(Goal): @goal_rule async def run( run_subsystem: RunSubsystem, + debug_adapter: DebugAdapterSubsystem, global_options: GlobalOptions, workspace: Workspace, build_root: BuildRoot, @@ -131,7 +154,11 @@ async def run( ), ) field_set = targets_to_valid_field_sets.field_sets[0] - request = await Get(RunRequest, RunFieldSet, field_set) + request = await ( + Get(RunRequest, RunFieldSet, field_set) + if not run_subsystem.debug_adapter + else Get(RunDebugAdapterRequest, RunFieldSet, field_set) + ) wrapped_target = await Get( WrappedTarget, WrappedTargetRequest(field_set.address, description_of_origin="") ) @@ -152,6 +179,16 @@ async def run( args = (arg.format(chroot=tmpdir) for arg in request.args) env = {**complete_env, **{k: v.format(chroot=tmpdir) for k, v in request.extra_env.items()}} + if run_subsystem.debug_adapter: + logger.info( + softwrap( + f""" + Launching debug adapter at '{debug_adapter.host}:{debug_adapter.port}', + which will wait for a client connection... + """ + ) + ) + result = await Effect( InteractiveProcessResult, InteractiveProcess( diff --git a/src/python/pants/core/goals/run_test.py b/src/python/pants/core/goals/run_test.py index 0bbc032c21d..9e85c83071b 100644 --- a/src/python/pants/core/goals/run_test.py +++ b/src/python/pants/core/goals/run_test.py @@ -8,7 +8,15 @@ import pytest from pants.base.build_root import BuildRoot -from pants.core.goals.run import Run, RunFieldSet, RunRequest, RunSubsystem, run +from pants.core.goals.run import ( + Run, + RunDebugAdapterRequest, + RunFieldSet, + RunRequest, + RunSubsystem, + run, +) +from pants.core.subsystems.debug_adapter import DebugAdapterSubsystem from pants.engine.addresses import Address from pants.engine.fs import CreateDigest, Digest, FileContent, Workspace from pants.engine.process import InteractiveProcess, InteractiveProcessResult @@ -43,6 +51,12 @@ def create_mock_run_request(rule_runner: RuleRunner, program_text: bytes) -> Run return RunRequest(digest=digest, args=(os.path.join("{chroot}", "program.py"),)) +def create_mock_run_debug_adapter_request( + rule_runner: RuleRunner, program_text: bytes +) -> RunDebugAdapterRequest: + return cast(RunDebugAdapterRequest, create_mock_run_request(rule_runner, program_text)) + + def single_target_run( rule_runner: RuleRunner, address: Address, @@ -65,7 +79,12 @@ class TestBinaryTarget(Target): res = run_rule_with_mocks( run, rule_args=[ - create_goal_subsystem(RunSubsystem, args=[], cleanup=True), + create_goal_subsystem(RunSubsystem, args=[], cleanup=True, debug_adapter=False), + create_subsystem( + DebugAdapterSubsystem, + host="127.0.0.1", + port="5678", + ), create_subsystem( GlobalOptions, pants_workdir=rule_runner.pants_workdir, process_cleanup=True ), @@ -89,6 +108,11 @@ class TestBinaryTarget(Target): input_type=TestRunFieldSet, mock=lambda _: create_mock_run_request(rule_runner, program_text), ), + MockGet( + output_type=RunDebugAdapterRequest, + input_type=TestRunFieldSet, + mock=lambda _: create_mock_run_debug_adapter_request(rule_runner, program_text), + ), MockEffect( output_type=InteractiveProcessResult, input_type=InteractiveProcess, diff --git a/src/python/pants/core/goals/test.py b/src/python/pants/core/goals/test.py index 6cd8e55d77e..7ee1ca1d8b9 100644 --- a/src/python/pants/core/goals/test.py +++ b/src/python/pants/core/goals/test.py @@ -319,6 +319,7 @@ def activated(cls, union_membership: UnionMembership) -> bool: """ ), ) + # See also `run.py`'s same option debug_adapter = BoolOption( "--debug-adapter", default=False, diff --git a/src/python/pants/jvm/run_deploy_jar.py b/src/python/pants/jvm/run_deploy_jar.py index 91e6acbd2a4..2854ae13b1d 100644 --- a/src/python/pants/jvm/run_deploy_jar.py +++ b/src/python/pants/jvm/run_deploy_jar.py @@ -7,7 +7,7 @@ from typing import Iterable from pants.core.goals.package import BuiltPackage -from pants.core.goals.run import RunFieldSet, RunRequest +from pants.core.goals.run import RunDebugAdapterRequest, RunFieldSet, RunRequest from pants.engine.fs import EMPTY_DIGEST, Digest, MergeDigests from pants.engine.internals.native_engine import AddPrefix from pants.engine.process import Process, ProcessResult @@ -106,6 +106,15 @@ def prefixed(arg: str, prefixes: Iterable[str]) -> str: ) +@rule +async def run_deploy_jar_debug_adapter_binary( + field_set: DeployJarFieldSet, +) -> RunDebugAdapterRequest: + raise NotImplementedError( + "Debugging a deploy JAR using a debug adapter has not yet been implemented." + ) + + @rule async def ensure_jdk_for_pants_run(jdk: JdkEnvironment) -> __RuntimeJvm: # `tools.jar` is distributed with the JDK, so we can rely on it existing.