From b5781b5601afe372dcd14d6cc041a912e32d4d39 Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Mon, 19 Jul 2021 20:08:32 -0400 Subject: [PATCH] Provide a peek goal to easily view BUILD metadata from command line (#11347) ### Problem It would be convenient to have pants able to export build metadata, either for human consumption or programmatic consumption. This seems to be the essence of issue #4861, which the PR addresses. ### Solution This PR adds a new `peek` goal, which allows us to look at the BUILD files for given targets using either their raw content, or with targets mapped to JSON. --- src/python/pants/backend/project_info/peek.py | 154 +++++++++++++++++ .../pants/backend/project_info/peek_test.py | 160 ++++++++++++++++++ .../pants/backend/project_info/register.py | 2 + 3 files changed, 316 insertions(+) create mode 100644 src/python/pants/backend/project_info/peek.py create mode 100644 src/python/pants/backend/project_info/peek_test.py diff --git a/src/python/pants/backend/project_info/peek.py b/src/python/pants/backend/project_info/peek.py new file mode 100644 index 00000000000..1c87edbb9cc --- /dev/null +++ b/src/python/pants/backend/project_info/peek.py @@ -0,0 +1,154 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import collections.abc +import json +import os +from dataclasses import asdict, is_dataclass +from enum import Enum +from typing import Any, Iterable, Mapping, cast + +from pkg_resources import Requirement + +from pants.engine.addresses import Address, BuildFileAddress +from pants.engine.console import Console +from pants.engine.fs import DigestContents, FileContent, PathGlobs +from pants.engine.goal import Goal, GoalSubsystem, Outputting +from pants.engine.rules import Get, MultiGet, collect_rules, goal_rule +from pants.engine.target import Target, UnexpandedTargets + + +class OutputOptions(Enum): + RAW = "raw" + JSON = "json" + + +class PeekSubsystem(Outputting, GoalSubsystem): + """Display BUILD file info to the console. + + In its most basic form, `peek` just prints the contents of a BUILD file. It can also display + multiple BUILD files, or render normalized target metadata as JSON for consumption by other + programs. + """ + + name = "peek" + help = "Display BUILD target info" + + @classmethod + def register_options(cls, register): + super().register_options(register) + register( + "--output", + type=OutputOptions, + default=OutputOptions.JSON, + help=( + "Which output style peek should use: `json` will show each target as a seperate " + "entry, whereas `raw` will simply show the original non-normalized BUILD files." + ), + ) + register( + "--exclude-defaults", + type=bool, + default=False, + help=( + "Whether to leave off values that match the target-defined default values " + "when using `json` output." + ), + ) + + @property + def output_type(self) -> OutputOptions: + """Get the output type from options. + + Must be renamed here because `output` conflicts with `Outputting` class. + """ + return cast(OutputOptions, self.options.output) + + @property + def exclude_defaults(self) -> bool: + return cast(bool, self.options.exclude_defaults) + + +class Peek(Goal): + subsystem_cls = PeekSubsystem + + +def _render_raw(fcs: Iterable[FileContent]) -> str: + sorted_fcs = sorted(fcs, key=lambda fc: fc.path) + rendereds = map(_render_raw_build_file, sorted_fcs) + return os.linesep.join(rendereds) + + +def _render_raw_build_file(fc: FileContent, encoding: str = "utf-8") -> str: + dashes = "-" * len(fc.path) + content = fc.content.decode(encoding) + parts = [dashes, fc.path, dashes, content] + if not content.endswith(os.linesep): + parts.append("") + return os.linesep.join(parts) + + +_nothing = object() + + +def _render_json(ts: Iterable[Target], exclude_defaults: bool = False) -> str: + targets: Iterable[Mapping[str, Any]] = [ + { + "address": t.address.spec, + "target_type": t.alias, + **{ + k.alias: v.value + for k, v in t.field_values.items() + if not (exclude_defaults and getattr(k, "default", _nothing) == v.value) + }, + } + for t in ts + ] + return f"{json.dumps(targets, indent=2, cls=_PeekJsonEncoder)}\n" + + +class _PeekJsonEncoder(json.JSONEncoder): + """Allow us to serialize some commmonly-found types in BUILD files.""" + + safe_to_str_types = (Requirement,) + + def default(self, o): + """Return a serializable object for o.""" + if is_dataclass(o): + return asdict(o) + if isinstance(o, collections.abc.Mapping): + return dict(o) + if isinstance(o, collections.abc.Sequence): + return list(o) + try: + return super().default(o) + except TypeError: + return str(o) + + +@goal_rule +async def peek( + console: Console, + subsys: PeekSubsystem, + targets: UnexpandedTargets, +) -> Peek: + if subsys.output_type == OutputOptions.RAW: + build_file_addresses = await MultiGet( + Get(BuildFileAddress, Address, t.address) for t in targets + ) + build_file_paths = {a.rel_path for a in build_file_addresses} + digest_contents = await Get(DigestContents, PathGlobs(build_file_paths)) + output = _render_raw(digest_contents) + elif subsys.output_type == OutputOptions.JSON: + output = _render_json(targets, subsys.exclude_defaults) + else: + raise AssertionError(f"output_type not one of {tuple(OutputOptions)}") + + with subsys.output(console) as write_stdout: + write_stdout(output) + + return Peek(exit_code=0) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/project_info/peek_test.py b/src/python/pants/backend/project_info/peek_test.py new file mode 100644 index 00000000000..27576c5f03e --- /dev/null +++ b/src/python/pants/backend/project_info/peek_test.py @@ -0,0 +1,160 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from textwrap import dedent + +import pytest + +from pants.backend.project_info import peek +from pants.backend.project_info.peek import Peek +from pants.core.target_types import ArchiveTarget, Files +from pants.engine.addresses import Address +from pants.testutil.rule_runner import RuleRunner + + +@pytest.mark.parametrize( + "targets, exclude_defaults, expected_output", + [ + pytest.param( + [], + False, + "[]\n", + id="null-case", + ), + pytest.param( + [Files({"sources": []}, Address("example", target_name="files_target"))], + True, + dedent( + """\ + [ + { + "address": "example:files_target", + "target_type": "files", + "sources": [] + } + ] + """ + ), + id="single-files-target/exclude-defaults", + ), + pytest.param( + [Files({"sources": []}, Address("example", target_name="files_target"))], + False, + dedent( + """\ + [ + { + "address": "example:files_target", + "target_type": "files", + "dependencies": null, + "description": null, + "sources": [], + "tags": null + } + ] + """ + ), + id="single-files-target/include-defaults", + ), + pytest.param( + [ + Files( + {"sources": ["*.txt"], "tags": ["zippable"]}, + Address("example", target_name="files_target"), + ), + ArchiveTarget( + { + "output_path": "my-archive.zip", + "format": "zip", + "files": ["example:files_target"], + }, + Address("example", target_name="archive_target"), + ), + ], + True, + dedent( + """\ + [ + { + "address": "example:files_target", + "target_type": "files", + "sources": [ + "*.txt" + ], + "tags": [ + "zippable" + ] + }, + { + "address": "example:archive_target", + "target_type": "archive", + "files": [ + "example:files_target" + ], + "format": "zip", + "output_path": "my-archive.zip" + } + ] + """ + ), + id="single-files-target/exclude-defaults", + ), + ], +) +def test_render_targets_as_json(targets, exclude_defaults, expected_output): + actual_output = peek._render_json(targets, exclude_defaults) + assert actual_output == expected_output + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner(rules=peek.rules(), target_types=[Files]) + + +def test_raw_output_single_build_file(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("project", "# A comment\nfiles(sources=[])") + result = rule_runner.run_goal_rule(Peek, args=["--output=raw", "project"]) + expected_output = dedent( + """\ + ------------- + project/BUILD + ------------- + # A comment + files(sources=[]) + """ + ) + assert result.stdout == expected_output + + +def test_raw_output_two_build_files(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("project1", "# A comment\nfiles(sources=[])") + rule_runner.add_to_build_file("project2", "# Another comment\nfiles(sources=[])") + result = rule_runner.run_goal_rule(Peek, args=["--output=raw", "project1", "project2"]) + expected_output = dedent( + """\ + -------------- + project1/BUILD + -------------- + # A comment + files(sources=[]) + + -------------- + project2/BUILD + -------------- + # Another comment + files(sources=[]) + """ + ) + assert result.stdout == expected_output + + +def test_raw_output_non_matching_build_target(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("some_name", "files(sources=[])") + result = rule_runner.run_goal_rule(Peek, args=["--output=raw", "other_name"]) + assert result.stdout == "" + + +def test_standard_json_output_non_matching_build_target(rule_runner: RuleRunner) -> None: + rule_runner.add_to_build_file("some_name", "files(sources=[])") + result = rule_runner.run_goal_rule(Peek, args=["other_name"]) + assert result.stdout == "[]\n" diff --git a/src/python/pants/backend/project_info/register.py b/src/python/pants/backend/project_info/register.py index 54c6889fe46..f99be87b499 100644 --- a/src/python/pants/backend/project_info/register.py +++ b/src/python/pants/backend/project_info/register.py @@ -11,6 +11,7 @@ filter_targets, list_roots, list_targets, + peek, source_file_validator, ) @@ -24,5 +25,6 @@ def rules(): *filter_targets.rules(), *list_roots.rules(), *list_targets.rules(), + *peek.rules(), *source_file_validator.rules(), ]