Skip to content

Commit

Permalink
Support extra test runner output. (#11741)
Browse files Browse the repository at this point in the history
Test runners may, depending on their runtime configuration,
output extra data that we don't know about, but the user may
want to see.

For example, pytest may be configured to run the pytest-html
plugin, which will emit an HTML report into the process execution
sandbox, and we won't pick it up from there.

This change adds generic support for "extra output" from a test
runner. All the user needs to do is ensure this output goes
somewhere where the appropriate test execution rule can find it.
Pants will dump any extra output it finds into dist/.

In the specific case of pytest, we look for this output in the
extra-output/ subdir. This means that users must ensure that output
is directed there (e.g., using the --html=extra-output/report.html
flag to pytest). We must document this, of course.

This change also enables pytest-html in the Pants repo itself,
as a proof of concept. To get a report:

./pants test <tgt> -- --html=extra-output/report.html

[ci skip-rust]

[ci skip-build-wheels]
  • Loading branch information
benjyw authored Mar 19, 2021
1 parent 306d185 commit 7a6a8df
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 1 deletion.
1 change: 1 addition & 0 deletions pants.toml
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ release_notes = """
args = ["--no-header"]
pytest_plugins.add = [
"ipdb",
"pytest-html",
"pytest-icdiff",
"pygments",
]
Expand Down
15 changes: 15 additions & 0 deletions src/python/pants/backend/python/goals/pytest_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
GlobMatchErrorBehavior,
MergeDigests,
PathGlobs,
RemovePrefix,
Snapshot,
)
from pants.engine.process import (
Expand All @@ -73,6 +74,12 @@
logger = logging.getLogger()


# If a user wants extra pytest output (e.g., plugin output) to show up in dist/
# they must ensure that output goes under this directory. E.g.,
# ./pants test <target> -- --html=extra-output/report.html
_EXTRA_OUTPUT_DIR = "extra-output"


@dataclass(frozen=True)
class PythonTestFieldSet(TestFieldSet):
required_fields = (PythonTestsSources,)
Expand Down Expand Up @@ -251,6 +258,7 @@ async def setup_pytest_for_target(
argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files),
extra_env=extra_env,
input_digest=input_digest,
output_directories=(_EXTRA_OUTPUT_DIR,),
output_files=output_files,
timeout_seconds=request.field_set.timeout.calculate_from_global_options(pytest),
execution_slot_variable=pytest.options.execution_slot_var,
Expand Down Expand Up @@ -294,12 +302,19 @@ async def run_python_test(
)
else:
logger.warning(f"Failed to generate JUnit XML data for {field_set.address}.")
extra_output_snapshot = await Get(
Snapshot, DigestSubset(result.output_digest, PathGlobs([f"{_EXTRA_OUTPUT_DIR}/**"]))
)
extra_output_snapshot = await Get(
Snapshot, RemovePrefix(extra_output_snapshot.digest, _EXTRA_OUTPUT_DIR)
)

return TestResult.from_fallible_process_result(
result,
address=field_set.address,
coverage_data=coverage_data,
xml_results=xml_results_snapshot,
extra_output=extra_output_snapshot,
)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,17 @@ def run_pytest(
config: Optional[str] = None,
force: bool = False,
) -> TestResult:
# pytest-html==1.22.1 has an undeclared dep on setuptools. This, unfortunately,
# is the most recent version of pytest-html that works with the low version of
# pytest that we pin to.
plugins = ["zipp==1.0.0", "pytest-cov>=2.8.1,<2.9", "pytest-html==1.22.1", "setuptools"]
plugins_str = "['" + "', '".join(plugins) + "']"
args = [
"--backend-packages=pants.backend.python",
f"--source-root-patterns={SOURCE_ROOT}",
# pin to lower versions so that we can run Python 2 tests
"--pytest-version=pytest>=4.6.6,<4.7",
"--pytest-pytest-plugins=['zipp==1.0.0', 'pytest-cov>=2.8.1,<2.9']",
f"--pytest-pytest-plugins={plugins_str}",
]
if passthrough_args:
args.append(f"--pytest-args='{passthrough_args}'")
Expand Down Expand Up @@ -401,6 +406,17 @@ def test_junit(rule_runner: RuleRunner) -> None:
assert b"pants_test.test_good" in file.content


def test_extra_output(rule_runner: RuleRunner) -> None:
tgt = create_test_target(rule_runner, [GOOD_SOURCE])
result = run_pytest(rule_runner, tgt, passthrough_args="--html=extra-output/report.html")
assert result.exit_code == 0
assert f"{PACKAGE}/test_good.py ." in result.stdout
assert result.extra_output is not None
digest_contents = rule_runner.request(DigestContents, [result.extra_output.digest])
paths = {dc.path for dc in digest_contents}
assert {"assets/style.css", "report.html"} == paths


def test_coverage(rule_runner: RuleRunner) -> None:
tgt = create_test_target(rule_runner, [GOOD_SOURCE])
result = run_pytest(rule_runner, tgt, use_coverage=True)
Expand Down
12 changes: 12 additions & 0 deletions src/python/pants/core/goals/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from pathlib import PurePath
from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast

from pants.core.util_rules.distdir import DistDir
from pants.core.util_rules.filter_empty_sources import (
FieldSetsWithSources,
FieldSetsWithSourcesRequest,
Expand Down Expand Up @@ -49,6 +50,8 @@ class TestResult:
address: Address
coverage_data: Optional[CoverageData] = None
xml_results: Optional[Snapshot] = None
# Any extra output (such as from plugins) that the test runner was configured to output.
extra_output: Optional[Snapshot] = None

# Prevent this class from being detected by pytest as a test class.
__test__ = False
Expand All @@ -72,6 +75,7 @@ def from_fallible_process_result(
*,
coverage_data: Optional[CoverageData] = None,
xml_results: Optional[Snapshot] = None,
extra_output: Optional[Snapshot] = None,
) -> TestResult:
return cls(
exit_code=process_result.exit_code,
Expand All @@ -82,6 +86,7 @@ def from_fallible_process_result(
address=address,
coverage_data=coverage_data,
xml_results=xml_results,
extra_output=extra_output,
)

@property
Expand Down Expand Up @@ -365,6 +370,7 @@ async def run_tests(
interactive_runner: InteractiveRunner,
workspace: Workspace,
union_membership: UnionMembership,
dist_dir: DistDir,
) -> Test:
if test_subsystem.debug:
targets_to_valid_field_sets = await Get(
Expand Down Expand Up @@ -419,6 +425,11 @@ async def run_tests(
status = "failed"
exit_code = cast(int, result.exit_code)
console.print_stderr(f"{sigil} {result.address} {status}.")
if result.extra_output and result.extra_output.files:
workspace.write_digest(
result.extra_output.digest,
path_prefix=str(dist_dir.relpath / "test" / result.address.path_safe_spec),
)

merged_xml_results = await Get(
Digest,
Expand Down Expand Up @@ -497,6 +508,7 @@ def enrich_test_result(
address=test_result.address,
coverage_data=test_result.coverage_data,
xml_results=test_result.xml_results,
extra_output=test_result.extra_output,
output_setting=test_subsystem.output,
)

Expand Down
3 changes: 3 additions & 0 deletions src/python/pants/core/goals/test_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from textwrap import dedent
from typing import List, Optional, Tuple, Type

Expand All @@ -22,6 +23,7 @@
TestSubsystem,
run_tests,
)
from pants.core.util_rules.distdir import DistDir
from pants.core.util_rules.filter_empty_sources import (
FieldSetsWithSources,
FieldSetsWithSourcesRequest,
Expand Down Expand Up @@ -165,6 +167,7 @@ def mock_coverage_report_generation(
interactive_runner,
workspace,
union_membership,
DistDir(relpath=Path("dist")),
],
mock_gets=[
MockGet(
Expand Down

0 comments on commit 7a6a8df

Please sign in to comment.