Support extra test runner output. (#11741)

Test runners may, depending on their runtime configuration,
output extra data that we don't know about, but the user may
want to see.

For example, pytest may be configured to run the pytest-html
plugin, which will emit an HTML report into the process execution
sandbox, and we won't pick it up from there.

This change adds generic support for "extra output" from a test
runner. All the user needs to do is ensure this output goes
somewhere where the appropriate test execution rule can find it.
Pants will dump any extra output it finds into dist/.

In the specific case of pytest, we look for this output in the
extra-output/ subdir. This means that users must ensure that output
is directed there (e.g., using the --html=extra-output/report.html
flag to pytest). We must document this, of course.

This change also enables pytest-html in the Pants repo itself,
as a proof of concept. To get a report:

./pants test <tgt> -- --html=extra-output/report.html

[ci skip-rust]

[ci skip-build-wheels]

pants.toml

@@ -141,6 +141,7 @@ release_notes = """
 args = ["--no-header"]
 pytest_plugins.add = [
   "ipdb",
+  "pytest-html",
   "pytest-icdiff",
   "pygments",
 ]

src/python/pants/backend/python/goals/pytest_runner.py

@@ -49,6 +49,7 @@
     GlobMatchErrorBehavior,
     MergeDigests,
     PathGlobs,
+    RemovePrefix,
     Snapshot,
 )
 from pants.engine.process import (
@@ -73,6 +74,12 @@
 logger = logging.getLogger()
 
 
+# If a user wants extra pytest output (e.g., plugin output) to show up in dist/
+# they must ensure that output goes under this directory. E.g.,
+# ./pants test <target> -- --html=extra-output/report.html
+_EXTRA_OUTPUT_DIR = "extra-output"
+
+
 @dataclass(frozen=True)
 class PythonTestFieldSet(TestFieldSet):
     required_fields = (PythonTestsSources,)
@@ -251,6 +258,7 @@ async def setup_pytest_for_target(
             argv=(*pytest.options.args, *coverage_args, *field_set_source_files.files),
             extra_env=extra_env,
             input_digest=input_digest,
+            output_directories=(_EXTRA_OUTPUT_DIR,),
             output_files=output_files,
             timeout_seconds=request.field_set.timeout.calculate_from_global_options(pytest),
             execution_slot_variable=pytest.options.execution_slot_var,
@@ -294,12 +302,19 @@ async def run_python_test(
             )
         else:
             logger.warning(f"Failed to generate JUnit XML data for {field_set.address}.")
+    extra_output_snapshot = await Get(
+        Snapshot, DigestSubset(result.output_digest, PathGlobs([f"{_EXTRA_OUTPUT_DIR}/**"]))
+    )
+    extra_output_snapshot = await Get(
+        Snapshot, RemovePrefix(extra_output_snapshot.digest, _EXTRA_OUTPUT_DIR)
+    )
 
     return TestResult.from_fallible_process_result(
         result,
         address=field_set.address,
         coverage_data=coverage_data,
         xml_results=xml_results_snapshot,
+        extra_output=extra_output_snapshot,
     )
 
 

src/python/pants/backend/python/goals/pytest_runner_integration_test.py

@@ -147,12 +147,17 @@ def run_pytest(
     config: Optional[str] = None,
     force: bool = False,
 ) -> TestResult:
+    # pytest-html==1.22.1 has an undeclared dep on setuptools. This, unfortunately,
+    # is the most recent version of pytest-html that works with the low version of
+    # pytest that we pin to.
+    plugins = ["zipp==1.0.0", "pytest-cov>=2.8.1,<2.9", "pytest-html==1.22.1", "setuptools"]
+    plugins_str = "['" + "', '".join(plugins) + "']"
     args = [
         "--backend-packages=pants.backend.python",
         f"--source-root-patterns={SOURCE_ROOT}",
         # pin to lower versions so that we can run Python 2 tests
         "--pytest-version=pytest>=4.6.6,<4.7",
-        "--pytest-pytest-plugins=['zipp==1.0.0', 'pytest-cov>=2.8.1,<2.9']",
+        f"--pytest-pytest-plugins={plugins_str}",
     ]
     if passthrough_args:
         args.append(f"--pytest-args='{passthrough_args}'")
@@ -401,6 +406,17 @@ def test_junit(rule_runner: RuleRunner) -> None:
     assert b"pants_test.test_good" in file.content
 
 
+def test_extra_output(rule_runner: RuleRunner) -> None:
+    tgt = create_test_target(rule_runner, [GOOD_SOURCE])
+    result = run_pytest(rule_runner, tgt, passthrough_args="--html=extra-output/report.html")
+    assert result.exit_code == 0
+    assert f"{PACKAGE}/test_good.py ." in result.stdout
+    assert result.extra_output is not None
+    digest_contents = rule_runner.request(DigestContents, [result.extra_output.digest])
+    paths = {dc.path for dc in digest_contents}
+    assert {"assets/style.css", "report.html"} == paths
+
+
 def test_coverage(rule_runner: RuleRunner) -> None:
     tgt = create_test_target(rule_runner, [GOOD_SOURCE])
     result = run_pytest(rule_runner, tgt, use_coverage=True)

src/python/pants/core/goals/test.py

@@ -11,6 +11,7 @@
 from pathlib import PurePath
 from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar, Union, cast
 
+from pants.core.util_rules.distdir import DistDir
 from pants.core.util_rules.filter_empty_sources import (
     FieldSetsWithSources,
     FieldSetsWithSourcesRequest,
@@ -49,6 +50,8 @@ class TestResult:
     address: Address
     coverage_data: Optional[CoverageData] = None
     xml_results: Optional[Snapshot] = None
+    # Any extra output (such as from plugins) that the test runner was configured to output.
+    extra_output: Optional[Snapshot] = None
 
     # Prevent this class from being detected by pytest as a test class.
     __test__ = False
@@ -72,6 +75,7 @@ def from_fallible_process_result(
         *,
         coverage_data: Optional[CoverageData] = None,
         xml_results: Optional[Snapshot] = None,
+        extra_output: Optional[Snapshot] = None,
     ) -> TestResult:
         return cls(
             exit_code=process_result.exit_code,
@@ -82,6 +86,7 @@ def from_fallible_process_result(
             address=address,
             coverage_data=coverage_data,
             xml_results=xml_results,
+            extra_output=extra_output,
         )
 
     @property
@@ -365,6 +370,7 @@ async def run_tests(
     interactive_runner: InteractiveRunner,
     workspace: Workspace,
     union_membership: UnionMembership,
+    dist_dir: DistDir,
 ) -> Test:
     if test_subsystem.debug:
         targets_to_valid_field_sets = await Get(
@@ -419,6 +425,11 @@ async def run_tests(
             status = "failed"
             exit_code = cast(int, result.exit_code)
         console.print_stderr(f"{sigil} {result.address} {status}.")
+        if result.extra_output and result.extra_output.files:
+            workspace.write_digest(
+                result.extra_output.digest,
+                path_prefix=str(dist_dir.relpath / "test" / result.address.path_safe_spec),
+            )
 
     merged_xml_results = await Get(
         Digest,
@@ -497,6 +508,7 @@ def enrich_test_result(
         address=test_result.address,
         coverage_data=test_result.coverage_data,
         xml_results=test_result.xml_results,
+        extra_output=test_result.extra_output,
         output_setting=test_subsystem.output,
     )
 

src/python/pants/core/goals/test_test.py

@@ -4,6 +4,7 @@
 from abc import ABCMeta, abstractmethod
 from dataclasses import dataclass
 from functools import partial
+from pathlib import Path
 from textwrap import dedent
 from typing import List, Optional, Tuple, Type
 
@@ -22,6 +23,7 @@
     TestSubsystem,
     run_tests,
 )
+from pants.core.util_rules.distdir import DistDir
 from pants.core.util_rules.filter_empty_sources import (
     FieldSetsWithSources,
     FieldSetsWithSourcesRequest,
@@ -165,6 +167,7 @@ def mock_coverage_report_generation(
                 interactive_runner,
                 workspace,
                 union_membership,
+                DistDir(relpath=Path("dist")),
             ],
             mock_gets=[
                 MockGet(