test console_task which aggregates test results

src/python/pants/backend/python/BUILD

@@ -9,6 +9,7 @@ python_library(
     ':python_artifact',
     ':python_requirement',
     ':python_requirements',
+    'src/python/pants/backend/python/rules',
     'src/python/pants/backend/python/targets:python',
     'src/python/pants/backend/python/tasks',
     'src/python/pants/build_graph',

src/python/pants/backend/python/register.py

@@ -8,6 +8,7 @@
 from pants.backend.python.python_artifact import PythonArtifact
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
+from pants.backend.python.rules.python_test_runner import run_python_test
 from pants.backend.python.targets.python_app import PythonApp
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_distribution import PythonDistribution
@@ -73,3 +74,7 @@ def register_goals():
   task(name='isort-prep', action=IsortPrep).install('fmt')
   task(name='isort', action=IsortRun).install('fmt')
   task(name='py', action=PythonBundle).install('bundle')
+
+
+def rules():
+  return (run_python_test,)

src/python/pants/backend/python/rules/BUILD

@@ -0,0 +1 @@
+python_library()

src/python/pants/backend/python/rules/python_test_runner.py

@@ -0,0 +1,32 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from builtins import str
+
+from pants.engine.legacy.graph import HydratedTarget
+from pants.engine.rules import rule
+from pants.engine.selectors import Select
+from pants.rules.core.core_test_model import Status, TestResult
+
+
+# This class currently exists so that other rules could be added which turned a HydratedTarget into
+# a language-specific test result, and could be installed alongside run_python_test.
+# Hopefully https://github.com/pantsbuild/pants/issues/4535 should help resolve this.
+class PyTestResult(TestResult):
+  pass
+
+
+@rule(PyTestResult, [Select(HydratedTarget)])
+def run_python_test(target):
+  # TODO: Actually run tests (https://github.com/pantsbuild/pants/issues/6003)
+
+  if 'fail' in target.address.reference():
+    noun = 'failed'
+    status = Status.FAILURE
+  else:
+    noun = 'passed'
+    status = Status.SUCCESS
+  return PyTestResult(status=status, stdout=str('I am a python test which {}'.format(noun)))

src/python/pants/init/engine_initializer.py

@@ -10,6 +10,7 @@
 from pants.backend.docgen.targets.doc import Page
 from pants.backend.jvm.targets.jvm_app import JvmApp
 from pants.backend.jvm.targets.jvm_binary import JvmBinary
+from pants.backend.python.rules.python_test_runner import run_python_test
 from pants.backend.python.targets.python_app import PythonApp
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_library import PythonLibrary
@@ -345,6 +346,8 @@ def setup_legacy_graph_extended(
       create_graph_rules(address_mapper, symbol_table) +
       create_options_parsing_rules() +
       create_core_rules() +
+      # TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here:
+      [run_python_test] +
       rules
     )
 

src/python/pants/rules/core/core_test_model.py

@@ -0,0 +1,24 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from builtins import str
+
+from pants.util.objects import datatype
+
+
+class Status(object):
+  SUCCESS = str('SUCCESS')
+  FAILURE = str('FAILURE')
+
+
+class TestResult(datatype([
+  # One of the Status pseudo-enum values capturing whether the run was successful.
+  ('status', str),
+  # The stdout of the test runner (which may or may not include actual testcase output).
+  ('stdout', str)
+])):
+  # Prevent this class from being detected by pytest as a test class.
+  __test__ = False

src/python/pants/rules/core/register.py

@@ -5,9 +5,12 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from pants.rules.core.fastlist import fast_list
+from pants.rules.core.test import coordinator_of_tests, fast_test
 
 
 def create_core_rules():
   return [
     fast_list,
+    fast_test,
+    coordinator_of_tests,
   ]

src/python/pants/rules/core/test.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from builtins import str
+
+from pants.backend.python.rules.python_test_runner import PyTestResult
+from pants.build_graph.address import Address
+from pants.engine.addressable import BuildFileAddresses
+from pants.engine.console import Console
+from pants.engine.legacy.graph import HydratedTarget
+from pants.engine.rules import console_rule, rule
+from pants.engine.selectors import Get, Select
+from pants.rules.core.core_test_model import Status, TestResult
+from pants.rules.core.exceptions import GracefulTerminationException
+
+
+@console_rule('test', [Select(Console), Select(BuildFileAddresses)])
+def fast_test(console, addresses):
+  test_results = yield [Get(TestResult, Address, address.to_address()) for address in addresses]
+  wrote_any_stdout = False
+  did_any_fail = False
+  for test_result in test_results:
+    wrote_any_stdout |= bool(test_result.stdout)
+    # Assume \n-terminated
+    console.write_stdout(test_result.stdout)
+    if test_result.stdout and not test_result.stdout[-1] == '\n':
+      console.write_stdout(str('\n'))
+    if test_result.status == Status.FAILURE:
+      did_any_fail = True
+
+  if wrote_any_stdout:
+    console.write_stdout(str('\n'))
+
+  for address, test_result in zip(addresses, test_results):
+    console.print_stdout(str('{0:80}.....{1:>10}'.format(address.reference(), test_result.status)))
+
+  if did_any_fail:
+    raise GracefulTerminationException("Tests failed", exit_code=1)
+
+
+@rule(TestResult, [Select(HydratedTarget)])
+def coordinator_of_tests(target):
+  # This should do an instance match, or canonicalise the adaptor type, or something
+  #if isinstance(target.adaptor, PythonTestsAdaptor):
+  # See https://github.com/pantsbuild/pants/issues/4535
+  if target.adaptor.type_alias == 'python_tests':
+    result = yield Get(PyTestResult, HydratedTarget, target)
+    yield TestResult(status=result.status, stdout=result.stdout)
+  else:
+    raise Exception("Didn't know how to run tests for type {}".format(target.adaptor.type_alias))

tests/python/pants_test/engine/util.py

@@ -7,6 +7,7 @@
 import os
 import re
 from builtins import str
+from io import StringIO
 from types import GeneratorType
 
 from pants.base.file_system_project_tree import FileSystemProjectTree
@@ -146,3 +147,23 @@ def remove_locations_from_traceback(trace):
   new_trace = location_pattern.sub('LOCATION-INFO', trace)
   new_trace = address_pattern.sub('0xEEEEEEEEE', new_trace)
   return new_trace
+
+
+class MockConsole(object):
+  """An implementation of pants.engine.console.Console which captures output."""
+
+  def __init__(self):
+    self.stdout = StringIO()
+    self.stderr = StringIO()
+
+  def write_stdout(self, payload):
+    self.stdout.write(payload)
+
+  def write_stderr(self, payload):
+    self.stderr.write(payload)
+
+  def print_stdout(self, payload):
+    print(payload, file=self.stdout)
+
+  def print_stderr(self, payload):
+    print(payload, file=self.stderr)

tests/python/pants_test/rules/BUILD

@@ -0,0 +1,21 @@
+python_tests(
+  name='test',
+  sources=['test_test.py'],
+  dependencies=[
+    '3rdparty/python:future',
+    'src/python/pants/backend/python/rules',
+    'src/python/pants/util:objects',
+    'tests/python/pants_test:test_base',
+    'tests/python/pants_test/engine:scheduler_test_base',
+    'tests/python/pants_test/engine/examples:scheduler_inputs',
+  ]
+)
+
+python_tests(
+  name='test_integration',
+  sources=['test_test_integration.py'],
+  dependencies=[
+    'tests/python/pants_test:int-test',
+  ],
+  tags={'integration'},
+)

tests/python/pants_test/rules/test_test.py

@@ -0,0 +1,120 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from builtins import str
+
+from pants.backend.python.rules.python_test_runner import PyTestResult
+from pants.build_graph.address import Address, BuildFileAddress
+from pants.engine.legacy.graph import HydratedTarget
+from pants.engine.legacy.structs import PythonTestsAdaptor
+from pants.rules.core.exceptions import GracefulTerminationException
+from pants.rules.core.test import Status, TestResult, coordinator_of_tests, fast_test
+from pants.util.meta import AbstractClass
+from pants_test.engine.scheduler_test_base import SchedulerTestBase
+from pants_test.engine.util import MockConsole, run_rule
+from pants_test.test_base import TestBase
+
+
+class TestTest(TestBase, SchedulerTestBase, AbstractClass):
+  def single_target_test(self, result, expected_console_output):
+    console = MockConsole()
+
+    run_rule(fast_test, console, (self.make_build_target_address("some/target"),), {
+      (TestResult, Address): lambda _: result,
+    })
+
+    self.assertEquals(console.stdout.getvalue(), expected_console_output)
+
+  def make_build_target_address(self, spec):
+    address = Address.parse(spec)
+    return BuildFileAddress(
+      build_file=None,
+      target_name=address.target_name,
+      rel_path='{}/BUILD'.format(address.spec_path),
+    )
+
+  def test_outputs_success(self):
+    self.single_target_test(
+      TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test')),
+      """Here is some output from a test
+
+some/target                                                                     .....   SUCCESS
+"""
+    )
+
+  def test_output_failure(self):
+    with self.assertRaises(GracefulTerminationException) as cm:
+      self.single_target_test(
+        TestResult(status=Status.FAILURE, stdout=str('Here is some output from a test')),
+        """Here is some output from a test
+
+some/target                                                                     .....   FAILURE
+"""
+      )
+    self.assertEqual(1, cm.exception.exit_code)
+
+  def test_output_no_trailing_newline(self):
+    self.single_target_test(
+      TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test')),
+      """Here is some output from a test
+
+some/target                                                                     .....   SUCCESS
+"""
+    )
+
+  def test_output_training_newline(self):
+    self.single_target_test(
+      TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test\n')),
+      """Here is some output from a test
+
+some/target                                                                     .....   SUCCESS
+"""
+    )
+
+  def test_output_mixed(self):
+    console = MockConsole()
+    target1 = self.make_build_target_address("testprojects/tests/python/pants/passes")
+    target2 = self.make_build_target_address("testprojects/tests/python/pants/fails")
+
+    def make_result(target):
+      if target == target1:
+        return TestResult(status=Status.SUCCESS, stdout=str('I passed'))
+      elif target == target2:
+        return TestResult(status=Status.FAILURE, stdout=str('I failed'))
+      else:
+        raise Exception("Unrecognised target")
+
+    with self.assertRaises(GracefulTerminationException) as cm:
+      run_rule(fast_test, console, (target1, target2), {
+        (TestResult, Address): make_result,
+      })
+
+    self.assertEqual(1, cm.exception.exit_code)
+    self.assertEquals(console.stdout.getvalue(), """I passed
+I failed
+
+testprojects/tests/python/pants/passes                                          .....   SUCCESS
+testprojects/tests/python/pants/fails                                           .....   FAILURE
+""")
+
+  def test_coordinator_python_test(self):
+    target_adaptor = PythonTestsAdaptor(type_alias='python_tests')
+
+    result = run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
+      (PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout=str('foo')),
+    })
+
+    self.assertEqual(result, TestResult(status=Status.FAILURE, stdout=str('foo')))
+
+  def test_coordinator_unknown_test(self):
+    target_adaptor = PythonTestsAdaptor(type_alias='unknown_tests')
+
+    with self.assertRaises(Exception) as cm:
+      run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
+        (PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout=str('foo')),
+      })
+
+    self.assertEqual(str(cm.exception), "Didn't know how to run tests for type unknown_tests")