Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test console_task which aggregates test results #6646

Merged
merged 3 commits into from
Oct 22, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/python/pants/backend/python/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ python_library(
':python_artifact',
':python_requirement',
':python_requirements',
'src/python/pants/backend/python/rules',
'src/python/pants/backend/python/targets:python',
'src/python/pants/backend/python/tasks',
'src/python/pants/build_graph',
Expand Down
5 changes: 5 additions & 0 deletions src/python/pants/backend/python/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from pants.backend.python.python_artifact import PythonArtifact
from pants.backend.python.python_requirement import PythonRequirement
from pants.backend.python.python_requirements import PythonRequirements
from pants.backend.python.rules.python_test_runner import run_python_test
from pants.backend.python.targets.python_app import PythonApp
from pants.backend.python.targets.python_binary import PythonBinary
from pants.backend.python.targets.python_distribution import PythonDistribution
Expand Down Expand Up @@ -73,3 +74,7 @@ def register_goals():
task(name='isort-prep', action=IsortPrep).install('fmt')
task(name='isort', action=IsortRun).install('fmt')
task(name='py', action=PythonBundle).install('bundle')


def rules():
return (run_python_test,)
1 change: 1 addition & 0 deletions src/python/pants/backend/python/rules/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
python_library()
Empty file.
32 changes: 32 additions & 0 deletions src/python/pants/backend/python/rules/python_test_runner.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import str

from pants.engine.legacy.graph import HydratedTarget
from pants.engine.rules import rule
from pants.engine.selectors import Select
from pants.rules.core.core_test_model import Status, TestResult


# This class currently exists so that other rules could be added which turned a HydratedTarget into
# a language-specific test result, and could be installed alongside run_python_test.
# Hopefully https://github.com/pantsbuild/pants/issues/4535 should help resolve this.
class PyTestResult(TestResult):
pass


@rule(PyTestResult, [Select(HydratedTarget)])
def run_python_test(target):
# TODO: Actually run tests (https://github.com/pantsbuild/pants/issues/6003)

if 'fail' in target.address.reference():
noun = 'failed'
status = Status.FAILURE
else:
noun = 'passed'
status = Status.SUCCESS
return PyTestResult(status=status, stdout=str('I am a python test which {}'.format(noun)))
3 changes: 3 additions & 0 deletions src/python/pants/init/engine_initializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from pants.backend.docgen.targets.doc import Page
from pants.backend.jvm.targets.jvm_app import JvmApp
from pants.backend.jvm.targets.jvm_binary import JvmBinary
from pants.backend.python.rules.python_test_runner import run_python_test
from pants.backend.python.targets.python_app import PythonApp
from pants.backend.python.targets.python_binary import PythonBinary
from pants.backend.python.targets.python_library import PythonLibrary
Expand Down Expand Up @@ -345,6 +346,8 @@ def setup_legacy_graph_extended(
create_graph_rules(address_mapper, symbol_table) +
create_options_parsing_rules() +
create_core_rules() +
# TODO: This should happen automatically, but most tests (e.g. tests/python/pants_test/auth) fail if it's not here:
[run_python_test] +
rules
)

Expand Down
24 changes: 24 additions & 0 deletions src/python/pants/rules/core/core_test_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import str

from pants.util.objects import datatype


class Status(object):
SUCCESS = str('SUCCESS')
FAILURE = str('FAILURE')


class TestResult(datatype([
# One of the Status pseudo-enum values capturing whether the run was successful.
('status', str),
# The stdout of the test runner (which may or may not include actual testcase output).
('stdout', str)
])):
# Prevent this class from being detected by pytest as a test class.
__test__ = False
3 changes: 3 additions & 0 deletions src/python/pants/rules/core/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
from __future__ import absolute_import, division, print_function, unicode_literals

from pants.rules.core.fastlist import fast_list
from pants.rules.core.test import coordinator_of_tests, fast_test


def create_core_rules():
return [
fast_list,
fast_test,
coordinator_of_tests,
]
53 changes: 53 additions & 0 deletions src/python/pants/rules/core/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import str

from pants.backend.python.rules.python_test_runner import PyTestResult
from pants.build_graph.address import Address
from pants.engine.addressable import BuildFileAddresses
from pants.engine.console import Console
from pants.engine.legacy.graph import HydratedTarget
from pants.engine.rules import console_rule, rule
from pants.engine.selectors import Get, Select
from pants.rules.core.core_test_model import Status, TestResult
from pants.rules.core.exceptions import GracefulTerminationException


@console_rule('test', [Select(Console), Select(BuildFileAddresses)])
def fast_test(console, addresses):
test_results = yield [Get(TestResult, Address, address.to_address()) for address in addresses]
wrote_any_stdout = False
did_any_fail = False
for test_result in test_results:
wrote_any_stdout |= bool(test_result.stdout)
# Assume \n-terminated
console.write_stdout(test_result.stdout)
if test_result.stdout and not test_result.stdout[-1] == '\n':
console.write_stdout(str('\n'))
if test_result.status == Status.FAILURE:
did_any_fail = True

if wrote_any_stdout:
console.write_stdout(str('\n'))

for address, test_result in zip(addresses, test_results):
console.print_stdout(str('{0:80}.....{1:>10}'.format(address.reference(), test_result.status)))

if did_any_fail:
raise GracefulTerminationException("Tests failed", exit_code=1)


@rule(TestResult, [Select(HydratedTarget)])
def coordinator_of_tests(target):
# This should do an instance match, or canonicalise the adaptor type, or something
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should definitely reference #4535.

#if isinstance(target.adaptor, PythonTestsAdaptor):
# See https://github.com/pantsbuild/pants/issues/4535
if target.adaptor.type_alias == 'python_tests':
result = yield Get(PyTestResult, HydratedTarget, target)
yield TestResult(status=result.status, stdout=result.stdout)
else:
raise Exception("Didn't know how to run tests for type {}".format(target.adaptor.type_alias))
21 changes: 21 additions & 0 deletions tests/python/pants_test/engine/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import os
import re
from builtins import str
from io import StringIO
from types import GeneratorType

from pants.base.file_system_project_tree import FileSystemProjectTree
Expand Down Expand Up @@ -146,3 +147,23 @@ def remove_locations_from_traceback(trace):
new_trace = location_pattern.sub('LOCATION-INFO', trace)
new_trace = address_pattern.sub('0xEEEEEEEEE', new_trace)
return new_trace


class MockConsole(object):
"""An implementation of pants.engine.console.Console which captures output."""

def __init__(self):
self.stdout = StringIO()
self.stderr = StringIO()

def write_stdout(self, payload):
self.stdout.write(payload)

def write_stderr(self, payload):
self.stderr.write(payload)

def print_stdout(self, payload):
print(payload, file=self.stdout)

def print_stderr(self, payload):
print(payload, file=self.stderr)
21 changes: 21 additions & 0 deletions tests/python/pants_test/rules/BUILD
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
python_tests(
name='test',
sources=['test_test.py'],
dependencies=[
'3rdparty/python:future',
'src/python/pants/backend/python/rules',
'src/python/pants/util:objects',
'tests/python/pants_test:test_base',
'tests/python/pants_test/engine:scheduler_test_base',
'tests/python/pants_test/engine/examples:scheduler_inputs',
]
)

python_tests(
name='test_integration',
sources=['test_test_integration.py'],
dependencies=[
'tests/python/pants_test:int-test',
],
tags={'integration'},
)
120 changes: 120 additions & 0 deletions tests/python/pants_test/rules/test_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# coding=utf-8
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

from builtins import str

from pants.backend.python.rules.python_test_runner import PyTestResult
from pants.build_graph.address import Address, BuildFileAddress
from pants.engine.legacy.graph import HydratedTarget
from pants.engine.legacy.structs import PythonTestsAdaptor
from pants.rules.core.exceptions import GracefulTerminationException
from pants.rules.core.test import Status, TestResult, coordinator_of_tests, fast_test
from pants.util.meta import AbstractClass
from pants_test.engine.scheduler_test_base import SchedulerTestBase
from pants_test.engine.util import MockConsole, run_rule
from pants_test.test_base import TestBase


class TestTest(TestBase, SchedulerTestBase, AbstractClass):
def single_target_test(self, result, expected_console_output):
console = MockConsole()

run_rule(fast_test, console, (self.make_build_target_address("some/target"),), {
(TestResult, Address): lambda _: result,
})

self.assertEquals(console.stdout.getvalue(), expected_console_output)

def make_build_target_address(self, spec):
address = Address.parse(spec)
return BuildFileAddress(
build_file=None,
target_name=address.target_name,
rel_path='{}/BUILD'.format(address.spec_path),
)

def test_outputs_success(self):
self.single_target_test(
TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test')),
"""Here is some output from a test

some/target ..... SUCCESS
"""
)

def test_output_failure(self):
with self.assertRaises(GracefulTerminationException) as cm:
self.single_target_test(
TestResult(status=Status.FAILURE, stdout=str('Here is some output from a test')),
"""Here is some output from a test

some/target ..... FAILURE
"""
)
self.assertEqual(1, cm.exception.exit_code)

def test_output_no_trailing_newline(self):
self.single_target_test(
TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test')),
"""Here is some output from a test

some/target ..... SUCCESS
"""
)

def test_output_training_newline(self):
self.single_target_test(
TestResult(status=Status.SUCCESS, stdout=str('Here is some output from a test\n')),
"""Here is some output from a test

some/target ..... SUCCESS
"""
)

def test_output_mixed(self):
console = MockConsole()
target1 = self.make_build_target_address("testprojects/tests/python/pants/passes")
target2 = self.make_build_target_address("testprojects/tests/python/pants/fails")

def make_result(target):
if target == target1:
return TestResult(status=Status.SUCCESS, stdout=str('I passed'))
elif target == target2:
return TestResult(status=Status.FAILURE, stdout=str('I failed'))
else:
raise Exception("Unrecognised target")

with self.assertRaises(GracefulTerminationException) as cm:
run_rule(fast_test, console, (target1, target2), {
(TestResult, Address): make_result,
})

self.assertEqual(1, cm.exception.exit_code)
self.assertEquals(console.stdout.getvalue(), """I passed
I failed

testprojects/tests/python/pants/passes ..... SUCCESS
testprojects/tests/python/pants/fails ..... FAILURE
""")

def test_coordinator_python_test(self):
target_adaptor = PythonTestsAdaptor(type_alias='python_tests')

result = run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
(PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout=str('foo')),
})

self.assertEqual(result, TestResult(status=Status.FAILURE, stdout=str('foo')))

def test_coordinator_unknown_test(self):
target_adaptor = PythonTestsAdaptor(type_alias='unknown_tests')

with self.assertRaises(Exception) as cm:
run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
(PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout=str('foo')),
})

self.assertEqual(str(cm.exception), "Didn't know how to run tests for type unknown_tests")
Loading