-
-
Notifications
You must be signed in to change notification settings - Fork 636
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Move testinfra code from
tests/python/pants_tests
to `src/python/pa…
…nts/testutil` (#8400) ### Problem We are in the process of moving our tests to live within the `src` directory, rather than `tests` (#7489). We should consequently have our core utility code live in `src`. Further, the current utils are spread out across several folders, which makes discovery more difficult. ### Solution Create a new `pants/testutil` namespace with all of the files used by the `pants.testinfra` wheel. Specifically, it has this directory structure: ``` src/python/pants/testutil ├── BUILD ├── __init__.py ├── base │ ├── BUILD │ ├── __init__.py │ └── context_utils.py ├── console_rule_test_base.py ├── engine │ ├── BUILD │ ├── __init__.py │ ├── base_engine_test.py │ └── util.py ├── file_test_util.py ├── git_util.py ├── interpreter_selection_utils.py ├── jvm │ ├── BUILD │ ├── __init__.py │ ├── jar_task_test_base.py │ ├── jvm_task_test_base.py │ ├── jvm_tool_task_test_base.py │ └── nailgun_task_test_base.py ├── mock_logger.py ├── option │ ├── BUILD │ ├── __init__.py │ └── fakes.py ├── pants_run_integration_test.py ├── pexrc_util.py ├── process_test_util.py ├── subsystem │ ├── BUILD │ ├── __init__.py │ └── util.py ├── task_test_base.py └── test_base.py ``` We can't yet delete the equivalent files in `pants_test` due to a public API, so we instead have those files simply re-export the implementation in `pants/testutils`. #### Does not move some util code There are several util files not exposed to the `pantsbuild.testinfra` wheel, like `engine/scheduler_test_base.py`. To reduce the scope of this PR, we keep those there for now. Presumably, they will be able to be moved into `pants/testinfra` without a deprecation cycle. #### Creates new wheel `pantsbuild.pants.testutil` We had to create a new package, rather than using `pantsbuild.pants.testinfra`, due to rules about ownership of files (#8400 (comment)). In a followup, we will deprecate `pantsbuild.pants.testinfra`. ### Result We can now either use `pants.testutil` or the conventional `pants_test` imports. Both have the same functionality. In a followup PR, we will deprecate the `pants_test` version.
- Loading branch information
1 parent
7f4ef60
commit a0bcc23
Showing
74 changed files
with
3,621 additions
and
3,246 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,162 @@ | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_library( | ||
name='testutil_wheel', | ||
dependencies=[ | ||
':int-test-for-export', | ||
':test_base', | ||
':file_test_util', | ||
'src/python/pants/testutil/base:context_utils', | ||
'src/python/pants/testutil/engine:engine_test_base', | ||
'src/python/pants/testutil/engine:util', | ||
'src/python/pants/testutil/jvm:jar_task_test_base', | ||
'src/python/pants/testutil/jvm:nailgun_task_test_base', | ||
'src/python/pants/testutil/jvm:jvm_tool_task_test_base', | ||
'src/python/pants/testutil/option', | ||
'src/python/pants/testutil/subsystem', | ||
], | ||
provides=pants_setup_py( | ||
name='pantsbuild.pants.testutil', | ||
description='Test support for writing Pants plugins.', | ||
namespace_packages=['pants.testutil'], | ||
additional_classifiers=[ | ||
'Topic :: Software Development :: Testing', | ||
] | ||
) | ||
) | ||
|
||
python_library( | ||
name = 'int-test-for-export', | ||
sources = [ | ||
'pants_run_integration_test.py' | ||
], | ||
dependencies = [ | ||
'//:build_root', | ||
'//:pants_pex', | ||
':file_test_util', | ||
'3rdparty/python:ansicolors', | ||
'3rdparty/python:dataclasses', | ||
'src/python/pants/base:build_environment', | ||
'src/python/pants/base:build_file', | ||
'src/python/pants/base:exiter', | ||
'src/python/pants/fs', | ||
'src/python/pants/subsystem', | ||
'src/python/pants/util:contextutil', | ||
'src/python/pants/util:dirutil', | ||
'src/python/pants/util:osutil', | ||
'src/python/pants/util:process_handler', | ||
'src/python/pants/util:strutil', | ||
'src/python/pants:entry_point', | ||
] | ||
) | ||
|
||
target( | ||
name = 'int-test', | ||
dependencies=[ | ||
':int-test-for-export', | ||
# NB: 'pants_run_integration_test.py' runs ./pants in a subprocess, so test results will depend | ||
# on the pants binary and all of its transitive dependencies. Adding the dependencies below is | ||
# our best proxy for ensuring that any test target depending on this target will be invalidated | ||
# on changes to those undeclared dependencies. | ||
'src/python/pants/bin:pants_local_binary', | ||
'src/rust/engine', | ||
'//:pyproject', | ||
], | ||
) | ||
|
||
|
||
python_library( | ||
name = 'test_base', | ||
sources = ['test_base.py'], | ||
dependencies = [ | ||
'src/python/pants/base:build_root', | ||
'src/python/pants/base:cmd_line_spec_parser', | ||
'src/python/pants/base:exceptions', | ||
'src/python/pants/build_graph', | ||
'src/python/pants/init', | ||
'src/python/pants/source', | ||
'src/python/pants/subsystem', | ||
'src/python/pants/task', | ||
'src/python/pants/testutil/base:context_utils', | ||
'src/python/pants/testutil/engine:util', | ||
'src/python/pants/testutil/option', | ||
'src/python/pants/testutil/subsystem', | ||
'src/python/pants/util:collections', | ||
'src/python/pants/util:contextutil', | ||
'src/python/pants/util:dirutil', | ||
'src/python/pants/util:memo', | ||
'src/python/pants/util:meta', | ||
] | ||
) | ||
|
||
python_library( | ||
name = 'console_rule_test_base', | ||
sources = ['console_rule_test_base.py'], | ||
dependencies = [ | ||
':test_base', | ||
'src/python/pants/bin', | ||
'src/python/pants/init', | ||
'src/python/pants/util:meta', | ||
] | ||
) | ||
|
||
python_library( | ||
name = 'task_test_base', | ||
sources = ['task_test_base.py'], | ||
dependencies = [ | ||
'src/python/pants/goal:context', | ||
'src/python/pants/ivy', | ||
'src/python/pants/task', | ||
'src/python/pants/util:contextutil', | ||
'src/python/pants/util:meta', | ||
'src/python/pants/util:memo', | ||
'src/python/pants/util:objects', | ||
':test_base', | ||
] | ||
) | ||
|
||
python_library( | ||
name='file_test_util', | ||
sources=['file_test_util.py'], | ||
) | ||
|
||
python_library( | ||
name='git_util', | ||
sources=['git_util.py'], | ||
dependencies = [ | ||
'src/python/pants/base:revision', | ||
'src/python/pants/scm:git', | ||
'src/python/pants/util:contextutil', | ||
], | ||
) | ||
|
||
python_library( | ||
name='interpreter_selection_utils', | ||
sources=['interpreter_selection_utils.py'], | ||
) | ||
|
||
python_library( | ||
name='mock_logger', | ||
sources=['mock_logger.py'], | ||
dependencies = [ | ||
'src/python/pants/reporting', | ||
], | ||
) | ||
|
||
python_library( | ||
name='pexrc_util', | ||
sources=['pexrc_util.py'], | ||
dependencies = [ | ||
':git_util', | ||
], | ||
) | ||
|
||
python_library( | ||
name='process_test_util', | ||
sources=['process_test_util.py'], | ||
dependencies = [ | ||
'3rdparty/python:dataclasses', | ||
'3rdparty/python:psutil', | ||
], | ||
) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_library( | ||
name = 'context_utils', | ||
sources = ['context_utils.py'], | ||
dependencies = [ | ||
'3rdparty/python/twitter/commons:twitter.common.collections', | ||
'src/python/pants/base:workunit', | ||
'src/python/pants/build_graph', | ||
'src/python/pants/goal:context', | ||
'src/python/pants/goal:run_tracker', | ||
] | ||
) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,131 @@ | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
import logging | ||
import sys | ||
from contextlib import contextmanager | ||
|
||
from twitter.common.collections import maybe_list | ||
|
||
from pants.base.workunit import WorkUnit | ||
from pants.build_graph.target import Target | ||
from pants.goal.context import Context | ||
from pants.goal.run_tracker import RunTrackerLogger | ||
|
||
|
||
class TestContext(Context): | ||
"""A Context to use during unittesting. | ||
:API: public | ||
Stubs out various dependencies that we don't want to introduce in unit tests. | ||
TODO: Instead of extending the runtime Context class, create a Context interface and have | ||
TestContext and a runtime Context implementation extend that. This will also allow us to | ||
isolate the parts of the interface that a Task is allowed to use vs. the parts that the | ||
task-running machinery is allowed to use. | ||
""" | ||
class DummyWorkUnit: | ||
"""A workunit stand-in that sends all output to stderr. | ||
These outputs are typically only used by subprocesses spawned by code under test, not | ||
the code under test itself, and would otherwise go into some reporting black hole. The | ||
testing framework will only display the stderr output when a test fails. | ||
Provides no other tracking/labeling/reporting functionality. Does not require "opening" | ||
or "closing". | ||
""" | ||
|
||
def output(self, name): | ||
return sys.stderr | ||
|
||
def set_outcome(self, outcome): | ||
return sys.stderr.write('\nWorkUnit outcome: {}\n'.format(WorkUnit.outcome_string(outcome))) | ||
|
||
class DummyRunTracker: | ||
"""A runtracker stand-in that does no actual tracking.""" | ||
|
||
def __init__(self): | ||
self.logger = RunTrackerLogger(self) | ||
|
||
class DummyArtifactCacheStats: | ||
def add_hits(self, cache_name, targets): pass | ||
|
||
def add_misses(self, cache_name, targets, causes): pass | ||
|
||
artifact_cache_stats = DummyArtifactCacheStats() | ||
|
||
def report_target_info(self, scope, target, keys, val): pass | ||
|
||
|
||
class TestLogger(logging.getLoggerClass()): | ||
"""A logger that converts our structured records into flat ones. | ||
This is so we can use a regular logger in tests instead of our reporting machinery. | ||
""" | ||
|
||
def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, *pos_args, **kwargs): | ||
# Python 2 and Python 3 have different arguments for makeRecord(). | ||
# For cross-compatibility, we are unpacking arguments. | ||
# See https://stackoverflow.com/questions/44329421/logging-makerecord-takes-8-positional-arguments-but-11-were-given. | ||
msg = ''.join([msg] + [a[0] if isinstance(a, (list, tuple)) else a for a in args]) | ||
args = [] | ||
return super(TestContext.TestLogger, self).makeRecord( | ||
name, lvl, fn, lno, msg, args, exc_info, *pos_args, **kwargs) | ||
|
||
def __init__(self, *args, **kwargs): | ||
super().__init__(*args, **kwargs) | ||
logger_cls = logging.getLoggerClass() | ||
try: | ||
logging.setLoggerClass(self.TestLogger) | ||
self._logger = logging.getLogger('test') | ||
finally: | ||
logging.setLoggerClass(logger_cls) | ||
|
||
@contextmanager | ||
def new_workunit(self, name, labels=None, cmd='', log_config=None): | ||
""" | ||
:API: public | ||
""" | ||
sys.stderr.write('\nStarting workunit {}\n'.format(name)) | ||
yield TestContext.DummyWorkUnit() | ||
|
||
@property | ||
def log(self): | ||
""" | ||
:API: public | ||
""" | ||
return self._logger | ||
|
||
def submit_background_work_chain(self, work_chain, parent_workunit_name=None): | ||
""" | ||
:API: public | ||
""" | ||
# Just do the work synchronously, so we don't need a run tracker, background workers and so on. | ||
for work in work_chain: | ||
for args_tuple in work.args_tuples: | ||
work.func(*args_tuple) | ||
|
||
def subproc_map(self, f, items): | ||
""" | ||
:API: public | ||
""" | ||
# Just execute in-process. | ||
return list(map(f, items)) | ||
|
||
|
||
def create_context_from_options(options, target_roots=None, build_graph=None, | ||
build_configuration=None, address_mapper=None, | ||
console_outstream=None, workspace=None, scheduler=None): | ||
"""Creates a ``Context`` with the given options and no targets by default. | ||
:param options: An :class:`pants.option.options.Option`-alike object that supports read methods. | ||
Other params are as for ``Context``. | ||
""" | ||
run_tracker = TestContext.DummyRunTracker() | ||
target_roots = maybe_list(target_roots, Target) if target_roots else [] | ||
return TestContext(options=options, run_tracker=run_tracker, target_roots=target_roots, | ||
build_graph=build_graph, build_configuration=build_configuration, | ||
address_mapper=address_mapper, console_outstream=console_outstream, | ||
workspace=workspace, scheduler=scheduler) |
Oops, something went wrong.