From bc0536c86c41219d5f3aa1f19fea04e7c2b777ef Mon Sep 17 00:00:00 2001 From: Stu Hood Date: Thu, 14 Feb 2019 13:59:26 -0800 Subject: [PATCH] Remove deprecated test classes (#7243) ### Problem `BaseTest` and the v1-aware `TaskTestBase` are long deprecated. Additionally, the `GraphTest` classes used v2 APIs that existed before `TestBase` came around. ### Solution Delete deprecated classes, and port `GraphTest` to `TestBase`. --- .../testproject/buildfile_path/BUILD | 3 - .../org/pantsbuild/testproject/ordering/BUILD | 11 - .../org/pantsbuild/testproject/ordering/a | 0 .../org/pantsbuild/testproject/ordering/b | 0 .../org/pantsbuild/testproject/ordering/d | 0 .../org/pantsbuild/testproject/ordering/i | 0 .../org/pantsbuild/testproject/ordering/l | 0 .../org/pantsbuild/testproject/ordering/n | 0 .../org/pantsbuild/testproject/ordering/p | 0 .../org/pantsbuild/testproject/ordering/s | 0 .../org/pantsbuild/testproject/ordering/t | 0 .../org/pantsbuild/testproject/ordering/u | 0 tests/python/pants_test/BUILD | 25 +- .../pants_test/backend/python/tasks/BUILD | 1 - .../backend/python/tasks/native/BUILD | 1 - tests/python/pants_test/base_test.py | 504 ------------------ tests/python/pants_test/engine/legacy/BUILD | 1 + .../pants_test/engine/legacy/test_graph.py | 186 ++----- .../engine/legacy/test_graph_integration.py | 8 + tests/python/pants_test/tasks/BUILD | 14 - .../python/pants_test/tasks/task_test_base.py | 305 ----------- tests/python/pants_test/test_base.py | 14 +- 22 files changed, 52 insertions(+), 1021 deletions(-) delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/BUILD delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/a delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/b delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/d delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/i delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/l delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/n delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/p delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/s delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/t delete mode 100644 testprojects/src/resources/org/pantsbuild/testproject/ordering/u delete mode 100644 tests/python/pants_test/base_test.py delete mode 100644 tests/python/pants_test/tasks/task_test_base.py diff --git a/testprojects/src/resources/org/pantsbuild/testproject/buildfile_path/BUILD b/testprojects/src/resources/org/pantsbuild/testproject/buildfile_path/BUILD index 861ff069618..7e66776349a 100644 --- a/testprojects/src/resources/org/pantsbuild/testproject/buildfile_path/BUILD +++ b/testprojects/src/resources/org/pantsbuild/testproject/buildfile_path/BUILD @@ -1,6 +1,3 @@ target( - dependencies=[ - 'testprojects/src/resources/org/pantsbuild/testproject/ordering:literal' - ], description='''This target exists at path {}.'''.format(buildfile_path()), ) diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/BUILD b/testprojects/src/resources/org/pantsbuild/testproject/ordering/BUILD deleted file mode 100644 index af2b0e8d07f..00000000000 --- a/testprojects/src/resources/org/pantsbuild/testproject/ordering/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -SOURCES=['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd', 'p', 'a', 'n', 't', 's'] - -resources( - name='literal', - sources=SOURCES, -) - -resources( - name='globs', - sources=globs(*SOURCES), -) diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/a b/testprojects/src/resources/org/pantsbuild/testproject/ordering/a deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/b b/testprojects/src/resources/org/pantsbuild/testproject/ordering/b deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/d b/testprojects/src/resources/org/pantsbuild/testproject/ordering/d deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/i b/testprojects/src/resources/org/pantsbuild/testproject/ordering/i deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/l b/testprojects/src/resources/org/pantsbuild/testproject/ordering/l deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/n b/testprojects/src/resources/org/pantsbuild/testproject/ordering/n deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/p b/testprojects/src/resources/org/pantsbuild/testproject/ordering/p deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/s b/testprojects/src/resources/org/pantsbuild/testproject/ordering/s deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/t b/testprojects/src/resources/org/pantsbuild/testproject/ordering/t deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testprojects/src/resources/org/pantsbuild/testproject/ordering/u b/testprojects/src/resources/org/pantsbuild/testproject/ordering/u deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/python/pants_test/BUILD b/tests/python/pants_test/BUILD index fb9a4961b25..af7bd0b8845 100644 --- a/tests/python/pants_test/BUILD +++ b/tests/python/pants_test/BUILD @@ -5,7 +5,6 @@ python_library( name='test_infra', dependencies=[ '3rdparty/python:future', - ':base_test', 'tests/python/pants_test:int-test-for-export', 'tests/python/pants_test:test_base', 'tests/python/pants_test/jvm:jar_task_test_base', @@ -24,28 +23,6 @@ python_library( ) ) -python_library( - name = 'base_test', - sources = ['base_test.py'], - dependencies = [ - '3rdparty/python:future', - 'src/python/pants/base:build_file', - 'src/python/pants/base:build_root', - 'src/python/pants/base:cmd_line_spec_parser', - 'src/python/pants/base:deprecated', - 'src/python/pants/base:exceptions', - 'src/python/pants/base:project_tree', - 'src/python/pants/build_graph', - 'src/python/pants/init', - 'src/python/pants/source', - 'src/python/pants/subsystem', - 'src/python/pants/task', - 'src/python/pants/util:dirutil', - 'tests/python/pants_test/base:context_utils', - 'tests/python/pants_test/option/util', - ] -) - python_library( name = 'int-test-for-export', sources = [ @@ -119,7 +96,7 @@ python_tests( name = 'test_maven_layout', sources = ['test_maven_layout.py'], dependencies = [ - ':base_test', + ':test_base', 'src/python/pants/backend/jvm/subsystems:junit', 'src/python/pants/build_graph', 'src/python/pants/source', diff --git a/tests/python/pants_test/backend/python/tasks/BUILD b/tests/python/pants_test/backend/python/tasks/BUILD index 072b3380e38..cf8ef3dcd88 100644 --- a/tests/python/pants_test/backend/python/tasks/BUILD +++ b/tests/python/pants_test/backend/python/tasks/BUILD @@ -54,7 +54,6 @@ python_tests( 'tests/python/pants_test/backend/python/tasks/util', 'tests/python/pants_test/engine:scheduler_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', - 'tests/python/pants_test/tasks:task_test_base', 'tests/python/pants_test:task_test_base', ], timeout=600 diff --git a/tests/python/pants_test/backend/python/tasks/native/BUILD b/tests/python/pants_test/backend/python/tasks/native/BUILD index 7bbccd25921..278a8f16ecd 100644 --- a/tests/python/pants_test/backend/python/tasks/native/BUILD +++ b/tests/python/pants_test/backend/python/tasks/native/BUILD @@ -33,7 +33,6 @@ python_tests( 'tests/python/pants_test/backend/python/tasks/util', 'tests/python/pants_test/engine:scheduler_test_base', 'tests/python/pants_test/subsystem:subsystem_utils', - 'tests/python/pants_test/tasks:task_test_base', 'tests/python/pants_test:task_test_base', ], tags={'platform_specific_behavior'}, diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py deleted file mode 100644 index 4fef0bc5cee..00000000000 --- a/tests/python/pants_test/base_test.py +++ /dev/null @@ -1,504 +0,0 @@ -# coding=utf-8 -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import itertools -import logging -import os -import unittest -from builtins import object, open -from collections import defaultdict -from contextlib import contextmanager -from tempfile import mkdtemp -from textwrap import dedent - -from future.utils import PY2 - -from pants.base.build_file import BuildFile -from pants.base.build_root import BuildRoot -from pants.base.cmd_line_spec_parser import CmdLineSpecParser -from pants.base.deprecated import deprecated_module -from pants.base.exceptions import TaskError -from pants.base.file_system_project_tree import FileSystemProjectTree -from pants.build_graph.address import Address -from pants.build_graph.build_configuration import BuildConfiguration -from pants.build_graph.build_file_address_mapper import BuildFileAddressMapper -from pants.build_graph.build_file_aliases import BuildFileAliases -from pants.build_graph.build_file_parser import BuildFileParser -from pants.build_graph.mutable_build_graph import MutableBuildGraph -from pants.build_graph.target import Target -from pants.init.util import clean_global_runtime_state -from pants.option.options_bootstrapper import OptionsBootstrapper -from pants.option.scope import GLOBAL_SCOPE -from pants.source.source_root import SourceRootConfig -from pants.subsystem.subsystem import Subsystem -from pants.task.goal_options_mixin import GoalOptionsMixin -from pants.util.dirutil import safe_mkdir, safe_open, safe_rmtree -from pants_test.base.context_utils import create_context_from_options -from pants_test.option.util.fakes import create_options_for_optionables - - -# Fix this during a dev release -deprecated_module('1.13.0.dev2', 'Use pants_test.test_base instead') - - -class TestGenerator(object): - """A mixin that facilitates test generation at runtime.""" - - @classmethod - def generate_tests(cls): - """Generate tests for a given class. - - This should be called against the composing class in it's defining module, e.g. - - class ThingTest(TestGenerator): - ... - - ThingTest.generate_tests() - - """ - raise NotImplementedError() - - @classmethod - def add_test(cls, method_name, method): - """A classmethod that adds dynamic test methods to a given class. - - :param string method_name: The name of the test method (e.g. `test_thing_x`). - :param callable method: A callable representing the method. This should take a 'self' argument - as its first parameter for instance method binding. - """ - assert not hasattr(cls, method_name), ( - 'a test with name `{}` already exists on `{}`!'.format(method_name, cls.__name__) - ) - assert method_name.startswith('test_'), '{} is not a valid test name!'.format(method_name) - setattr(cls, method_name, method) - - -class BaseTest(unittest.TestCase): - """A baseclass useful for tests requiring a temporary buildroot. - - :API: public - - """ - - def build_path(self, relpath): - """Returns the canonical BUILD file path for the given relative build path. - - :API: public - """ - if os.path.basename(relpath).startswith('BUILD'): - return relpath - else: - return os.path.join(relpath, 'BUILD') - - def create_dir(self, relpath): - """Creates a directory under the buildroot. - - :API: public - - relpath: The relative path to the directory from the build root. - """ - path = os.path.join(self.build_root, relpath) - safe_mkdir(path) - return path - - def create_workdir_dir(self, relpath): - """Creates a directory under the work directory. - - :API: public - - relpath: The relative path to the directory from the work directory. - """ - path = os.path.join(self.pants_workdir, relpath) - safe_mkdir(path) - return path - - def create_file(self, relpath, contents='', mode='w'): - """Writes to a file under the buildroot. - - :API: public - - relpath: The relative path to the file from the build root. - contents: A string containing the contents of the file - '' by default.. - mode: The mode to write to the file in - over-write by default. - """ - path = os.path.join(self.build_root, relpath) - with safe_open(path, mode=mode) as fp: - fp.write(contents) - return path - - def create_workdir_file(self, relpath, contents='', mode='w'): - """Writes to a file under the work directory. - - :API: public - - relpath: The relative path to the file from the work directory. - contents: A string containing the contents of the file - '' by default.. - mode: The mode to write to the file in - over-write by default. - """ - path = os.path.join(self.pants_workdir, relpath) - with safe_open(path, mode=mode) as fp: - fp.write(contents) - return path - - def add_to_build_file(self, relpath, target): - """Adds the given target specification to the BUILD file at relpath. - - :API: public - - relpath: The relative path to the BUILD file from the build root. - target: A string containing the target definition as it would appear in a BUILD file. - """ - self.create_file(self.build_path(relpath), target, mode='a') - return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) - - def make_target(self, - spec='', - target_type=Target, - dependencies=None, - derived_from=None, - synthetic=False, - **kwargs): - """Creates a target and injects it into the test's build graph. - - :API: public - - :param string spec: The target address spec that locates this target. - :param type target_type: The concrete target subclass to create this new target from. - :param list dependencies: A list of target instances this new target depends on. - :param derived_from: The target this new target was derived from. - :type derived_from: :class:`pants.build_graph.target.Target` - """ - address = Address.parse(spec) - target = target_type(name=address.target_name, - address=address, - build_graph=self.build_graph, - **kwargs) - dependencies = dependencies or [] - - self.build_graph.apply_injectables([target]) - self.build_graph.inject_target(target, - dependencies=[dep.address for dep in dependencies], - derived_from=derived_from, - synthetic=synthetic) - - # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. - # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. - traversables = [target.compute_dependency_specs(payload=target.payload)] - - for dependency_spec in itertools.chain(*traversables): - dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path) - dependency_target = self.build_graph.get_target(dependency_address) - if not dependency_target: - raise ValueError('Tests must make targets for dependency specs ahead of them ' - 'being traversed, {} tried to traverse {} which does not exist.' - .format(target, dependency_address)) - if dependency_target not in target.dependencies: - self.build_graph.inject_dependency(dependent=target.address, - dependency=dependency_address) - target.mark_transitive_invalidation_hash_dirty() - - return target - - @property - def alias_groups(self): - """ - :API: public - """ - return BuildFileAliases(targets={'target': Target}) - - @property - def build_ignore_patterns(self): - """ - :API: public - """ - return None - - def setUp(self): - """ - :API: public - """ - super(BaseTest, self).setUp() - # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. - clean_global_runtime_state(reset_subsystem=True) - - self.real_build_root = BuildRoot().path - - self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) - self.subprocess_dir = os.path.join(self.build_root, '.pids') - self.addCleanup(safe_rmtree, self.build_root) - - self.pants_workdir = os.path.join(self.build_root, '.pants.d') - safe_mkdir(self.pants_workdir) - - self.options = defaultdict(dict) # scope -> key-value mapping. - self.options[GLOBAL_SCOPE] = { - 'pants_workdir': self.pants_workdir, - 'pants_supportdir': os.path.join(self.build_root, 'build-support'), - 'pants_distdir': os.path.join(self.build_root, 'dist'), - 'pants_configdir': os.path.join(self.build_root, 'config'), - 'pants_subprocessdir': self.subprocess_dir, - 'cache_key_gen_version': '0-test', - } - self.options['cache'] = { - 'read_from': [], - 'write_to': [], - } - - BuildRoot().path = self.build_root - self.addCleanup(BuildRoot().reset) - - self._build_configuration = BuildConfiguration() - self._build_configuration.register_aliases(self.alias_groups) - self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) - self.project_tree = FileSystemProjectTree(self.build_root) - self.reset_build_graph() - - def buildroot_files(self, relpath=None): - """Returns the set of all files under the test build root. - - :API: public - - :param string relpath: If supplied, only collect files from this subtree. - :returns: All file paths found. - :rtype: set - """ - def scan(): - for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')): - for f in files: - yield os.path.relpath(os.path.join(root, f), self.build_root) - return set(scan()) - - def reset_build_graph(self): - """Start over with a fresh build graph with no targets in it.""" - self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, - build_ignore_patterns=self.build_ignore_patterns) - self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper) - - def set_options_for_scope(self, scope, **kwargs): - self.options[scope].update(kwargs) - - def context(self, for_task_types=None, for_subsystems=None, options=None, - target_roots=None, console_outstream=None, workspace=None, - scheduler=None, **kwargs): - """ - :API: public - - :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`. - """ - # Many tests use source root functionality via the SourceRootConfig.global_instance(). - # (typically accessed via Target.target_base), so we always set it up, for convenience. - for_subsystems = set(for_subsystems or ()) - for subsystem in for_subsystems: - if subsystem.options_scope is None: - raise TaskError('You must set a scope on your subsystem type before using it in tests.') - - optionables = {SourceRootConfig} | self._build_configuration.optionables() | for_subsystems - - for_task_types = for_task_types or () - for task_type in for_task_types: - scope = task_type.options_scope - if scope is None: - raise TaskError('You must set a scope on your task type before using it in tests.') - optionables.add(task_type) - # If task is expected to inherit goal-level options, register those directly on the task, - # by subclassing the goal options registrar and settings its scope to the task scope. - if issubclass(task_type, GoalOptionsMixin): - subclass_name = 'test_{}_{}_{}'.format( - task_type.__name__, task_type.goal_options_registrar_cls.options_scope, - task_type.options_scope) - if PY2: - subclass_name = subclass_name.encode('utf-8') - optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ), - {'options_scope': task_type.options_scope})) - - # Now expand to all deps. - all_optionables = set() - for optionable in optionables: - all_optionables.update(si.optionable_cls for si in optionable.known_scope_infos()) - - # Now default the option values and override with any caller-specified values. - # TODO(benjy): Get rid of the options arg, and require tests to call set_options. - options = options.copy() if options else {} - for s, opts in self.options.items(): - scoped_opts = options.setdefault(s, {}) - scoped_opts.update(opts) - - fake_options = create_options_for_optionables( - all_optionables, options=options, **kwargs) - - Subsystem.reset(reset_options=True) - Subsystem.set_options(fake_options) - - context = create_context_from_options(fake_options, - target_roots=target_roots, - build_graph=self.build_graph, - build_file_parser=self.build_file_parser, - address_mapper=self.address_mapper, - console_outstream=console_outstream, - workspace=workspace, - scheduler=scheduler) - return context - - def tearDown(self): - """ - :API: public - """ - super(BaseTest, self).tearDown() - BuildFile.clear_cache() - Subsystem.reset() - - def target(self, spec): - """Resolves the given target address to a Target object. - - :API: public - - address: The BUILD target address to resolve. - - Returns the corresponding Target or else None if the address does not point to a defined Target. - """ - address = Address.parse(spec) - self.build_graph.inject_address_closure(address) - return self.build_graph.get_target(address) - - def targets(self, spec): - """Resolves a target spec to one or more Target objects. - - :API: public - - spec: Either BUILD target address or else a target glob using the siblings ':' or - descendants '::' suffixes. - - Returns the set of all Targets found. - """ - - spec = CmdLineSpecParser(self.build_root).parse_spec(spec) - addresses = list(self.address_mapper.scan_specs([spec])) - for address in addresses: - self.build_graph.inject_address_closure(address) - targets = [self.build_graph.get_target(address) for address in addresses] - return targets - - def create_files(self, path, files): - """Writes to a file under the buildroot with contents same as file name. - - :API: public - - path: The relative path to the file from the build root. - files: List of file names. - """ - for f in files: - self.create_file(os.path.join(path, f), contents=f) - - def create_library(self, path, target_type, name, sources=None, **kwargs): - """Creates a library target of given type at the BUILD file at path with sources - - :API: public - - path: The relative path to the BUILD file from the build root. - target_type: valid pants target type. - name: Name of the library target. - sources: List of source file at the path relative to path. - **kwargs: Optional attributes that can be set for any library target. - Currently it includes support for resources, java_sources, provides - and dependencies. - """ - if sources: - self.create_files(path, sources) - self.add_to_build_file(path, dedent(''' - %(target_type)s(name='%(name)s', - %(sources)s - %(java_sources)s - %(provides)s - %(dependencies)s - ) - ''' % dict(target_type=target_type, - name=name, - sources=('sources=%s,' % repr(sources) - if sources else ''), - java_sources=('java_sources=[%s],' - % ','.join('"%s"' % str_target for str_target in kwargs.get('java_sources')) - if 'java_sources' in kwargs else ''), - provides=('provides=%s,' % kwargs.get('provides') - if 'provides' in kwargs else ''), - dependencies=('dependencies=%s,' % kwargs.get('dependencies') - if 'dependencies' in kwargs else ''), - ))) - return self.target('%s:%s' % (path, name)) - - def create_resources(self, path, name, *sources): - """ - :API: public - """ - return self.create_library(path, 'resources', name, sources) - - def assertUnorderedPrefixEqual(self, expected, actual_iter): - """Consumes len(expected) items from the given iter, and asserts that they match, unordered. - - :API: public - """ - actual = list(itertools.islice(actual_iter, len(expected))) - self.assertEqual(sorted(expected), sorted(actual)) - - def assertPrefixEqual(self, expected, actual_iter): - """Consumes len(expected) items from the given iter, and asserts that they match, in order. - - :API: public - """ - self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) - - def assertInFile(self, string, file_path): - """Verifies that a string appears in a file - - :API: public - """ - - with open(file_path, 'r') as f: - content = f.read() - self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content)) - - def get_bootstrap_options(self, cli_options=()): - """Retrieves bootstrap options. - - :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. - """ - # Can't parse any options without a pants.ini. - self.create_file('pants.ini') - return OptionsBootstrapper.create(args=cli_options).bootstrap_options.for_global_scope() - - class LoggingRecorder(object): - """Simple logging handler to record warnings.""" - - def __init__(self): - self._records = [] - self.level = logging.DEBUG - - def handle(self, record): - self._records.append(record) - - def _messages_for_level(self, levelname): - return ['{}: {}'.format(record.name, record.getMessage()) - for record in self._records if record.levelname == levelname] - - def infos(self): - return self._messages_for_level('INFO') - - def warnings(self): - return self._messages_for_level('WARNING') - - @contextmanager - def captured_logging(self, level=None): - root_logger = logging.getLogger() - - old_level = root_logger.level - root_logger.setLevel(level or logging.NOTSET) - - handler = self.LoggingRecorder() - root_logger.addHandler(handler) - try: - yield handler - finally: - root_logger.setLevel(old_level) - root_logger.removeHandler(handler) diff --git a/tests/python/pants_test/engine/legacy/BUILD b/tests/python/pants_test/engine/legacy/BUILD index 367f31219c2..308bd57441a 100644 --- a/tests/python/pants_test/engine/legacy/BUILD +++ b/tests/python/pants_test/engine/legacy/BUILD @@ -129,6 +129,7 @@ python_tests( 'src/python/pants/engine/legacy:graph', 'src/python/pants/init', 'tests/python/pants_test/engine:util', + 'tests/python/pants_test:test_base' ] ) diff --git a/tests/python/pants_test/engine/legacy/test_graph.py b/tests/python/pants_test/engine/legacy/test_graph.py index 18fb8e997e6..959612a9d63 100644 --- a/tests/python/pants_test/engine/legacy/test_graph.py +++ b/tests/python/pants_test/engine/legacy/test_graph.py @@ -6,23 +6,12 @@ import functools import os -import unittest from builtins import str -from contextlib import contextmanager -import mock - -from pants.build_graph.address import Address from pants.build_graph.address_lookup_error import AddressLookupError from pants.build_graph.build_file_aliases import BuildFileAliases, TargetMacro -from pants.build_graph.target import Target -from pants.init.engine_initializer import EngineInitializer -from pants.init.options_initializer import BuildConfigInitializer -from pants.init.target_roots_calculator import TargetRootsCalculator -from pants.option.options_bootstrapper import OptionsBootstrapper -from pants.subsystem.subsystem import Subsystem -from pants.util.contextutil import temporary_dir -from pants_test.engine.util import init_native +from pants.build_graph.files import Files +from pants_test.test_base import TestBase # Macro that adds the specified tag. @@ -32,164 +21,59 @@ def macro(target_cls, tag, parse_context, tags=None, **kwargs): parse_context.create_object(target_cls, tags=tags, **kwargs) -class GraphTestBase(unittest.TestCase): - - _native = init_native() - - def _make_setup_args(self, specs): - options = mock.Mock(target_specs=specs) - options.for_scope.return_value = mock.Mock(diffspec=None, changes_since=None) - options.for_global_scope.return_value = mock.Mock(owner_of=None) - return options - - def _default_build_config(self, options_bootstrapper, build_file_aliases=None): - # TODO: Get default BuildFileAliases by extending BaseTest post - # https://github.com/pantsbuild/pants/issues/4401 - build_config = BuildConfigInitializer.get(options_bootstrapper) - if build_file_aliases: - build_config.register_aliases(build_file_aliases) - return build_config - - @contextmanager - def graph_helper(self, - build_configuration=None, - build_file_imports_behavior='allow', - include_trace_on_error=True, - path_ignore_patterns=None): - - with temporary_dir() as work_dir: - with temporary_dir() as local_store_dir: - path_ignore_patterns = path_ignore_patterns or [] - options_bootstrapper = OptionsBootstrapper.create() - build_config = build_configuration or self._default_build_config(options_bootstrapper) - # TODO: This test should be swapped to using TestBase. - graph_helper = EngineInitializer.setup_legacy_graph_extended( - path_ignore_patterns, - work_dir, - local_store_dir, - build_file_imports_behavior, - options_bootstrapper=options_bootstrapper, - build_configuration=build_config, - native=self._native, - include_trace_on_error=include_trace_on_error - ) - yield graph_helper - - @contextmanager - def open_scheduler(self, specs, build_configuration=None): - with self.graph_helper(build_configuration=build_configuration) as graph_helper: - graph, target_roots = self.create_graph_from_specs(graph_helper, specs) - addresses = tuple(graph.inject_roots_closure(target_roots)) - yield graph, addresses, graph_helper.scheduler.new_session() - - def create_graph_from_specs(self, graph_helper, specs): - Subsystem.reset() - session = graph_helper.new_session() - target_roots = self.create_target_roots(specs, session, session.symbol_table) - graph = session.create_build_graph(target_roots)[0] - return graph, target_roots - - def create_target_roots(self, specs, session, symbol_table): - return TargetRootsCalculator.create(self._make_setup_args(specs), session, symbol_table) - - -class GraphTargetScanFailureTests(GraphTestBase): +class GraphTest(TestBase): + + _TAG = 'tag_added_by_macro' + + @classmethod + def alias_groups(cls): + return super(GraphTest, cls).alias_groups().merge( + BuildFileAliases(targets={ + 'files': Files, + 'tagged_files': TargetMacro.Factory.wrap(functools.partial(macro, Files, cls._TAG), Files), + })) def test_with_missing_target_in_existing_build_file(self): + self.create_library('3rdparty/python', 'target', 'Markdown') + self.create_library('3rdparty/python', 'target', 'Pygments') # When a target is missing, # the suggestions should be in order # and there should only be one copy of the error if tracing is off. - with self.assertRaises(AddressLookupError) as cm: - with self.graph_helper(include_trace_on_error=False) as graph_helper: - self.create_graph_from_specs(graph_helper, ['3rdparty/python:rutabaga']) - self.fail('Expected an exception.') - - error_message = str(cm.exception) expected_message = '"rutabaga" was not found in namespace "3rdparty/python".' \ - ' Did you mean one of:\n' \ - ' :Markdown\n' \ - ' :Pygments\n' - self.assertIn(expected_message, error_message) - self.assertTrue(error_message.count(expected_message) == 1) + '.*Did you mean one of:\n' \ + '.*:Markdown\n' \ + '.*:Pygments\n' + with self.assertRaisesRegexp(AddressLookupError, expected_message): + self.targets('3rdparty/python:rutabaga') def test_with_missing_directory_fails(self): with self.assertRaises(AddressLookupError) as cm: - with self.graph_helper() as graph_helper: - self.create_graph_from_specs(graph_helper, ['no-such-path:']) + self.targets('no-such-path:') self.assertIn('Path "no-such-path" does not contain any BUILD files', str(cm.exception)) - def test_with_existing_directory_with_no_build_files_fails(self): - with self.assertRaises(AddressLookupError) as cm: - path_ignore_patterns=[ - # This is a symlink that points out of the build root. - '/build-support/bin/native/src' - ] - with self.graph_helper(path_ignore_patterns=path_ignore_patterns) as graph_helper: - self.create_graph_from_specs(graph_helper, ['build-support/bin::']) - - self.assertIn('does not match any targets.', str(cm.exception)) - - def test_inject_bad_dir(self): - with self.assertRaises(AddressLookupError) as cm: - with self.graph_helper() as graph_helper: - graph, target_roots = self.create_graph_from_specs(graph_helper, ['3rdparty/python:']) - - graph.inject_address_closure(Address('build-support/bin', 'wat')) - - self.assertIn('Path "build-support/bin" does not contain any BUILD files', - str(cm.exception)) - - -class GraphInvalidationTest(GraphTestBase): - def test_invalidate_fsnode(self): # NB: Invalidation is now more directly tested in unit tests in the `graph` crate. - with self.open_scheduler(['3rdparty/python::']) as (_, _, scheduler): - invalidated_count = scheduler.invalidate_files(['3rdparty/python/BUILD']) - self.assertGreater(invalidated_count, 0) + self.create_library('src/example', 'target', 'things') + self.targets('src/example::') + invalidated_count = self.invalidate_for('src/example/BUILD') + self.assertGreater(invalidated_count, 0) - def test_invalidate_fsnode_incremental(self): - # NB: Invalidation is now more directly tested in unit tests in the `graph` crate. - with self.open_scheduler(['//:', '3rdparty/::']) as (_, _, scheduler): - # Invalidate the '3rdparty/python' DirectoryListing, the `3rdparty` DirectoryListing, - # and then the root DirectoryListing by "touching" files/dirs. - for filename in ('3rdparty/python/BUILD', '3rdparty/jvm', 'non_existing_file'): - invalidated_count = scheduler.invalidate_files([filename]) - self.assertGreater(invalidated_count, - 0, - 'File {} did not invalidate any Nodes.'.format(filename)) - - def _ordering_test(self, spec, expected_sources=None): - expected_sources = expected_sources or ['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'] - with self.open_scheduler([spec]) as (graph, _, _): - target = graph.get_target(Address.parse(spec)) - sources = [os.path.basename(s) for s in target.sources_relative_to_buildroot()] - self.assertEqual(expected_sources, sources) - - def test_sources_ordering_literal(self): - self._ordering_test('testprojects/src/resources/org/pantsbuild/testproject/ordering:literal') - - def test_sources_ordering_glob(self): - self._ordering_test('testprojects/src/resources/org/pantsbuild/testproject/ordering:globs') + def test_sources_ordering(self): + expected_sources = ['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'] + self.create_library('src/example', 'files', 'things', sources=expected_sources) + + target = self.target('src/example:things') + sources = [os.path.basename(s) for s in target.sources_relative_to_buildroot()] + self.assertEqual(expected_sources, sources) def test_target_macro_override(self): """Tests that we can "wrap" an existing target type with additional functionality. Installs an additional TargetMacro that wraps `target` aliases to add a tag to all definitions. """ - spec = 'testprojects/tests/python/pants/build_parsing:' - - tag = 'tag_added_by_macro' - target_cls = Target - tag_macro = functools.partial(macro, target_cls, tag) - target_symbols = {'target': TargetMacro.Factory.wrap(tag_macro, target_cls)} - - build_config = self._default_build_config(OptionsBootstrapper.create(), BuildFileAliases(targets=target_symbols)) - # Confirm that python_tests in a small directory are marked. - with self.open_scheduler([spec], build_configuration=build_config) as (graph, addresses, _): - self.assertTrue(len(addresses) > 0, 'No targets matched by {}'.format(addresses)) - for address in addresses: - self.assertIn(tag, graph.get_target(address).tags) + files = self.create_library('src/example', 'tagged_files', 'things') + self.assertIn(self._TAG, files.tags) + self.assertEqual(type(files), Files) diff --git a/tests/python/pants_test/engine/legacy/test_graph_integration.py b/tests/python/pants_test/engine/legacy/test_graph_integration.py index 2f9b9b167d3..f517d3953c0 100644 --- a/tests/python/pants_test/engine/legacy/test_graph_integration.py +++ b/tests/python/pants_test/engine/legacy/test_graph_integration.py @@ -138,6 +138,14 @@ def test_existing_bundles(self): self.assert_success(pants_run) self.assertNotIn("WARN]", pants_run.stderr_data) + def test_existing_directory_with_no_build_files_fails(self): + options = [ + '--pants-ignore=+["/build-support/bin/native/src"]', + ] + pants_run = self.run_pants(options + ['list', 'build-support/bin::']) + self.assert_failure(pants_run) + self.assertIn("does not match any targets.", pants_run.stderr_data) + def test_error_message(self): for k in self._ERR_TARGETS: self._list_target_check_error(k) diff --git a/tests/python/pants_test/tasks/BUILD b/tests/python/pants_test/tasks/BUILD index 44cf4c6b7e6..52705df8233 100644 --- a/tests/python/pants_test/tasks/BUILD +++ b/tests/python/pants_test/tasks/BUILD @@ -1,20 +1,6 @@ # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -python_library( - name = 'task_test_base', - sources = ['task_test_base.py'], - dependencies = [ - 'src/python/pants/base:deprecated', - 'src/python/pants/goal:context', - 'src/python/pants/ivy', - 'src/python/pants/task', - 'src/python/pants/util:contextutil', - 'src/python/pants/util:process_handler', - 'tests/python/pants_test:base_test', - ] -) - python_tests( name = 'scalastyle_integration', sources = ['test_scalastyle_integration.py'], diff --git a/tests/python/pants_test/tasks/task_test_base.py b/tests/python/pants_test/tasks/task_test_base.py deleted file mode 100644 index 0a634187aa7..00000000000 --- a/tests/python/pants_test/tasks/task_test_base.py +++ /dev/null @@ -1,305 +0,0 @@ -# coding=utf-8 -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import absolute_import, division, print_function, unicode_literals - -import glob -import os -from contextlib import closing, contextmanager -from io import BytesIO - -from future.utils import PY2 - -from pants.base.deprecated import deprecated_module -from pants.goal.goal import Goal -from pants.ivy.bootstrapper import Bootstrapper -from pants.task.console_task import ConsoleTask -from pants.util.contextutil import temporary_dir -from pants.util.process_handler import subprocess -from pants_test.base_test import BaseTest - - -deprecated_module('1.12.0.dev2', 'Use pants_test.TaskTestBase instead') - - -# TODO: Find a better home for this? -def is_exe(name): - result = subprocess.call(['which', name], stdout=open(os.devnull, 'w'), stderr=subprocess.STDOUT) - return result == 0 - - -def ensure_cached(task_cls, expected_num_artifacts=None): - """Decorator for a task-executing unit test. Asserts that after running the - decorated test function, the cache for task_cls contains - expected_num_artifacts. - - Uses a new temp dir for the artifact cache, and uses a glob based on the - task's synthesized subtype to find the cache directories within the new temp - dir which were generated by the actions performed within the test method. - - :API: public - - :param task_cls: Class of the task to check the artifact cache - for (e.g. JarCreate). - :param expected_num_artifacts: Expected number of artifacts to be in the - task's cache after running the test. If - unspecified, will assert that the number of - artifacts in the cache is non-zero. - """ - def decorator(test_fn): - def wrapper(self, *args, **kwargs): - with self.cache_check(expected_num_artifacts=expected_num_artifacts): - test_fn(self, *args, **kwargs) - return wrapper - return decorator - - -class TaskTestBase(BaseTest): - """A baseclass useful for testing a single Task type. - - :API: public - """ - - options_scope = 'test_scope' - - @classmethod - def task_type(cls): - """Subclasses must return the type of the Task subclass under test. - - :API: public - """ - raise NotImplementedError() - - def setUp(self): - """ - :API: public - """ - super(TaskTestBase, self).setUp() - self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope) - # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot. - # BaseTest cleans this up, so we don't need to. We give it a stable name, so that we can - # use artifact caching to speed up tests. - self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name()) - os.mkdir(self._test_workdir) - # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific - # action in this non-JVM-specific, high-level base class. - Bootstrapper.reset_instance() - - @property - def test_workdir(self): - """ - :API: public - """ - return self._test_workdir - - def synthesize_task_subtype(self, task_type, options_scope): - """Creates a synthetic subclass of the task type. - - Note that passing in a stable options scope will speed up some tests, as the scope may appear - in the paths of tools used by the task, and if these are stable, tests can get artifact - cache hits when bootstrapping these tools. This doesn't hurt test isolation, as we reset - class-level state between each test. - - # TODO: Use the task type directly once we re-do the Task lifecycle. - - :API: public - - :param task_type: The task type to subtype. - :param options_scope: The scope to give options on the generated task type. - :return: A pair (type, options_scope) - """ - subclass_name = 'test_{0}_{1}'.format(task_type.__name__, options_scope) - if PY2: - subclass_name = subclass_name.encode('utf-8') - return type(subclass_name, (task_type,), {'_stable_name': task_type._compute_stable_name(), - 'options_scope': options_scope}) - - def set_options(self, **kwargs): - """ - :API: public - """ - self.set_options_for_scope(self.options_scope, **kwargs) - - def context(self, for_task_types=None, **kwargs): - """ - :API: public - """ - # Add in our task type. - for_task_types = [self._testing_task_type] + (for_task_types or []) - return super(TaskTestBase, self).context(for_task_types=for_task_types, **kwargs) - - def create_task(self, context, workdir=None): - """ - :API: public - """ - if workdir is None: - workdir = self.test_workdir - return self._testing_task_type(context, workdir) - - @contextmanager - def cache_check(self, expected_num_artifacts=None): - """Sets up a temporary artifact cache and checks that the yielded-to code populates it. - - :param expected_num_artifacts: Expected number of artifacts to be in the cache after yielding. - If unspecified, will assert that the number of artifacts in the - cache is non-zero. - """ - with temporary_dir() as artifact_cache: - self.set_options_for_scope('cache.{}'.format(self.options_scope), - write_to=[artifact_cache]) - - yield - - cache_subdir_glob_str = os.path.join(artifact_cache, '*/') - cache_subdirs = glob.glob(cache_subdir_glob_str) - - if expected_num_artifacts == 0: - self.assertEqual(len(cache_subdirs), 0) - return - - self.assertEqual(len(cache_subdirs), 1) - task_cache = cache_subdirs[0] - - num_artifacts = 0 - for (_, _, files) in os.walk(task_cache): - num_artifacts += len(files) - - if expected_num_artifacts is None: - self.assertNotEqual(num_artifacts, 0) - else: - self.assertEqual(num_artifacts, expected_num_artifacts) - - -class ConsoleTaskTestBase(TaskTestBase): - """A base class useful for testing ConsoleTasks. - - :API: public - """ - - def setUp(self): - """ - :API: public - """ - Goal.clear() - super(ConsoleTaskTestBase, self).setUp() - - task_type = self.task_type() - assert issubclass(task_type, ConsoleTask), \ - 'task_type() must return a ConsoleTask subclass, got %s' % task_type - - def execute_task(self, targets=None, options=None): - """Creates a new task and executes it with the given config, command line args and targets. - - :API: public - - :param targets: Optional list of Target objects passed on the command line. - Returns the text output of the task. - """ - options = options or {} - with closing(BytesIO()) as output: - self.set_options(**options) - context = self.context(target_roots=targets, console_outstream=output) - task = self.create_task(context) - task.execute() - return output.getvalue() - - def execute_console_task(self, targets=None, extra_targets=None, options=None, - passthru_args=None, workspace=None, scheduler=None): - """Creates a new task and executes it with the given config, command line args and targets. - - :API: public - - :param options: option values. - :param targets: optional list of Target objects passed on the command line. - :param extra_targets: optional list of extra targets in the context in addition to those - passed on the command line. - :param passthru_args: optional list of passthru_args - :param workspace: optional Workspace to pass into the context. - - Returns the list of items returned from invoking the console task's console_output method. - """ - options = options or {} - self.set_options(**options) - context = self.context( - target_roots=targets, - passthru_args=passthru_args, - workspace=workspace, - scheduler=scheduler - ) - return self.execute_console_task_given_context(context, extra_targets=extra_targets) - - def execute_console_task_given_context(self, context, extra_targets=None): - """Creates a new task and executes it with the context and extra targets. - - :API: public - - :param context: The pants run context to use. - :param extra_targets: An optional list of extra targets in the context in addition to those - passed on the command line. - :returns: The list of items returned from invoking the console task's console_output method. - :rtype: list of strings - """ - task = self.create_task(context) - return list(task.console_output(list(task.context.targets()) + list(extra_targets or ()))) - - def assert_entries(self, sep, *output, **kwargs): - """Verifies the expected output text is flushed by the console task under test. - - NB: order of entries is not tested, just presence. - - :API: public - - sep: the expected output separator. - *output: the output entries expected between the separators - **options: additional options passed to execute_task. - """ - # We expect each output line to be suffixed with the separator, so for , and [1,2,3] we expect: - # '1,2,3,' - splitting this by the separator we should get ['1', '2', '3', ''] - always an extra - # empty string if the separator is properly always a suffix and not applied just between - # entries. - self.assertEqual(sorted(list(output) + ['']), sorted((self.execute_task(**kwargs)).split(sep))) - - def assert_console_output(self, *output, **kwargs): - """Verifies the expected output entries are emitted by the console task under test. - - NB: order of entries is not tested, just presence. - - :API: public - - *output: the expected output entries - **kwargs: additional kwargs passed to execute_console_task. - """ - self.assertEqual(sorted(output), sorted(self.execute_console_task(**kwargs))) - - def assert_console_output_contains(self, output, **kwargs): - """Verifies the expected output string is emitted by the console task under test. - - :API: public - - output: the expected output entry(ies) - **kwargs: additional kwargs passed to execute_console_task. - """ - self.assertIn(output, self.execute_console_task(**kwargs)) - - def assert_console_output_ordered(self, *output, **kwargs): - """Verifies the expected output entries are emitted by the console task under test. - - NB: order of entries is tested. - - :API: public - - *output: the expected output entries in expected order - **kwargs: additional kwargs passed to execute_console_task. - """ - self.assertEqual(list(output), self.execute_console_task(**kwargs)) - - def assert_console_raises(self, exception, **kwargs): - """Verifies the expected exception is raised by the console task under test. - - :API: public - - **kwargs: additional kwargs are passed to execute_console_task. - """ - with self.assertRaises(exception): - self.execute_console_task(**kwargs) diff --git a/tests/python/pants_test/test_base.py b/tests/python/pants_test/test_base.py index 38653ea25b5..abe240977a2 100644 --- a/tests/python/pants_test/test_base.py +++ b/tests/python/pants_test/test_base.py @@ -107,7 +107,7 @@ def create_dir(self, relpath): """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) - self._invalidate_for(relpath) + self.invalidate_for(relpath) return path def create_workdir_dir(self, relpath): @@ -119,10 +119,10 @@ def create_workdir_dir(self, relpath): """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) - self._invalidate_for(relpath) + self.invalidate_for(relpath) return path - def _invalidate_for(self, *relpaths): + def invalidate_for(self, *relpaths): """Invalidates all files from the relpath, recursively up to the root. Many python operations implicitly create parent directories, so we assume that touching a @@ -131,7 +131,7 @@ def _invalidate_for(self, *relpaths): if self._scheduler is None: return files = {f for relpath in relpaths for f in recursive_dirname(relpath)} - self._scheduler.invalidate_files(files) + return self._scheduler.invalidate_files(files) def create_link(self, relsrc, reldst): """Creates a symlink within the buildroot. @@ -144,7 +144,7 @@ def create_link(self, relsrc, reldst): src = os.path.join(self.build_root, relsrc) dst = os.path.join(self.build_root, reldst) relative_symlink(src, dst) - self._invalidate_for(reldst) + self.invalidate_for(reldst) def create_file(self, relpath, contents='', mode='w'): """Writes to a file under the buildroot. @@ -158,7 +158,7 @@ def create_file(self, relpath, contents='', mode='w'): path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) - self._invalidate_for(relpath) + self.invalidate_for(relpath) return path def create_files(self, path, files): @@ -432,7 +432,7 @@ def reset_build_graph(self, reset_build_files=False, delete_build_files=False): if delete_build_files: for f in files: os.remove(os.path.join(self.build_root, f)) - self._invalidate_for(*files) + self.invalidate_for(*files) if self._build_graph is not None: self._build_graph.reset()