Move testinfra code from tests/python/pants_tests to `src/python/pa…

…nts/testutil` (#8400)

### Problem
We are in the process of moving our tests to live within the `src` directory, rather than `tests` (#7489). We should consequently have our core utility code live in `src`.

Further, the current utils are spread out across several folders, which makes discovery more difficult.

### Solution
Create a new `pants/testutil` namespace with all of the files used by the `pants.testinfra` wheel. Specifically, it has this directory structure:

```
src/python/pants/testutil
├── BUILD
├── __init__.py
├── base
│   ├── BUILD
│   ├── __init__.py
│   └── context_utils.py
├── console_rule_test_base.py
├── engine
│   ├── BUILD
│   ├── __init__.py
│   ├── base_engine_test.py
│   └── util.py
├── file_test_util.py
├── git_util.py
├── interpreter_selection_utils.py
├── jvm
│   ├── BUILD
│   ├── __init__.py
│   ├── jar_task_test_base.py
│   ├── jvm_task_test_base.py
│   ├── jvm_tool_task_test_base.py
│   └── nailgun_task_test_base.py
├── mock_logger.py
├── option
│   ├── BUILD
│   ├── __init__.py
│   └── fakes.py
├── pants_run_integration_test.py
├── pexrc_util.py
├── process_test_util.py
├── subsystem
│   ├── BUILD
│   ├── __init__.py
│   └── util.py
├── task_test_base.py
└── test_base.py
```

We can't yet delete the equivalent files in `pants_test` due to a public API, so we instead have those files simply re-export the implementation in `pants/testutils`.

#### Does not move some util code
There are several util files not exposed to the `pantsbuild.testinfra` wheel, like `engine/scheduler_test_base.py`. To reduce the scope of this PR, we keep those there for now. Presumably, they will be able to be moved into `pants/testinfra` without a deprecation cycle.

#### Creates new wheel `pantsbuild.pants.testutil`
We had to create a new package, rather than using `pantsbuild.pants.testinfra`, due to rules about ownership of files (#8400 (comment)).

In a followup, we will deprecate `pantsbuild.pants.testinfra`.

### Result
We can now either use `pants.testutil` or the conventional `pants_test` imports. Both have the same functionality.

In a followup PR, we will deprecate the `pants_test` version.

build-support/bin/release.sh

@@ -98,6 +98,14 @@ function pkg_pants_install_test() {
      == "${version}" ]] || die "Installed version of pants does not match requested version!"
 }
 
+function pkg_testutil_install_test() {
+  local version=$1
+  shift
+  local PIP_ARGS=("$@")
+  pip install "${PIP_ARGS[@]}" "pantsbuild.pants.testutil==${version}" && \
+  python -c "import pants.testutil"
+}
+
 function pkg_testinfra_install_test() {
   local version=$1
   shift

contrib/jax_ws/tests/python/pants_test/contrib/jax_ws/tasks/BUILD

@@ -16,6 +16,7 @@ python_tests(
   sources=['test_jax_ws_gen_integration.py'],
   dependencies=[
     'tests/python/pants_test:int-test',
+    'tests/python/pants_test/testutils:file_test_util',
     'contrib/jax_ws:wsdl_tests_directory',
   ],
   tags={'integration'},

contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/BUILD

@@ -7,7 +7,7 @@ python_tests(
   dependencies=[
     'contrib/scrooge/src/python/pants/contrib/scrooge/tasks:java_thrift_library_fingerprint_strategy',
     'src/python/pants/backend/codegen/thrift/java',
-    'tests/python/pants_test:task_test_base',
+    'tests/python/pants_test:test_base',
   ]
 )
 

src/python/pants/backend/python/subsystems/BUILD

@@ -26,7 +26,7 @@ python_tests(
     'src/python/pants/backend/python/targets',
     'src/python/pants/build_graph',
     'src/python/pants/util:contextutil',
-    'tests/python/pants_test/subsystem',
+    'tests/python/pants_test/subsystem:subsystem_utils',
     'tests/python/pants_test:test_base',
   ],
 )

src/python/pants/releases/packages.py

@@ -82,6 +82,7 @@ def core_packages():
   bdist_wheel_flags = ("--py-limited-api", "cp36")
   return {
     Package("pantsbuild.pants", "//src/python/pants:pants-packaged", bdist_wheel_flags=bdist_wheel_flags),
+    Package("pantsbuild.pants.testutil", "//src/python/pants/testutil:testutil_wheel"),
     Package("pantsbuild.pants.testinfra", "//tests/python/pants_test:test_infra"),
   }
 

src/python/pants/rules/core/BUILD

@@ -20,8 +20,10 @@ python_tests(
   name = "tests",
   dependencies = [
     ':core',
+    'tests/python/pants_test:test_base',
     'tests/python/pants_test:console_rule_test_base',
-    'tests/python/pants_test/engine:util'
+    'tests/python/pants_test/engine:util',
+    'tests/python/pants_test/subsystem:subsystem_utils',
   ]
 )
 

src/python/pants/testutil/BUILD

@@ -0,0 +1,162 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  name='testutil_wheel',
+  dependencies=[
+    ':int-test-for-export',
+    ':test_base',
+    ':file_test_util',
+    'src/python/pants/testutil/base:context_utils',
+    'src/python/pants/testutil/engine:engine_test_base',
+    'src/python/pants/testutil/engine:util',
+    'src/python/pants/testutil/jvm:jar_task_test_base',
+    'src/python/pants/testutil/jvm:nailgun_task_test_base',
+    'src/python/pants/testutil/jvm:jvm_tool_task_test_base',
+    'src/python/pants/testutil/option',
+    'src/python/pants/testutil/subsystem',
+  ],
+  provides=pants_setup_py(
+    name='pantsbuild.pants.testutil',
+    description='Test support for writing Pants plugins.',
+    namespace_packages=['pants.testutil'],
+    additional_classifiers=[
+      'Topic :: Software Development :: Testing',
+    ]
+  )
+)
+
+python_library(
+  name = 'int-test-for-export',
+  sources = [
+    'pants_run_integration_test.py'
+  ],
+  dependencies = [
+    '//:build_root',
+    '//:pants_pex',
+    ':file_test_util',
+    '3rdparty/python:ansicolors',
+    '3rdparty/python:dataclasses',
+    'src/python/pants/base:build_environment',
+    'src/python/pants/base:build_file',
+    'src/python/pants/base:exiter',
+    'src/python/pants/fs',
+    'src/python/pants/subsystem',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/util:dirutil',
+    'src/python/pants/util:osutil',
+    'src/python/pants/util:process_handler',
+    'src/python/pants/util:strutil',
+    'src/python/pants:entry_point',
+  ]
+)
+
+target(
+  name = 'int-test',
+  dependencies=[
+    ':int-test-for-export',
+    # NB: 'pants_run_integration_test.py' runs ./pants in a subprocess, so test results will depend
+    # on the pants binary and all of its transitive dependencies. Adding the dependencies below is
+    # our best proxy for ensuring that any test target depending on this target will be invalidated
+    # on changes to those undeclared dependencies.
+    'src/python/pants/bin:pants_local_binary',
+    'src/rust/engine',
+    '//:pyproject',
+  ],
+)
+
+
+python_library(
+  name = 'test_base',
+  sources = ['test_base.py'],
+  dependencies = [
+    'src/python/pants/base:build_root',
+    'src/python/pants/base:cmd_line_spec_parser',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/build_graph',
+    'src/python/pants/init',
+    'src/python/pants/source',
+    'src/python/pants/subsystem',
+    'src/python/pants/task',
+    'src/python/pants/testutil/base:context_utils',
+    'src/python/pants/testutil/engine:util',
+    'src/python/pants/testutil/option',
+    'src/python/pants/testutil/subsystem',
+    'src/python/pants/util:collections',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/util:dirutil',
+    'src/python/pants/util:memo',
+    'src/python/pants/util:meta',
+  ]
+)
+
+python_library(
+  name = 'console_rule_test_base',
+  sources = ['console_rule_test_base.py'],
+  dependencies = [
+    ':test_base',
+    'src/python/pants/bin',
+    'src/python/pants/init',
+    'src/python/pants/util:meta',
+  ]
+)
+
+python_library(
+  name = 'task_test_base',
+  sources = ['task_test_base.py'],
+  dependencies = [
+    'src/python/pants/goal:context',
+    'src/python/pants/ivy',
+    'src/python/pants/task',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/util:meta',
+    'src/python/pants/util:memo',
+    'src/python/pants/util:objects',
+    ':test_base',
+  ]
+)
+
+python_library(
+  name='file_test_util',
+  sources=['file_test_util.py'],
+)
+
+python_library(
+  name='git_util',
+  sources=['git_util.py'],
+  dependencies = [
+    'src/python/pants/base:revision',
+    'src/python/pants/scm:git',
+    'src/python/pants/util:contextutil',
+  ],
+)
+
+python_library(
+  name='interpreter_selection_utils',
+  sources=['interpreter_selection_utils.py'],
+)
+
+python_library(
+  name='mock_logger',
+  sources=['mock_logger.py'],
+  dependencies = [
+    'src/python/pants/reporting',
+  ],
+)
+
+python_library(
+  name='pexrc_util',
+  sources=['pexrc_util.py'],
+  dependencies = [
+    ':git_util',
+  ],
+)
+
+python_library(
+  name='process_test_util',
+  sources=['process_test_util.py'],
+  dependencies = [
+    '3rdparty/python:dataclasses',
+    '3rdparty/python:psutil',
+  ],
+)

src/python/pants/testutil/base/BUILD

@@ -0,0 +1,14 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  name = 'context_utils',
+  sources = ['context_utils.py'],
+  dependencies = [
+    '3rdparty/python/twitter/commons:twitter.common.collections',
+    'src/python/pants/base:workunit',
+    'src/python/pants/build_graph',
+    'src/python/pants/goal:context',
+    'src/python/pants/goal:run_tracker',
+  ]
+)

src/python/pants/testutil/base/context_utils.py

@@ -0,0 +1,131 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import logging
+import sys
+from contextlib import contextmanager
+
+from twitter.common.collections import maybe_list
+
+from pants.base.workunit import WorkUnit
+from pants.build_graph.target import Target
+from pants.goal.context import Context
+from pants.goal.run_tracker import RunTrackerLogger
+
+
+class TestContext(Context):
+  """A Context to use during unittesting.
+
+  :API: public
+
+  Stubs out various dependencies that we don't want to introduce in unit tests.
+
+  TODO: Instead of extending the runtime Context class, create a Context interface and have
+  TestContext and a runtime Context implementation extend that. This will also allow us to
+  isolate the parts of the interface that a Task is allowed to use vs. the parts that the
+  task-running machinery is allowed to use.
+  """
+  class DummyWorkUnit:
+    """A workunit stand-in that sends all output to stderr.
+
+   These outputs are typically only used by subprocesses spawned by code under test, not
+   the code under test itself, and would otherwise go into some reporting black hole.  The
+   testing framework will only display the stderr output when a test fails.
+
+   Provides no other tracking/labeling/reporting functionality. Does not require "opening"
+   or "closing".
+   """
+
+    def output(self, name):
+      return sys.stderr
+
+    def set_outcome(self, outcome):
+      return sys.stderr.write('\nWorkUnit outcome: {}\n'.format(WorkUnit.outcome_string(outcome)))
+
+  class DummyRunTracker:
+    """A runtracker stand-in that does no actual tracking."""
+
+    def __init__(self):
+      self.logger = RunTrackerLogger(self)
+
+    class DummyArtifactCacheStats:
+      def add_hits(self, cache_name, targets): pass
+
+      def add_misses(self, cache_name, targets, causes): pass
+
+    artifact_cache_stats = DummyArtifactCacheStats()
+
+    def report_target_info(self, scope, target, keys, val): pass
+
+
+  class TestLogger(logging.getLoggerClass()):
+    """A logger that converts our structured records into flat ones.
+
+    This is so we can use a regular logger in tests instead of our reporting machinery.
+    """
+
+    def makeRecord(self, name, lvl, fn, lno, msg, args, exc_info, *pos_args, **kwargs):
+      # Python 2 and Python 3 have different arguments for makeRecord().
+      # For cross-compatibility, we are unpacking arguments.
+      # See https://stackoverflow.com/questions/44329421/logging-makerecord-takes-8-positional-arguments-but-11-were-given.
+      msg = ''.join([msg] + [a[0] if isinstance(a, (list, tuple)) else a for a in args])
+      args = []
+      return super(TestContext.TestLogger, self).makeRecord(
+        name, lvl, fn, lno, msg, args, exc_info, *pos_args, **kwargs)
+
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    logger_cls = logging.getLoggerClass()
+    try:
+      logging.setLoggerClass(self.TestLogger)
+      self._logger = logging.getLogger('test')
+    finally:
+      logging.setLoggerClass(logger_cls)
+
+  @contextmanager
+  def new_workunit(self, name, labels=None, cmd='', log_config=None):
+    """
+    :API: public
+    """
+    sys.stderr.write('\nStarting workunit {}\n'.format(name))
+    yield TestContext.DummyWorkUnit()
+
+  @property
+  def log(self):
+    """
+    :API: public
+    """
+    return self._logger
+
+  def submit_background_work_chain(self, work_chain, parent_workunit_name=None):
+    """
+    :API: public
+    """
+    # Just do the work synchronously, so we don't need a run tracker, background workers and so on.
+    for work in work_chain:
+      for args_tuple in work.args_tuples:
+        work.func(*args_tuple)
+
+  def subproc_map(self, f, items):
+    """
+    :API: public
+    """
+    # Just execute in-process.
+    return list(map(f, items))
+
+
+def create_context_from_options(options, target_roots=None, build_graph=None,
+                                build_configuration=None, address_mapper=None,
+                                console_outstream=None, workspace=None, scheduler=None):
+  """Creates a ``Context`` with the given options and no targets by default.
+
+  :param options: An :class:`pants.option.options.Option`-alike object that supports read methods.
+
+  Other params are as for ``Context``.
+  """
+  run_tracker = TestContext.DummyRunTracker()
+  target_roots = maybe_list(target_roots, Target) if target_roots else []
+  return TestContext(options=options, run_tracker=run_tracker, target_roots=target_roots,
+                     build_graph=build_graph, build_configuration=build_configuration,
+                     address_mapper=address_mapper, console_outstream=console_outstream,
+                     workspace=workspace, scheduler=scheduler)