Support mypy plugins and 3rdpary type definitions.

Add support to the Pants mypy contrib plugin for loading type
definitions and mypy plugins from requirements in the transitive closure
of targets being checked.

Fixes #8263.

contrib/mypy/examples/src/python/mypy_plugin/BUILD

@@ -0,0 +1,42 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_requirement_library(
+  name='django',
+  requirements=[
+    python_requirement('Django==2.2.5'),
+  ]
+)
+
+python_requirement_library(
+  name='django-stubs',
+  requirements=[
+    python_requirement('django-stubs==1.1.0'),
+  ]
+)
+
+python_library(
+  name='settings',
+  source='settings.py',
+  dependencies=[
+    ':django-stubs',
+  ],
+)
+
+python_library(
+  name='valid',
+  source='valid.py',
+  dependencies=[
+    ':django',
+    ':settings',
+  ],
+)
+
+python_library(
+  name='invalid',
+  source='invalid.py',
+  dependencies=[
+    ':django',
+    ':settings',
+  ],
+)

contrib/mypy/examples/src/python/mypy_plugin/invalid.py

@@ -0,0 +1,7 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from django.utils import text
+
+
+assert '42' == text.slugify(42)

contrib/mypy/examples/src/python/mypy_plugin/mypy.ini

@@ -0,0 +1,6 @@
+[mypy]
+plugins =
+  mypy_django_plugin.main
+
+[mypy.plugins.django-stubs]
+django_settings_module = mypy_plugin.settings

contrib/mypy/examples/src/python/mypy_plugin/settings.py

@@ -0,0 +1,11 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from django.urls import URLPattern
+
+
+DEBUG: bool = True
+DEFAULT_FROM_EMAIL: str = 'webmaster@example.com'
+SECRET_KEY: str = 'not so secret'
+
+MY_SETTING: URLPattern = URLPattern(pattern='foo', callback=lambda: None)

contrib/mypy/examples/src/python/mypy_plugin/valid.py

@@ -0,0 +1,7 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from django.utils import text
+
+
+assert 'forty-two' == text.slugify('forty two')

contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py

@@ -2,21 +2,24 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 import os
-import subprocess
+from pathlib import Path
+from textwrap import dedent
 from typing import List
 
 from pants.backend.python.interpreter_cache import PythonInterpreterCache
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_target import PythonTarget
 from pants.backend.python.targets.python_tests import PythonTests
+from pants.backend.python.tasks.resolve_requirements import ResolveRequirements
 from pants.backend.python.tasks.resolve_requirements_task_base import ResolveRequirementsTaskBase
+from pants.base import hash_utils
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
-from pants.base.workunit import WorkUnit, WorkUnitLabel
+from pants.base.workunit import WorkUnitLabel
 from pants.build_graph.target import Target
 from pants.task.lint_task_mixin import LintTaskMixin
-from pants.util.contextutil import temporary_file_path
+from pants.util.contextutil import temporary_file, temporary_file_path
 from pants.util.memo import memoized_property
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
@@ -51,10 +54,16 @@ class MypyTask(LintTaskMixin, ResolveRequirementsTaskBase):
   def prepare(cls, options, round_manager):
     super().prepare(options, round_manager)
     round_manager.require_data(PythonInterpreter)
+    if options.include_requirements:
+      round_manager.require_data(ResolveRequirements.REQUIREMENTS_PEX)
 
   @classmethod
   def register_options(cls, register):
-    register('--mypy-version', default='0.710', help='The version of mypy to use.')
+    register('--mypy-version', default='0.720', help='The version of mypy to use.')
+    register('--include-requirements', type=bool, default=False,
+             help='Whether to include the transitive requirements of targets being checked. This is'
+                  'useful if those targets depend on mypy plugins or distributions that provide '
+                  'type stubs that should be active in the check.')
     register('--config-file', default=None,
              help='Path mypy configuration file, relative to buildroot.')
     register('--whitelist-tag-name', default=None,
@@ -144,19 +153,53 @@ def _collect_source_roots(self):
   def _interpreter_cache(self):
     return PythonInterpreterCache.global_instance()
 
-  def _run_mypy(self, py3_interpreter, mypy_args, **kwargs):
-    pex_info = PexInfo.default()
-    pex_info.entry_point = 'mypy'
+  def _get_mypy_pex(self, py3_interpreter: PythonInterpreter, *extra_pexes: PEX) -> PEX:
     mypy_version = self.get_options().mypy_version
-
-    mypy_requirement_pex = self.resolve_requirement_strings(
-      py3_interpreter, [f'mypy=={mypy_version}'])
-
-    path = os.path.realpath(os.path.join(self.workdir, str(py3_interpreter.identity), mypy_version))
-    if not os.path.isdir(path):
-      self.merge_pexes(path, pex_info, py3_interpreter, [mypy_requirement_pex])
-    pex = PEX(path, py3_interpreter)
-    return pex.run(mypy_args, **kwargs)
+    extras_hash = hash_utils.hash_all(hash_utils.hash_dir(Path(extra_pex.path()))
+                                      for extra_pex in extra_pexes)
+
+    path = Path(self.workdir,
+                str(py3_interpreter.identity),
+                f'{mypy_version}-{extras_hash}')
+    pex_dir = str(path)
+    if not path.is_dir():
+      mypy_requirement_pex = self.resolve_requirement_strings(
+        py3_interpreter,
+        [f'mypy=={mypy_version}']
+      )
+      pex_info = PexInfo.default()
+      pex_info.entry_point = 'pants_mypy_launcher'
+      with self.merged_pex(path=pex_dir,
+                           pex_info=pex_info,
+                           interpreter=py3_interpreter,
+                           pexes=[mypy_requirement_pex, *extra_pexes]) as builder:
+        with temporary_file(binary_mode=False) as exe_fp:
+          # MyPy searches for types for a package in packages containing a `py.types` marker file
+          # or else in a sibling `<package>-stubs` package as per PEP-0561. Going further than that
+          # PEP, MyPy restricts its search to `site-packages`. Since PEX deliberately isolates
+          # itself from `site-packages` as part of its raison d'etre, we monkey-patch
+          # `site.getsitepackages` to look inside the scrubbed PEX sys.path before handing off to
+          # `mypy`.
+          #
+          # See:
+          #   https://mypy.readthedocs.io/en/stable/installed_packages.html#installed-packages
+          #   https://www.python.org/dev/peps/pep-0561/#stub-only-packages
+          exe_fp.write(dedent("""
+            import runpy
+            import site
+            import sys
+
+
+            site.getsitepackages = lambda: sys.path[:]
+
+            
+            runpy.run_module('mypy', run_name='__main__')
+          """))
+          exe_fp.flush()
+          builder.set_executable(filename=exe_fp.name, env_filename=f'{pex_info.entry_point}.py')
+        builder.freeze(bytecode_compile=False)
+
+    return PEX(pex_dir, py3_interpreter)
 
   def execute(self):
     mypy_interpreter = self.find_mypy_interpreter()
@@ -176,6 +219,18 @@ def execute(self):
     if not interpreter_for_targets:
       raise TaskError('No Python interpreter compatible with specified sources.')
 
+    extra_pexes = []
+    if self.get_options().include_requirements:
+      if interpreter_for_targets.identity.matches(self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT):
+        extra_pexes.append(self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX))
+      else:
+        self.context.log.warn(
+          f"The --include-requirements option is set, but the current target's requirements have "
+          f"been resolved for {interpreter_for_targets.identity} which is not compatible with mypy "
+          f"which needs {self._MYPY_COMPATIBLE_INTERPETER_CONSTRAINT}: omitting resolved "
+          f"requirements from the mypy PYTHONPATH."
+        )
+
     with temporary_file_path() as sources_list_path:
       with open(sources_list_path, 'w') as f:
         for source in sources:
@@ -186,20 +241,25 @@ def execute(self):
         cmd.append(f'--config-file={os.path.join(get_buildroot(), self.get_options().config_file)}')
       cmd.extend(self.get_passthru_args())
       cmd.append(f'@{sources_list_path}')
-      self.context.log.debug(f'mypy command: {" ".join(cmd)}')
+
+      with self.context.new_workunit(name='create_mypy_pex', labels=[WorkUnitLabel.PREP]):
+        mypy_pex = self._get_mypy_pex(mypy_interpreter, *extra_pexes)
 
       # Collect source roots for the targets being checked.
-      source_roots = self._collect_source_roots()
+      buildroot = Path(get_buildroot())
+      sources_path = os.pathsep.join(str(buildroot.joinpath(root))
+                                     for root in self._collect_source_roots())
 
-      mypy_path = os.pathsep.join([os.path.join(get_buildroot(), root) for root in source_roots])
       # Execute mypy.
-      with self.context.new_workunit(
-        name='check',
-        labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],
-        log_config=WorkUnit.LogConfig(level=self.get_options().level,
-                                      colors=self.get_options().colors),
-        cmd=' '.join(cmd)) as workunit:
-        returncode = self._run_mypy(mypy_interpreter, cmd,
-          env={'MYPYPATH': mypy_path}, stdout=workunit.output('stdout'), stderr=subprocess.STDOUT)
+      with self.context.new_workunit(name='check',
+                                     labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN],
+                                     cmd=' '.join(mypy_pex.cmdline(cmd))) as workunit:
+        returncode = mypy_pex.run(cmd,
+                                  env=dict(
+                                    PYTHONPATH=sources_path,
+                                    PEX_INHERIT_PATH='fallback'
+                                  ),
+                                  stdout=workunit.output('stdout'),
+                                  stderr=workunit.output('stderr'))
         if returncode != 0:
           raise MypyTaskError(f'mypy failed: code={returncode}')

contrib/mypy/tests/python/pants_test/contrib/mypy/tasks/test_mypy_integration.py

@@ -1,17 +1,56 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from pathlib import Path
+
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
 
 class MypyIntegrationTest(PantsRunIntegrationTest):
 
   cmdline = ['--backend-packages=pants.contrib.mypy', 'lint']
 
+  def target(self, name):
+    return f'contrib/mypy/examples/src/python/simple:{name}'
+
   def test_valid_type_hints(self):
-    result = self.run_pants([*self.cmdline, 'contrib/mypy/examples/src/python:valid'])
+    result = self.run_pants([*self.cmdline, self.target('valid')])
     self.assert_success(result)
 
   def test_invalid_type_hints(self):
-    result = self.run_pants([*self.cmdline, 'contrib/mypy/examples/src/python:invalid'])
+    result = self.run_pants([*self.cmdline, self.target('invalid')])
+    self.assert_failure(result)
+
+
+class MypyPluginIntegrationTest(PantsRunIntegrationTest):
+
+  example_dir = Path('contrib/mypy/examples/src/python/mypy_plugin')
+
+  @classmethod
+  def cmdline(cls, *, include_requirements):
+    cmd = [
+      '--backend-packages=pants.contrib.mypy',
+      'lint.mypy',
+      f'--config-file={cls.example_dir / "mypy.ini"}'
+    ]
+    if include_requirements:
+      cmd.append('--include-requirements')
+    return cmd
+
+  @classmethod
+  def target(cls, name):
+    return f'{cls.example_dir}:{name}'
+
+  def test_valid_library_use_include_requirements(self):
+    result = self.run_pants([*self.cmdline(include_requirements=True), self.target('valid')])
+    self.assert_success(result)
+
+  def test_invalid_library_use_include_requirements(self):
+    result = self.run_pants([*self.cmdline(include_requirements=True), self.target('invalid')])
+    self.assert_failure(result)
+
+  def test_valid_library_use_exclude_requirements(self):
+    # The target is valid, but we fail to include the mypy plugin and type information needed via
+    # requirements and so the check fails.
+    result = self.run_pants([*self.cmdline(include_requirements=False), self.target('valid')])
     self.assert_failure(result)