Add support for MyPy to Pants v2 (#10132)

This is not a perfect implementation yet. See #10131 for remaining TODOs. But, this gives an initial implementation to build off of.

A key decision of this PR is to add MyPy to the `lint` goal, rather than a new `typecheck` goal. Users shared feedback that they prefer this, as it's neat to run all your linters in parallel. Technically, MyPy is a linter, only a supercharged one.

To activate, add `pants.backend.python.lint.mypy` to `backend_packages2`.

[ci skip-rust-tests]
[ci skip-jvm-tests]

src/python/pants/backend/python/lint/isort/subsystem.py

@@ -18,21 +18,18 @@ def register_options(cls, register):
             "--skip",
             type=bool,
             default=False,
-            fingerprint=True,
             help="Don't use isort when running `./pants fmt` and `./pants lint`",
         )
         register(
             "--args",
             type=list,
             member_type=shell_str,
-            fingerprint=True,
             help="Arguments to pass directly to isort, e.g. "
             '`--isort-args="--case-sensitive --trailing-comma"`',
         )
         register(
             "--config",
             type=list,
             member_type=file_option,
-            fingerprint=True,
             help="Path to `isort.cfg` or alternative isort config file(s)",
         )

src/python/pants/backend/python/lint/mypy/BUILD

@@ -0,0 +1,42 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies=[
+    '3rdparty/python:dataclasses',
+    'src/python/pants/backend/python/lint',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/backend/python/rules',
+    'src/python/pants/core/goals',
+    'src/python/pants/core/util_rules',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:process',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
+    'src/python/pants/option',
+    'src/python/pants/python',
+  ],
+  tags = {"partially_type_checked"},
+)
+
+python_tests(
+  name='integration',
+  sources=['*_integration_test.py'],
+  dependencies=[
+    ':mypy',
+    'src/python/pants/backend/python/lint',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/core/goals',
+    'src/python/pants/engine:addresses',
+    'src/python/pants/engine:fs',
+    'src/python/pants/engine:rules',
+    'src/python/pants/engine:selectors',
+    'src/python/pants/engine:unions',
+    'src/python/pants/engine/legacy:structs',
+    'src/python/pants/source',
+    'src/python/pants/testutil:interpreter_selection_utils',
+    'src/python/pants/testutil:external_tool_test_base',
+    'src/python/pants/testutil/option',
+  ],
+  tags = {'integration', 'partially_type_checked'},
+)

src/python/pants/backend/python/lint/mypy/register.py

@@ -0,0 +1,14 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+"""Type checker for Python.
+
+See https://pants.readme.io/docs/python-linters-and-formatters and
+https://mypy.readthedocs.io/en/stable/.
+"""
+
+from pants.backend.python.lint.mypy import rules as mypy_rules
+
+
+def rules():
+    return mypy_rules.rules()

src/python/pants/backend/python/lint/mypy/rules.py

@@ -0,0 +1,144 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from dataclasses import dataclass
+from typing import Tuple
+
+from pants.backend.python.lint.mypy.subsystem import MyPy
+from pants.backend.python.rules import download_pex_bin, importable_python_sources, pex
+from pants.backend.python.rules.importable_python_sources import ImportablePythonSources
+from pants.backend.python.rules.pex import (
+    Pex,
+    PexInterpreterConstraints,
+    PexRequest,
+    PexRequirements,
+)
+from pants.backend.python.subsystems import python_native_code, subprocess_environment
+from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
+from pants.backend.python.target_types import PythonSources
+from pants.core.goals.lint import LintRequest, LintResult, LintResults
+from pants.core.util_rules import determine_source_files, strip_source_roots
+from pants.engine.addresses import Addresses
+from pants.engine.fs import (
+    Digest,
+    FileContent,
+    InputFilesContent,
+    MergeDigests,
+    PathGlobs,
+    Snapshot,
+)
+from pants.engine.process import FallibleProcessResult, Process
+from pants.engine.rules import SubsystemRule, rule
+from pants.engine.selectors import Get, MultiGet
+from pants.engine.target import FieldSetWithOrigin, Targets, TransitiveTargets
+from pants.engine.unions import UnionRule
+from pants.option.global_options import GlobMatchErrorBehavior
+from pants.python.python_setup import PythonSetup
+from pants.util.strutil import pluralize
+
+
+@dataclass(frozen=True)
+class MyPyFieldSet(FieldSetWithOrigin):
+    required_fields = (PythonSources,)
+
+    sources: PythonSources
+
+
+class MyPyRequest(LintRequest):
+    field_set_type = MyPyFieldSet
+
+
+def generate_args(mypy: MyPy, *, file_list_path: str) -> Tuple[str, ...]:
+    args = []
+    if mypy.config:
+        args.append(f"--config-file={mypy.config}")
+    args.extend(mypy.args)
+    args.append(f"@{file_list_path}")
+    return tuple(args)
+
+
+# TODO(#10131): Improve performance, e.g. by leveraging the MyPy cache.
+# TODO(#10131): Support plugins and type stubs.
+@rule(desc="Lint using MyPy")
+async def mypy_lint(
+    request: MyPyRequest,
+    mypy: MyPy,
+    python_setup: PythonSetup,
+    subprocess_encoding_environment: SubprocessEncodingEnvironment,
+) -> LintResults:
+    if mypy.skip:
+        return LintResults()
+
+    transitive_targets = await Get(
+        TransitiveTargets, Addresses(fs.address for fs in request.field_sets)
+    )
+
+    prepared_sources_request = Get(ImportablePythonSources, Targets(transitive_targets.closure))
+    pex_request = Get(
+        Pex,
+        PexRequest(
+            output_filename="mypy.pex",
+            requirements=PexRequirements(mypy.get_requirement_specs()),
+            # TODO(#10131): figure out how to robustly handle interpreter constraints. Unlike other
+            #  linters, the version of Python used to run MyPy can be different than the version of
+            #  the code.
+            interpreter_constraints=PexInterpreterConstraints(mypy.default_interpreter_constraints),
+            entry_point=mypy.get_entry_point(),
+        ),
+    )
+    config_snapshot_request = Get(
+        Snapshot,
+        PathGlobs(
+            globs=[mypy.config] if mypy.config else [],
+            glob_match_error_behavior=GlobMatchErrorBehavior.error,
+            description_of_origin="the option `--mypy-config`",
+        ),
+    )
+    prepared_sources, pex, config_snapshot = await MultiGet(
+        prepared_sources_request, pex_request, config_snapshot_request
+    )
+
+    file_list_path = "__files.txt"
+    file_list = await Get(
+        Digest,
+        InputFilesContent(
+            [FileContent(file_list_path, "\n".join(prepared_sources.snapshot.files).encode())]
+        ),
+    )
+
+    merged_input_files = await Get(
+        Digest,
+        MergeDigests(
+            [file_list, prepared_sources.snapshot.digest, pex.digest, config_snapshot.digest]
+        ),
+    )
+
+    address_references = ", ".join(sorted(tgt.address.spec for tgt in transitive_targets.closure))
+    process = pex.create_process(
+        python_setup=python_setup,
+        subprocess_encoding_environment=subprocess_encoding_environment,
+        pex_path=pex.output_filename,
+        pex_args=generate_args(mypy, file_list_path=file_list_path),
+        input_digest=merged_input_files,
+        description=(
+            f"Run MyPy on {pluralize(len(transitive_targets.closure), 'target')}: "
+            f"{address_references}."
+        ),
+    )
+    result = await Get(FallibleProcessResult, Process, process)
+    return LintResults([LintResult.from_fallible_process_result(result, linter_name="MyPy")])
+
+
+def rules():
+    return [
+        mypy_lint,
+        SubsystemRule(MyPy),
+        UnionRule(LintRequest, MyPyRequest),
+        *download_pex_bin.rules(),
+        *determine_source_files.rules(),
+        *importable_python_sources.rules(),
+        *pex.rules(),
+        *python_native_code.rules(),
+        *strip_source_roots.rules(),
+        *subprocess_environment.rules(),
+    ]