From 6b6775cbfc78f03eeff8c94b156191ac18309213 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 5 Feb 2020 22:48:19 -0800
Subject: [PATCH 01/22] stubtest: move into mypy

---
 {scripts => mypy}/stubtest.py | 0
 tox.ini                       | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename {scripts => mypy}/stubtest.py (100%)

diff --git a/scripts/stubtest.py b/mypy/stubtest.py
similarity index 100%
rename from scripts/stubtest.py
rename to mypy/stubtest.py
diff --git a/tox.ini b/tox.ini
index 18cf56f9c3a8..ac7cdc72fdb7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -51,7 +51,7 @@ description = type check ourselves
 basepython = python3.7
 commands =
     python -m mypy --config-file mypy_self_check.ini -p mypy -p mypyc
-    python -m mypy --config-file mypy_self_check.ini misc/proper_plugin.py scripts/stubtest.py scripts/mypyc
+    python -m mypy --config-file mypy_self_check.ini misc/proper_plugin.py scripts/mypyc
 
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs

From 60100166ed31b06f7b90e7f80a7455f3ca2a3cd1 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 5 Feb 2020 22:51:40 -0800
Subject: [PATCH 02/22] stubtest: add entry point to setup.py

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 1a66f51c5bed..37995c3d7018 100644
--- a/setup.py
+++ b/setup.py
@@ -182,6 +182,7 @@ def run(self):
       scripts=['scripts/mypyc'],
       entry_points={'console_scripts': ['mypy=mypy.__main__:console_entry',
                                         'stubgen=mypy.stubgen:main',
+                                        'stubtest=mypy.stubtest:main',
                                         'dmypy=mypy.dmypy.client:console_entry',
                                         ]},
       classifiers=classifiers,

From 1ec9503db71a80fbfe60c75a7589b7d7784e1415 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 5 Feb 2020 23:32:27 -0800
Subject: [PATCH 03/22] stubtest: use mypy.utils.check_python_version

---
 mypy/stubtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 00475b78168d..941a235f2461 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -976,7 +976,7 @@ def strip_comments(s: str) -> str:
 
 
 def main() -> int:
-    assert sys.version_info >= (3, 5), "This script requires at least Python 3.5"
+    mypy.util.check_python_version("stubtest")
 
     parser = argparse.ArgumentParser(
         description="Compares stubs to objects introspected from the runtime."

From ac4ae2af00b76dfc24409d5795212c1df03ccf2f Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Wed, 5 Feb 2020 23:47:15 -0800
Subject: [PATCH 04/22] stubtest: split up main to make it easier to test

---
 mypy/stubtest.py | 95 ++++++++++++++++++++++++++----------------------
 1 file changed, 52 insertions(+), 43 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 941a235f2461..15cf7bfd015c 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -917,7 +917,7 @@ def build_stubs(modules: List[str], options: Options, find_submodules: bool = Fa
     if res.errors:
         output = [_style("error: ", color="red", bold=True), " failed mypy build.\n"]
         print("".join(output) + "\n".join(res.errors))
-        sys.exit(1)
+        raise RuntimeError
 
     global _all_stubs
     _all_stubs = res.files
@@ -975,47 +975,8 @@ def strip_comments(s: str) -> str:
                 yield entry
 
 
-def main() -> int:
-    mypy.util.check_python_version("stubtest")
-
-    parser = argparse.ArgumentParser(
-        description="Compares stubs to objects introspected from the runtime."
-    )
-    parser.add_argument("modules", nargs="*", help="Modules to test")
-    parser.add_argument("--concise", action="store_true", help="Make output concise")
-    parser.add_argument(
-        "--ignore-missing-stub",
-        action="store_true",
-        help="Ignore errors for stub missing things that are present at runtime",
-    )
-    parser.add_argument(
-        "--ignore-positional-only",
-        action="store_true",
-        help="Ignore errors for whether an argument should or shouldn't be positional-only",
-    )
-    parser.add_argument(
-        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
-    )
-    parser.add_argument(
-        "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
-    )
-    parser.add_argument(
-        "--whitelist",
-        action="append",
-        metavar="FILE",
-        default=[],
-        help=(
-            "Use file as a whitelist. Can be passed multiple times to combine multiple "
-            "whitelists. Whitelist can be created with --generate-whitelist"
-        ),
-    )
-    parser.add_argument(
-        "--generate-whitelist",
-        action="store_true",
-        help="Print a whitelist (to stdout) to be used with --whitelist",
-    )
-    args = parser.parse_args()
-
+def test_stubs(args: argparse.Namespace) -> int:
+    """This is stubtest! It's time to test the stubs!"""
     # Load the whitelist. This is a series of strings corresponding to Error.object_desc
     # Values in the dict will store whether we used the whitelist entry or not.
     whitelist = {
@@ -1039,7 +1000,10 @@ def main() -> int:
     options.incremental = False
     options.custom_typeshed_dir = args.custom_typeshed_dir
 
-    modules = build_stubs(modules, options, find_submodules=not args.check_typeshed)
+    try:
+        modules = build_stubs(modules, options, find_submodules=not args.check_typeshed)
+    except RuntimeError:
+        return 1
 
     exit_code = 0
     for module in modules:
@@ -1075,5 +1039,50 @@ def main() -> int:
     return exit_code
 
 
+def parse_options(args: List[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Compares stubs to objects introspected from the runtime."
+    )
+    parser.add_argument("modules", nargs="*", help="Modules to test")
+    parser.add_argument("--concise", action="store_true", help="Make output concise")
+    parser.add_argument(
+        "--ignore-missing-stub",
+        action="store_true",
+        help="Ignore errors for stub missing things that are present at runtime",
+    )
+    parser.add_argument(
+        "--ignore-positional-only",
+        action="store_true",
+        help="Ignore errors for whether an argument should or shouldn't be positional-only",
+    )
+    parser.add_argument(
+        "--custom-typeshed-dir", metavar="DIR", help="Use the custom typeshed in DIR"
+    )
+    parser.add_argument(
+        "--check-typeshed", action="store_true", help="Check all stdlib modules in typeshed"
+    )
+    parser.add_argument(
+        "--whitelist",
+        action="append",
+        metavar="FILE",
+        default=[],
+        help=(
+            "Use file as a whitelist. Can be passed multiple times to combine multiple "
+            "whitelists. Whitelist can be created with --generate-whitelist"
+        ),
+    )
+    parser.add_argument(
+        "--generate-whitelist",
+        action="store_true",
+        help="Print a whitelist (to stdout) to be used with --whitelist",
+    )
+    return parser.parse_args(args)
+
+
+def main() -> int:
+    mypy.util.check_python_version("stubtest")
+    return test_stubs(parse_options(sys.argv[1:]))
+
+
 if __name__ == "__main__":
     sys.exit(main())

From 0e6bf85c0602d8a0cbe518e2710342dd3a6941bd Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 13:57:07 -0800
Subject: [PATCH 05/22] stubtest: improvements to signature checking

Fixes some false negatives and a minor false positive. Makes the logic
more readable and improve comments.
---
 mypy/stubtest.py | 61 +++++++++++++++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 15cf7bfd015c..30f8de84c010 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -532,26 +532,34 @@ def _verify_signature(
                 "(remove leading double underscore)".format(stub_arg.variable.name)
             )
 
-    # Checks involving *args
-    if len(stub.pos) == len(runtime.pos):
-        if stub.varpos is None and runtime.varpos is not None:
-            yield 'stub does not have *args argument "{}"'.format(runtime.varpos.name)
-        if stub.varpos is not None and runtime.varpos is None:
-            yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
-    elif len(stub.pos) > len(runtime.pos):
+    # Check unmatched positional args
+    if len(stub.pos) > len(runtime.pos):
+        # There are cases where the stub exhaustively lists out the extra parameters the function
+        # would take through *args. Hence, a) we can't check that the runtime actually takes those
+        # parameters and b) below, we don't enforce that the stub takes *args, since runtime logic
+        # may prevent those arguments from actually being accepted.
         if runtime.varpos is None:
             for stub_arg in stub.pos[len(runtime.pos) :]:
                 # If the variable is in runtime.kwonly, it's just mislabelled as not a
-                # keyword-only argument; we report the error while checking keyword-only arguments
+                # keyword-only argument
                 if stub_arg.variable.name not in runtime.kwonly:
                     yield 'runtime does not have argument "{}"'.format(stub_arg.variable.name)
-        # We do not check whether stub takes *args when the runtime does, for cases where the stub
-        # just listed out the extra parameters the function takes
+                else:
+                    yield 'stub argument "{}" is not keyword-only'.format(stub_arg.variable.name)
+            if stub.varpos is not None:
+                yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
     elif len(stub.pos) < len(runtime.pos):
-        if stub.varpos is None:
-            for runtime_arg in runtime.pos[len(stub.pos) :]:
+        for runtime_arg in runtime.pos[len(stub.pos) :]:
+            if runtime_arg.name not in stub.kwonly:
                 yield 'stub does not have argument "{}"'.format(runtime_arg.name)
-        elif runtime.pos is None:
+            else:
+                yield 'runtime argument "{}" is not keyword-only'.format(runtime_arg.name)
+
+    # Checks involving *args
+    if len(stub.pos) <= len(runtime.pos) or runtime.varpos is None:
+        if stub.varpos is None and runtime.varpos is not None:
+            yield 'stub does not have *args argument "{}"'.format(runtime.varpos.name)
+        if stub.varpos is not None and runtime.varpos is None:
             yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
 
     # Check keyword-only args
@@ -560,26 +568,31 @@ def _verify_signature(
         yield from _verify_arg_name(stub_arg, runtime_arg, function_name)
         yield from _verify_arg_default_value(stub_arg, runtime_arg)
 
+    # Check unmatched keyword-only args
+    if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)):
+        for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)):
+            yield 'runtime does not have argument "{}"'.format(arg)
+    if stub.varkw is None or not set(stub.kwonly).issubset(set(runtime.kwonly)):
+        for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)):
+            if arg in set(stub_arg.variable.name for stub_arg in stub.pos):
+                # Don't report this if we've reported it before
+                if len(stub.pos) > len(runtime.pos) and runtime.varpos is not None:
+                    yield 'stub argument "{}" is not keyword-only'.format(arg)
+            else:
+                yield 'stub does not have argument "{}"'.format(arg)
+
     # Checks involving **kwargs
     if stub.varkw is None and runtime.varkw is not None:
-        # We do not check whether stub takes **kwargs when the runtime does, for cases where the
-        # stub just listed out the extra keyword parameters the function takes
+        # There are cases where the stub exhaustively lists out the extra parameters the function
+        # would take through **kwargs, so we don't enforce that the stub takes **kwargs.
         # Also check against positional parameters, to avoid a nitpicky message when an argument
         # isn't marked as keyword-only
         stub_pos_names = set(stub_arg.variable.name for stub_arg in stub.pos)
+        # Ideally we'd do a strict subset check, but in practice the errors from that aren't useful
         if not set(runtime.kwonly).issubset(set(stub.kwonly) | stub_pos_names):
             yield 'stub does not have **kwargs argument "{}"'.format(runtime.varkw.name)
     if stub.varkw is not None and runtime.varkw is None:
         yield 'runtime does not have **kwargs argument "{}"'.format(stub.varkw.variable.name)
-    if runtime.varkw is None or not set(runtime.kwonly).issubset(set(stub.kwonly)):
-        for arg in sorted(set(stub.kwonly) - set(runtime.kwonly)):
-            yield 'runtime does not have argument "{}"'.format(arg)
-    if stub.varkw is None or not set(stub.kwonly).issubset(set(runtime.kwonly)):
-        for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)):
-            if arg in set(stub_arg.variable.name for stub_arg in stub.pos):
-                yield 'stub argument "{}" is not keyword-only'.format(arg)
-            else:
-                yield 'stub does not have argument "{}"'.format(arg)
 
 
 @verify.register(nodes.FuncItem)

From 0efa73df5fcbc34580472974f34c9917bfc75bf7 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 16:56:36 -0800
Subject: [PATCH 06/22] stubtest: [minor] follow project style / track coverage
 better

---
 mypy/stubtest.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 30f8de84c010..334c37938ca4 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -356,21 +356,21 @@ def get_name(arg: Any) -> str:
                 return arg.name
             if isinstance(arg, nodes.Argument):
                 return arg.variable.name
-            raise ValueError
+            raise AssertionError
 
         def get_type(arg: Any) -> Optional[str]:
             if isinstance(arg, inspect.Parameter):
                 return None
             if isinstance(arg, nodes.Argument):
                 return str(arg.variable.type or arg.type_annotation)
-            raise ValueError
+            raise AssertionError
 
         def has_default(arg: Any) -> bool:
             if isinstance(arg, inspect.Parameter):
                 return arg.default != inspect.Parameter.empty
             if isinstance(arg, nodes.Argument):
                 return arg.kind in (nodes.ARG_OPT, nodes.ARG_NAMED_OPT)
-            raise ValueError
+            raise AssertionError
 
         def get_desc(arg: Any) -> str:
             arg_type = get_type(arg)
@@ -403,7 +403,7 @@ def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
             elif stub_arg.kind == nodes.ARG_STAR2:
                 stub_sig.varkw = stub_arg
             else:
-                raise ValueError
+                raise AssertionError
         return stub_sig
 
     @staticmethod
@@ -422,7 +422,7 @@ def from_inspect_signature(signature: inspect.Signature,) -> "Signature[inspect.
             elif runtime_arg.kind == inspect.Parameter.VAR_KEYWORD:
                 runtime_sig.varkw = runtime_arg
             else:
-                raise ValueError
+                raise AssertionError
         return runtime_sig
 
     @staticmethod
@@ -500,7 +500,7 @@ def get_kind(arg_name: str) -> int:
             elif arg.kind == nodes.ARG_STAR2:
                 sig.varkw = arg
             else:
-                raise ValueError
+                raise AssertionError
         return sig
 
 

From a54dc9f26490bf753180c1e7b174479e1a17144d Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 18:18:54 -0800
Subject: [PATCH 07/22] stubtest: [minor] output filename more consistently

---
 mypy/stubtest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 334c37938ca4..8e89757919c2 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -105,7 +105,7 @@ def get_description(self, concise: bool = False) -> str:
         if stub_line:
             stub_loc_str += " at line {}".format(stub_line)
         if stub_file:
-            stub_loc_str += " in file {}".format(stub_file)
+            stub_loc_str += " in file {}".format(Path(stub_file))
 
         runtime_line = None
         runtime_file = None
@@ -123,7 +123,7 @@ def get_description(self, concise: bool = False) -> str:
         if runtime_line:
             runtime_loc_str += " at line {}".format(runtime_line)
         if runtime_file:
-            runtime_loc_str += " in file {}".format(runtime_file)
+            runtime_loc_str += " in file {}".format(Path(runtime_file))
 
         output = [
             _style("error: ", color="red", bold=True),

From 3162788f0702ce2561351b7ef3659484998d9cc5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 20:23:28 -0800
Subject: [PATCH 08/22] stubtest: [minor] remove no longer necessary optional

---
 mypy/stubtest.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 8e89757919c2..de0f402e5835 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -971,10 +971,7 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
     return sorted(modules)
 
 
-def get_whitelist_entries(whitelist_file: Optional[str]) -> Iterator[str]:
-    if not whitelist_file:
-        return
-
+def get_whitelist_entries(whitelist_file: str) -> Iterator[str]:
     def strip_comments(s: str) -> str:
         try:
             return s[: s.index("#")].strip()

From ea3f83fa866d59d76400df502bdfbff8182cfdbc Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 22:56:50 -0800
Subject: [PATCH 09/22] stubtest: fix module level variables missing at runtime

Dumb mistake causing false negatives, mainly seems to surface a lot of
platform differences
---
 mypy/stubtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index de0f402e5835..5b8d71be6c18 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -655,7 +655,7 @@ def verify_var(
 ) -> Iterator[Error]:
     if isinstance(runtime, Missing):
         # Don't always yield an error here, because we often can't find instance variables
-        if len(object_path) <= 1:
+        if len(object_path) <= 2:
             yield Error(object_path, "is not present at runtime", stub, runtime)
         return
 

From 6d63451ab03c95fe704bebbdfd041108457359b3 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Thu, 6 Feb 2020 23:52:00 -0800
Subject: [PATCH 10/22] stubtest: handle compile errors

---
 mypy/stubtest.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 5b8d71be6c18..ee8c63fad3eb 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -926,9 +926,14 @@ def build_stubs(modules: List[str], options: Options, find_submodules: bool = Fa
             sources.extend(found_sources)
             all_modules.extend(s.module for s in found_sources if s.module not in all_modules)
 
-    res = mypy.build.build(sources=sources, options=options)
+    try:
+        res = mypy.build.build(sources=sources, options=options)
+    except mypy.errors.CompileError as e:
+        output = [_style("error: ", color="red", bold=True), "failed mypy compile.\n", str(e)]
+        print("".join(output))
+        raise RuntimeError
     if res.errors:
-        output = [_style("error: ", color="red", bold=True), " failed mypy build.\n"]
+        output = [_style("error: ", color="red", bold=True), "failed mypy build.\n"]
         print("".join(output) + "\n".join(res.errors))
         raise RuntimeError
 

From 3ed99bb6e5bdd616fcfd9dc546434167b0052798 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 01:00:35 -0800
Subject: [PATCH 11/22] stubtest: [minor] remove black's colon spaces

To comply with project style
---
 mypy/stubtest.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index ee8c63fad3eb..412a2fd4636d 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -276,7 +276,7 @@ def _verify_arg_name(
         return
 
     def strip_prefix(s: str, prefix: str) -> str:
-        return s[len(prefix) :] if s.startswith(prefix) else s
+        return s[len(prefix):] if s.startswith(prefix) else s
 
     if strip_prefix(stub_arg.variable.name, "__") == runtime_arg.name:
         return
@@ -539,7 +539,7 @@ def _verify_signature(
         # parameters and b) below, we don't enforce that the stub takes *args, since runtime logic
         # may prevent those arguments from actually being accepted.
         if runtime.varpos is None:
-            for stub_arg in stub.pos[len(runtime.pos) :]:
+            for stub_arg in stub.pos[len(runtime.pos):]:
                 # If the variable is in runtime.kwonly, it's just mislabelled as not a
                 # keyword-only argument
                 if stub_arg.variable.name not in runtime.kwonly:
@@ -549,7 +549,7 @@ def _verify_signature(
             if stub.varpos is not None:
                 yield 'runtime does not have *args argument "{}"'.format(stub.varpos.variable.name)
     elif len(stub.pos) < len(runtime.pos):
-        for runtime_arg in runtime.pos[len(stub.pos) :]:
+        for runtime_arg in runtime.pos[len(stub.pos):]:
             if runtime_arg.name not in stub.kwonly:
                 yield 'stub does not have argument "{}"'.format(runtime_arg.name)
             else:

From ac5b12a66e63eed9608928f8a7b265d0b31f4d20 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 00:26:03 -0800
Subject: [PATCH 12/22] stubtest: [minor] remove black's commas

---
 mypy/stubtest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 412a2fd4636d..6e466d4ae8b9 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -407,7 +407,7 @@ def from_funcitem(stub: nodes.FuncItem) -> "Signature[nodes.Argument]":
         return stub_sig
 
     @staticmethod
-    def from_inspect_signature(signature: inspect.Signature,) -> "Signature[inspect.Parameter]":
+    def from_inspect_signature(signature: inspect.Signature) -> "Signature[inspect.Parameter]":
         runtime_sig = Signature()  # type: Signature[inspect.Parameter]
         for runtime_arg in signature.parameters.values():
             if runtime_arg.kind in (
@@ -426,7 +426,7 @@ def from_inspect_signature(signature: inspect.Signature,) -> "Signature[inspect.
         return runtime_sig
 
     @staticmethod
-    def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef,) -> "Signature[nodes.Argument]":
+    def from_overloadedfuncdef(stub: nodes.OverloadedFuncDef) -> "Signature[nodes.Argument]":
         """Returns a Signature from an OverloadedFuncDef.
 
         If life were simple, to verify_overloadedfuncdef, we'd just verify_funcitem for each of its

From 37fe91c8bdc540976ecbdfe351a8354f7067bccf Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 00:39:51 -0800
Subject: [PATCH 13/22] stubtest: [minor] handle a case in
 get_mypy_type_of_runtime_value

Doesn't make a difference to typeshed
---
 mypy/stubtest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 6e466d4ae8b9..d7369e95c940 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -870,6 +870,8 @@ def get_mypy_type_of_runtime_value(runtime: Any) -> Optional[mypy.types.Type]:
     if type_name not in stub.names:
         return None
     type_info = stub.names[type_name].node
+    if isinstance(type_info, nodes.Var):
+        return type_info.type
     if not isinstance(type_info, nodes.TypeInfo):
         return None
 

From 5e0949c1c7337444a04b0a61247d1ca40d8c46f0 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 00:53:45 -0800
Subject: [PATCH 14/22] stubtest: add tests

---
 mypy/test/teststubtest.py | 663 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 663 insertions(+)
 create mode 100644 mypy/test/teststubtest.py

diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py
new file mode 100644
index 000000000000..069de7e3bfd7
--- /dev/null
+++ b/mypy/test/teststubtest.py
@@ -0,0 +1,663 @@
+import contextlib
+import inspect
+import io
+import os
+import re
+import sys
+import tempfile
+import textwrap
+import unittest
+from typing import Any, Callable, Iterator, List, Optional
+
+import mypy.stubtest
+from mypy.stubtest import parse_options, test_stubs
+
+
+@contextlib.contextmanager
+def use_tmp_dir() -> Iterator[None]:
+    current = os.getcwd()
+    with tempfile.TemporaryDirectory() as tmp:
+        try:
+            os.chdir(tmp)
+            yield
+        finally:
+            os.chdir(current)
+
+
+TEST_MODULE_NAME = "test_module"
+
+
+def run_stubtest(stub: str, runtime: str, options: List[str]) -> str:
+    with use_tmp_dir():
+        with open("{}.pyi".format(TEST_MODULE_NAME), "w") as f:
+            f.write(stub)
+        with open("{}.py".format(TEST_MODULE_NAME), "w") as f:
+            f.write(runtime)
+
+        if sys.path[0] != ".":
+            sys.path.insert(0, ".")
+        if TEST_MODULE_NAME in sys.modules:
+            del sys.modules[TEST_MODULE_NAME]
+
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output):
+            test_stubs(parse_options([TEST_MODULE_NAME] + options))
+
+        return output.getvalue()
+
+
+class Case:
+    def __init__(self, stub: str, runtime: str, error: Optional[str]):
+        self.stub = stub
+        self.runtime = runtime
+        self.error = error
+
+
+def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]:
+    """Repeatedly invoking run_stubtest is slow, so use this decorator to combine cases.
+
+    We could also manually combine cases, but this allows us to keep the contrasting stub and
+    runtime definitions next to each other.
+
+    """
+
+    def test(*args: Any, **kwargs: Any) -> None:
+        cases = list(fn(*args, **kwargs))
+        expected_errors = set(
+            "{}.{}".format(TEST_MODULE_NAME, c.error) for c in cases if c.error is not None
+        )
+        output = run_stubtest(
+            stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
+            runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
+            options=["--generate-whitelist"]
+        )
+
+        actual_errors = set(output.splitlines())
+        assert actual_errors == expected_errors, output
+
+    return test
+
+
+class StubtestUnit(unittest.TestCase):
+    @collect_cases
+    def test_basic_good(self) -> Iterator[Case]:
+        yield Case(
+            stub="def f(number: int, text: str) -> None: ...",
+            runtime="def f(number, text): pass",
+            error=None,
+        )
+        yield Case(
+            stub="""
+            class X:
+                def f(self, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class X:
+                def f(self, number, text): pass
+            """,
+            error=None,
+        )
+
+    @collect_cases
+    def test_types(self) -> Iterator[Case]:
+        yield Case(
+            stub="def mistyped_class() -> None: ...",
+            runtime="class mistyped_class: pass",
+            error="mistyped_class",
+        )
+        yield Case(
+            stub="class mistyped_fn: ...", runtime="def mistyped_fn(): pass", error="mistyped_fn"
+        )
+        yield Case(
+            stub="""
+            class X:
+                def mistyped_var(self) -> int: ...
+            """,
+            runtime="""
+            class X:
+                mistyped_var = 1
+            """,
+            error="X.mistyped_var",
+        )
+
+    @collect_cases
+    def test_arg_name(self) -> Iterator[Case]:
+        yield Case(
+            stub="def bad(number: int, text: str) -> None: ...",
+            runtime="def bad(num, text) -> None: pass",
+            error="bad",
+        )
+        if sys.version_info >= (3, 8):
+            yield Case(
+                stub="def good_posonly(__number: int, text: str) -> None: ...",
+                runtime="def good_posonly(num, /, text): pass",
+                error=None,
+            )
+            yield Case(
+                stub="def bad_posonly(__number: int, text: str) -> None: ...",
+                runtime="def bad_posonly(flag, /, text): pass",
+                error="bad_posonly",
+            )
+        yield Case(
+            stub="""
+            class BadMethod:
+                def f(self, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class BadMethod:
+                def f(self, n, text): pass
+            """,
+            error="BadMethod.f",
+        )
+        yield Case(
+            stub="""
+            class GoodDunder:
+                def __exit__(self, t, v, tb) -> None: ...
+            """,
+            runtime="""
+            class GoodDunder:
+                def __exit__(self, exc_type, exc_val, exc_tb): pass
+            """,
+            error=None,
+        )
+
+    @collect_cases
+    def test_arg_kind(self) -> Iterator[Case]:
+        yield Case(
+            stub="def runtime_kwonly(number: int, text: str) -> None: ...",
+            runtime="def runtime_kwonly(number, *, text): pass",
+            error="runtime_kwonly",
+        )
+        yield Case(
+            stub="def stub_kwonly(number: int, *, text: str) -> None: ...",
+            runtime="def stub_kwonly(number, text): pass",
+            error="stub_kwonly",
+        )
+        yield Case(
+            stub="def stub_posonly(__number: int, text: str) -> None: ...",
+            runtime="def stub_posonly(number, text): pass",
+            error="stub_posonly",
+        )
+        if sys.version_info >= (3, 8):
+            yield Case(
+                stub="def good_posonly(__number: int, text: str) -> None: ...",
+                runtime="def good_posonly(number, /, text): pass",
+                error=None,
+            )
+            yield Case(
+                stub="def runtime_posonly(number: int, text: str) -> None: ...",
+                runtime="def runtime_posonly(number, /, text): pass",
+                error="runtime_posonly",
+            )
+
+    @collect_cases
+    def test_default_value(self) -> Iterator[Case]:
+        yield Case(
+            stub="def f1(text: str = ...) -> None: ...",
+            runtime="def f1(text = 'asdf'): pass",
+            error=None,
+        )
+        yield Case(
+            stub="def f2(text: str = ...) -> None: ...", runtime="def f2(text): pass", error="f2"
+        )
+        yield Case(
+            stub="def f3(text: str) -> None: ...",
+            runtime="def f3(text = 'asdf'): pass",
+            error="f3",
+        )
+        yield Case(
+            stub="def f4(text: str = ...) -> None: ...",
+            runtime="def f4(text = None): pass",
+            error="f4",
+        )
+        yield Case(
+            stub="def f5(data: bytes = ...) -> None: ...",
+            runtime="def f5(data = 'asdf'): pass",
+            error="f5",
+        )
+        yield Case(
+            stub="""
+            from typing import TypeVar
+            T = TypeVar("T", bound=str)
+            def f6(text: T = ...) -> None: ...
+            """,
+            runtime="def f6(text = None): pass",
+            error="f6",
+        )
+
+    @collect_cases
+    def test_static_class_method(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            class Good:
+                @classmethod
+                def f(cls, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class Good:
+                @classmethod
+                def f(cls, number, text): pass
+            """,
+            error=None,
+        )
+        yield Case(
+            stub="""
+            class Bad1:
+                def f(cls, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class Bad1:
+                @classmethod
+                def f(cls, number, text): pass
+            """,
+            error="Bad1.f",
+        )
+        yield Case(
+            stub="""
+            class Bad2:
+                @classmethod
+                def f(cls, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class Bad2:
+                @staticmethod
+                def f(self, number, text): pass
+            """,
+            error="Bad2.f",
+        )
+        yield Case(
+            stub="""
+            class Bad3:
+                @staticmethod
+                def f(cls, number: int, text: str) -> None: ...
+            """,
+            runtime="""
+            class Bad3:
+                @classmethod
+                def f(self, number, text): pass
+            """,
+            error="Bad3.f",
+        )
+        yield Case(
+            stub="""
+            class GoodNew:
+                def __new__(cls, *args, **kwargs): ...
+            """,
+            runtime="""
+            class GoodNew:
+                def __new__(cls, *args, **kwargs): pass
+            """,
+            error=None,
+        )
+
+    @collect_cases
+    def test_arg_mismatch(self) -> Iterator[Case]:
+        yield Case(
+            stub="def f1(a, *, b, c) -> None: ...", runtime="def f1(a, *, b, c): pass", error=None
+        )
+        yield Case(
+            stub="def f2(a, *, b) -> None: ...", runtime="def f2(a, *, b, c): pass", error="f2"
+        )
+        yield Case(
+            stub="def f3(a, *, b, c) -> None: ...", runtime="def f3(a, *, b): pass", error="f3"
+        )
+        yield Case(
+            stub="def f4(a, *, b, c) -> None: ...", runtime="def f4(a, b, *, c): pass", error="f4"
+        )
+        yield Case(
+            stub="def f5(a, b, *, c) -> None: ...", runtime="def f5(a, *, b, c): pass", error="f5"
+        )
+
+    @collect_cases
+    def test_varargs_varkwargs(self) -> Iterator[Case]:
+        yield Case(
+            stub="def f1(*args, **kwargs) -> None: ...",
+            runtime="def f1(*args, **kwargs): pass",
+            error=None,
+        )
+        yield Case(
+            stub="def f2(*args, **kwargs) -> None: ...",
+            runtime="def f2(**kwargs): pass",
+            error="f2",
+        )
+        yield Case(
+            stub="def g1(a, b, c, d) -> None: ...", runtime="def g1(a, *args): pass", error=None
+        )
+        yield Case(
+            stub="def g2(a, b, c, d, *args) -> None: ...", runtime="def g2(a): pass", error="g2"
+        )
+        yield Case(
+            stub="def g3(a, b, c, d, *args) -> None: ...",
+            runtime="def g3(a, *args): pass",
+            error=None,
+        )
+        yield Case(
+            stub="def h1(a) -> None: ...", runtime="def h1(a, b, c, d, *args): pass", error="h1"
+        )
+        yield Case(
+            stub="def h2(a, *args) -> None: ...", runtime="def h2(a, b, c, d): pass", error="h2"
+        )
+        yield Case(
+            stub="def h3(a, *args) -> None: ...",
+            runtime="def h3(a, b, c, d, *args): pass",
+            error="h3",
+        )
+        yield Case(
+            stub="def j1(a: int, *args) -> None: ...", runtime="def j1(a): pass", error="j1"
+        )
+        yield Case(
+            stub="def j2(a: int) -> None: ...", runtime="def j2(a, *args): pass", error="j2"
+        )
+        yield Case(
+            stub="def j3(a, b, c) -> None: ...", runtime="def j3(a, *args, c): pass", error="j3"
+        )
+        yield Case(stub="def k1(a, **kwargs) -> None: ...", runtime="def k1(a): pass", error="k1")
+        yield Case(
+            # In theory an error, but led to worse results in practice
+            stub="def k2(a) -> None: ...",
+            runtime="def k2(a, **kwargs): pass",
+            error=None,
+        )
+        yield Case(
+            stub="def k3(a, b) -> None: ...", runtime="def k3(a, **kwargs): pass", error="k3"
+        )
+        yield Case(
+            stub="def k4(a, *, b) -> None: ...", runtime="def k4(a, **kwargs): pass", error=None
+        )
+        yield Case(
+            stub="def k5(a, *, b) -> None: ...",
+            runtime="def k5(a, *, b, c, **kwargs): pass",
+            error="k5",
+        )
+
+    @collect_cases
+    def test_overload(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            from typing import overload
+
+            @overload
+            def f1(a: int, *, c: int = ...) -> int: ...
+            @overload
+            def f1(a: int, b: int, c: int = ...) -> str: ...
+            """,
+            runtime="def f1(a, b = 0, c = 0): pass",
+            error=None,
+        )
+        yield Case(
+            stub="""
+            @overload
+            def f2(a: int, *, c: int = ...) -> int: ...
+            @overload
+            def f2(a: int, b: int, c: int = ...) -> str: ...
+            """,
+            runtime="def f2(a, b, c = 0): pass",
+            error="f2",
+        )
+        yield Case(
+            stub="""
+            @overload
+            def f3(a: int) -> int: ...
+            @overload
+            def f3(a: int, b: str) -> str: ...
+            """,
+            runtime="def f3(a, b = None): pass",
+            error="f3",
+        )
+        yield Case(
+            stub="""
+            @overload
+            def f4(a: int, *args, b: int, **kwargs) -> int: ...
+            @overload
+            def f4(a: str, *args, b: int, **kwargs) -> str: ...
+            """,
+            runtime="def f4(a, *args, b, **kwargs): pass",
+            error=None,
+        )
+        if sys.version_info >= (3, 8):
+            yield Case(
+                stub="""
+                @overload
+                def f5(__a: int) -> int: ...
+                @overload
+                def f5(__b: str) -> str: ...
+                """,
+                runtime="def f5(x, /): pass",
+                error=None,
+            )
+
+    @collect_cases
+    def test_property(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            class Good:
+                @property
+                def f(self) -> int: ...
+            """,
+            runtime="""
+            class Good:
+                @property
+                def f(self) -> int: return 1
+            """,
+            error=None,
+        )
+        yield Case(
+            stub="""
+            class Bad:
+                @property
+                def f(self) -> int: ...
+            """,
+            runtime="""
+            class Bad:
+                def f(self) -> int: return 1
+            """,
+            error="Bad.f",
+        )
+        yield Case(
+            stub="""
+            class GoodReadOnly:
+                @property
+                def f(self) -> int: ...
+            """,
+            runtime="""
+            class GoodReadOnly:
+                f = 1
+            """,
+            error=None,
+        )
+        yield Case(
+            stub="""
+            class BadReadOnly:
+                @property
+                def f(self) -> str: ...
+            """,
+            runtime="""
+            class BadReadOnly:
+                f = 1
+            """,
+            error="BadReadOnly.f",
+        )
+
+    @collect_cases
+    def test_var(self) -> Iterator[Case]:
+        yield Case(stub="x1: int", runtime="x1 = 5", error=None)
+        yield Case(stub="x2: str", runtime="x2 = 5", error="x2")
+        yield Case("from typing import Tuple", "", None)  # dummy case
+        yield Case(
+            stub="""
+            x3: Tuple[int, int]
+            """,
+            runtime="x3 = (1, 3)",
+            error=None,
+        )
+        yield Case(
+            stub="""
+            x4: Tuple[int, int]
+            """,
+            runtime="x4 = (1, 3, 5)",
+            error="x4",
+        )
+        yield Case(
+            stub="""
+            class X:
+                f: int
+            """,
+            runtime="""
+            class X:
+                def __init__(self):
+                    self.f = "asdf"
+            """,
+            error=None,
+        )
+
+    @collect_cases
+    def test_enum(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            import enum
+            class X(enum.Enum):
+                a: int
+                b: str
+                c: str
+            """,
+            runtime="""
+            import enum
+            class X(enum.Enum):
+                a = 1
+                b = "asdf"
+                c = 2
+            """,
+            error="X.c",
+        )
+
+    @collect_cases
+    def test_decorator(self) -> Iterator[Case]:
+        yield Case(
+            stub="""
+            from typing import Any, Callable
+            def decorator(f: Callable[[], int]) -> Callable[..., Any]: ...
+            @decorator
+            def f() -> Any: ...
+            """,
+            runtime="""
+            def decorator(f): return f
+            @decorator
+            def f(): return 3
+            """,
+            error=None,
+        )
+
+    @collect_cases
+    def test_missing(self) -> Iterator[Case]:
+        yield Case(stub="x = 5", runtime="", error="x")
+        yield Case(stub="def f(): ...", runtime="", error="f")
+        yield Case(stub="class X: ...", runtime="", error="X")
+        yield Case(
+            stub="""
+            from typing import overload
+            @overload
+            def h(x: int): ...
+            @overload
+            def h(x: str): ...
+            """,
+            runtime="",
+            error="h",
+        )
+        yield Case("", "__all__ = []", None)  # dummy case
+        yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y")
+        yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g")
+
+
+def remove_color_code(s: str) -> str:
+    return re.sub("\\x1b.*?m", "", s)  # this works!
+
+
+class StubtestMiscUnit(unittest.TestCase):
+    def test_output(self) -> None:
+        output = run_stubtest(
+            stub="def bad(number: int, text: str) -> None: ...",
+            runtime="def bad(num, text): pass",
+            options=[],
+        )
+        expected = (
+            'error: {0}.bad is inconsistent, stub argument "number" differs from runtime '
+            'argument "num"\nStub: at line 1\ndef (number: builtins.int, text: builtins.str)\n'
+            "Runtime: at line 1 in file {0}.py\ndef (num, text)\n\n".format(TEST_MODULE_NAME)
+        )
+        assert remove_color_code(output) == expected
+
+        output = run_stubtest(
+            stub="def bad(number: int, text: str) -> None: ...",
+            runtime="def bad(num, text): pass",
+            options=["--concise"],
+        )
+        expected = (
+            "{}.bad is inconsistent, "
+            'stub argument "number" differs from runtime argument "num"\n'.format(TEST_MODULE_NAME)
+        )
+        assert remove_color_code(output) == expected
+
+    def test_ignore_flags(self) -> None:
+        output = run_stubtest(
+            stub="", runtime="__all__ = ['f']\ndef f(): pass", options=["--ignore-missing-stub"]
+        )
+        assert not output
+
+        output = run_stubtest(
+            stub="def f(__a): ...", runtime="def f(a): pass", options=["--ignore-positional-only"]
+        )
+        assert not output
+
+    def test_whitelist(self) -> None:
+        with tempfile.NamedTemporaryFile() as f:
+            f.write("{}.bad\n# a comment".format(TEST_MODULE_NAME).encode("utf-8"))
+            f.flush()
+
+            output = run_stubtest(
+                stub="def bad(number: int, text: str) -> None: ...",
+                runtime="def bad(num, text) -> None: pass",
+                options=["--whitelist", f.name],
+            )
+            assert not output
+
+            output = run_stubtest(stub="", runtime="", options=["--whitelist", f.name])
+            assert output == "note: unused whitelist entry {}.bad\n".format(TEST_MODULE_NAME)
+
+    def test_mypy_build(self) -> None:
+        output = run_stubtest(stub="+", runtime="", options=[])
+        assert remove_color_code(output) == (
+            "error: failed mypy compile.\n{}.pyi:1: "
+            "error: invalid syntax\n".format(TEST_MODULE_NAME)
+        )
+
+        output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[])
+        assert remove_color_code(output) == (
+            "error: failed mypy build.\n{}.pyi:2: "
+            "error: Name 'f' already defined on line 1\n".format(TEST_MODULE_NAME)
+        )
+
+    def test_missing_stubs(self) -> None:
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output):
+            test_stubs(parse_options(["not_a_module"]))
+        assert "error: not_a_module failed to find stubs" in remove_color_code(output.getvalue())
+
+    def test_get_typeshed_stdlib_modules(self) -> None:
+        stdlib = mypy.stubtest.get_typeshed_stdlib_modules(None)
+        assert "builtins" in stdlib
+        assert "os" in stdlib
+
+    def test_signature(self) -> None:
+        def f(a: int, b: int, *, c: int, d: int = 0, **kwargs: Any) -> None:
+            pass
+
+        assert (
+            str(mypy.stubtest.Signature.from_inspect_signature(inspect.signature(f)))
+            == "def (a, b, *, c, d = ..., **kwargs)"
+        )
+
+
+class StubtestIntegration(unittest.TestCase):
+    def test_typeshed(self) -> None:
+        # check we don't crash while checking typeshed
+        test_stubs(parse_options(["--check-typeshed"]))

From b788ddb11a9b50042df81e08ff6ca51c5a8c903c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 12:06:14 -0800
Subject: [PATCH 15/22] stubtest: [minor] catch more warnings

---
 mypy/stubtest.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index d7369e95c940..f4256283d292 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -157,12 +157,13 @@ def test_module(module_name: str) -> Iterator[Error]:
         return
 
     try:
-        runtime = importlib.import_module(module_name)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            runtime = importlib.import_module(module_name)
     except Exception as e:
         yield Error([module_name], "failed to import: {}".format(e), stub, MISSING)
         return
 
-    # collections likes to warn us about the things we're doing
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         yield from verify(stub, runtime, [module_name])

From 882fb46fc939f4ae7d9cc31adfd82cc1fb80fbb2 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 12:49:53 -0800
Subject: [PATCH 16/22] stubtest: add annotation to help mypyc out

---
 mypy/stubtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index f4256283d292..aa41e593a699 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -96,7 +96,7 @@ def get_description(self, concise: bool = False) -> str:
             return _style(self.object_desc, bold=True) + " " + self.message
 
         stub_line = None
-        stub_file = None
+        stub_file = None  # type: None
         if not isinstance(self.stub_object, Missing):
             stub_line = self.stub_object.line
         # TODO: Find a way of getting the stub file

From 1641fe47802650235ae7ab1c5f2c4a9cd7ad2860 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 12:33:07 -0800
Subject: [PATCH 17/22] stubtest: replace use of find for Windows compatibility

This is nicer too
---
 mypy/stubtest.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index aa41e593a699..1f49e2ee5d21 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -9,7 +9,6 @@
 import enum
 import importlib
 import inspect
-import subprocess
 import sys
 import types
 import warnings
@@ -970,9 +969,7 @@ def get_typeshed_stdlib_modules(custom_typeshed_dir: Optional[str]) -> List[str]
     for version in versions:
         base = typeshed_dir / "stdlib" / version
         if base.exists():
-            output = subprocess.check_output(["find", str(base), "-type", "f"]).decode("utf-8")
-            paths = [Path(p) for p in output.splitlines()]
-            for path in paths:
+            for path in base.rglob("*.pyi"):
                 if path.stem == "__init__":
                     path = path.parent
                 modules.append(".".join(path.relative_to(base).parts[:-1] + (path.stem,)))

From 7e7fe2509874ee324fecf4058f6fddb68604902d Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 12:48:19 -0800
Subject: [PATCH 18/22] teststubtest: NamedTemporaryFile doesn't work on
 Windows

---
 mypy/test/teststubtest.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/mypy/test/teststubtest.py b/mypy/test/teststubtest.py
index 069de7e3bfd7..d48fb12ccd10 100644
--- a/mypy/test/teststubtest.py
+++ b/mypy/test/teststubtest.py
@@ -69,7 +69,7 @@ def test(*args: Any, **kwargs: Any) -> None:
         output = run_stubtest(
             stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
             runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
-            options=["--generate-whitelist"]
+            options=["--generate-whitelist"],
         )
 
         actual_errors = set(output.splitlines())
@@ -609,19 +609,23 @@ def test_ignore_flags(self) -> None:
         assert not output
 
     def test_whitelist(self) -> None:
-        with tempfile.NamedTemporaryFile() as f:
-            f.write("{}.bad\n# a comment".format(TEST_MODULE_NAME).encode("utf-8"))
-            f.flush()
+        # Can't use this as a context because Windows
+        whitelist = tempfile.NamedTemporaryFile(mode="w", delete=False)
+        try:
+            with whitelist:
+                whitelist.write("{}.bad\n# a comment".format(TEST_MODULE_NAME))
 
             output = run_stubtest(
                 stub="def bad(number: int, text: str) -> None: ...",
                 runtime="def bad(num, text) -> None: pass",
-                options=["--whitelist", f.name],
+                options=["--whitelist", whitelist.name],
             )
             assert not output
 
-            output = run_stubtest(stub="", runtime="", options=["--whitelist", f.name])
+            output = run_stubtest(stub="", runtime="", options=["--whitelist", whitelist.name])
             assert output == "note: unused whitelist entry {}.bad\n".format(TEST_MODULE_NAME)
+        finally:
+            os.unlink(whitelist.name)
 
     def test_mypy_build(self) -> None:
         output = run_stubtest(stub="+", runtime="", options=[])

From 7f6e0d5f499fb24bd7ca0c1c0f8dbdb5b01fbcbe Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 14:14:13 -0800
Subject: [PATCH 19/22] stubtest: [minor] make str(signature) deterministic

---
 mypy/stubtest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 1f49e2ee5d21..0637a3b3fa79 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -380,11 +380,12 @@ def get_desc(arg: Any) -> str:
                 + (" = ..." if has_default(arg) else "")
             )
 
+        kw_only = sorted(self.kwonly.values(), key=lambda a: (has_default(a), get_name(a)))
         ret = "def ("
         ret += ", ".join(
             [get_desc(arg) for arg in self.pos]
             + (["*" + get_name(self.varpos)] if self.varpos else (["*"] if self.kwonly else []))
-            + [get_desc(arg) for arg in self.kwonly.values()]
+            + [get_desc(arg) for arg in kw_only]
             + (["**" + get_name(self.varkw)] if self.varkw else [])
         )
         ret += ")"

From 436586f8ffb34016ddacd146f28e90ef1aea9d36 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 7 Feb 2020 16:48:00 -0800
Subject: [PATCH 20/22] mypyc: exclude stubtest.py

---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index 37995c3d7018..a393c4035205 100644
--- a/setup.py
+++ b/setup.py
@@ -100,6 +100,9 @@ def run(self):
         # We don't populate __file__ properly at the top level or something?
         # Also I think there would be problems with how we generate version.py.
         'version.py',
+
+        # Written by someone who doesn't know how to deal with mypyc
+        'stubtest.py',
     )) + (
         # Don't want to grab this accidentally
         os.path.join('mypyc', 'lib-rt', 'setup.py'),

From 6d1f84fdccd26d1f36491a34f18054ef3f97fc65 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 8 Feb 2020 14:35:12 -0800
Subject: [PATCH 21/22] stubtest: fix LiteralType misuse for mypyc

EAFP, since bytes and enums should work, and default value error
messages can be more informative with literal types
---
 mypy/stubtest.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 0637a3b3fa79..951c86a89f2c 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -881,17 +881,22 @@ def anytype() -> mypy.types.AnyType:
 
     if isinstance(runtime, tuple):
         # Special case tuples so we construct a valid mypy.types.TupleType
-        opt_items = [get_mypy_type_of_runtime_value(v) for v in runtime]
-        items = [(i if i is not None else anytype()) for i in opt_items]
+        optional_items = [get_mypy_type_of_runtime_value(v) for v in runtime]
+        items = [(i if i is not None else anytype()) for i in optional_items]
         fallback = mypy.types.Instance(type_info, [anytype()])
         return mypy.types.TupleType(items, fallback)
 
-    # Technically, Literals are supposed to be only bool, int, str or bytes, but this
-    # seems to work fine
-    return mypy.types.LiteralType(
-        value=runtime,
-        fallback=mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars]),
-    )
+    fallback = mypy.types.Instance(type_info, [anytype() for _ in type_info.type_vars])
+    try:
+        # Literals are supposed to be only bool, int, str, bytes or enums, but this seems to work
+        # well (when not using mypyc, for which bytes and enums are also problematic).
+        return mypy.types.LiteralType(
+            value=runtime,
+            fallback=fallback,
+        )
+    except TypeError:
+        # Ask for forgiveness if we're using mypyc.
+        return fallback
 
 
 _all_stubs = {}  # type: Dict[str, nodes.MypyFile]

From 8fda109d6b07a23670cc1148155e4696d94654e0 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Mon, 10 Feb 2020 16:09:05 -0800
Subject: [PATCH 22/22] stubtest: work around a bug in early versions of py35

---
 mypy/stubtest.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/mypy/stubtest.py b/mypy/stubtest.py
index 951c86a89f2c..8d87f6e7f5a0 100644
--- a/mypy/stubtest.py
+++ b/mypy/stubtest.py
@@ -36,7 +36,17 @@ def __repr__(self) -> str:
 MISSING = Missing()
 
 T = TypeVar("T")
-MaybeMissing = Union[T, Missing]
+if sys.version_info >= (3, 5, 3):
+    MaybeMissing = Union[T, Missing]
+else:
+    # work around a bug in 3.5.2 and earlier's typing.py
+    class MaybeMissingMeta(type):
+        def __getitem__(self, arg: Any) -> Any:
+            return Union[arg, Missing]
+
+    class MaybeMissing(metaclass=MaybeMissingMeta):  # type: ignore
+        pass
+
 
 _formatter = FancyFormatter(sys.stdout, sys.stderr, False)