From e4b5274c110edfb8f70f4f8c78c8527d12f7c22f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 13 Dec 2017 21:49:15 -0800 Subject: [PATCH 001/100] Simplify find_module cache and document PEP 561 --- docs/source/checking_installed_packages.rst | 36 +++++++++++++++++++++ mypy/build.py | 8 ++--- 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 docs/source/checking_installed_packages.rst diff --git a/docs/source/checking_installed_packages.rst b/docs/source/checking_installed_packages.rst new file mode 100644 index 000000000000..9e95a2144a2b --- /dev/null +++ b/docs/source/checking_installed_packages.rst @@ -0,0 +1,36 @@ +.. _checking-installed-packages: + +Using and Creating Typed Packages for Distribution +================================================== + +`PEP 561 `_ specifies how to mark +a package as supporting type checking. Below is a summary of how to use this +feature and create PEP 561 compatible packages. + + +Creating Typed Packages +*********************** + +For a typed package to be picked up by mypy, you must put a file named +``py.typed`` in each top level package installed. For example, your directory +structure may look like: + +.. code:: + + setup.py + my_pkg/ + __init__.py + py.typed + file.py + +Note that if ``my_pkg`` has subpackages, they do *not* need to have their own +``py.typed`` file marker. + + +Checking Typed Packages +*********************** + +Installed packages for the Python being checked should be picked up if they +opt into type checking. If the Python version being checked is different +from the version running mypy, you also need to point mypy to find it via +``--python``. \ No newline at end of file diff --git a/mypy/build.py b/mypy/build.py index 21fde0239c16..d48306348d25 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -739,7 +739,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: # elements of lib_path have even the subdirectory they'd need for the module # to exist. This is shared among different module ids when they differ only # in the last component. -find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] +find_module_dir_cache = {} # type: Dict[str, List[str]] # Cache directory listings. We assume that while one os.listdir() # call may be more expensive than one os.stat() call, a small number @@ -808,7 +808,7 @@ def find() -> Optional[str]: # that will require the same subdirectory. components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if (dir_chain, lib_path) not in find_module_dir_cache: + if dir_chain not in find_module_dir_cache: dirs = [] for pathitem in lib_path: # e.g., '/usr/lib/python3.4/foo/bar' @@ -819,8 +819,8 @@ def find() -> Optional[str]: find_module_isdir_cache[pathitem, dir_chain] = isdir if isdir: dirs.append(dir) - find_module_dir_cache[dir_chain, lib_path] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain, lib_path] + find_module_dir_cache[dir_chain] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain] # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the From 6b6a97d2533d7a5c3a8416befcfd8e6e2039cf3d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 15 Dec 2017 13:00:42 -0800 Subject: [PATCH 002/100] More work on PEP 561 impl --- mypy/build.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index d48306348d25..92ee66d880e5 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -797,9 +797,22 @@ def is_file(path: str) -> bool: return res +package_dirs_cache = [] + + +def get_package_dirs() -> List[str]: + try: + user_dir = site.getusersitepackages() + package_dirs = site.getsitepackages() + [user_dir] + except AttributeError: + package_dirs = [get_python_lib()] + return package_dirs + + def find_module(id: str, lib_path_arg: Iterable[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) + package_dirs = get_package_dirs() def find() -> Optional[str]: # If we're looking for a module like 'foo.bar.baz', it's likely that most of the @@ -810,6 +823,8 @@ def find() -> Optional[str]: dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' if dir_chain not in find_module_dir_cache: dirs = [] + + # Regular packages on the PATH for pathitem in lib_path: # e.g., '/usr/lib/python3.4/foo/bar' isdir = find_module_isdir_cache.get((pathitem, dir_chain)) @@ -819,6 +834,19 @@ def find() -> Optional[str]: find_module_isdir_cache[pathitem, dir_chain] = isdir if isdir: dirs.append(dir) + + # Third-party stub/typed packages + for pkg_dir in package_dirs: + stub_name = components[0] + '_stubs' + stub_pkg = os.path.join(pkg_dir, stub_name) + dir = os.path.join(pkg_dir, dir_chain) + if os.path.isfile(os.path.join(stub_pkg, 'py.typed')): + components[0] = stub_name + dirs.append(os.path.join(pkg_dir, os.sep.join(components[:-1]))) + elif os.path.isfile(os.path.join(pkg_dir, components[0], 'py.typed')) \ + and os.path.isdir(dir): + dirs.append(os.path.join(pkg_dir, dir_chain)) + find_module_dir_cache[dir_chain] = dirs candidate_base_dirs = find_module_dir_cache[dir_chain] From 2c936e10ba04419584a4506470561a7420edf685 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 16 Dec 2017 17:19:00 -0600 Subject: [PATCH 003/100] Scaffold testing and fix bugs. --- mypy/build.py | 96 +++++++++++-------- test-data/packages/setup.py | 3 + .../packages/typed_pkg_stubs/__init__.pyi | 0 test-data/packages/typedpkg/__init__.py | 0 4 files changed, 59 insertions(+), 40 deletions(-) create mode 100644 test-data/packages/setup.py create mode 100644 test-data/packages/typed_pkg_stubs/__init__.pyi create mode 100644 test-data/packages/typedpkg/__init__.py diff --git a/mypy/build.py b/mypy/build.py index 92ee66d880e5..d6faa8734679 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -754,6 +754,9 @@ def remove_cwd_prefix_from_path(p: str) -> str: # Cache for isdir(join(head, tail)) find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] +# Cache packages for Python executable +package_dirs_cache = [] # type: List[str] + def find_module_clear_caches() -> None: find_module_cache.clear() @@ -797,19 +800,28 @@ def is_file(path: str) -> bool: return res -package_dirs_cache = [] - - -def get_package_dirs() -> List[str]: - try: - user_dir = site.getusersitepackages() - package_dirs = site.getsitepackages() + [user_dir] - except AttributeError: - package_dirs = [get_python_lib()] - return package_dirs +def get_package_dirs(python: Optional[str]) -> List[str]: + """Find package directories for given python (default to Python running + mypy.""" + global package_dirs_cache + if package_dirs_cache: + return package_dirs_cache + if python: + # Use subprocess to get the package directory of given Python + # executable + return [] + else: + # Use running Python's package dirs + try: + user_dir = site.getusersitepackages() + package_dirs = site.getsitepackages() + [user_dir] + except AttributeError: + package_dirs = [get_python_lib()] + package_dirs_cache = package_dirs + return package_dirs -def find_module(id: str, lib_path_arg: Iterable[str]) -> Optional[str]: +def find_module(id: str, lib_path_arg: Iterable[str], python: str) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) package_dirs = get_package_dirs() @@ -821,35 +833,37 @@ def find() -> Optional[str]: # that will require the same subdirectory. components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if dir_chain not in find_module_dir_cache: - dirs = [] - - # Regular packages on the PATH - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - isdir = find_module_isdir_cache.get((pathitem, dir_chain)) - if isdir is None: - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - isdir = os.path.isdir(dir) - find_module_isdir_cache[pathitem, dir_chain] = isdir - if isdir: - dirs.append(dir) - - # Third-party stub/typed packages - for pkg_dir in package_dirs: - stub_name = components[0] + '_stubs' - stub_pkg = os.path.join(pkg_dir, stub_name) - dir = os.path.join(pkg_dir, dir_chain) - if os.path.isfile(os.path.join(stub_pkg, 'py.typed')): - components[0] = stub_name - dirs.append(os.path.join(pkg_dir, os.sep.join(components[:-1]))) - elif os.path.isfile(os.path.join(pkg_dir, components[0], 'py.typed')) \ - and os.path.isdir(dir): - dirs.append(os.path.join(pkg_dir, dir_chain)) - - find_module_dir_cache[dir_chain] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain] - + if dir_chain: + if dir_chain not in find_module_dir_cache: + dirs = [] + + # Regular packages on the PATH + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + isdir = find_module_isdir_cache.get((pathitem, dir_chain)) + if isdir is None: + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + isdir = os.path.isdir(dir) + find_module_isdir_cache[pathitem, dir_chain] = isdir + if isdir: + dirs.append(dir) + + # Third-party stub/typed packages + for pkg_dir in package_dirs: + stub_name = components[0] + '_stubs' + stub_pkg = os.path.join(pkg_dir, stub_name) + dir = os.path.join(pkg_dir, dir_chain) + if os.path.isfile(os.path.join(stub_pkg, 'py.typed')): + components[0] = stub_name + dirs.append(os.path.join(pkg_dir, os.sep.join(components[:-1]))) + elif os.path.isfile(os.path.join(pkg_dir, components[0], 'py.typed')) \ + and os.path.isdir(dir): + dirs.append(os.path.join(pkg_dir, dir_chain)) + + find_module_dir_cache[dir_chain] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain] + else: + candidate_base_dirs = lib_path + tuple(package_dirs) # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the # elements of lib_path. This is probably much shorter than lib_path itself. @@ -1551,6 +1565,8 @@ def __init__(self, file_id = '__builtin__' path = find_module(file_id, manager.lib_path) if path: + if any((path.startswith(d) for d in package_dirs_cache)): + self.ignore_all = True # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze # - silent -> analyze but silence errors diff --git a/test-data/packages/setup.py b/test-data/packages/setup.py new file mode 100644 index 000000000000..7cbe1e231887 --- /dev/null +++ b/test-data/packages/setup.py @@ -0,0 +1,3 @@ +""" +This setup file install packages to test mypy's PEP 561 +""" \ No newline at end of file diff --git a/test-data/packages/typed_pkg_stubs/__init__.pyi b/test-data/packages/typed_pkg_stubs/__init__.pyi new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/__init__.py b/test-data/packages/typedpkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 8f64f81172e8726116b11c5887c63b98d66b4b1c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 16 Dec 2017 22:15:31 -0500 Subject: [PATCH 004/100] Add python arg for find_module --- mypy/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index d6faa8734679..9ab8b7e38ba6 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -802,7 +802,7 @@ def is_file(path: str) -> bool: def get_package_dirs(python: Optional[str]) -> List[str]: """Find package directories for given python (default to Python running - mypy.""" + mypy).""" global package_dirs_cache if package_dirs_cache: return package_dirs_cache @@ -821,10 +821,10 @@ def get_package_dirs(python: Optional[str]) -> List[str]: return package_dirs -def find_module(id: str, lib_path_arg: Iterable[str], python: str) -> Optional[str]: +def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - package_dirs = get_package_dirs() + package_dirs = get_package_dirs(python) def find() -> Optional[str]: # If we're looking for a module like 'foo.bar.baz', it's likely that most of the From ddc526faac6e74421db4e4e37601d68ed90082eb Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 16 Dec 2017 22:56:17 -0500 Subject: [PATCH 005/100] Get packages from Python executables --- mypy/build.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 9ab8b7e38ba6..c20811a7ce1c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -21,6 +21,7 @@ import re import site import stat +from subprocess import check_output, STDOUT import sys import time from os.path import dirname, basename @@ -800,16 +801,45 @@ def is_file(path: str) -> bool: return res +SITE_PACKAGE_COMMANDS = ( + # User site packages + '"import site;print(site.getusersitepackages())"', + # Usual site packages/python directory + '"import site;print(*site.getsitepackages(), sep=\'\\n\')"', + # for virtualenvs + '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"', +) + + +def call_python(python: str, command) -> str: + return check_output([python, '-c', command], stderr=STDOUT).decode('UTF-8') + + def get_package_dirs(python: Optional[str]) -> List[str]: """Find package directories for given python (default to Python running mypy).""" global package_dirs_cache if package_dirs_cache: return package_dirs_cache + package_dirs = [] if python: # Use subprocess to get the package directory of given Python # executable - return [] + check = check_output([python, '-V'], stderr=STDOUT).decode('UTF-8') + if not check.startswith('Python'): + return package_dirs + # If we have a working python executable, query information from it + for command in SITE_PACKAGE_COMMANDS[:2]: + output = call_python(python, command) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) + if not package_dirs: + # if no paths are found, we fall back on sysconfig + output = call_python(python, SITE_PACKAGE_COMMANDS[2]) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) else: # Use running Python's package dirs try: @@ -817,8 +847,8 @@ def get_package_dirs(python: Optional[str]) -> List[str]: package_dirs = site.getsitepackages() + [user_dir] except AttributeError: package_dirs = [get_python_lib()] - package_dirs_cache = package_dirs - return package_dirs + package_dirs_cache = package_dirs + return package_dirs def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: From 329dc6843b319473134a0894db0c66bc254554e6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 00:14:52 -0500 Subject: [PATCH 006/100] Add support for non-running Python, fix bug in impl --- mypy/build.py | 40 +++++++++---------- mypy/main.py | 3 +- mypy/options.py | 1 + mypy/stubgen.py | 3 +- test-data/packages/setup.py | 7 +++- test-data/packages/typed_pkg_stubs/sample.pyi | 2 + test-data/packages/typedpkg/sample.py | 4 ++ 7 files changed, 34 insertions(+), 26 deletions(-) create mode 100644 test-data/packages/typed_pkg_stubs/sample.pyi create mode 100644 test-data/packages/typedpkg/sample.py diff --git a/mypy/build.py b/mypy/build.py index c20811a7ce1c..1ecba7cf8ae4 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -631,7 +631,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path) is not None + return find_module(id, self.lib_path, self.options.python) is not None def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile: """Parse the source of a file with the given name. @@ -803,15 +803,13 @@ def is_file(path: str) -> bool: SITE_PACKAGE_COMMANDS = ( # User site packages - '"import site;print(site.getusersitepackages())"', - # Usual site packages/python directory - '"import site;print(*site.getsitepackages(), sep=\'\\n\')"', + '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"', # for virtualenvs '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"', ) -def call_python(python: str, command) -> str: +def call_python(python: str, command: str) -> str: return check_output([python, '-c', command], stderr=STDOUT).decode('UTF-8') @@ -821,7 +819,7 @@ def get_package_dirs(python: Optional[str]) -> List[str]: global package_dirs_cache if package_dirs_cache: return package_dirs_cache - package_dirs = [] + package_dirs = [] # type: List[str] if python: # Use subprocess to get the package directory of given Python # executable @@ -829,14 +827,13 @@ def get_package_dirs(python: Optional[str]) -> List[str]: if not check.startswith('Python'): return package_dirs # If we have a working python executable, query information from it - for command in SITE_PACKAGE_COMMANDS[:2]: - output = call_python(python, command) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) + output = call_python(python, SITE_PACKAGE_COMMANDS[0]) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) if not package_dirs: # if no paths are found, we fall back on sysconfig - output = call_python(python, SITE_PACKAGE_COMMANDS[2]) + output = call_python(python, SITE_PACKAGE_COMMANDS[1]) for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) @@ -882,18 +879,17 @@ def find() -> Optional[str]: for pkg_dir in package_dirs: stub_name = components[0] + '_stubs' stub_pkg = os.path.join(pkg_dir, stub_name) - dir = os.path.join(pkg_dir, dir_chain) + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') if os.path.isfile(os.path.join(stub_pkg, 'py.typed')): components[0] = stub_name dirs.append(os.path.join(pkg_dir, os.sep.join(components[:-1]))) - elif os.path.isfile(os.path.join(pkg_dir, components[0], 'py.typed')) \ - and os.path.isdir(dir): + elif os.path.isfile(typed_file): dirs.append(os.path.join(pkg_dir, dir_chain)) find_module_dir_cache[dir_chain] = dirs candidate_base_dirs = find_module_dir_cache[dir_chain] else: - candidate_base_dirs = lib_path + tuple(package_dirs) + candidate_base_dirs = list(lib_path_arg) + package_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the # elements of lib_path. This is probably much shorter than lib_path itself. @@ -920,8 +916,8 @@ def find() -> Optional[str]: return find_module_cache[key] -def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path) +def find_modules_recursive(module: str, lib_path: List[str], python: Optional[str]) -> List[BuildSource]: + module_path = find_module(module, lib_path, python) if not module_path: return [] result = [BuildSource(module_path, module, None)] @@ -941,14 +937,14 @@ def find_modules_recursive(module: str, lib_path: List[str]) -> List[BuildSource (os.path.isfile(os.path.join(abs_path, '__init__.py')) or os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path) + result += find_modules_recursive(module + '.' + item, lib_path, python) elif item != '__init__.py' and item != '__init__.pyi' and \ item.endswith(('.py', '.pyi')): mod = item.split('.')[0] if mod not in hits: hits.add(mod) - result += find_modules_recursive( - module + '.' + mod, lib_path) + result += find_modules_recursive(module + '.' + mod, + lib_path, python) return result @@ -1593,7 +1589,7 @@ def __init__(self, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = find_module(file_id, manager.lib_path) + path = find_module(file_id, manager.lib_path, manager.options.python) if path: if any((path.startswith(d) for d in package_dirs_cache)): self.ignore_all = True diff --git a/mypy/main.py b/mypy/main.py index ef30c02d7ef4..5134a4ebf02e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -245,6 +245,7 @@ def add_invertible_flag(flag: str, version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y') + parser.add_argument('--python', action='store', help="Point to a Python executable.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -508,7 +509,7 @@ def add_invertible_flag(flag: str, .format(special_opts.package)) options.build_type = BuildType.MODULE lib_path = [os.getcwd()] + build.mypy_path() - targets = build.find_modules_recursive(special_opts.package, lib_path) + targets = build.find_modules_recursive(special_opts.package, lib_path, options.python) if not targets: fail("Can't find package '{}'".format(special_opts.package)) return targets, options diff --git a/mypy/options.py b/mypy/options.py index dd9bfe08c095..b88e4d99e946 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -51,6 +51,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = defaults.PYTHON3_VERSION + self.python = None # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 707fb32f06d3..894da92ea025 100644 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -156,7 +156,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], module_all = getattr(mod, '__all__', None) else: # Find module by going through search path. - module_path = mypy.build.find_module(module, ['.'] + search_path) + module_path = mypy.build.find_module(module, ['.'] + search_path, + interpreter) if not module_path: raise SystemExit( "Can't find module '{}' (consider using --search-path)".format(module)) diff --git a/test-data/packages/setup.py b/test-data/packages/setup.py index 7cbe1e231887..bab24a0e8051 100644 --- a/test-data/packages/setup.py +++ b/test-data/packages/setup.py @@ -1,3 +1,6 @@ """ -This setup file install packages to test mypy's PEP 561 -""" \ No newline at end of file +This setup file install packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + diff --git a/test-data/packages/typed_pkg_stubs/sample.pyi b/test-data/packages/typed_pkg_stubs/sample.pyi new file mode 100644 index 000000000000..ffd321129946 --- /dev/null +++ b/test-data/packages/typed_pkg_stubs/sample.pyi @@ -0,0 +1,2 @@ +from typing import Iterable, List +def ex(a: Iterable[str]) -> List[str]: ... \ No newline at end of file diff --git a/test-data/packages/typedpkg/sample.py b/test-data/packages/typedpkg/sample.py new file mode 100644 index 000000000000..fdd21fcfc321 --- /dev/null +++ b/test-data/packages/typedpkg/sample.py @@ -0,0 +1,4 @@ +from typing import Iterable, List +def ex(a: Iterable[str]) -> List[str]: + """Example typed package. This intentionally has an error.""" + return a + ['Hello'] \ No newline at end of file From 55766296f8a55d09d406a7eebd2e2e41baa18e80 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 00:23:59 -0500 Subject: [PATCH 007/100] Fix mypy self check --- mypy/build.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 1ecba7cf8ae4..13892908fe6e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -848,7 +848,8 @@ def get_package_dirs(python: Optional[str]) -> List[str]: return package_dirs -def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: +def find_module(id: str, lib_path_arg: Iterable[str], + python: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) package_dirs = get_package_dirs(python) @@ -916,7 +917,8 @@ def find() -> Optional[str]: return find_module_cache[key] -def find_modules_recursive(module: str, lib_path: List[str], python: Optional[str]) -> List[BuildSource]: +def find_modules_recursive(module: str, lib_path: List[str], + python: Optional[str]) -> List[BuildSource]: module_path = find_module(module, lib_path, python) if not module_path: return [] From 23977633b9da6a07303369f57da397f57c7f7eca Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 18:50:27 -0500 Subject: [PATCH 008/100] Fix weird subprocess bug that works in pydevd, but not otherwise --- mypy/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 13892908fe6e..874ed502046d 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -810,7 +810,8 @@ def is_file(path: str) -> bool: def call_python(python: str, command: str) -> str: - return check_output([python, '-c', command], stderr=STDOUT).decode('UTF-8') + return check_output(python + ' -c ' + command, + stderr=STDOUT).decode('UTF-8') def get_package_dirs(python: Optional[str]) -> List[str]: From fda2a2c3ba0b3e0231f998bee8fb4712ad700172 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 19:32:00 -0500 Subject: [PATCH 009/100] Add docs for PEP561 impl --- docs/source/index.rst | 1 + docs/source/installed_packages.rst | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docs/source/installed_packages.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 90cc74941da8..582c1c4ee1b8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -30,6 +30,7 @@ Mypy is a static type checker for Python. command_line config_file python36 + installed_packages faq cheat_sheet cheat_sheet_py3 diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst new file mode 100644 index 000000000000..df82aa926321 --- /dev/null +++ b/docs/source/installed_packages.rst @@ -0,0 +1,20 @@ +Using Installed Packages +======================== + + +Making PEP 561 compatible packages +********************************** + +Packages that supply type information should put a ``py.typed``. + +Using PEP 561 compatible packages with mypy +******************************************* + +PEP 561 specifies a format to indicate a package installed in site-packages or +dist-packages supports providing type information. Generally, you do not need +to do anything to use these packages. They should be automatically picked up by +mypy and used for type checking. + +If you use mypy to type check a Python other than the version running mypy, you +can use the ``--python`` flag to point to the executable, and mypy will pick up +the site/dist-packages for the Python executable pointed to. \ No newline at end of file From 59f0c49b2ae9cfec1b55ecc61aea4cebca6fddf1 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 21:19:42 -0500 Subject: [PATCH 010/100] Add initial tests for PEP 561 checking The tests install a typed package and verify that the modules can be found correctly. It then does the same for a stub package. --- mypy/test/config.py | 1 + mypy/test/helpers.py | 31 ++++++++++++- mypy/test/testpackages.py | 45 +++++++++++++++++++ mypy/test/testpythoneval.py | 33 +------------- runtests.py | 1 + test-data/{packages/typedpkg => }/__init__.py | 0 test-data/packages/setup.py | 6 --- test-data/packages/stubs/setup.py | 13 ++++++ .../typedpkg_stubs}/__init__.pyi | 0 .../packages/stubs/typedpkg_stubs/py.typed | 0 .../typedpkg_stubs}/sample.pyi | 0 test-data/packages/typed/setup.py | 13 ++++++ test-data/packages/typed/typedpkg/__init__.py | 0 .../packages/{ => typed}/typedpkg/sample.py | 2 + 14 files changed, 107 insertions(+), 38 deletions(-) create mode 100644 mypy/test/testpackages.py rename test-data/{packages/typedpkg => }/__init__.py (100%) delete mode 100644 test-data/packages/setup.py create mode 100644 test-data/packages/stubs/setup.py rename test-data/packages/{typed_pkg_stubs => stubs/typedpkg_stubs}/__init__.pyi (100%) create mode 100644 test-data/packages/stubs/typedpkg_stubs/py.typed rename test-data/packages/{typed_pkg_stubs => stubs/typedpkg_stubs}/sample.pyi (100%) create mode 100644 test-data/packages/typed/setup.py create mode 100644 test-data/packages/typed/typedpkg/__init__.py rename test-data/packages/{ => typed}/typedpkg/sample.py (98%) diff --git a/mypy/test/config.py b/mypy/test/config.py index 5dbe791e593a..5e6792fb0d57 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -5,6 +5,7 @@ # Location of test data files such as test case descriptions. test_data_prefix = os.path.join(PREFIX, 'test-data', 'unit') +package_path = os.path.join(PREFIX, 'test-data', 'packages') assert os.path.isdir(test_data_prefix), \ 'Test data prefix ({}) not set correctly'.format(test_data_prefix) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 8bd3a615694d..b69d90d105f3 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -1,12 +1,14 @@ import os import re +import subprocess import sys import time -from typing import List, Dict, Tuple, Callable, Any +from typing import Any, Callable, Dict, List, Optional, Tuple from mypy import defaults from mypy.myunit import AssertionFailure +from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase @@ -308,3 +310,30 @@ def retry_on_error(func: Callable[[], Any], max_wait: float = 1.0) -> None: # Done enough waiting, the error seems persistent. raise time.sleep(wait_time) + + +def split_lines(*streams: bytes) -> List[str]: + """Returns a single list of string lines from the byte streams in args.""" + return [ + s.rstrip('\n\r') + for stream in streams + for s in str(stream, 'utf8').splitlines() + ] + + +def run(cmdline: List[str], *, env: Optional[Dict[str, str]] = None, + timeout: int = 300, cwd: str = test_temp_dir) -> Tuple[int, List[str]]: + """A poor man's subprocess.run() for 3.4 compatibility.""" + process = subprocess.Popen( + cmdline, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=cwd, + ) + try: + out, err = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + out = err = b'' + process.kill() + return process.returncode, split_lines(out, err) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py new file mode 100644 index 000000000000..885597e59548 --- /dev/null +++ b/mypy/test/testpackages.py @@ -0,0 +1,45 @@ +import os +import sys + +from mypy.build import find_module, get_package_dirs +from mypy.test.config import package_path +from unittest import TestCase, main +from mypy.test.helpers import run + + +class TestPackages(TestCase): + + def install_pkg(self, pkg: str) -> None: + working_dir = os.path.join(package_path, pkg) + out, lines = run([sys.executable, 'setup.py', 'install'], + cwd=working_dir) + if out != 0: + self.fail('\n'.join(lines)) + + def setUp(self) -> None: + self.install_pkg('typed') + self.dirs = get_package_dirs(None) + self.assertNotEqual(self.dirs, []) + + def find_package(self, pkg: str) -> None: + path = find_module(pkg, [], None) + assert path is not None + self.assertTrue(os.path.exists(path)) + for dir in self.dirs: + if path.startswith(dir): + break + else: + self.fail("Could not locate {}, path is {}".format(pkg, path)) + + def test_find_typed_package(self) -> None: + self.find_package('typedpkg') + self.find_package('typedpkg.sample') + + def test_find_stub_pacakage(self) -> None: + self.install_pkg('stubs') + self.find_package('typedpkg') + self.find_package('typedpkg.sample') + + +if __name__ == '__main__': + main() diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 222fa6ff32c2..2d08091871c4 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -13,15 +13,14 @@ import os import os.path import re -import subprocess import sys import pytest # type: ignore # no pytest in typeshed -from typing import Dict, List, Tuple, Optional +from typing import List from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite -from mypy.test.helpers import assert_string_arrays_equal +from mypy.test.helpers import assert_string_arrays_equal, run from mypy.util import try_find_python2_interpreter from mypy import api @@ -88,35 +87,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: testcase.file, testcase.line)) -def split_lines(*streams: bytes) -> List[str]: - """Returns a single list of string lines from the byte streams in args.""" - return [ - s.rstrip('\n\r') - for stream in streams - for s in str(stream, 'utf8').splitlines() - ] - - def adapt_output(testcase: DataDrivenTestCase) -> List[str]: """Translates the generic _program.py into the actual filename.""" program = '_' + testcase.name + '.py' return [program_re.sub(program, line) for line in testcase.output] - - -def run( - cmdline: List[str], *, env: Optional[Dict[str, str]] = None, timeout: int = 300 -) -> Tuple[int, List[str]]: - """A poor man's subprocess.run() for 3.3 and 3.4 compatibility.""" - process = subprocess.Popen( - cmdline, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=test_temp_dir, - ) - try: - out, err = process.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - out = err = b'' - process.kill() - return process.returncode, split_lines(out, err) diff --git a/runtests.py b/runtests.py index d4712bbfbabb..edb7b0414ccf 100755 --- a/runtests.py +++ b/runtests.py @@ -212,6 +212,7 @@ def test_path(*names: str): 'testmerge', 'testtransform', 'testtypegen', + 'testpackages', 'testparse', 'testsemanal' ) diff --git a/test-data/packages/typedpkg/__init__.py b/test-data/__init__.py similarity index 100% rename from test-data/packages/typedpkg/__init__.py rename to test-data/__init__.py diff --git a/test-data/packages/setup.py b/test-data/packages/setup.py deleted file mode 100644 index bab24a0e8051..000000000000 --- a/test-data/packages/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -This setup file install packages to test mypy's PEP 561 implementation -""" - -from distutils.core import setup - diff --git a/test-data/packages/stubs/setup.py b/test-data/packages/stubs/setup.py new file mode 100644 index 000000000000..f106a0c381b4 --- /dev/null +++ b/test-data/packages/stubs/setup.py @@ -0,0 +1,13 @@ +""" +This setup file install packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg_stubs', + author="The mypy team", + version='0.1', + package_data={'typedpkg_stubs': ['py.typed']}, + packages=['typedpkg_stubs'], +) \ No newline at end of file diff --git a/test-data/packages/typed_pkg_stubs/__init__.pyi b/test-data/packages/stubs/typedpkg_stubs/__init__.pyi similarity index 100% rename from test-data/packages/typed_pkg_stubs/__init__.pyi rename to test-data/packages/stubs/typedpkg_stubs/__init__.pyi diff --git a/test-data/packages/stubs/typedpkg_stubs/py.typed b/test-data/packages/stubs/typedpkg_stubs/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typed_pkg_stubs/sample.pyi b/test-data/packages/stubs/typedpkg_stubs/sample.pyi similarity index 100% rename from test-data/packages/typed_pkg_stubs/sample.pyi rename to test-data/packages/stubs/typedpkg_stubs/sample.pyi diff --git a/test-data/packages/typed/setup.py b/test-data/packages/typed/setup.py new file mode 100644 index 000000000000..0632964ef30f --- /dev/null +++ b/test-data/packages/typed/setup.py @@ -0,0 +1,13 @@ +""" +This setup file install packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg', + author="The mypy team", + version='0.1', + package_data={'typedpkg': ['py.typed']}, + packages=['typedpkg'], +) \ No newline at end of file diff --git a/test-data/packages/typed/typedpkg/__init__.py b/test-data/packages/typed/typedpkg/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/sample.py b/test-data/packages/typed/typedpkg/sample.py similarity index 98% rename from test-data/packages/typedpkg/sample.py rename to test-data/packages/typed/typedpkg/sample.py index fdd21fcfc321..ad69fca662e2 100644 --- a/test-data/packages/typedpkg/sample.py +++ b/test-data/packages/typed/typedpkg/sample.py @@ -1,4 +1,6 @@ from typing import Iterable, List + + def ex(a: Iterable[str]) -> List[str]: """Example typed package. This intentionally has an error.""" return a + ['Hello'] \ No newline at end of file From c97600833a7718b187eced005144f2b0be083253 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 21 Dec 2017 22:45:56 -0500 Subject: [PATCH 011/100] Clean up tests a bit --- mypy/test/testpackages.py | 19 ++++++++++++------- .../packages/{typed => typedpkg}/setup.py | 0 .../{typed => typedpkg}/typedpkg/__init__.py | 0 .../{typed => typedpkg}/typedpkg/sample.py | 0 .../{stubs => typedpkg_stubs}/setup.py | 0 .../typedpkg_stubs/__init__.pyi | 0 .../typedpkg_stubs/py.typed | 0 .../typedpkg_stubs/sample.pyi | 0 8 files changed, 12 insertions(+), 7 deletions(-) rename test-data/packages/{typed => typedpkg}/setup.py (100%) rename test-data/packages/{typed => typedpkg}/typedpkg/__init__.py (100%) rename test-data/packages/{typed => typedpkg}/typedpkg/sample.py (100%) rename test-data/packages/{stubs => typedpkg_stubs}/setup.py (100%) rename test-data/packages/{stubs => typedpkg_stubs}/typedpkg_stubs/__init__.pyi (100%) rename test-data/packages/{stubs => typedpkg_stubs}/typedpkg_stubs/py.typed (100%) rename test-data/packages/{stubs => typedpkg_stubs}/typedpkg_stubs/sample.pyi (100%) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 885597e59548..528dfaf394a7 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -1,3 +1,4 @@ +from contextlib import contextmanager import os import sys @@ -9,15 +10,18 @@ class TestPackages(TestCase): - def install_pkg(self, pkg: str) -> None: + @contextmanager + def installed_package(self, pkg: str) -> None: working_dir = os.path.join(package_path, pkg) out, lines = run([sys.executable, 'setup.py', 'install'], cwd=working_dir) if out != 0: self.fail('\n'.join(lines)) + yield + out, _ = run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=working_dir) + assert out == 0 def setUp(self) -> None: - self.install_pkg('typed') self.dirs = get_package_dirs(None) self.assertNotEqual(self.dirs, []) @@ -32,13 +36,14 @@ def find_package(self, pkg: str) -> None: self.fail("Could not locate {}, path is {}".format(pkg, path)) def test_find_typed_package(self) -> None: - self.find_package('typedpkg') - self.find_package('typedpkg.sample') + with self.installed_package('typedpkg'): + self.find_package('typedpkg') + self.find_package('typedpkg.sample') def test_find_stub_pacakage(self) -> None: - self.install_pkg('stubs') - self.find_package('typedpkg') - self.find_package('typedpkg.sample') + with self.installed_package('typedpkg_stubs'): + self.find_package('typedpkg') + self.find_package('typedpkg.sample') if __name__ == '__main__': diff --git a/test-data/packages/typed/setup.py b/test-data/packages/typedpkg/setup.py similarity index 100% rename from test-data/packages/typed/setup.py rename to test-data/packages/typedpkg/setup.py diff --git a/test-data/packages/typed/typedpkg/__init__.py b/test-data/packages/typedpkg/typedpkg/__init__.py similarity index 100% rename from test-data/packages/typed/typedpkg/__init__.py rename to test-data/packages/typedpkg/typedpkg/__init__.py diff --git a/test-data/packages/typed/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py similarity index 100% rename from test-data/packages/typed/typedpkg/sample.py rename to test-data/packages/typedpkg/typedpkg/sample.py diff --git a/test-data/packages/stubs/setup.py b/test-data/packages/typedpkg_stubs/setup.py similarity index 100% rename from test-data/packages/stubs/setup.py rename to test-data/packages/typedpkg_stubs/setup.py diff --git a/test-data/packages/stubs/typedpkg_stubs/__init__.pyi b/test-data/packages/typedpkg_stubs/typedpkg_stubs/__init__.pyi similarity index 100% rename from test-data/packages/stubs/typedpkg_stubs/__init__.pyi rename to test-data/packages/typedpkg_stubs/typedpkg_stubs/__init__.pyi diff --git a/test-data/packages/stubs/typedpkg_stubs/py.typed b/test-data/packages/typedpkg_stubs/typedpkg_stubs/py.typed similarity index 100% rename from test-data/packages/stubs/typedpkg_stubs/py.typed rename to test-data/packages/typedpkg_stubs/typedpkg_stubs/py.typed diff --git a/test-data/packages/stubs/typedpkg_stubs/sample.pyi b/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi similarity index 100% rename from test-data/packages/stubs/typedpkg_stubs/sample.pyi rename to test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi From 8df8b8de3e7c3b3cd2d0bf85c99dd8689c2c13b6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 22 Dec 2017 12:36:47 -0800 Subject: [PATCH 012/100] Get tests passing --- mypy/build.py | 22 +++--- mypy/test/testpackages.py | 67 +++++++++++++------ test-data/packages/typedpkg/setup.py | 1 + test-data/packages/typedpkg/typedpkg/py.typed | 0 .../packages/typedpkg/typedpkg/sample.py | 6 +- test-data/packages/typedpkg_stubs/setup.py | 2 +- 6 files changed, 65 insertions(+), 33 deletions(-) create mode 100644 test-data/packages/typedpkg/typedpkg/py.typed diff --git a/mypy/build.py b/mypy/build.py index 874ed502046d..292e2f507243 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -734,7 +734,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: # Cache find_module: (id, lib_path) -> result. -find_module_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], Optional[str]] +find_module_cache = {} # type: Dict[str, Optional[str]] # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module @@ -880,13 +880,15 @@ def find() -> Optional[str]: # Third-party stub/typed packages for pkg_dir in package_dirs: stub_name = components[0] + '_stubs' - stub_pkg = os.path.join(pkg_dir, stub_name) typed_file = os.path.join(pkg_dir, components[0], 'py.typed') - if os.path.isfile(os.path.join(stub_pkg, 'py.typed')): + stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') + if os.path.isfile(stub_typed_file): components[0] = stub_name - dirs.append(os.path.join(pkg_dir, os.sep.join(components[:-1]))) + path = os.path.join(pkg_dir, stub_name, *components[1:-1]) + dirs.append(path) elif os.path.isfile(typed_file): - dirs.append(os.path.join(pkg_dir, dir_chain)) + path = os.path.join(pkg_dir, dir_chain) + dirs.append(path) find_module_dir_cache[dir_chain] = dirs candidate_base_dirs = find_module_dir_cache[dir_chain] @@ -903,8 +905,11 @@ def find() -> Optional[str]: # Prefer package over module, i.e. baz/__init__.py* over baz.py*. for extension in PYTHON_EXTENSIONS: path = base_path + sepinit + extension + path_stubs = base_path + '_stubs' + sepinit + extension if is_file(path) and verify_module(id, path): return path + elif is_file(path_stubs) and verify_module(id, path_stubs): + return path_stubs # No package, look for module. for extension in PYTHON_EXTENSIONS: path = base_path + extension @@ -912,10 +917,9 @@ def find() -> Optional[str]: return path return None - key = (id, lib_path) - if key not in find_module_cache: - find_module_cache[key] = find() - return find_module_cache[key] + if id not in find_module_cache: + find_module_cache[id] = find() + return find_module_cache[id] def find_modules_recursive(module: str, lib_path: List[str], diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 528dfaf394a7..3a5c442b5fb7 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -1,49 +1,76 @@ from contextlib import contextmanager import os import sys +from typing import Generator +from unittest import TestCase, main +from mypy.api import run as run_mypy from mypy.build import find_module, get_package_dirs from mypy.test.config import package_path -from unittest import TestCase, main from mypy.test.helpers import run +SIMPLE_PROGRAM = """ +from typedpkg.sample import ex +a = ex(['']) +reveal_type(a) +""" + + class TestPackages(TestCase): + def setUp(self) -> None: + self.dirs = get_package_dirs(None) + self.assertNotEqual(self.dirs, []) + with open('simple.py', 'w') as f: + f.write(SIMPLE_PROGRAM) + + def tearDown(self) -> None: + os.remove('simple.py') + @contextmanager - def installed_package(self, pkg: str) -> None: + def installed_package(self, pkg: str) -> Generator[None, None, None]: + """Context manager to install a package in test-data/packages/pkg/. + Uninstalls the package afterward.""" working_dir = os.path.join(package_path, pkg) - out, lines = run([sys.executable, 'setup.py', 'install'], + out, lines = run([sys.executable, '-m', 'pip', 'install', '.'], cwd=working_dir) if out != 0: self.fail('\n'.join(lines)) - yield - out, _ = run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=working_dir) - assert out == 0 - - def setUp(self) -> None: - self.dirs = get_package_dirs(None) - self.assertNotEqual(self.dirs, []) + try: + yield + finally: + run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) def find_package(self, pkg: str) -> None: path = find_module(pkg, [], None) - assert path is not None - self.assertTrue(os.path.exists(path)) + assert path is not None, (self.dirs, pkg) + self.assertTrue(os.path.exists(path), path) for dir in self.dirs: if path.startswith(dir): break else: self.fail("Could not locate {}, path is {}".format(pkg, path)) - def test_find_typed_package(self) -> None: - with self.installed_package('typedpkg'): - self.find_package('typedpkg') - self.find_package('typedpkg.sample') - - def test_find_stub_pacakage(self) -> None: + def test_typed_package(self) -> None: with self.installed_package('typedpkg_stubs'): - self.find_package('typedpkg') - self.find_package('typedpkg.sample') + out, err, ret = run_mypy(['simple.py']) + assert ret == 1 + assert \ + out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + assert err == '' + with self.installed_package('typedpkg'): + out, err, ret = run_mypy(['simple.py']) + assert ret == 1 + assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + assert err == '' + with self.installed_package('typedpkg'): + with self.installed_package('typedpkg_stubs'): + out, err, ret = run_mypy(['simple.py']) + assert ret == 1 + assert \ + out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + assert err == '' if __name__ == '__main__': diff --git a/test-data/packages/typedpkg/setup.py b/test-data/packages/typedpkg/setup.py index 0632964ef30f..855594551632 100644 --- a/test-data/packages/typedpkg/setup.py +++ b/test-data/packages/typedpkg/setup.py @@ -10,4 +10,5 @@ version='0.1', package_data={'typedpkg': ['py.typed']}, packages=['typedpkg'], + include_package_data=True, ) \ No newline at end of file diff --git a/test-data/packages/typedpkg/typedpkg/py.typed b/test-data/packages/typedpkg/typedpkg/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py index ad69fca662e2..1b203078fd6d 100644 --- a/test-data/packages/typedpkg/typedpkg/sample.py +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -1,6 +1,6 @@ -from typing import Iterable, List +from typing import Iterable, Tuple -def ex(a: Iterable[str]) -> List[str]: +def ex(a: Iterable[str]) -> Tuple[str, ...]: """Example typed package. This intentionally has an error.""" - return a + ['Hello'] \ No newline at end of file + return a + ('Hello') \ No newline at end of file diff --git a/test-data/packages/typedpkg_stubs/setup.py b/test-data/packages/typedpkg_stubs/setup.py index f106a0c381b4..e2f6f0e30834 100644 --- a/test-data/packages/typedpkg_stubs/setup.py +++ b/test-data/packages/typedpkg_stubs/setup.py @@ -8,6 +8,6 @@ name='typedpkg_stubs', author="The mypy team", version='0.1', - package_data={'typedpkg_stubs': ['py.typed']}, + package_data={'typedpkg_stubs': ['py.typed', 'sample.pyi', '__init__.pyi']}, packages=['typedpkg_stubs'], ) \ No newline at end of file From 4ca6939473fbe49e23fe57114527888a06ec582c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 22 Dec 2017 12:53:08 -0800 Subject: [PATCH 013/100] Add note about tests --- mypy/test/testpackages.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 3a5c442b5fb7..e6a0704ad72c 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -53,6 +53,9 @@ def find_package(self, pkg: str) -> None: self.fail("Could not locate {}, path is {}".format(pkg, path)) def test_typed_package(self) -> None: + """Tests checking information based on installed packages. + This test CANNOT be split up, concurrency means that simultaneously + installing/uninstalling will break tests""" with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) assert ret == 1 From a96601c7651578e96a8d0871a490d552859565dd Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 3 Jan 2018 13:22:48 -0800 Subject: [PATCH 014/100] Clear site-packages from cache. --- mypy/build.py | 72 ++++++++++++++++++++++----------------- mypy/test/testpackages.py | 24 +++++-------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 292e2f507243..6b304f871894 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -854,46 +854,47 @@ def find_module(id: str, lib_path_arg: Iterable[str], """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) package_dirs = get_package_dirs(python) + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' def find() -> Optional[str]: # If we're looking for a module like 'foo.bar.baz', it's likely that most of the # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover # that only once and cache it for when we look for modules like 'foo.bar.blah' # that will require the same subdirectory. - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - if dir_chain: - if dir_chain not in find_module_dir_cache: - dirs = [] - - # Regular packages on the PATH - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - isdir = find_module_isdir_cache.get((pathitem, dir_chain)) - if isdir is None: - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - isdir = os.path.isdir(dir) - find_module_isdir_cache[pathitem, dir_chain] = isdir - if isdir: - dirs.append(dir) - - # Third-party stub/typed packages - for pkg_dir in package_dirs: - stub_name = components[0] + '_stubs' - typed_file = os.path.join(pkg_dir, components[0], 'py.typed') - stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') - if os.path.isfile(stub_typed_file): - components[0] = stub_name - path = os.path.join(pkg_dir, stub_name, *components[1:-1]) - dirs.append(path) - elif os.path.isfile(typed_file): - path = os.path.join(pkg_dir, dir_chain) + + if dir_chain not in find_module_dir_cache: + dirs = [] + + # Regular packages on the PATH + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + isdir = find_module_isdir_cache.get((pathitem, dir_chain)) + if isdir is None: + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + isdir = os.path.isdir(dir) + find_module_isdir_cache[pathitem, dir_chain] = isdir + if isdir: + dirs.append(dir) + + # Third-party stub/typed packages + for pkg_dir in package_dirs: + stub_name = components[0] + '_stubs' + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') + stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') + if os.path.isfile(stub_typed_file): + components[0] = stub_name + rest = components[:-1] + path = os.path.join(pkg_dir, *rest) + if os.path.isdir(path): dirs.append(path) + elif os.path.isfile(typed_file): + path = os.path.join(pkg_dir, dir_chain) + dirs.append(path) + + find_module_dir_cache[dir_chain] = dirs + candidate_base_dirs = find_module_dir_cache[dir_chain] - find_module_dir_cache[dir_chain] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain] - else: - candidate_base_dirs = list(lib_path_arg) + package_dirs # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now # contains just the subdirectories 'foo/bar' that actually exist under the # elements of lib_path. This is probably much shorter than lib_path itself. @@ -919,6 +920,13 @@ def find() -> Optional[str]: if id not in find_module_cache: find_module_cache[id] = find() + + # If we searched for items with a base directory of site-packages/ we need to + # remove it to avoid searching it for non-typed ids. + if len(find_module_dir_cache[dir_chain]) > 0 and \ + find_module_dir_cache[dir_chain][-1] in package_dirs: + find_module_dir_cache[dir_chain].pop() + return find_module_cache[id] diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index e6a0704ad72c..11c636692d5f 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -5,7 +5,7 @@ from unittest import TestCase, main from mypy.api import run as run_mypy -from mypy.build import find_module, get_package_dirs +from mypy.build import get_package_dirs from mypy.test.config import package_path from mypy.test.helpers import run @@ -20,8 +20,6 @@ class TestPackages(TestCase): def setUp(self) -> None: - self.dirs = get_package_dirs(None) - self.assertNotEqual(self.dirs, []) with open('simple.py', 'w') as f: f.write(SIMPLE_PROGRAM) @@ -42,15 +40,9 @@ def installed_package(self, pkg: str) -> Generator[None, None, None]: finally: run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) - def find_package(self, pkg: str) -> None: - path = find_module(pkg, [], None) - assert path is not None, (self.dirs, pkg) - self.assertTrue(os.path.exists(path), path) - for dir in self.dirs: - if path.startswith(dir): - break - else: - self.fail("Could not locate {}, path is {}".format(pkg, path)) + def test_get_package_dirs(self) -> None: + dirs = get_package_dirs(None) + assert dirs def test_typed_package(self) -> None: """Tests checking information based on installed packages. @@ -59,21 +51,23 @@ def test_typed_package(self) -> None: with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) assert ret == 1 + assert err == '' assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert err == '' + with self.installed_package('typedpkg'): out, err, ret = run_mypy(['simple.py']) assert ret == 1 - assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" assert err == '' + assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + with self.installed_package('typedpkg'): with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) assert ret == 1 + assert err == '' assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert err == '' if __name__ == '__main__': From 336fb6be995cd035d60c6716b7ab5c35a86ea290 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 3 Jan 2018 13:49:08 -0800 Subject: [PATCH 015/100] Show sample package layout, change help text Also added newlines and refactored call_python in build and get_package_dirs. --- docs/source/installed_packages.rst | 26 ++++++++++++++++++- mypy/build.py | 15 ++++------- mypy/main.py | 3 ++- test-data/packages/typedpkg/setup.py | 2 +- .../packages/typedpkg/typedpkg/sample.py | 2 +- test-data/packages/typedpkg_stubs/setup.py | 2 +- .../typedpkg_stubs/typedpkg_stubs/sample.pyi | 2 +- 7 files changed, 36 insertions(+), 16 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index df82aa926321..ced6e6c197f7 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -5,7 +5,31 @@ Using Installed Packages Making PEP 561 compatible packages ********************************** -Packages that supply type information should put a ``py.typed``. +Packages that supply type information should put a ``py.typed`` in their package +directory. For example, with a directory structure as follows: + +.. code-block:: text + +setup.py +package_a/ + __init__.py + lib.py + py.typed + +the setup.py might look like: + +.. code-block:: python + +from distutils.core import setup + +setup( + name="SuperPackage", + author="Me", + version="0.1", + package_data={"package_a": ["py.typed"]}, + packages=["package_a"] +) + Using PEP 561 compatible packages with mypy ******************************************* diff --git a/mypy/build.py b/mypy/build.py index 6b304f871894..d92b6e417497 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -801,17 +801,12 @@ def is_file(path: str) -> bool: return res -SITE_PACKAGE_COMMANDS = ( - # User site packages - '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"', - # for virtualenvs - '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"', -) +USER_SITE_PACKAGES = '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"' +VIRTUALENV_SITE_PACKAGES = '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"' def call_python(python: str, command: str) -> str: - return check_output(python + ' -c ' + command, - stderr=STDOUT).decode('UTF-8') + return check_output(python + ' -c ' + command).decode(sys.stdout.encoding) def get_package_dirs(python: Optional[str]) -> List[str]: @@ -828,13 +823,13 @@ def get_package_dirs(python: Optional[str]) -> List[str]: if not check.startswith('Python'): return package_dirs # If we have a working python executable, query information from it - output = call_python(python, SITE_PACKAGE_COMMANDS[0]) + output = call_python(python, USER_SITE_PACKAGES) for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) if not package_dirs: # if no paths are found, we fall back on sysconfig - output = call_python(python, SITE_PACKAGE_COMMANDS[1]) + output = call_python(python, VIRTUALENV_SITE_PACKAGES) for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) diff --git a/mypy/main.py b/mypy/main.py index 5134a4ebf02e..c50b0c3232bb 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -245,7 +245,8 @@ def add_invertible_flag(flag: str, version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y') - parser.add_argument('--python', action='store', help="Point to a Python executable.") + parser.add_argument('--python', action='store', + help="Python executable whose installed packages will be used in typechecking.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") diff --git a/test-data/packages/typedpkg/setup.py b/test-data/packages/typedpkg/setup.py index 855594551632..c9472735d0bc 100644 --- a/test-data/packages/typedpkg/setup.py +++ b/test-data/packages/typedpkg/setup.py @@ -11,4 +11,4 @@ package_data={'typedpkg': ['py.typed']}, packages=['typedpkg'], include_package_data=True, -) \ No newline at end of file +) diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py index 1b203078fd6d..a01d857169ce 100644 --- a/test-data/packages/typedpkg/typedpkg/sample.py +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -3,4 +3,4 @@ def ex(a: Iterable[str]) -> Tuple[str, ...]: """Example typed package. This intentionally has an error.""" - return a + ('Hello') \ No newline at end of file + return a + ('Hello') diff --git a/test-data/packages/typedpkg_stubs/setup.py b/test-data/packages/typedpkg_stubs/setup.py index e2f6f0e30834..b158bd7cfef6 100644 --- a/test-data/packages/typedpkg_stubs/setup.py +++ b/test-data/packages/typedpkg_stubs/setup.py @@ -10,4 +10,4 @@ version='0.1', package_data={'typedpkg_stubs': ['py.typed', 'sample.pyi', '__init__.pyi']}, packages=['typedpkg_stubs'], -) \ No newline at end of file +) diff --git a/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi b/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi index ffd321129946..355deefd6a2d 100644 --- a/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi +++ b/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi @@ -1,2 +1,2 @@ from typing import Iterable, List -def ex(a: Iterable[str]) -> List[str]: ... \ No newline at end of file +def ex(a: Iterable[str]) -> List[str]: ... From 1f005d719b35a1f221236dfce76e371cc1fa4284 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 31 Jan 2018 23:14:12 -0800 Subject: [PATCH 016/100] Fix deletion of site-packages from cache --- mypy/build.py | 43 ++++++++++++++++++++------------------- mypy/test/testpackages.py | 18 +++++++++------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 4b207f4c3102..76211ec7db7b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -858,9 +858,8 @@ def find() -> Optional[str]: # that only once and cache it for when we look for modules like 'foo.bar.blah' # that will require the same subdirectory. - if dir_chain not in find_module_dir_cache: - dirs = [] - + dirs = find_module_dir_cache.get(dir_chain, []) + if not dirs: # Regular packages on the PATH for pathitem in lib_path: # e.g., '/usr/lib/python3.4/foo/bar' @@ -872,20 +871,20 @@ def find() -> Optional[str]: if isdir: dirs.append(dir) - # Third-party stub/typed packages - for pkg_dir in package_dirs: - stub_name = components[0] + '_stubs' - typed_file = os.path.join(pkg_dir, components[0], 'py.typed') - stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') - if os.path.isfile(stub_typed_file): - components[0] = stub_name - rest = components[:-1] - path = os.path.join(pkg_dir, *rest) - if os.path.isdir(path): - dirs.append(path) - elif os.path.isfile(typed_file): - path = os.path.join(pkg_dir, dir_chain) + # Third-party stub/typed packages + for pkg_dir in package_dirs: + stub_name = components[0] + '_stubs' + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') + stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') + if os.path.isfile(stub_typed_file): + components[0] = stub_name + rest = components[:-1] + path = os.path.join(pkg_dir, *rest) + if os.path.isdir(path): dirs.append(path) + elif os.path.isfile(typed_file): + path = os.path.join(pkg_dir, dir_chain) + dirs.append(path) find_module_dir_cache[dir_chain] = dirs candidate_base_dirs = find_module_dir_cache[dir_chain] @@ -918,9 +917,9 @@ def find() -> Optional[str]: # If we searched for items with a base directory of site-packages/ we need to # remove it to avoid searching it for non-typed ids. - if len(find_module_dir_cache[dir_chain]) > 0 and \ - find_module_dir_cache[dir_chain][-1] in package_dirs: - find_module_dir_cache[dir_chain].pop() + for dir in package_dirs: + if dir + os.sep in find_module_dir_cache[dir_chain]: + find_module_dir_cache[dir_chain].remove(dir + os.sep) return find_module_cache[id] @@ -1601,8 +1600,10 @@ def __init__(self, file_id = '__builtin__' path = find_module(file_id, manager.lib_path, manager.options.python) if path: - if any((path.startswith(d) for d in package_dirs_cache)): - self.ignore_all = True + if os.path.isabs(path): + for d in package_dirs_cache: + if os.path.commonpath([d, path]) == d: + self.ignore_all = True # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze # - silent -> analyze but silence errors diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 11c636692d5f..21e4521d746f 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -24,11 +24,14 @@ def setUp(self) -> None: f.write(SIMPLE_PROGRAM) def tearDown(self) -> None: - os.remove('simple.py') + try: + os.remove('simple.py') + except PermissionError: + pass @contextmanager def installed_package(self, pkg: str) -> Generator[None, None, None]: - """Context manager to install a package in test-data/packages/pkg/. + """Context manager to install a package from test-data/packages/pkg/. Uninstalls the package afterward.""" working_dir = os.path.join(package_path, pkg) out, lines = run([sys.executable, '-m', 'pip', 'install', '.'], @@ -48,26 +51,27 @@ def test_typed_package(self) -> None: """Tests checking information based on installed packages. This test CANNOT be split up, concurrency means that simultaneously installing/uninstalling will break tests""" + with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) - assert ret == 1 - assert err == '' assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + assert ret == 1 + assert err == '' with self.installed_package('typedpkg'): out, err, ret = run_mypy(['simple.py']) + assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" assert ret == 1 assert err == '' - assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" with self.installed_package('typedpkg'): with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) - assert ret == 1 - assert err == '' assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + assert ret == 1 + assert err == '' if __name__ == '__main__': From 45afe58bd0d815749e0f8da4002805e5e116dac0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 31 Jan 2018 23:37:31 -0800 Subject: [PATCH 017/100] Simplify testcase file creation/deletion --- mypy/build.py | 2 +- mypy/test/helpers.py | 2 +- mypy/test/testpackages.py | 14 +++----------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 76211ec7db7b..6681b77b585f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -806,7 +806,7 @@ def is_file(path: str) -> bool: def call_python(python: str, command: str) -> str: - return check_output(python + ' -c ' + command).decode(sys.stdout.encoding) + return check_output([python, '-c', command]).decode(sys.stdout.encoding) def get_package_dirs(python: Optional[str]) -> List[str]: diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 67e4f761c291..c26761b09afa 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -349,7 +349,7 @@ def split_lines(*streams: bytes) -> List[str]: return [ s.rstrip('\n\r') for stream in streams - for s in str(stream, 'utf8').splitlines() + for s in stream.decode('utf8').splitlines() ] diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 21e4521d746f..8800066fe456 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -19,16 +19,6 @@ class TestPackages(TestCase): - def setUp(self) -> None: - with open('simple.py', 'w') as f: - f.write(SIMPLE_PROGRAM) - - def tearDown(self) -> None: - try: - os.remove('simple.py') - except PermissionError: - pass - @contextmanager def installed_package(self, pkg: str) -> Generator[None, None, None]: """Context manager to install a package from test-data/packages/pkg/. @@ -51,6 +41,8 @@ def test_typed_package(self) -> None: """Tests checking information based on installed packages. This test CANNOT be split up, concurrency means that simultaneously installing/uninstalling will break tests""" + with open('simple.py', 'w') as f: + f.write(SIMPLE_PROGRAM) with self.installed_package('typedpkg_stubs'): out, err, ret = run_mypy(['simple.py']) @@ -72,7 +64,7 @@ def test_typed_package(self) -> None: out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" assert ret == 1 assert err == '' - + os.remove('simple.py') if __name__ == '__main__': main() From e7f23bb86f16c00c6d826c4e8dcbaeb4d41eea06 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 1 Feb 2018 00:03:29 -0800 Subject: [PATCH 018/100] Don't use common path and fix lint errors --- mypy/build.py | 13 +++++++++---- mypy/main.py | 3 ++- mypy/test/helpers.py | 1 - mypy/test/testpackages.py | 1 + 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 6681b77b585f..5c550ef4d48a 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -801,8 +801,10 @@ def is_file(path: str) -> bool: return res -USER_SITE_PACKAGES = '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"' -VIRTUALENV_SITE_PACKAGES = '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"' +USER_SITE_PACKAGES = \ + '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"' +VIRTUALENV_SITE_PACKAGES = \ + '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"' def call_python(python: str, command: str) -> str: @@ -1601,8 +1603,11 @@ def __init__(self, path = find_module(file_id, manager.lib_path, manager.options.python) if path: if os.path.isabs(path): - for d in package_dirs_cache: - if os.path.commonpath([d, path]) == d: + for dir in package_dirs_cache: + # if dir is /foo and path is /foo/bar, the next character after + # the prefix is / + dir_len = len(dir) + if path[dir_len:dir_len + 1] == os.sep: self.ignore_all = True # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze diff --git a/mypy/main.py b/mypy/main.py index da68e714c823..8cd4edfbfda6 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -246,7 +246,8 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y') parser.add_argument('--python', action='store', - help="Python executable whose installed packages will be used in typechecking.") + help="Python executable whose installed packages will be" + " used in typechecking.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index c26761b09afa..4c0e065b313e 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -314,7 +314,6 @@ def retry_on_error(func: Callable[[], Any], max_wait: float = 1.0) -> None: time.sleep(wait_time) - def parse_options(program_text: str, testcase: DataDrivenTestCase, incremental_step: int) -> Options: """Parse comments like '# flags: --foo' in a test case.""" diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 8800066fe456..afba1254a29f 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -66,5 +66,6 @@ def test_typed_package(self) -> None: assert err == '' os.remove('simple.py') + if __name__ == '__main__': main() From 58228843934db7702a9ea800ca4fd59f70de69e9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 03:51:18 -0800 Subject: [PATCH 019/100] Update PEP561 implementation, docs In response to Guido's feedback: - Stub only package libraries have suffix `-stubs` - Simplify code for silencing installed package modules - change `--python` flag to be `--python-executable` - other small code cleanup and fixes --- docs/source/checking_installed_packages.rst | 36 --------------- docs/source/installed_packages.rst | 45 +++++++++++++++---- mypy/build.py | 28 ++++++------ mypy/main.py | 5 ++- mypy/options.py | 2 +- mypy/test/helpers.py | 4 +- mypy/test/testpackages.py | 34 +++++++------- mypy/test/testpythoneval.py | 4 +- test-data/packages/typedpkg-stubs/setup.py | 13 ++++++ .../typedpkg-stubs}/__init__.pyi | 0 .../typedpkg-stubs}/py.typed | 0 .../typedpkg-stubs}/sample.pyi | 0 test-data/packages/typedpkg/setup.py | 2 +- test-data/packages/typedpkg_stubs/setup.py | 13 ------ 14 files changed, 90 insertions(+), 96 deletions(-) delete mode 100644 docs/source/checking_installed_packages.rst create mode 100644 test-data/packages/typedpkg-stubs/setup.py rename test-data/packages/{typedpkg_stubs/typedpkg_stubs => typedpkg-stubs/typedpkg-stubs}/__init__.pyi (100%) rename test-data/packages/{typedpkg_stubs/typedpkg_stubs => typedpkg-stubs/typedpkg-stubs}/py.typed (100%) rename test-data/packages/{typedpkg_stubs/typedpkg_stubs => typedpkg-stubs/typedpkg-stubs}/sample.pyi (100%) delete mode 100644 test-data/packages/typedpkg_stubs/setup.py diff --git a/docs/source/checking_installed_packages.rst b/docs/source/checking_installed_packages.rst deleted file mode 100644 index 9e95a2144a2b..000000000000 --- a/docs/source/checking_installed_packages.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. _checking-installed-packages: - -Using and Creating Typed Packages for Distribution -================================================== - -`PEP 561 `_ specifies how to mark -a package as supporting type checking. Below is a summary of how to use this -feature and create PEP 561 compatible packages. - - -Creating Typed Packages -*********************** - -For a typed package to be picked up by mypy, you must put a file named -``py.typed`` in each top level package installed. For example, your directory -structure may look like: - -.. code:: - - setup.py - my_pkg/ - __init__.py - py.typed - file.py - -Note that if ``my_pkg`` has subpackages, they do *not* need to have their own -``py.typed`` file marker. - - -Checking Typed Packages -*********************** - -Installed packages for the Python being checked should be picked up if they -opt into type checking. If the Python version being checked is different -from the version running mypy, you also need to point mypy to find it via -``--python``. \ No newline at end of file diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index ced6e6c197f7..7bd1bf9fdbd4 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -1,12 +1,18 @@ +.. _installed-packages: + Using Installed Packages ======================== +`PEP 561 `_ specifies how to mark +a package as supporting type checking. Below is a summary of how to create +PEP 561 compatible packages and have mypy use them in type checking. Making PEP 561 compatible packages ********************************** -Packages that supply type information should put a ``py.typed`` in their package -directory. For example, with a directory structure as follows: +Packages that must be imported at runtime that supply type information should +put a ``py.typed`` in their package directory. For example, with a directory +structure as follows: .. code-block:: text @@ -23,22 +29,45 @@ the setup.py might look like: from distutils.core import setup setup( - name="SuperPackage", + name="SuperPackageA", author="Me", version="0.1", package_data={"package_a": ["py.typed"]}, packages=["package_a"] ) +If the package is entirely made up of stub (``*.pyi``) files, the package +should have a suffix of ``-stubs``. For example, if we had stubs for +``package_b``, we might do the following: + +.. code-block:: text + +setup.py +package_b-stubs/ + __init__.pyi + lib.pyi + +the setup.py might look like: + +.. code-block:: python + +from distutils.core import setup + +setup( + name="SuperPackageB", + author="Me", + version="0.1", + package_data={"package_b-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_b-stubs"] +) Using PEP 561 compatible packages with mypy ******************************************* -PEP 561 specifies a format to indicate a package installed in site-packages or -dist-packages supports providing type information. Generally, you do not need -to do anything to use these packages. They should be automatically picked up by +Generally, you do not need to do anything to use installed packages for the +Python executable used to run mypy. They should be automatically picked up by mypy and used for type checking. If you use mypy to type check a Python other than the version running mypy, you -can use the ``--python`` flag to point to the executable, and mypy will pick up -the site/dist-packages for the Python executable pointed to. \ No newline at end of file +can use the ``--python-executable`` flag to point to the executable, and mypy +will pick up the site/dist-packages for the Python executable pointed to. \ No newline at end of file diff --git a/mypy/build.py b/mypy/build.py index 86baf6c760a8..812d55f734c9 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -684,7 +684,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" - return find_module(id, self.lib_path, self.options.python) is not None + return find_module(id, self.lib_path, self.options.python_executable) is not None def parse_file(self, id: str, path: str, source: str, ignore_errors: bool) -> MypyFile: """Parse the source of a file with the given name. @@ -797,7 +797,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: return p -# Cache find_module: (id, lib_path) -> result. +# Cache find_module: (id) -> result. find_module_cache = {} # type: Dict[str, Optional[str]] # Cache some repeated work within distinct find_module calls: finding which @@ -866,9 +866,9 @@ def is_file(path: str) -> bool: USER_SITE_PACKAGES = \ - '"import site;print(site.getusersitepackages());print(*site.getsitepackages(), sep=\'\\n\')"' + '"import site; print(site.getusersitepackages()); print(*site.getsitepackages(), sep=\'\\n\')"' VIRTUALENV_SITE_PACKAGES = \ - '"from distutils.sysconfig import get_python_lib;print(get_python_lib())"' + '"from distutils.sysconfig import get_python_lib; print(get_python_lib())"' def call_python(python: str, command: str) -> str: @@ -876,9 +876,9 @@ def call_python(python: str, command: str) -> str: def get_package_dirs(python: Optional[str]) -> List[str]: - """Find package directories for given python (default to Python running - mypy).""" - global package_dirs_cache + """Find package directories for given python + + This defaults to the Python running mypy.""" if package_dirs_cache: return package_dirs_cache package_dirs = [] # type: List[str] @@ -906,7 +906,7 @@ def get_package_dirs(python: Optional[str]) -> List[str]: package_dirs = site.getsitepackages() + [user_dir] except AttributeError: package_dirs = [get_python_lib()] - package_dirs_cache = package_dirs + package_dirs_cache[:] = package_dirs return package_dirs @@ -939,7 +939,7 @@ def find() -> Optional[str]: # Third-party stub/typed packages for pkg_dir in package_dirs: - stub_name = components[0] + '_stubs' + stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') if os.path.isfile(stub_typed_file): @@ -1685,14 +1685,14 @@ def __init__(self, # difference and just assume 'builtins' everywhere, # which simplifies code. file_id = '__builtin__' - path = find_module(file_id, manager.lib_path, manager.options.python) + path = find_module(file_id, manager.lib_path, manager.options.python_executable) if path: + # Installed package modules should be silenced. They are all under absolute + # paths. if os.path.isabs(path): + # Silence errors from module if it is in a package directory for dir in package_dirs_cache: - # if dir is /foo and path is /foo/bar, the next character after - # the prefix is / - dir_len = len(dir) - if path[dir_len:dir_len + 1] == os.sep: + if path.startswith(dir + os.sep): self.ignore_all = True # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze diff --git a/mypy/main.py b/mypy/main.py index c346a82ef7d8..91139e04bba6 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -256,7 +256,7 @@ def add_invertible_flag(flag: str, version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', help='use Python x.y') - parser.add_argument('--python', action='store', + parser.add_argument('--python-executable', action='store', help="Python executable whose installed packages will be" " used in typechecking.") parser.add_argument('--platform', action='store', metavar='PLATFORM', @@ -527,7 +527,8 @@ def add_invertible_flag(flag: str, .format(special_opts.package)) options.build_type = BuildType.MODULE lib_path = [os.getcwd()] + build.mypy_path() - targets = build.find_modules_recursive(special_opts.package, lib_path, options.python) + targets = build.find_modules_recursive(special_opts.package, lib_path, + options.python_executable) if not targets: fail("Can't find package '{}'".format(special_opts.package)) return targets, options diff --git a/mypy/options.py b/mypy/options.py index efd425b6e511..90ed337c2f42 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -53,7 +53,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = defaults.PYTHON3_VERSION - self.python = None # type: Optional[str] + self.python_executable = None # type: Optional[str] self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index ac6c5b053ccb..4fd285af5355 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -341,8 +341,8 @@ def split_lines(*streams: bytes) -> List[str]: ] -def run(cmdline: List[str], *, env: Optional[Dict[str, str]] = None, - timeout: int = 300, cwd: str = test_temp_dir) -> Tuple[int, List[str]]: +def run_command(cmdline: List[str], *, env: Optional[Dict[str, str]] = None, + timeout: int = 300, cwd: str = test_temp_dir) -> Tuple[int, List[str]]: """A poor man's subprocess.run() for 3.4 compatibility.""" process = subprocess.Popen( cmdline, diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index afba1254a29f..8e1717e7c68a 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -4,10 +4,10 @@ from typing import Generator from unittest import TestCase, main -from mypy.api import run as run_mypy +import mypy.api from mypy.build import get_package_dirs from mypy.test.config import package_path -from mypy.test.helpers import run +from mypy.test.helpers import run_command SIMPLE_PROGRAM = """ @@ -20,18 +20,18 @@ class TestPackages(TestCase): @contextmanager - def installed_package(self, pkg: str) -> Generator[None, None, None]: + def install_package(self, pkg: str) -> Generator[None, None, None]: """Context manager to install a package from test-data/packages/pkg/. Uninstalls the package afterward.""" working_dir = os.path.join(package_path, pkg) - out, lines = run([sys.executable, '-m', 'pip', 'install', '.'], - cwd=working_dir) - if out != 0: + returncode, lines = run_command([sys.executable, '-m', 'pip', 'install', '.'], + cwd=working_dir) + if returncode != 0: self.fail('\n'.join(lines)) try: yield finally: - run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) + run_command([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) def test_get_package_dirs(self) -> None: dirs = get_package_dirs(None) @@ -44,25 +44,25 @@ def test_typed_package(self) -> None: with open('simple.py', 'w') as f: f.write(SIMPLE_PROGRAM) - with self.installed_package('typedpkg_stubs'): - out, err, ret = run_mypy(['simple.py']) + with self.install_package('typedpkg-stubs'): + out, err, returncode = mypy.api.run(['simple.py']) assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert ret == 1 + assert returncode == 1 assert err == '' - with self.installed_package('typedpkg'): - out, err, ret = run_mypy(['simple.py']) + with self.install_package('typedpkg'): + out, err, returncode = mypy.api.run(['simple.py']) assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - assert ret == 1 + assert returncode == 1 assert err == '' - with self.installed_package('typedpkg'): - with self.installed_package('typedpkg_stubs'): - out, err, ret = run_mypy(['simple.py']) + with self.install_package('typedpkg'): + with self.install_package('typedpkg-stubs'): + out, err, returncode = mypy.api.run(['simple.py']) assert \ out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert ret == 1 + assert returncode == 1 assert err == '' os.remove('simple.py') diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 2d08091871c4..918395cfd490 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -20,7 +20,7 @@ from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase, DataSuite -from mypy.test.helpers import assert_string_arrays_equal, run +from mypy.test.helpers import assert_string_arrays_equal, run_command from mypy.util import try_find_python2_interpreter from mypy import api @@ -78,7 +78,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: output.append(line.rstrip("\r\n")) if returncode == 0: # Execute the program. - returncode, interp_out = run([interpreter, program]) + returncode, interp_out = run_command([interpreter, program]) output.extend(interp_out) # Remove temp file. os.remove(program_path) diff --git a/test-data/packages/typedpkg-stubs/setup.py b/test-data/packages/typedpkg-stubs/setup.py new file mode 100644 index 000000000000..cb183b408cad --- /dev/null +++ b/test-data/packages/typedpkg-stubs/setup.py @@ -0,0 +1,13 @@ +""" +This setup file installs packages to test mypy's PEP 561 implementation +""" + +from distutils.core import setup + +setup( + name='typedpkg-stubs', + author="The mypy team", + version='0.1', + package_data={'typedpkg-stubs': ['py.typed', 'sample.pyi', '__init__.pyi']}, + packages=['typedpkg-stubs'], +) diff --git a/test-data/packages/typedpkg_stubs/typedpkg_stubs/__init__.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi similarity index 100% rename from test-data/packages/typedpkg_stubs/typedpkg_stubs/__init__.pyi rename to test-data/packages/typedpkg-stubs/typedpkg-stubs/__init__.pyi diff --git a/test-data/packages/typedpkg_stubs/typedpkg_stubs/py.typed b/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed similarity index 100% rename from test-data/packages/typedpkg_stubs/typedpkg_stubs/py.typed rename to test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed diff --git a/test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi b/test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi similarity index 100% rename from test-data/packages/typedpkg_stubs/typedpkg_stubs/sample.pyi rename to test-data/packages/typedpkg-stubs/typedpkg-stubs/sample.pyi diff --git a/test-data/packages/typedpkg/setup.py b/test-data/packages/typedpkg/setup.py index c9472735d0bc..6da37d2f6629 100644 --- a/test-data/packages/typedpkg/setup.py +++ b/test-data/packages/typedpkg/setup.py @@ -1,5 +1,5 @@ """ -This setup file install packages to test mypy's PEP 561 implementation +This setup file installs packages to test mypy's PEP 561 implementation """ from distutils.core import setup diff --git a/test-data/packages/typedpkg_stubs/setup.py b/test-data/packages/typedpkg_stubs/setup.py deleted file mode 100644 index b158bd7cfef6..000000000000 --- a/test-data/packages/typedpkg_stubs/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -""" -This setup file install packages to test mypy's PEP 561 implementation -""" - -from distutils.core import setup - -setup( - name='typedpkg_stubs', - author="The mypy team", - version='0.1', - package_data={'typedpkg_stubs': ['py.typed', 'sample.pyi', '__init__.pyi']}, - packages=['typedpkg_stubs'], -) From c734bc28d5323faee82aa84c78e7b6b827ae193b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 04:02:40 -0800 Subject: [PATCH 020/100] Clarify python-executable purpose --- docs/source/installed_packages.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 7bd1bf9fdbd4..812bde1ab831 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -70,4 +70,4 @@ mypy and used for type checking. If you use mypy to type check a Python other than the version running mypy, you can use the ``--python-executable`` flag to point to the executable, and mypy -will pick up the site/dist-packages for the Python executable pointed to. \ No newline at end of file +will find packages installed for that python executable. \ No newline at end of file From 1d7cc7ef72f4a826844412a31309c4d850080f03 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 15:22:12 -0800 Subject: [PATCH 021/100] Document --python-executable --- docs/source/command_line.rst | 58 +++++++++++++++++++++--------------- mypy/build.py | 2 +- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 99f7d5b58409..968a3b2d57d1 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -8,30 +8,34 @@ summary of command line flags can always be printed using the ``-h`` flag (or its long form ``--help``):: $ mypy -h - usage: mypy [-h] [-v] [-V] [--python-version x.y] [--platform PLATFORM] [-2] - [--ignore-missing-imports] - [--follow-imports {normal,silent,skip,error}] - [--disallow-any-{unimported,expr,decorated,explicit,generics}] - [--disallow-untyped-calls] [--disallow-untyped-defs] - [--check-untyped-defs] [--disallow-subclassing-any] - [--warn-incomplete-stub] [--warn-redundant-casts] - [--no-warn-no-return] [--warn-return-any] [--warn-unused-ignores] - [--show-error-context] [--no-implicit-optional] [-i] - [--quick-and-dirty] [--cache-dir DIR] [--skip-version-check] - [--strict-optional] - [--strict-optional-whitelist [GLOB [GLOB ...]]] - [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] - [--inferstats] [--custom-typing MODULE] - [--custom-typeshed-dir DIR] [--scripts-are-modules] - [--config-file CONFIG_FILE] [--show-column-numbers] - [--find-occurrences CLASS.MEMBER] [--strict] - [--shadow-file SOURCE_FILE SHADOW_FILE] [--any-exprs-report DIR] - [--cobertura-xml-report DIR] [--html-report DIR] - [--linecount-report DIR] [--linecoverage-report DIR] - [--memory-xml-report DIR] - [--txt-report DIR] [--xml-report DIR] [--xslt-html-report DIR] - [--xslt-txt-report DIR] [-m MODULE] [-c PROGRAM_TEXT] [-p PACKAGE] - [files [files ...]] + usage: mypy [-h] [-v] [-V] [--python-version x.y] + [--python-executable PYTHON_EXECUTABLE] [--platform PLATFORM] [-2] + [--ignore-missing-imports] + [--follow-imports {normal,silent,skip,error}] + [--disallow-any-unimported] [--disallow-any-expr] + [--disallow-any-decorated] [--disallow-any-explicit] + [--disallow-any-generics] [--disallow-untyped-calls] + [--disallow-untyped-defs] [--disallow-incomplete-defs] + [--check-untyped-defs] [--disallow-subclassing-any] + [--warn-incomplete-stub] [--disallow-untyped-decorators] + [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any] + [--warn-unused-ignores] [--warn-unused-configs] + [--show-error-context] [--no-implicit-optional] [-i] + [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained] + [--skip-version-check] [--strict-optional] + [--strict-optional-whitelist [GLOB [GLOB ...]]] + [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] + [--inferstats] [--custom-typing MODULE] + [--custom-typeshed-dir DIR] [--scripts-are-modules] + [--config-file CONFIG_FILE] [--show-column-numbers] + [--find-occurrences CLASS.MEMBER] [--strict] + [--shadow-file SOURCE_FILE SHADOW_FILE] [--any-exprs-report DIR] + [--cobertura-xml-report DIR] [--html-report DIR] + [--linecount-report DIR] [--linecoverage-report DIR] + [--memory-xml-report DIR] [--txt-report DIR] [--xml-report DIR] + [--xslt-html-report DIR] [--xslt-txt-report DIR] [-m MODULE] + [-c PROGRAM_TEXT] [-p PACKAGE] + [files [files ...]] (etc., too long to show everything here) @@ -366,6 +370,12 @@ Here are some more useful flags: updates the cache, but regular incremental mode ignores cache files written by quick mode. +- ``--python-executable EXECUTABLE`` will have mypy collect type information + from PEP 561 compliant packages installed with the given Python executable. + By default, mypy will use PEP 561 compliant packages installed for the Python + executable running mypy. See :ref:`installed_packages` for more on making + PEP 561 compliant packages. + - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and diff --git a/mypy/build.py b/mypy/build.py index 812d55f734c9..70e6bd78b189 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1688,7 +1688,7 @@ def __init__(self, path = find_module(file_id, manager.lib_path, manager.options.python_executable) if path: # Installed package modules should be silenced. They are all under absolute - # paths. + # paths. When 3.4 is dropped, this should just use os.path.commonpath. if os.path.isabs(path): # Silence errors from module if it is in a package directory for dir in package_dirs_cache: From 1ac42538b29ba23def261d37b5a8f50a5e709bc8 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 15:33:42 -0800 Subject: [PATCH 022/100] Assure dir cache is always set for a key --- mypy/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 70e6bd78b189..f51cefc9372b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -886,8 +886,8 @@ def get_package_dirs(python: Optional[str]) -> List[str]: # Use subprocess to get the package directory of given Python # executable check = check_output([python, '-V'], stderr=STDOUT).decode('UTF-8') - if not check.startswith('Python'): - return package_dirs + assert check.startswith('Python'), \ + "Mypy could not use the Python executable: {}".format(python) # If we have a working python executable, query information from it output = call_python(python, USER_SITE_PACKAGES) for line in output.splitlines(): @@ -952,7 +952,7 @@ def find() -> Optional[str]: path = os.path.join(pkg_dir, dir_chain) dirs.append(path) - find_module_dir_cache[dir_chain] = dirs + find_module_dir_cache[dir_chain] = dirs candidate_base_dirs = find_module_dir_cache[dir_chain] # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now From f681bf9b541f7e2eb1b83df0704c2f3314796c1b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 15:52:18 -0800 Subject: [PATCH 023/100] Fix indentation of code and command line --- docs/source/command_line.rst | 54 ++++++++++++++-------------- docs/source/installed_packages.rst | 58 +++++++++++++++--------------- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 968a3b2d57d1..3d27211cae35 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -9,33 +9,33 @@ flag (or its long form ``--help``):: $ mypy -h usage: mypy [-h] [-v] [-V] [--python-version x.y] - [--python-executable PYTHON_EXECUTABLE] [--platform PLATFORM] [-2] - [--ignore-missing-imports] - [--follow-imports {normal,silent,skip,error}] - [--disallow-any-unimported] [--disallow-any-expr] - [--disallow-any-decorated] [--disallow-any-explicit] - [--disallow-any-generics] [--disallow-untyped-calls] - [--disallow-untyped-defs] [--disallow-incomplete-defs] - [--check-untyped-defs] [--disallow-subclassing-any] - [--warn-incomplete-stub] [--disallow-untyped-decorators] - [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any] - [--warn-unused-ignores] [--warn-unused-configs] - [--show-error-context] [--no-implicit-optional] [-i] - [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained] - [--skip-version-check] [--strict-optional] - [--strict-optional-whitelist [GLOB [GLOB ...]]] - [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] - [--inferstats] [--custom-typing MODULE] - [--custom-typeshed-dir DIR] [--scripts-are-modules] - [--config-file CONFIG_FILE] [--show-column-numbers] - [--find-occurrences CLASS.MEMBER] [--strict] - [--shadow-file SOURCE_FILE SHADOW_FILE] [--any-exprs-report DIR] - [--cobertura-xml-report DIR] [--html-report DIR] - [--linecount-report DIR] [--linecoverage-report DIR] - [--memory-xml-report DIR] [--txt-report DIR] [--xml-report DIR] - [--xslt-html-report DIR] [--xslt-txt-report DIR] [-m MODULE] - [-c PROGRAM_TEXT] [-p PACKAGE] - [files [files ...]] + [--python-executable PYTHON_EXECUTABLE] [--platform PLATFORM] [-2] + [--ignore-missing-imports] + [--follow-imports {normal,silent,skip,error}] + [--disallow-any-unimported] [--disallow-any-expr] + [--disallow-any-decorated] [--disallow-any-explicit] + [--disallow-any-generics] [--disallow-untyped-calls] + [--disallow-untyped-defs] [--disallow-incomplete-defs] + [--check-untyped-defs] [--disallow-subclassing-any] + [--warn-incomplete-stub] [--disallow-untyped-decorators] + [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any] + [--warn-unused-ignores] [--warn-unused-configs] + [--show-error-context] [--no-implicit-optional] [-i] + [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained] + [--skip-version-check] [--strict-optional] + [--strict-optional-whitelist [GLOB [GLOB ...]]] + [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] + [--inferstats] [--custom-typing MODULE] + [--custom-typeshed-dir DIR] [--scripts-are-modules] + [--config-file CONFIG_FILE] [--show-column-numbers] + [--find-occurrences CLASS.MEMBER] [--strict] + [--shadow-file SOURCE_FILE SHADOW_FILE] [--any-exprs-report DIR] + [--cobertura-xml-report DIR] [--html-report DIR] + [--linecount-report DIR] [--linecoverage-report DIR] + [--memory-xml-report DIR] [--txt-report DIR] [--xml-report DIR] + [--xslt-html-report DIR] [--xslt-txt-report DIR] [-m MODULE] + [-c PROGRAM_TEXT] [-p PACKAGE] + [files [files ...]] (etc., too long to show everything here) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 812bde1ab831..5ff8841f401a 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -16,25 +16,25 @@ structure as follows: .. code-block:: text -setup.py -package_a/ - __init__.py - lib.py - py.typed + setup.py + package_a/ + __init__.py + lib.py + py.typed -the setup.py might look like: + the setup.py might look like: -.. code-block:: python + .. code-block:: python -from distutils.core import setup + from distutils.core import setup -setup( - name="SuperPackageA", - author="Me", - version="0.1", - package_data={"package_a": ["py.typed"]}, - packages=["package_a"] -) + setup( + name="SuperPackageA", + author="Me", + version="0.1", + package_data={"package_a": ["py.typed"]}, + packages=["package_a"] + ) If the package is entirely made up of stub (``*.pyi``) files, the package should have a suffix of ``-stubs``. For example, if we had stubs for @@ -42,24 +42,24 @@ should have a suffix of ``-stubs``. For example, if we had stubs for .. code-block:: text -setup.py -package_b-stubs/ - __init__.pyi - lib.pyi + setup.py + package_b-stubs/ + __init__.pyi + lib.pyi -the setup.py might look like: + the setup.py might look like: -.. code-block:: python + .. code-block:: python -from distutils.core import setup + from distutils.core import setup -setup( - name="SuperPackageB", - author="Me", - version="0.1", - package_data={"package_b-stubs": ["__init__.pyi", "lib.pyi"]}, - packages=["package_b-stubs"] -) + setup( + name="SuperPackageB", + author="Me", + version="0.1", + package_data={"package_b-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_b-stubs"] + ) Using PEP 561 compatible packages with mypy ******************************************* From 85f233c678cac6599af0af6a01963fbb0d98e002 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 16:00:11 -0800 Subject: [PATCH 024/100] Fix silly indentation mistake (again) --- docs/source/installed_packages.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 5ff8841f401a..cfeb9340fbf2 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -22,9 +22,9 @@ structure as follows: lib.py py.typed - the setup.py might look like: +the setup.py might look like: - .. code-block:: python +.. code-block:: python from distutils.core import setup From 3726f18ecf56f0e8e5c8a7e75c90e524a4e338ef Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 16:24:46 -0800 Subject: [PATCH 025/100] Add check for Python packages if executable given --- mypy/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/build.py b/mypy/build.py index f51cefc9372b..b917d55fcd9b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -915,6 +915,8 @@ def find_module(id: str, lib_path_arg: Iterable[str], """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) package_dirs = get_package_dirs(python) + if python: + assert package_dirs, "Could not find package directories for Python '{}'".format(python) components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' From 59a2004fde2001d02f7867340d294184f500af4d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 17:41:47 -0800 Subject: [PATCH 026/100] Finish implementation and tests for alternate executable --- mypy/build.py | 39 ++++++++++++++++++++---------------- mypy/test/testpackages.py | 42 ++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index b917d55fcd9b..feb3c9eece9f 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -21,7 +21,7 @@ import re import site import stat -from subprocess import check_output, STDOUT +import subprocess import sys import time from os.path import dirname, basename @@ -819,8 +819,8 @@ def remove_cwd_prefix_from_path(p: str) -> str: # Cache for isdir(join(head, tail)) find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] -# Cache packages for Python executable -package_dirs_cache = [] # type: List[str] +# Cache packages for each Python executable +package_dirs_cache = {} # type: Dict[str, List[str]] def find_module_clear_caches() -> None: @@ -866,34 +866,36 @@ def is_file(path: str) -> bool: USER_SITE_PACKAGES = \ - '"import site; print(site.getusersitepackages()); print(*site.getsitepackages(), sep=\'\\n\')"' + 'from __future__ import print_function; import site; print(site.getusersitepackages());' \ + 'print(*site.getsitepackages(), sep="\\n")' VIRTUALENV_SITE_PACKAGES = \ - '"from distutils.sysconfig import get_python_lib; print(get_python_lib())"' + 'from distutils.sysconfig import get_python_lib; print(get_python_lib())' def call_python(python: str, command: str) -> str: - return check_output([python, '-c', command]).decode(sys.stdout.encoding) + return subprocess.check_output([python, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') -def get_package_dirs(python: Optional[str]) -> List[str]: +def get_package_dirs(python: str) -> List[str]: """Find package directories for given python This defaults to the Python running mypy.""" - if package_dirs_cache: - return package_dirs_cache + if package_dirs_cache.get(python, None): + return package_dirs_cache[python] package_dirs = [] # type: List[str] if python: # Use subprocess to get the package directory of given Python # executable - check = check_output([python, '-V'], stderr=STDOUT).decode('UTF-8') + check = subprocess.check_output([python, '-V'], stderr=subprocess.STDOUT).decode('UTF-8') assert check.startswith('Python'), \ "Mypy could not use the Python executable: {}".format(python) # If we have a working python executable, query information from it - output = call_python(python, USER_SITE_PACKAGES) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) - if not package_dirs: + try: + output = call_python(python, USER_SITE_PACKAGES) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) + except subprocess.CalledProcessError: # if no paths are found, we fall back on sysconfig output = call_python(python, VIRTUALENV_SITE_PACKAGES) for line in output.splitlines(): @@ -906,7 +908,7 @@ def get_package_dirs(python: Optional[str]) -> List[str]: package_dirs = site.getsitepackages() + [user_dir] except AttributeError: package_dirs = [get_python_lib()] - package_dirs_cache[:] = package_dirs + package_dirs_cache[python] = package_dirs return package_dirs @@ -914,6 +916,8 @@ def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) + if not python: + python = sys.executable package_dirs = get_package_dirs(python) if python: assert package_dirs, "Could not find package directories for Python '{}'".format(python) @@ -1693,7 +1697,8 @@ def __init__(self, # paths. When 3.4 is dropped, this should just use os.path.commonpath. if os.path.isabs(path): # Silence errors from module if it is in a package directory - for dir in package_dirs_cache: + python = manager.options.python_executable or sys.executable + for dir in package_dirs_cache.get(python, []): if path.startswith(dir + os.sep): self.ignore_all = True # For non-stubs, look at options.follow_imports: diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 8e1717e7c68a..f004b1b71cf4 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -8,6 +8,7 @@ from mypy.build import get_package_dirs from mypy.test.config import package_path from mypy.test.helpers import run_command +from mypy.util import try_find_python2_interpreter SIMPLE_PROGRAM = """ @@ -20,21 +21,25 @@ class TestPackages(TestCase): @contextmanager - def install_package(self, pkg: str) -> Generator[None, None, None]: + def install_package(self, pkg: str, + python: str = sys.executable) -> Generator[None, None, None]: """Context manager to install a package from test-data/packages/pkg/. Uninstalls the package afterward.""" working_dir = os.path.join(package_path, pkg) - returncode, lines = run_command([sys.executable, '-m', 'pip', 'install', '.'], - cwd=working_dir) + install_cmd = [python, '-m', 'pip', 'install', '.'] + # if we aren't in a virtualenv, install in the user package directory so we don't need sudo + if not hasattr(sys, 'real_prefix') or python != sys.executable: + install_cmd.append('--user') + returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: self.fail('\n'.join(lines)) try: yield finally: - run_command([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) + run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) def test_get_package_dirs(self) -> None: - dirs = get_package_dirs(None) + dirs = get_package_dirs(sys.executable) assert dirs def test_typed_package(self) -> None: @@ -51,6 +56,33 @@ def test_typed_package(self) -> None: assert returncode == 1 assert err == '' + python2 = try_find_python2_interpreter() + if python2: + with self.install_package('typedpkg-stubs', python2): + out, err, returncode = mypy.api.run( + ['--python-executable={}'.format(python2), 'simple.py']) + assert \ + out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + assert returncode == 1 + assert err == '' + with self.install_package('typedpkg', python2): + out, err, returncode = mypy.api.run( + ['--python-executable={}'.format(python2), 'simple.py']) + assert out == "simple.py:4: error: Revealed type is " \ + "'builtins.tuple[builtins.str]'\n" + assert returncode == 1 + assert err == '' + + with self.install_package('typedpkg', python2): + with self.install_package('typedpkg-stubs', python2): + out, err, returncode = mypy.api.run( + ['--python-executable={}'.format(python2), 'simple.py']) + assert \ + out == "simple.py:4: error: Revealed type is " \ + "'builtins.list[builtins.str]'\n" + assert returncode == 1 + assert err == '' + with self.install_package('typedpkg'): out, err, returncode = mypy.api.run(['simple.py']) assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" From 77b3b6684db7c7bf706e1803d5d94f17b8647296 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 17:44:58 -0800 Subject: [PATCH 027/100] Add note about advanced import methods --- docs/source/installed_packages.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index cfeb9340fbf2..ff76b298f832 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -70,4 +70,7 @@ mypy and used for type checking. If you use mypy to type check a Python other than the version running mypy, you can use the ``--python-executable`` flag to point to the executable, and mypy -will find packages installed for that python executable. \ No newline at end of file +will find packages installed for that python executable. + +Note that mypy does not support some more dynamic import features, such as zip +imports and advanced custom import hooks. \ No newline at end of file From 1da6e58dc592c21e7dbb87e9021b002281c5672e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 17:56:34 -0800 Subject: [PATCH 028/100] Remove undocumented flag --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 3d27211cae35..81bcd1d95b9f 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -21,7 +21,7 @@ flag (or its long form ``--help``):: [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any] [--warn-unused-ignores] [--warn-unused-configs] [--show-error-context] [--no-implicit-optional] [-i] - [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained] + [--quick-and-dirty] [--cache-dir DIR] [--skip-version-check] [--strict-optional] [--strict-optional-whitelist [GLOB [GLOB ...]]] [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] From 056dc8f67252e50098dd90b73369c66fef8ee8c3 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 02:21:12 -0800 Subject: [PATCH 029/100] Clean up testpackages docstrings --- mypy/test/testpackages.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index f004b1b71cf4..a8cdb922598a 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -23,12 +23,11 @@ class TestPackages(TestCase): @contextmanager def install_package(self, pkg: str, python: str = sys.executable) -> Generator[None, None, None]: - """Context manager to install a package from test-data/packages/pkg/. - Uninstalls the package afterward.""" + """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) install_cmd = [python, '-m', 'pip', 'install', '.'] # if we aren't in a virtualenv, install in the user package directory so we don't need sudo - if not hasattr(sys, 'real_prefix') or python != sys.executable: + if not hasattr(sys, 'real_prefix'): install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: @@ -39,11 +38,13 @@ def install_package(self, pkg: str, run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) def test_get_package_dirs(self) -> None: + """Check that get_package_dirs works.""" dirs = get_package_dirs(sys.executable) assert dirs def test_typed_package(self) -> None: - """Tests checking information based on installed packages. + """Tests type checking based on installed packages. + This test CANNOT be split up, concurrency means that simultaneously installing/uninstalling will break tests""" with open('simple.py', 'w') as f: From e35e7aa5e6932c6cec0c4e9f9b8d4c335f0f1a64 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 02:45:08 -0800 Subject: [PATCH 030/100] User install on Python not executing tests --- mypy/test/testpackages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index a8cdb922598a..9f5be22cc885 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -27,7 +27,7 @@ def install_package(self, pkg: str, working_dir = os.path.join(package_path, pkg) install_cmd = [python, '-m', 'pip', 'install', '.'] # if we aren't in a virtualenv, install in the user package directory so we don't need sudo - if not hasattr(sys, 'real_prefix'): + if not hasattr(sys, 'real_prefix') or python != sys.executable: install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: From ecb702e5ce3b6613b5bcc2b0db28ff8b3280c640 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 19:30:31 -0800 Subject: [PATCH 031/100] Better debug info on failed typecheck --- mypy/test/testpackages.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 9f5be22cc885..424c82b04dbd 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -34,6 +34,10 @@ def install_package(self, pkg: str, self.fail('\n'.join(lines)) try: yield + except AssertionError as e: + raise AssertionError("Failed to typecheck with installed package {}.\n" + "Package directories checked:\n{}" + "Error:\n{}".format(pkg, get_package_dirs(python), e)) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From e54025f489ec5dcaca6de1f2b300c6df2cdfe682 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 15 Feb 2018 19:44:33 -0800 Subject: [PATCH 032/100] Maintain source information in traceback --- mypy/test/testpackages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 424c82b04dbd..cc182a8fd091 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -36,8 +36,9 @@ def install_package(self, pkg: str, yield except AssertionError as e: raise AssertionError("Failed to typecheck with installed package {}.\n" - "Package directories checked:\n{}" - "Error:\n{}".format(pkg, get_package_dirs(python), e)) + "Package directories checked:\n{}\n" + "Error traceback:\n{}\n".format(pkg, get_package_dirs(python), e) + ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From 297fac6131830dfb8502537e4361f648e42d287c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 02:29:07 -0800 Subject: [PATCH 033/100] Even better tracebacks for package tests --- mypy/test/testpackages.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index cc182a8fd091..91ff27bc24be 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -35,9 +35,12 @@ def install_package(self, pkg: str, try: yield except AssertionError as e: + package_dirs = get_package_dirs(python) + possible_paths = [os.path.join(dir, pkg) for dir in package_dirs] + checked_paths = {path: os.path.exists(path) for path in possible_paths} raise AssertionError("Failed to typecheck with installed package {}.\n" - "Package directories checked:\n{}\n" - "Error traceback:\n{}\n".format(pkg, get_package_dirs(python), e) + "Package paths checked:\n{}\n" + "Error traceback:\n{}\n".format(pkg, checked_paths, e) ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From 1a4126233b79b9333417db1a5facc4c0f56a6b24 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 03:04:38 -0800 Subject: [PATCH 034/100] Check if package was installed user --- mypy/test/testpackages.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 91ff27bc24be..758f80a17ced 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -26,7 +26,8 @@ def install_package(self, pkg: str, """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) install_cmd = [python, '-m', 'pip', 'install', '.'] - # if we aren't in a virtualenv, install in the user package directory so we don't need sudo + # if we aren't in a virtualenv, install in the + # user package directory so we don't need sudo if not hasattr(sys, 'real_prefix') or python != sys.executable: install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) @@ -39,8 +40,10 @@ def install_package(self, pkg: str, possible_paths = [os.path.join(dir, pkg) for dir in package_dirs] checked_paths = {path: os.path.exists(path) for path in possible_paths} raise AssertionError("Failed to typecheck with installed package {}.\n" - "Package paths checked:\n{}\n" - "Error traceback:\n{}\n".format(pkg, checked_paths, e) + "Package paths exist?:\n{}\n" + "Installed user?: {}\n" + "Error traceback:\n{}\n".format(pkg, checked_paths, + '--user' in install_cmd, e) ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From 31dab8da82f86c056bc20de61db99c3a11bb38cd Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 03:29:48 -0800 Subject: [PATCH 035/100] Check if user site exits --- mypy/test/testpackages.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 758f80a17ced..224c13fa7cc7 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import os +import site import sys from typing import Generator from unittest import TestCase, main @@ -42,8 +43,10 @@ def install_package(self, pkg: str, raise AssertionError("Failed to typecheck with installed package {}.\n" "Package paths exist?:\n{}\n" "Installed user?: {}\n" + "User-site?: {}\n" "Error traceback:\n{}\n".format(pkg, checked_paths, - '--user' in install_cmd, e) + '--user' in install_cmd, + site.getusersitepackages, e) ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From ff150575f398d8333d8513a9cd03a117c2882ebf Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 03:38:17 -0800 Subject: [PATCH 036/100] Parentheses have haunted me since Scheme --- mypy/test/testpackages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 224c13fa7cc7..9b3a1b55098b 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -46,7 +46,7 @@ def install_package(self, pkg: str, "User-site?: {}\n" "Error traceback:\n{}\n".format(pkg, checked_paths, '--user' in install_cmd, - site.getusersitepackages, e) + site.getusersitepackages(), e) ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From 944276560bdb6f9c3775e7e223ac4aaa3c2cbecd Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 04:02:21 -0800 Subject: [PATCH 037/100] Don't unconditionally set python --- mypy/build.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index feb3c9eece9f..4d9750fa71ea 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -876,7 +876,7 @@ def call_python(python: str, command: str) -> str: return subprocess.check_output([python, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') -def get_package_dirs(python: str) -> List[str]: +def get_package_dirs(python: Optional[str]) -> List[str]: """Find package directories for given python This defaults to the Python running mypy.""" @@ -916,8 +916,6 @@ def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if not python: - python = sys.executable package_dirs = get_package_dirs(python) if python: assert package_dirs, "Could not find package directories for Python '{}'".format(python) @@ -1697,7 +1695,7 @@ def __init__(self, # paths. When 3.4 is dropped, this should just use os.path.commonpath. if os.path.isabs(path): # Silence errors from module if it is in a package directory - python = manager.options.python_executable or sys.executable + python = manager.options.python_executable or None for dir in package_dirs_cache.get(python, []): if path.startswith(dir + os.sep): self.ignore_all = True From a34e7e886ec33da49560d35c185b4b3d7de600c4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 04:41:51 -0800 Subject: [PATCH 038/100] Cleanup tests --- mypy/test/testpackages.py | 76 +++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 9b3a1b55098b..e232c119f12f 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -2,7 +2,7 @@ import os import site import sys -from typing import Generator +from typing import Generator, List from unittest import TestCase, main import mypy.api @@ -21,6 +21,10 @@ class TestPackages(TestCase): + def tearDown(self): + if os.path.isfile('simple.py'): + os.remove('simple.py') + @contextmanager def install_package(self, pkg: str, python: str = sys.executable) -> Generator[None, None, None]: @@ -56,6 +60,17 @@ def test_get_package_dirs(self) -> None: dirs = get_package_dirs(sys.executable) assert dirs + @staticmethod + def check_mypy_run(cmd_line: List[str], + expected_out: str, + expected_err: str = '', + expected_returncode: int = 1) -> None: + """Helper to run mypy and check the output.""" + out, err, returncode = mypy.api.run(cmd_line) + assert out == expected_out + assert err == expected_err + assert expected_returncode == returncode + def test_typed_package(self) -> None: """Tests type checking based on installed packages. @@ -65,52 +80,45 @@ def test_typed_package(self) -> None: f.write(SIMPLE_PROGRAM) with self.install_package('typedpkg-stubs'): - out, err, returncode = mypy.api.run(['simple.py']) - assert \ - out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + # The Python 2 tests are intentionally placed after a Python 3 test to check + # the package_dir_cache is behaving correctly. python2 = try_find_python2_interpreter() if python2: with self.install_package('typedpkg-stubs', python2): - out, err, returncode = mypy.api.run( - ['--python-executable={}'.format(python2), 'simple.py']) - assert \ - out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) with self.install_package('typedpkg', python2): - out, err, returncode = mypy.api.run( - ['--python-executable={}'.format(python2), 'simple.py']) - assert out == "simple.py:4: error: Revealed type is " \ - "'builtins.tuple[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) with self.install_package('typedpkg', python2): with self.install_package('typedpkg-stubs', python2): - out, err, returncode = mypy.api.run( - ['--python-executable={}'.format(python2), 'simple.py']) - assert \ - out == "simple.py:4: error: Revealed type is " \ - "'builtins.list[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) with self.install_package('typedpkg'): - out, err, returncode = mypy.api.run(['simple.py']) - assert out == "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) with self.install_package('typedpkg'): with self.install_package('typedpkg-stubs'): - out, err, returncode = mypy.api.run(['simple.py']) - assert \ - out == "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - assert returncode == 1 - assert err == '' + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) os.remove('simple.py') From 0fd592a8bc17d265a76901cce6fda3606c46f576 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 04:49:54 -0800 Subject: [PATCH 039/100] Fix for mypy check --- mypy/build.py | 2 +- mypy/test/testpackages.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 4d9750fa71ea..04e40ba32d74 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -820,7 +820,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] # Cache packages for each Python executable -package_dirs_cache = {} # type: Dict[str, List[str]] +package_dirs_cache = {} # type: Dict[Optional[str], List[str]] def find_module_clear_caches() -> None: diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index e232c119f12f..16594b991c68 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -21,7 +21,7 @@ class TestPackages(TestCase): - def tearDown(self): + def tearDown(self) -> None: if os.path.isfile('simple.py'): os.remove('simple.py') From 4cbe21a1758cf9dc4c4e03055334211822ac2c5b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 05:04:28 -0800 Subject: [PATCH 040/100] Remove debugging info --- mypy/test/testpackages.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 16594b991c68..8faaaaf91671 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -40,18 +40,6 @@ def install_package(self, pkg: str, self.fail('\n'.join(lines)) try: yield - except AssertionError as e: - package_dirs = get_package_dirs(python) - possible_paths = [os.path.join(dir, pkg) for dir in package_dirs] - checked_paths = {path: os.path.exists(path) for path in possible_paths} - raise AssertionError("Failed to typecheck with installed package {}.\n" - "Package paths exist?:\n{}\n" - "Installed user?: {}\n" - "User-site?: {}\n" - "Error traceback:\n{}\n".format(pkg, checked_paths, - '--user' in install_cmd, - site.getusersitepackages(), e) - ).with_traceback(sys.exc_info()[2]) finally: run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) From f566ed0c1e4a671b8e334a7a7e6129bb66f7779d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 13:09:50 -0800 Subject: [PATCH 041/100] Try to delete simple.py after all tests in case have run --- mypy/test/testpackages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 8faaaaf91671..e91bfbeedb77 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -21,7 +21,7 @@ class TestPackages(TestCase): - def tearDown(self) -> None: + def tearDownClass(cls) -> None: if os.path.isfile('simple.py'): os.remove('simple.py') @@ -55,8 +55,8 @@ def check_mypy_run(cmd_line: List[str], expected_returncode: int = 1) -> None: """Helper to run mypy and check the output.""" out, err, returncode = mypy.api.run(cmd_line) - assert out == expected_out - assert err == expected_err + assert out == expected_out, err + assert err == expected_err, out assert expected_returncode == returncode def test_typed_package(self) -> None: From 206f70b0ae2b91fff9fc2702ed41b9661d9ead4f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 13:31:00 -0800 Subject: [PATCH 042/100] Make tearDownClass a classmethod --- mypy/test/testpackages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index e91bfbeedb77..dc7d09a4b0b1 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -21,6 +21,7 @@ class TestPackages(TestCase): + @classmethod def tearDownClass(cls) -> None: if os.path.isfile('simple.py'): os.remove('simple.py') From 2dbff9727e583dcefbbb1f5c247ea1a27eaa3588 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 13:42:22 -0800 Subject: [PATCH 043/100] Don't clean up twice, as it can break stubgen tests --- mypy/test/testpackages.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index dc7d09a4b0b1..221a46055aef 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -21,11 +21,6 @@ class TestPackages(TestCase): - @classmethod - def tearDownClass(cls) -> None: - if os.path.isfile('simple.py'): - os.remove('simple.py') - @contextmanager def install_package(self, pkg: str, python: str = sys.executable) -> Generator[None, None, None]: From e33ceb781e10c374a8b71fc55d05bfa57cfbdb84 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 14:12:51 -0800 Subject: [PATCH 044/100] Set python version from executable --- mypy/build.py | 4 ---- mypy/main.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 04e40ba32d74..ff5c151063c9 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -886,10 +886,6 @@ def get_package_dirs(python: Optional[str]) -> List[str]: if python: # Use subprocess to get the package directory of given Python # executable - check = subprocess.check_output([python, '-V'], stderr=subprocess.STDOUT).decode('UTF-8') - assert check.startswith('Python'), \ - "Mypy could not use the Python executable: {}".format(python) - # If we have a working python executable, query information from it try: output = call_python(python, USER_SITE_PACKAGES) for line in output.splitlines(): diff --git a/mypy/main.py b/mypy/main.py index 91139e04bba6..6395ad32aac3 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -5,6 +5,7 @@ import fnmatch import os import re +import subprocess import sys import time @@ -516,6 +517,15 @@ def add_invertible_flag(flag: str, if options.quick_and_dirty: options.incremental = True + if options.python_executable: + check = subprocess.check_output([options.python_executable, '-V'], + stderr=subprocess.STDOUT).decode('UTF-8') + assert check.startswith('Python'), \ + "Mypy could not use the Python executable: {}".format(options.python_executable) + ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) + if ver: + options.python_version = (int(v) for v in ver.groups()) + # Set target. if special_opts.modules: options.build_type = BuildType.MODULE From 644e9a5496f27437ff4067db42d9bec40da495e0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 16 Feb 2018 15:45:44 -0800 Subject: [PATCH 045/100] Fix mypy self check and doc indentation --- mypy/build.py | 2 +- mypy/main.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ff5c151063c9..a33cb082179b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -880,7 +880,7 @@ def get_package_dirs(python: Optional[str]) -> List[str]: """Find package directories for given python This defaults to the Python running mypy.""" - if package_dirs_cache.get(python, None): + if package_dirs_cache.get(python, None) is not None: return package_dirs_cache[python] package_dirs = [] # type: List[str] if python: diff --git a/mypy/main.py b/mypy/main.py index 6395ad32aac3..6ff83400948e 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -524,7 +524,8 @@ def add_invertible_flag(flag: str, "Mypy could not use the Python executable: {}".format(options.python_executable) ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) if ver: - options.python_version = (int(v) for v in ver.groups()) + python_ver = int(ver.group(1)), int(ver.group(2)) + options.python_version = python_ver # Set target. if special_opts.modules: From af38a12f63cd049495a18e64b659d049503758c9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 18 Feb 2018 14:24:44 -0800 Subject: [PATCH 046/100] Dedent docs --- docs/source/installed_packages.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index ff76b298f832..da915a7b53b8 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -47,9 +47,9 @@ should have a suffix of ``-stubs``. For example, if we had stubs for __init__.pyi lib.pyi - the setup.py might look like: +the setup.py might look like: - .. code-block:: python +.. code-block:: python from distutils.core import setup @@ -73,4 +73,4 @@ can use the ``--python-executable`` flag to point to the executable, and mypy will find packages installed for that python executable. Note that mypy does not support some more dynamic import features, such as zip -imports and advanced custom import hooks. \ No newline at end of file +imports, namespace packages, and advanced custom import hooks. \ No newline at end of file From b7f07884f105a61b01ab4fd083805b11b2d84f6f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 18 Feb 2018 15:02:56 -0800 Subject: [PATCH 047/100] Add missed flag back --- docs/source/command_line.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 81bcd1d95b9f..3d27211cae35 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -21,7 +21,7 @@ flag (or its long form ``--help``):: [--warn-redundant-casts] [--no-warn-no-return] [--warn-return-any] [--warn-unused-ignores] [--warn-unused-configs] [--show-error-context] [--no-implicit-optional] [-i] - [--quick-and-dirty] [--cache-dir DIR] + [--quick-and-dirty] [--cache-dir DIR] [--cache-fine-grained] [--skip-version-check] [--strict-optional] [--strict-optional-whitelist [GLOB [GLOB ...]]] [--junit-xml JUNIT_XML] [--pdb] [--show-traceback] [--stats] From 0564a5c5e40df25e75d56c85ac122d0447b58f03 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 15:55:13 -0800 Subject: [PATCH 048/100] Remove silencing of errors in installed packages --- mypy/build.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a33cb082179b..89f6f19f7097 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1687,14 +1687,6 @@ def __init__(self, file_id = '__builtin__' path = find_module(file_id, manager.lib_path, manager.options.python_executable) if path: - # Installed package modules should be silenced. They are all under absolute - # paths. When 3.4 is dropped, this should just use os.path.commonpath. - if os.path.isabs(path): - # Silence errors from module if it is in a package directory - python = manager.options.python_executable or None - for dir in package_dirs_cache.get(python, []): - if path.startswith(dir + os.sep): - self.ignore_all = True # For non-stubs, look at options.follow_imports: # - normal (default) -> fully analyze # - silent -> analyze but silence errors From d17582bbeb5ac218a13a9b066ca99b46c41688f6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 15:59:20 -0800 Subject: [PATCH 049/100] Clean up get_package_dirs --- mypy/build.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 89f6f19f7097..ad8f84e44df8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -797,7 +797,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: return p -# Cache find_module: (id) -> result. +# Cache find_module: id -> result. find_module_cache = {} # type: Dict[str, Optional[str]] # Cache some repeated work within distinct find_module calls: finding which @@ -876,14 +876,22 @@ def call_python(python: str, command: str) -> str: return subprocess.check_output([python, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') -def get_package_dirs(python: Optional[str]) -> List[str]: +def get_package_dirs(python: str) -> List[str]: """Find package directories for given python - This defaults to the Python running mypy.""" - if package_dirs_cache.get(python, None) is not None: + This defaults to the Python running mypy. + """ + if python in package_dirs_cache: return package_dirs_cache[python] package_dirs = [] # type: List[str] - if python: + if python == sys.executable: + # Use running Python's package dirs + try: + user_dir = site.getusersitepackages() + package_dirs = site.getsitepackages() + [user_dir] + except AttributeError: + package_dirs = [get_python_lib()] + else: # Use subprocess to get the package directory of given Python # executable try: @@ -897,13 +905,6 @@ def get_package_dirs(python: Optional[str]) -> List[str]: for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) - else: - # Use running Python's package dirs - try: - user_dir = site.getusersitepackages() - package_dirs = site.getsitepackages() + [user_dir] - except AttributeError: - package_dirs = [get_python_lib()] package_dirs_cache[python] = package_dirs return package_dirs @@ -912,6 +913,8 @@ def find_module(id: str, lib_path_arg: Iterable[str], python: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) + if not python: + python = sys.executable package_dirs = get_package_dirs(python) if python: assert package_dirs, "Could not find package directories for Python '{}'".format(python) From 085d00202ac4054f07954757f7737f387066a864 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:09:24 -0800 Subject: [PATCH 050/100] Make assert a user facing error --- mypy/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ad8f84e44df8..ab3178d3dab7 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -916,8 +916,9 @@ def find_module(id: str, lib_path_arg: Iterable[str], if not python: python = sys.executable package_dirs = get_package_dirs(python) - if python: - assert package_dirs, "Could not find package directories for Python '{}'".format(python) + if python and not package_dirs: + print("Could not find package directories for Python '{}'".format(python), file=sys.stderr) + sys.exit(2) components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' From 8958dc79987764f74a54dcd1c4061a3a3de3dd00 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:19:34 -0800 Subject: [PATCH 051/100] Refactor get_package_dirs --- mypy/build.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ab3178d3dab7..3183ed45f5dc 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -896,15 +896,17 @@ def get_package_dirs(python: str) -> List[str]: # executable try: output = call_python(python, USER_SITE_PACKAGES) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) except subprocess.CalledProcessError: - # if no paths are found, we fall back on sysconfig - output = call_python(python, VIRTUALENV_SITE_PACKAGES) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) + output = '' + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) + # if no paths are found, we fall back on sysconfig, the python is likely in a + # virtual environment, thus lacking needed site methods + output = call_python(python, VIRTUALENV_SITE_PACKAGES) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) package_dirs_cache[python] = package_dirs return package_dirs From 84fd527722a8479a588d13d4a617eb8e23f7899e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:20:10 -0800 Subject: [PATCH 052/100] Make test package typesafe to avoid error --- test-data/packages/typedpkg/typedpkg/sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py index a01d857169ce..6497c315f373 100644 --- a/test-data/packages/typedpkg/typedpkg/sample.py +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -3,4 +3,4 @@ def ex(a: Iterable[str]) -> Tuple[str, ...]: """Example typed package. This intentionally has an error.""" - return a + ('Hello') + return tuple(a) From 4c0ff1b920c615a1f3379ea9d53e051dd851036f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:23:28 -0800 Subject: [PATCH 053/100] Clarify docs of unsupported import features --- docs/source/installed_packages.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index da915a7b53b8..fb05d8041038 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -72,5 +72,5 @@ If you use mypy to type check a Python other than the version running mypy, you can use the ``--python-executable`` flag to point to the executable, and mypy will find packages installed for that python executable. -Note that mypy does not support some more dynamic import features, such as zip -imports, namespace packages, and advanced custom import hooks. \ No newline at end of file +Note that mypy does not support some more advanced import features, such as zip +imports, namespace packages, and custom import hooks. \ No newline at end of file From a917ff5ffd1c919eae0fc4690917c7a4787f8d07 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:24:44 -0800 Subject: [PATCH 054/100] Reverse equality order --- mypy/test/testpackages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 221a46055aef..a253e791e873 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -53,7 +53,7 @@ def check_mypy_run(cmd_line: List[str], out, err, returncode = mypy.api.run(cmd_line) assert out == expected_out, err assert err == expected_err, out - assert expected_returncode == returncode + assert returncode == expected_returncode, returncode def test_typed_package(self) -> None: """Tests type checking based on installed packages. From 83ab97319712ab2e78aef0a5f8256705433f37b2 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:30:38 -0800 Subject: [PATCH 055/100] Correct to more sensible key type for package_dir_cache --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 3183ed45f5dc..eb51c8ec8cd2 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -820,7 +820,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] # Cache packages for each Python executable -package_dirs_cache = {} # type: Dict[Optional[str], List[str]] +package_dirs_cache = {} # type: Dict[str, List[str]] def find_module_clear_caches() -> None: From cfac326a11897404de11c752bebdace940253a76 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 16:37:47 -0800 Subject: [PATCH 056/100] python -> python_executable --- mypy/build.py | 27 ++++++++++++++------------- mypy/test/testpackages.py | 8 ++++---- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index eb51c8ec8cd2..14a131e857d3 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -876,15 +876,15 @@ def call_python(python: str, command: str) -> str: return subprocess.check_output([python, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') -def get_package_dirs(python: str) -> List[str]: +def get_package_dirs(python_executable: str) -> List[str]: """Find package directories for given python This defaults to the Python running mypy. """ - if python in package_dirs_cache: - return package_dirs_cache[python] + if python_executable in package_dirs_cache: + return package_dirs_cache[python_executable] package_dirs = [] # type: List[str] - if python == sys.executable: + if python_executable == sys.executable: # Use running Python's package dirs try: user_dir = site.getusersitepackages() @@ -895,7 +895,7 @@ def get_package_dirs(python: str) -> List[str]: # Use subprocess to get the package directory of given Python # executable try: - output = call_python(python, USER_SITE_PACKAGES) + output = call_python(python_executable, USER_SITE_PACKAGES) except subprocess.CalledProcessError: output = '' for line in output.splitlines(): @@ -903,23 +903,24 @@ def get_package_dirs(python: str) -> List[str]: package_dirs.append(line) # if no paths are found, we fall back on sysconfig, the python is likely in a # virtual environment, thus lacking needed site methods - output = call_python(python, VIRTUALENV_SITE_PACKAGES) + output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) - package_dirs_cache[python] = package_dirs + package_dirs_cache[python_executable] = package_dirs return package_dirs def find_module(id: str, lib_path_arg: Iterable[str], - python: Optional[str] = None) -> Optional[str]: + python_executable: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if not python: - python = sys.executable - package_dirs = get_package_dirs(python) - if python and not package_dirs: - print("Could not find package directories for Python '{}'".format(python), file=sys.stderr) + if not python_executable: + python_executable = sys.executable + package_dirs = get_package_dirs(python_executable) + if python_executable and not package_dirs: + print("Could not find package directories for Python '{}'".format( + python_executable), file=sys.stderr) sys.exit(2) components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index a253e791e873..269d32cccd75 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -23,13 +23,13 @@ class TestPackages(TestCase): @contextmanager def install_package(self, pkg: str, - python: str = sys.executable) -> Generator[None, None, None]: + python_executable: str = sys.executable) -> Generator[None, None, None]: """Context manager to temporarily install a package from test-data/packages/pkg/""" working_dir = os.path.join(package_path, pkg) - install_cmd = [python, '-m', 'pip', 'install', '.'] + install_cmd = [python_executable, '-m', 'pip', 'install', '.'] # if we aren't in a virtualenv, install in the # user package directory so we don't need sudo - if not hasattr(sys, 'real_prefix') or python != sys.executable: + if not hasattr(sys, 'real_prefix') or python_executable != sys.executable: install_cmd.append('--user') returncode, lines = run_command(install_cmd, cwd=working_dir) if returncode != 0: @@ -37,7 +37,7 @@ def install_package(self, pkg: str, try: yield finally: - run_command([python, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) + run_command([python_executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) def test_get_package_dirs(self) -> None: """Check that get_package_dirs works.""" From aefcb9625c0a8d6f5320f13c28a6197c203f7bba Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 17:23:18 -0800 Subject: [PATCH 057/100] Set python_executable based on python_version --- mypy/main.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 6ff83400948e..6dc29e1287fc 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -206,6 +206,12 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +if sys.platform == 'win32': + python_executable_prefix = 'py -{}' +else: + python_executable_prefix = 'python{}' + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -517,15 +523,31 @@ def add_invertible_flag(flag: str, if options.quick_and_dirty: options.incremental = True + # try setting a valid Python executable based on a specified version + if options.python_version: + if not options.python_executable: + try: + output = subprocess.check_output( + [python_executable_prefix.format(options.python_version)], + stderr=subprocess.STDOUT).decode() + except FileNotFoundError: + output = '' + if output.startswith('Python '): + options.python_executable = python_executable_prefix.format(options.python_version) + + # Set Python version if given Python executable, but no version if options.python_executable: - check = subprocess.check_output([options.python_executable, '-V'], - stderr=subprocess.STDOUT).decode('UTF-8') - assert check.startswith('Python'), \ - "Mypy could not use the Python executable: {}".format(options.python_executable) - ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) - if ver: - python_ver = int(ver.group(1)), int(ver.group(2)) - options.python_version = python_ver + if not options.python_version: + check = subprocess.check_output([options.python_executable, '-V'], + stderr=subprocess.STDOUT).decode() + if not check.startswith('Python '): + print("Mypy could not use the Python executable: {}".format( + options.python_executable), file=sys.stderr) + sys.exit(2) + ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) + if ver: + python_ver = int(ver.group(1)), int(ver.group(2)) + options.python_version = python_ver # Set target. if special_opts.modules: From 380e30134862976cfe789fff388c2c1c03677317 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:06:46 -0800 Subject: [PATCH 058/100] Fix weird indentation --- mypy/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 14a131e857d3..93c521264214 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -879,8 +879,8 @@ def call_python(python: str, command: str) -> str: def get_package_dirs(python_executable: str) -> List[str]: """Find package directories for given python - This defaults to the Python running mypy. - """ + This defaults to the Python running mypy. + """ if python_executable in package_dirs_cache: return package_dirs_cache[python_executable] package_dirs = [] # type: List[str] From 47d8e0d136df5b3970973ff130c7a687e380e4f0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:09:22 -0800 Subject: [PATCH 059/100] Refactor and clarify checks for package information --- mypy/build.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 93c521264214..8dcc3b6b3a1b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -898,15 +898,17 @@ def get_package_dirs(python_executable: str) -> List[str]: output = call_python(python_executable, USER_SITE_PACKAGES) except subprocess.CalledProcessError: output = '' - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) - # if no paths are found, we fall back on sysconfig, the python is likely in a - # virtual environment, thus lacking needed site methods - output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) + if output: + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) + else: + # if no paths are found, we fall back on sysconfig, the python is likely in a + # virtual environment, thus lacking needed site methods + output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) + for line in output.splitlines(): + if os.path.isdir(line): + package_dirs.append(line) package_dirs_cache[python_executable] = package_dirs return package_dirs From 5c19e8ccdfd297908005633d2ca3bfceacc67a59 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:12:07 -0800 Subject: [PATCH 060/100] Simplify check --- mypy/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/build.py b/mypy/build.py index 8dcc3b6b3a1b..59d2ecd54f81 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -920,7 +920,7 @@ def find_module(id: str, lib_path_arg: Iterable[str], if not python_executable: python_executable = sys.executable package_dirs = get_package_dirs(python_executable) - if python_executable and not package_dirs: + if not package_dirs: print("Could not find package directories for Python '{}'".format( python_executable), file=sys.stderr) sys.exit(2) From 56dfcd403ef8d3ad187aca2d42f63116b8fb0a44 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:12:49 -0800 Subject: [PATCH 061/100] Last python -> python_executable --- mypy/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 59d2ecd54f81..e715cd91349c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -872,8 +872,8 @@ def is_file(path: str) -> bool: 'from distutils.sysconfig import get_python_lib; print(get_python_lib())' -def call_python(python: str, command: str) -> str: - return subprocess.check_output([python, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') +def call_python(python_executable: str, command: str) -> str: + return subprocess.check_output([python_executable, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') def get_package_dirs(python_executable: str) -> List[str]: From 4293dadeb9a1cd4d7a7642653102c0d45b4714d2 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:18:55 -0800 Subject: [PATCH 062/100] Use lru_cache on get_package_dirs --- mypy/build.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index e715cd91349c..ea1cc0811f49 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -14,6 +14,7 @@ import collections import contextlib from distutils.sysconfig import get_python_lib +import functools import gc import hashlib import json @@ -819,8 +820,6 @@ def remove_cwd_prefix_from_path(p: str) -> str: # Cache for isdir(join(head, tail)) find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] -# Cache packages for each Python executable -package_dirs_cache = {} # type: Dict[str, List[str]] def find_module_clear_caches() -> None: @@ -876,13 +875,12 @@ def call_python(python_executable: str, command: str) -> str: return subprocess.check_output([python_executable, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') +@functools.lru_cache(maxsize=None) def get_package_dirs(python_executable: str) -> List[str]: """Find package directories for given python This defaults to the Python running mypy. """ - if python_executable in package_dirs_cache: - return package_dirs_cache[python_executable] package_dirs = [] # type: List[str] if python_executable == sys.executable: # Use running Python's package dirs @@ -909,7 +907,6 @@ def get_package_dirs(python_executable: str) -> List[str]: for line in output.splitlines(): if os.path.isdir(line): package_dirs.append(line) - package_dirs_cache[python_executable] = package_dirs return package_dirs From b810d3e0a3a091e00f5e636be193a819783a03fe Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 18:27:48 -0800 Subject: [PATCH 063/100] Fix lint --- mypy/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index ea1cc0811f49..e53d9f5a8697 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -821,7 +821,6 @@ def remove_cwd_prefix_from_path(p: str) -> str: find_module_isdir_cache = {} # type: Dict[Tuple[str, str], bool] - def find_module_clear_caches() -> None: find_module_cache.clear() find_module_dir_cache.clear() @@ -872,7 +871,8 @@ def is_file(path: str) -> bool: def call_python(python_executable: str, command: str) -> str: - return subprocess.check_output([python_executable, '-c', command], stderr=subprocess.PIPE).decode('UTF-8') + return subprocess.check_output([python_executable, '-c', command], + stderr=subprocess.PIPE).decode('UTF-8') @functools.lru_cache(maxsize=None) From 17488ad59272c1cab4b5d53472c5fdceb4da4b3d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 20:32:29 -0800 Subject: [PATCH 064/100] Refactor version setting from executable --- mypy/main.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 6dc29e1287fc..6955555d8d36 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -212,6 +212,18 @@ def invert_flag_name(flag: str) -> str: python_executable_prefix = 'python{}' +def _python_version_from_executable(python_executable: str) -> Optional[Tuple[int, int]]: + check = subprocess.check_output([python_executable, '-V'], + stderr=subprocess.STDOUT).decode() + if not check.startswith('Python '): + print("Mypy could not use the Python executable: {}".format( + python_executable), file=sys.stderr) + sys.exit(2) + ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) + if ver: + return int(ver.group(1)), int(ver.group(2)) + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -538,15 +550,8 @@ def add_invertible_flag(flag: str, # Set Python version if given Python executable, but no version if options.python_executable: if not options.python_version: - check = subprocess.check_output([options.python_executable, '-V'], - stderr=subprocess.STDOUT).decode() - if not check.startswith('Python '): - print("Mypy could not use the Python executable: {}".format( - options.python_executable), file=sys.stderr) - sys.exit(2) - ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) - if ver: - python_ver = int(ver.group(1)), int(ver.group(2)) + python_ver = _python_version_from_executable(options.python_executable) + if python_ver: options.python_version = python_ver # Set target. From e348471eb6518f86b0fa9103a03e237d2bfcccea Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 23:14:38 -0800 Subject: [PATCH 065/100] Handle python executable with spaces --- mypy/build.py | 22 +++++++--------------- mypy/main.py | 45 ++++++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index e53d9f5a8697..905cb136dbe1 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -871,8 +871,8 @@ def is_file(path: str) -> bool: def call_python(python_executable: str, command: str) -> str: - return subprocess.check_output([python_executable, '-c', command], - stderr=subprocess.PIPE).decode('UTF-8') + return subprocess.check_output(python_executable.split(' ') + ['-c', command], + stderr=subprocess.PIPE).decode() @functools.lru_cache(maxsize=None) @@ -881,7 +881,6 @@ def get_package_dirs(python_executable: str) -> List[str]: This defaults to the Python running mypy. """ - package_dirs = [] # type: List[str] if python_executable == sys.executable: # Use running Python's package dirs try: @@ -895,18 +894,11 @@ def get_package_dirs(python_executable: str) -> List[str]: try: output = call_python(python_executable, USER_SITE_PACKAGES) except subprocess.CalledProcessError: - output = '' - if output: - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) - else: - # if no paths are found, we fall back on sysconfig, the python is likely in a - # virtual environment, thus lacking needed site methods + # if no paths are found (raising a CalledProcessError, we fall back on sysconfig, + # the python executable is likely in a virtual environment, thus lacking + # needed site methods output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) - for line in output.splitlines(): - if os.path.isdir(line): - package_dirs.append(line) + return [line for line in output.splitlines() if os.path.isdir(line)] return package_dirs @@ -914,7 +906,7 @@ def find_module(id: str, lib_path_arg: Iterable[str], python_executable: Optional[str] = None) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if not python_executable: + if python_executable is None: python_executable = sys.executable package_dirs = get_package_dirs(python_executable) if not package_dirs: diff --git a/mypy/main.py b/mypy/main.py index 6955555d8d36..6525c817227f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -212,16 +212,28 @@ def invert_flag_name(flag: str) -> str: python_executable_prefix = 'python{}' -def _python_version_from_executable(python_executable: str) -> Optional[Tuple[int, int]]: - check = subprocess.check_output([python_executable, '-V'], - stderr=subprocess.STDOUT).decode() - if not check.startswith('Python '): - print("Mypy could not use the Python executable: {}".format( - python_executable), file=sys.stderr) - sys.exit(2) - ver = re.fullmatch(r'Python (\d)\.(\d)\.\d\s*', check) - if ver: - return int(ver.group(1)), int(ver.group(2)) +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output(python_executable.split(' ') + + ['-c', 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + except subprocess.CalledProcessError: + return sys.version_info[:2] + else: + ver = re.fullmatch(r'(\d)\.(\d+)\s*', check) + if ver: + return int(ver.group(1)), int(ver.group(2)) + else: + return sys.version_info[:2] + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: + str_ver = '.'.join(map(str, python_version)) + python_ver = _python_version_from_executable(python_executable_prefix.format(str_ver)) + if python_ver == sys.version_info[:2]: + return None + else: + return python_executable_prefix.format(str_ver) def process_options(args: List[str], @@ -538,21 +550,12 @@ def add_invertible_flag(flag: str, # try setting a valid Python executable based on a specified version if options.python_version: if not options.python_executable: - try: - output = subprocess.check_output( - [python_executable_prefix.format(options.python_version)], - stderr=subprocess.STDOUT).decode() - except FileNotFoundError: - output = '' - if output.startswith('Python '): - options.python_executable = python_executable_prefix.format(options.python_version) + options.python_executable = _python_executable_from_version(options.python_version) # Set Python version if given Python executable, but no version if options.python_executable: if not options.python_version: - python_ver = _python_version_from_executable(options.python_executable) - if python_ver: - options.python_version = python_ver + options.python_version = _python_version_from_executable(options.python_executable) # Set target. if special_opts.modules: From 81849376320cb0c997d20d310a10f2e45606fd9f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 23:16:19 -0800 Subject: [PATCH 066/100] Return early in get_package_dirs --- mypy/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 905cb136dbe1..107849878509 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -885,9 +885,10 @@ def get_package_dirs(python_executable: str) -> List[str]: # Use running Python's package dirs try: user_dir = site.getusersitepackages() - package_dirs = site.getsitepackages() + [user_dir] + return site.getsitepackages() + [user_dir] except AttributeError: - package_dirs = [get_python_lib()] + # fall back on get_python_lib for virtualenvs + return [get_python_lib()] else: # Use subprocess to get the package directory of given Python # executable @@ -899,7 +900,6 @@ def get_package_dirs(python_executable: str) -> List[str]: # needed site methods output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) return [line for line in output.splitlines() if os.path.isdir(line)] - return package_dirs def find_module(id: str, lib_path_arg: Iterable[str], From 97f29cc3063845e8e6a6ecef30344b029aad9d1f Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Tue, 20 Feb 2018 23:43:11 -0800 Subject: [PATCH 067/100] Refactor and clean up testpackages --- mypy/test/testpackages.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 269d32cccd75..ee3caba7f60a 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -1,6 +1,5 @@ from contextlib import contextmanager import os -import site import sys from typing import Generator, List from unittest import TestCase, main @@ -11,7 +10,6 @@ from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter - SIMPLE_PROGRAM = """ from typedpkg.sample import ex a = ex(['']) @@ -20,7 +18,6 @@ class TestPackages(TestCase): - @contextmanager def install_package(self, pkg: str, python_executable: str = sys.executable) -> Generator[None, None, None]: @@ -69,8 +66,18 @@ def test_typed_package(self) -> None: "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - # The Python 2 tests are intentionally placed after a Python 3 test to check - # the package_dir_cache is behaving correctly. + with self.install_package('typedpkg'): + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg'), self.install_package('typedpkg-stubs'): + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + python2 = try_find_python2_interpreter() if python2: with self.install_package('typedpkg-stubs', python2): @@ -84,23 +91,10 @@ def test_typed_package(self) -> None: "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) - with self.install_package('typedpkg', python2): - with self.install_package('typedpkg-stubs', python2): - self.check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - ) - - with self.install_package('typedpkg'): - self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - ) - - with self.install_package('typedpkg'): - with self.install_package('typedpkg-stubs'): + with self.install_package('typedpkg', python2), \ + self.install_package('typedpkg-stubs', python2): self.check_mypy_run( - ['simple.py'], + ['--python-executable={}'.format(python2), 'simple.py'], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) os.remove('simple.py') From 928683a88fb6f6575489c90de2a6fc3929415fc4 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Feb 2018 00:06:34 -0800 Subject: [PATCH 068/100] Fix executable flag and version inference --- mypy/build.py | 2 +- mypy/main.py | 34 ++++++++++++++++++---------------- mypy/test/testpackages.py | 33 ++++++++++++++++++--------------- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 107849878509..28cea2935615 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -871,7 +871,7 @@ def is_file(path: str) -> bool: def call_python(python_executable: str, command: str) -> str: - return subprocess.check_output(python_executable.split(' ') + ['-c', command], + return subprocess.check_output([python_executable, '-c', command], stderr=subprocess.PIPE).decode() diff --git a/mypy/main.py b/mypy/main.py index 6525c817227f..087a4d64b1e1 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -1,6 +1,7 @@ """Mypy type checker command line tool.""" import argparse +import ast import configparser import fnmatch import os @@ -214,26 +215,29 @@ def invert_flag_name(flag: str) -> str: def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: try: - check = subprocess.check_output(python_executable.split(' ') + - ['-c', 'import sys; print(repr(sys.version_info[:2]))'], + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], stderr=subprocess.STDOUT).decode() except subprocess.CalledProcessError: return sys.version_info[:2] else: - ver = re.fullmatch(r'(\d)\.(\d+)\s*', check) - if ver: - return int(ver.group(1)), int(ver.group(2)) - else: - return sys.version_info[:2] + return ast.literal_eval(check) def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: str_ver = '.'.join(map(str, python_version)) - python_ver = _python_version_from_executable(python_executable_prefix.format(str_ver)) - if python_ver == sys.version_info[:2]: + try: + sys_exe = subprocess.check_output(python_executable_prefix.format(str_ver).split(' ') + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + except subprocess.CalledProcessError: return None else: - return python_executable_prefix.format(str_ver) + # don't set if its the sys.executable as this will break things later + if sys_exe.lower() != sys.executable: + return sys_exe + else: + return None def process_options(args: List[str], @@ -548,14 +552,12 @@ def add_invertible_flag(flag: str, options.incremental = True # try setting a valid Python executable based on a specified version - if options.python_version: - if not options.python_executable: - options.python_executable = _python_executable_from_version(options.python_version) + if options.python_version and not options.python_executable: + options.python_executable = _python_executable_from_version(options.python_version) # Set Python version if given Python executable, but no version - if options.python_executable: - if not options.python_version: - options.python_version = _python_version_from_executable(options.python_executable) + if options.python_executable and not options.python_version: + options.python_version = _python_version_from_executable(options.python_executable) # Set target. if special_opts.modules: diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index ee3caba7f60a..3bd77b69e2ab 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -66,18 +66,8 @@ def test_typed_package(self) -> None: "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - with self.install_package('typedpkg'): - self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - ) - - with self.install_package('typedpkg'), self.install_package('typedpkg-stubs'): - self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - ) - + # The Python 2 tests are intentionally placed after a Python 3 test to check + # the package_dir_cache is behaving correctly. python2 = try_find_python2_interpreter() if python2: with self.install_package('typedpkg-stubs', python2): @@ -91,10 +81,23 @@ def test_typed_package(self) -> None: "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) - with self.install_package('typedpkg', python2), \ - self.install_package('typedpkg-stubs', python2): + with self.install_package('typedpkg', python2): + with self.install_package('typedpkg-stubs', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + self.check_mypy_run( + ['simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) + + with self.install_package('typedpkg'): + with self.install_package('typedpkg-stubs'): self.check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], + ['simple.py'], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) os.remove('simple.py') From ae65564b9ff8192a3216ce4ab06fcd8be0a6bff8 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Feb 2018 10:39:39 -0800 Subject: [PATCH 069/100] Clean up Python version/executable inference --- mypy/main.py | 80 +++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 087a4d64b1e1..c872e293783f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -45,6 +45,37 @@ def stat_proxy(path: str) -> os.stat_result: return st +if sys.platform == 'win32': + def python_executable_prefix(v: str) -> List[str]: + return ['py', '-{}'.format(v)] +else: + def python_executable_prefix(v: str) -> List[str]: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + except (subprocess.CalledProcessError, FileNotFoundError): + return sys.version_info[:2] + else: + return ast.literal_eval(check) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: + str_ver = '.'.join(map(str, python_version)) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + else: + return sys_exe + + def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: """Main entry point to the type checker. @@ -64,6 +95,14 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] sources, options = process_options(args) + # try setting a valid Python executable based on a specified version + if options.python_version and not options.python_executable: + options.python_executable = _python_executable_from_version(options.python_version) + + # Set Python version if given Python executable, but no version + if options.python_executable and not options.python_version: + options.python_version = _python_version_from_executable(options.python_executable) + messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: @@ -207,39 +246,6 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) -if sys.platform == 'win32': - python_executable_prefix = 'py -{}' -else: - python_executable_prefix = 'python{}' - - -def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: - try: - check = subprocess.check_output([python_executable, '-c', - 'import sys; print(repr(sys.version_info[:2]))'], - stderr=subprocess.STDOUT).decode() - except subprocess.CalledProcessError: - return sys.version_info[:2] - else: - return ast.literal_eval(check) - - -def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: - str_ver = '.'.join(map(str, python_version)) - try: - sys_exe = subprocess.check_output(python_executable_prefix.format(str_ver).split(' ') + - ['-c', 'import sys; print(sys.executable)'], - stderr=subprocess.STDOUT).decode().strip() - except subprocess.CalledProcessError: - return None - else: - # don't set if its the sys.executable as this will break things later - if sys_exe.lower() != sys.executable: - return sys_exe - else: - return None - - def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -551,14 +557,6 @@ def add_invertible_flag(flag: str, if options.quick_and_dirty: options.incremental = True - # try setting a valid Python executable based on a specified version - if options.python_version and not options.python_executable: - options.python_executable = _python_executable_from_version(options.python_version) - - # Set Python version if given Python executable, but no version - if options.python_executable and not options.python_version: - options.python_version = _python_version_from_executable(options.python_executable) - # Set target. if special_opts.modules: options.build_type = BuildType.MODULE From f6c43277eae385a0f18330fb20f3837c4cde3895 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Feb 2018 12:44:08 -0800 Subject: [PATCH 070/100] Make typedpkg python2 compatible --- test-data/packages/typedpkg/typedpkg/sample.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-data/packages/typedpkg/typedpkg/sample.py b/test-data/packages/typedpkg/typedpkg/sample.py index 6497c315f373..59f6aec1548e 100644 --- a/test-data/packages/typedpkg/typedpkg/sample.py +++ b/test-data/packages/typedpkg/typedpkg/sample.py @@ -1,6 +1,7 @@ from typing import Iterable, Tuple -def ex(a: Iterable[str]) -> Tuple[str, ...]: +def ex(a): + # type: (Iterable[str]) -> Tuple[str, ...] """Example typed package. This intentionally has an error.""" return tuple(a) From d475d4b3e3ba44bd769f3dd518c9a1c535f69d7d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Feb 2018 12:45:02 -0800 Subject: [PATCH 071/100] Overwrite default Python version if python_executable set --- mypy/main.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index c872e293783f..7b59a9fb2ba0 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -59,7 +59,8 @@ def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: 'import sys; print(repr(sys.version_info[:2]))'], stderr=subprocess.STDOUT).decode() except (subprocess.CalledProcessError, FileNotFoundError): - return sys.version_info[:2] + print('Error: invalid Python executable {}'.format(python_executable), file=sys.stderr) + sys.exit(2) else: return ast.literal_eval(check) @@ -95,14 +96,20 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] sources, options = process_options(args) - # try setting a valid Python executable based on a specified version - if options.python_version and not options.python_executable: - options.python_executable = _python_executable_from_version(options.python_version) - - # Set Python version if given Python executable, but no version - if options.python_executable and not options.python_version: + # Set Python version if given Python executable, but the version is default + if options.python_executable and options.python_version == defaults.PYTHON3_VERSION: options.python_version = _python_version_from_executable(options.python_executable) + # try setting a valid Python executable based on a specified version + if options.python_version: + if options.python_executable: + py_exe_ver = _python_version_from_executable(options.python_executable) + if py_exe_ver != options.python_version: + print('Error: Python version and executable are mismatched.') + sys.exit(2) + else: + options.python_executable = _python_executable_from_version(options.python_version) + messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: From 03cd3788126f9c8acc113ebd6bc561ab062eef4b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Feb 2018 12:48:18 -0800 Subject: [PATCH 072/100] Move returns into try blocks --- mypy/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 7b59a9fb2ba0..9b8a26e02918 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -58,11 +58,10 @@ def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: check = subprocess.check_output([python_executable, '-c', 'import sys; print(repr(sys.version_info[:2]))'], stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) except (subprocess.CalledProcessError, FileNotFoundError): print('Error: invalid Python executable {}'.format(python_executable), file=sys.stderr) sys.exit(2) - else: - return ast.literal_eval(check) def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: @@ -71,10 +70,9 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + ['-c', 'import sys; print(sys.executable)'], stderr=subprocess.STDOUT).decode().strip() + return sys_exe except (subprocess.CalledProcessError, FileNotFoundError): return None - else: - return sys_exe def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: From 3a8ca60535652a9313b0616dab7d7d482f250fe6 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 02:23:41 -0800 Subject: [PATCH 073/100] Refactor python-version/executable inference --- mypy/main.py | 96 ++++++++++++++++++++++--------------------- mypy/options.py | 4 +- mypy/test/testargs.py | 2 +- 3 files changed, 53 insertions(+), 49 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 9b8a26e02918..8e5684c2e5e0 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -45,36 +45,6 @@ def stat_proxy(path: str) -> os.stat_result: return st -if sys.platform == 'win32': - def python_executable_prefix(v: str) -> List[str]: - return ['py', '-{}'.format(v)] -else: - def python_executable_prefix(v: str) -> List[str]: - return ['python{}'.format(v)] - - -def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: - try: - check = subprocess.check_output([python_executable, '-c', - 'import sys; print(repr(sys.version_info[:2]))'], - stderr=subprocess.STDOUT).decode() - return ast.literal_eval(check) - except (subprocess.CalledProcessError, FileNotFoundError): - print('Error: invalid Python executable {}'.format(python_executable), file=sys.stderr) - sys.exit(2) - - -def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: - str_ver = '.'.join(map(str, python_version)) - try: - sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + - ['-c', 'import sys; print(sys.executable)'], - stderr=subprocess.STDOUT).decode().strip() - return sys_exe - except (subprocess.CalledProcessError, FileNotFoundError): - return None - - def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: """Main entry point to the type checker. @@ -94,20 +64,6 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: args = sys.argv[1:] sources, options = process_options(args) - # Set Python version if given Python executable, but the version is default - if options.python_executable and options.python_version == defaults.PYTHON3_VERSION: - options.python_version = _python_version_from_executable(options.python_executable) - - # try setting a valid Python executable based on a specified version - if options.python_version: - if options.python_executable: - py_exe_ver = _python_version_from_executable(options.python_executable) - if py_exe_ver != options.python_version: - print('Error: Python version and executable are mismatched.') - sys.exit(2) - else: - options.python_executable = _python_executable_from_version(options.python_version) - messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: @@ -251,6 +207,36 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +if sys.platform == 'win32': + def python_executable_prefix(v: str) -> List[str]: + return ['py', '-{}'.format(v)] +else: + def python_executable_prefix(v: str) -> List[str]: + return ['python{}'.format(v)] + + +def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: + try: + check = subprocess.check_output([python_executable, '-c', + 'import sys; print(repr(sys.version_info[:2]))'], + stderr=subprocess.STDOUT).decode() + return ast.literal_eval(check) + except (subprocess.CalledProcessError, FileNotFoundError): + print('Error: invalid Python executable {}'.format(python_executable), file=sys.stderr) + sys.exit(2) + + +def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: + str_ver = '.'.join(map(str, python_version)) + try: + sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + + ['-c', 'import sys; print(sys.executable)'], + stderr=subprocess.STDOUT).decode().strip() + return sys_exe + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + def process_options(args: List[str], require_targets: bool = True, server_options: bool = False, @@ -301,10 +287,10 @@ def add_invertible_flag(flag: str, parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__) parser.add_argument('--python-version', type=parse_version, metavar='x.y', - help='use Python x.y') + help='use Python x.y', dest='special-opts:python_version') parser.add_argument('--python-executable', action='store', help="Python executable whose installed packages will be" - " used in typechecking.") + " used in typechecking.", dest='special-opts:python_executable') parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -528,6 +514,24 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + print('Error: Python version and executable are mismatched.') + sys.exit(2) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + elif special_opts.python_executable is None and special_opts.python_version is not None: + py_exe = _python_executable_from_version(special_opts.python_version) + if py_exe is not None: + options.python_executable = py_exe + options.python_version = special_opts.python_version + elif special_opts.python_version is None and special_opts.python_executable is not None: + options.python_version = _python_version_from_executable(special_opts.python_executable) + options.python_executable = special_opts.python_executable + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules, diff --git a/mypy/options.py b/mypy/options.py index 90ed337c2f42..c30aece467d3 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -52,8 +52,8 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD - self.python_version = defaults.PYTHON3_VERSION - self.python_executable = None # type: Optional[str] + self.python_version = sys.version_info[:2] # type: Tuple[int, int] + self.python_executable = sys.executable # type: str self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 20db610cda18..43ef7aaebb03 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -16,4 +16,4 @@ def test_coherence(self) -> None: _, parsed_options = process_options([], require_targets=False) # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file - assert_equal(options, parsed_options) + assert_equal(dir(options), dir(parsed_options)) From e1f949493c564fdd2b9bce876b918afef7a0c35b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 02:50:37 -0800 Subject: [PATCH 074/100] Clarify searching for packages and mixed packages --- docs/source/installed_packages.rst | 57 ++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index fb05d8041038..4dd1745c905e 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -10,7 +10,7 @@ PEP 561 compatible packages and have mypy use them in type checking. Making PEP 561 compatible packages ********************************** -Packages that must be imported at runtime that supply type information should +Packages that must be imported at runtime and supply type information should put a ``py.typed`` in their package directory. For example, with a directory structure as follows: @@ -36,16 +36,17 @@ the setup.py might look like: packages=["package_a"] ) -If the package is entirely made up of stub (``*.pyi``) files, the package -should have a suffix of ``-stubs``. For example, if we had stubs for -``package_b``, we might do the following: +Some packages have a mix of stub files and runtime files. These packages also require +a ``py.typed`` file. An example can be seen below: .. code-block:: text setup.py - package_b-stubs/ - __init__.pyi + package_b/ + __init__.py + lib.py lib.pyi + py.typed the setup.py might look like: @@ -57,8 +58,37 @@ the setup.py might look like: name="SuperPackageB", author="Me", version="0.1", - package_data={"package_b-stubs": ["__init__.pyi", "lib.pyi"]}, - packages=["package_b-stubs"] + package_data={"package_b": ["py.typed", "lib.pyi"]}, + packages=["package_b"] + ) + +In this example, both ``lib.py`` and ``lib.pyi`` exist. At runtime, ``lib.py`` +will be used, however mypy will use ``lib.pyi``. + +If the package is stub-only (not imported at runtime), the package should have +a prefix of the runtime package name and a suffix of ``-stubs``. +A ``py.typed`` file is not needed for stub-only packages. For example, if we +had stubs for ``package_c``, we might do the following: + +.. code-block:: text + + setup.py + package_c-stubs/ + __init__.pyi + lib.pyi + +the setup.py might look like: + +.. code-block:: python + + from distutils.core import setup + + setup( + name="SuperPackageC", + author="Me", + version="0.1", + package_data={"package_c-stubs": ["__init__.pyi", "lib.pyi"]}, + packages=["package_c-stubs"] ) Using PEP 561 compatible packages with mypy @@ -68,9 +98,14 @@ Generally, you do not need to do anything to use installed packages for the Python executable used to run mypy. They should be automatically picked up by mypy and used for type checking. -If you use mypy to type check a Python other than the version running mypy, you -can use the ``--python-executable`` flag to point to the executable, and mypy -will find packages installed for that python executable. +By default, mypy searches for packages installed for the Python executable +running mypy. It is highly unlikely you want this situation if you have +installed typed packages in another Python's package directory. + +Generally, you can use the ``--python-version`` flag and mypy will try to find +the correct package directory. If that fails, you can use the +``--python-executable`` flag to point to the exact executable, and mypy will +find packages installed for that Python executable. Note that mypy does not support some more advanced import features, such as zip imports, namespace packages, and custom import hooks. \ No newline at end of file From b5cbd095e4e751e4be0bbbd20efcb53e88c1b51b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 02:54:52 -0800 Subject: [PATCH 075/100] Clarify default behaviour of python-executable flag --- docs/source/command_line.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 3d27211cae35..cfeac05baa2b 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -371,10 +371,10 @@ Here are some more useful flags: written by quick mode. - ``--python-executable EXECUTABLE`` will have mypy collect type information - from PEP 561 compliant packages installed with the given Python executable. - By default, mypy will use PEP 561 compliant packages installed for the Python - executable running mypy. See :ref:`installed_packages` for more on making - PEP 561 compliant packages. + from PEP 561 compliant packages installed for the Python executable + ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages + installed for the Python executable running mypy. See + :ref:`installed_packages` for more on making PEP 561 compliant packages. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using From fc58e4c98edcf566ee4f54ace00df3117636da72 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 02:59:02 -0800 Subject: [PATCH 076/100] Comment and refactor getsitepackage usage --- mypy/build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 28cea2935615..aefc55521ff8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -883,12 +883,12 @@ def get_package_dirs(python_executable: str) -> List[str]: """ if python_executable == sys.executable: # Use running Python's package dirs - try: + if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): user_dir = site.getusersitepackages() return site.getsitepackages() + [user_dir] - except AttributeError: - # fall back on get_python_lib for virtualenvs - return [get_python_lib()] + # If site doesn't have get(user)sitepackages, we are running in a + # virtualenv, and should fall back to get_python_lib + return [get_python_lib()] else: # Use subprocess to get the package directory of given Python # executable From 96e2f556df7d5fce8be247d93066039beece7a96 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 17:15:33 -0800 Subject: [PATCH 077/100] Make python_executable mandatory for find_packages --- mypy/build.py | 20 +++++++++----------- mypy/test/testcheck.py | 3 ++- mypy/test/testdmypy.py | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index aefc55521ff8..0095a6a06a77 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -903,11 +903,9 @@ def get_package_dirs(python_executable: str) -> List[str]: def find_module(id: str, lib_path_arg: Iterable[str], - python_executable: Optional[str] = None) -> Optional[str]: + python_executable: str) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if python_executable is None: - python_executable = sys.executable package_dirs = get_package_dirs(python_executable) if not package_dirs: print("Could not find package directories for Python '{}'".format( @@ -939,8 +937,8 @@ def find() -> Optional[str]: for pkg_dir in package_dirs: stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') - stub_typed_file = os.path.join(pkg_dir, stub_name, 'py.typed') - if os.path.isfile(stub_typed_file): + stub_dir = os.path.join(pkg_dir, stub_name) + if os.path.isdir(stub_dir): components[0] = stub_name rest = components[:-1] path = os.path.join(pkg_dir, *rest) @@ -964,7 +962,7 @@ def find() -> Optional[str]: # Prefer package over module, i.e. baz/__init__.py* over baz.py*. for extension in PYTHON_EXTENSIONS: path = base_path + sepinit + extension - path_stubs = base_path + '_stubs' + sepinit + extension + path_stubs = base_path + '-stubs' + sepinit + extension if is_file(path) and verify_module(id, path): return path elif is_file(path_stubs) and verify_module(id, path_stubs): @@ -979,7 +977,7 @@ def find() -> Optional[str]: if id not in find_module_cache: find_module_cache[id] = find() - # If we searched for items with a base directory of site-packages/ we need to + # If we searched for items with a base directory of site-packages/ we need to # remove it to avoid searching it for non-typed ids. for dir in package_dirs: if dir + os.sep in find_module_dir_cache[dir_chain]: @@ -989,8 +987,8 @@ def find() -> Optional[str]: def find_modules_recursive(module: str, lib_path: List[str], - python: Optional[str]) -> List[BuildSource]: - module_path = find_module(module, lib_path, python) + python_executable: str) -> List[BuildSource]: + module_path = find_module(module, lib_path, python_executable) if not module_path: return [] result = [BuildSource(module_path, module, None)] @@ -1010,14 +1008,14 @@ def find_modules_recursive(module: str, lib_path: List[str], (os.path.isfile(os.path.join(abs_path, '__init__.py')) or os.path.isfile(os.path.join(abs_path, '__init__.pyi'))): hits.add(item) - result += find_modules_recursive(module + '.' + item, lib_path, python) + result += find_modules_recursive(module + '.' + item, lib_path, python_executable) elif item != '__init__.py' and item != '__init__.pyi' and \ item.endswith(('.py', '.pyi')): mod = item.split('.')[0] if mod not in hits: hits.add(mod) result += find_modules_recursive(module + '.' + mod, - lib_path, python) + lib_path, python_executable) return result diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 0b1f1573760e..37be5cd4a8af 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -3,6 +3,7 @@ import os import re import shutil +import sys from typing import Dict, List, Optional, Set, Tuple @@ -312,7 +313,7 @@ def parse_module(self, module_names = m.group(1) out = [] for module_name in module_names.split(' '): - path = build.find_module(module_name, [test_temp_dir]) + path = build.find_module(module_name, [test_temp_dir], sys.executable) assert path is not None, "Can't find ad hoc case file" with open(path) as f: program_text = f.read() diff --git a/mypy/test/testdmypy.py b/mypy/test/testdmypy.py index 26f2adcd7b6e..87356704998f 100644 --- a/mypy/test/testdmypy.py +++ b/mypy/test/testdmypy.py @@ -270,7 +270,7 @@ def parse_module(self, module_names = m.group(1) out = [] # type: List[Tuple[str, str, Optional[str]]] for module_name in module_names.split(' '): - path = build.find_module(module_name, [test_temp_dir]) + path = build.find_module(module_name, [test_temp_dir], sys.executable) if path is None and module_name.startswith(NON_EXISTENT_PREFIX): # This is a special name for a file that we don't want to exist. assert '.' not in module_name # TODO: Packages not supported here From 80f4a35d309f464a54803279111ad38fb485df7c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Feb 2018 17:29:05 -0800 Subject: [PATCH 078/100] Remove py.typed file from -stubs test package --- test-data/packages/typedpkg-stubs/setup.py | 2 +- test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed | 0 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed diff --git a/test-data/packages/typedpkg-stubs/setup.py b/test-data/packages/typedpkg-stubs/setup.py index cb183b408cad..b90e3a011f23 100644 --- a/test-data/packages/typedpkg-stubs/setup.py +++ b/test-data/packages/typedpkg-stubs/setup.py @@ -8,6 +8,6 @@ name='typedpkg-stubs', author="The mypy team", version='0.1', - package_data={'typedpkg-stubs': ['py.typed', 'sample.pyi', '__init__.pyi']}, + package_data={'typedpkg-stubs': ['sample.pyi', '__init__.pyi']}, packages=['typedpkg-stubs'], ) diff --git a/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed b/test-data/packages/typedpkg-stubs/typedpkg-stubs/py.typed deleted file mode 100644 index e69de29bb2d1..000000000000 From dc29e516e5683bf2f4299fa2d35c50972d7f6c71 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Feb 2018 13:30:56 -0800 Subject: [PATCH 079/100] Write testpackages to test folder --- mypy/test/testpackages.py | 70 ++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/mypy/test/testpackages.py b/mypy/test/testpackages.py index 3bd77b69e2ab..bd91467ddc30 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpackages.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import os +import shutil import sys from typing import Generator, List from unittest import TestCase, main @@ -57,50 +58,57 @@ def test_typed_package(self) -> None: This test CANNOT be split up, concurrency means that simultaneously installing/uninstalling will break tests""" - with open('simple.py', 'w') as f: + test_file = 'simple.py' + if not os.path.isdir('test-packages-data'): + os.mkdir('test-packages-data') + old_cwd = os.getcwd() + os.chdir('test-packages-data') + with open(test_file, 'w') as f: f.write(SIMPLE_PROGRAM) - - with self.install_package('typedpkg-stubs'): - self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" - ) - - # The Python 2 tests are intentionally placed after a Python 3 test to check - # the package_dir_cache is behaving correctly. - python2 = try_find_python2_interpreter() - if python2: - with self.install_package('typedpkg-stubs', python2): + try: + with self.install_package('typedpkg-stubs'): self.check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], + [test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) - with self.install_package('typedpkg', python2): - self.check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - ) - with self.install_package('typedpkg', python2): + # The Python 2 tests are intentionally placed after a Python 3 test to check + # the package_dir_cache is behaving correctly. + python2 = try_find_python2_interpreter() + if python2: with self.install_package('typedpkg-stubs', python2): self.check_mypy_run( - ['--python-executable={}'.format(python2), 'simple.py'], + ['--python-executable={}'.format(python2), test_file], "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" ) + with self.install_package('typedpkg', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), 'simple.py'], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" + ) - with self.install_package('typedpkg'): - self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" - ) + with self.install_package('typedpkg', python2): + with self.install_package('typedpkg-stubs', python2): + self.check_mypy_run( + ['--python-executable={}'.format(python2), test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) - with self.install_package('typedpkg'): - with self.install_package('typedpkg-stubs'): + with self.install_package('typedpkg'): self.check_mypy_run( - ['simple.py'], - "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + [test_file], + "simple.py:4: error: Revealed type is 'builtins.tuple[builtins.str]'\n" ) - os.remove('simple.py') + + with self.install_package('typedpkg'): + with self.install_package('typedpkg-stubs'): + self.check_mypy_run( + [test_file], + "simple.py:4: error: Revealed type is 'builtins.list[builtins.str]'\n" + ) + finally: + os.chdir(old_cwd) + shutil.rmtree('test-packages-data') if __name__ == '__main__': From be733c584f96b5affac4d78965e3c93a93fff8a9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Feb 2018 13:37:03 -0800 Subject: [PATCH 080/100] Refactor tests to not be run with package check --- mypy/test/{testpackages.py => testpep561.py} | 6 +++--- runtests.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) rename mypy/test/{testpackages.py => testpep561.py} (97%) diff --git a/mypy/test/testpackages.py b/mypy/test/testpep561.py similarity index 97% rename from mypy/test/testpackages.py rename to mypy/test/testpep561.py index bd91467ddc30..8de2094fca23 100644 --- a/mypy/test/testpackages.py +++ b/mypy/test/testpep561.py @@ -18,7 +18,7 @@ """ -class TestPackages(TestCase): +class TestPEP561(TestCase): @contextmanager def install_package(self, pkg: str, python_executable: str = sys.executable) -> Generator[None, None, None]: @@ -37,7 +37,7 @@ def install_package(self, pkg: str, finally: run_command([python_executable, '-m', 'pip', 'uninstall', '-y', pkg], cwd=package_path) - def test_get_package_dirs(self) -> None: + def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" dirs = get_package_dirs(sys.executable) assert dirs @@ -53,7 +53,7 @@ def check_mypy_run(cmd_line: List[str], assert err == expected_err, out assert returncode == expected_returncode, returncode - def test_typed_package(self) -> None: + def test_typed_pkg(self) -> None: """Tests type checking based on installed packages. This test CANNOT be split up, concurrency means that simultaneously diff --git a/runtests.py b/runtests.py index 204d52b2cb73..b8f746989c25 100755 --- a/runtests.py +++ b/runtests.py @@ -212,7 +212,6 @@ def test_path(*names: str): 'testmerge', 'testtransform', 'testtypegen', - 'testpackages', 'testparse', 'testsemanal', 'testerrorstream', @@ -232,6 +231,8 @@ def test_path(*names: str): 'testpythoneval', 'testcmdline', 'teststubgen', + # non-data-driven: + 'testpep561', ) for f in find_files('mypy', prefix='test', suffix='.py'): From c1264a01d88228b37d5c03cf965b98efd4b977a9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Feb 2018 16:26:58 -0800 Subject: [PATCH 081/100] Disable pep561 searching with --no-site-packages --- mypy/build.py | 15 +++-- mypy/main.py | 55 ++++++++++++----- mypy/options.py | 1 + test-data/unit/check-async-await.test | 22 +++---- test-data/unit/check-class-namedtuple.test | 68 +++++++++++----------- test-data/unit/check-classes.test | 2 +- test-data/unit/check-expressions.test | 4 +- test-data/unit/check-fastparse.test | 2 +- test-data/unit/check-functions.test | 8 +-- test-data/unit/check-generics.test | 2 +- test-data/unit/check-inference.test | 2 +- test-data/unit/check-newsyntax.test | 36 ++++++------ test-data/unit/check-overloading.test | 2 +- test-data/unit/check-python2.test | 2 +- test-data/unit/check-tuples.test | 2 +- test-data/unit/check-typeddict.test | 24 ++++---- test-data/unit/check-underscores.test | 6 +- test-data/unit/check-unreachable-code.test | 4 +- 18 files changed, 144 insertions(+), 113 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 0095a6a06a77..53add848145c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -903,14 +903,17 @@ def get_package_dirs(python_executable: str) -> List[str]: def find_module(id: str, lib_path_arg: Iterable[str], - python_executable: str) -> Optional[str]: + python_executable: Optional[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - package_dirs = get_package_dirs(python_executable) - if not package_dirs: - print("Could not find package directories for Python '{}'".format( - python_executable), file=sys.stderr) - sys.exit(2) + if python_executable: + package_dirs = get_package_dirs(python_executable) + if not package_dirs: + print("Could not find package directories for Python '{}'".format( + python_executable), file=sys.stderr) + sys.exit(2) + else: + package_dirs = [] components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' diff --git a/mypy/main.py b/mypy/main.py index 8e5684c2e5e0..7d9aa9ad619f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -207,6 +207,10 @@ def invert_flag_name(flag: str) -> str: return '--no-{}'.format(flag[2:]) +class PythonExecutableInferenceError(Exception): + """Represents a failure to infer the version or executable while searching.""" + + if sys.platform == 'win32': def python_executable_prefix(v: str) -> List[str]: return ['py', '-{}'.format(v)] @@ -222,11 +226,13 @@ def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: stderr=subprocess.STDOUT).decode() return ast.literal_eval(check) except (subprocess.CalledProcessError, FileNotFoundError): - print('Error: invalid Python executable {}'.format(python_executable), file=sys.stderr) - sys.exit(2) + raise PythonExecutableInferenceError( + 'Error: invalid Python executable {}'.format(python_executable)) def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: + if sys.version_info[:2] == python_version: + return sys.executable str_ver = '.'.join(map(str, python_version)) try: sys_exe = subprocess.check_output(python_executable_prefix(str_ver) + @@ -234,7 +240,9 @@ def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional stderr=subprocess.STDOUT).decode().strip() return sys_exe except (subprocess.CalledProcessError, FileNotFoundError): - return None + raise PythonExecutableInferenceError( + 'Error: failed to find a Python executable matching version {},' + ' perhaps try --python-executable, or --no-site-packages?'.format(python_version)) def process_options(args: List[str], @@ -291,6 +299,8 @@ def add_invertible_flag(flag: str, parser.add_argument('--python-executable', action='store', help="Python executable whose installed packages will be" " used in typechecking.", dest='special-opts:python_executable') + parser.add_argument('--no-site-packages', action='store_true', + help="Do not search for PEP 561 packages in the package directory.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " "(defaults to sys.platform).") @@ -516,21 +526,38 @@ def add_invertible_flag(flag: str, # Infer Python version and/or executable if one is not given if special_opts.python_executable is not None and special_opts.python_version is not None: - py_exe_ver = _python_version_from_executable(special_opts.python_executable) - if py_exe_ver != special_opts.python_version: - print('Error: Python version and executable are mismatched.') + try: + py_exe_ver = _python_version_from_executable(special_opts.python_executable) + if py_exe_ver != special_opts.python_version: + parser.error( + 'Error: Python version {} did not match executable {}, got version {}.'.format( + special_opts.python_version, special_opts.python_executable, py_exe_ver + )) + sys.exit(2) + else: + options.python_version = special_opts.python_version + options.python_executable = special_opts.python_executable + except PythonExecutableInferenceError as e: + parser.error(e) sys.exit(2) - else: - options.python_version = special_opts.python_version - options.python_executable = special_opts.python_executable elif special_opts.python_executable is None and special_opts.python_version is not None: - py_exe = _python_executable_from_version(special_opts.python_version) - if py_exe is not None: + try: + py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe - options.python_version = special_opts.python_version + except PythonExecutableInferenceError as e: + if not options.no_site_packages: + # raise error if we cannot find site-packages and PEP 561 searching is not disabled + parser.error(e) + sys.exit(2) + finally: + options.python_version = special_opts.python_version elif special_opts.python_version is None and special_opts.python_executable is not None: - options.python_version = _python_version_from_executable(special_opts.python_executable) - options.python_executable = special_opts.python_executable + try: + options.python_version = _python_version_from_executable(special_opts.python_executable) + options.python_executable = special_opts.python_executable + except PythonExecutableInferenceError as e: + parser.error(e) + sys.exit(2) # Check for invalid argument combinations. if require_targets: diff --git a/mypy/options.py b/mypy/options.py index c30aece467d3..6f75abc9fe4a 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -54,6 +54,7 @@ def __init__(self) -> None: self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] self.python_executable = sys.executable # type: str + self.no_site_packages = False self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/test-data/unit/check-async-await.test b/test-data/unit/check-async-await.test index e243bd50f415..227ba6279e23 100644 --- a/test-data/unit/check-async-await.test +++ b/test-data/unit/check-async-await.test @@ -184,7 +184,7 @@ async def f() -> None: [typing fixtures/typing-full.pyi] [case testAsyncForComprehension] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import Generic, Iterable, TypeVar, AsyncIterator, Tuple T = TypeVar('T') @@ -224,7 +224,7 @@ async def generatorexp(obj: Iterable[int]): [typing fixtures/typing-full.pyi] [case testAsyncForComprehensionErrors] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import Generic, Iterable, TypeVar, AsyncIterator, Tuple T = TypeVar('T') @@ -346,7 +346,7 @@ async def f() -> None: [typing fixtures/typing-full.pyi] [case testNoYieldInAsyncDef] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages async def f(): yield None # E: 'yield' in async function @@ -432,7 +432,7 @@ def f() -> Generator[int, str, int]: -- --------------------------------------------------------------------- [case testAsyncGenerator] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import AsyncGenerator, Generator async def f() -> int: @@ -460,7 +460,7 @@ async def wrong_return() -> Generator[int, None, None]: # E: The return type of [typing fixtures/typing-full.pyi] [case testAsyncGeneratorReturnIterator] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import AsyncIterator async def gen() -> AsyncIterator[int]: @@ -476,7 +476,7 @@ async def use_gen() -> None: [typing fixtures/typing-full.pyi] [case testAsyncGeneratorManualIter] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def genfunc() -> AsyncGenerator[int, None]: @@ -494,7 +494,7 @@ async def user() -> None: [typing fixtures/typing-full.pyi] [case testAsyncGeneratorAsend] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def f() -> None: @@ -515,7 +515,7 @@ async def h() -> None: [typing fixtures/typing-full.pyi] [case testAsyncGeneratorAthrow] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def gen() -> AsyncGenerator[str, int]: @@ -534,7 +534,7 @@ async def h() -> None: [typing fixtures/typing-full.pyi] [case testAsyncGeneratorNoSyncIteration] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def gen() -> AsyncGenerator[int, None]: @@ -553,7 +553,7 @@ main:9: error: Iterable expected main:9: error: "AsyncGenerator[int, None]" has no attribute "__iter__"; maybe "__aiter__"? [case testAsyncGeneratorNoYieldFrom] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def f() -> AsyncGenerator[int, None]: @@ -566,7 +566,7 @@ async def gen() -> AsyncGenerator[int, None]: [typing fixtures/typing-full.pyi] [case testAsyncGeneratorNoReturnWithValue] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import AsyncGenerator async def return_int() -> AsyncGenerator[int, None]: diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index 2e6d5cff55fa..2f838fb4458b 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -1,12 +1,12 @@ [case testNewNamedTupleOldPythonVersion] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages from typing import NamedTuple class E(NamedTuple): # E: NamedTuple class syntax is only supported in Python 3.6 pass [case testNewNamedTupleNoUnderscoreFields] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -15,7 +15,7 @@ class X(NamedTuple): _z: int # E: NamedTuple field name cannot start with an underscore: _z [case testNewNamedTupleAccessingAttributes] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -28,7 +28,7 @@ x.y x.z # E: "X" has no attribute "z" [case testNewNamedTupleAttributesAreReadOnly] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -43,7 +43,7 @@ a: A a.x = 5 # E: Property "x" defined in "A" is read-only [case testNewNamedTupleCreateWithPositionalArguments] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -57,7 +57,7 @@ x = X(1) # E: Too few arguments for "X" x = X(1, '2', 3) # E: Too many arguments for "X" [case testNewNamedTupleShouldBeSingleBase] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A: ... @@ -65,7 +65,7 @@ class X(NamedTuple, A): # E: NamedTuple should be a single base pass [case testCreateNewNamedTupleWithKeywordArguments] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -78,7 +78,7 @@ x = X(x=1, z=1) # E: Unexpected keyword argument "z" for "X" x = X(y='x') # E: Missing positional argument "x" in call to "X" [case testNewNamedTupleCreateAndUseAsTuple] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -90,7 +90,7 @@ a, b = x a, b, c = x # E: Need more than 2 values to unpack (3 expected) [case testNewNamedTupleWithItemTypes] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class N(NamedTuple): @@ -106,7 +106,7 @@ x, y = n x = y # E: Incompatible types in assignment (expression has type "str", variable has type "int") [case testNewNamedTupleConstructorArgumentTypes] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class N(NamedTuple): @@ -119,7 +119,7 @@ N(1, 'x') N(b='x', a=1) [case testNewNamedTupleAsBaseClass] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class N(NamedTuple): @@ -136,7 +136,7 @@ i, s = x s, s = x # E: Incompatible types in assignment (expression has type "int", variable has type "str") [case testNewNamedTupleSelfTypeWithNamedTupleAsBase] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A(NamedTuple): @@ -155,7 +155,7 @@ class B(A): [out] [case testNewNamedTupleTypeReferenceToClassDerivedFrom] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A(NamedTuple): @@ -177,7 +177,7 @@ class B(A): [out] [case testNewNamedTupleSubtyping] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple, Tuple class A(NamedTuple): @@ -197,7 +197,7 @@ t = b a = b [case testNewNamedTupleSimpleTypeInference] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple, Tuple class A(NamedTuple): @@ -214,7 +214,7 @@ a = (1,) # E: Incompatible types in assignment (expression has type "Tuple[int] [builtins fixtures/list.pyi] [case testNewNamedTupleMissingClassAttribute] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class MyNamedTuple(NamedTuple): @@ -224,14 +224,14 @@ class MyNamedTuple(NamedTuple): MyNamedTuple.x # E: "Type[MyNamedTuple]" has no attribute "x" [case testNewNamedTupleEmptyItems] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A(NamedTuple): ... [case testNewNamedTupleForwardRef] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A(NamedTuple): @@ -243,7 +243,7 @@ a = A(B()) a = A(1) # E: Argument 1 to "A" has incompatible type "int"; expected "B" [case testNewNamedTupleProperty] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class A(NamedTuple): @@ -260,7 +260,7 @@ C(2).b [builtins fixtures/property.pyi] [case testNewNamedTupleAsDict] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple, Any class X(NamedTuple): @@ -273,7 +273,7 @@ reveal_type(x._asdict()) # E: Revealed type is 'builtins.dict[builtins.str, Any [builtins fixtures/dict.pyi] [case testNewNamedTupleReplaceTyped] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -286,7 +286,7 @@ x._replace(x=5) x._replace(y=5) # E: Argument "y" to "_replace" of "X" has incompatible type "int"; expected "str" [case testNewNamedTupleFields] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -301,7 +301,7 @@ reveal_type(X.__annotations__) # E: Revealed type is 'builtins.dict[builtins.st [builtins fixtures/dict.pyi] [case testNewNamedTupleUnit] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -312,7 +312,7 @@ x._replace() x._fields[0] # E: Tuple index out of range [case testNewNamedTupleJoinNamedTuple] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -327,7 +327,7 @@ reveal_type([X(3, 'b'), Y(1, 'a')]) # E: Revealed type is 'builtins.list[Tuple[ [builtins fixtures/list.pyi] [case testNewNamedTupleJoinTuple] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -340,7 +340,7 @@ reveal_type([X(1, 'a'), (3, 'b')]) # E: Revealed type is 'builtins.list[Tuple[b [builtins fixtures/list.pyi] [case testNewNamedTupleWithTooManyArguments] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -349,7 +349,7 @@ class X(NamedTuple): def f(self): pass [case testNewNamedTupleWithInvalidItems2] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages import typing class X(typing.NamedTuple): @@ -369,7 +369,7 @@ main:9: error: Non-default NamedTuple fields cannot follow default fields [builtins fixtures/list.pyi] [case testNewNamedTupleWithoutTypesSpecified] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -377,7 +377,7 @@ class X(NamedTuple): y = 2 # E: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" [case testTypeUsingTypeCNamedTuple] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import NamedTuple, Type class N(NamedTuple): @@ -391,7 +391,7 @@ def f(a: Type[N]): main:8: error: Unsupported type Type["N"] [case testNewNamedTupleWithDefaults] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import List, NamedTuple, Optional class X(NamedTuple): @@ -431,7 +431,7 @@ UserDefined(1) # E: Argument 1 to "UserDefined" has incompatible type "int"; ex [builtins fixtures/list.pyi] [case testNewNamedTupleWithDefaultsStrictOptional] -# flags: --fast-parser --strict-optional --python-version 3.6 +# flags: --fast-parser --strict-optional --python-version 3.6 --no-site-packages from typing import List, NamedTuple, Optional class HasNone(NamedTuple): @@ -450,7 +450,7 @@ class CannotBeNone(NamedTuple): [builtins fixtures/list.pyi] [case testNewNamedTupleWrongType] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): @@ -458,14 +458,14 @@ class X(NamedTuple): y: int = 'not an int' # E: Incompatible types in assignment (expression has type "str", variable has type "int") [case testNewNamedTupleErrorInDefault] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): x: int = 1 + '1' # E: Unsupported operand types for + ("int" and "str") [case testNewNamedTupleInheritance] -# flags: --fast-parser --python-version 3.6 +# flags: --fast-parser --python-version 3.6 --no-site-packages from typing import NamedTuple class X(NamedTuple): diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 9137345d141e..772b2a76f1ca 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -4135,7 +4135,7 @@ class CQA(Q1): pass class CQW(six.with_metaclass(M, Q1)): pass # E: Inconsistent metaclass structure for 'CQW' [case testSixMetaclassErrors_python2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages import six class M(type): pass class C4(six.with_metaclass(M)): # E: Multiple metaclass definitions diff --git a/test-data/unit/check-expressions.test b/test-data/unit/check-expressions.test index 2d5d88e7b152..124e61fe7c5e 100644 --- a/test-data/unit/check-expressions.test +++ b/test-data/unit/check-expressions.test @@ -1066,13 +1066,13 @@ a = None # type: Any '%b' % 1 # E: Format character 'b' is only supported on bytes patterns [case testStringInterPolationPython2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages b'%b' % 1 # E: Format character 'b' is only supported in Python 3.5 and later b'%s' % 1 b'%a' % 1 # E: Format character 'a' is only supported in Python 3 [case testBytesInterpolationBefore35] -# flags: --python-version 3.4 +# flags: --python-version 3.4 --no-site-packages b'%b' % 1 # E: Unsupported left operand type for % ("bytes") [case testBytesInterpolation] diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index e6aaa04f1f35..38f286602cde 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -67,7 +67,7 @@ def f7(x): # E: invalid type comment or annotation [case testFastParseInvalidTypes3] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages # All of these should not crash from typing import Callable, Tuple, Iterable diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 183561ca2836..30c2a5861f28 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -406,7 +406,7 @@ class B: pass class A: pass [case testDefaultArgumentExpressionsPython2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages from typing import Tuple def f(x = B()): # E: Incompatible default for argument "x" (default has type "B", argument has type "A") # type: (A) -> None @@ -417,7 +417,7 @@ class B: pass class A: pass [case testDefaultTupleArgumentExpressionsPython2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages from typing import Tuple def f((x, y) = (A(), B())): # E: Incompatible default for tuple argument 1 (default has type "Tuple[A, B]", argument has type "Tuple[B, B]") # type: (Tuple[B, B]) -> None @@ -2116,12 +2116,12 @@ a.__eq__(other=a) # E: Unexpected keyword argument "other" for "__eq__" of "A" [builtins fixtures/bool.pyi] [case testTupleArguments] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages def f(a, (b, c), d): pass [case testTupleArgumentsFastparse] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages def f(a, (b, c), d): pass diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 9c99d7b038e9..ea551efdcf71 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -586,7 +586,7 @@ reveal_type(wrap(z)) # E: Revealed type is '__main__.Node[builtins.int, builtins main:13: error: Argument 2 to "Node" has incompatible type "int"; expected "str" [case testGenericTypeAliasesWrongAliases] -# flags: --show-column-numbers --python-version 3.6 +# flags: --show-column-numbers --python-version 3.6 --no-site-packages from typing import TypeVar, Generic, List, Callable, Tuple, Union T = TypeVar('T') S = TypeVar('S') diff --git a/test-data/unit/check-inference.test b/test-data/unit/check-inference.test index 3bcdf2d6d276..2c18f76d731f 100644 --- a/test-data/unit/check-inference.test +++ b/test-data/unit/check-inference.test @@ -719,7 +719,7 @@ def call(c: Callable[[int], Any], i: int) -> None: [out] [case testCallableMeetAndJoin] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import Callable, Any, TypeVar class A: ... diff --git a/test-data/unit/check-newsyntax.test b/test-data/unit/check-newsyntax.test index 2ed26e4f8a81..25e13c20f9c2 100644 --- a/test-data/unit/check-newsyntax.test +++ b/test-data/unit/check-newsyntax.test @@ -1,15 +1,15 @@ [case testNewSyntaxRequire36] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages x: int = 5 # E: Variable annotation syntax is only supported in Python 3.6 and greater [out] [case testNewSyntaxSyntaxError] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages x: int: int # E: invalid syntax [out] [case testNewSyntaxBasics] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages x: int x = 5 y: int = 5 @@ -23,7 +23,7 @@ zzz: str # E: Name 'zzz' already defined [out] [case testNewSyntaxWithDict] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import Dict, Any d: Dict[int, str] = {} @@ -34,7 +34,7 @@ d['ab'] = 'ab' # E: Invalid index type "str" for "Dict[int, str]"; expected typ [out] [case testNewSyntaxWithRevealType] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from typing import Dict def tst_local(dct: Dict[int, T]) -> Dict[T, int]: @@ -46,7 +46,7 @@ reveal_type(tst_local({1: 'a'})) # E: Revealed type is 'builtins.dict[builtins. [out] [case testNewSyntaxWithInstanceVars] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages class TstInstance: a: str def __init__(self) -> None: @@ -59,27 +59,27 @@ TstInstance().a = 'ab' [out] [case testNewSyntaxWithClassVars] -# flags: --strict-optional --python-version 3.6 +# flags: --strict-optional --python-version 3.6 --no-site-packages class CCC: a: str = None # E: Incompatible types in assignment (expression has type "None", variable has type "str") [out] [case testNewSyntaxWithStrictOptional] -# flags: --strict-optional --python-version 3.6 +# flags: --strict-optional --python-version 3.6 --no-site-packages strict: int strict = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") strict2: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") [out] [case testNewSyntaxWithStrictOptionalFunctions] -# flags: --strict-optional --python-version 3.6 +# flags: --strict-optional --python-version 3.6 --no-site-packages def f() -> None: x: int x = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") [out] [case testNewSyntaxWithStrictOptionalClasses] -# flags: --strict-optional --python-version 3.6 +# flags: --strict-optional --python-version 3.6 --no-site-packages class C: def meth(self) -> None: x: int = None # E: Incompatible types in assignment (expression has type "None", variable has type "int") @@ -87,7 +87,7 @@ class C: [out] [case testNewSyntaxSpecialAssign] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages class X: x: str x[0]: int @@ -100,17 +100,17 @@ main:5: error: Type cannot be declared in assignment to non-self attribute main:5: error: "str" has no attribute "x" [case testNewSyntaxAsyncComprehensionError] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages async def f(): results = [i async for i in aiter() if i % 2] # E: Async comprehensions are only supported in Python 3.6 and greater [case testNewSyntaxFstringError] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages f'' # E: Format strings are only supported in Python 3.6 and greater [case testNewSyntaxFStringBasics] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages f'foobar' f'{"foobar"}' f'foo{"bar"}' @@ -122,13 +122,13 @@ a = f'{"foobar"}' [builtins fixtures/f_string.pyi] [case testNewSyntaxFStringExpressionsOk] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages f'.{1 + 1}.' f'.{1 + 1}.{"foo" + "bar"}' [builtins fixtures/f_string.pyi] [case testNewSyntaxFStringExpressionsErrors] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages f'{1 + ""}' f'.{1 + ""}' [builtins fixtures/f_string.pyi] @@ -137,7 +137,7 @@ main:2: error: Unsupported operand types for + ("int" and "str") main:3: error: Unsupported operand types for + ("int" and "str") [case testNewSyntaxFStringParseFormatOptions] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages value = 10.5142 width = 10 precision = 4 @@ -145,7 +145,7 @@ f'result: {value:{width}.{precision}}' [builtins fixtures/f_string.pyi] [case testNewSyntaxFStringSingleField] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages v = 1 reveal_type(f'{v}') # E: Revealed type is 'builtins.str' reveal_type(f'{1}') # E: Revealed type is 'builtins.str' diff --git a/test-data/unit/check-overloading.test b/test-data/unit/check-overloading.test index 3b0abadbf408..614f924580bd 100644 --- a/test-data/unit/check-overloading.test +++ b/test-data/unit/check-overloading.test @@ -153,7 +153,7 @@ class B: pass [builtins fixtures/isinstance.pyi] [case testTypeCheckOverloadWithImplementationPy2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages from typing import overload @overload diff --git a/test-data/unit/check-python2.test b/test-data/unit/check-python2.test index 4011ef57f4e7..04474dbf6149 100644 --- a/test-data/unit/check-python2.test +++ b/test-data/unit/check-python2.test @@ -246,7 +246,7 @@ class Foo: pass exec('print 1 + 1') [case testUnicodeDocStrings] -# flags: --python-version=2.7 +# flags: --python-version=2.7 --no-site-packages __doc__ = u"unicode" class A: diff --git a/test-data/unit/check-tuples.test b/test-data/unit/check-tuples.test index 3796fb1cb663..2b8f10932f09 100644 --- a/test-data/unit/check-tuples.test +++ b/test-data/unit/check-tuples.test @@ -271,7 +271,7 @@ class B: pass [builtins fixtures/tuple.pyi] [case testMultipleAssignmentWithSquareBracketTuplesPython2] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages from typing import Tuple def avoid_confusing_test_parser(): diff --git a/test-data/unit/check-typeddict.test b/test-data/unit/check-typeddict.test index 757ad6be8c60..e56dd7b3b896 100644 --- a/test-data/unit/check-typeddict.test +++ b/test-data/unit/check-typeddict.test @@ -78,7 +78,7 @@ p = Point(x='meaning_of_life', y=1337) # E: Incompatible types (expression has -- Define TypedDict (Class syntax) [case testCanCreateTypedDictWithClass] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point(TypedDict): @@ -90,7 +90,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.Point', {'x': builtin [builtins fixtures/dict.pyi] [case testCanCreateTypedDictWithSubclass] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point1D(TypedDict): @@ -104,7 +104,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.Point2D', {'x': built [builtins fixtures/dict.pyi] [case testCanCreateTypedDictWithSubclass2] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point1D(TypedDict): @@ -117,7 +117,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.Point2D', {'x': built [builtins fixtures/dict.pyi] [case testCanCreateTypedDictClassEmpty] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class EmptyDict(TypedDict): @@ -131,7 +131,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.EmptyDict', {})' -- Define TypedDict (Class syntax errors) [case testCanCreateTypedDictWithClassOldVersion] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages from mypy_extensions import TypedDict class Point(TypedDict): # E: TypedDict class syntax is only supported in Python 3.6 @@ -139,7 +139,7 @@ class Point(TypedDict): # E: TypedDict class syntax is only supported in Python [builtins fixtures/dict.pyi] [case testCannotCreateTypedDictWithClassOtherBases] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class A: pass @@ -154,7 +154,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.Point2D', {'x': built [builtins fixtures/dict.pyi] [case testCannotCreateTypedDictWithClassWithOtherStuff] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point(TypedDict): @@ -173,7 +173,7 @@ Point = TypedDict('Point', {'x': int, 'y': int, '_fallback': object}) [builtins fixtures/dict.pyi] [case testCanCreateTypedDictWithClassUnderscores] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point(TypedDict): @@ -185,7 +185,7 @@ reveal_type(p) # E: Revealed type is 'TypedDict('__main__.Point', {'x': builtins [builtins fixtures/dict.pyi] [case testCannotCreateTypedDictWithClassOverwriting] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Bad(TypedDict): @@ -197,7 +197,7 @@ reveal_type(b) # E: Revealed type is 'TypedDict('__main__.Bad', {'x': builtins.i [builtins fixtures/dict.pyi] [case testCannotCreateTypedDictWithClassOverwriting2] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point1(TypedDict): @@ -212,7 +212,7 @@ reveal_type(b) # E: Revealed type is 'TypedDict('__main__.Bad', {'x': builtins.i [builtins fixtures/dict.pyi] [case testCannotCreateTypedDictWithClassOverwriting2] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point1(TypedDict): @@ -647,7 +647,7 @@ reveal_type(p['y']) # E: Revealed type is 'builtins.int' [builtins fixtures/dict.pyi] [case testCanGetItemOfTypedDictWithValidBytesOrUnicodeLiteralKey] -# flags: --python-version 2.7 +# flags: --python-version 2.7 --no-site-packages from mypy_extensions import TypedDict Cell = TypedDict('Cell', {'value': int}) c = Cell(value=42) diff --git a/test-data/unit/check-underscores.test b/test-data/unit/check-underscores.test index 88f95ef4d2fa..adaaec88336f 100644 --- a/test-data/unit/check-underscores.test +++ b/test-data/unit/check-underscores.test @@ -1,15 +1,15 @@ [case testUnderscoresRequire36] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages x = 1000_000 # E: Underscores in numeric literals are only supported in Python 3.6 and greater [out] [case testUnderscoresSyntaxError] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages x = 1000_000_ # E: invalid token [out] [case testUnderscoresBasics] -# flags: --python-version 3.6 +# flags: --python-version 3.6 --no-site-packages x: int x = 1000_000 x = 0x_FF_FF_FF_FF diff --git a/test-data/unit/check-unreachable-code.test b/test-data/unit/check-unreachable-code.test index b86154302f8c..cbfcb3974383 100644 --- a/test-data/unit/check-unreachable-code.test +++ b/test-data/unit/check-unreachable-code.test @@ -434,7 +434,7 @@ x = 1 [out] [case testCustomSysVersionInfo] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages import sys if sys.version_info == (3, 5): x = "foo" @@ -445,7 +445,7 @@ reveal_type(x) # E: Revealed type is 'builtins.str' [out] [case testCustomSysVersionInfo2] -# flags: --python-version 3.5 +# flags: --python-version 3.5 --no-site-packages import sys if sys.version_info == (3, 6): x = "foo" From ceacbfff7d19dad23be706fefb656e6e1014c18e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Feb 2018 16:35:50 -0800 Subject: [PATCH 082/100] Fix return type to be non-optional --- mypy/main.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 7d9aa9ad619f..0da3a39a2109 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -230,7 +230,7 @@ def _python_version_from_executable(python_executable: str) -> Tuple[int, int]: 'Error: invalid Python executable {}'.format(python_executable)) -def _python_executable_from_version(python_version: Tuple[int, int]) -> Optional[str]: +def _python_executable_from_version(python_version: Tuple[int, int]) -> str: if sys.version_info[:2] == python_version: return sys.executable str_ver = '.'.join(map(str, python_version)) @@ -538,25 +538,27 @@ def add_invertible_flag(flag: str, options.python_version = special_opts.python_version options.python_executable = special_opts.python_executable except PythonExecutableInferenceError as e: - parser.error(e) + parser.error(str(e)) sys.exit(2) elif special_opts.python_executable is None and special_opts.python_version is not None: - try: - py_exe = _python_executable_from_version(special_opts.python_version) - options.python_executable = py_exe - except PythonExecutableInferenceError as e: - if not options.no_site_packages: - # raise error if we cannot find site-packages and PEP 561 searching is not disabled - parser.error(e) - sys.exit(2) - finally: - options.python_version = special_opts.python_version + options.python_version = special_opts.python_version + if not options.no_site_packages: + try: + py_exe = _python_executable_from_version(special_opts.python_version) + options.python_executable = py_exe + except PythonExecutableInferenceError as e: + if not options.no_site_packages: + # raise error if we cannot find site-packages and PEP 561 + # searching is not disabled + parser.error(str(e)) + sys.exit(2) elif special_opts.python_version is None and special_opts.python_executable is not None: try: - options.python_version = _python_version_from_executable(special_opts.python_executable) + options.python_version = _python_version_from_executable( + special_opts.python_executable) options.python_executable = special_opts.python_executable except PythonExecutableInferenceError as e: - parser.error(e) + parser.error(str(e)) sys.exit(2) # Check for invalid argument combinations. From f389f6493da2e60802f50b49a2099651cbc95c9c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Feb 2018 19:51:29 -0800 Subject: [PATCH 083/100] Set python_executable to None if not inferred --- mypy/build.py | 2 +- mypy/main.py | 10 +++++++--- mypy/options.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 53add848145c..8ac78a110cf9 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -990,7 +990,7 @@ def find() -> Optional[str]: def find_modules_recursive(module: str, lib_path: List[str], - python_executable: str) -> List[BuildSource]: + python_executable: Optional[str]) -> List[BuildSource]: module_path = find_module(module, lib_path, python_executable) if not module_path: return [] diff --git a/mypy/main.py b/mypy/main.py index 0da3a39a2109..cdb8988ba093 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -528,6 +528,11 @@ def add_invertible_flag(flag: str, if special_opts.python_executable is not None and special_opts.python_version is not None: try: py_exe_ver = _python_version_from_executable(special_opts.python_executable) + + except PythonExecutableInferenceError as e: + parser.error(str(e)) + sys.exit(2) + else: if py_exe_ver != special_opts.python_version: parser.error( 'Error: Python version {} did not match executable {}, got version {}.'.format( @@ -537,9 +542,6 @@ def add_invertible_flag(flag: str, else: options.python_version = special_opts.python_version options.python_executable = special_opts.python_executable - except PythonExecutableInferenceError as e: - parser.error(str(e)) - sys.exit(2) elif special_opts.python_executable is None and special_opts.python_version is not None: options.python_version = special_opts.python_version if not options.no_site_packages: @@ -552,6 +554,8 @@ def add_invertible_flag(flag: str, # searching is not disabled parser.error(str(e)) sys.exit(2) + else: + options.python_executable = None elif special_opts.python_version is None and special_opts.python_executable is not None: try: options.python_version = _python_version_from_executable( diff --git a/mypy/options.py b/mypy/options.py index 6f75abc9fe4a..aeea9da70fda 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -53,7 +53,7 @@ def __init__(self) -> None: # -- build options -- self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] - self.python_executable = sys.executable # type: str + self.python_executable = sys.executable # type: Optional[str] self.no_site_packages = False self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] From 1bd9f6604a1ca04ccd8519a4cabfdc76f0572c63 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 24 Feb 2018 02:35:13 -0800 Subject: [PATCH 084/100] Set flags for tests needing 3.6 --- mypy/test/testcmdline.py | 1 + mypy/test/testdiff.py | 2 ++ mypy/test/testmerge.py | 2 ++ mypy/test/testpythoneval.py | 3 ++- mypy/test/testsemanal.py | 2 ++ test-data/unit/cmdline.test | 14 +++++++------- test-data/unit/diff.test | 1 + test-data/unit/reports.test | 2 +- 8 files changed, 18 insertions(+), 9 deletions(-) diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 57910c1a1dc0..88ad272fab0d 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -47,6 +47,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: file.write('{}\n'.format(s)) args = parse_args(testcase.input[0]) args.append('--show-traceback') + args.append('--no-site-packages') # Type check the program. fixed = [python3_path, os.path.join(testcase.old_cwd, 'scripts', 'mypy')] diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py index b1cfc65a4a29..bb0403b8db4c 100644 --- a/mypy/test/testdiff.py +++ b/mypy/test/testdiff.py @@ -53,6 +53,8 @@ def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]: options.use_builtins_fixtures = True options.show_traceback = True options.cache_dir = os.devnull + options.python_version = (3, 6) + options.no_site_packages = True try: result = build.build(sources=[BuildSource('main', None, source)], options=options, diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index cc79958d5c15..cdccead1c7b7 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -102,6 +102,8 @@ def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[st options.fine_grained_incremental = True options.use_builtins_fixtures = True options.show_traceback = True + options.python_version = (3, 6) + options.no_site_packages = True main_path = os.path.join(test_temp_dir, 'main') with open(main_path, 'w') as f: f.write(source) diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index 918395cfd490..a9d8e35b4396 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -47,7 +47,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: version. """ assert testcase.old_cwd is not None, "test was not properly set up" - mypy_cmdline = ['--show-traceback'] + mypy_cmdline = ['--show-traceback', '--no-site-packages'] py2 = testcase.name.lower().endswith('python2') if py2: mypy_cmdline.append('--py2') @@ -59,6 +59,7 @@ def test_python_evaluation(testcase: DataDrivenTestCase) -> None: return else: interpreter = python3_path + mypy_cmdline.append('--python-version=3.6') # Write the program to a file. program = '_' + testcase.name + '.py' diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 98f3ef64b26c..64d094c05fe0 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -38,6 +38,8 @@ def get_semanal_options() -> Options: options.use_builtins_fixtures = True options.semantic_analysis_only = True options.show_traceback = True + options.python_version = (3, 6) + options.no_site_packages = True return options diff --git a/test-data/unit/cmdline.test b/test-data/unit/cmdline.test index 86aae2d344f4..4becdb18e23f 100644 --- a/test-data/unit/cmdline.test +++ b/test-data/unit/cmdline.test @@ -581,7 +581,7 @@ m.py:6: error: Explicit "Any" is not allowed m.py:9: error: Explicit "Any" is not allowed [case testDisallowAnyExplicitVarDeclaration] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] @@ -601,7 +601,7 @@ m.py:3: error: Explicit "Any" is not allowed m.py:5: error: Explicit "Any" is not allowed [case testDisallowAnyExplicitGenericVarDeclaration] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] @@ -785,7 +785,7 @@ N = TypedDict('N', {'x': str, 'y': List}) # no error m.py:4: error: Explicit "Any" is not allowed [case testDisallowAnyGenericsTupleNoTypeParams] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] [[mypy-m] @@ -821,7 +821,7 @@ def g(s: List[Tuple[str, str]]) -> None: pass # no error m.py:3: error: Missing type parameters for generic type [case testDisallowAnyGenericsTypeType] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] [[mypy-m] @@ -858,7 +858,7 @@ def g(l: L[str]) -> None: pass # no error m.py:5: error: Missing type parameters for generic type [case testDisallowAnyGenericsGenericAlias] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] [[mypy-m] @@ -882,7 +882,7 @@ m.py:7: error: Missing type parameters for generic type m.py:11: error: Missing type parameters for generic type [case testDisallowAnyGenericsPlainList] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] [[mypy-m] @@ -906,7 +906,7 @@ m.py:8: error: Need type annotation for 'x' m.py:9: error: Missing type parameters for generic type [case testDisallowAnyGenericsCustomGenericClass] -# cmd: mypy m.py +# cmd: mypy --python-version=3.6 m.py [file mypy.ini] [[mypy] [[mypy-m] diff --git a/test-data/unit/diff.test b/test-data/unit/diff.test index 5e5d81294de9..1dc82fffd91a 100644 --- a/test-data/unit/diff.test +++ b/test-data/unit/diff.test @@ -577,6 +577,7 @@ __main__.Point __main__.p [case testTypedDict2] +# flags: --python-version 3.6 --no-site-packages from mypy_extensions import TypedDict class Point(TypedDict): x: int diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 8343212fc8df..850ed128b883 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -281,7 +281,7 @@ Total 0 16 100.00% [case testAnyExpressionsReportTypesOfAny] -# cmd: mypy --any-exprs-report report n.py +# cmd: mypy --no-site-packages --python-version=3.6 --any-exprs-report report n.py [file n.py] from typing import Any, List From 806fbefa15bc38e140590cb9ed0bb2b607d03e7d Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 24 Feb 2018 02:46:20 -0800 Subject: [PATCH 085/100] Document no-site-packages --- docs/source/command_line.rst | 13 +++++++++++-- docs/source/installed_packages.rst | 5 ++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index cfeac05baa2b..9abfce348722 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -10,7 +10,7 @@ flag (or its long form ``--help``):: $ mypy -h usage: mypy [-h] [-v] [-V] [--python-version x.y] [--python-executable PYTHON_EXECUTABLE] [--platform PLATFORM] [-2] - [--ignore-missing-imports] + [--ignore-missing-imports] [--no-site-packages] [--follow-imports {normal,silent,skip,error}] [--disallow-any-unimported] [--disallow-any-expr] [--disallow-any-decorated] [--disallow-any-explicit] @@ -380,7 +380,16 @@ Here are some more useful flags: run under Python version X.Y. Without this option, mypy will default to using whatever version of Python is running mypy. Note that the ``-2`` and ``--py2`` flags are aliases for ``--python-version 2.7``. See - :ref:`version_and_platform_checks` for more about this feature. + :ref:`version_and_platform_checks` for more about this feature. This flag + will attempt to find a Python executable of the corresponding version to + search for PEP 561 compliant packages. If you'd like to disable this, see + ``--no-site-packages`` below. + +- ``--no-site-packages`` will disable searching for PEP 561 compliant packages. + This will also disable searching for a usable Python executable. Use this + flag if mypy cannot find a Python executable for the version of Python being + checked, and you don't need to use PEP 561 typed packages. Otherwise, use + ``--python-executable``. - ``--platform PLATFORM`` will make mypy typecheck your code as if it were run under the the given operating system. Without this option, mypy will diff --git a/docs/source/installed_packages.rst b/docs/source/installed_packages.rst index 4dd1745c905e..01384f508e18 100644 --- a/docs/source/installed_packages.rst +++ b/docs/source/installed_packages.rst @@ -108,4 +108,7 @@ the correct package directory. If that fails, you can use the find packages installed for that Python executable. Note that mypy does not support some more advanced import features, such as zip -imports, namespace packages, and custom import hooks. \ No newline at end of file +imports, namespace packages, and custom import hooks. + +If you do not want to use typed packages, use the ``--no-site-packages`` flag +to disable searching. \ No newline at end of file From c9e33615fb50e096632e8cd05cc9da81d10b69c5 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 24 Feb 2018 12:56:58 -0800 Subject: [PATCH 086/100] Fix bugs with running mypy on 3.4 --- mypy/waiter.py | 2 +- runtests.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mypy/waiter.py b/mypy/waiter.py index 7e475d3e61ae..76d6fd335b0b 100644 --- a/mypy/waiter.py +++ b/mypy/waiter.py @@ -160,7 +160,7 @@ def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]: test_log = json.load(fp) except FileNotFoundError: test_log = [] - except json.JSONDecodeError: + except ValueError: print('corrupt test log file {}'.format(self.FULL_LOG_FILENAME), file=sys.stderr) test_log = [] return test_log diff --git a/runtests.py b/runtests.py index b8f746989c25..2e64a2b29389 100755 --- a/runtests.py +++ b/runtests.py @@ -9,6 +9,7 @@ import itertools import os from os.path import join, isdir +import re import sys @@ -73,14 +74,16 @@ def add_mypy_cmd(self, name: str, mypy_args: List[str], cwd: Optional[str] = Non return args = [sys.executable, self.mypy] + mypy_args args.append('--show-traceback') + args.append('--no-site-packages') self.waiter.add(LazySubprocess(full_name, args, cwd=cwd, env=self.env)) def add_mypy(self, name: str, *args: str, cwd: Optional[str] = None) -> None: self.add_mypy_cmd(name, list(args), cwd=cwd) def add_mypy_modules(self, name: str, modules: Iterable[str], - cwd: Optional[str] = None) -> None: - args = list(itertools.chain(*(['-m', mod] for mod in modules))) + cwd: Optional[str] = None, extra_args: List[str] = None) -> None: + args = extra_args or [] + args.extend(list(itertools.chain(*(['-m', mod] for mod in modules)))) self.add_mypy_cmd(name, args, cwd=cwd) def add_mypy_package(self, name: str, packagename: str, *flags: str) -> None: @@ -258,7 +261,7 @@ def add_stubs(driver: Driver) -> None: module = file_to_module(f[len(stubdir) + 1:]) modules.add(module) - driver.add_mypy_modules('stubs', sorted(modules)) + driver.add_mypy_modules('stubs', sorted(modules), extra_args=['--python-version=3.5']) def add_stdlibsamples(driver: Driver) -> None: @@ -278,7 +281,11 @@ def add_stdlibsamples(driver: Driver) -> None: def add_samples(driver: Driver) -> None: for f in find_files(os.path.join('test-data', 'samples'), suffix='.py'): - driver.add_mypy('file %s' % f, f) + if f == os.path.join('test-data', 'samples', 'crawl2.py'): + # This test requires 3.5 + driver.add_mypy_cmd('file {}'.format(f), ['--python-version=3.5', f]) + else: + driver.add_mypy('file %s' % f, f) def usage(status: int) -> None: From 5fc9d52355e45e62b4c6b237c6862b8e58c6b846 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sat, 24 Feb 2018 19:59:57 -0800 Subject: [PATCH 087/100] Disable PEP561 searching if --no-site-packages is passed --- mypy/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 2e81023e70e3..1031695a6c88 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -557,8 +557,6 @@ def add_invertible_flag(flag: str, # searching is not disabled parser.error(str(e)) sys.exit(2) - else: - options.python_executable = None elif special_opts.python_version is None and special_opts.python_executable is not None: try: options.python_version = _python_version_from_executable( @@ -568,6 +566,9 @@ def add_invertible_flag(flag: str, parser.error(str(e)) sys.exit(2) + if options.no_site_packages: + options.python_executable = None + # Check for invalid argument combinations. if require_targets: code_methods = sum(bool(c) for c in [special_opts.modules, From ae86abd61fff2abf8e30d981c8cd296c310eb99e Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 21:14:36 -0800 Subject: [PATCH 088/100] Clean up tests --- mypy/test/helpers.py | 2 +- mypy/test/testargs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index 4fd285af5355..a26ad27221b5 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -335,7 +335,7 @@ def parse_options(program_text: str, testcase: DataDrivenTestCase, def split_lines(*streams: bytes) -> List[str]: """Returns a single list of string lines from the byte streams in args.""" return [ - s.rstrip('\n\r') + s for stream in streams for s in stream.decode('utf8').splitlines() ] diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 43ef7aaebb03..20db610cda18 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -16,4 +16,4 @@ def test_coherence(self) -> None: _, parsed_options = process_options([], require_targets=False) # FIX: test this too. Requires changing working dir to avoid finding 'setup.cfg' options.config_file = parsed_options.config_file - assert_equal(dir(options), dir(parsed_options)) + assert_equal(options, parsed_options) From 0bf024fa50fbbfa11734c01741bffd085f5d9d93 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 21:36:12 -0800 Subject: [PATCH 089/100] Move no-site-packages to special-opts --- mypy/build.py | 12 ++++++------ mypy/main.py | 7 ++++--- mypy/options.py | 1 - mypy/test/testdiff.py | 2 +- mypy/test/testmerge.py | 2 +- mypy/test/testsemanal.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 8ac78a110cf9..89331f821083 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -906,14 +906,14 @@ def find_module(id: str, lib_path_arg: Iterable[str], python_executable: Optional[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if python_executable: - package_dirs = get_package_dirs(python_executable) - if not package_dirs: + if python_executable is not None: + site_packages_dirs = get_package_dirs(python_executable) + if not site_packages_dirs: print("Could not find package directories for Python '{}'".format( python_executable), file=sys.stderr) sys.exit(2) else: - package_dirs = [] + site_packages_dirs = [] components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' @@ -937,7 +937,7 @@ def find() -> Optional[str]: dirs.append(dir) # Third-party stub/typed packages - for pkg_dir in package_dirs: + for pkg_dir in site_packages_dirs: stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_dir = os.path.join(pkg_dir, stub_name) @@ -982,7 +982,7 @@ def find() -> Optional[str]: # If we searched for items with a base directory of site-packages/ we need to # remove it to avoid searching it for non-typed ids. - for dir in package_dirs: + for dir in site_packages_dirs: if dir + os.sep in find_module_dir_cache[dir_chain]: find_module_dir_cache[dir_chain].remove(dir + os.sep) diff --git a/mypy/main.py b/mypy/main.py index 1031695a6c88..a0fff49499aa 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -300,6 +300,7 @@ def add_invertible_flag(flag: str, help="Python executable whose installed packages will be" " used in typechecking.", dest='special-opts:python_executable') parser.add_argument('--no-site-packages', action='store_true', + dest='special-opts:no_site_packages', help="Do not search for PEP 561 packages in the package directory.") parser.add_argument('--platform', action='store', metavar='PLATFORM', help="typecheck special-cased code for the given OS platform " @@ -547,12 +548,12 @@ def add_invertible_flag(flag: str, options.python_executable = special_opts.python_executable elif special_opts.python_executable is None and special_opts.python_version is not None: options.python_version = special_opts.python_version - if not options.no_site_packages: + if not special_opts.no_site_packages: try: py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe except PythonExecutableInferenceError as e: - if not options.no_site_packages: + if not special_opts.no_site_packages: # raise error if we cannot find site-packages and PEP 561 # searching is not disabled parser.error(str(e)) @@ -566,7 +567,7 @@ def add_invertible_flag(flag: str, parser.error(str(e)) sys.exit(2) - if options.no_site_packages: + if special_opts.no_site_packages: options.python_executable = None # Check for invalid argument combinations. diff --git a/mypy/options.py b/mypy/options.py index 51d8efbdae88..c3d07df08191 100644 --- a/mypy/options.py +++ b/mypy/options.py @@ -54,7 +54,6 @@ def __init__(self) -> None: self.build_type = BuildType.STANDARD self.python_version = sys.version_info[:2] # type: Tuple[int, int] self.python_executable = sys.executable # type: Optional[str] - self.no_site_packages = False self.platform = sys.platform self.custom_typing_module = None # type: Optional[str] self.custom_typeshed_dir = None # type: Optional[str] diff --git a/mypy/test/testdiff.py b/mypy/test/testdiff.py index bb0403b8db4c..552b49be36ec 100644 --- a/mypy/test/testdiff.py +++ b/mypy/test/testdiff.py @@ -54,7 +54,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]: options.show_traceback = True options.cache_dir = os.devnull options.python_version = (3, 6) - options.no_site_packages = True + options.python_executable = None try: result = build.build(sources=[BuildSource('main', None, source)], options=options, diff --git a/mypy/test/testmerge.py b/mypy/test/testmerge.py index cdccead1c7b7..283947861791 100644 --- a/mypy/test/testmerge.py +++ b/mypy/test/testmerge.py @@ -103,7 +103,7 @@ def build(self, source: str) -> Tuple[List[str], Optional[BuildManager], Dict[st options.use_builtins_fixtures = True options.show_traceback = True options.python_version = (3, 6) - options.no_site_packages = True + options.python_executable = None main_path = os.path.join(test_temp_dir, 'main') with open(main_path, 'w') as f: f.write(source) diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 64d094c05fe0..ef593f19710e 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -39,7 +39,7 @@ def get_semanal_options() -> Options: options.semantic_analysis_only = True options.show_traceback = True options.python_version = (3, 6) - options.no_site_packages = True + options.python_executable = None return options From c9c35c1e7dcdc526dd1e7e123915044438ff237b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 23:19:38 -0800 Subject: [PATCH 090/100] Remove duplicate exit and error message --- mypy/main.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index a0fff49499aa..032407a1dd8d 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -539,10 +539,9 @@ def add_invertible_flag(flag: str, else: if py_exe_ver != special_opts.python_version: parser.error( - 'Error: Python version {} did not match executable {}, got version {}.'.format( + 'Python version {} did not match executable {}, got version {}.'.format( special_opts.python_version, special_opts.python_executable, py_exe_ver )) - sys.exit(2) else: options.python_version = special_opts.python_version options.python_executable = special_opts.python_executable @@ -557,7 +556,6 @@ def add_invertible_flag(flag: str, # raise error if we cannot find site-packages and PEP 561 # searching is not disabled parser.error(str(e)) - sys.exit(2) elif special_opts.python_version is None and special_opts.python_executable is not None: try: options.python_version = _python_version_from_executable( @@ -565,7 +563,6 @@ def add_invertible_flag(flag: str, options.python_executable = special_opts.python_executable except PythonExecutableInferenceError as e: parser.error(str(e)) - sys.exit(2) if special_opts.no_site_packages: options.python_executable = None From dca9c5474e2737625513e3bb411cff9d554e6d86 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 23:34:16 -0800 Subject: [PATCH 091/100] Refactor name, add missing paren --- mypy/build.py | 11 ++++------- mypy/main.py | 7 +++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 89331f821083..617558b50e6e 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -876,11 +876,8 @@ def call_python(python_executable: str, command: str) -> str: @functools.lru_cache(maxsize=None) -def get_package_dirs(python_executable: str) -> List[str]: - """Find package directories for given python - - This defaults to the Python running mypy. - """ +def get_site_packages_dirs(python_executable: str) -> List[str]: + """Find package directories for given python.""" if python_executable == sys.executable: # Use running Python's package dirs if hasattr(site, 'getusersitepackages') and hasattr(site, 'getsitepackages'): @@ -895,7 +892,7 @@ def get_package_dirs(python_executable: str) -> List[str]: try: output = call_python(python_executable, USER_SITE_PACKAGES) except subprocess.CalledProcessError: - # if no paths are found (raising a CalledProcessError, we fall back on sysconfig, + # if no paths are found (raising a CalledProcessError), we fall back on sysconfig, # the python executable is likely in a virtual environment, thus lacking # needed site methods output = call_python(python_executable, VIRTUALENV_SITE_PACKAGES) @@ -907,7 +904,7 @@ def find_module(id: str, lib_path_arg: Iterable[str], """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) if python_executable is not None: - site_packages_dirs = get_package_dirs(python_executable) + site_packages_dirs = get_site_packages_dirs(python_executable) if not site_packages_dirs: print("Could not find package directories for Python '{}'".format( python_executable), file=sys.stderr) diff --git a/mypy/main.py b/mypy/main.py index 032407a1dd8d..1b3acbfb8d40 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -552,10 +552,9 @@ def add_invertible_flag(flag: str, py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe except PythonExecutableInferenceError as e: - if not special_opts.no_site_packages: - # raise error if we cannot find site-packages and PEP 561 - # searching is not disabled - parser.error(str(e)) + # raise error if we cannot find site-packages and PEP 561 + # searching is not disabled + parser.error(str(e)) elif special_opts.python_version is None and special_opts.python_executable is not None: try: options.python_version = _python_version_from_executable( From a929b46c1fd659faa900f65c101881a0fd586ae0 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 23:39:46 -0800 Subject: [PATCH 092/100] Fix test import --- mypy/test/testpep561.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 8de2094fca23..3282e92ceb46 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -6,7 +6,7 @@ from unittest import TestCase, main import mypy.api -from mypy.build import get_package_dirs +from mypy.build import get_site_packages_dirs from mypy.test.config import package_path from mypy.test.helpers import run_command from mypy.util import try_find_python2_interpreter From e56ffe3392d14b64a708693996ae0d4488e15564 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 25 Feb 2018 23:54:04 -0800 Subject: [PATCH 093/100] Fix silly name mistake --- mypy/main.py | 29 +++++++++-------------------- mypy/test/testpep561.py | 2 +- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/mypy/main.py b/mypy/main.py index 1b3acbfb8d40..8b8bfc16d871 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -528,15 +528,10 @@ def add_invertible_flag(flag: str, print("Warning: --no-fast-parser no longer has any effect. The fast parser " "is now mypy's default and only parser.") - # Infer Python version and/or executable if one is not given - if special_opts.python_executable is not None and special_opts.python_version is not None: - try: + try: + # Infer Python version and/or executable if one is not given + if special_opts.python_executable is not None and special_opts.python_version is not None: py_exe_ver = _python_version_from_executable(special_opts.python_executable) - - except PythonExecutableInferenceError as e: - parser.error(str(e)) - sys.exit(2) - else: if py_exe_ver != special_opts.python_version: parser.error( 'Python version {} did not match executable {}, got version {}.'.format( @@ -545,23 +540,17 @@ def add_invertible_flag(flag: str, else: options.python_version = special_opts.python_version options.python_executable = special_opts.python_executable - elif special_opts.python_executable is None and special_opts.python_version is not None: - options.python_version = special_opts.python_version - if not special_opts.no_site_packages: - try: + elif special_opts.python_executable is None and special_opts.python_version is not None: + options.python_version = special_opts.python_version + if not special_opts.no_site_packages: py_exe = _python_executable_from_version(special_opts.python_version) options.python_executable = py_exe - except PythonExecutableInferenceError as e: - # raise error if we cannot find site-packages and PEP 561 - # searching is not disabled - parser.error(str(e)) - elif special_opts.python_version is None and special_opts.python_executable is not None: - try: + elif special_opts.python_version is None and special_opts.python_executable is not None: options.python_version = _python_version_from_executable( special_opts.python_executable) options.python_executable = special_opts.python_executable - except PythonExecutableInferenceError as e: - parser.error(str(e)) + except PythonExecutableInferenceError as e: + parser.error(str(e)) if special_opts.no_site_packages: options.python_executable = None diff --git a/mypy/test/testpep561.py b/mypy/test/testpep561.py index 3282e92ceb46..5ab2bbf3cb3b 100644 --- a/mypy/test/testpep561.py +++ b/mypy/test/testpep561.py @@ -39,7 +39,7 @@ def install_package(self, pkg: str, def test_get_pkg_dirs(self) -> None: """Check that get_package_dirs works.""" - dirs = get_package_dirs(sys.executable) + dirs = get_site_packages_dirs(sys.executable) assert dirs @staticmethod From 72d7b0a575363ead7160c51a4c9b20c9037fbc16 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 26 Feb 2018 01:03:29 -0800 Subject: [PATCH 094/100] Link to PEP 561 and add note about inference --- docs/source/command_line.rst | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index 9abfce348722..c1b1c8b7fb9e 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -371,10 +371,12 @@ Here are some more useful flags: written by quick mode. - ``--python-executable EXECUTABLE`` will have mypy collect type information - from PEP 561 compliant packages installed for the Python executable - ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages - installed for the Python executable running mypy. See - :ref:`installed_packages` for more on making PEP 561 compliant packages. + from `PEP 561 `_ compliant + packages installed for the Python executable ``EXECUTABLE``. If not provided, + mypy will use PEP 561 compliant packages installed for the Python executable + running mypy. See :ref:`installed_packages` for more on making PEP 561 + compliant packages. This flag will attempt to set ``--python-version`` if not + already set. - ``--python-version X.Y`` will make mypy typecheck your code as if it were run under Python version X.Y. Without this option, mypy will default to using @@ -382,10 +384,11 @@ Here are some more useful flags: ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag will attempt to find a Python executable of the corresponding version to - search for PEP 561 compliant packages. If you'd like to disable this, see - ``--no-site-packages`` below. + search for `PEP 561 `_ compliant + packages. If you'd like to disable this, see ``--no-site-packages`` below. -- ``--no-site-packages`` will disable searching for PEP 561 compliant packages. +- ``--no-site-packages`` will disable searching for + `PEP 561 `_ compliant packages. This will also disable searching for a usable Python executable. Use this flag if mypy cannot find a Python executable for the version of Python being checked, and you don't need to use PEP 561 typed packages. Otherwise, use From c9f6bc258b1b478a270eaa93b25287efcc423f73 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 26 Feb 2018 13:05:06 -0800 Subject: [PATCH 095/100] Fixup docs --- docs/source/command_line.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index c1b1c8b7fb9e..a50499649794 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -371,10 +371,10 @@ Here are some more useful flags: written by quick mode. - ``--python-executable EXECUTABLE`` will have mypy collect type information - from `PEP 561 `_ compliant + from `PEP 561`_ compliant packages installed for the Python executable ``EXECUTABLE``. If not provided, mypy will use PEP 561 compliant packages installed for the Python executable - running mypy. See :ref:`installed_packages` for more on making PEP 561 + running mypy. See :ref:`installed-packages` for more on making PEP 561 compliant packages. This flag will attempt to set ``--python-version`` if not already set. @@ -384,11 +384,11 @@ Here are some more useful flags: ``--py2`` flags are aliases for ``--python-version 2.7``. See :ref:`version_and_platform_checks` for more about this feature. This flag will attempt to find a Python executable of the corresponding version to - search for `PEP 561 `_ compliant + search for `PEP 561`_ compliant packages. If you'd like to disable this, see ``--no-site-packages`` below. - ``--no-site-packages`` will disable searching for - `PEP 561 `_ compliant packages. + `PEP 561`_ compliant packages. This will also disable searching for a usable Python executable. Use this flag if mypy cannot find a Python executable for the version of Python being checked, and you don't need to use PEP 561 typed packages. Otherwise, use @@ -475,6 +475,9 @@ For the remaining flags you can read the full ``mypy -h`` output. Command line flags are liable to change between releases. + +.. _PEP 561: https://www.python.org/dev/peps/pep-0561/ + .. _integrating-mypy: Integrating mypy into another Python application From 4698d232090db54c95e3dec6dcd0a674d6a01999 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 28 Feb 2018 23:30:05 -0800 Subject: [PATCH 096/100] Refactor find_module and clarify comment --- mypy/build.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 617558b50e6e..a8ded6511623 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -911,8 +911,6 @@ def find_module(id: str, lib_path_arg: Iterable[str], sys.exit(2) else: site_packages_dirs = [] - components = id.split('.') - dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' def find() -> Optional[str]: # If we're looking for a module like 'foo.bar.baz', it's likely that most of the @@ -975,13 +973,15 @@ def find() -> Optional[str]: return None if id not in find_module_cache: + components = id.split('.') + dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' find_module_cache[id] = find() - - # If we searched for items with a base directory of site-packages/ we need to - # remove it to avoid searching it for non-typed ids. - for dir in site_packages_dirs: - if dir + os.sep in find_module_dir_cache[dir_chain]: - find_module_dir_cache[dir_chain].remove(dir + os.sep) + # If we searched for items with a base directory of site-packages/ we need to + # remove it to avoid searching it for untyped modules and packages e.g. + # site-packages/file.py + for dir in site_packages_dirs: + if dir + os.sep in find_module_dir_cache[dir_chain]: + find_module_dir_cache[dir_chain].remove(dir + os.sep) return find_module_cache[id] From db0680a87ea542369d0311474387365cfeb6ba70 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 4 Mar 2018 23:05:39 -0800 Subject: [PATCH 097/100] Refactor find_module to not be monolithic --- mypy/build.py | 154 ++++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 73 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index a8ded6511623..b4b1eb7ae3ce 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -799,7 +799,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: # Cache find_module: id -> result. -find_module_cache = {} # type: Dict[str, Optional[str]] +find_module_cache = {} # type: Dict[Tuple[str, Optional[str], Tuple[str, ...]], Optional[str]] # Cache some repeated work within distinct find_module calls: finding which # elements of lib_path have even the subdirectory they'd need for the module @@ -899,83 +899,91 @@ def get_site_packages_dirs(python_executable: str) -> List[str]: return [line for line in output.splitlines() if os.path.isdir(line)] +def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[str], + site_packages_dirs: List[str]) -> Tuple[Iterable[str], List[str]]: + # If we're looking for a module like 'foo.bar.baz', it's likely that most of the + # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover + # that only once and cache it for when we look for modules like 'foo.bar.blah' + # that will require the same subdirectory. + dirs = find_module_dir_cache.get(dir_chain, []) + if not dirs: + # Regular packages on the PATH + for pathitem in lib_path: + # e.g., '/usr/lib/python3.4/foo/bar' + isdir = find_module_isdir_cache.get((pathitem, dir_chain)) + if isdir is None: + dir = os.path.normpath(os.path.join(pathitem, dir_chain)) + isdir = os.path.isdir(dir) + find_module_isdir_cache[pathitem, dir_chain] = isdir + if isdir: + dirs.append(dir) + + # Third-party stub/typed packages + for pkg_dir in site_packages_dirs: + stub_name = components[0] + '-stubs' + typed_file = os.path.join(pkg_dir, components[0], 'py.typed') + stub_dir = os.path.join(pkg_dir, stub_name) + if os.path.isdir(stub_dir): + components[0] = stub_name + rest = components[:-1] + path = os.path.join(pkg_dir, *rest) + if os.path.isdir(path): + dirs.append(path) + elif os.path.isfile(typed_file): + path = os.path.join(pkg_dir, dir_chain) + dirs.append(path) + + find_module_dir_cache[dir_chain] = dirs + return tuple(find_module_dir_cache[dir_chain]), components + + +def find_module_in_base_dirs(id: str, candidate_base_dirs: Iterable[str], + last_component: str) -> Optional[str]: + # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now + # contains just the subdirectories 'foo/bar' that actually exist under the + # elements of lib_path. This is probably much shorter than lib_path itself. + # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. + seplast = os.sep + last_component + sepinit = os.sep + '__init__' + for base_dir in candidate_base_dirs: + base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' + # Prefer package over module, i.e. baz/__init__.py* over baz.py*. + for extension in PYTHON_EXTENSIONS: + path = base_path + sepinit + extension + path_stubs = base_path + '-stubs' + sepinit + extension + if is_file(path) and verify_module(id, path): + return path + elif is_file(path_stubs) and verify_module(id, path_stubs): + return path_stubs + # No package, look for module. + for extension in PYTHON_EXTENSIONS: + path = base_path + extension + if is_file(path) and verify_module(id, path): + return path + return None + + def find_module(id: str, lib_path_arg: Iterable[str], python_executable: Optional[str]) -> Optional[str]: """Return the path of the module source file, or None if not found.""" lib_path = tuple(lib_path_arg) - if python_executable is not None: - site_packages_dirs = get_site_packages_dirs(python_executable) - if not site_packages_dirs: - print("Could not find package directories for Python '{}'".format( - python_executable), file=sys.stderr) - sys.exit(2) - else: - site_packages_dirs = [] - - def find() -> Optional[str]: - # If we're looking for a module like 'foo.bar.baz', it's likely that most of the - # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover - # that only once and cache it for when we look for modules like 'foo.bar.blah' - # that will require the same subdirectory. - - dirs = find_module_dir_cache.get(dir_chain, []) - if not dirs: - # Regular packages on the PATH - for pathitem in lib_path: - # e.g., '/usr/lib/python3.4/foo/bar' - isdir = find_module_isdir_cache.get((pathitem, dir_chain)) - if isdir is None: - dir = os.path.normpath(os.path.join(pathitem, dir_chain)) - isdir = os.path.isdir(dir) - find_module_isdir_cache[pathitem, dir_chain] = isdir - if isdir: - dirs.append(dir) - - # Third-party stub/typed packages - for pkg_dir in site_packages_dirs: - stub_name = components[0] + '-stubs' - typed_file = os.path.join(pkg_dir, components[0], 'py.typed') - stub_dir = os.path.join(pkg_dir, stub_name) - if os.path.isdir(stub_dir): - components[0] = stub_name - rest = components[:-1] - path = os.path.join(pkg_dir, *rest) - if os.path.isdir(path): - dirs.append(path) - elif os.path.isfile(typed_file): - path = os.path.join(pkg_dir, dir_chain) - dirs.append(path) - - find_module_dir_cache[dir_chain] = dirs - candidate_base_dirs = find_module_dir_cache[dir_chain] - - # If we're looking for a module like 'foo.bar.baz', then candidate_base_dirs now - # contains just the subdirectories 'foo/bar' that actually exist under the - # elements of lib_path. This is probably much shorter than lib_path itself. - # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories. - seplast = os.sep + components[-1] # so e.g. '/baz' - sepinit = os.sep + '__init__' - for base_dir in candidate_base_dirs: - base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz' - # Prefer package over module, i.e. baz/__init__.py* over baz.py*. - for extension in PYTHON_EXTENSIONS: - path = base_path + sepinit + extension - path_stubs = base_path + '-stubs' + sepinit + extension - if is_file(path) and verify_module(id, path): - return path - elif is_file(path_stubs) and verify_module(id, path_stubs): - return path_stubs - # No package, look for module. - for extension in PYTHON_EXTENSIONS: - path = base_path + extension - if is_file(path) and verify_module(id, path): - return path - return None - - if id not in find_module_cache: + if (id, python_executable, lib_path) not in find_module_cache: components = id.split('.') dir_chain = os.sep.join(components[:-1]) # e.g., 'foo/bar' - find_module_cache[id] = find() + + if python_executable is not None: + site_packages_dirs = get_site_packages_dirs(python_executable) + if not site_packages_dirs: + print("Could not find package directories for Python '{}'".format( + python_executable), file=sys.stderr) + sys.exit(2) + else: + site_packages_dirs = [] + base_dirs, components = find_base_dirs(lib_path, dir_chain, components, site_packages_dirs) + find_module_cache[id, + python_executable, + lib_path] = find_module_in_base_dirs(id, base_dirs, components[-1]) + # If we searched for items with a base directory of site-packages/ we need to # remove it to avoid searching it for untyped modules and packages e.g. # site-packages/file.py @@ -983,7 +991,7 @@ def find() -> Optional[str]: if dir + os.sep in find_module_dir_cache[dir_chain]: find_module_dir_cache[dir_chain].remove(dir + os.sep) - return find_module_cache[id] + return find_module_cache[id, python_executable, lib_path] def find_modules_recursive(module: str, lib_path: List[str], From 8367e57a1838c23b98fde817730c72aa9a031f2c Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 4 Mar 2018 23:13:57 -0800 Subject: [PATCH 098/100] Key find_module_dir_cache with lib_path as well --- mypy/build.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index b4b1eb7ae3ce..de029464d19b 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -805,7 +805,7 @@ def remove_cwd_prefix_from_path(p: str) -> str: # elements of lib_path have even the subdirectory they'd need for the module # to exist. This is shared among different module ids when they differ only # in the last component. -find_module_dir_cache = {} # type: Dict[str, List[str]] +find_module_dir_cache = {} # type: Dict[Tuple[str, Tuple[str, ...]], List[str]] # Cache directory listings. We assume that while one os.listdir() # call may be more expensive than one os.stat() call, a small number @@ -905,7 +905,7 @@ def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[s # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover # that only once and cache it for when we look for modules like 'foo.bar.blah' # that will require the same subdirectory. - dirs = find_module_dir_cache.get(dir_chain, []) + dirs = find_module_dir_cache.get((dir_chain, lib_path), []) if not dirs: # Regular packages on the PATH for pathitem in lib_path: @@ -933,8 +933,8 @@ def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[s path = os.path.join(pkg_dir, dir_chain) dirs.append(path) - find_module_dir_cache[dir_chain] = dirs - return tuple(find_module_dir_cache[dir_chain]), components + find_module_dir_cache[dir_chain, lib_path] = dirs + return tuple(find_module_dir_cache[dir_chain, lib_path]), components def find_module_in_base_dirs(id: str, candidate_base_dirs: Iterable[str], @@ -988,8 +988,8 @@ def find_module(id: str, lib_path_arg: Iterable[str], # remove it to avoid searching it for untyped modules and packages e.g. # site-packages/file.py for dir in site_packages_dirs: - if dir + os.sep in find_module_dir_cache[dir_chain]: - find_module_dir_cache[dir_chain].remove(dir + os.sep) + if dir + os.sep in find_module_dir_cache[dir_chain, lib_path]: + find_module_dir_cache[dir_chain, lib_path].remove(dir + os.sep) return find_module_cache[id, python_executable, lib_path] From 24b6742bb79744864f7720d35174cef8112c03f9 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 5 Mar 2018 13:04:07 -0800 Subject: [PATCH 099/100] Don't mutate dir cache with site packages --- mypy/build.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index de029464d19b..d5793c9f1964 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -917,9 +917,11 @@ def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[s find_module_isdir_cache[pathitem, dir_chain] = isdir if isdir: dirs.append(dir) - + find_module_dir_cache[dir_chain, lib_path] = dirs + third_party_dirs = [] # Third-party stub/typed packages for pkg_dir in site_packages_dirs: + prefix = components[0] stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_dir = os.path.join(pkg_dir, stub_name) @@ -928,13 +930,14 @@ def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[s rest = components[:-1] path = os.path.join(pkg_dir, *rest) if os.path.isdir(path): - dirs.append(path) + third_party_dirs.append(path) + components[0] = prefix elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) - dirs.append(path) + third_party_dirs.append(path) - find_module_dir_cache[dir_chain, lib_path] = dirs - return tuple(find_module_dir_cache[dir_chain, lib_path]), components + return tuple(third_party_dirs + + find_module_dir_cache[dir_chain, lib_path]), components def find_module_in_base_dirs(id: str, candidate_base_dirs: Iterable[str], @@ -984,13 +987,6 @@ def find_module(id: str, lib_path_arg: Iterable[str], python_executable, lib_path] = find_module_in_base_dirs(id, base_dirs, components[-1]) - # If we searched for items with a base directory of site-packages/ we need to - # remove it to avoid searching it for untyped modules and packages e.g. - # site-packages/file.py - for dir in site_packages_dirs: - if dir + os.sep in find_module_dir_cache[dir_chain, lib_path]: - find_module_dir_cache[dir_chain, lib_path].remove(dir + os.sep) - return find_module_cache[id, python_executable, lib_path] From afb80ea15062f9251b90f859fa9e6e92957e8c43 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Mon, 5 Mar 2018 15:17:05 -0800 Subject: [PATCH 100/100] Refactor stub package searching --- mypy/build.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index d5793c9f1964..e756f5fbe743 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -900,11 +900,12 @@ def get_site_packages_dirs(python_executable: str) -> List[str]: def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[str], - site_packages_dirs: List[str]) -> Tuple[Iterable[str], List[str]]: + site_packages_dirs: List[str]) -> Tuple[str, ...]: # If we're looking for a module like 'foo.bar.baz', it's likely that most of the # many elements of lib_path don't even have a subdirectory 'foo/bar'. Discover # that only once and cache it for when we look for modules like 'foo.bar.blah' # that will require the same subdirectory. + # TODO (ethanhs): refactor to use lru_cache on each of these searches dirs = find_module_dir_cache.get((dir_chain, lib_path), []) if not dirs: # Regular packages on the PATH @@ -921,23 +922,20 @@ def find_base_dirs(lib_path: Tuple[str, ...], dir_chain: str, components: List[s third_party_dirs = [] # Third-party stub/typed packages for pkg_dir in site_packages_dirs: - prefix = components[0] stub_name = components[0] + '-stubs' typed_file = os.path.join(pkg_dir, components[0], 'py.typed') stub_dir = os.path.join(pkg_dir, stub_name) if os.path.isdir(stub_dir): - components[0] = stub_name - rest = components[:-1] - path = os.path.join(pkg_dir, *rest) + stub_components = [stub_name] + components[1:] + path = os.path.join(pkg_dir, *stub_components[:-1]) if os.path.isdir(path): third_party_dirs.append(path) - components[0] = prefix elif os.path.isfile(typed_file): path = os.path.join(pkg_dir, dir_chain) third_party_dirs.append(path) return tuple(third_party_dirs + - find_module_dir_cache[dir_chain, lib_path]), components + find_module_dir_cache[dir_chain, lib_path]) def find_module_in_base_dirs(id: str, candidate_base_dirs: Iterable[str], @@ -982,7 +980,7 @@ def find_module(id: str, lib_path_arg: Iterable[str], sys.exit(2) else: site_packages_dirs = [] - base_dirs, components = find_base_dirs(lib_path, dir_chain, components, site_packages_dirs) + base_dirs = find_base_dirs(lib_path, dir_chain, components, site_packages_dirs) find_module_cache[id, python_executable, lib_path] = find_module_in_base_dirs(id, base_dirs, components[-1])