Namespace packages (PEP 420) (#5691)

Tentative implementation of PEP 420. Fixes #1645.

Clarification of the implementation:

- `candidate_base_dirs` is a list of `(dir, verify)` tuples, laboriously pre-computed.
- The old code just looped over this list, looking for `dir/<module>`, and if found, it would verify that there were `__init__.py` files in all the right places (except if `verify` is false); the first success would be the hit;
- In PEP 420 mode, if the above approach finds no hits, we do something different for those paths that failed due to `__init__` verification; essentially we narrow down the list of candidate paths by checking for `__init__` files from the top down. Hopefully the last test added clarifies this.

mypy/modulefinder.py

@@ -170,6 +170,7 @@ def _find_module(self, id: str) -> Optional[str]:
  # Now just look for 'baz.pyi', 'baz/__init__.py', etc., inside those directories.
  seplast = os.sep + components[-1] # so e.g. '/baz'
  sepinit = os.sep + '__init__'
+ near_misses = [] # Collect near misses for namespace mode (see below).
  for base_dir, verify in candidate_base_dirs:
  base_path = base_dir + seplast # so e.g. '/usr/lib/python3.4/foo/bar/baz'
  # Prefer package over module, i.e. baz/__init__.py* over baz.py*.
@@ -178,19 +179,49 @@ def _find_module(self, id: str) -> Optional[str]:
  path_stubs = base_path + '-stubs' + sepinit + extension
  if fscache.isfile_case(path):
  if verify and not verify_module(fscache, id, path):
+ near_misses.append(path)
  continue
  return path
  elif fscache.isfile_case(path_stubs):
  if verify and not verify_module(fscache, id, path_stubs):
+ near_misses.append(path_stubs)
  continue
  return path_stubs
  # No package, look for module.
  for extension in PYTHON_EXTENSIONS:
  path = base_path + extension
  if fscache.isfile_case(path):
  if verify and not verify_module(fscache, id, path):
+ near_misses.append(path)
  continue
  return path
+
+ # In namespace mode, re-check those entries that had 'verify'.
+ # Assume search path entries xxx, yyy and zzz, and we're
+ # looking for foo.bar.baz. Suppose near_misses has:
+ #
+ # - xxx/foo/bar/baz.py
+ # - yyy/foo/bar/baz/__init__.py
+ # - zzz/foo/bar/baz.pyi
+ #
+ # If any of the foo directories has __init__.py[i], it wins.
+ # Else, we look for foo/bar/__init__.py[i], etc. If there are
+ # none, the first hit wins. Note that this does not take into
+ # account whether the lowest-level module is a file (baz.py),
+ # a package (baz/__init__.py), or a stub file (baz.pyi) -- for
+ # these the first one encountered along the search path wins.
+ #
+ # The helper function highest_init_level() returns an int that
+ # indicates the highest level at which a __init__.py[i] file
+ # is found; if no __init__ was found it returns 0, if we find
+ # only foo/bar/__init__.py it returns 1, and if we have
+ # foo/__init__.py it returns 2 (regardless of what's in
+ # foo/bar). It doesn't look higher than that.
+ if self.options and self.options.namespace_packages and near_misses:
+ levels = [highest_init_level(fscache, id, path) for path in near_misses]
+ index = levels.index(max(levels))
+ return near_misses[index]
+
  return None
 
  def find_modules_recursive(self, module: str) -> List[BuildSource]:
@@ -236,6 +267,19 @@ def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool:
  return True
 
 
+def highest_init_level(fscache: FileSystemCache, id: str, path: str) -> int:
+ """Compute the highest level where an __init__ file is found."""
+ if path.endswith(('__init__.py', '__init__.pyi')):
+ path = os.path.dirname(path)
+ level = 0
+ for i in range(id.count('.')):
+ path = os.path.dirname(path)
+ if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension)))
+ for extension in PYTHON_EXTENSIONS):
+ level = i + 1
+ return level
+
+
 def mypy_path() -> List[str]:
  path_env = os.getenv('MYPYPATH')
  if not path_env:

test-data/unit/check-modules.test

@@ -1,4 +1,5 @@
 -- Type checker test cases dealing with modules and imports.
+-- Towards the end there are tests for PEP 420 (namespace packages, i.e. __init__.py-less packages).
 
 [case testAccessImportedDefinitions]
 import m
@@ -2525,3 +2526,97 @@ def __radd__(self) -> int: ...
 
 [case testFunctionWithInPlaceDunderName]
 def __iadd__(self) -> int: ...
+
+-- Tests for PEP 420 namespace packages.
+
+[case testClassicPackage]
+from foo.bar import x
+[file foo/__init__.py]
+# empty
+[file foo/bar.py]
+x = 0
+
+[case testClassicNotPackage]
+from foo.bar import x
+[file foo/bar.py]
+x = 0
+[out]
+main:1: error: Cannot find module named 'foo.bar'
+main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
+
+[case testNamespacePackage]
+# flags: --namespace-packages
+from foo.bar import x
+reveal_type(x) # E: Revealed type is 'builtins.int'
+[file foo/bar.py]
+x = 0
+
+[case testNamespacePackageWithMypyPath]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bax import x
+from foo.bay import y
+from foo.baz import z
+reveal_type(x) # E: Revealed type is 'builtins.int'
+reveal_type(y) # E: Revealed type is 'builtins.int'
+reveal_type(z) # E: Revealed type is 'builtins.int'
+[file xx/foo/bax.py]
+x = 0
+[file yy/foo/bay.py]
+y = 0
+[file foo/baz.py]
+z = 0
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testClassicPackageIgnoresEarlierNamespacePackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar import y
+reveal_type(y) # E: Revealed type is 'builtins.int'
+[file xx/foo/bar.py]
+x = ''
+[file yy/foo/bar.py]
+y = 0
+[file yy/foo/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testNamespacePackagePickFirstOnMypyPath]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar import x
+reveal_type(x) # E: Revealed type is 'builtins.int'
+[file xx/foo/bar.py]
+x = 0
+[file yy/foo/bar.py]
+x = ''
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testNamespacePackageInsideClassicPackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar.baz import x
+reveal_type(x) # E: Revealed type is 'builtins.int'
+[file xx/foo/bar/baz.py]
+x = ''
+[file yy/foo/bar/baz.py]
+x = 0
+[file yy/foo/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy
+
+[case testClassicPackageInsideNamespacePackage]
+# flags: --namespace-packages --config-file tmp/mypy.ini
+from foo.bar.baz.boo import x
+reveal_type(x) # E: Revealed type is 'builtins.int'
+[file xx/foo/bar/baz/boo.py]
+x = ''
+[file xx/foo/bar/baz/__init__.py]
+[file yy/foo/bar/baz/boo.py]
+x = 0
+[file yy/foo/bar/__init__.py]
+[file mypy.ini]
+[[mypy]
+mypy_path = tmp/xx, tmp/yy