[WIP] Namespace implementation

mypy/main.py

@@ -323,6 +323,8 @@ def add_invertible_flag(flag: str,
     parser.add_argument('--show-traceback', '--tb', action='store_true',
                         help="show traceback on fatal error")
     parser.add_argument('--stats', action='store_true', dest='dump_type_stats', help="dump stats")
+    parser.add_argument('--namespace-packages', action='store_true', dest='namespace_packages',
+                        help='Allow implicit namespace packages (PEP420)')
     parser.add_argument('--inferstats', action='store_true', dest='dump_inference_stats',
                         help="dump type inference stats")
     parser.add_argument('--custom-typing', metavar='MODULE', dest='custom_typing_module',
@@ -508,7 +510,8 @@ def add_invertible_flag(flag: str,
                  .format(special_opts.package))
         options.build_type = BuildType.MODULE
         lib_path = [os.getcwd()] + build.mypy_path()
-        targets = build.find_modules_recursive(special_opts.package, lib_path)
+        mod_discovery = build.ModuleDiscovery(lib_path, options.namespace_packages)
+        targets = mod_discovery.find_modules_recursive(special_opts.package)
         if not targets:
             fail("Can't find package '{}'".format(special_opts.package))
         return targets, options

mypy/options.py

@@ -164,6 +164,9 @@ def __init__(self) -> None:
         # Use stub builtins fixtures to speed up tests
         self.use_builtins_fixtures = False
 
+        # Allow implicit namespace packages (PEP420)
+        self.namespace_packages = False
+
         # -- experimental options --
         self.shadow_file = None  # type: Optional[Tuple[str, str]]
         self.show_column_numbers = False  # type: bool

mypy/semanal.py

@@ -182,8 +182,6 @@ class SemanticAnalyzerPass2(NodeVisitor[None]):
     This is the second phase of semantic analysis.
     """
 
-    # Library search paths
-    lib_path = None  # type: List[str]
     # Module name space
     modules = None  # type: Dict[str, MypyFile]
     # Global name space for current module
@@ -229,13 +227,9 @@ class SemanticAnalyzerPass2(NodeVisitor[None]):
     def __init__(self,
                  modules: Dict[str, MypyFile],
                  missing_modules: Set[str],
-                 lib_path: List[str], errors: Errors,
+                 errors: Errors,
                  plugin: Plugin) -> None:
-        """Construct semantic analyzer.
-
-        Use lib_path to search for modules, and report analysis errors
-        using the Errors instance.
-        """
+        """Construct semantic analyzer."""
         self.locals = [None]
         self.imports = set()
         self.type = None
@@ -244,7 +238,6 @@ def __init__(self,
         self.function_stack = []
         self.block_depth = [0]
         self.loop_depth = 0
-        self.lib_path = lib_path
         self.errors = errors
         self.modules = modules
         self.msg = MessageBuilder(errors, modules)

mypy/stubgen.py

@@ -156,10 +156,12 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
             module_all = getattr(mod, '__all__', None)
     else:
         # Find module by going through search path.
-        module_path = mypy.build.find_module(module, ['.'] + search_path)
-        if not module_path:
+        md = mypy.build.ModuleDiscovery(['.'] + search_path)
+        src = md.find_module(module)
+        if not (src and src.path):
             raise SystemExit(
                 "Can't find module '{}' (consider using --search-path)".format(module))
+        module_path = src.path
         module_all = None
     return module_path, module_all
 

mypy/test/testcheck.py

@@ -76,6 +76,7 @@
     'check-incomplete-fixture.test',
     'check-custom-plugin.test',
     'check-default-plugin.test',
+    'check-namespaces.test',
 ]
 
 
@@ -320,11 +321,12 @@ def parse_module(self,
             module_names = m.group(1)
             out = []
             for module_name in module_names.split(' '):
-                path = build.find_module(module_name, [test_temp_dir])
-                assert path is not None, "Can't find ad hoc case file"
-                with open(path) as f:
+                md = build.ModuleDiscovery([test_temp_dir], namespaces_allowed=False)
+                src = md.find_module(module_name)
+                assert src is not None and src.path is not None, "Can't find ad hoc case file"
+                with open(src.path) as f:
                     program_text = f.read()
-                out.append((module_name, path, program_text))
+                out.append((module_name, src.path, program_text))
             return out
         else:
             return [('__main__', 'main', program_text)]

mypy/test/testdmypy.py

@@ -253,11 +253,12 @@ def parse_module(self,
             module_names = m.group(1)
             out = []
             for module_name in module_names.split(' '):
-                path = build.find_module(module_name, [test_temp_dir])
-                assert path is not None, "Can't find ad hoc case file"
-                with open(path) as f:
+                md = build.ModuleDiscovery([test_temp_dir])
+                src = md.find_module(module_name)
+                assert src is not None and src.path is not None, "Can't find ad hoc case file"
+                with open(src.path) as f:
                     program_text = f.read()
-                out.append((module_name, path, program_text))
+                out.append((module_name, src.path, program_text))
             return out
         else:
             return [('__main__', 'main', program_text)]

mypy/test/testgraph.py

@@ -3,7 +3,7 @@
 from typing import AbstractSet, Dict, Set, List
 
 from mypy.myunit import Suite, assert_equal
-from mypy.build import BuildManager, State, BuildSourceSet
+from mypy.build import BuildManager, State, BuildSourceSet, ModuleDiscovery
 from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc
 from mypy.version import __version__
 from mypy.options import Options
@@ -41,14 +41,14 @@ def _make_manager(self) -> BuildManager:
         options = Options()
         manager = BuildManager(
             data_dir='',
-            lib_path=[],
             ignore_prefix='',
             source_set=BuildSourceSet([]),
             reports=Reports('', {}),
             options=options,
             version_id=__version__,
             plugin=Plugin(options),
             errors=errors,
+            module_discovery=ModuleDiscovery([]),
         )
         return manager
 

mypy/test/testmodulediscovery.py

@@ -0,0 +1,155 @@
+import os
+
+from unittest import TestCase, mock
+from typing import List, Set, Union
+
+from mypy.build import ModuleDiscovery, find_module_clear_caches
+
+
+class TestModuleDiscovery(TestCase):
+    def setUp(self) -> None:
+        self.files = set()  # type: Union[Set[str], List[str]]
+        self._setup_mock_filesystem()
+
+    def tearDown(self) -> None:
+        self._teardown_mock_filesystem()
+        find_module_clear_caches()
+
+    def _list_dir(self, path: str) -> List[str]:
+        res = []
+
+        if not path.endswith(os.path.sep):
+            path = path + os.path.sep
+
+        for item in self.files:
+            if item.startswith(path):
+                remnant = item.replace(path, '')
+                segments = remnant.split(os.path.sep)
+                if segments:
+                    res.append(segments[0])
+
+        return res
+
+    def _is_file(self, path: str) -> bool:
+        return path in self.files
+
+    def _is_dir(self, path: str) -> bool:
+        for item in self.files:
+            if not item.endswith(os.path.sep):
+                item += os.path.sep
+            if item.startswith(path):
+                return True
+        return False
+
+    def _setup_mock_filesystem(self) -> None:
+        self._listdir_patcher = mock.patch('os.listdir', side_effect=self._list_dir)
+        self._listdir_mock = self._listdir_patcher.start()
+        self._isfile_patcher = mock.patch('os.path.isfile', side_effect=self._is_file)
+        self._isfile_mock = self._isfile_patcher.start()
+        self._isdir_patcher = mock.patch('os.path.isdir', side_effect=self._is_dir)
+        self._isdir_mock = self._isdir_patcher.start()
+
+    def _teardown_mock_filesystem(self) -> None:
+        self._listdir_patcher.stop()
+        self._isfile_patcher.stop()
+        self._isdir_patcher.stop()
+
+    def test_module_vs_package(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod.py'),
+            os.path.join('dir2', 'mod', '__init__.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=False)
+        src = m.find_module('mod')
+        assert src is not None
+        assert src.path == os.path.join('dir1', 'mod.py')
+
+        m = ModuleDiscovery(['dir2', 'dir1'], namespaces_allowed=False)
+        src = m.find_module('mod')
+        assert src is not None
+        assert src.path == os.path.join('dir2', 'mod', '__init__.py')
+
+    def test_stubs_priority_module(self) -> None:
+        self.files = [
+            os.path.join('dir1', 'mod.py'),
+            os.path.join('dir1', 'mod.pyi'),
+        ]
+        m = ModuleDiscovery(['dir1'], namespaces_allowed=False)
+        src = m.find_module('mod')
+        assert src is not None
+        assert src.path == os.path.join('dir1', 'mod.pyi')
+
+    def test_package_in_different_directories(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['./dir1', './dir2'], namespaces_allowed=False)
+        src = m.find_module('mod.a')
+        assert src is not None
+        assert src.path == os.path.join('dir1', 'mod', 'a.py')
+
+        src = m.find_module('mod.b')
+        assert src is None
+
+    def test_package_with_stubs(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'b.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=False)
+        src = m.find_module('mod.a')
+        assert src is not None
+        assert src.path == os.path.join('dir1', 'mod', 'a.py')
+
+        src = m.find_module('mod.b')
+        assert src is None
+
+    def test_namespaces(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        src = m.find_module('mod.a')
+        assert src is not None
+        assert src.path == os.path.join('dir1', 'mod', 'a.py')
+
+        src = m.find_module('mod.b')
+        assert src is not None
+        assert src.path == os.path.join('dir2', 'mod', 'b.py')
+
+    def test_find_modules_recursive(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'b.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert [s.module for s in srcs] == ['mod', 'mod.a']
+
+    def test_find_modules_recursive_with_namespace(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', 'b.py'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert [s.module for s in srcs] == ['mod', 'mod.a', 'mod.b']
+
+    def test_find_modules_recursive_with_stubs(self) -> None:
+        self.files = {
+            os.path.join('dir1', 'mod', '__init__.py'),
+            os.path.join('dir1', 'mod', 'a.py'),
+            os.path.join('dir2', 'mod', '__init__.pyi'),
+            os.path.join('dir2', 'mod', 'a.pyi'),
+        }
+        m = ModuleDiscovery(['dir1', 'dir2'], namespaces_allowed=True)
+        srcs = m.find_modules_recursive('mod')
+        assert [s.module for s in srcs] == ['mod', 'mod.a']

runtests.py

@@ -213,7 +213,8 @@ def test_path(*names: str):
     'testtransform',
     'testtypegen',
     'testparse',
-    'testsemanal'
+    'testsemanal',
+    'testmodulediscovery',
 )
 
 SLOW_FILES = test_path(