Namespaces refactor (#5686)

In preparation of working on PEP 420 namespaces, here's some initial work. IT DOES NOT IMPLEMENT NAMESPACES YET. What it does do:

- Move nearly 400 lines from build.py to a new file, `modulefinder.py`.  This includes `SearchPaths`, `BuildSource`, `FindModuleCache`, `mypy-path`, `default_lib_path`, `get_site_packages` (now without underscore!), and `compute_search_paths`.
- Slight refactor to `FindModuleCache` so that the `search_paths` are passed to the constructor instead of to `find_module`.
- Removed `search_paths` and `python_executable` from the signature of `find_module` and `find_modules_recursive`, and also from the cache key (**this may be the most controversial change** -- it's not just a refactor).
- Add a (non-functional) `--namespace-packages` flag to `main.py` and add new global config option `namespace_packages`. (These are not used yet.)

I'm presenting this as a separate PR because it's a significant refactor without change of functionality -- everything I plan to do after this will mostly tweak the code in `modulefinder.py`. It seems fairer to reviewers to be able to review this massive code move without having to worry too much about "what else changed".

(Note that this is part of my general plan to move functionality out of `build.py` -- that file is (still) way too bulky.)

mypy/find_sources.py

@@ -4,7 +4,7 @@
 
 from typing import List, Sequence, Set, Tuple, Optional, Dict
 
-from mypy.build import BuildSource, PYTHON_EXTENSIONS
+from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
 

mypy/main.py

@@ -15,7 +15,8 @@
 from mypy import defaults
 from mypy import experiments
 from mypy import util
-from mypy.build import BuildSource, BuildResult, SearchPaths
+from mypy.build import BuildResult
+from mypy.modulefinder import BuildSource, FindModuleCache, mypy_path, SearchPaths
 from mypy.find_sources import create_source_list, InvalidSourceList
 from mypy.fscache import FileSystemCache
 from mypy.errors import CompileError
@@ -453,6 +454,10 @@ def add_invertible_flag(flag: str,
     imports_group.add_argument(
         '--no-silence-site-packages', action='store_true',
         help="Do not silence errors in PEP 561 compliant installed packages")
+    add_invertible_flag(
+        '--namespace-packages', default=False,
+        help="Support namespace packages (PEP 420, __init__.py-less)",
+        group=imports_group)
 
     platform_group = parser.add_argument_group(
         title='Platform configuration',
@@ -880,14 +885,14 @@ def add_invertible_flag(flag: str,
     # Set target.
     if special_opts.modules + special_opts.packages:
         options.build_type = BuildType.MODULE
-        search_paths = SearchPaths((os.getcwd(),), tuple(build.mypy_path()), (), ())
+        search_paths = SearchPaths((os.getcwd(),), tuple(mypy_path()), (), ())
         targets = []
         # TODO: use the same cache that the BuildManager will
-        cache = build.FindModuleCache(fscache)
+        cache = FindModuleCache(search_paths, fscache)
         for p in special_opts.packages:
             if os.sep in p or os.altsep and os.altsep in p:
                 fail("Package name '{}' cannot have a slash in it.".format(p))
-            p_targets = cache.find_modules_recursive(p, search_paths, options.python_executable)
+            p_targets = cache.find_modules_recursive(p)
             if not p_targets:
                 fail("Can't find package '{}'".format(p))
             targets.extend(p_targets)

mypy/options.py

@@ -80,7 +80,9 @@ def __init__(self) -> None:
         self.follow_imports = 'normal'  # normal|silent|skip|error
         # Whether to respect the follow_imports setting even for stub files.
         # Intended to be used for disabling specific stubs.
-        self.follow_imports_for_stubs = False  # type: bool
+        self.follow_imports_for_stubs = False
+        # PEP 420 namespace packages
+        self.namespace_packages = False
 
         # disallow_any options
         self.disallow_any_generics = False

mypy/stubgen.py

@@ -57,6 +57,7 @@
 import mypy.traverser
 import mypy.util
 from mypy import defaults
+from mypy.modulefinder import FindModuleCache, SearchPaths
 from mypy.nodes import (
     Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr, TupleExpr,
     ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr,
@@ -165,9 +166,8 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int],
             module_all = getattr(mod, '__all__', None)
     else:
         # Find module by going through search path.
-        search_paths = mypy.build.SearchPaths(('.',) + tuple(search_path), (), (), ())
-        module_path = mypy.build.FindModuleCache().find_module(module, search_paths,
-                                                               interpreter)
+        search_paths = SearchPaths(('.',) + tuple(search_path), (), (), ())
+        module_path = FindModuleCache(search_paths).find_module(module)
         if not module_path:
             raise SystemExit(
                 "Can't find module '{}' (consider using --search-path)".format(module))

mypy/test/testcheck.py

@@ -7,7 +7,8 @@
 from typing import Dict, List, Set, Tuple
 
 from mypy import build
-from mypy.build import BuildSource, Graph, SearchPaths
+from mypy.build import Graph
+from mypy.modulefinder import BuildSource, SearchPaths
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, UpdateFile
 from mypy.test.helpers import (
@@ -286,8 +287,7 @@ def parse_module(self,
             out = []
             search_paths = SearchPaths((test_temp_dir,), (), (), ())
             for module_name in module_names.split(' '):
-                path = build.FindModuleCache().find_module(module_name, search_paths,
-                                                           sys.executable)
+                path = build.FindModuleCache(search_paths).find_module(module_name)
                 assert path is not None, "Can't find ad hoc case file"
                 with open(path) as f:
                     program_text = f.read()

mypy/test/testdeps.py

@@ -8,7 +8,7 @@
 from collections import defaultdict
 
 from mypy import build, defaults
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile, Expression
 from mypy.options import Options

mypy/test/testdiff.py

@@ -4,7 +4,7 @@
 from typing import List, Tuple, Dict, Optional
 
 from mypy import build
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import CompileError
 from mypy.nodes import MypyFile

mypy/test/testerrorstream.py

@@ -4,7 +4,7 @@
 from mypy import build
 from mypy.test.helpers import assert_string_arrays_equal
 from mypy.test.data import DataDrivenTestCase, DataSuite
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.errors import CompileError
 from mypy.options import Options
 

mypy/test/testfinegrained.py

@@ -18,7 +18,7 @@
 from typing import List, cast
 
 from mypy import build
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.errors import CompileError
 from mypy.options import Options
 from mypy.test.config import test_temp_dir

mypy/test/testgraph.py

@@ -3,7 +3,8 @@
 from typing import AbstractSet, Dict, Set, List
 
 from mypy.test.helpers import assert_equal, Suite
-from mypy.build import BuildManager, State, BuildSourceSet, SearchPaths
+from mypy.build import BuildManager, State, BuildSourceSet
+from mypy.modulefinder import SearchPaths
 from mypy.build import topsort, strongly_connected_components, sorted_components, order_ascc
 from mypy.version import __version__
 from mypy.options import Options

mypy/test/testmerge.py

@@ -5,7 +5,8 @@
 from typing import List, Tuple, Dict, Optional
 
 from mypy import build
-from mypy.build import BuildSource, BuildResult
+from mypy.build import BuildResult
+from mypy.modulefinder import BuildSource
 from mypy.defaults import PYTHON3_VERSION
 from mypy.errors import CompileError
 from mypy.nodes import (

mypy/test/testpep561.py

@@ -6,7 +6,8 @@
 from unittest import TestCase, main
 
 import mypy.api
-from mypy.build import _get_site_packages_dirs, FileSystemCache
+from mypy.build import FileSystemCache
+from mypy.modulefinder import get_site_packages_dirs
 from mypy.test.config import package_path
 from mypy.test.helpers import run_command
 from mypy.util import try_find_python2_interpreter
@@ -129,7 +130,7 @@ def tearDown(self) -> None:
 
     def test_get_pkg_dirs(self) -> None:
         """Check that get_package_dirs works."""
-        dirs = _get_site_packages_dirs(sys.executable, FileSystemCache())
+        dirs = get_site_packages_dirs(sys.executable, FileSystemCache())
         assert dirs
 
     def test_typedpkg_stub_package(self) -> None:

mypy/test/testsemanal.py

@@ -5,7 +5,7 @@
 from typing import Dict, List
 
 from mypy import build
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.defaults import PYTHON3_VERSION
 from mypy.test.helpers import (
     assert_string_arrays_equal, normalize_error_messages, testfile_pyversion,

mypy/test/testtransform.py

@@ -3,7 +3,7 @@
 import os.path
 
 from mypy import build
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.test.helpers import (
     assert_string_arrays_equal, testfile_pyversion, normalize_error_messages
 )

mypy/test/testtypegen.py

@@ -3,7 +3,7 @@
 import re
 
 from mypy import build
-from mypy.build import BuildSource
+from mypy.modulefinder import BuildSource
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal