From ccb5e94469283fa303e3420e323984d832b539f3 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 21 Nov 2020 20:54:07 -0800
Subject: [PATCH 01/14] cosmetic changes to modulefinder

Just so that the logic mirrors closely
---
 mypy/modulefinder.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py
index d61c65e279bf..bdc71d7a7e58 100644
--- a/mypy/modulefinder.py
+++ b/mypy/modulefinder.py
@@ -361,7 +361,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
         module_path = self.find_module(module)
         if isinstance(module_path, ModuleNotFoundReason):
             return []
-        result = [BuildSource(module_path, module, None)]
+        sources = [BuildSource(module_path, module, None)]
 
         package_path = None
         if module_path.endswith(('__init__.py', '__init__.pyi')):
@@ -369,7 +369,7 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
         elif self.fscache.isdir(module_path):
             package_path = module_path
         if package_path is None:
-            return result
+            return sources
 
         # This logic closely mirrors that in find_sources. One small but important difference is
         # that we do not sort names with keyfunc. The recursive call to find_modules_recursive
@@ -382,16 +382,16 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
             # Skip certain names altogether
             if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
                 continue
-            path = os.path.join(package_path, name)
+            subpath = os.path.join(package_path, name)
 
-            if self.fscache.isdir(path):
+            if self.fscache.isdir(subpath):
                 # Only recurse into packages
                 if (self.options and self.options.namespace_packages) or (
-                    self.fscache.isfile(os.path.join(path, "__init__.py"))
-                    or self.fscache.isfile(os.path.join(path, "__init__.pyi"))
+                    self.fscache.isfile(os.path.join(subpath, "__init__.py"))
+                    or self.fscache.isfile(os.path.join(subpath, "__init__.pyi"))
                 ):
                     seen.add(name)
-                    result.extend(self.find_modules_recursive(module + '.' + name))
+                    sources.extend(self.find_modules_recursive(module + '.' + name))
             else:
                 stem, suffix = os.path.splitext(name)
                 if stem == '__init__':
@@ -400,8 +400,8 @@ def find_modules_recursive(self, module: str) -> List[BuildSource]:
                     # (If we sorted names) we could probably just make the BuildSource ourselves,
                     # but this ensures compatibility with find_module / the cache
                     seen.add(stem)
-                    result.extend(self.find_modules_recursive(module + '.' + stem))
-        return result
+                    sources.extend(self.find_modules_recursive(module + '.' + stem))
+        return sources
 
 
 def verify_module(fscache: FileSystemCache, id: str, path: str, prefix: str) -> bool:

From a5a24c9f059a5ca3f1826bfbba8e5402ed75acf5 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 20 Nov 2020 19:27:58 -0800
Subject: [PATCH 02/14] better crawling for namespace packages, explicit base
 dirs, abs paths

---
 mypy/find_sources.py           | 194 ++++++++++++++++-------------
 mypy/main.py                   |  26 ++--
 mypy/options.py                |   1 +
 mypy/suggestions.py            |   2 +-
 mypy/test/test_find_sources.py | 220 +++++++++++++++++++++++++++++++++
 5 files changed, 347 insertions(+), 96 deletions(-)
 create mode 100644 mypy/test/test_find_sources.py

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index d20f0ac9832f..c09121cf3e32 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -1,11 +1,12 @@
 """Routines for finding the sources that mypy will check"""
 
-import os.path
+import functools
+import os
 
-from typing import List, Sequence, Set, Tuple, Optional, Dict
+from typing import List, Sequence, Set, Tuple, Optional
 from typing_extensions import Final
 
-from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS
+from mypy.modulefinder import BuildSource, PYTHON_EXTENSIONS, mypy_path
 from mypy.fscache import FileSystemCache
 from mypy.options import Options
 
@@ -24,7 +25,7 @@ def create_source_list(paths: Sequence[str], options: Options,
     Raises InvalidSourceList on errors.
     """
     fscache = fscache or FileSystemCache()
-    finder = SourceFinder(fscache)
+    finder = SourceFinder(fscache, options)
 
     sources = []
     for path in paths:
@@ -34,7 +35,7 @@ def create_source_list(paths: Sequence[str], options: Options,
             name, base_dir = finder.crawl_up(path)
             sources.append(BuildSource(path, name, None, base_dir))
         elif fscache.isdir(path):
-            sub_sources = finder.find_sources_in_dir(path, explicit_package_roots=None)
+            sub_sources = finder.find_sources_in_dir(path)
             if not sub_sources and not allow_empty_dir:
                 raise InvalidSourceList(
                     "There are no .py[i] files in directory '{}'".format(path)
@@ -58,112 +59,138 @@ def keyfunc(name: str) -> Tuple[int, str]:
     return (-1, name)
 
 
+def normalise_package_base(root: str) -> str:
+    if not root:
+        root = os.curdir
+    root = os.path.normpath(os.path.abspath(root))
+    if root.endswith(os.sep):
+        root = root[:-1]
+    return root
+
+
+def get_explicit_package_bases(options: Options) -> Optional[List[str]]:
+    if not options.explicit_package_bases:
+        return None
+    roots = mypy_path() + options.mypy_path + [os.getcwd()]
+    return [normalise_package_base(root) for root in roots]
+
+
 class SourceFinder:
-    def __init__(self, fscache: FileSystemCache) -> None:
+    def __init__(self, fscache: FileSystemCache, options: Options) -> None:
         self.fscache = fscache
-        # A cache for package names, mapping from directory path to module id and base dir
-        self.package_cache = {}  # type: Dict[str, Tuple[str, str]]
-
-    def find_sources_in_dir(
-        self, path: str, explicit_package_roots: Optional[List[str]]
-    ) -> List[BuildSource]:
-        if explicit_package_roots is None:
-            mod_prefix, root_dir = self.crawl_up_dir(path)
-        else:
-            mod_prefix = os.path.basename(path)
-            root_dir = os.path.dirname(path) or "."
-        if mod_prefix:
-            mod_prefix += "."
-        return self.find_sources_in_dir_helper(path, mod_prefix, root_dir, explicit_package_roots)
-
-    def find_sources_in_dir_helper(
-        self, dir_path: str, mod_prefix: str, root_dir: str,
-        explicit_package_roots: Optional[List[str]]
-    ) -> List[BuildSource]:
-        assert not mod_prefix or mod_prefix.endswith(".")
-
-        init_file = self.get_init_file(dir_path)
-        # If the current directory is an explicit package root, explore it as such.
-        # Alternatively, if we aren't given explicit package roots and we don't have an __init__
-        # file, recursively explore this directory as a new package root.
-        if (
-            (explicit_package_roots is not None and dir_path in explicit_package_roots)
-            or (explicit_package_roots is None and init_file is None)
-        ):
-            mod_prefix = ""
-            root_dir = dir_path
+        self.explicit_package_bases = get_explicit_package_bases(options)
+        self.namespace_packages = options.namespace_packages
 
-        seen = set()  # type: Set[str]
-        sources = []
+    def is_explicit_package_base(self, path: str) -> bool:
+        assert self.explicit_package_bases
+        return normalise_package_base(path) in self.explicit_package_bases
 
-        if init_file:
-            sources.append(BuildSource(init_file, mod_prefix.rstrip("."), None, root_dir))
+    def find_sources_in_dir(self, path: str) -> List[BuildSource]:
+        sources = []
 
-        names = self.fscache.listdir(dir_path)
-        names.sort(key=keyfunc)
+        seen = set()  # type: Set[str]
+        names = sorted(self.fscache.listdir(path), key=keyfunc)
         for name in names:
             # Skip certain names altogether
             if name == '__pycache__' or name.startswith('.') or name.endswith('~'):
                 continue
-            path = os.path.join(dir_path, name)
+            subpath = os.path.join(path, name)
 
-            if self.fscache.isdir(path):
-                sub_sources = self.find_sources_in_dir_helper(
-                    path, mod_prefix + name + '.', root_dir, explicit_package_roots
-                )
+            if self.fscache.isdir(subpath):
+                sub_sources = self.find_sources_in_dir(subpath)
                 if sub_sources:
                     seen.add(name)
                     sources.extend(sub_sources)
             else:
                 stem, suffix = os.path.splitext(name)
-                if stem == '__init__':
-                    continue
-                if stem not in seen and '.' not in stem and suffix in PY_EXTENSIONS:
+                if stem not in seen and suffix in PY_EXTENSIONS:
                     seen.add(stem)
-                    src = BuildSource(path, mod_prefix + stem, None, root_dir)
-                    sources.append(src)
+                    module, base_dir = self.crawl_up(subpath)
+                    sources.append(BuildSource(subpath, module, None, base_dir))
 
         return sources
 
     def crawl_up(self, path: str) -> Tuple[str, str]:
-        """Given a .py[i] filename, return module and base directory
+        """Given a .py[i] filename, return module and base directory.
 
-        We crawl up the path until we find a directory without
-        __init__.py[i], or until we run out of path components.
+        For example, given "xxx/yyy/foo/bar.py", we might return something like:
+        ("foo.bar", "xxx/yyy")
+
+        If namespace packages is off, we crawl upwards until we find a directory without
+        an __init__.py
+
+        If namespace packages is on, we crawl upwards until the nearest explicit base directory.
+        Failing that, we return one past the highest directory containing an __init__.py
+
+        We won't crawl past directories with invalid package names.
+        The base directory returned is an absolute path.
         """
+        path = os.path.normpath(os.path.abspath(path))
         parent, filename = os.path.split(path)
-        module_name = strip_py(filename) or os.path.basename(filename)
-        module_prefix, base_dir = self.crawl_up_dir(parent)
-        if module_name == '__init__' or not module_name:
-            module = module_prefix
-        else:
-            module = module_join(module_prefix, module_name)
 
+        module_name = strip_py(filename) or filename
+        if not module_name.isidentifier():
+            return module_name, parent
+
+        parent_module, base_dir = self.crawl_up_dir(parent)
+        if module_name == "__init__":
+            return parent_module, base_dir
+
+        module = module_join(parent_module, module_name)
         return module, base_dir
 
     def crawl_up_dir(self, dir: str) -> Tuple[str, str]:
-        """Given a directory name, return the corresponding module name and base directory
+        return self._crawl_up_helper(dir) or ("", dir)
 
-        Use package_cache to cache results.
-        """
-        if dir in self.package_cache:
-            return self.package_cache[dir]
+    @functools.lru_cache()
+    def _crawl_up_helper(self, dir: str) -> Optional[Tuple[str, str]]:
+        """Given a directory, maybe returns module and base directory.
 
-        parent_dir, base = os.path.split(dir)
-        if not dir or not self.get_init_file(dir) or not base:
-            module = ''
-            base_dir = dir or '.'
-        else:
-            # Ensure that base is a valid python module name
-            if base.endswith('-stubs'):
-                base = base[:-6]  # PEP-561 stub-only directory
-            if not base.isidentifier():
-                raise InvalidSourceList('{} is not a valid Python package name'.format(base))
-            parent_module, base_dir = self.crawl_up_dir(parent_dir)
-            module = module_join(parent_module, base)
-
-        self.package_cache[dir] = module, base_dir
-        return module, base_dir
+        We return a non-None value if we were able to find something clearly intended as a base
+        directory (as adjudicated by being an explicit base directory or by containing a package
+        with __init__.py).
+
+        This distinction is necessary for namespace packages, so that we know when to treat
+        ourselves as a subpackage.
+        """
+        # stop crawling if we're an explicit base directory
+        if self.explicit_package_bases is not None and self.is_explicit_package_base(dir):
+            return "", dir
+
+        parent, name = os.path.split(dir)
+        if name.endswith('-stubs'):
+            name = name[:-6]  # PEP-561 stub-only directory
+
+        # recurse if there's an __init__.py
+        init_file = self.get_init_file(dir)
+        if init_file is not None:
+            if not name.isidentifier():
+                # in most cases the directory name is invalid, we'll just stop crawling upwards
+                # but if there's an __init__.py in the directory, something is messed up
+                raise InvalidSourceList("{} is not a valid Python package name".format(name))
+            # we're definitely a package, so we always return a non-None value
+            mod_prefix, base_dir = self.crawl_up_dir(parent)
+            return module_join(mod_prefix, name), base_dir
+
+        # stop crawling if we're out of path components or our name is an invalid identifier
+        if not name or not parent or not name.isidentifier():
+            return None
+
+        # stop crawling if namespace packages is off (since we don't have an __init__.py)
+        if not self.namespace_packages:
+            return None
+
+        # at this point: namespace packages is on, we don't have an __init__.py and we're not an
+        # explicit base directory
+        result = self._crawl_up_helper(parent)
+        if result is None:
+            # we're not an explicit base directory and we don't have an __init__.py
+            # and none of our parents are either, so return
+            return None
+        # one of our parents was an explicit base directory or had an __init__.py, so we're
+        # definitely a subpackage! chain our name to the module.
+        mod_prefix, base_dir = result
+        return module_join(mod_prefix, name), base_dir
 
     def get_init_file(self, dir: str) -> Optional[str]:
         """Check whether a directory contains a file named __init__.py[i].
@@ -185,8 +212,7 @@ def module_join(parent: str, child: str) -> str:
     """Join module ids, accounting for a possibly empty parent."""
     if parent:
         return parent + '.' + child
-    else:
-        return child
+    return child
 
 
 def strip_py(arg: str) -> Optional[str]:
diff --git a/mypy/main.py b/mypy/main.py
index 763bd9e95638..ddabfc0fbeb1 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -786,6 +786,9 @@ def add_invertible_flag(flag: str,
         title="Running code",
         description="Specify the code you want to type check. For more details, see "
                     "mypy.readthedocs.io/en/latest/running_mypy.html#running-mypy")
+    code_group.add_argument(
+        '--explicit-package-bases', action='store_true',
+        help="Use current directory and MYPYPATH to determine module names of files passed")
     code_group.add_argument(
         '-m', '--module', action='append', metavar='MODULE',
         default=[],
@@ -862,6 +865,11 @@ def set_strict_flags() -> None:
             parser.error("Missing target module, package, files, or command.")
         elif code_methods > 1:
             parser.error("May only specify one of: module/package, files, or command.")
+    if options.explicit_package_bases and not options.namespace_packages:
+        parser.error(
+            "Can only use --explicit-base-dirs with --namespace-packages, since otherwise "
+            "examining __init__.py's is sufficient to determine module names for files"
+        )
 
     # Check for overlapping `--always-true` and `--always-false` flags.
     overlap = set(options.always_true) & set(options.always_false)
@@ -966,10 +974,7 @@ def process_package_roots(fscache: Optional[FileSystemCache],
     assert fscache is not None  # Since mypy doesn't know parser.error() raises.
     # Do some stuff with drive letters to make Windows happy (esp. tests).
     current_drive, _ = os.path.splitdrive(os.getcwd())
-    dot = os.curdir
-    dotslash = os.curdir + os.sep
     dotdotslash = os.pardir + os.sep
-    trivial_paths = {dot, dotslash}
     package_root = []
     for root in options.package_root:
         if os.path.isabs(root):
@@ -978,14 +983,13 @@ def process_package_roots(fscache: Optional[FileSystemCache],
         if drive and drive != current_drive:
             parser.error("Package root must be on current drive: %r" % (drive + root))
         # Empty package root is always okay.
-        if root:
-            root = os.path.relpath(root)  # Normalize the heck out of it.
-            if root.startswith(dotdotslash):
-                parser.error("Package root cannot be above current directory: %r" % root)
-            if root in trivial_paths:
-                root = ''
-            elif not root.endswith(os.sep):
-                root = root + os.sep
+        if not root:
+            root = os.curdir
+        if os.path.relpath(root).startswith(dotdotslash):
+            parser.error("Package root cannot be above current directory: %r" % root)
+        root = os.path.normpath(os.path.abspath(root))
+        if not root.endswith(os.sep):
+            root += os.sep
         package_root.append(root)
     options.package_root = package_root
     # Pass the package root on the the filesystem cache.
diff --git a/mypy/options.py b/mypy/options.py
index 901b90f28f53..fa8c3fca1e56 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -88,6 +88,7 @@ def __init__(self) -> None:
         self.follow_imports_for_stubs = False
         # PEP 420 namespace packages
         self.namespace_packages = False
+        self.explicit_package_bases = False
 
         # disallow_any options
         self.disallow_any_generics = False
diff --git a/mypy/suggestions.py b/mypy/suggestions.py
index 0a41b134db6f..b66ba6d6118d 100644
--- a/mypy/suggestions.py
+++ b/mypy/suggestions.py
@@ -220,7 +220,7 @@ def __init__(self, fgmanager: FineGrainedBuildManager,
         self.manager = fgmanager.manager
         self.plugin = self.manager.plugin
         self.graph = fgmanager.graph
-        self.finder = SourceFinder(self.manager.fscache)
+        self.finder = SourceFinder(self.manager.fscache, self.manager.options)
 
         self.give_json = json
         self.no_errors = no_errors
diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
new file mode 100644
index 000000000000..2b6b0178c32e
--- /dev/null
+++ b/mypy/test/test_find_sources.py
@@ -0,0 +1,220 @@
+from mypy.modulefinder import BuildSource
+import os
+from typing import Any, List, Optional, Set, Tuple, cast
+from unittest import TestCase
+from mypy.find_sources import SourceFinder
+from mypy.modulefinder import BuildSource
+from mypy.options import Options
+
+
+class _FakeFSCache:
+    def __init__(self, files: Set[str]) -> None:
+        assert all(os.path.isabs(f) for f in files)
+        self.files = files
+
+    def isfile(self, file: str) -> bool:
+        return file in self.files
+
+    def isdir(self, dir: str) -> bool:
+        if not dir.endswith(os.sep):
+            dir += os.sep
+        return any(f.startswith(dir) for f in self.files)
+
+    def listdir(self, dir: str) -> List[str]:
+        if not dir.endswith(os.sep):
+            dir += os.sep
+        return list(set(f[len(dir):].split(os.sep)[0] for f in self.files if f.startswith(dir)))
+
+    def init_under_package_root(self, file: str) -> bool:
+        return False
+
+
+FakeFSCache = cast(Any, _FakeFSCache)
+
+
+def normalise_build_source_list(sources: List[BuildSource]) -> List[Tuple[str, Optional[str]]]:
+    return sorted((s.module, s.base_dir) for s in sources)
+
+
+class SourceFinderSuite(TestCase):
+    def test_crawl_no_namespace(self) -> None:
+        options = Options()
+        options.namespace_packages = False
+
+        finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
+        assert finder.crawl_up("/setup.py") == ("setup", "/")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+
+    def test_crawl_namespace(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+
+        finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
+        assert finder.crawl_up("/setup.py") == ("setup", "/")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("a.b.setup", "/")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/b/c/setup.py") == ("a.b.c.setup", "/")
+
+    def test_crawl_namespace_explicit_base(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+        options.explicit_package_bases = True
+
+        finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
+        assert finder.crawl_up("/setup.py") == ("setup", "/")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+
+        finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+
+        finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
+        assert finder.crawl_up("/a/b/setup.py") == ("a.b.setup", "/")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/b/c/setup.py") == ("a.b.c.setup", "/")
+
+        # set mypy path, so we actually have some explicit base dirs
+        options.mypy_path = ["/a/b"]
+
+        finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
+        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+
+        finder = SourceFinder(
+            FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
+            options,
+        )
+        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+
+        options.mypy_path = ["/a/b", "/a/b/c"]
+        finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
+        assert finder.crawl_up("/a/b/c/setup.py") == ("setup", "/a/b/c")
+
+    def test_find_sources_no_namespace(self) -> None:
+        options = Options()
+        options.namespace_packages = False
+
+        files = {
+            "/pkg/a1/b/c/d/e.py",
+            "/pkg/a1/b/f.py",
+            "/pkg/a2/__init__.py",
+            "/pkg/a2/b/c/d/e.py",
+            "/pkg/a2/b/f.py",
+        }
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+            ("a2", "/pkg"),
+            ("e", "/pkg/a1/b/c/d"),
+            ("e", "/pkg/a2/b/c/d"),
+            ("f", "/pkg/a1/b"),
+            ("f", "/pkg/a2/b"),
+        ]
+
+    def test_find_sources_namespace(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+
+        files = {
+            "/pkg/a1/b/c/d/e.py",
+            "/pkg/a1/b/f.py",
+            "/pkg/a2/__init__.py",
+            "/pkg/a2/b/c/d/e.py",
+            "/pkg/a2/b/f.py",
+        }
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+            ("a2", "/pkg"),
+            ("a2.b.c.d.e", "/pkg"),
+            ("a2.b.f", "/pkg"),
+            ("e", "/pkg/a1/b/c/d"),
+            ("f", "/pkg/a1/b"),
+        ]
+
+    def test_find_sources_namespace_explicit_base(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+        options.explicit_package_bases = True
+        options.mypy_path = ["/"]
+
+        files = {
+            "/pkg/a1/b/c/d/e.py",
+            "/pkg/a1/b/f.py",
+            "/pkg/a2/__init__.py",
+            "/pkg/a2/b/c/d/e.py",
+            "/pkg/a2/b/f.py",
+        }
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+            ("pkg.a1.b.c.d.e", "/"),
+            ("pkg.a1.b.f", "/"),
+            ("pkg.a2", "/"),
+            ("pkg.a2.b.c.d.e", "/"),
+            ("pkg.a2.b.f", "/"),
+        ]
+
+        options.mypy_path = ["/pkg"]
+        finder = SourceFinder(FakeFSCache(files), options)
+        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+            ("a1.b.c.d.e", "/pkg"),
+            ("a1.b.f", "/pkg"),
+            ("a2", "/pkg"),
+            ("a2.b.c.d.e", "/pkg"),
+            ("a2.b.f", "/pkg"),
+        ]

From f228956c1cf22aad2dac11b6859eecaaaa562a35 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 21 Nov 2020 21:59:53 -0800
Subject: [PATCH 03/14] fix package root validation

---
 mypy/main.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mypy/main.py b/mypy/main.py
index ddabfc0fbeb1..02d7dec148a5 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -974,7 +974,6 @@ def process_package_roots(fscache: Optional[FileSystemCache],
     assert fscache is not None  # Since mypy doesn't know parser.error() raises.
     # Do some stuff with drive letters to make Windows happy (esp. tests).
     current_drive, _ = os.path.splitdrive(os.getcwd())
-    dotdotslash = os.pardir + os.sep
     package_root = []
     for root in options.package_root:
         if os.path.isabs(root):
@@ -985,7 +984,7 @@ def process_package_roots(fscache: Optional[FileSystemCache],
         # Empty package root is always okay.
         if not root:
             root = os.curdir
-        if os.path.relpath(root).startswith(dotdotslash):
+        if os.path.relpath(root).split(os.sep)[0] == os.pardir:
             parser.error("Package root cannot be above current directory: %r" % root)
         root = os.path.normpath(os.path.abspath(root))
         if not root.endswith(os.sep):

From c3cfef22a35dc9f21defcd07098f743aa9f2cf8c Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 21 Nov 2020 22:43:28 -0800
Subject: [PATCH 04/14] fix test failing from absolute path change

---
 mypy/build.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/build.py b/mypy/build.py
index d70b421c380b..e6f597af31bc 100644
--- a/mypy/build.py
+++ b/mypy/build.py
@@ -2396,7 +2396,7 @@ def find_module_and_diagnose(manager: BuildManager,
                 and not options.use_builtins_fixtures
                 and not options.custom_typeshed_dir):
             raise CompileError([
-                'mypy: "%s" shadows library module "%s"' % (result, id),
+                'mypy: "%s" shadows library module "%s"' % (os.path.relpath(result), id),
                 'note: A user-defined top-level module with name "%s" is not supported' % id
             ])
         return (result, follow_imports)

From dec730ad1981d636bf9ba54f12c6169f69a3f1e8 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 21 Nov 2020 23:24:39 -0800
Subject: [PATCH 05/14] fix build source ordering

Apparently this is something mypy is sensitive to. Thanks mypy primer!
---
 mypy/find_sources.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index c09121cf3e32..995f97dcafe4 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -47,16 +47,18 @@ def create_source_list(paths: Sequence[str], options: Options,
     return sources
 
 
-def keyfunc(name: str) -> Tuple[int, str]:
+def keyfunc(name: str) -> Tuple[bool, int, str]:
     """Determines sort order for directory listing.
 
-    The desirable property is foo < foo.pyi < foo.py.
+    The desirable propertes are:
+    1) foo < foo.pyi < foo.py
+    2) __init__.py[i] < foo
     """
     base, suffix = os.path.splitext(name)
     for i, ext in enumerate(PY_EXTENSIONS):
         if suffix == ext:
-            return (i, base)
-    return (-1, name)
+            return (base != "__init__", i, base)
+    return (base != "__init__", -1, name)
 
 
 def normalise_package_base(root: str) -> str:

From dfaf104f019fbb3502234c9c86eb7a0cefa4b470 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sat, 21 Nov 2020 23:42:45 -0800
Subject: [PATCH 06/14] actually inherit from FileSystemCache to appease mypyc

This means we could accidentally fallback to calling a FileSystemCache
method if stuff gets moved around.
---
 mypy/fscache.py                | 2 ++
 mypy/test/test_find_sources.py | 8 +++-----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/mypy/fscache.py b/mypy/fscache.py
index 0677aaee7645..71fdd5d7381d 100644
--- a/mypy/fscache.py
+++ b/mypy/fscache.py
@@ -32,8 +32,10 @@
 import stat
 from typing import Dict, List, Set
 from mypy.util import hash_digest
+from mypy_extensions import mypyc_attr
 
 
+@mypyc_attr(allow_interpreted_subclasses=True)  # for tests
 class FileSystemCache:
     def __init__(self) -> None:
         # The package root is not flushed with the caches.
diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
index 2b6b0178c32e..8d16fcf925e5 100644
--- a/mypy/test/test_find_sources.py
+++ b/mypy/test/test_find_sources.py
@@ -1,13 +1,14 @@
 from mypy.modulefinder import BuildSource
 import os
-from typing import Any, List, Optional, Set, Tuple, cast
+from typing import List, Optional, Set, Tuple
 from unittest import TestCase
 from mypy.find_sources import SourceFinder
+from mypy.fscache import FileSystemCache
 from mypy.modulefinder import BuildSource
 from mypy.options import Options
 
 
-class _FakeFSCache:
+class FakeFSCache(FileSystemCache):
     def __init__(self, files: Set[str]) -> None:
         assert all(os.path.isabs(f) for f in files)
         self.files = files
@@ -29,9 +30,6 @@ def init_under_package_root(self, file: str) -> bool:
         return False
 
 
-FakeFSCache = cast(Any, _FakeFSCache)
-
-
 def normalise_build_source_list(sources: List[BuildSource]) -> List[Tuple[str, Optional[str]]]:
     return sorted((s.module, s.base_dir) for s in sources)
 

From 739f06778111c31bfd4d0a8f37dafe6d342665af Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 22 Nov 2020 00:17:46 -0800
Subject: [PATCH 07/14] don't change base dir if module name is invalid

mypy_primer points out two undesirable effects:
1) scripts causing search path confusion
2) scripts with the same names causing issues (e.g. migrations in zulip)
---
 mypy/find_sources.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index 995f97dcafe4..63744e8caee1 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -131,13 +131,13 @@ def crawl_up(self, path: str) -> Tuple[str, str]:
         parent, filename = os.path.split(path)
 
         module_name = strip_py(filename) or filename
-        if not module_name.isidentifier():
-            return module_name, parent
 
         parent_module, base_dir = self.crawl_up_dir(parent)
         if module_name == "__init__":
             return parent_module, base_dir
 
+        # Note that module_name might not actually be a valid identifier, but that's okay
+        # Ignoring this possibility sidesteps some search path confusion
         module = module_join(parent_module, module_name)
         return module, base_dir
 

From 3a762b0d9b53741aaa9b8b0f377ad206ebaf0316 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 22 Nov 2020 16:05:10 -0800
Subject: [PATCH 08/14] fix test find sources on windows

---
 mypy/find_sources.py           |  4 +-
 mypy/test/test_find_sources.py | 83 ++++++++++++++++++++--------------
 2 files changed, 52 insertions(+), 35 deletions(-)

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index 63744e8caee1..6249aa76c1ca 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -64,7 +64,7 @@ def keyfunc(name: str) -> Tuple[bool, int, str]:
 def normalise_package_base(root: str) -> str:
     if not root:
         root = os.curdir
-    root = os.path.normpath(os.path.abspath(root))
+    root = os.path.abspath(root)
     if root.endswith(os.sep):
         root = root[:-1]
     return root
@@ -127,7 +127,7 @@ def crawl_up(self, path: str) -> Tuple[str, str]:
         We won't crawl past directories with invalid package names.
         The base directory returned is an absolute path.
         """
-        path = os.path.normpath(os.path.abspath(path))
+        path = os.path.abspath(path)
         parent, filename = os.path.split(path)
 
         module_name = strip_py(filename) or filename
diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
index 8d16fcf925e5..67cd028c79d3 100644
--- a/mypy/test/test_find_sources.py
+++ b/mypy/test/test_find_sources.py
@@ -1,7 +1,7 @@
 from mypy.modulefinder import BuildSource
 import os
+import unittest
 from typing import List, Optional, Set, Tuple
-from unittest import TestCase
 from mypy.find_sources import SourceFinder
 from mypy.fscache import FileSystemCache
 from mypy.modulefinder import BuildSource
@@ -10,8 +10,7 @@
 
 class FakeFSCache(FileSystemCache):
     def __init__(self, files: Set[str]) -> None:
-        assert all(os.path.isabs(f) for f in files)
-        self.files = files
+        self.files = {os.path.abspath(f) for f in files}
 
     def isfile(self, file: str) -> bool:
         return file in self.files
@@ -30,72 +29,90 @@ def init_under_package_root(self, file: str) -> bool:
         return False
 
 
+def normalise_path(path: str) -> str:
+    path = os.path.splitdrive(path)[1]
+    path = path.replace(os.sep, "/")
+    return path
+
+
 def normalise_build_source_list(sources: List[BuildSource]) -> List[Tuple[str, Optional[str]]]:
-    return sorted((s.module, s.base_dir) for s in sources)
+    return sorted(
+        (s.module, normalise_path(s.base_dir) if s.base_dir is not None else None)
+        for s in sources
+    )
+
+
+def crawl(finder: SourceFinder, f: str) -> Tuple[str, str]:
+    module, base_dir = finder.crawl_up(f)
+    return module, normalise_path(base_dir)
+
+
+def find_sources(finder: SourceFinder, f: str) -> List[Tuple[str, Optional[str]]]:
+    return normalise_build_source_list(finder.find_sources_in_dir(os.path.abspath(f)))
 
 
-class SourceFinderSuite(TestCase):
+class SourceFinderSuite(unittest.TestCase):
     def test_crawl_no_namespace(self) -> None:
         options = Options()
         options.namespace_packages = False
 
         finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
-        assert finder.crawl_up("/setup.py") == ("setup", "/")
+        assert crawl(finder, "/setup.py") == ("setup", "/")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+        assert crawl(finder, "/a/setup.py") == ("setup", "/a")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+        assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+        assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
 
         finder = SourceFinder(
             FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+        assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+        assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
 
         finder = SourceFinder(
             FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+        assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
 
     def test_crawl_namespace(self) -> None:
         options = Options()
         options.namespace_packages = True
 
         finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
-        assert finder.crawl_up("/setup.py") == ("setup", "/")
+        assert crawl(finder, "/setup.py") == ("setup", "/")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+        assert crawl(finder, "/a/setup.py") == ("setup", "/a")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+        assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+        assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
 
         finder = SourceFinder(
             FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+        assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("a.b.setup", "/")
+        assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/")
 
         finder = SourceFinder(
             FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/b/c/setup.py") == ("a.b.c.setup", "/")
+        assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/")
 
     def test_crawl_namespace_explicit_base(self) -> None:
         options = Options()
@@ -103,47 +120,47 @@ def test_crawl_namespace_explicit_base(self) -> None:
         options.explicit_package_bases = True
 
         finder = SourceFinder(FakeFSCache({"/setup.py"}), options)
-        assert finder.crawl_up("/setup.py") == ("setup", "/")
+        assert crawl(finder, "/setup.py") == ("setup", "/")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("setup", "/a")
+        assert crawl(finder, "/a/setup.py") == ("setup", "/a")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("setup", "/a/b")
+        assert crawl(finder, "/a/b/setup.py") == ("setup", "/a/b")
 
         finder = SourceFinder(FakeFSCache({"/a/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/setup.py") == ("a.setup", "/")
+        assert crawl(finder, "/a/setup.py") == ("a.setup", "/")
 
         finder = SourceFinder(
             FakeFSCache({"/a/invalid-name/setup.py", "/a/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
+        assert crawl(finder, "/a/invalid-name/setup.py") == ("setup", "/a/invalid-name")
 
         finder = SourceFinder(FakeFSCache({"/a/b/setup.py", "/a/__init__.py"}), options)
-        assert finder.crawl_up("/a/b/setup.py") == ("a.b.setup", "/")
+        assert crawl(finder, "/a/b/setup.py") == ("a.b.setup", "/")
 
         finder = SourceFinder(
             FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/b/c/setup.py") == ("a.b.c.setup", "/")
+        assert crawl(finder, "/a/b/c/setup.py") == ("a.b.c.setup", "/")
 
         # set mypy path, so we actually have some explicit base dirs
         options.mypy_path = ["/a/b"]
 
         finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
-        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+        assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
 
         finder = SourceFinder(
             FakeFSCache({"/a/b/c/setup.py", "/a/__init__.py", "/a/b/c/__init__.py"}),
             options,
         )
-        assert finder.crawl_up("/a/b/c/setup.py") == ("c.setup", "/a/b")
+        assert crawl(finder, "/a/b/c/setup.py") == ("c.setup", "/a/b")
 
         options.mypy_path = ["/a/b", "/a/b/c"]
         finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
-        assert finder.crawl_up("/a/b/c/setup.py") == ("setup", "/a/b/c")
+        assert crawl(finder, "/a/b/c/setup.py") == ("setup", "/a/b/c")
 
     def test_find_sources_no_namespace(self) -> None:
         options = Options()
@@ -157,7 +174,7 @@ def test_find_sources_no_namespace(self) -> None:
             "/pkg/a2/b/f.py",
         }
         finder = SourceFinder(FakeFSCache(files), options)
-        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+        assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
             ("e", "/pkg/a1/b/c/d"),
             ("e", "/pkg/a2/b/c/d"),
@@ -177,7 +194,7 @@ def test_find_sources_namespace(self) -> None:
             "/pkg/a2/b/f.py",
         }
         finder = SourceFinder(FakeFSCache(files), options)
-        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+        assert find_sources(finder, "/") == [
             ("a2", "/pkg"),
             ("a2.b.c.d.e", "/pkg"),
             ("a2.b.f", "/pkg"),
@@ -199,7 +216,7 @@ def test_find_sources_namespace_explicit_base(self) -> None:
             "/pkg/a2/b/f.py",
         }
         finder = SourceFinder(FakeFSCache(files), options)
-        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+        assert find_sources(finder, "/") == [
             ("pkg.a1.b.c.d.e", "/"),
             ("pkg.a1.b.f", "/"),
             ("pkg.a2", "/"),
@@ -209,7 +226,7 @@ def test_find_sources_namespace_explicit_base(self) -> None:
 
         options.mypy_path = ["/pkg"]
         finder = SourceFinder(FakeFSCache(files), options)
-        assert normalise_build_source_list(finder.find_sources_in_dir("/")) == [
+        assert find_sources(finder, "/") == [
             ("a1.b.c.d.e", "/pkg"),
             ("a1.b.f", "/pkg"),
             ("a2", "/pkg"),

From 6b78138b04b30b32b2e1e64817ed6d58b0a31cdc Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Sun, 22 Nov 2020 16:13:37 -0800
Subject: [PATCH 09/14] make changes to package root logic more minimal

---
 mypy/fscache.py |  2 ++
 mypy/main.py    | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/mypy/fscache.py b/mypy/fscache.py
index 71fdd5d7381d..368120ea904e 100644
--- a/mypy/fscache.py
+++ b/mypy/fscache.py
@@ -114,6 +114,8 @@ def init_under_package_root(self, path: str) -> bool:
                 return False
         ok = False
         drive, path = os.path.splitdrive(path)  # Ignore Windows drive name
+        if os.path.isabs(path):
+            path = os.path.relpath(path)
         path = os.path.normpath(path)
         for root in self.package_root:
             if path.startswith(root):
diff --git a/mypy/main.py b/mypy/main.py
index 02d7dec148a5..efd356747271 100644
--- a/mypy/main.py
+++ b/mypy/main.py
@@ -974,6 +974,10 @@ def process_package_roots(fscache: Optional[FileSystemCache],
     assert fscache is not None  # Since mypy doesn't know parser.error() raises.
     # Do some stuff with drive letters to make Windows happy (esp. tests).
     current_drive, _ = os.path.splitdrive(os.getcwd())
+    dot = os.curdir
+    dotslash = os.curdir + os.sep
+    dotdotslash = os.pardir + os.sep
+    trivial_paths = {dot, dotslash}
     package_root = []
     for root in options.package_root:
         if os.path.isabs(root):
@@ -982,13 +986,14 @@ def process_package_roots(fscache: Optional[FileSystemCache],
         if drive and drive != current_drive:
             parser.error("Package root must be on current drive: %r" % (drive + root))
         # Empty package root is always okay.
-        if not root:
-            root = os.curdir
-        if os.path.relpath(root).split(os.sep)[0] == os.pardir:
-            parser.error("Package root cannot be above current directory: %r" % root)
-        root = os.path.normpath(os.path.abspath(root))
-        if not root.endswith(os.sep):
-            root += os.sep
+        if root:
+            root = os.path.relpath(root)  # Normalize the heck out of it.
+            if not root.endswith(os.sep):
+                root = root + os.sep
+            if root.startswith(dotdotslash):
+                parser.error("Package root cannot be above current directory: %r" % root)
+            if root in trivial_paths:
+                root = ''
         package_root.append(root)
     options.package_root = package_root
     # Pass the package root on the the filesystem cache.

From c07e26ec7ff283b649c1ccb615901d371a6c07d0 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 11 Dec 2020 11:23:12 -0800
Subject: [PATCH 10/14] [minor] fix typo

---
 mypy/find_sources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index 6249aa76c1ca..59614cb53c3e 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -50,7 +50,7 @@ def create_source_list(paths: Sequence[str], options: Options,
 def keyfunc(name: str) -> Tuple[bool, int, str]:
     """Determines sort order for directory listing.
 
-    The desirable propertes are:
+    The desirable properties are:
     1) foo < foo.pyi < foo.py
     2) __init__.py[i] < foo
     """

From bce2e077aeea18b67adcbd057f479b909e7b5926 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 11 Dec 2020 11:33:19 -0800
Subject: [PATCH 11/14] add docstring to get_explicit_package_bases

---
 mypy/find_sources.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/mypy/find_sources.py b/mypy/find_sources.py
index 59614cb53c3e..47d686cddcbc 100644
--- a/mypy/find_sources.py
+++ b/mypy/find_sources.py
@@ -71,6 +71,15 @@ def normalise_package_base(root: str) -> str:
 
 
 def get_explicit_package_bases(options: Options) -> Optional[List[str]]:
+    """Returns explicit package bases to use if the option is enabled, or None if disabled.
+
+    We currently use MYPYPATH and the current directory as the package bases. In the future,
+    when --namespace-packages is the default could also use the values passed with the
+    --package-root flag, see #9632.
+
+    Values returned are normalised so we can use simple string comparisons in
+    SourceFinder.is_explicit_package_base
+    """
     if not options.explicit_package_bases:
         return None
     roots = mypy_path() + options.mypy_path + [os.getcwd()]

From bcdcf06727a10c8a2290394c1452a844fefcc5d7 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 11 Dec 2020 11:49:28 -0800
Subject: [PATCH 12/14] options: document namespace_packages and
 explicit_package_bases

---
 mypy/options.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/mypy/options.py b/mypy/options.py
index fa8c3fca1e56..f1805bc292a7 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -87,7 +87,15 @@ def __init__(self) -> None:
         # Intended to be used for disabling specific stubs.
         self.follow_imports_for_stubs = False
         # PEP 420 namespace packages
+        # This allows definitions of packages without __init__.py and allows packages to span
+        # multiple directories. This flag affects both import discovery and the association of
+        # input files/modules/packages to the relevant file and fully qualified module name.
         self.namespace_packages = False
+        # Use current directory and MYPYPATH to determine fully qualified module names of files
+        # passed by automatically considering their subdirectories as packages. This is only
+        # relevant if namespace packages are enabled, since otherwise examining __init__.py's is
+        # sufficient to determine module names for files. As a possible alternative, add a single
+        # top-level __init__.py to your packages.
         self.explicit_package_bases = False
 
         # disallow_any options

From 73ae65b162d82c4f62c2d3f7aa0acb7f70646f75 Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 11 Dec 2020 11:54:15 -0800
Subject: [PATCH 13/14] [minor] fix style nit

---
 mypy/test/test_find_sources.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
index 67cd028c79d3..608b835b58a7 100644
--- a/mypy/test/test_find_sources.py
+++ b/mypy/test/test_find_sources.py
@@ -37,7 +37,7 @@ def normalise_path(path: str) -> str:
 
 def normalise_build_source_list(sources: List[BuildSource]) -> List[Tuple[str, Optional[str]]]:
     return sorted(
-        (s.module, normalise_path(s.base_dir) if s.base_dir is not None else None)
+        (s.module, (normalise_path(s.base_dir) if s.base_dir is not None else None))
         for s in sources
     )
 

From d6a53399ec91995746b62cfe491e3f44aee1e98f Mon Sep 17 00:00:00 2001
From: hauntsaninja <>
Date: Fri, 11 Dec 2020 12:00:32 -0800
Subject: [PATCH 14/14] test_find_sources: add multiple directory tests

---
 mypy/test/test_find_sources.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/mypy/test/test_find_sources.py b/mypy/test/test_find_sources.py
index 608b835b58a7..5cedec338bbc 100644
--- a/mypy/test/test_find_sources.py
+++ b/mypy/test/test_find_sources.py
@@ -162,6 +162,16 @@ def test_crawl_namespace_explicit_base(self) -> None:
         finder = SourceFinder(FakeFSCache({"/a/b/c/setup.py"}), options)
         assert crawl(finder, "/a/b/c/setup.py") == ("setup", "/a/b/c")
 
+    def test_crawl_namespace_multi_dir(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+        options.explicit_package_bases = True
+        options.mypy_path = ["/a", "/b"]
+
+        finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options)
+        assert crawl(finder, "/a/pkg/a.py") == ("pkg.a", "/a")
+        assert crawl(finder, "/b/pkg/b.py") == ("pkg.b", "/b")
+
     def test_find_sources_no_namespace(self) -> None:
         options = Options()
         options.namespace_packages = False
@@ -233,3 +243,12 @@ def test_find_sources_namespace_explicit_base(self) -> None:
             ("a2.b.c.d.e", "/pkg"),
             ("a2.b.f", "/pkg"),
         ]
+
+    def test_find_sources_namespace_multi_dir(self) -> None:
+        options = Options()
+        options.namespace_packages = True
+        options.explicit_package_bases = True
+        options.mypy_path = ["/a", "/b"]
+
+        finder = SourceFinder(FakeFSCache({"/a/pkg/a.py", "/b/pkg/b.py"}), options)
+        assert find_sources(finder, "/") == [("pkg.a", "/a"), ("pkg.b", "/b")]