Fix imports being unsilenced when checking stale SCCs

mypy/build.py

@@ -48,6 +48,9 @@
 PYTHON_EXTENSIONS = ['.pyi', '.py']
 
 
+Graph = Dict[str, 'State']
+
+
 class BuildResult:
     """The result of a successful build.
 
@@ -703,12 +706,14 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
         id, path, manager.options.cache_dir, manager.options.python_version)
     manager.trace('Looking for {} {}'.format(id, data_json))
     if not os.path.exists(meta_json):
+        manager.trace('Could not load cache for {}: could not find {}'.format(id, meta_json))
         return None
     with open(meta_json, 'r') as f:
         meta_str = f.read()
         manager.trace('Meta {} {}'.format(id, meta_str.rstrip()))
         meta = json.loads(meta_str)  # TODO: Errors
     if not isinstance(meta, dict):
+        manager.trace('Could not load cache for {}: meta cache is not a dict'.format(id))
         return None
     path = os.path.abspath(path)
     m = CacheMeta(
@@ -728,32 +733,36 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache
     if (m.id != id or m.path != path or
             m.mtime is None or m.size is None or
             m.dependencies is None or m.data_mtime is None):
+        manager.trace('Metadata abandoned for {}: attributes are missing'.format(id))
         return None
 
     # Ignore cache if generated by an older mypy version.
     if (m.version_id != manager.version_id
             or m.options is None
             or len(m.dependencies) != len(m.dep_prios)):
+        manager.trace('Metadata abandoned for {}: new attributes are missing'.format(id))
         return None
 
     # Ignore cache if (relevant) options aren't the same.
     cached_options = m.options
     current_options = select_options_affecting_cache(manager.options)
     if cached_options != current_options:
+        manager.trace('Metadata abandoned for {}: options differ'.format(id))
         return None
 
     # TODO: Share stat() outcome with find_module()
     st = os.stat(path)  # TODO: Errors
     if st.st_mtime != m.mtime or st.st_size != m.size:
-        manager.log('Metadata abandoned because of modified file {}'.format(path))
+        manager.log('Metadata abandoned for {}: file {} is modified'.format(id, path))
         return None
 
     # It's a match on (id, path, mtime, size).
     # Check data_json; assume if its mtime matches it's good.
     # TODO: stat() errors
     if os.path.getmtime(data_json) != m.data_mtime:
+        manager.log('Metadata abandoned for {}: data cache is modified'.format(id))
         return None
-    manager.log('Found {} {}'.format(id, meta_json))
+    manager.log('Found {} {} (metadata is fresh)'.format(id, meta_json))
     return m
 
 
@@ -1200,6 +1209,40 @@ def fix_cross_refs(self) -> None:
     def calculate_mros(self) -> None:
         fixup_module_pass_two(self.tree, self.manager.modules)
 
+    def fix_suppressed_dependencies(self, graph: Graph) -> None:
+        """Corrects whether dependencies are considered stale or not when using silent_imports.
+
+        This method is a hack to correct imports in silent_imports + incremental mode.
+        In particular, the problem is that when running mypy with a cold cache, the
+        `parse_file(...)` function is called *at the start* of the `load_graph(...)` function.
+        Note that load_graph will mark some dependencies as suppressed if they weren't specified
+        on the command line in silent_imports mode.
+
+        However, if the interface for a module is changed, parse_file will be called within
+        `process_stale_scc` -- *after* load_graph is finished, wiping out the changes load_graph
+        previously made.
+
+        This method is meant to be run after parse_file finishes in process_stale_scc and will
+        recompute what modules should be considered suppressed in silent_import mode.
+        """
+        # TODO: See if it's possible to move this check directly into parse_file in some way.
+        # TODO: Find a way to write a test case for this fix.
+        silent_mode = self.manager.options.silent_imports or self.manager.options.almost_silent
+        if not silent_mode:
+            return
+
+        new_suppressed = []
+        new_dependencies = []
+        entry_points = self.manager.source_set.source_modules
+        for dep in self.dependencies + self.suppressed:
+            ignored = dep in self.suppressed and dep not in entry_points
+            if ignored or dep not in graph:
+                new_suppressed.append(dep)
+            else:
+                new_dependencies.append(dep)
+        self.dependencies = new_dependencies
+        self.suppressed = new_suppressed
+
     # Methods for processing modules from source code.
 
     def parse_file(self) -> None:
@@ -1325,9 +1368,6 @@ def write_cache(self) -> None:
                         self.manager)
 
 
-Graph = Dict[str, State]
-
-
 def dispatch(sources: List[BuildSource], manager: BuildManager) -> None:
     manager.log("Mypy version %s" % __version__)
     graph = load_graph(sources, manager)
@@ -1556,6 +1596,7 @@ def process_stale_scc(graph: Graph, scc: List[str]) -> None:
         # We may already have parsed the module, or not.
         # If the former, parse_file() is a no-op.
         graph[id].parse_file()
+        graph[id].fix_suppressed_dependencies(graph)
     for id in scc:
         graph[id].patch_parent()
     for id in scc: