From bf0626d774cc2bd78529f3164cf99c73e163a4a9 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 17 Aug 2016 18:35:59 -0700 Subject: [PATCH 1/2] Add more logging to find_cache_meta --- mypy/build.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 848d746f07da..02f6300630b0 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -703,12 +703,14 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache id, path, manager.options.cache_dir, manager.options.python_version) manager.trace('Looking for {} {}'.format(id, data_json)) if not os.path.exists(meta_json): + manager.trace('Could not load cache for {}: could not find {}'.format(id, meta_json)) return None with open(meta_json, 'r') as f: meta_str = f.read() manager.trace('Meta {} {}'.format(id, meta_str.rstrip())) meta = json.loads(meta_str) # TODO: Errors if not isinstance(meta, dict): + manager.trace('Could not load cache for {}: meta cache is not a dict'.format(id)) return None path = os.path.abspath(path) m = CacheMeta( @@ -728,32 +730,36 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> Optional[Cache if (m.id != id or m.path != path or m.mtime is None or m.size is None or m.dependencies is None or m.data_mtime is None): + manager.trace('Metadata abandoned for {}: attributes are missing'.format(id)) return None # Ignore cache if generated by an older mypy version. if (m.version_id != manager.version_id or m.options is None or len(m.dependencies) != len(m.dep_prios)): + manager.trace('Metadata abandoned for {}: new attributes are missing'.format(id)) return None # Ignore cache if (relevant) options aren't the same. cached_options = m.options current_options = select_options_affecting_cache(manager.options) if cached_options != current_options: + manager.trace('Metadata abandoned for {}: options differ'.format(id)) return None # TODO: Share stat() outcome with find_module() st = os.stat(path) # TODO: Errors if st.st_mtime != m.mtime or st.st_size != m.size: - manager.log('Metadata abandoned because of modified file {}'.format(path)) + manager.log('Metadata abandoned for {}: file {} is modified'.format(id, path)) return None # It's a match on (id, path, mtime, size). # Check data_json; assume if its mtime matches it's good. # TODO: stat() errors if os.path.getmtime(data_json) != m.data_mtime: + manager.log('Metadata abandoned for {}: data cache is modified'.format(id)) return None - manager.log('Found {} {}'.format(id, meta_json)) + manager.log('Found {} {} (metadata is fresh)'.format(id, meta_json)) return m From 8a5349c9200d3f90ae866c8424ef8ce3911da9d4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 17 Aug 2016 18:37:21 -0700 Subject: [PATCH 2/2] Fix silenced files being ignored on multiple runs This commit fixes a bug where silenced imports were being unsilenced after incremental mode + silent_imports was run several times after forcing an SCC to be rechecked. --- mypy/build.py | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 02f6300630b0..54268ad98626 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -48,6 +48,9 @@ PYTHON_EXTENSIONS = ['.pyi', '.py'] +Graph = Dict[str, 'State'] + + class BuildResult: """The result of a successful build. @@ -1206,6 +1209,40 @@ def fix_cross_refs(self) -> None: def calculate_mros(self) -> None: fixup_module_pass_two(self.tree, self.manager.modules) + def fix_suppressed_dependencies(self, graph: Graph) -> None: + """Corrects whether dependencies are considered stale or not when using silent_imports. + + This method is a hack to correct imports in silent_imports + incremental mode. + In particular, the problem is that when running mypy with a cold cache, the + `parse_file(...)` function is called *at the start* of the `load_graph(...)` function. + Note that load_graph will mark some dependencies as suppressed if they weren't specified + on the command line in silent_imports mode. + + However, if the interface for a module is changed, parse_file will be called within + `process_stale_scc` -- *after* load_graph is finished, wiping out the changes load_graph + previously made. + + This method is meant to be run after parse_file finishes in process_stale_scc and will + recompute what modules should be considered suppressed in silent_import mode. + """ + # TODO: See if it's possible to move this check directly into parse_file in some way. + # TODO: Find a way to write a test case for this fix. + silent_mode = self.manager.options.silent_imports or self.manager.options.almost_silent + if not silent_mode: + return + + new_suppressed = [] + new_dependencies = [] + entry_points = self.manager.source_set.source_modules + for dep in self.dependencies + self.suppressed: + ignored = dep in self.suppressed and dep not in entry_points + if ignored or dep not in graph: + new_suppressed.append(dep) + else: + new_dependencies.append(dep) + self.dependencies = new_dependencies + self.suppressed = new_suppressed + # Methods for processing modules from source code. def parse_file(self) -> None: @@ -1331,9 +1368,6 @@ def write_cache(self) -> None: self.manager) -Graph = Dict[str, State] - - def dispatch(sources: List[BuildSource], manager: BuildManager) -> None: manager.log("Mypy version %s" % __version__) graph = load_graph(sources, manager) @@ -1562,6 +1596,7 @@ def process_stale_scc(graph: Graph, scc: List[str]) -> None: # We may already have parsed the module, or not. # If the former, parse_file() is a no-op. graph[id].parse_file() + graph[id].fix_suppressed_dependencies(graph) for id in scc: graph[id].patch_parent() for id in scc: