From 23c370cff409d4e32b631edc126aec79c5470ca4 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 28 Sep 2018 20:38:53 -0700 Subject: [PATCH] Namespaces refactor part 2 (#5690) * Get rid of trivial type_check_only() function * Remove bin_dir -- it's no longer used * Fix wrong import in testcheck.py * Fix indent of compute_search_paths() args * Minor cleanup in mypy/test/testcheck.py * Cleanups in mypy/dmypy_server.py and mypy/server/ * Fix lint * Fix lint fix --- mypy/build.py | 9 ++------- mypy/dmypy_server.py | 20 +++++++++++--------- mypy/main.py | 34 +++------------------------------- mypy/modulefinder.py | 8 ++++---- mypy/server/update.py | 3 ++- mypy/test/testcheck.py | 20 +++++++++++--------- 6 files changed, 33 insertions(+), 61 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 748735b9dd38..21b76956d2f1 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -122,7 +122,6 @@ def is_source(self, file: MypyFile) -> bool: def build(sources: List[BuildSource], options: Options, alt_lib_path: Optional[str] = None, - bin_dir: Optional[str] = None, flush_errors: Optional[Callable[[List[str], bool], None]] = None, fscache: Optional[FileSystemCache] = None, ) -> BuildResult: @@ -144,8 +143,6 @@ def build(sources: List[BuildSource], options: build options alt_lib_path: an additional directory for looking up library modules (takes precedence over other directories) - bin_dir: directory containing the mypy script, used for finding data - directories; if omitted, use '.' as the data directory flush_errors: optional function to flush errors after a file is processed fscache: optionally a file-system cacher @@ -160,8 +157,7 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: flush_errors = flush_errors or default_flush_errors try: - result = _build(sources, options, alt_lib_path, bin_dir, - flush_errors, fscache) + result = _build(sources, options, alt_lib_path, flush_errors, fscache) result.errors = messages return result except CompileError as e: @@ -178,7 +174,6 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: def _build(sources: List[BuildSource], options: Options, alt_lib_path: Optional[str], - bin_dir: Optional[str], flush_errors: Callable[[List[str], bool], None], fscache: Optional[FileSystemCache], ) -> BuildResult: @@ -424,7 +419,7 @@ class BuildManager(BuildManagerBase): Attributes: data_dir: Mypy data directory (contains stubs) - lib_path: Library path for looking up modules + search_paths: SearchPaths instance indicating where to look for modules modules: Mapping of module ID to MypyFile (shared by the passes) semantic_analyzer: Semantic analyzer, pass 2 diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 95a2d0dda068..21112a1daaf8 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -24,6 +24,7 @@ from mypy.dmypy_util import STATUS_FILE, receive from mypy.fscache import FileSystemCache from mypy.fswatcher import FileSystemWatcher, FileData +from mypy.modulefinder import BuildSource, compute_search_paths from mypy.options import Options from mypy.typestate import reset_global_state from mypy.version import __version__ @@ -231,7 +232,7 @@ def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" return {} - last_sources = None # type: List[mypy.build.BuildSource] + last_sources = None # type: List[BuildSource] def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" @@ -264,7 +265,7 @@ def cmd_recheck(self) -> Dict[str, object]: return {'error': "Command 'recheck' is only valid after a 'check' command"} return self.check(self.last_sources) - def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: + def check(self, sources: List[BuildSource]) -> Dict[str, Any]: """Check using fine-grained incremental mode.""" if not self.fine_grained_manager: res = self.initialize_fine_grained(sources) @@ -273,7 +274,7 @@ def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: self.fscache.flush() return res - def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: + def initialize_fine_grained(self, sources: List[BuildSource]) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) self.update_sources(sources) try: @@ -326,15 +327,16 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict status = 1 if messages else 0 return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} - def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: + def fine_grained_increment(self, sources: List[BuildSource]) -> Dict[str, Any]: assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() self.update_sources(sources) changed, removed = self.find_changed(sources) - manager.search_paths = mypy.build.compute_search_paths( - sources, manager.options, manager.data_dir, mypy.build.FileSystemCache()) + # TODO: Why create a new FileSystemCache rather than using self.fscache? + manager.search_paths = compute_search_paths( + sources, manager.options, manager.data_dir, FileSystemCache()) t1 = time.time() messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() @@ -345,12 +347,12 @@ def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[ self.previous_sources = sources return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} - def update_sources(self, sources: List[mypy.build.BuildSource]) -> None: + def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] self.fswatcher.add_watched_paths(paths) - def find_changed(self, sources: List[mypy.build.BuildSource]) -> Tuple[List[Tuple[str, str]], - List[Tuple[str, str]]]: + def find_changed(self, sources: List[BuildSource]) -> Tuple[List[Tuple[str, str]], + List[Tuple[str, str]]]: changed_paths = self.fswatcher.find_changed() # Find anything that has been added or modified changed = [(source.module, source.path) diff --git a/mypy/main.py b/mypy/main.py index 951c9ce5ef0a..ff8a330fdd6f 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -65,10 +65,6 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: t0 = time.time() # To log stat() calls: os.stat = stat_proxy - if script_path: - bin_dir = find_bin_directory(script_path) # type: Optional[str] - else: - bin_dir = None sys.setrecursionlimit(2 ** 14) if args is None: args = sys.argv[1:] @@ -90,10 +86,11 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: serious = False blockers = False + res = None try: # Keep a dummy reference (res) for memory profiling below, as otherwise # the result could be freed. - res = type_check_only(sources, bin_dir, options, flush_errors, fscache) # noqa + res = build.build(sources, options, None, flush_errors, fscache) except CompileError as e: blockers = True if not e.use_stdout: @@ -111,6 +108,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile() + del res # Now it's safe to delete code = 0 if messages: @@ -124,20 +122,6 @@ def flush_errors(new_messages: List[str], serious: bool) -> None: sys.exit(code) -def find_bin_directory(script_path: str) -> str: - """Find the directory that contains this script. - - This is used by build to find stubs and other data files. - """ - # Follow up to 5 symbolic links (cap to avoid cycles). - for i in range(5): - if os.path.islink(script_path): - script_path = readlinkabs(script_path) - else: - break - return os.path.dirname(script_path) - - def readlinkabs(link: str) -> str: """Return an absolute path to symbolic link destination.""" # Adapted from code by Greg Smith. @@ -148,18 +132,6 @@ def readlinkabs(link: str) -> str: return os.path.join(os.path.dirname(link), path) -def type_check_only(sources: List[BuildSource], bin_dir: Optional[str], - options: Options, - flush_errors: Optional[Callable[[List[str], bool], None]], - fscache: FileSystemCache) -> BuildResult: - # Type-check the program and dependencies. - return build.build(sources=sources, - bin_dir=bin_dir, - options=options, - flush_errors=flush_errors, - fscache=fscache) - - class SplitNamespace(argparse.Namespace): def __init__(self, standard_namespace: object, alt_namespace: object, alt_prefix: str) -> None: self.__dict__['_standard_namespace'] = standard_namespace diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index d98bea3c2c83..b9b79e3b4f09 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -320,10 +320,10 @@ def make_abspath(path: str, root: str) -> str: def compute_search_paths(sources: List[BuildSource], - options: Options, - data_dir: str, - fscache: FileSystemCache, - alt_lib_path: Optional[str] = None) -> SearchPaths: + options: Options, + data_dir: str, + fscache: FileSystemCache, + alt_lib_path: Optional[str] = None) -> SearchPaths: """Compute the search paths as specified in PEP 561. There are the following 4 members created: diff --git a/mypy/server/update.py b/mypy/server/update.py index 7943ffaee95b..a1b095679e5a 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -119,9 +119,10 @@ ) from mypy.build import ( - BuildManager, State, BuildSource, BuildResult, Graph, load_graph, + BuildManager, State, BuildResult, Graph, load_graph, process_fresh_modules, DEBUG_FINE_GRAINED, ) +from mypy.modulefinder import BuildSource from mypy.checker import FineGrainedDeferredNode from mypy.errors import CompileError from mypy.nodes import ( diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 7869ec07d7c3..c44472b889c1 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -8,7 +8,7 @@ from mypy import build from mypy.build import Graph -from mypy.modulefinder import BuildSource, SearchPaths +from mypy.modulefinder import BuildSource, SearchPaths, FindModuleCache from mypy.test.config import test_temp_dir, test_data_prefix from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, UpdateFile from mypy.test.helpers import ( @@ -271,13 +271,14 @@ def parse_module(self, Return a list of tuples (module name, file name, program text). """ m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE) - regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step) - alt_m = re.search(regex, program_text, flags=re.MULTILINE) - if alt_m is not None and incremental_step > 1: - # Optionally return a different command if in a later step - # of incremental mode, otherwise default to reusing the - # original cmd. - m = alt_m + if incremental_step > 1: + alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step) + alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE) + if alt_m is not None: + # Optionally return a different command if in a later step + # of incremental mode, otherwise default to reusing the + # original cmd. + m = alt_m if m: # The test case wants to use a non-default main @@ -286,8 +287,9 @@ def parse_module(self, module_names = m.group(1) out = [] search_paths = SearchPaths((test_temp_dir,), (), (), ()) + cache = FindModuleCache(search_paths) for module_name in module_names.split(' '): - path = build.FindModuleCache(search_paths).find_module(module_name) + path = cache.find_module(module_name) assert path is not None, "Can't find ad hoc case file" with open(path) as f: program_text = f.read()