Namespaces refactor part 2 (#5690)

* Get rid of trivial type_check_only() function

* Remove bin_dir -- it's no longer used

* Fix wrong import in testcheck.py

* Fix indent of compute_search_paths() args

* Minor cleanup in mypy/test/testcheck.py

* Cleanups in mypy/dmypy_server.py and mypy/server/

* Fix lint

* Fix lint fix

mypy/build.py

@@ -122,7 +122,6 @@ def is_source(self, file: MypyFile) -> bool:
 def build(sources: List[BuildSource],
  options: Options,
  alt_lib_path: Optional[str] = None,
- bin_dir: Optional[str] = None,
  flush_errors: Optional[Callable[[List[str], bool], None]] = None,
  fscache: Optional[FileSystemCache] = None,
  ) -> BuildResult:
@@ -144,8 +143,6 @@ def build(sources: List[BuildSource],
  options: build options
  alt_lib_path: an additional directory for looking up library modules
  (takes precedence over other directories)
- bin_dir: directory containing the mypy script, used for finding data
- directories; if omitted, use '.' as the data directory
  flush_errors: optional function to flush errors after a file is processed
  fscache: optionally a file-system cacher
 
@@ -160,8 +157,7 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None:
  flush_errors = flush_errors or default_flush_errors
 
  try:
- result = _build(sources, options, alt_lib_path, bin_dir,
- flush_errors, fscache)
+ result = _build(sources, options, alt_lib_path, flush_errors, fscache)
  result.errors = messages
  return result
  except CompileError as e:
@@ -178,7 +174,6 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None:
 def _build(sources: List[BuildSource],
  options: Options,
  alt_lib_path: Optional[str],
- bin_dir: Optional[str],
  flush_errors: Callable[[List[str], bool], None],
  fscache: Optional[FileSystemCache],
  ) -> BuildResult:
@@ -424,7 +419,7 @@ class BuildManager(BuildManagerBase):
 
  Attributes:
  data_dir: Mypy data directory (contains stubs)
- lib_path:  Library path for looking up modules
+ search_paths: SearchPaths instance indicating where to look for modules
  modules: Mapping of module ID to MypyFile (shared by the passes)
  semantic_analyzer:
  Semantic analyzer, pass 2

mypy/dmypy_server.py

@@ -24,6 +24,7 @@
 from mypy.dmypy_util import STATUS_FILE, receive
 from mypy.fscache import FileSystemCache
 from mypy.fswatcher import FileSystemWatcher, FileData
+from mypy.modulefinder import BuildSource, compute_search_paths
 from mypy.options import Options
 from mypy.typestate import reset_global_state
 from mypy.version import __version__
@@ -231,7 +232,7 @@ def cmd_stop(self) -> Dict[str, object]:
  """Stop daemon."""
  return {}
 
- last_sources = None # type: List[mypy.build.BuildSource]
+ last_sources = None # type: List[BuildSource]
 
  def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]:
  """Check a list of files, triggering a restart if needed."""
@@ -264,7 +265,7 @@ def cmd_recheck(self) -> Dict[str, object]:
  return {'error': "Command 'recheck' is only valid after a 'check' command"}
  return self.check(self.last_sources)
 
- def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+ def check(self, sources: List[BuildSource]) -> Dict[str, Any]:
  """Check using fine-grained incremental mode."""
  if not self.fine_grained_manager:
  res = self.initialize_fine_grained(sources)
@@ -273,7 +274,7 @@ def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
  self.fscache.flush()
  return res
 
- def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+ def initialize_fine_grained(self, sources: List[BuildSource]) -> Dict[str, Any]:
  self.fswatcher = FileSystemWatcher(self.fscache)
  self.update_sources(sources)
  try:
@@ -326,15 +327,16 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
  status = 1 if messages else 0
  return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
- def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+ def fine_grained_increment(self, sources: List[BuildSource]) -> Dict[str, Any]:
  assert self.fine_grained_manager is not None
  manager = self.fine_grained_manager.manager
 
  t0 = time.time()
  self.update_sources(sources)
  changed, removed = self.find_changed(sources)
- manager.search_paths = mypy.build.compute_search_paths(
- sources, manager.options, manager.data_dir, mypy.build.FileSystemCache())
+ # TODO: Why create a new FileSystemCache rather than using self.fscache?
+ manager.search_paths = compute_search_paths(
+ sources, manager.options, manager.data_dir, FileSystemCache())
  t1 = time.time()
  messages = self.fine_grained_manager.update(changed, removed)
  t2 = time.time()
@@ -345,12 +347,12 @@ def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[
  self.previous_sources = sources
  return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
- def update_sources(self, sources: List[mypy.build.BuildSource]) -> None:
+ def update_sources(self, sources: List[BuildSource]) -> None:
  paths = [source.path for source in sources if source.path is not None]
  self.fswatcher.add_watched_paths(paths)
 
- def find_changed(self, sources: List[mypy.build.BuildSource]) -> Tuple[List[Tuple[str, str]],
-  List[Tuple[str, str]]]:
+ def find_changed(self, sources: List[BuildSource]) -> Tuple[List[Tuple[str, str]],
+ List[Tuple[str, str]]]:
  changed_paths = self.fswatcher.find_changed()
  # Find anything that has been added or modified
  changed = [(source.module, source.path)

mypy/main.py

@@ -65,10 +65,6 @@ def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None:
 
  t0 = time.time()
  # To log stat() calls: os.stat = stat_proxy
- if script_path:
- bin_dir = find_bin_directory(script_path) # type: Optional[str]
- else:
- bin_dir = None
  sys.setrecursionlimit(2 ** 14)
  if args is None:
  args = sys.argv[1:]
@@ -90,10 +86,11 @@ def flush_errors(new_messages: List[str], serious: bool) -> None:
 
  serious = False
  blockers = False
+ res = None
  try:
  # Keep a dummy reference (res) for memory profiling below, as otherwise
  # the result could be freed.
- res = type_check_only(sources, bin_dir, options, flush_errors, fscache) # noqa
+ res = build.build(sources, options, None, flush_errors, fscache)
  except CompileError as e:
  blockers = True
  if not e.use_stdout:
@@ -111,6 +108,7 @@ def flush_errors(new_messages: List[str], serious: bool) -> None:
  if MEM_PROFILE:
  from mypy.memprofile import print_memory_profile
  print_memory_profile()
+ del res # Now it's safe to delete
 
  code = 0
  if messages:
@@ -124,20 +122,6 @@ def flush_errors(new_messages: List[str], serious: bool) -> None:
  sys.exit(code)
 
 
-def find_bin_directory(script_path: str) -> str:
- """Find the directory that contains this script.
-
- This is used by build to find stubs and other data files.
- """
- # Follow up to 5 symbolic links (cap to avoid cycles).
- for i in range(5):
- if os.path.islink(script_path):
- script_path = readlinkabs(script_path)
- else:
- break
- return os.path.dirname(script_path)
-
-
 def readlinkabs(link: str) -> str:
  """Return an absolute path to symbolic link destination."""
  # Adapted from code by Greg Smith.
@@ -148,18 +132,6 @@ def readlinkabs(link: str) -> str:
  return os.path.join(os.path.dirname(link), path)
 
 
-def type_check_only(sources: List[BuildSource], bin_dir: Optional[str],
- options: Options,
- flush_errors: Optional[Callable[[List[str], bool], None]],
- fscache: FileSystemCache) -> BuildResult:
- # Type-check the program and dependencies.
- return build.build(sources=sources,
- bin_dir=bin_dir,
- options=options,
- flush_errors=flush_errors,
- fscache=fscache)
-
-
 class SplitNamespace(argparse.Namespace):
  def __init__(self, standard_namespace: object, alt_namespace: object, alt_prefix: str) -> None:
  self.__dict__['_standard_namespace'] = standard_namespace

mypy/modulefinder.py

@@ -320,10 +320,10 @@ def make_abspath(path: str, root: str) -> str:
 
 
 def compute_search_paths(sources: List[BuildSource],
- options: Options,
- data_dir: str,
- fscache: FileSystemCache,
- alt_lib_path: Optional[str] = None) -> SearchPaths:
+  options: Options,
+  data_dir: str,
+  fscache: FileSystemCache,
+  alt_lib_path: Optional[str] = None) -> SearchPaths:
  """Compute the search paths as specified in PEP 561.
 
  There are the following 4 members created:

mypy/server/update.py

@@ -119,9 +119,10 @@
 )
 
 from mypy.build import (
- BuildManager, State, BuildSource, BuildResult, Graph, load_graph,
+ BuildManager, State, BuildResult, Graph, load_graph,
  process_fresh_modules, DEBUG_FINE_GRAINED,
 )
+from mypy.modulefinder import BuildSource
 from mypy.checker import FineGrainedDeferredNode
 from mypy.errors import CompileError
 from mypy.nodes import (

mypy/test/testcheck.py

@@ -8,7 +8,7 @@
 
 from mypy import build
 from mypy.build import Graph
-from mypy.modulefinder import BuildSource, SearchPaths
+from mypy.modulefinder import BuildSource, SearchPaths, FindModuleCache
 from mypy.test.config import test_temp_dir, test_data_prefix
 from mypy.test.data import DataDrivenTestCase, DataSuite, FileOperation, UpdateFile
 from mypy.test.helpers import (
@@ -271,13 +271,14 @@ def parse_module(self,
  Return a list of tuples (module name, file name, program text).
  """
  m = re.search('# cmd: mypy -m ([a-zA-Z0-9_. ]+)$', program_text, flags=re.MULTILINE)
- regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step)
- alt_m = re.search(regex, program_text, flags=re.MULTILINE)
- if alt_m is not None and incremental_step > 1:
- # Optionally return a different command if in a later step
- # of incremental mode, otherwise default to reusing the
- # original cmd.
- m = alt_m
+ if incremental_step > 1:
+ alt_regex = '# cmd{}: mypy -m ([a-zA-Z0-9_. ]+)$'.format(incremental_step)
+ alt_m = re.search(alt_regex, program_text, flags=re.MULTILINE)
+ if alt_m is not None:
+ # Optionally return a different command if in a later step
+ # of incremental mode, otherwise default to reusing the
+ # original cmd.
+ m = alt_m
 
  if m:
  # The test case wants to use a non-default main
@@ -286,8 +287,9 @@ def parse_module(self,
  module_names = m.group(1)
  out = []
  search_paths = SearchPaths((test_temp_dir,), (), (), ())
+ cache = FindModuleCache(search_paths)
  for module_name in module_names.split(' '):
- path = build.FindModuleCache(search_paths).find_module(module_name)
+ path = cache.find_module(module_name)
  assert path is not None, "Can't find ad hoc case file"
  with open(path) as f:
  program_text = f.read()