Various fixes to the fine-grained incremental mode (#4438)

1) Fixes to refreshing class definitions

Bundling these in a single commit since a single test case reproduces
both of these issues and initially it wasn't clear if they are a single
issue.

1. Fix AST merge of ClassDefs.
2. If there is a previous error in a class body, refresh module top
   level instead of the whole class.
3. Generally fix the previous target handling in case of nested
   definitions (full test coverage missing).

2) Fix stripping of type alias in class body

3) Bind class type variables when refreshing a method

4) Fix stripping of super expressions

Not sure if this will make a difference anywhere, but this seems
like the right thing to do.

5) Reset subtype caches when stripping the AST

6) Merge type variable definitions

7) Fix refresh of a named tuple subclass

8) Fix refresh of NewType expressions

9) Fixes to stale TypeInfos escaping from AST merge

I didn't add tests since it's unclear if these actually caused any
visible issues.

10) Fix up additional references in AST merge

Not sure if these changes fix user-visible issues, but this at least
makes things cleaner.

11) Fix up another TypeInfo reference in AST merge

Not sure if this fixes an user-visible issue, but at least this
makes things more consistent.

12) Fix AST diff of functional enums

13) Fix refresh of functional Enum definition

mypy/errors.py

@@ -134,10 +134,20 @@ class Errors:
  # Set to True to show column numbers in error messages.
  show_column_numbers = False # type: bool
 
- # Stack of active fine-grained incremental checking targets within
- # a module. The first item is always the current module id.
+ # State for keeping track of the current fine-grained incremental mode target.
  # (See mypy.server.update for more about targets.)
- target = None # type: List[str]
+ #
+ # Current module id.
+ target_module = None # type: Optional[str]
+ # Partially qualified name of target class; without module prefix (examples: 'C' is top-level,
+ # 'C.D' nested).
+ target_class = None # type: Optional[str]
+ # Short name of the outermost function/method.
+ target_function = None # type: Optional[str]
+ # Nesting depth of functions/classes within the outermost function/method. These aren't
+ # separate targets and they are included in the surrounding function, but we this counter
+ # for internal bookkeeping.
+ target_ignore_depth = 0
 
  def __init__(self, show_error_context: bool = False,
  show_column_numbers: bool = False) -> None:
@@ -155,7 +165,10 @@ def initialize(self) -> None:
  self.used_ignored_lines = defaultdict(set)
  self.ignored_files = set()
  self.only_once_messages = set()
- self.target = []
+ self.target_module = None
+ self.target_class = None
+ self.target_function = None
+ self.target_ignore_depth = 0
 
  def reset(self) -> None:
  self.initialize()
@@ -166,7 +179,10 @@ def copy(self) -> 'Errors':
  new.import_ctx = self.import_ctx[:]
  new.type_name = self.type_name[:]
  new.function_or_member = self.function_or_member[:]
- new.target = self.target[:]
+ new.target_module = self.target_module
+ new.target_class = self.target_class
+ new.target_function = self.target_function
+ new.target_ignore_depth = self.target_ignore_depth
  return new
 
  def set_ignore_prefix(self, prefix: str) -> None:
@@ -191,8 +207,10 @@ def set_file(self, file: str,
  # reporting errors for files other than the one currently being
  # processed.
  self.file = file
- if module:
- self.target = [module]
+ self.target_module = module
+ self.target_class = None
+ self.target_function = None
+ self.target_ignore_depth = 0
 
  def set_file_ignored_lines(self, file: str,
  ignored_lines: Set[int],
@@ -203,12 +221,18 @@ def set_file_ignored_lines(self, file: str,
 
  def push_function(self, name: str) -> None:
  """Set the current function or member short name (it can be None)."""
- self.push_target_component(name)
+ if self.target_function is None:
+ self.target_function = name
+ else:
+ self.target_ignore_depth += 1
  self.function_or_member.append(name)
 
  def pop_function(self) -> None:
  self.function_or_member.pop()
- self.pop_target_component()
+ if self.target_ignore_depth > 0:
+ self.target_ignore_depth -= 1
+ else:
+ self.target_function = None
 
  @contextmanager
  def enter_function(self, name: str) -> Iterator[None]:
@@ -218,30 +242,41 @@ def enter_function(self, name: str) -> Iterator[None]:
 
  def push_type(self, name: str) -> None:
  """Set the short name of the current type (it can be None)."""
- self.push_target_component(name)
+ if self.target_function is not None:
+ self.target_ignore_depth += 1
+ elif self.target_class is None:
+ self.target_class = name
+ else:
+ self.target_class += '.' + name
  self.type_name.append(name)
 
  def pop_type(self) -> None:
  self.type_name.pop()
- self.pop_target_component()
-
- def push_target_component(self, name: str) -> None:
- if self.target and not self.function_or_member[-1]:
- self.target.append('{}.{}'.format(self.target[-1], name))
-
- def pop_target_component(self) -> None:
- if self.target and not self.function_or_member[-1]:
- self.target.pop()
+ if self.target_ignore_depth > 0:
+ self.target_ignore_depth -= 1
+ else:
+ assert self.target_class is not None
+ if '.' in self.target_class:
+ self.target_class = '.'.join(self.target_class.split('.')[:-1])
+ else:
+ self.target_class = None
 
  def current_target(self) -> Optional[str]:
- if self.target:
- return self.target[-1]
- return None
+ if self.target_module is None:
+ return None
+ target = self.target_module
+ if self.target_function is not None:
+ # Only include class name if we are inside a method, since a class
+ # target also includes all methods, which is not what we want
+ # here. Instead, the current target for a class body is the
+ # enclosing module top level.
+ if self.target_class is not None:
+ target += '.' + self.target_class
+ target += '.' + self.target_function
+ return target
 
  def current_module(self) -> Optional[str]:
- if self.target:
- return self.target[0]
- return None
+ return self.target_module
 
  @contextmanager
  def enter_type(self, name: str) -> Iterator[None]:

mypy/nodes.py

@@ -324,6 +324,7 @@ class FuncBase(Node):
  # Original, not semantically analyzed type (used for reprocessing)
  unanalyzed_type = None # type: Optional[mypy.types.Type]
  # If method, reference to TypeInfo
+ # TODO: The type should be Optional[TypeInfo]
  info = None # type: TypeInfo
  is_property = False
  _fullname = None # type: str # Name with module prefix
@@ -597,6 +598,7 @@ class Var(SymbolNode):
 
  _name = None # type: str # Name without module prefix
  _fullname = None # type: str # Name with module prefix
+ # TODO: The following should be Optional[TypeInfo]
  info = None # type: TypeInfo # Defining class (for member variables)
  type = None # type: Optional[mypy.types.Type] # Declared or inferred type, or None
  # Is this the first argument to an ordinary method (usually "self")?
@@ -1468,7 +1470,7 @@ class SuperExpr(Expression):
  """Expression super().name"""
 
  name = ''
- info = None # type: TypeInfo # Type that contains this super expression
+ info = None # type: Optional[TypeInfo] # Type that contains this super expression
  call = None # type: CallExpr # The expression super(...)
 
  def __init__(self, name: str, call: CallExpr) -> None:

mypy/semanal.py

@@ -201,8 +201,6 @@ class SemanticAnalyzerPass2(NodeVisitor[None], SemanticAnalyzerPluginInterface):
  type = None # type: Optional[TypeInfo]
  # Stack of outer classes (the second tuple item contains tvars).
  type_stack = None # type: List[Optional[TypeInfo]]
- # Type variables that are bound by the directly enclosing class
- bound_tvars = None # type: List[SymbolTableNode]
  # Type variables bound by the current scope, be it class or function
  tvar_scope = None # type: TypeVarScope
  # Per-module options
@@ -332,9 +330,11 @@ def file_context(self, file_node: MypyFile, fnam: str, options: Options,
  self.is_stub_file = fnam.lower().endswith('.pyi')
  self.is_typeshed_stub_file = self.errors.is_typeshed_file(file_node.path)
  self.globals = file_node.names
+ self.tvar_scope = TypeVarScope()
  if active_type:
  self.enter_class(active_type.defn.info)
- # TODO: Bind class type vars
+ for tvar in active_type.defn.type_vars:
+ self.tvar_scope.bind_existing(tvar)
 
  yield
 
@@ -909,7 +909,8 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None:
  del defn.base_type_exprs[i]
  tvar_defs = [] # type: List[TypeVarDef]
  for name, tvar_expr in declared_tvars:
- tvar_defs.append(self.tvar_scope.bind(name, tvar_expr))
+ tvar_def = self.tvar_scope.bind_new(name, tvar_expr)
+ tvar_defs.append(tvar_def)
  defn.type_vars = tvar_defs
 
  def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]:
@@ -2844,6 +2845,7 @@ def build_enum_call_typeinfo(self, name: str, items: List[str], fullname: str) -
  var = Var(item)
  var.info = info
  var.is_property = True
+ var._fullname = '{}.{}'.format(self.qualified_name(name), item)
  info.names[item] = SymbolTableNode(MDEF, var)
  return info
 

mypy/server/astmerge.py

@@ -49,13 +49,14 @@
 
 from mypy.nodes import (
  Node, MypyFile, SymbolTable, Block, AssignmentStmt, NameExpr, MemberExpr, RefExpr, TypeInfo,
- FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, MDEF
+ FuncDef, ClassDef, NamedTupleExpr, SymbolNode, Var, Statement, SuperExpr, NewTypeExpr,
+ OverloadedFuncDef, LambdaExpr, TypedDictExpr, EnumCallExpr, MDEF
 )
 from mypy.traverser import TraverserVisitor
 from mypy.types import (
  Type, TypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType,
  TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType,
- Overloaded
+ Overloaded, TypeVarDef
 )
 from mypy.util import get_prefix
 
@@ -157,14 +158,29 @@ def visit_func_def(self, node: FuncDef) -> None:
  node = self.fixup(node)
  if node.type:
  self.fixup_type(node.type)
+ if node.info:
+ node.info = self.fixup(node.info)
  super().visit_func_def(node)
 
+ def visit_overloaded_func_def(self, node: OverloadedFuncDef) -> None:
+ if node.info:
+ node.info = self.fixup(node.info)
+ super().visit_overloaded_func_def(node)
+
  def visit_class_def(self, node: ClassDef) -> None:
- # TODO additional things like the MRO
+ # TODO additional things?
  node.defs.body = self.replace_statements(node.defs.body)
+ node.info = self.fixup(node.info)
+ for tv in node.type_vars:
+ self.process_type_var_def(tv)
  self.process_type_info(node.info)
  super().visit_class_def(node)
 
+ def process_type_var_def(self, tv: TypeVarDef) -> None:
+ for value in tv.values:
+ self.fixup_type(value)
+ self.fixup_type(tv.upper_bound)
+
  def visit_assignment_stmt(self, node: AssignmentStmt) -> None:
  if node.type:
  self.fixup_type(node.type)
@@ -191,7 +207,39 @@ def visit_namedtuple_expr(self, node: NamedTupleExpr) -> None:
 
  def visit_super_expr(self, node: SuperExpr) -> None:
  super().visit_super_expr(node)
+ if node.info is not None:
+ node.info = self.fixup(node.info)
+
+ def visit_newtype_expr(self, node: NewTypeExpr) -> None:
+ if node.info:
+ node.info = self.fixup(node.info)
+ self.process_type_info(node.info)
+ if node.old_type:
+ self.fixup_type(node.old_type)
+ super().visit_newtype_expr(node)
+
+ def visit_lambda_expr(self, node: LambdaExpr) -> None:
+ if node.info:
+ node.info = self.fixup(node.info)
+ super().visit_lambda_expr(node)
+
+ def visit_typeddict_expr(self, node: TypedDictExpr) -> None:
+ node.info = self.fixup(node.info)
+ super().visit_typeddict_expr(node)
+
+ def visit_enum_call_expr(self, node: EnumCallExpr) -> None:
  node.info = self.fixup(node.info)
+ self.process_type_info(node.info)
+ super().visit_enum_call_expr(node)
+
+ # Others
+
+ def visit_var(self, node: Var) -> None:
+ if node.info:
+ node.info = self.fixup(node.info)
+ if node.type:
+ self.fixup_type(node.type)
+ super().visit_var(node)
 
  # Helpers
 
@@ -206,7 +254,13 @@ def fixup_type(self, typ: Type) -> None:
  typ.accept(TypeReplaceVisitor(self.replacements))
 
  def process_type_info(self, info: TypeInfo) -> None:
- # TODO additional things like the MRO
+ # TODO: Additional things:
+ # - declared_metaclass
+ # - metaclass_type
+ # - _promote
+ # - tuple_type
+ # - typeddict_type
+ # - replaced
  replace_nodes_in_symbol_table(info.names, self.replacements)
  for i, item in enumerate(info.mro):
  info.mro[i] = self.fixup(info.mro[i])

mypy/server/aststrip.py

@@ -43,7 +43,7 @@
 from mypy.nodes import (
  Node, FuncDef, NameExpr, MemberExpr, RefExpr, MypyFile, FuncItem, ClassDef, AssignmentStmt,
  ImportFrom, Import, TypeInfo, SymbolTable, Var, CallExpr, Decorator, OverloadedFuncDef,
- UNBOUND_IMPORTED, GDEF
+ SuperExpr, UNBOUND_IMPORTED, GDEF, MDEF
 )
 from mypy.traverser import TraverserVisitor
 
@@ -83,6 +83,10 @@ def visit_class_def(self, node: ClassDef) -> None:
  node.info.abstract_attributes = []
  node.info.mro = []
  node.info.add_type_vars()
+ node.info.tuple_type = None
+ node.info.typeddict_type = None
+ node.info._cache = set()
+ node.info._cache_proper = set()
  node.base_type_exprs.extend(node.removed_base_type_exprs)
  node.removed_base_type_exprs = []
  with self.enter_class(node.info):
@@ -136,20 +140,8 @@ def enter_method(self, info: TypeInfo) -> Iterator[None]:
 
  def visit_assignment_stmt(self, node: AssignmentStmt) -> None:
  node.type = node.unanalyzed_type
- if node.type and self.is_class_body:
- # Remove attribute defined in the class body from the class namespace to avoid
- # bogus "Name already defined" errors.
- #
- # TODO: Handle multiple assignment, other lvalues
- # TODO: What about assignments without type annotations?
- assert len(node.lvalues) == 1
- lvalue = node.lvalues[0]
- assert isinstance(lvalue, NameExpr)
- assert self.type is not None # Because self.is_class_body is True
- del self.type.names[lvalue.name]
  if self.type and not self.is_class_body:
  # TODO: Handle multiple assignment
- # TODO: Merge with above
  if len(node.lvalues) == 1:
  lvalue = node.lvalues[0]
  if isinstance(lvalue, MemberExpr) and lvalue.is_new_def:
@@ -192,6 +184,9 @@ def visit_name_expr(self, node: NameExpr) -> None:
  # Global assignments are processed in semantic analysis pass 1, and we
  # only want to strip changes made in passes 2 or later.
  if not (node.kind == GDEF and node.is_new_def):
+ # Remove defined attributes so that they can recreated during semantic analysis.
+ if node.kind == MDEF and node.is_new_def:
+ self.strip_class_attr(node.name)
  self.strip_ref_expr(node)
 
  def visit_member_expr(self, node: MemberExpr) -> None:
@@ -205,12 +200,14 @@ def visit_member_expr(self, node: MemberExpr) -> None:
  # defines an attribute with the same name, and we can't have
  # multiple definitions for an attribute. Defer to the base class
  # definition.
- if self.type is not None:
- del self.type.names[node.name]
- node.is_inferred_def = False
+ self.strip_class_attr(node.name)
  node.def_var = None
  super().visit_member_expr(node)
 
+ def strip_class_attr(self, name: str) -> None:
+ if self.type is not None:
+ del self.type.names[name]
+
  def is_duplicate_attribute_def(self, node: MemberExpr) -> bool:
  if not node.is_inferred_def:
  return False
@@ -223,11 +220,17 @@ def strip_ref_expr(self, node: RefExpr) -> None:
  node.kind = None
  node.node = None
  node.fullname = None
+ node.is_new_def = False
+ node.is_inferred_def = False
 
  def visit_call_expr(self, node: CallExpr) -> None:
  node.analyzed = None
  super().visit_call_expr(node)
 
+ def visit_super_expr(self, node: SuperExpr) -> None:
+ node.info = None
+ super().visit_super_expr(node)
+
  # TODO: handle more node types