Better support for type variables in fine grained mode (python#4743)

This adds support for type variables with value restrictions and upper 
bounds in fine grained mode. This also fixes several related and 
unrelated (triggered by new tests) issues.

I try not to add too many new dependencies. In particular, this is done 
by making class type variables included fully in the class snapshot 
instead of just their names.

This also adds missing test cases.

Fixes python#4393.

mypy/checkexpr.py

@@ -161,6 +161,11 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
  elif isinstance(node, TypeInfo):
  # Reference to a type object.
  result = type_object_type(node, self.named_type)
+ if isinstance(result, CallableType) and isinstance(result.ret_type, Instance):
+ # We need to set correct line and column
+ # TODO: always do this in type_object_type by passing the original context
+ result.ret_type.line = e.line
+ result.ret_type.column = e.column
  if isinstance(self.type_context[-1], TypeType):
  # This is the type in a Type[] expression, so substitute type
  # variables with Any.

mypy/server/astdiff.py

@@ -196,7 +196,15 @@ def snapshot_definition(node: Optional[SymbolNode],
  snapshot_optional_type(node.tuple_type),
  snapshot_optional_type(node.typeddict_type),
  [base.fullname() for base in node.mro],
- node.type_vars,
+ # Note that the structure of type variables is a part of the external interface,
+ # since creating instances might fail, for example:
+ # T = TypeVar('T', bound=int)
+ # class C(Generic[T]):
+ # ...
+ # x: C[str] <- this is invalid, and needs to be re-checked if `T` changes.
+ # An alternative would be to create both deps: <...> -> C, and <...> -> <C>,
+ # but this currently seems a bit ad hoc.
+ tuple(snapshot_type(TypeVarType(tdef)) for tdef in node.defn.type_vars),
  [snapshot_type(base) for base in node.bases],
  snapshot_optional_type(node._promote))
  prefix = node.fullname()

mypy/server/deps.py

@@ -170,7 +170,6 @@ def __init__(self,
  # await
  # protocols
  # metaclasses
- # type variable with value restriction
 
  def visit_mypy_file(self, o: MypyFile) -> None:
  self.scope.enter_file(o.fullname())
@@ -195,6 +194,10 @@ def visit_func_def(self, o: FuncDef) -> None:
  self.add_dependency(make_trigger(base.fullname() + '.' + o.name()))
  self.add_type_alias_deps(self.scope.current_target())
  super().visit_func_def(o)
+ variants = set(o.expanded) - {o}
+ for ex in variants:
+ if isinstance(ex, FuncDef):
+ super().visit_func_def(ex)
  self.scope.leave()
 
  def visit_decorator(self, o: Decorator) -> None:
@@ -274,10 +277,11 @@ def visit_block(self, o: Block) -> None:
  def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
  rvalue = o.rvalue
  if isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, TypeVarExpr):
- # TODO: Support type variable value restriction
  analyzed = rvalue.analyzed
  self.add_type_dependencies(analyzed.upper_bound,
  target=make_trigger(analyzed.fullname()))
+ for val in analyzed.values:
+ self.add_type_dependencies(val, target=make_trigger(analyzed.fullname()))
  elif isinstance(rvalue, CallExpr) and isinstance(rvalue.analyzed, NamedTupleExpr):
  # Depend on types of named tuple items.
  info = rvalue.analyzed.info
@@ -685,10 +689,13 @@ def visit_forwardref_type(self, typ: ForwardRef) -> List[str]:
  assert False, 'Internal error: Leaked forward reference object {}'.format(typ)
 
  def visit_type_var(self, typ: TypeVarType) -> List[str]:
- # TODO: bound (values?)
  triggers = []
  if typ.fullname:
  triggers.append(make_trigger(typ.fullname))
+ if typ.upper_bound:
+ triggers.extend(get_type_triggers(typ.upper_bound))
+ for val in typ.values:
+ triggers.extend(get_type_triggers(val))
  return triggers
 
  def visit_typeddict_type(self, typ: TypedDictType) -> List[str]:

mypy/server/update.py

@@ -869,7 +869,12 @@ def key(node: DeferredNode) -> int:
  scope=manager.semantic_analyzer_pass3.scope):
  manager.semantic_analyzer_pass3.refresh_partial(deferred.node, patches)
 
- apply_semantic_analyzer_patches(patches)
+ with semantic_analyzer.file_context(
+ file_node=file_node,
+ fnam=file_node.path,
+ options=manager.options,
+ active_type=None):
+ apply_semantic_analyzer_patches(patches)
 
  # Merge symbol tables to preserve identities of AST nodes. The file node will remain
  # the same, but other nodes may have been recreated with different identities, such as

test-data/unit/deps-generics.test

@@ -121,8 +121,8 @@ T = TypeVar('T', bound=Tuple[A, B])
 def f(x: T) -> T:
  return x
 [out]
-<m.A> -> <m.T>, m.A
-<m.B> -> <m.T>, m.B
+<m.A> -> <m.T>, <m.f>, m.A, m.f
+<m.B> -> <m.T>, <m.f>, m.B, m.f
 <m.T> -> <m.f>, m.f
 
 [case testTypeVarBoundOperations]
@@ -140,5 +140,58 @@ def f(x: T) -> None:
 [out]
 <m.A.__add__> -> m.f
 <m.A.f> -> m.f
-<m.A> -> <m.T>, m.A
+<m.A> -> <m.T>, <m.f>, m.A, m.f
 <m.T> -> <m.f>, m.f
+
+[case testTypeVarValues]
+from typing import TypeVar
+class A: pass
+class B: pass
+class C: pass
+class D: pass
+T = TypeVar('T', A, B)
+S = TypeVar('S', C, D)
+
+def f(x: T) -> S:
+ pass
+[out]
+<m.A> -> <m.T>, <m.f>, m.A, m.f
+<m.B> -> <m.T>, <m.f>, m.B, m.f
+<m.C> -> <m.S>, <m.f>, m.C, m.f
+<m.D> -> <m.S>, <m.f>, m.D, m.f
+<m.S> -> <m.f>, m.f
+<m.T> -> <m.f>, m.f
+
+[case testTypeVarValuesMethod]
+from typing import TypeVar, Generic
+class C: pass
+class D: pass
+S = TypeVar('S', C, D)
+
+class G(Generic[S]):
+ def f(self) -> S:
+ pass
+[out]
+<m.C> -> <m.G.f>, <m.S>, m.C, m.G.f
+<m.D> -> <m.G.f>, <m.S>, m.D, m.G.f
+<m.G> -> m.G
+<m.S> -> <m.G.f>, m.G, m.G.f
+
+[case testTypeVarValuesMethodAttr]
+from typing import TypeVar, Generic
+class A:
+ x: int
+class B:
+ x: int
+T = TypeVar('T', A, B)
+
+class G(Generic[T]):
+ def f(self, x: T) -> None:
+ x.x
+[out]
+<m.A.x> -> m.G.f
+<m.A> -> <m.G.f>, <m.T>, m.A, m.G.f
+<m.B.x> -> m.G.f
+<m.B> -> <m.G.f>, <m.T>, m.B, m.G.f
+<m.G> -> m.G
+<m.T> -> <m.G.f>, m.G, m.G.f

test-data/unit/fine-grained.test

@@ -2976,6 +2976,202 @@ class User:
 [out]
 ==
 
+[case testTypeVarValuesFunction]
+import a
+[file a.py]
+from typing import TypeVar
+from c import A, B
+T = TypeVar('T', A, B)
+
+def f(x: T) -> T:
+ x.x = int()
+ return x
+[file c.py]
+class A:
+ x: int
+class B:
+ x: int
+[file c.py.2]
+class A:
+ x: int
+class B:
+ x: str
+[out]
+==
+a.py:6: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+
+[case testTypeVarValuesClass]
+import a
+[file a.py]
+import c
+class C:
+ x: c.D[c.A]
+[file c.py]
+from typing import TypeVar, Generic
+class A: pass
+class B: pass
+class C: pass
+T = TypeVar('T', A, B, C)
+class D(Generic[T]):
+ pass
+[file c.py.2]
+from typing import TypeVar, Generic
+class A: pass
+class B: pass
+class C: pass
+T = TypeVar('T', B, C)
+class D(Generic[T]):
+ pass
+[out]
+==
+a.py:3: error: Value of type variable "T" of "D" cannot be "A"
+
+[case testTypeVarValuesMethod1]
+import a
+[file a.py]
+from typing import Generic
+import c
+class G(Generic[c.T]):
+ def f(self, x: c.T) -> None:
+ x.x = int()
+[file c.py]
+from typing import TypeVar
+class A:
+ x: int
+class B:
+ x: int
+class C:
+ x: str
+T = TypeVar('T', A, B, C)
+[file c.py.2]
+from typing import TypeVar
+class A:
+ x: int
+class B:
+ x: int
+class C:
+ x: str
+T = TypeVar('T', A, B)
+[out]
+a.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+==
+
+[case testTypeVarValuesMethod2]
+import a
+[file a.py]
+from typing import Generic
+import c
+class G(Generic[c.T]):
+ def f(self, x: c.T) -> None:
+ x.x = int()
+[file c.py]
+from typing import TypeVar
+class A:
+ x: int
+class B:
+ x: int
+T = TypeVar('T', A, B)
+[file c.py.2]
+from typing import TypeVar
+class A:
+ x: int
+class B:
+ x: str
+T = TypeVar('T', A, B)
+[out]
+==
+a.py:5: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+
+[case testTypeVarBoundFunction]
+import a
+[file a.py]
+from typing import TypeVar
+from c import B
+T = TypeVar('T', bound=B)
+
+def f(x: T) -> T:
+ x.x = int()
+ return x
+[file c.py]
+class B:
+ x: int
+[file c.py.2]
+class B:
+ x: str
+[out]
+==
+a.py:6: error: Incompatible types in assignment (expression has type "int", variable has type "str")
+
+[case testTypeVarBoundClass]
+import a
+[file a.py]
+import c
+class C:
+ x: c.D[c.A]
+[file c.py]
+from typing import TypeVar, Generic
+class A: pass
+class B: pass
+T = TypeVar('T', bound=A)
+class D(Generic[T]):
+ pass
+[file c.py.2]
+from typing import TypeVar, Generic
+class A: pass
+class B: pass
+T = TypeVar('T', bound=B)
+class D(Generic[T]):
+ pass
+[out]
+==
+a.py:3: error: Type argument "c.A" of "D" must be a subtype of "c.B"
+
+[case testTypeVarValuesRuntime]
+from mod import I, S, D
+A = I
+x = D[S, A]()
+[file mod.py]
+import submod
+from typing import Generic
+class D(Generic[submod.T, submod.U]): pass
+class I: pass
+class S: pass
+[file submod.py]
+from typing import TypeVar
+T = TypeVar('T')
+U = TypeVar('U')
+[file submod.py.2]
+from typing import TypeVar
+T = TypeVar('T', int, str)
+U = TypeVar('U', int, str)
+[out]
+==
+main:3: error: Value of type variable "submod.T" of "D" cannot be "S"
+main:3: error: Value of type variable "submod.U" of "D" cannot be "I"
+
+[case testTypeVarBoundRuntime]
+from mod import I, S, D
+A = I
+x = D[S, A]()
+[file mod.py]
+import submod
+from typing import Generic
+class D(Generic[submod.T, submod.U]): pass
+class I: pass
+class S: pass
+[file submod.py]
+from typing import TypeVar
+T = TypeVar('T', bound=int)
+U = TypeVar('U', bound=int)
+[file submod.py.2]
+from typing import TypeVar
+T = TypeVar('T')
+U = TypeVar('U')
+[out]
+main:3: error: Value of type variable "submod.T" of "D" cannot be "S"
+main:3: error: Value of type variable "submod.U" of "D" cannot be "I"
+==
+
 [case testRefreshClassBasedEnum]
 import aa
 [file aa.py]