Refactor type aliases (#5221)

This refactors type aliases by introducing a dedicated symbol node `TypeAlias`, and removing all _ad-hoc_ attributes from `SymbolTableNode`. This simplifies the overall logic, and also the fine grained dependency tracking (since now aliases have names).

Most of the logic is straightforward. Few comments:
* It looks like `.type_override` was also used for some overload hacks, I replaced those with a proper fix.
* Somehow my changes exposed a small `ImportedName` bug, I can't find a simple repro (typeshed check test was failing), but the fix is straightforward.
* In some tests revealed type don't have `*` anymore, I think this is more consistent.
* There few other small changes in tests, I think they are harmless.

Author: ilevkivskyi

mypy/build.py

@@ -2856,6 +2856,13 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
         # If the former, parse_file() is a no-op.
         graph[id].parse_file()
         graph[id].fix_suppressed_dependencies(graph)
+    if 'typing' in scc:
+        # For historical reasons we need to manually add typing aliases
+        # for built-in generic collections, see docstring of
+        # SemanticAnalyzerPass2.add_builtin_aliases for details.
+        typing_mod = graph['typing'].tree
+        assert typing_mod, "The typing module was not parsed"
+        manager.semantic_analyzer.add_builtin_aliases(typing_mod)
     for id in fresh:
         graph[id].fix_cross_refs()
     for id in stale:

mypy/checker.py

@@ -1441,10 +1441,10 @@ def check_compatibility(self, name: str, base1: TypeInfo,
         first = base1[name]
         second = base2[name]
         first_type = first.type
-        if first_type is None and isinstance(first.node, FuncDef):
+        if first_type is None and isinstance(first.node, FuncBase):
             first_type = self.function_type(first.node)
         second_type = second.type
-        if second_type is None and isinstance(second.node, FuncDef):
+        if second_type is None and isinstance(second.node, FuncBase):
             second_type = self.function_type(second.node)
         # TODO: What if some classes are generic?
         if (isinstance(first_type, FunctionLike) and
@@ -2835,7 +2835,7 @@ def intersect_instance_callable(self, typ: Instance, callable_type: CallableType
         func_def = FuncDef('__call__', [], Block([]), callable_type)
         func_def._fullname = cdef.fullname + '.__call__'
         func_def.info = info
-        info.names['__call__'] = SymbolTableNode(MDEF, func_def, callable_type)
+        info.names['__call__'] = SymbolTableNode(MDEF, func_def)
 
         cur_module.names[gen_name] = SymbolTableNode(GDEF, info)
 

mypy/checkexpr.py

@@ -4,7 +4,9 @@
 from typing import cast, Dict, Set, List, Tuple, Callable, Union, Optional, Iterable, Sequence, Any
 
 from mypy.errors import report_internal_error
-from mypy.typeanal import has_any_from_unimported_type, check_for_explicit_any, set_any_tvars
+from mypy.typeanal import (
+    has_any_from_unimported_type, check_for_explicit_any, set_any_tvars, expand_type_alias
+)
 from mypy.types import (
     Type, AnyType, CallableType, Overloaded, NoneTyp, TypeVarDef,
     TupleType, TypedDictType, Instance, TypeVarType, ErasedType, UnionType,
@@ -21,8 +23,8 @@
     ConditionalExpr, ComparisonExpr, TempNode, SetComprehension,
     DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr,
     YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr,
-    TypeAliasExpr, BackquoteExpr, EnumCallExpr,
-    ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, TVAR, LITERAL_TYPE, REVEAL_TYPE
+    TypeAliasExpr, BackquoteExpr, EnumCallExpr, TypeAlias, ClassDef, Block, SymbolTable,
+    ARG_POS, ARG_OPT, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, LITERAL_TYPE, REVEAL_TYPE
 )
 from mypy.literals import literal
 from mypy import nodes
@@ -181,6 +183,14 @@ def analyze_ref_expr(self, e: RefExpr, lvalue: bool = False) -> Type:
                 result = self.named_type('builtins.object')
         elif isinstance(node, Decorator):
             result = self.analyze_var_ref(node.var, e)
+        elif isinstance(node, TypeAlias):
+            # Something that refers to a type alias appears in runtime context.
+            # Note that we suppress bogus errors for alias redefinitions,
+            # they are already reported in semanal.py.
+            result = self.alias_type_in_runtime_context(node.target, node.alias_tvars,
+                                                        node.no_args, e,
+                                                        alias_definition=e.is_alias_rvalue
+                                                        or lvalue)
         else:
             # Unknown reference; use any type implicitly to avoid
             # generating extra type errors.
@@ -220,8 +230,8 @@ def visit_call_expr(self, e: CallExpr, allow_none_return: bool = False) -> Type:
                         pass
                 if ((isinstance(typ, IndexExpr)
                         and isinstance(typ.analyzed, (TypeApplication, TypeAliasExpr)))
-                        # node.kind == TYPE_ALIAS only for aliases like It = Iterable[int].
-                        or (isinstance(typ, NameExpr) and node and node.kind == nodes.TYPE_ALIAS)):
+                        or (isinstance(typ, NameExpr) and node and
+                            isinstance(node.node, TypeAlias) and not node.node.no_args)):
                     self.msg.type_arguments_not_allowed(e)
                 if isinstance(typ, RefExpr) and isinstance(typ.node, TypeInfo):
                     if typ.node.typeddict_type:
@@ -2094,60 +2104,119 @@ def visit_reveal_expr(self, expr: RevealExpr) -> Type:
             return NoneTyp()
 
     def visit_type_application(self, tapp: TypeApplication) -> Type:
-        """Type check a type application (expr[type, ...])."""
-        tp = self.accept(tapp.expr)
-        if isinstance(tp, CallableType):
-            if not tp.is_type_obj():
+        """Type check a type application (expr[type, ...]).
+
+        There are two different options here, depending on whether expr refers
+        to a type alias or directly to a generic class. In the first case we need
+        to use a dedicated function typeanal.expand_type_aliases. This
+        is due to the fact that currently type aliases machinery uses
+        unbound type variables, while normal generics use bound ones;
+        see TypeAlias docstring for more details.
+        """
+        if isinstance(tapp.expr, RefExpr) and isinstance(tapp.expr.node, TypeAlias):
+            # Subscription of a (generic) alias in runtime context, expand the alias.
+            target = tapp.expr.node.target
+            all_vars = tapp.expr.node.alias_tvars
+            item = expand_type_alias(target, all_vars, tapp.types, self.chk.fail,
+                                     tapp.expr.node.no_args, tapp)
+            if isinstance(item, Instance):
+                tp = type_object_type(item.type, self.named_type)
+                return self.apply_type_arguments_to_callable(tp, item.args, tapp)
+            else:
                 self.chk.fail(messages.ONLY_CLASS_APPLICATION, tapp)
-            if len(tp.variables) != len(tapp.types):
-                self.msg.incompatible_type_application(len(tp.variables),
-                                                       len(tapp.types), tapp)
                 return AnyType(TypeOfAny.from_error)
-            return self.apply_generic_arguments(tp, tapp.types, tapp)
-        elif isinstance(tp, Overloaded):
+        # Type application of a normal generic class in runtime context.
+        # This is typically used as `x = G[int]()`.
+        tp = self.accept(tapp.expr)
+        if isinstance(tp, (CallableType, Overloaded)):
             if not tp.is_type_obj():
                 self.chk.fail(messages.ONLY_CLASS_APPLICATION, tapp)
-            for item in tp.items():
-                if len(item.variables) != len(tapp.types):
-                    self.msg.incompatible_type_application(len(item.variables),
-                                                           len(tapp.types), tapp)
-                    return AnyType(TypeOfAny.from_error)
-            return Overloaded([self.apply_generic_arguments(item, tapp.types, tapp)
-                               for item in tp.items()])
+            return self.apply_type_arguments_to_callable(tp, tapp.types, tapp)
         if isinstance(tp, AnyType):
             return AnyType(TypeOfAny.from_another_any, source_any=tp)
         return AnyType(TypeOfAny.special_form)
 
     def visit_type_alias_expr(self, alias: TypeAliasExpr) -> Type:
-        """Get type of a type alias (could be generic) in a runtime expression."""
-        if isinstance(alias.type, Instance) and alias.type.invalid:
+        """Right hand side of a type alias definition.
+
+        It has the same type as if the alias itself was used in a runtime context.
+        For example, here:
+
+            A = reveal_type(List[T])
+            reveal_type(A)
+
+        both `reveal_type` instances will reveal the same type `def (...) -> builtins.list[Any]`.
+        Note that type variables are implicitly substituted with `Any`.
+        """
+        return self.alias_type_in_runtime_context(alias.type, alias.tvars, alias.no_args,
+                                                  alias, alias_definition=True)
+
+    def alias_type_in_runtime_context(self, target: Type, alias_tvars: List[str],
+                                      no_args: bool, ctx: Context,
+                                      *,
+                                      alias_definition: bool = False) -> Type:
+        """Get type of a type alias (could be generic) in a runtime expression.
+
+        Note that this function can be called only if the alias appears _not_
+        as a target of type application, which is treated separately in the
+        visit_type_application method. Some examples where this method is called are
+        casts and instantiation:
+
+            class LongName(Generic[T]): ...
+            A = LongName[int]
+
+            x = A()
+            y = cast(A, ...)
+        """
+        if isinstance(target, Instance) and target.invalid:
             # An invalid alias, error already has been reported
             return AnyType(TypeOfAny.from_error)
-        item = alias.type
-        if not alias.in_runtime:
-            # We don't replace TypeVar's with Any for alias used as Alias[T](42).
-            item = set_any_tvars(item, alias.tvars, alias.line, alias.column)
+        # If this is a generic alias, we set all variables to `Any`.
+        # For example:
+        #     A = List[Tuple[T, T]]
+        #     x = A() <- same as List[Tuple[Any, Any]], see PEP 484.
+        item = set_any_tvars(target, alias_tvars, ctx.line, ctx.column)
         if isinstance(item, Instance):
             # Normally we get a callable type (or overloaded) with .is_type_obj() true
             # representing the class's constructor
             tp = type_object_type(item.type, self.named_type)
+            if no_args:
+                return tp
+            return self.apply_type_arguments_to_callable(tp, item.args, ctx)
+        elif (isinstance(item, TupleType) and
+              # Tuple[str, int]() fails at runtime, only named tuples and subclasses work.
+              item.fallback.type.fullname() != 'builtins.tuple'):
+            return type_object_type(item.fallback.type, self.named_type)
+        elif isinstance(item, AnyType):
+            return AnyType(TypeOfAny.from_another_any, source_any=item)
         else:
-            # This type is invalid in most runtime contexts
-            # and corresponding an error will be reported.
-            return alias.fallback
+            if alias_definition:
+                return AnyType(TypeOfAny.special_form)
+            # This type is invalid in most runtime contexts.
+            self.msg.alias_invalid_in_runtime_context(item, ctx)
+            return AnyType(TypeOfAny.from_error)
+
+    def apply_type_arguments_to_callable(self, tp: Type, args: List[Type], ctx: Context) -> Type:
+        """Apply type arguments to a generic callable type coming from a type object.
+
+        This will first perform type arguments count checks, report the
+        error as needed, and return the correct kind of Any. As a special
+        case this returns Any for non-callable types, because if type object type
+        is not callable, then an error should be already reported.
+        """
         if isinstance(tp, CallableType):
-            if len(tp.variables) != len(item.args):
+            if len(tp.variables) != len(args):
                 self.msg.incompatible_type_application(len(tp.variables),
-                                                       len(item.args), item)
+                                                       len(args), ctx)
                 return AnyType(TypeOfAny.from_error)
-            return self.apply_generic_arguments(tp, item.args, item)
-        elif isinstance(tp, Overloaded):
+            return self.apply_generic_arguments(tp, args, ctx)
+        if isinstance(tp, Overloaded):
             for it in tp.items():
-                if len(it.variables) != len(item.args):
+                if len(it.variables) != len(args):
                     self.msg.incompatible_type_application(len(it.variables),
-                                                           len(item.args), item)
+                                                           len(args), ctx)
                     return AnyType(TypeOfAny.from_error)
-            return Overloaded([self.apply_generic_arguments(it, item.args, item)
+            return Overloaded([self.apply_generic_arguments(it, args, ctx)
                                for it in tp.items()])
         return AnyType(TypeOfAny.special_form)
 

mypy/checkmember.py

@@ -9,15 +9,15 @@
 )
 from mypy.nodes import (
     TypeInfo, FuncBase, Var, FuncDef, SymbolNode, Context, MypyFile, TypeVarExpr,
-    ARG_POS, ARG_STAR, ARG_STAR2,
-    Decorator, OverloadedFuncDef,
+    ARG_POS, ARG_STAR, ARG_STAR2, Decorator, OverloadedFuncDef, TypeAlias
 )
 from mypy.messages import MessageBuilder
 from mypy.maptype import map_instance_to_supertype
 from mypy.expandtype import expand_type_by_instance, expand_type, freshen_function_type_vars
 from mypy.infer import infer_type_arguments
 from mypy.typevars import fill_typevars
 from mypy.plugin import AttributeContext
+from mypy.typeanal import set_any_tvars
 from mypy import messages
 from mypy import subtypes
 from mypy import meet
@@ -249,6 +249,17 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo,
         v = Var(name, type=type_object_type(vv, builtin_type))
         v.info = info
 
+    if isinstance(vv, TypeAlias) and isinstance(vv.target, Instance):
+        # Similar to the above TypeInfo case, we allow using
+        # qualified type aliases in runtime context if it refers to an
+        # instance type. For example:
+        #     class C:
+        #         A = List[int]
+        #     x = C.A() <- this is OK
+        typ = instance_alias_type(vv, builtin_type)
+        v = Var(name, type=typ)
+        v.info = info
+
     if isinstance(v, Var):
         return analyze_var(name, v, itype, info, node, is_lvalue, msg,
                            original_type, not_ready_callback, chk=chk)
@@ -291,6 +302,19 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo,
         return msg.has_no_attr(original_type, itype, name, node)
 
 
+def instance_alias_type(alias: TypeAlias,
+                        builtin_type: Callable[[str], Instance]) -> Type:
+    """Type of a type alias node targeting an instance, when appears in runtime context.
+
+    As usual, we first erase any unbound type variables to Any.
+    """
+    assert isinstance(alias.target, Instance), "Must be called only with aliases to classes"
+    target = set_any_tvars(alias.target, alias.alias_tvars, alias.line, alias.column)
+    assert isinstance(target, Instance)
+    tp = type_object_type(target.type, builtin_type)
+    return expand_type_by_instance(tp, target)
+
+
 def analyze_var(name: str, var: Var, itype: Instance, info: TypeInfo, node: Context,
                 is_lvalue: bool, msg: MessageBuilder, original_type: Type,
                 not_ready_callback: Callable[[str, Context], None], *,
@@ -430,7 +454,7 @@ def analyze_class_attribute_access(itype: Instance,
         return None
 
     is_decorated = isinstance(node.node, Decorator)
-    is_method = is_decorated or isinstance(node.node, FuncDef)
+    is_method = is_decorated or isinstance(node.node, FuncBase)
     if is_lvalue:
         if is_method:
             msg.cant_assign_to_method(context)
@@ -467,6 +491,9 @@ def analyze_class_attribute_access(itype: Instance,
         # Reference to a module object.
         return builtin_type('types.ModuleType')
 
+    if isinstance(node.node, TypeAlias) and isinstance(node.node.target, Instance):
+        return instance_alias_type(node.node, builtin_type)
+
     if is_decorated:
         assert isinstance(node.node, Decorator)
         if node.node.type:

mypy/fixup.py

@@ -5,7 +5,7 @@
 from mypy.nodes import (
     MypyFile, SymbolNode, SymbolTable, SymbolTableNode,
     TypeInfo, FuncDef, OverloadedFuncDef, Decorator, Var,
-    TypeVarExpr, ClassDef, Block, TYPE_ALIAS
+    TypeVarExpr, ClassDef, Block, TypeAlias,
 )
 from mypy.types import (
     CallableType, Instance, Overloaded, TupleType, TypedDictType,
@@ -76,26 +76,17 @@ def visit_symbol_table(self, symtab: SymbolTable) -> None:
                                                      self.quick_and_dirty)
                     if stnode is not None:
                         value.node = stnode.node
-                        value.type_override = stnode.type_override
-                        if (self.quick_and_dirty and value.kind == TYPE_ALIAS and
-                                stnode.type_override is None):
-                            value.type_override = Instance(stale_info(self.modules), [])
-                        value.alias_tvars = stnode.alias_tvars or []
                     elif not self.quick_and_dirty:
                         assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,)
                     else:
                         # We have a missing crossref in quick mode, need to put something
                         value.node = stale_info(self.modules)
-                        if value.kind == TYPE_ALIAS:
-                            value.type_override = Instance(stale_info(self.modules), [])
             else:
                 if isinstance(value.node, TypeInfo):
                     # TypeInfo has no accept().  TODO: Add it?
                     self.visit_type_info(value.node)
                 elif value.node is not None:
                     value.node.accept(self)
-                if value.type_override is not None:
-                    value.type_override.accept(self.type_fixer)
 
     def visit_func_def(self, func: FuncDef) -> None:
         if self.current_info is not None:
@@ -140,6 +131,9 @@ def visit_var(self, v: Var) -> None:
         if v.type is not None:
             v.type.accept(self.type_fixer)
 
+    def visit_type_alias(self, a: TypeAlias) -> None:
+        a.target.accept(self.type_fixer)
+
 
 class TypeFixer(TypeVisitor[None]):
     def __init__(self, modules: Dict[str, MypyFile], quick_and_dirty: bool) -> None: