From 679a172c5e70c0d0d70b4bcd05b5f5abb0d539ab Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 01:46:23 -0700 Subject: [PATCH 01/37] Make it an error to use a class-attribute type var outside a type Previously, e9d28a01 fixed a crash when you tried to access a class-attribute type variable. The test in that commit involved assigning a variable the value of the typevar. It resulted in no crash, but rather treating the variable as being an instance of the type the typevar bound to, later, which is incorrect. Instead, this PR treats such an assignment as an error, and gives you the same message as when you try to alias a typevar directly. Also test a *correct* alias with the typevar access method in question -- it works. --- mypy/checkmember.py | 4 +++- mypy/typeanal.py | 3 +++ test-data/unit/check-typevar-values.test | 11 ++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index ede8b9189698..056148c96c92 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -406,7 +406,9 @@ def analyze_class_attribute_access(itype: Instance, return AnyType() if isinstance(node.node, TypeVarExpr): - return TypeVarType(node.tvar_def, node.tvar_def.line, node.tvar_def.column) + msg.fail('Type variable "{}" is invalid as target for type alias'.format( + node.node.fullname() or node.node.name()), context) + return AnyType() if isinstance(node.node, TypeInfo): return type_object_type(node.node, builtin_type) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index bd0ec722171b..d6084b2cb61d 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -43,6 +43,9 @@ def analyze_type_alias(node: Expression, # that we don't support straight string literals as type aliases # (only string literals within index expressions). if isinstance(node, RefExpr): + # Note that this misses the case where someone tried to use a + # class-referenced type variable as a type alias. It's easier to catch + # that one in checkmemeber.py if node.kind == UNBOUND_TVAR or node.kind == BOUND_TVAR: fail_func('Type variable "{}" is invalid as target for type alias'.format( node.fullname), node) diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index f41710ee155b..bd008d5b44b6 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -496,12 +496,17 @@ def outer(x: T) -> T: [out] [case testClassMemberTypeVarInFunctionBody] -from typing import TypeVar +from typing import TypeVar, List class C: T = TypeVar('T', bound=int) def f(self, x: T) -> T: - A = C.T - return x + L = List[C.T] # a valid type alias + l: L = [] + l.append(x) + A = C.T # E: Type variable "T" is invalid as target for type alias + return l[0] + +[builtins fixtures/list.pyi] [case testParameterLessGenericAsRestriction] from typing import Sequence, Iterable, TypeVar From 74e9262d933940b8b37a0c286b034596484629e6 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 02:09:52 -0700 Subject: [PATCH 02/37] Fix the error message to handle more cases --- mypy/checkmember.py | 4 ++-- test-data/unit/check-generics.test | 1 + test-data/unit/check-typevar-values.test | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 056148c96c92..a70292f3d736 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -406,8 +406,8 @@ def analyze_class_attribute_access(itype: Instance, return AnyType() if isinstance(node.node, TypeVarExpr): - msg.fail('Type variable "{}" is invalid as target for type alias'.format( - node.node.fullname() or node.node.name()), context) + msg.fail('Type variable "{}" is invalid except as a parameter to another type'.format( + node.node.fullname() or node.node.name()), context) return AnyType() if isinstance(node.node, TypeInfo): diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 7d035d0ed992..a0f4dbc5ad9a 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -14,6 +14,7 @@ class A(Generic[T]): class B: pass class C: pass + [out] main:4: error: Incompatible types in assignment (expression has type "B", variable has type "C") diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index bd008d5b44b6..86350344e2be 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -503,7 +503,8 @@ class C: L = List[C.T] # a valid type alias l: L = [] l.append(x) - A = C.T # E: Type variable "T" is invalid as target for type alias + C.T # E: Type variable "T" is invalid except as a parameter to another type + A = C.T # E: Type variable "T" is invalid except as a parameter to another type return l[0] [builtins fixtures/list.pyi] From 5470c778372859950039afbc6eda562ce602bd6f Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 02:19:52 -0700 Subject: [PATCH 03/37] oops accidentally changed a file --- test-data/unit/check-generics.test | 1 - 1 file changed, 1 deletion(-) diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index a0f4dbc5ad9a..7d035d0ed992 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -14,7 +14,6 @@ class A(Generic[T]): class B: pass class C: pass - [out] main:4: error: Incompatible types in assignment (expression has type "B", variable has type "C") From 12def8b72f297b529175e353371b3ede64657948 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 10:45:30 -0700 Subject: [PATCH 04/37] Typo fix --- mypy/typeanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d6084b2cb61d..e89a17a1a998 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -45,7 +45,7 @@ def analyze_type_alias(node: Expression, if isinstance(node, RefExpr): # Note that this misses the case where someone tried to use a # class-referenced type variable as a type alias. It's easier to catch - # that one in checkmemeber.py + # that one in checkmember.py if node.kind == UNBOUND_TVAR or node.kind == BOUND_TVAR: fail_func('Type variable "{}" is invalid as target for type alias'.format( node.fullname), node) From 52782fe559c4c58d884a117fb43041afe87b60f8 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 11:14:47 -0700 Subject: [PATCH 05/37] more checking things in tests --- test-data/unit/check-typevar-values.test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index 86350344e2be..56ffbf3a5489 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -502,6 +502,8 @@ class C: def f(self, x: T) -> T: L = List[C.T] # a valid type alias l: L = [] + reveal_type(l) # E: Revealed type is 'builtins.list[T`-1]' + y: C.T = x l.append(x) C.T # E: Type variable "T" is invalid except as a parameter to another type A = C.T # E: Type variable "T" is invalid except as a parameter to another type From 01845955e90bb136c3302f5dace9465ca2044e60 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 11:28:38 -0700 Subject: [PATCH 06/37] Change error message --- mypy/checkmember.py | 4 ++-- test-data/unit/check-typevar-values.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index a70292f3d736..83c4660cb1b1 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -406,8 +406,8 @@ def analyze_class_attribute_access(itype: Instance, return AnyType() if isinstance(node.node, TypeVarExpr): - msg.fail('Type variable "{}" is invalid except as a parameter to another type'.format( - node.node.fullname() or node.node.name()), context) + msg.fail('Type variable "{}.{}" is invalid in a runtime context'.format( + itype.type.name(), name), context) return AnyType() if isinstance(node.node, TypeInfo): diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index 56ffbf3a5489..29c72add0624 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -505,8 +505,8 @@ class C: reveal_type(l) # E: Revealed type is 'builtins.list[T`-1]' y: C.T = x l.append(x) - C.T # E: Type variable "T" is invalid except as a parameter to another type - A = C.T # E: Type variable "T" is invalid except as a parameter to another type + C.T # E: Type variable "C.T" is invalid in a runtime context + A = C.T # E: Type variable "C.T" is invalid in a runtime context return l[0] [builtins fixtures/list.pyi] From 757a6b842b674b397df3c01820e4a15b441caf46 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 11:30:27 -0700 Subject: [PATCH 07/37] Change error message --- mypy/checkmember.py | 2 +- test-data/unit/check-typevar-values.test | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 83c4660cb1b1..6c934dccb022 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -406,7 +406,7 @@ def analyze_class_attribute_access(itype: Instance, return AnyType() if isinstance(node.node, TypeVarExpr): - msg.fail('Type variable "{}.{}" is invalid in a runtime context'.format( + msg.fail('Type variable "{}.{}" cannot be used as an expression'.format( itype.type.name(), name), context) return AnyType() diff --git a/test-data/unit/check-typevar-values.test b/test-data/unit/check-typevar-values.test index 29c72add0624..36df2235a209 100644 --- a/test-data/unit/check-typevar-values.test +++ b/test-data/unit/check-typevar-values.test @@ -505,8 +505,8 @@ class C: reveal_type(l) # E: Revealed type is 'builtins.list[T`-1]' y: C.T = x l.append(x) - C.T # E: Type variable "C.T" is invalid in a runtime context - A = C.T # E: Type variable "C.T" is invalid in a runtime context + C.T # E: Type variable "C.T" cannot be used as an expression + A = C.T # E: Type variable "C.T" cannot be used as an expression return l[0] [builtins fixtures/list.pyi] From 57bca93d3e7042f9ada2e3cdad6bf2eede8b9f08 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 29 Mar 2017 16:05:18 -0700 Subject: [PATCH 08/37] Make TypeQuery more general, handling nonboolean queries. Instead of TypeQuery always returning a boolean and having the strategy be an enum, the strategy is now a Callable describing how to combine partial results, and the two default strategies are plain old funcitons. To preserve the short-circuiting behavior of the previous code, this PR uses an exception. This is a pure refactor that I am using in my experimentation regarding fixing https://github.com/python/mypy/issues/1551. It should result in exactly no change to current behavior. It's separable from the other things I'm experimenting with, so I'm filing it as a separate pull request now. It enables me to rewrite the code that pulls type variables out of types as a TypeQuery. Consider waiting to merge this PR until I have some code that uses it ready for review. Or merge it now, if you think it's a pleasant cleanup instead of an ugly complication. I'm of two minds on that particular question. --- mypy/checkexpr.py | 6 +-- mypy/constraints.py | 2 +- mypy/stats.py | 2 +- mypy/types.py | 108 ++++++++++++++++++++++---------------------- 4 files changed, 59 insertions(+), 59 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index ecccd267c791..7410d08de950 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -2375,7 +2375,7 @@ def replace_callable_return_type(c: CallableType, new_ret_type: Type) -> Callabl return c.copy_modified(ret_type=new_ret_type) -class ArgInferSecondPassQuery(types.TypeQuery): +class ArgInferSecondPassQuery(types.TypeQuery[bool]): """Query whether an argument type should be inferred in the second pass. The result is True if the type has a type variable in a callable return @@ -2389,7 +2389,7 @@ def visit_callable_type(self, t: CallableType) -> bool: return self.query_types(t.arg_types) or t.accept(HasTypeVarQuery()) -class HasTypeVarQuery(types.TypeQuery): +class HasTypeVarQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has a type variable component.""" def __init__(self) -> None: super().__init__(False, types.ANY_TYPE_STRATEGY) @@ -2402,7 +2402,7 @@ def has_erased_component(t: Type) -> bool: return t is not None and t.accept(HasErasedComponentsQuery()) -class HasErasedComponentsQuery(types.TypeQuery): +class HasErasedComponentsQuery(types.TypeQuery[bool]): """Visitor for querying whether a type has an erased component.""" def __init__(self) -> None: super().__init__(False, types.ANY_TYPE_STRATEGY) diff --git a/mypy/constraints.py b/mypy/constraints.py index d6e44bea857d..e1c5f47b8f99 100644 --- a/mypy/constraints.py +++ b/mypy/constraints.py @@ -250,7 +250,7 @@ def is_complete_type(typ: Type) -> bool: return typ.accept(CompleteTypeVisitor()) -class CompleteTypeVisitor(TypeQuery): +class CompleteTypeVisitor(TypeQuery[bool]): def __init__(self) -> None: super().__init__(default=True, strategy=ALL_TYPES_STRATEGY) diff --git a/mypy/stats.py b/mypy/stats.py index 2b809a6d6267..39c954bfac90 100644 --- a/mypy/stats.py +++ b/mypy/stats.py @@ -226,7 +226,7 @@ def is_imprecise(t: Type) -> bool: return t.accept(HasAnyQuery()) -class HasAnyQuery(TypeQuery): +class HasAnyQuery(TypeQuery[bool]): def __init__(self) -> None: super().__init__(False, ANY_TYPE_STRATEGY) diff --git a/mypy/types.py b/mypy/types.py index 41ef8ec238f5..54d45a14eaf2 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -5,7 +5,7 @@ from collections import OrderedDict from typing import ( Any, TypeVar, Dict, List, Tuple, cast, Generic, Set, Sequence, Optional, Union, Iterable, - NamedTuple, + NamedTuple, Callable, ) import mypy.nodes @@ -1500,112 +1500,112 @@ def keywords_str(self, a: Iterable[Tuple[str, Type]]) -> str: ]) -# These constants define the method used by TypeQuery to combine multiple -# query results, e.g. for tuple types. The strategy is not used for empty -# result lists; in that case the default value takes precedence. -ANY_TYPE_STRATEGY = 0 # Return True if any of the results are True. -ALL_TYPES_STRATEGY = 1 # Return True if all of the results are True. +# Combination strategies for boolean type queries +def ANY_TYPE_STRATEGY(current: bool, accumulated: bool) -> bool: + """True if any type's result is True""" + if accumulated: + raise ShortCircuitQuery() + return current -class TypeQuery(TypeVisitor[bool]): - """Visitor for performing simple boolean queries of types. +def ALL_TYPES_STRATEGY(current: bool, accumulated: bool) -> bool: + """True if all types' results are True""" + if not accumulated: + raise ShortCircuitQuery() + return current - This class allows defining the default value for leafs to simplify the - implementation of many queries. - """ - default = False # Default result - strategy = 0 # Strategy for combining multiple values (ANY_TYPE_STRATEGY or ALL_TYPES_...). +class ShortCircuitQuery(Exception): + pass - def __init__(self, default: bool, strategy: int) -> None: - """Construct a query visitor. - Use the given default result and strategy for combining - multiple results. The strategy must be either - ANY_TYPE_STRATEGY or ALL_TYPES_STRATEGY. - """ +class TypeQuery(Generic[T], TypeVisitor[T]): + """Visitor for performing queries of types. + + default is used as the query result unless a method for that type is + overridden. + + strategy is used to combine a partial result with a result for a particular + type in a series of types. + + Common use cases involve a boolean query using ANY_TYPE_STRATEGY and a + default of False or ALL_TYPES_STRATEGY and a default of True. + """ + + def __init__(self, default: T, strategy: Callable[[T, T], T]) -> None: self.default = default self.strategy = strategy - def visit_unbound_type(self, t: UnboundType) -> bool: + def visit_unbound_type(self, t: UnboundType) -> T: return self.default - def visit_type_list(self, t: TypeList) -> bool: + def visit_type_list(self, t: TypeList) -> T: return self.default - def visit_error_type(self, t: ErrorType) -> bool: + def visit_error_type(self, t: ErrorType) -> T: return self.default - def visit_any(self, t: AnyType) -> bool: + def visit_any(self, t: AnyType) -> T: return self.default - def visit_uninhabited_type(self, t: UninhabitedType) -> bool: + def visit_uninhabited_type(self, t: UninhabitedType) -> T: return self.default - def visit_none_type(self, t: NoneTyp) -> bool: + def visit_none_type(self, t: NoneTyp) -> T: return self.default - def visit_erased_type(self, t: ErasedType) -> bool: + def visit_erased_type(self, t: ErasedType) -> T: return self.default - def visit_deleted_type(self, t: DeletedType) -> bool: + def visit_deleted_type(self, t: DeletedType) -> T: return self.default - def visit_type_var(self, t: TypeVarType) -> bool: + def visit_type_var(self, t: TypeVarType) -> T: return self.default - def visit_partial_type(self, t: PartialType) -> bool: + def visit_partial_type(self, t: PartialType) -> T: return self.default - def visit_instance(self, t: Instance) -> bool: + def visit_instance(self, t: Instance) -> T: return self.query_types(t.args) - def visit_callable_type(self, t: CallableType) -> bool: + def visit_callable_type(self, t: CallableType) -> T: # FIX generics return self.query_types(t.arg_types + [t.ret_type]) - def visit_tuple_type(self, t: TupleType) -> bool: + def visit_tuple_type(self, t: TupleType) -> T: return self.query_types(t.items) - def visit_typeddict_type(self, t: TypedDictType) -> bool: + def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) - def visit_star_type(self, t: StarType) -> bool: + def visit_star_type(self, t: StarType) -> T: return t.type.accept(self) - def visit_union_type(self, t: UnionType) -> bool: + def visit_union_type(self, t: UnionType) -> T: return self.query_types(t.items) - def visit_overloaded(self, t: Overloaded) -> bool: + def visit_overloaded(self, t: Overloaded) -> T: return self.query_types(t.items()) - def visit_type_type(self, t: TypeType) -> bool: + def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) - def query_types(self, types: Iterable[Type]) -> bool: + def query_types(self, types: Iterable[Type]) -> T: """Perform a query for a list of types. - Use the strategy constant to combine the results. + Use the strategy to combine the results. """ if not types: # Use default result for empty list. return self.default - if self.strategy == ANY_TYPE_STRATEGY: - # Return True if at least one component is true. - res = False + res = self.default + try: for t in types: - res = res or t.accept(self) - if res: - break - return res - else: - # Return True if all components are true. - res = True - for t in types: - res = res and t.accept(self) - if not res: - break - return res + res = self.strategy(t.accept(self), res) + except ShortCircuitQuery: + pass + return res def strip_type(typ: Type) -> Type: From 3b8bc23a6def253b4fb5f7d337d8a30217a8bc49 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 29 Mar 2017 16:59:37 -0700 Subject: [PATCH 09/37] Remove redundant empty check --- mypy/types.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 54d45a14eaf2..96ffb6621a3e 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1596,9 +1596,6 @@ def query_types(self, types: Iterable[Type]) -> T: Use the strategy to combine the results. """ - if not types: - # Use default result for empty list. - return self.default res = self.default try: for t in types: From 82da3359e9d0d63b191f35c51fcf27b025232f4e Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:24:45 -0700 Subject: [PATCH 10/37] Some types in the visitor were missing recursion --- mypy/types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/types.py b/mypy/types.py index 96ffb6621a3e..1636d41688ed 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1537,10 +1537,10 @@ def __init__(self, default: T, strategy: Callable[[T, T], T]) -> None: self.strategy = strategy def visit_unbound_type(self, t: UnboundType) -> T: - return self.default + return self.query_types(t.args) def visit_type_list(self, t: TypeList) -> T: - return self.default + return self.query_types(t.items) def visit_error_type(self, t: ErrorType) -> T: return self.default @@ -1564,7 +1564,7 @@ def visit_type_var(self, t: TypeVarType) -> T: return self.default def visit_partial_type(self, t: PartialType) -> T: - return self.default + return self.query_types(t.inner_types) def visit_instance(self, t: Instance) -> T: return self.query_types(t.args) From 9f1264493930bead4180b1512830123f96b1a7d9 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:45:00 -0700 Subject: [PATCH 11/37] Add ellipsis type handler to TypeQuery --- mypy/types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/types.py b/mypy/types.py index 1636d41688ed..197d6633aef5 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -1591,6 +1591,9 @@ def visit_overloaded(self, t: Overloaded) -> T: def visit_type_type(self, t: TypeType) -> T: return t.item.accept(self) + def visit_ellipsis_type(self, t: EllipsisType) -> T: + return self.default + def query_types(self, types: Iterable[Type]) -> T: """Perform a query for a list of types. From 8cf4fa6c1d46d3f178d72fc2a4cd215ee5fe295b Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:25:45 -0700 Subject: [PATCH 12/37] Typechecks, but gives spurious error. Need to move variable binding? --- mypy/semanal.py | 160 ++++++++++++++++++---------- test-data/unit/check-functions.test | 39 +++++++ 2 files changed, 143 insertions(+), 56 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 8f285c135242..b6cd30d8c845 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -77,6 +77,7 @@ NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, TypeVarDef, TypeType, TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType, + TypeTranslator, TypeQuery ) from mypy.nodes import implicit_module_attrs from mypy.typeanal import ( @@ -152,7 +153,7 @@ FUNCTION_SECOND_PHASE = 2 # Only analyze body -class SemanticAnalyzer(NodeVisitor): +class SemanticAnalyzer(NodeVisitor, TypeTranslator): """Semantically analyze parsed mypy files. The analyzer binds names and does various consistency checks for a @@ -273,7 +274,9 @@ def visit_func_def(self, defn: FuncDef) -> None: self.function_stack.append(defn) # First phase of analysis for function. self.errors.push_function(defn.name()) - self.update_function_type_variables(defn) + if defn.type: + assert isinstance(defn.type, CallableType) + self.update_function_type_variables(defn.type, defn) self.errors.pop_function() self.function_stack.pop() @@ -366,67 +369,61 @@ def set_original_def(self, previous: Node, new: FuncDef) -> bool: else: return False - def update_function_type_variables(self, defn: FuncDef) -> None: + def update_function_type_variables(self, fun_type: CallableType, defn: Context) -> None: """Make any type variables in the signature of defn explicit. Update the signature of defn to contain type variable definitions if defn is generic. """ - if defn.type: - assert isinstance(defn.type, CallableType) - typevars = self.infer_type_variables(defn.type) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - defn.type.variables = defs + # MOVEABLE + typevars = self.infer_type_variables(fun_type) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + if typevars: + next_tvar_id = self.next_function_tvar_id() + defs = [TypeVarDef(tvar[0], next_tvar_id - i, + tvar[1].values, tvar[1].upper_bound, + tvar[1].variance) + for i, tvar in enumerate(typevars)] + fun_type.variables = defs def infer_type_variables(self, type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + # MOVEABLE """Return list of unique type variables referred to in a callable.""" names = [] # type: List[str] tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types + [type.ret_type]: - for name, tvar_expr in self.find_type_variables_in_type(arg): + for arg in type.arg_types: + for name, tvar_expr in self.find_type_variables_in_type(arg, include_callables=True): if name not in names: names.append(name) tvars.append(tvar_expr) + for name, tvar_expr in self.find_type_variables_in_type(type.ret_type, include_callables=False): + if name not in names: + names.append(name) + tvars.append(tvar_expr) return list(zip(names, tvars)) - def find_type_variables_in_type(self, type: Type) -> List[Tuple[str, TypeVarExpr]]: + def _seems_like_callable(self, type: UnboundType) -> bool: + # MOVEABLE + if not type.args: + return False + if isinstance(type.args[0], (EllipsisType, TypeList)): + return True + return False + + def find_type_variables_in_type(self, type: Type, *, include_callables: bool) -> List[Tuple[str, TypeVarExpr]]: """Return a list of all unique type variable references in type. This effectively does partial name binding, results of which are mostly thrown away. """ - result = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(type, UnboundType): - name = type.name - node = self.lookup_qualified(name, type) - if node and node.kind == UNBOUND_TVAR: - assert isinstance(node.node, TypeVarExpr) - result.append((name, node.node)) - for arg in type.args: - result.extend(self.find_type_variables_in_type(arg)) - elif isinstance(type, TypeList): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, UnionType): - for item in type.items: - result.extend(self.find_type_variables_in_type(item)) - elif isinstance(type, AnyType): - pass - elif isinstance(type, EllipsisType) or isinstance(type, TupleType): - pass - else: - assert False, 'Unsupported type %s' % type - return result + # MOVEABLE + ret = type.accept(TypeVariableQuery(self.lookup_qualified, include_callables)) + return ret def is_defined_type_var(self, tvar: str, context: Context) -> bool: + # USED ONCE; MOVEABLE return self.lookup_qualified(tvar, context).kind == BOUND_TVAR def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: @@ -547,7 +544,11 @@ def next_function_tvar_id(self) -> int: def analyze_function(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - tvarnodes = self.add_func_type_variables_to_symbol_table(defn) + if defn.type: + assert isinstance(defn.type, CallableType) + tvarnodes = self.add_func_type_variables_to_symbol_table(defn.type, defn) + else: + tvarnodes = [] next_function_tvar_id = min([self.next_function_tvar_id()] + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) self.next_function_tvar_id_stack.append(next_function_tvar_id) @@ -611,20 +612,17 @@ def check_classvar_in_signature(self, typ: Type) -> None: break def add_func_type_variables_to_symbol_table( - self, defn: FuncItem) -> List[SymbolTableNode]: + self, tt: CallableType, defn: Context) -> List[SymbolTableNode]: nodes = [] # type: List[SymbolTableNode] - if defn.type: - tt = defn.type - assert isinstance(tt, CallableType) - items = tt.variables - names = self.type_var_names() - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) + items = tt.variables + names = self.type_var_names() + for item in items: + name = item.name + if name in names: + self.name_already_defined(name, defn) + node = self.bind_type_var(name, item, defn) + nodes.append(node) + names.add(name) return nodes def type_var_names(self) -> Set[str]: @@ -1457,7 +1455,8 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file) - return t.accept(a) + return t.accept(a).accept(self) + else: return None @@ -3084,6 +3083,20 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: self.fail("'await' outside coroutine ('async def')", expr) expr.expr.accept(self) + + # + # Visitors for the TypeTranslator nature. Used for binding type vars. + # + def visit_callable_type(self, t: CallableType) -> CallableType: + print("visiting", t) + self.update_function_type_variables(t, t) + self.add_func_type_variables_to_symbol_table(t, t) + t = t.copy_modified( + arg_types=[self.anal_type(a) for a in t.arg_types], + ret_type=self.anal_type(t.ret_type), + ) + print("visited", t) + return t # # Helpers # @@ -3723,6 +3736,41 @@ def builtin_type(self, name: str, args: List[Type] = None) -> Instance: assert isinstance(sym.node, TypeInfo) return Instance(sym.node, args or []) +TypeVarList = List[Tuple[str, TypeVarExpr]] + +def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: + return a + b + +class TypeVariableQuery(TypeQuery[TypeVarList]): + + def __init__(self, lookup: Callable[[str, Context], SymbolTableNode], include_callables: bool): + self.include_callables = include_callables + self.lookup = lookup + super().__init__(default=[], strategy=_concat_type_var_lists) + + def _seems_like_callable(self, type: UnboundType) -> bool: + if not type.args: + return False + if isinstance(type.args[0], (EllipsisType, TypeList)): + return True + return False + + def visit_unbound_type(self, t: UnboundType) -> TypeVarList: + name = t.name + node = self.lookup(name, t) + if node and node.kind == UNBOUND_TVAR: + assert isinstance(node.node, TypeVarExpr) + return[(name, node.node)] + elif not self.include_callables and self._seems_like_callable(t): + return [] + else: + return super().visit_unbound_type(t) + + def visit_callable_type(self, t: CallableType) -> TypeVarList: + if self.include_callables: + return super().visit_callable_type(t) + else: + return self.default def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 0099d501b9f6..4e95ac18583e 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1703,3 +1703,42 @@ def f(a, (b, c), d): pass # flags: --python-version 2.7 def f(a, (b, c), d): pass + +-- Type variable shenanagins +-- ------------------------- + +[case testGenericFunctionTypeDecl] +from typing import Callable, TypeVar + +T = TypeVar('T') + +f: Callable[[T], T] +reveal_type(f) +def g(__x: T) -> T: pass +f = g +reveal_type(f) +i = f(3) +reveal_type(i) + +[case testFunctionReturningGenericFunction] +from typing import Callable, TypeVar + +T = TypeVar('T') +def deco() -> Callable[[T], T]: pass +reveal_type(deco) +f = deco() +reveal_type(f) +i = f(3) +reveal_type(i) + +[case testGenericFunctionOnReturnTypeOnly] +from typing import TypeVar, List + +T = TypeVar('T') + +def make_list() -> List[T]: pass + +l: List[int] = make_list() + +bad = make_list() # E: Need type annotation for variable +[builtins fixtures/list.pyi] From 2634277fdbd5aa0c735c5137453f49e414b0b237 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:49:41 -0700 Subject: [PATCH 13/37] Small change: Move tvar id generation nearer type analysis --- mypy/semanal.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index b6cd30d8c845..ffeb649c9946 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -544,16 +544,13 @@ def next_function_tvar_id(self) -> int: def analyze_function(self, defn: FuncItem) -> None: is_method = self.is_class_scope() + tvarnodes = [] if defn.type: assert isinstance(defn.type, CallableType) tvarnodes = self.add_func_type_variables_to_symbol_table(defn.type, defn) - else: - tvarnodes = [] - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) - - if defn.type: + next_function_tvar_id = min([self.next_function_tvar_id()] + + [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) + self.next_function_tvar_id_stack.append(next_function_tvar_id) # Signature must be analyzed in the surrounding scope so that # class-level imported names and type variables are in scope. self.check_classvar_in_signature(defn.type) @@ -561,6 +558,8 @@ def analyze_function(self, defn: FuncItem) -> None: self.check_function_signature(defn) if isinstance(defn, FuncDef): defn.type = set_callable_name(defn.type, defn) + else: + self.next_function_tvar_id_stack.append(self.next_function_tvar_id()) for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) From 30dba10a61b577a156b218a87f6edfbceb3795d6 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 11:51:10 -0700 Subject: [PATCH 14/37] Even nearer --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index ffeb649c9946..580aae7edf1f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -546,6 +546,7 @@ def analyze_function(self, defn: FuncItem) -> None: tvarnodes = [] if defn.type: + self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) tvarnodes = self.add_func_type_variables_to_symbol_table(defn.type, defn) next_function_tvar_id = min([self.next_function_tvar_id()] + @@ -553,7 +554,6 @@ def analyze_function(self, defn: FuncItem) -> None: self.next_function_tvar_id_stack.append(next_function_tvar_id) # Signature must be analyzed in the surrounding scope so that # class-level imported names and type variables are in scope. - self.check_classvar_in_signature(defn.type) defn.type = self.anal_type(defn.type) self.check_function_signature(defn) if isinstance(defn, FuncDef): From 7b4f9f21669e5fb09cb1cac555f7e952db39879d Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 16:08:48 -0700 Subject: [PATCH 15/37] Replace the concept of BOUND_TVAR and UNBOUND_TVAR as mutative flags with a scope object --- mypy/checkexpr.py | 5 +- mypy/nodes.py | 22 +-- mypy/semanal.py | 308 +++++++++++++---------------- mypy/typeanal.py | 21 +- test-data/unit/check-callable.test | 4 +- 5 files changed, 163 insertions(+), 197 deletions(-) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index 7410d08de950..e6c47d7ed3ff 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -21,8 +21,7 @@ DictionaryComprehension, ComplexExpr, EllipsisExpr, StarExpr, AwaitExpr, YieldExpr, YieldFromExpr, TypedDictExpr, PromoteExpr, NewTypeExpr, NamedTupleExpr, TypeVarExpr, TypeAliasExpr, BackquoteExpr, EnumCallExpr, - ARG_POS, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, - UNBOUND_TVAR, BOUND_TVAR, LITERAL_TYPE + ARG_POS, ARG_NAMED, ARG_STAR, ARG_STAR2, MODULE_REF, TVAR, LITERAL_TYPE, ) from mypy import nodes import mypy.checker @@ -1610,7 +1609,7 @@ def replace_tvars_any(self, tp: Type) -> Type: sym = self.chk.lookup_qualified(arg.name) except KeyError: pass - if sym and (sym.kind == UNBOUND_TVAR or sym.kind == BOUND_TVAR): + if sym and (sym.kind == TVAR): new_args[i] = AnyType() else: new_args[i] = self.replace_tvars_any(arg) diff --git a/mypy/nodes.py b/mypy/nodes.py index 4584245b9904..9898a176890d 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -39,12 +39,12 @@ def get_column(self) -> int: pass GDEF = 1 # type: int MDEF = 2 # type: int MODULE_REF = 3 # type: int -# Type variable declared using TypeVar(...) has kind UNBOUND_TVAR. It's not -# valid as a type. A type variable is valid as a type (kind BOUND_TVAR) within +# Type variable declared using TypeVar(...) has kind TVAR. It's not +# valid as a type unless bound in a TypeVarScope. That happens within: # (1) a generic class that uses the type variable as a type argument or # (2) a generic function that refers to the type variable in its signature. -UNBOUND_TVAR = 4 # type: int -BOUND_TVAR = 5 # type: int +TVAR = 4 # type: int + TYPE_ALIAS = 6 # type: int # Placeholder for a name imported via 'from ... import'. Second phase of # semantic will replace this the actual imported reference. This is @@ -65,8 +65,7 @@ def get_column(self) -> int: pass GDEF: 'Gdef', MDEF: 'Mdef', MODULE_REF: 'ModuleRef', - UNBOUND_TVAR: 'UnboundTvar', - BOUND_TVAR: 'Tvar', + TVAR: 'Tvar', TYPE_ALIAS: 'TypeAlias', UNBOUND_IMPORTED: 'UnboundImported', } @@ -2181,8 +2180,7 @@ class SymbolTableNode: # - LDEF: local definition (of any kind) # - GDEF: global (module-level) definition # - MDEF: class member definition - # - UNBOUND_TVAR: TypeVar(...) definition, not bound - # - TVAR: type variable in a bound scope (generic function / generic clas) + # - TVAR: TypeVar(...) definition # - MODULE_REF: reference to a module # - TYPE_ALIAS: type alias # - UNBOUND_IMPORTED: temporary kind for imported names @@ -2190,8 +2188,6 @@ class SymbolTableNode: # AST node of definition (FuncDef/Var/TypeInfo/Decorator/TypeVarExpr, # or None for a bound type variable). node = None # type: Optional[SymbolNode] - # Type variable definition (for bound type variables only) - tvar_def = None # type: Optional[mypy.types.TypeVarDef] # Module id (e.g. "foo.bar") or None mod_id = '' # If this not None, override the type of the 'node' attribute. @@ -2207,13 +2203,11 @@ class SymbolTableNode: def __init__(self, kind: int, node: Optional[SymbolNode], mod_id: str = None, typ: 'mypy.types.Type' = None, - tvar_def: 'mypy.types.TypeVarDef' = None, module_public: bool = True, normalized: bool = False) -> None: self.kind = kind self.node = node self.type_override = typ self.mod_id = mod_id - self.tvar_def = tvar_def self.module_public = module_public self.normalized = normalized @@ -2257,8 +2251,6 @@ def serialize(self, prefix: str, name: str) -> JsonDict: data = {'.class': 'SymbolTableNode', 'kind': node_kinds[self.kind], } # type: JsonDict - if self.tvar_def: - data['tvar_def'] = self.tvar_def.serialize() if not self.module_public: data['module_public'] = False if self.kind == MODULE_REF: @@ -2300,8 +2292,6 @@ def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': if 'type_override' in data: typ = mypy.types.deserialize_type(data['type_override']) stnode = SymbolTableNode(kind, node, typ=typ) - if 'tvar_def' in data: - stnode.tvar_def = mypy.types.TypeVarDef.deserialize(data['tvar_def']) if 'module_public' in data: stnode.module_public = data['module_public'] return stnode diff --git a/mypy/semanal.py b/mypy/semanal.py index 580aae7edf1f..c3a7f71b48f4 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -44,6 +44,8 @@ """ from collections import OrderedDict +from contextlib import contextmanager + from typing import ( List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable ) @@ -57,7 +59,7 @@ ForStmt, BreakStmt, ContinueStmt, IfStmt, TryStmt, WithStmt, DelStmt, PassStmt, GlobalDecl, SuperExpr, DictExpr, CallExpr, RefExpr, OpExpr, UnaryExpr, SliceExpr, CastExpr, RevealTypeExpr, TypeApplication, Context, SymbolTable, - SymbolTableNode, BOUND_TVAR, UNBOUND_TVAR, ListComprehension, GeneratorExpr, + SymbolTableNode, TVAR, ListComprehension, GeneratorExpr, LambdaExpr, MDEF, FuncBase, Decorator, SetExpr, TypeVarExpr, NewTypeExpr, StrExpr, BytesExpr, PrintStmt, ConditionalExpr, PromoteExpr, ComparisonExpr, StarExpr, ARG_POS, ARG_NAMED, ARG_NAMED_OPT, MroError, type_aliases, @@ -68,6 +70,7 @@ COVARIANT, CONTRAVARIANT, INVARIANT, UNBOUND_IMPORTED, LITERAL_YES, ARG_OPT, nongen_builtins, collections_type_aliases, get_member_expr_fullname, ) +from mypy.tvar_scope import TypeVarScope from mypy.typevars import has_no_typevars, fill_typevars from mypy.visitor import NodeVisitor from mypy.traverser import TraverserVisitor @@ -184,14 +187,12 @@ class SemanticAnalyzer(NodeVisitor, TypeTranslator): # Type variables that are bound by the directly enclosing class bound_tvars = None # type: List[SymbolTableNode] # Stack of type variables that were bound by outer classess - tvar_stack = None # type: List[List[SymbolTableNode]] + tvar_scope = None # type: TypeVarScope # Per-module options options = None # type: Options # Stack of functions being analyzed function_stack = None # type: List[FuncItem] - # Stack of next available function type variable ids - next_function_tvar_id_stack = None # type: List[int] # Status of postponing analysis of nested function bodies. By using this we # can have mutually recursive nested functions. Values are FUNCTION_x @@ -220,10 +221,8 @@ def __init__(self, self.imports = set() self.type = None self.type_stack = [] - self.bound_tvars = None - self.tvar_stack = [] + self.tvar_scope = TypeVarScope() self.function_stack = [] - self.next_function_tvar_id_stack = [-1] self.block_depth = [0] self.loop_depth = 0 self.lib_path = lib_path @@ -269,74 +268,75 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None: del self.options def visit_func_def(self, defn: FuncDef) -> None: - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - if defn.type: - assert isinstance(defn.type, CallableType) - self.update_function_type_variables(defn.type, defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if not self.set_original_def(n, defn): - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if not self.set_original_def(n, defn): - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - if not self.set_original_def(symbol.node, defn): - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - if defn.is_coroutine and isinstance(defn.type, CallableType): - if defn.is_async_generator: - # Async generator types are handled elsewhere - pass + with self.tvar_scope_frame(): + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + if defn.type: + assert isinstance(defn.type, CallableType) + self.update_function_type_variables(defn.type, defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if not self.set_original_def(n, defn): + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if not self.set_original_def(n, defn): + self.name_already_defined(defn.name(), defn) + else: + self.add_local(defn, defn) else: - # A coroutine defined as `async def foo(...) -> T: ...` - # has external return type `Awaitable[T]`. - defn.type = defn.type.copy_modified( - ret_type = self.named_type_or_none('typing.Awaitable', - [defn.type.ret_type])) - self.errors.pop_function() + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + if not self.set_original_def(symbol.node, defn): + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + if defn.is_coroutine and isinstance(defn.type, CallableType): + if defn.is_async_generator: + # Async generator types are handled elsewhere + pass + else: + # A coroutine defined as `async def foo(...) -> T: ...` + # has external return type `Awaitable[T]`. + defn.type = defn.type.copy_modified( + ret_type = self.named_type_or_none('typing.Awaitable', + [defn.type.ret_type])) + self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: """Check basic signature validity and tweak annotation of self/cls argument.""" @@ -376,17 +376,19 @@ def update_function_type_variables(self, fun_type: CallableType, defn: Context) if defn is generic. """ # MOVEABLE + if fun_type.variables: + print("Already got to function", fun_type) + return typevars = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. typevars = [(name, tvar) for name, tvar in typevars if not self.is_defined_type_var(name, defn)] - if typevars: - next_tvar_id = self.next_function_tvar_id() - defs = [TypeVarDef(tvar[0], next_tvar_id - i, - tvar[1].values, tvar[1].upper_bound, - tvar[1].variance) - for i, tvar in enumerate(typevars)] - fun_type.variables = defs + defs = [] # type: List[TypeVarDef] + for name, tvar in typevars: + self.tvar_scope.bind_fun_tvar(name, tvar) + defs.append(self.tvar_scope.get_binding(tvar.fullname())) + print("Setting variables of ", fun_type, "to", defs) + fun_type.variables = defs def infer_type_variables(self, type: CallableType) -> List[Tuple[str, TypeVarExpr]]: @@ -419,12 +421,12 @@ def find_type_variables_in_type(self, type: Type, *, include_callables: bool) -> This effectively does partial name binding, results of which are mostly thrown away. """ # MOVEABLE - ret = type.accept(TypeVariableQuery(self.lookup_qualified, include_callables)) + ret = type.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope, include_callables)) return ret def is_defined_type_var(self, tvar: str, context: Context) -> bool: # USED ONCE; MOVEABLE - return self.lookup_qualified(tvar, context).kind == BOUND_TVAR + return self.tvar_scope.get_binding(self.lookup_qualified(tvar, context)) is not None def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # OverloadedFuncDef refers to any legitimate situation where you have @@ -538,28 +540,18 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - else: self.fail("Decorated property not supported", item) - def next_function_tvar_id(self) -> int: - return self.next_function_tvar_id_stack[-1] - def analyze_function(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - - tvarnodes = [] if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) - tvarnodes = self.add_func_type_variables_to_symbol_table(defn.type, defn) - next_function_tvar_id = min([self.next_function_tvar_id()] + - [n.tvar_def.id.raw_id - 1 for n in tvarnodes]) - self.next_function_tvar_id_stack.append(next_function_tvar_id) # Signature must be analyzed in the surrounding scope so that # class-level imported names and type variables are in scope. defn.type = self.anal_type(defn.type) self.check_function_signature(defn) if isinstance(defn, FuncDef): defn.type = set_callable_name(defn.type, defn) - else: - self.next_function_tvar_id_stack.append(self.next_function_tvar_id()) + for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) @@ -590,9 +582,6 @@ def analyze_function(self, defn: FuncItem) -> None: self.postpone_nested_functions_stack.pop() self.postponed_functions_stack.pop() - self.next_function_tvar_id_stack.pop() - disable_typevars(tvarnodes) - self.leave() self.function_stack.pop() @@ -633,8 +622,7 @@ def type_var_names(self) -> Set[str]: def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, context: Context) -> SymbolTableNode: node = self.lookup_qualified(fullname, context) - node.kind = BOUND_TVAR - node.tvar_def = tvar_def + self.tvar_scope.bind(node, tvar_def) return node def check_function_signature(self, fdef: FuncItem) -> None: @@ -649,36 +637,33 @@ def check_function_signature(self, fdef: FuncItem) -> None: self.fail('Type signature has too many arguments', fdef, blocker=True) def visit_class_def(self, defn: ClassDef) -> None: - self.clean_up_bases_and_infer_type_variables(defn) - if self.analyze_typeddict_classdef(defn): - return - if self.analyze_namedtuple_classdef(defn): - # just analyze the class body so we catch type errors in default values - self.enter_class(defn) - defn.defs.accept(self) - self.leave_class() - else: - self.setup_class_def_analysis(defn) - - self.bind_class_type_vars(defn) - - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) + with self.new_tvar_scope(): + self.clean_up_bases_and_infer_type_variables(defn) + if self.analyze_typeddict_classdef(defn): + return + if self.analyze_namedtuple_classdef(defn): + # just analyze the class body so we catch type errors in default values + self.enter_class(defn) + defn.defs.accept(self) + self.leave_class() + else: + self.setup_class_def_analysis(defn) + # self.bind_class_type_variables_in_symbol_table(defn.info) + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) - self.enter_class(defn) + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) - # Analyze class body. - defn.defs.accept(self) + self.enter_class(defn) - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) + # Analyze class body. + defn.defs.accept(self) - self.leave_class() + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) - self.unbind_class_type_vars() + self.leave_class() def enter_class(self, defn: ClassDef) -> None: # Remember previous active class @@ -695,24 +680,6 @@ def leave_class(self) -> None: self.locals.pop() self.type = self.type_stack.pop() - def bind_class_type_vars(self, defn: ClassDef) -> None: - """ Unbind type variables of previously active class and bind - the type variables for the active class. - """ - if self.bound_tvars: - disable_typevars(self.bound_tvars) - self.tvar_stack.append(self.bound_tvars) - self.bound_tvars = self.bind_class_type_variables_in_symbol_table(defn.info) - - def unbind_class_type_vars(self) -> None: - """ Unbind the active class' type vars and rebind the - type vars of the previously active class. - """ - disable_typevars(self.bound_tvars) - self.bound_tvars = self.tvar_stack.pop() - if self.bound_tvars: - enable_typevars(self.bound_tvars) - def analyze_class_decorator(self, defn: ClassDef, decorator: Expression) -> None: decorator.accept(self) @@ -801,15 +768,16 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: declared_tvars = self.remove_dups(declared_tvars + all_tvars) else: declared_tvars = all_tvars - for j, (name, tvar_expr) in enumerate(declared_tvars): - type_vars.append(TypeVarDef(name, j + 1, tvar_expr.values, - tvar_expr.upper_bound, tvar_expr.variance)) - if type_vars: - defn.type_vars = type_vars + if declared_tvars: if defn.info: - defn.info.type_vars = [tv.name for tv in type_vars] + defn.info.type_vars = [name for name, _ in declared_tvars] for i in reversed(removed): del defn.base_type_exprs[i] + tvar_defs = [] # type: List[TypeVarDef] + for name, tvar_expr in declared_tvars: + self.tvar_scope.bind_class_tvar(name, tvar_expr) + tvar_defs.append(self.tvar_scope.get_binding(tvar_expr.fullname())) + defn.type_vars = tvar_defs def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: if not isinstance(t, UnboundType): @@ -835,7 +803,7 @@ def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: return None unbound = t sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == UNBOUND_TVAR: + if sym is not None and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None: assert isinstance(sym.node, TypeVarExpr) return unbound.name, sym.node return None @@ -1450,6 +1418,7 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False, if t: a = TypeAnalyser(self.lookup_qualified, self.lookup_fully_qualified, + self.tvar_scope, self.fail, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal, @@ -1859,7 +1828,7 @@ def process_typevar_declaration(self, s: AssignmentStmt) -> None: # Yes, it's a valid type variable definition! Add it to the symbol table. node = self.lookup(name, s) - node.kind = UNBOUND_TVAR + node.kind = TVAR TypeVar = TypeVarExpr(name, node.fullname, values, upper_bound, variance) TypeVar.line = call.line call.analyzed = TypeVar @@ -2716,7 +2685,7 @@ def visit_exec_stmt(self, s: ExecStmt) -> None: def visit_name_expr(self, expr: NameExpr) -> None: n = self.lookup(expr.name, expr) if n: - if n.kind == BOUND_TVAR: + if n.kind == TVAR and self.tvar_scope.get_binding(n): self.fail("'{}' is a type variable and only valid in type " "context".format(expr.name), expr) else: @@ -3087,19 +3056,30 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: # Visitors for the TypeTranslator nature. Used for binding type vars. # def visit_callable_type(self, t: CallableType) -> CallableType: - print("visiting", t) self.update_function_type_variables(t, t) - self.add_func_type_variables_to_symbol_table(t, t) t = t.copy_modified( arg_types=[self.anal_type(a) for a in t.arg_types], ret_type=self.anal_type(t.ret_type), ) - print("visited", t) return t # # Helpers # + @contextmanager + def new_tvar_scope(self): + old_scope = self.tvar_scope + self.tvar_scope = TypeVarScope() + yield + self.tvar_scope = old_scope + + @contextmanager + def tvar_scope_frame(self): + old_scope = self.tvar_scope + self.tvar_scope = TypeVarScope(self.tvar_scope) + yield + self.tvar_scope = old_scope + def lookup(self, name: str, ctx: Context) -> SymbolTableNode: """Look up an unqualified name in all active namespaces.""" # 1a. Name declared using 'global x' takes precedence @@ -3742,9 +3722,13 @@ def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: class TypeVariableQuery(TypeQuery[TypeVarList]): - def __init__(self, lookup: Callable[[str, Context], SymbolTableNode], include_callables: bool): + def __init__(self, + lookup: Callable[[str, Context], SymbolTableNode], + scope: 'TypeVarScope', + include_callables: bool): self.include_callables = include_callables self.lookup = lookup + self.scope = scope super().__init__(default=[], strategy=_concat_type_var_lists) def _seems_like_callable(self, type: UnboundType) -> bool: @@ -3757,7 +3741,7 @@ def _seems_like_callable(self, type: UnboundType) -> bool: def visit_unbound_type(self, t: UnboundType) -> TypeVarList: name = t.name node = self.lookup(name, t) - if node and node.kind == UNBOUND_TVAR: + if node and node.kind == TVAR and self.scope.get_binding(node) is None: assert isinstance(node.node, TypeVarExpr) return[(name, node.node)] elif not self.include_callables and self._seems_like_callable(t): @@ -3825,18 +3809,6 @@ def find_duplicate(list: List[T]) -> T: return None -def disable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = UNBOUND_TVAR - - -def enable_typevars(nodes: List[SymbolTableNode]) -> None: - for node in nodes: - assert node.kind in (BOUND_TVAR, UNBOUND_TVAR) - node.kind = BOUND_TVAR - - def remove_imported_names_from_symtable(names: SymbolTable, module: str) -> None: """Remove all imported names from the symbol table of a module.""" diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e89a17a1a998..3e99d65a9ac1 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -10,10 +10,11 @@ get_type_vars, ) from mypy.nodes import ( - BOUND_TVAR, UNBOUND_TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, + TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, IndexExpr, RefExpr, nongen_builtins, ) +from mypy.tvar_scope import TypeVarScope from mypy.sametypes import is_same_type from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.subtypes import is_subtype @@ -46,7 +47,7 @@ def analyze_type_alias(node: Expression, # Note that this misses the case where someone tried to use a # class-referenced type variable as a type alias. It's easier to catch # that one in checkmember.py - if node.kind == UNBOUND_TVAR or node.kind == BOUND_TVAR: + if node.kind == TVAR: fail_func('Type variable "{}" is invalid as target for type alias'.format( node.fullname), node) return None @@ -72,7 +73,7 @@ def analyze_type_alias(node: Expression, except TypeTranslationError: fail_func('Invalid type alias', node) return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, fail_func, aliasing=True, + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, TypeVarScope(), fail_func, aliasing=True, allow_unnormalized=allow_unnormalized) return type.accept(analyzer) @@ -94,6 +95,7 @@ class TypeAnalyser(TypeVisitor[Type]): def __init__(self, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], + tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], *, aliasing: bool = False, allow_tuple_literal: bool = False, @@ -101,6 +103,7 @@ def __init__(self, self.lookup = lookup_func self.lookup_fqn_func = lookup_fqn_func self.fail = fail_func + self.tvar_scope = tvar_scope self.aliasing = aliasing self.allow_tuple_literal = allow_tuple_literal # Positive if we are analyzing arguments of another (outer) type @@ -124,12 +127,13 @@ def visit_unbound_type(self, t: UnboundType) -> Type: if (fullname in nongen_builtins and t.args and not sym.normalized and not self.allow_unnormalized): self.fail(no_subscript_builtin_alias(fullname), t) - if sym.kind == BOUND_TVAR: + if sym.kind == TVAR and self.tvar_scope.get_binding(sym) is not None: + # XXX: Don't fetch twice + tvar_def = self.tvar_scope.get_binding(sym) if len(t.args) > 0: self.fail('Type variable "{}" used with arguments'.format( t.name), t) - assert sym.tvar_def is not None - return TypeVarType(sym.tvar_def, t.line) + return TypeVarType(tvar_def, t.line) elif fullname == 'builtins.None': return NoneTyp() elif fullname == 'typing.Any' or fullname == 'builtins.Any': @@ -207,7 +211,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # is pretty minor. return AnyType() # Allow unbound type variables when defining an alias - if not (self.aliasing and sym.kind == UNBOUND_TVAR): + if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) == None): + print("scope is", self.tvar_scope, "looking for", sym.fullname) self.fail('Invalid type "{}"'.format(name), t) return t info = sym.node # type: TypeInfo @@ -272,7 +277,7 @@ def get_tvar_name(self, t: Type) -> Optional[str]: if not isinstance(t, UnboundType): return None sym = self.lookup(t.name, t) - if sym is not None and (sym.kind == UNBOUND_TVAR or sym.kind == BOUND_TVAR): + if sym is not None and sym.kind == TVAR: return t.name return None diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test index 429ad4494e0a..3bc02bb3f8f0 100644 --- a/test-data/unit/check-callable.test +++ b/test-data/unit/check-callable.test @@ -253,9 +253,9 @@ T = TypeVar('T', int, Callable[[], int], Union[str, Callable[[], str]]) def f(t: T) -> None: if callable(t): - reveal_type(t()) # E: Revealed type is 'builtins.int' # E: Revealed type is 'builtins.str' + reveal_type(t()) # E: Revealed type is 'Union[builtins.int, builtins.str]' else: - reveal_type(t) # E: Revealed type is 'builtins.int*' # E: Revealed type is 'builtins.str' + reveal_type(t) # E: # E: Revealed type is 'Union[builtins.int, builtins.str]' [builtins fixtures/callable.pyi] From daf959237a3276da639d7265cbf6bbafcd17737c Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 16:28:31 -0700 Subject: [PATCH 16/37] Mid debugging gotta push now --- mypy/semanal.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index c3a7f71b48f4..0ae4a3dd07b2 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -377,7 +377,9 @@ def update_function_type_variables(self, fun_type: CallableType, defn: Context) """ # MOVEABLE if fun_type.variables: - print("Already got to function", fun_type) + print("Already got to function", fun_type, "scope", self.tvar_scope) + for var in fun_type.variables: + self.tvar_scope.bind_fun_tvar(var.name, self.lookup_qualified(var.name, var).node) return typevars = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. @@ -387,7 +389,7 @@ def update_function_type_variables(self, fun_type: CallableType, defn: Context) for name, tvar in typevars: self.tvar_scope.bind_fun_tvar(name, tvar) defs.append(self.tvar_scope.get_binding(tvar.fullname())) - print("Setting variables of ", fun_type, "to", defs) + print("Setting variables of ", fun_type, "to", defs, "scope now", self.tvar_scope) fun_type.variables = defs def infer_type_variables(self, @@ -1423,6 +1425,7 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file) + print("analyzing", t, "scope", self.tvar_scope) return t.accept(a).accept(self) else: From 826b9cfbcc16988c603e2808c690e52fa2e1269d Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 23:36:27 -0700 Subject: [PATCH 17/37] Checkpoint. Most tests pass, classvar type aliases not yet --- mypy/checkmember.py | 8 +- mypy/expandtype.py | 1 + mypy/semanal.py | 337 +++++++++++------------------ mypy/typeanal.py | 132 +++++++++-- test-data/unit/check-callable.test | 4 +- test-data/unit/check-classes.test | 2 + 6 files changed, 255 insertions(+), 229 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 6c934dccb022..34bbde9803de 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -53,6 +53,7 @@ def analyze_member_access(name: str, original_type is always the type used in the initial call. """ if isinstance(typ, Instance): + print("Getting member", name, "of", typ) if name == '__init__' and not is_super: # Accessing __init__ in statically typed code would compromise # type safety unless used via super(). @@ -73,6 +74,7 @@ def analyze_member_access(name: str, # Look up the member. First look up the method dictionary. method = info.get_method(name) if method: + print("It is a method!", method) if method.is_property: assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) @@ -213,6 +215,8 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, """ # It was not a method. Try looking up a variable. v = lookup_member_var_or_accessor(info, name, is_lvalue) + print("It's a var?", v, "itype", itype, "type", v.type) + print("names", info.names) vv = v if isinstance(vv, Decorator): @@ -490,7 +494,9 @@ def type_object_type(info: TypeInfo, builtin_type: Callable[[str], Instance]) -> return class_callable(sig, info, fallback, None) # Construct callable type based on signature of __init__. Adjust # return type and insert type arguments. - return type_object_type_from_function(init_method, info, fallback) + ret = type_object_type_from_function(init_method, info, fallback) + print("Making callable based on __init__ of", info.name(), ret) + return ret def type_object_type_from_function(init_or_new: FuncBase, info: TypeInfo, diff --git a/mypy/expandtype.py b/mypy/expandtype.py index 41c74daf6ea3..fd44c5d5b279 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -23,6 +23,7 @@ def expand_type_by_instance(typ: Type, instance: Instance) -> Type: if instance.args == []: return typ else: + print("Ok so the instance", instance, "has some args", instance.args) variables = {} # type: Dict[TypeVarId, Type] for binder, arg in zip(instance.type.defn.type_vars, instance.args): variables[binder.id] = arg diff --git a/mypy/semanal.py b/mypy/semanal.py index 0ae4a3dd07b2..877d866edab1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -268,75 +268,75 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options) -> None: del self.options def visit_func_def(self, defn: FuncDef) -> None: - with self.tvar_scope_frame(): - phase_info = self.postpone_nested_functions_stack[-1] - if phase_info != FUNCTION_SECOND_PHASE: - self.function_stack.append(defn) - # First phase of analysis for function. - self.errors.push_function(defn.name()) - if defn.type: - assert isinstance(defn.type, CallableType) - self.update_function_type_variables(defn.type, defn) - self.errors.pop_function() - self.function_stack.pop() - - defn.is_conditional = self.block_depth[-1] > 0 - - # TODO(jukka): Figure out how to share the various cases. It doesn't - # make sense to have (almost) duplicate code (here and elsewhere) for - # 3 cases: module-level, class-level and local names. Maybe implement - # a common stack of namespaces. As the 3 kinds of namespaces have - # different semantics, this wouldn't always work, but it might still - # be a win. - if self.is_class_scope(): - # Method definition - defn.info = self.type - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.type.names: - # Redefinition. Conditional redefinition is okay. - n = self.type.names[defn.name()].node - if not self.set_original_def(n, defn): - self.name_already_defined(defn.name(), defn) - self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) - self.prepare_method_signature(defn) - elif self.is_func_scope(): - # Nested function - if not defn.is_decorated and not defn.is_overload: - if defn.name() in self.locals[-1]: - # Redefinition. Conditional redefinition is okay. - n = self.locals[-1][defn.name()].node - if not self.set_original_def(n, defn): - self.name_already_defined(defn.name(), defn) - else: - self.add_local(defn, defn) - else: - # Top-level function - if not defn.is_decorated and not defn.is_overload: - symbol = self.globals.get(defn.name()) - if isinstance(symbol.node, FuncDef) and symbol.node != defn: - # This is redefinition. Conditional redefinition is okay. - if not self.set_original_def(symbol.node, defn): - # Report error. - self.check_no_global(defn.name(), defn, True) - if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Postpone this function (for the second phase). - self.postponed_functions_stack[-1].append(defn) - return - if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: - # Second phase of analysis for function. - self.errors.push_function(defn.name()) - self.analyze_function(defn) - if defn.is_coroutine and isinstance(defn.type, CallableType): - if defn.is_async_generator: - # Async generator types are handled elsewhere - pass + + phase_info = self.postpone_nested_functions_stack[-1] + if phase_info != FUNCTION_SECOND_PHASE: + self.function_stack.append(defn) + # First phase of analysis for function. + self.errors.push_function(defn.name()) + if defn.type: + assert isinstance(defn.type, CallableType) + self.update_function_type_variables(defn.type, defn) + self.errors.pop_function() + self.function_stack.pop() + + defn.is_conditional = self.block_depth[-1] > 0 + + # TODO(jukka): Figure out how to share the various cases. It doesn't + # make sense to have (almost) duplicate code (here and elsewhere) for + # 3 cases: module-level, class-level and local names. Maybe implement + # a common stack of namespaces. As the 3 kinds of namespaces have + # different semantics, this wouldn't always work, but it might still + # be a win. + if self.is_class_scope(): + # Method definition + defn.info = self.type + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.type.names: + # Redefinition. Conditional redefinition is okay. + n = self.type.names[defn.name()].node + if not self.set_original_def(n, defn): + self.name_already_defined(defn.name(), defn) + self.type.names[defn.name()] = SymbolTableNode(MDEF, defn) + self.prepare_method_signature(defn) + elif self.is_func_scope(): + # Nested function + if not defn.is_decorated and not defn.is_overload: + if defn.name() in self.locals[-1]: + # Redefinition. Conditional redefinition is okay. + n = self.locals[-1][defn.name()].node + if not self.set_original_def(n, defn): + self.name_already_defined(defn.name(), defn) else: - # A coroutine defined as `async def foo(...) -> T: ...` - # has external return type `Awaitable[T]`. - defn.type = defn.type.copy_modified( - ret_type = self.named_type_or_none('typing.Awaitable', - [defn.type.ret_type])) - self.errors.pop_function() + self.add_local(defn, defn) + else: + # Top-level function + if not defn.is_decorated and not defn.is_overload: + symbol = self.globals.get(defn.name()) + if isinstance(symbol.node, FuncDef) and symbol.node != defn: + # This is redefinition. Conditional redefinition is okay. + if not self.set_original_def(symbol.node, defn): + # Report error. + self.check_no_global(defn.name(), defn, True) + if phase_info == FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Postpone this function (for the second phase). + self.postponed_functions_stack[-1].append(defn) + return + if phase_info != FUNCTION_FIRST_PHASE_POSTPONE_SECOND: + # Second phase of analysis for function. + self.errors.push_function(defn.name()) + self.analyze_function(defn) + if defn.is_coroutine and isinstance(defn.type, CallableType): + if defn.is_async_generator: + # Async generator types are handled elsewhere + pass + else: + # A coroutine defined as `async def foo(...) -> T: ...` + # has external return type `Awaitable[T]`. + defn.type = defn.type.copy_modified( + ret_type = self.named_type_or_none('typing.Awaitable', + [defn.type.ret_type])) + self.errors.pop_function() def prepare_method_signature(self, func: FuncDef) -> None: """Check basic signature validity and tweak annotation of self/cls argument.""" @@ -375,60 +375,10 @@ def update_function_type_variables(self, fun_type: CallableType, defn: Context) Update the signature of defn to contain type variable definitions if defn is generic. """ - # MOVEABLE - if fun_type.variables: - print("Already got to function", fun_type, "scope", self.tvar_scope) - for var in fun_type.variables: - self.tvar_scope.bind_fun_tvar(var.name, self.lookup_qualified(var.name, var).node) - return - typevars = self.infer_type_variables(fun_type) - # Do not define a new type variable if already defined in scope. - typevars = [(name, tvar) for name, tvar in typevars - if not self.is_defined_type_var(name, defn)] - defs = [] # type: List[TypeVarDef] - for name, tvar in typevars: - self.tvar_scope.bind_fun_tvar(name, tvar) - defs.append(self.tvar_scope.get_binding(tvar.fullname())) - print("Setting variables of ", fun_type, "to", defs, "scope now", self.tvar_scope) - fun_type.variables = defs - - def infer_type_variables(self, - type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - # MOVEABLE - """Return list of unique type variables referred to in a callable.""" - names = [] # type: List[str] - tvars = [] # type: List[TypeVarExpr] - for arg in type.arg_types: - for name, tvar_expr in self.find_type_variables_in_type(arg, include_callables=True): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - for name, tvar_expr in self.find_type_variables_in_type(type.ret_type, include_callables=False): - if name not in names: - names.append(name) - tvars.append(tvar_expr) - return list(zip(names, tvars)) - - def _seems_like_callable(self, type: UnboundType) -> bool: - # MOVEABLE - if not type.args: - return False - if isinstance(type.args[0], (EllipsisType, TypeList)): - return True - return False - - def find_type_variables_in_type(self, type: Type, *, include_callables: bool) -> List[Tuple[str, TypeVarExpr]]: - """Return a list of all unique type variable references in type. - - This effectively does partial name binding, results of which are mostly thrown away. - """ - # MOVEABLE - ret = type.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope, include_callables)) - return ret - def is_defined_type_var(self, tvar: str, context: Context) -> bool: - # USED ONCE; MOVEABLE - return self.tvar_scope.get_binding(self.lookup_qualified(tvar, context)) is not None + a = self.type_analyzer(tvar_scope=TypeVarScope(self.tvar_scope)) + variables = a.bind_function_type_variables(fun_type, defn) + fun_type.variables = variables def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # OverloadedFuncDef refers to any legitimate situation where you have @@ -557,35 +507,40 @@ def analyze_function(self, defn: FuncItem) -> None: for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) - self.function_stack.append(defn) - self.enter() - for arg in defn.arguments: - self.add_local(arg.variable, defn) - for arg in defn.arguments: - if arg.initialization_statement: - lvalue = arg.initialization_statement.lvalues[0] - lvalue.accept(self) - - # The first argument of a non-static, non-class method is like 'self' - # (though the name could be different), having the enclosing class's - # instance type. - if is_method and not defn.is_static and not defn.is_class and defn.arguments: - defn.arguments[0].variable.is_self = True - - # First analyze body of the function but ignore nested functions. - self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) - self.postponed_functions_stack.append([]) - defn.body.accept(self) - - # Analyze nested functions (if any) as a second phase. - self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE - for postponed in self.postponed_functions_stack[-1]: - postponed.accept(self) - self.postpone_nested_functions_stack.pop() - self.postponed_functions_stack.pop() - - self.leave() - self.function_stack.pop() + with self.tvar_scope_frame(): + # Bind the type variables again to visit the body. + if defn.type: + a = self.type_analyzer() + a.bind_function_type_variables(defn.type, defn) + self.function_stack.append(defn) + self.enter() + for arg in defn.arguments: + self.add_local(arg.variable, defn) + for arg in defn.arguments: + if arg.initialization_statement: + lvalue = arg.initialization_statement.lvalues[0] + lvalue.accept(self) + + # The first argument of a non-static, non-class method is like 'self' + # (though the name could be different), having the enclosing class's + # instance type. + if is_method and not defn.is_static and not defn.is_class and defn.arguments: + defn.arguments[0].variable.is_self = True + + # First analyze body of the function but ignore nested functions. + self.postpone_nested_functions_stack.append(FUNCTION_FIRST_PHASE_POSTPONE_SECOND) + self.postponed_functions_stack.append([]) + defn.body.accept(self) + + # Analyze nested functions (if any) as a second phase. + self.postpone_nested_functions_stack[-1] = FUNCTION_SECOND_PHASE + for postponed in self.postponed_functions_stack[-1]: + postponed.accept(self) + self.postpone_nested_functions_stack.pop() + self.postponed_functions_stack.pop() + + self.leave() + self.function_stack.pop() def check_classvar_in_signature(self, typ: Type) -> None: t = None # type: Type @@ -658,6 +613,7 @@ def visit_class_def(self, defn: ClassDef) -> None: self.analyze_class_decorator(defn, decorator) self.enter_class(defn) + print("Entered class", defn.name, self.tvar_scope) # Analyze class body. defn.defs.accept(self) @@ -1415,23 +1371,34 @@ def visit_block_maybe(self, b: Block) -> None: if b: self.visit_block(b) + def type_analyzer(self, + tvar_scope: Optional[TypeVarScope] = None, + allow_tuple_literal: bool = False, + aliasing: bool = False) -> TypeAnalyser: + if tvar_scope is None: + tvar_scope = self.tvar_scope + return TypeAnalyser(self.lookup_qualified, + self.lookup_fully_qualified, + tvar_scope, + self.fail, + aliasing=aliasing, + allow_tuple_literal=allow_tuple_literal, + allow_unnormalized=self.is_stub_file) + def anal_type(self, t: Type, allow_tuple_literal: bool = False, aliasing: bool = False) -> Type: if t: - a = TypeAnalyser(self.lookup_qualified, - self.lookup_fully_qualified, - self.tvar_scope, - self.fail, - aliasing=aliasing, - allow_tuple_literal=allow_tuple_literal, - allow_unnormalized=self.is_stub_file) + a = self.type_analyzer( + aliasing=aliasing, + allow_tuple_literal=allow_tuple_literal) print("analyzing", t, "scope", self.tvar_scope) - return t.accept(a).accept(self) + return t.accept(a) else: return None def visit_assignment_stmt(self, s: AssignmentStmt) -> None: + print("Visiting assignment", s) for lval in s.lvalues: self.analyze_lvalue(lval, explicit_type=s.type is not None) self.check_classvar(s) @@ -1439,6 +1406,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if s.type: allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) s.type = self.anal_type(s.type, allow_tuple_literal) + print("declared type", s.type) else: # For simple assignments, allow binding type aliases. # Also set the type if the rvalue is a simple literal. @@ -1457,8 +1425,10 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: node.kind = TYPE_ALIAS node.type_override = res if isinstance(s.rvalue, IndexExpr): + print("Making type alias with scope", self.tvar_scope) s.rvalue.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res)) + print("Made", s.rvalue.analyzed) if s.type: # Store type into nodes. for lvalue in s.lvalues: @@ -1521,6 +1491,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: # Only a definition can create a type alias, not regular assignment. return rvalue = s.rvalue + print("Setting up type alias", rvalue) if isinstance(rvalue, RefExpr): node = rvalue.node if isinstance(node, TypeInfo): @@ -3054,17 +3025,6 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: self.fail("'await' outside coroutine ('async def')", expr) expr.expr.accept(self) - - # - # Visitors for the TypeTranslator nature. Used for binding type vars. - # - def visit_callable_type(self, t: CallableType) -> CallableType: - self.update_function_type_variables(t, t) - t = t.copy_modified( - arg_types=[self.anal_type(a) for a in t.arg_types], - ret_type=self.anal_type(t.ret_type), - ) - return t # # Helpers # @@ -3718,45 +3678,6 @@ def builtin_type(self, name: str, args: List[Type] = None) -> Instance: assert isinstance(sym.node, TypeInfo) return Instance(sym.node, args or []) -TypeVarList = List[Tuple[str, TypeVarExpr]] - -def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: - return a + b - -class TypeVariableQuery(TypeQuery[TypeVarList]): - - def __init__(self, - lookup: Callable[[str, Context], SymbolTableNode], - scope: 'TypeVarScope', - include_callables: bool): - self.include_callables = include_callables - self.lookup = lookup - self.scope = scope - super().__init__(default=[], strategy=_concat_type_var_lists) - - def _seems_like_callable(self, type: UnboundType) -> bool: - if not type.args: - return False - if isinstance(type.args[0], (EllipsisType, TypeList)): - return True - return False - - def visit_unbound_type(self, t: UnboundType) -> TypeVarList: - name = t.name - node = self.lookup(name, t) - if node and node.kind == TVAR and self.scope.get_binding(node) is None: - assert isinstance(node.node, TypeVarExpr) - return[(name, node.node)] - elif not self.include_callables and self._seems_like_callable(t): - return [] - else: - return super().visit_unbound_type(t) - - def visit_callable_type(self, t: CallableType) -> TypeVarList: - if self.include_callables: - return super().visit_callable_type(t) - else: - return self.default def replace_implicit_first_type(sig: FunctionLike, new: Type) -> FunctionLike: if isinstance(sig, CallableType): diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 3e99d65a9ac1..97d9262a8b28 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,18 +1,20 @@ """Semantic analysis of types""" from collections import OrderedDict -from typing import Callable, List, Optional, Set +from typing import Callable, List, Optional, Set, Tuple + +from contextlib import contextmanager from mypy.types import ( Type, UnboundType, TypeVarType, TupleType, TypedDictType, UnionType, Instance, AnyType, CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, - get_type_vars, + get_type_vars, TypeQuery, ) from mypy.nodes import ( TVAR, TYPE_ALIAS, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, - IndexExpr, RefExpr, nongen_builtins, + IndexExpr, RefExpr, nongen_builtins, TypeVarExpr ) from mypy.tvar_scope import TypeVarScope from mypy.sametypes import is_same_type @@ -321,10 +323,13 @@ def visit_type_var(self, t: TypeVarType) -> Type: return t def visit_callable_type(self, t: CallableType) -> Type: - return t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=False), - ret_type=self.anal_type(t.ret_type, nested=False), - fallback=t.fallback or self.builtin_type('builtins.function'), - variables=self.anal_var_defs(t.variables)) + with self.tvar_scope_frame(): + variables = self.bind_function_type_variables(t, t) + ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=False), + ret_type=self.anal_type(t.ret_type, nested=False), + fallback=t.fallback or self.builtin_type('builtins.function'), + variables=self.anal_var_defs(variables)) + return ret def visit_tuple_type(self, t: TupleType) -> Type: # Types such as (t1, t2, ...) only allowed in assignment statements. They'll @@ -371,25 +376,25 @@ def analyze_callable_type(self, t: UnboundType) -> Type: fallback = self.builtin_type('builtins.function') if len(t.args) == 0: # Callable (bare). Treat as Callable[..., Any]. - return CallableType([AnyType(), AnyType()], + ret = CallableType([AnyType(), AnyType()], [nodes.ARG_STAR, nodes.ARG_STAR2], [None, None], ret_type=AnyType(), fallback=fallback, is_ellipsis_args=True) elif len(t.args) == 2: - ret_type = self.anal_type(t.args[1]) + ret_type = t.args[1] if isinstance(t.args[0], TypeList): # Callable[[ARG, ...], RET] (ordinary callable type) args = t.args[0].items - return CallableType(self.anal_array(args), - [nodes.ARG_POS] * len(args), - [None] * len(args), - ret_type=ret_type, - fallback=fallback) + ret = CallableType(args, + [nodes.ARG_POS] * len(args), + [None] * len(args), + ret_type=ret_type, + fallback=fallback) elif isinstance(t.args[0], EllipsisType): # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) - return CallableType([AnyType(), AnyType()], + ret = CallableType([AnyType(), AnyType()], [nodes.ARG_STAR, nodes.ARG_STAR2], [None, None], ret_type=ret_type, @@ -398,9 +403,60 @@ def analyze_callable_type(self, t: UnboundType) -> Type: else: self.fail('The first argument to Callable must be a list of types or "..."', t) return AnyType() - - self.fail('Invalid function type', t) - return AnyType() + else: + self.fail('Invalid function type', t) + return AnyType() + assert isinstance(ret, CallableType) + return ret.accept(self) + + @contextmanager + def tvar_scope_frame(self): + old_scope = self.tvar_scope + self.tvar_scope = TypeVarScope(self.tvar_scope) + yield + self.tvar_scope = old_scope + + def infer_type_variables(self, + type: CallableType) -> List[Tuple[str, TypeVarExpr]]: + # MOVEABLE + """Return list of unique type variables referred to in a callable.""" + names = [] # type: List[str] + tvars = [] # type: List[TypeVarExpr] + for arg in type.arg_types: + for name, tvar_expr in self.find_type_variables_in_type(arg, include_callables=True): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + for name, tvar_expr in self.find_type_variables_in_type(type.ret_type, include_callables=False): + if name not in names: + names.append(name) + tvars.append(tvar_expr) + return list(zip(names, tvars)) + + def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> None: + """Find the type variables of the function type and bind them in our tvar_scope. + """ + # MOVEABLE + if fun_type.variables: + for var in fun_type.variables: + self.tvar_scope.bind_fun_tvar(var.name, self.lookup(var.name, var).node) + return fun_type.variables + typevars = self.infer_type_variables(fun_type) + # Do not define a new type variable if already defined in scope. + typevars = [(name, tvar) for name, tvar in typevars + if not self.is_defined_type_var(name, defn)] + defs = [] # type: List[TypeVarDef] + for name, tvar in typevars: + self.tvar_scope.bind_fun_tvar(name, tvar) + defs.append(self.tvar_scope.get_binding(tvar.fullname())) + return defs + + def is_defined_type_var(self, tvar: str, context: Context) -> bool: + return self.tvar_scope.get_binding(self.lookup(tvar, context)) is not None + + def find_type_variables_in_type(self, type: Type, *, include_callables: bool) -> List[Tuple[str, TypeVarExpr]]: + ret = type.accept(TypeVariableQuery(self.lookup, self.tvar_scope, include_callables)) + return ret def anal_array(self, a: List[Type], nested: bool = True) -> List[Type]: res = [] # type: List[Type] @@ -565,3 +621,43 @@ def visit_partial_type(self, t: PartialType) -> None: def visit_type_type(self, t: TypeType) -> None: pass + +TypeVarList = List[Tuple[str, TypeVarExpr]] + +def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: + return b + a + +class TypeVariableQuery(TypeQuery[TypeVarList]): + + def __init__(self, + lookup: Callable[[str, Context], SymbolTableNode], + scope: 'TypeVarScope', + include_callables: bool): + self.include_callables = include_callables + self.lookup = lookup + self.scope = scope + super().__init__(default=[], strategy=_concat_type_var_lists) + + def _seems_like_callable(self, type: UnboundType) -> bool: + if not type.args: + return False + if isinstance(type.args[0], (EllipsisType, TypeList)): + return True + return False + + def visit_unbound_type(self, t: UnboundType) -> TypeVarList: + name = t.name + node = self.lookup(name, t) + if node and node.kind == TVAR and self.scope.get_binding(node) is None: + assert isinstance(node.node, TypeVarExpr) + return[(name, node.node)] + elif not self.include_callables and self._seems_like_callable(t): + return [] + else: + return super().visit_unbound_type(t) + + def visit_callable_type(self, t: CallableType) -> TypeVarList: + if self.include_callables: + return super().visit_callable_type(t) + else: + return self.default diff --git a/test-data/unit/check-callable.test b/test-data/unit/check-callable.test index 3bc02bb3f8f0..429ad4494e0a 100644 --- a/test-data/unit/check-callable.test +++ b/test-data/unit/check-callable.test @@ -253,9 +253,9 @@ T = TypeVar('T', int, Callable[[], int], Union[str, Callable[[], str]]) def f(t: T) -> None: if callable(t): - reveal_type(t()) # E: Revealed type is 'Union[builtins.int, builtins.str]' + reveal_type(t()) # E: Revealed type is 'builtins.int' # E: Revealed type is 'builtins.str' else: - reveal_type(t) # E: # E: Revealed type is 'Union[builtins.int, builtins.str]' + reveal_type(t) # E: Revealed type is 'builtins.int*' # E: Revealed type is 'builtins.str' [builtins fixtures/callable.pyi] diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index e401738c4dbf..96cbfd26c5dc 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -1867,6 +1867,8 @@ class A(Generic[T]): class B(Generic[T]): a = A[T] +reveal_type(B[int]()) + reveal_type(B[int]().a) # E: Revealed type is 'def (x: builtins.int*) -> __main__.A[builtins.int*]' B[int]().a('hi') # E: Argument 1 has incompatible type "str"; expected "int" From 4141f94d105f5a9fdf3b90256f4d2e24b3ae7a24 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 30 Mar 2017 23:37:45 -0700 Subject: [PATCH 18/37] forgot this file --- mypy/tvar_scope.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 mypy/tvar_scope.py diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py new file mode 100644 index 000000000000..0704c9335bd8 --- /dev/null +++ b/mypy/tvar_scope.py @@ -0,0 +1,51 @@ +from typing import Optional, Dict, Union + +from mypy.types import TypeVarDef, TypeVarId + +from mypy.nodes import TypeVarExpr, SymbolTableNode + +class TypeVarScope: + + def __init__(self, parent: Optional['TypeVarScope'] = None): + self.scope = {} # type: Dict[str, TypeVarDef] + self.parent = parent + if parent is None: + self.func_id = 0 + self.class_id = 0 + else: + self.func_id = parent.func_id + self.class_id = parent.class_id + + def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr): + self.func_id -= 1 + self._bind(name, tvar_expr, self.func_id) + + def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr): + self.class_id += 1 + self._bind(name, tvar_expr, self.class_id) + + def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int): + tvar_def = TypeVarDef( + name, i, values=tvar_expr.values, + upper_bound=tvar_expr.upper_bound, variance=tvar_expr.variance, + line=tvar_expr.line, column=tvar_expr.column) + self.scope[tvar_expr.fullname()] = tvar_def + + def get_binding(self, item: Union[str, SymbolTableNode]): + if isinstance(item, SymbolTableNode): + fullname = item.fullname + else: + fullname = item + if fullname in self.scope: + return self.scope[fullname] + elif self.parent is not None: + return self.parent.get_binding(fullname) + else: + return None + + def __str__(self): + me = ", ".join('{}: {}`{}'.format(k, v.name, v.id) for k, v in self.scope.items()) + if self.parent is None: + return me + else: + return "{} <- {}".format(str(self.parent), me) From 58e3a2adb74356fd812b57ff1bdc3182d83158f7 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 31 Mar 2017 00:29:30 -0700 Subject: [PATCH 19/37] Fix aliasing to not bind type vars within a callable while aliasing --- mypy/checkmember.py | 6 ------ mypy/semanal.py | 2 ++ mypy/typeanal.py | 16 ++++++++++++++-- test-data/unit/check-classes.test | 2 -- test-data/unit/check-functions.test | 12 ++++++------ 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 34bbde9803de..6534f1a431d6 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -53,7 +53,6 @@ def analyze_member_access(name: str, original_type is always the type used in the initial call. """ if isinstance(typ, Instance): - print("Getting member", name, "of", typ) if name == '__init__' and not is_super: # Accessing __init__ in statically typed code would compromise # type safety unless used via super(). @@ -74,7 +73,6 @@ def analyze_member_access(name: str, # Look up the member. First look up the method dictionary. method = info.get_method(name) if method: - print("It is a method!", method) if method.is_property: assert isinstance(method, OverloadedFuncDef) first_item = cast(Decorator, method.items[0]) @@ -215,9 +213,6 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, """ # It was not a method. Try looking up a variable. v = lookup_member_var_or_accessor(info, name, is_lvalue) - print("It's a var?", v, "itype", itype, "type", v.type) - print("names", info.names) - vv = v if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. @@ -495,7 +490,6 @@ def type_object_type(info: TypeInfo, builtin_type: Callable[[str], Instance]) -> # Construct callable type based on signature of __init__. Adjust # return type and insert type arguments. ret = type_object_type_from_function(init_method, info, fallback) - print("Making callable based on __init__ of", info.name(), ret) return ret diff --git a/mypy/semanal.py b/mypy/semanal.py index 877d866edab1..719bcb364fb1 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1417,6 +1417,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: res = analyze_type_alias(s.rvalue, self.lookup_qualified, self.lookup_fully_qualified, + self.tvar_scope, self.fail, allow_unnormalized=True) if res and (not isinstance(res, Instance) or res.args): # TODO: What if this gets reassigned? @@ -2884,6 +2885,7 @@ def visit_index_expr(self, expr: IndexExpr) -> None: res = analyze_type_alias(expr, self.lookup_qualified, self.lookup_fully_qualified, + self.tvar_scope, self.fail, allow_unnormalized=self.is_stub_file) expr.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res), in_runtime=True) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 97d9262a8b28..0fe5ed755142 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -36,6 +36,7 @@ def analyze_type_alias(node: Expression, lookup_func: Callable[[str, Context], SymbolTableNode], lookup_fqn_func: Callable[[str], SymbolTableNode], + tvar_scope: TypeVarScope, fail_func: Callable[[str, Context], None], allow_unnormalized: bool = False) -> Type: """Return type if node is valid as a type alias rvalue. @@ -75,7 +76,7 @@ def analyze_type_alias(node: Expression, except TypeTranslationError: fail_func('Invalid type alias', node) return None - analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, TypeVarScope(), fail_func, aliasing=True, + analyzer = TypeAnalyser(lookup_func, lookup_fqn_func, tvar_scope, fail_func, aliasing=True, allow_unnormalized=allow_unnormalized) return type.accept(analyzer) @@ -187,8 +188,14 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return UninhabitedType(is_noreturn=True) elif sym.kind == TYPE_ALIAS: override = sym.type_override + print("t is", t) + print("t.class", t.__class__.__name__) + an_args = self.anal_array(t.args) + print("Analyzed args", an_args) + print("Type override is", override) all_vars = self.get_type_var_names(override) + print("All vars", all_vars) exp_len = len(all_vars) act_len = len(an_args) if exp_len > 0 and act_len == 0: @@ -258,6 +265,7 @@ def get_type_var_names(self, tp: Type) -> List[str]: """ tvars = [] # type: List[str] typ_args = get_typ_args(tp) + print("typ_args", typ_args, "of", tp) for arg in typ_args: tvar = self.get_tvar_name(arg) if tvar: @@ -273,6 +281,7 @@ def get_type_var_names(self, tp: Type) -> List[str]: if t in all_tvars: new_tvars.append(t) all_tvars.remove(t) + print("consolidates to", new_tvars, "of", tp) return new_tvars def get_tvar_name(self, t: Type) -> Optional[str]: @@ -324,7 +333,10 @@ def visit_type_var(self, t: TypeVarType) -> Type: def visit_callable_type(self, t: CallableType) -> Type: with self.tvar_scope_frame(): - variables = self.bind_function_type_variables(t, t) + if self.aliasing: + variables = t.variables + else: + variables = self.bind_function_type_variables(t, t) ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=False), ret_type=self.anal_type(t.ret_type, nested=False), fallback=t.fallback or self.builtin_type('builtins.function'), diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test index 96cbfd26c5dc..e401738c4dbf 100644 --- a/test-data/unit/check-classes.test +++ b/test-data/unit/check-classes.test @@ -1867,8 +1867,6 @@ class A(Generic[T]): class B(Generic[T]): a = A[T] -reveal_type(B[int]()) - reveal_type(B[int]().a) # E: Revealed type is 'def (x: builtins.int*) -> __main__.A[builtins.int*]' B[int]().a('hi') # E: Argument 1 has incompatible type "str"; expected "int" diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 4e95ac18583e..30e63a73b224 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1713,23 +1713,23 @@ from typing import Callable, TypeVar T = TypeVar('T') f: Callable[[T], T] -reveal_type(f) +reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> T`-1' def g(__x: T) -> T: pass f = g -reveal_type(f) +reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> T`-1' i = f(3) -reveal_type(i) +reveal_type(i) # E: Revealed type is 'builtins.int*' [case testFunctionReturningGenericFunction] from typing import Callable, TypeVar T = TypeVar('T') def deco() -> Callable[[T], T]: pass -reveal_type(deco) +reveal_type(deco) # E: Revealed type is 'def () -> def [T] (T`-1) -> T`-1' f = deco() -reveal_type(f) +reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> T`-1' i = f(3) -reveal_type(i) +reveal_type(i) # E: Revealed type is 'builtins.int*' [case testGenericFunctionOnReturnTypeOnly] from typing import TypeVar, List From c061a09f389ae3b36cbbf155934d1211305feecf Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 01:16:06 -0700 Subject: [PATCH 20/37] removed some now-unused code --- mypy/checkmember.py | 1 + mypy/semanal.py | 38 +------------------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 6534f1a431d6..65091c0931ec 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -442,6 +442,7 @@ class B(A): pass # TODO: verify consistency between Q and T info = itype.type # type: TypeInfo if isinstance(t, CallableType): + print("Adding class tvars", info.type_vars) # TODO: Should we propagate type variable values? tvars = [TypeVarDef(n, i + 1, None, builtin_type('builtins.object'), tv.variance) for (i, n), tv in zip(enumerate(info.type_vars), info.defn.type_vars)] diff --git a/mypy/semanal.py b/mypy/semanal.py index 719bcb364fb1..60e61fcbbf4f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -156,7 +156,7 @@ FUNCTION_SECOND_PHASE = 2 # Only analyze body -class SemanticAnalyzer(NodeVisitor, TypeTranslator): +class SemanticAnalyzer(NodeVisitor): """Semantically analyze parsed mypy files. The analyzer binds names and does various consistency checks for a @@ -556,32 +556,12 @@ def check_classvar_in_signature(self, typ: Type) -> None: # Show only one error per signature break - def add_func_type_variables_to_symbol_table( - self, tt: CallableType, defn: Context) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - items = tt.variables - names = self.type_var_names() - for item in items: - name = item.name - if name in names: - self.name_already_defined(name, defn) - node = self.bind_type_var(name, item, defn) - nodes.append(node) - names.add(name) - return nodes - def type_var_names(self) -> Set[str]: if not self.type: return set() else: return set(self.type.type_vars) - def bind_type_var(self, fullname: str, tvar_def: TypeVarDef, - context: Context) -> SymbolTableNode: - node = self.lookup_qualified(fullname, context) - self.tvar_scope.bind(node, tvar_def) - return node - def check_function_signature(self, fdef: FuncItem) -> None: sig = fdef.type assert isinstance(sig, CallableType) @@ -605,7 +585,6 @@ def visit_class_def(self, defn: ClassDef) -> None: self.leave_class() else: self.setup_class_def_analysis(defn) - # self.bind_class_type_variables_in_symbol_table(defn.info) self.analyze_base_classes(defn) self.analyze_metaclass(defn) @@ -613,7 +592,6 @@ def visit_class_def(self, defn: ClassDef) -> None: self.analyze_class_decorator(defn, decorator) self.enter_class(defn) - print("Entered class", defn.name, self.tvar_scope) # Analyze class body. defn.defs.accept(self) @@ -1071,14 +1049,6 @@ def named_type_or_none(self, qualified_name: str, args: List[Type] = None) -> In assert isinstance(sym.node, TypeInfo) return Instance(sym.node, args or []) - def bind_class_type_variables_in_symbol_table( - self, info: TypeInfo) -> List[SymbolTableNode]: - nodes = [] # type: List[SymbolTableNode] - for var, binder in zip(info.type_vars, info.defn.type_vars): - node = self.bind_type_var(var, binder, info) - nodes.append(node) - return nodes - def is_typeddict(self, expr: Expression) -> bool: return (isinstance(expr, RefExpr) and isinstance(expr.node, TypeInfo) and expr.node.typeddict_type is not None) @@ -1391,14 +1361,12 @@ def anal_type(self, t: Type, allow_tuple_literal: bool = False, a = self.type_analyzer( aliasing=aliasing, allow_tuple_literal=allow_tuple_literal) - print("analyzing", t, "scope", self.tvar_scope) return t.accept(a) else: return None def visit_assignment_stmt(self, s: AssignmentStmt) -> None: - print("Visiting assignment", s) for lval in s.lvalues: self.analyze_lvalue(lval, explicit_type=s.type is not None) self.check_classvar(s) @@ -1406,7 +1374,6 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: if s.type: allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) s.type = self.anal_type(s.type, allow_tuple_literal) - print("declared type", s.type) else: # For simple assignments, allow binding type aliases. # Also set the type if the rvalue is a simple literal. @@ -1426,10 +1393,8 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: node.kind = TYPE_ALIAS node.type_override = res if isinstance(s.rvalue, IndexExpr): - print("Making type alias with scope", self.tvar_scope) s.rvalue.analyzed = TypeAliasExpr(res, fallback=self.alias_fallback(res)) - print("Made", s.rvalue.analyzed) if s.type: # Store type into nodes. for lvalue in s.lvalues: @@ -1492,7 +1457,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None: # Only a definition can create a type alias, not regular assignment. return rvalue = s.rvalue - print("Setting up type alias", rvalue) if isinstance(rvalue, RefExpr): node = rvalue.node if isinstance(node, TypeInfo): From 603013d293904b28ac6bb2faf6589f64bf0b78df Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sat, 1 Apr 2017 12:13:00 -0700 Subject: [PATCH 21/37] my own code review, first pass --- mypy/checkmember.py | 5 ++--- mypy/expandtype.py | 1 - mypy/semanal.py | 2 +- mypy/typeanal.py | 14 ++------------ 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/mypy/checkmember.py b/mypy/checkmember.py index 65091c0931ec..6c934dccb022 100644 --- a/mypy/checkmember.py +++ b/mypy/checkmember.py @@ -213,6 +213,7 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo, """ # It was not a method. Try looking up a variable. v = lookup_member_var_or_accessor(info, name, is_lvalue) + vv = v if isinstance(vv, Decorator): # The associated Var node of a decorator contains the type. @@ -442,7 +443,6 @@ class B(A): pass # TODO: verify consistency between Q and T info = itype.type # type: TypeInfo if isinstance(t, CallableType): - print("Adding class tvars", info.type_vars) # TODO: Should we propagate type variable values? tvars = [TypeVarDef(n, i + 1, None, builtin_type('builtins.object'), tv.variance) for (i, n), tv in zip(enumerate(info.type_vars), info.defn.type_vars)] @@ -490,8 +490,7 @@ def type_object_type(info: TypeInfo, builtin_type: Callable[[str], Instance]) -> return class_callable(sig, info, fallback, None) # Construct callable type based on signature of __init__. Adjust # return type and insert type arguments. - ret = type_object_type_from_function(init_method, info, fallback) - return ret + return type_object_type_from_function(init_method, info, fallback) def type_object_type_from_function(init_or_new: FuncBase, info: TypeInfo, diff --git a/mypy/expandtype.py b/mypy/expandtype.py index fd44c5d5b279..41c74daf6ea3 100644 --- a/mypy/expandtype.py +++ b/mypy/expandtype.py @@ -23,7 +23,6 @@ def expand_type_by_instance(typ: Type, instance: Instance) -> Type: if instance.args == []: return typ else: - print("Ok so the instance", instance, "has some args", instance.args) variables = {} # type: Dict[TypeVarId, Type] for binder, arg in zip(instance.type.defn.type_vars, instance.args): variables[binder.id] = arg diff --git a/mypy/semanal.py b/mypy/semanal.py index 60e61fcbbf4f..71e15bd8d8ac 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -80,7 +80,7 @@ NoneTyp, CallableType, Overloaded, Instance, Type, TypeVarType, AnyType, FunctionLike, UnboundType, TypeList, TypeVarDef, TypeType, TupleType, UnionType, StarType, EllipsisType, function_type, TypedDictType, - TypeTranslator, TypeQuery + TypeQuery ) from mypy.nodes import implicit_module_attrs from mypy.typeanal import ( diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 0fe5ed755142..d1833b991e56 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -188,14 +188,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return UninhabitedType(is_noreturn=True) elif sym.kind == TYPE_ALIAS: override = sym.type_override - print("t is", t) - print("t.class", t.__class__.__name__) - an_args = self.anal_array(t.args) - print("Analyzed args", an_args) - print("Type override is", override) all_vars = self.get_type_var_names(override) - print("All vars", all_vars) exp_len = len(all_vars) act_len = len(an_args) if exp_len > 0 and act_len == 0: @@ -220,8 +214,8 @@ def visit_unbound_type(self, t: UnboundType) -> Type: # is pretty minor. return AnyType() # Allow unbound type variables when defining an alias - if not (self.aliasing and sym.kind == TVAR and self.tvar_scope.get_binding(sym) == None): - print("scope is", self.tvar_scope, "looking for", sym.fullname) + if not (self.aliasing and sym.kind == TVAR and + self.tvar_scope.get_binding(sym) == None): self.fail('Invalid type "{}"'.format(name), t) return t info = sym.node # type: TypeInfo @@ -265,7 +259,6 @@ def get_type_var_names(self, tp: Type) -> List[str]: """ tvars = [] # type: List[str] typ_args = get_typ_args(tp) - print("typ_args", typ_args, "of", tp) for arg in typ_args: tvar = self.get_tvar_name(arg) if tvar: @@ -281,7 +274,6 @@ def get_type_var_names(self, tp: Type) -> List[str]: if t in all_tvars: new_tvars.append(t) all_tvars.remove(t) - print("consolidates to", new_tvars, "of", tp) return new_tvars def get_tvar_name(self, t: Type) -> Optional[str]: @@ -430,7 +422,6 @@ def tvar_scope_frame(self): def infer_type_variables(self, type: CallableType) -> List[Tuple[str, TypeVarExpr]]: - # MOVEABLE """Return list of unique type variables referred to in a callable.""" names = [] # type: List[str] tvars = [] # type: List[TypeVarExpr] @@ -448,7 +439,6 @@ def infer_type_variables(self, def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> None: """Find the type variables of the function type and bind them in our tvar_scope. """ - # MOVEABLE if fun_type.variables: for var in fun_type.variables: self.tvar_scope.bind_fun_tvar(var.name, self.lookup(var.name, var).node) From c6fec6388887988de6fbbee0431b35f93c372f3d Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 00:56:27 -0700 Subject: [PATCH 22/37] Use type var query instead of another query fn --- mypy/typeanal.py | 31 ++++++++++--------------------- typeshed | 2 +- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index d1833b991e56..1fb0bce539ac 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -257,24 +257,9 @@ def get_type_var_names(self, tp: Type) -> List[str]: """Get all type variable names that are present in a generic type alias in order of textual appearance (recursively, if needed). """ - tvars = [] # type: List[str] - typ_args = get_typ_args(tp) - for arg in typ_args: - tvar = self.get_tvar_name(arg) - if tvar: - tvars.append(tvar) - else: - subvars = self.get_type_var_names(arg) - if subvars: - tvars.extend(subvars) - # Get unique type variables in order of appearance - all_tvars = set(tvars) - new_tvars = [] - for t in tvars: - if t in all_tvars: - new_tvars.append(t) - all_tvars.remove(t) - return new_tvars + return [name for name, _ + in tp.accept(TypeVariableQuery(self.lookup, self.tvar_scope, + include_callables=True,include_bound=True))] def get_tvar_name(self, t: Type) -> Optional[str]: if not isinstance(t, UnboundType): @@ -627,17 +612,20 @@ def visit_type_type(self, t: TypeType) -> None: TypeVarList = List[Tuple[str, TypeVarExpr]] def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: - return b + a + return b + [v for v in a if v not in b] class TypeVariableQuery(TypeQuery[TypeVarList]): def __init__(self, lookup: Callable[[str, Context], SymbolTableNode], scope: 'TypeVarScope', - include_callables: bool): + include_callables: bool, + *, + include_bound: bool = False): self.include_callables = include_callables self.lookup = lookup self.scope = scope + self.include_bound = include_bound super().__init__(default=[], strategy=_concat_type_var_lists) def _seems_like_callable(self, type: UnboundType) -> bool: @@ -650,7 +638,8 @@ def _seems_like_callable(self, type: UnboundType) -> bool: def visit_unbound_type(self, t: UnboundType) -> TypeVarList: name = t.name node = self.lookup(name, t) - if node and node.kind == TVAR and self.scope.get_binding(node) is None: + if node and node.kind == TVAR and ( + self.include_bound or self.scope.get_binding(node) is None): assert isinstance(node.node, TypeVarExpr) return[(name, node.node)] elif not self.include_callables and self._seems_like_callable(t): diff --git a/typeshed b/typeshed index 1ea3d2de5794..48920fea7ee6 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 1ea3d2de5794c2224a1bc086aa471d0097699bf1 +Subproject commit 48920fea7ee6c1d413fba12fe709c652da065393 From f46d52a9f8225ac4c84a866965f7a6542681c4c6 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 01:32:05 -0700 Subject: [PATCH 23/37] Tighten code --- mypy/semanal.py | 31 ++++--------------------------- mypy/tvar_scope.py | 15 +++++---------- mypy/typeanal.py | 14 +++++--------- 3 files changed, 14 insertions(+), 46 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 71e15bd8d8ac..1066dff90901 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -85,6 +85,7 @@ from mypy.nodes import implicit_module_attrs from mypy.typeanal import ( TypeAnalyser, TypeAnalyserPass3, analyze_type_alias, no_subscript_builtin_alias, + TypeVariableQuery, ) from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.sametypes import is_same_type @@ -375,10 +376,8 @@ def update_function_type_variables(self, fun_type: CallableType, defn: Context) Update the signature of defn to contain type variable definitions if defn is generic. """ - a = self.type_analyzer(tvar_scope=TypeVarScope(self.tvar_scope)) - variables = a.bind_function_type_variables(fun_type, defn) - fun_type.variables = variables + fun_type.variables = a.bind_function_type_variables(fun_type, defn) def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # OverloadedFuncDef refers to any legitimate situation where you have @@ -503,7 +502,6 @@ def analyze_function(self, defn: FuncItem) -> None: self.check_function_signature(defn) if isinstance(defn, FuncDef): defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: if arg.initializer: arg.initializer.accept(self) @@ -556,12 +554,6 @@ def check_classvar_in_signature(self, typ: Type) -> None: # Show only one error per signature break - def type_var_names(self) -> Set[str]: - if not self.type: - return set() - else: - return set(self.type.type_vars) - def check_function_signature(self, fdef: FuncItem) -> None: sig = fdef.type assert isinstance(sig, CallableType) @@ -754,23 +746,8 @@ def get_all_bases_tvars(self, defn: ClassDef, except TypeTranslationError: # This error will be caught later. continue - tvars.extend(self.get_tvars(base)) - return self.remove_dups(tvars) - - def get_tvars(self, tp: Type) -> List[Tuple[str, TypeVarExpr]]: - tvars = [] # type: List[Tuple[str, TypeVarExpr]] - if isinstance(tp, UnboundType): - tp_args = tp.args - elif isinstance(tp, TypeList): - tp_args = tp.items - else: - return tvars - for arg in tp_args: - tvar = self.analyze_unbound_tvar(arg) - if tvar: - tvars.append(tvar) - else: - tvars.extend(self.get_tvars(arg)) + tvars.extend(base.accept(TypeVariableQuery( + self.lookup_qualified, self.tvar_scope))) return self.remove_dups(tvars) def remove_dups(self, tvars: List[T]) -> List[T]: diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 0704c9335bd8..0f62a23bf0b3 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -9,10 +9,9 @@ class TypeVarScope: def __init__(self, parent: Optional['TypeVarScope'] = None): self.scope = {} # type: Dict[str, TypeVarDef] self.parent = parent - if parent is None: - self.func_id = 0 - self.class_id = 0 - else: + self.func_id = 0 + self.class_id = 0 + if parent is not None: self.func_id = parent.func_id self.class_id = parent.class_id @@ -32,10 +31,7 @@ def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int): self.scope[tvar_expr.fullname()] = tvar_def def get_binding(self, item: Union[str, SymbolTableNode]): - if isinstance(item, SymbolTableNode): - fullname = item.fullname - else: - fullname = item + fullname = item.fullname if isinstance(item, SymbolTableNode) else item if fullname in self.scope: return self.scope[fullname] elif self.parent is not None: @@ -47,5 +43,4 @@ def __str__(self): me = ", ".join('{}: {}`{}'.format(k, v.name, v.id) for k, v in self.scope.items()) if self.parent is None: return me - else: - return "{} <- {}".format(str(self.parent), me) + return "{} <- {}".format(str(self.parent), me) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1fb0bce539ac..9d79502d3277 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -131,7 +131,6 @@ def visit_unbound_type(self, t: UnboundType) -> Type: not sym.normalized and not self.allow_unnormalized): self.fail(no_subscript_builtin_alias(fullname), t) if sym.kind == TVAR and self.tvar_scope.get_binding(sym) is not None: - # XXX: Don't fetch twice tvar_def = self.tvar_scope.get_binding(sym) if len(t.args) > 0: self.fail('Type variable "{}" used with arguments'.format( @@ -259,7 +258,7 @@ def get_type_var_names(self, tp: Type) -> List[str]: """ return [name for name, _ in tp.accept(TypeVariableQuery(self.lookup, self.tvar_scope, - include_callables=True,include_bound=True))] + include_callables=True, include_bound=True))] def get_tvar_name(self, t: Type) -> Optional[str]: if not isinstance(t, UnboundType): @@ -411,11 +410,12 @@ def infer_type_variables(self, names = [] # type: List[str] tvars = [] # type: List[TypeVarExpr] for arg in type.arg_types: - for name, tvar_expr in self.find_type_variables_in_type(arg, include_callables=True): + for name, tvar_expr in arg.accept(TypeVariableQuery(self.lookup, self.tvar_scope)): if name not in names: names.append(name) tvars.append(tvar_expr) - for name, tvar_expr in self.find_type_variables_in_type(type.ret_type, include_callables=False): + for name, tvar_expr in type.ret_type.accept( + TypeVariableQuery(self.lookup, self.tvar_scope, include_callables=False)): if name not in names: names.append(name) tvars.append(tvar_expr) @@ -441,10 +441,6 @@ def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> def is_defined_type_var(self, tvar: str, context: Context) -> bool: return self.tvar_scope.get_binding(self.lookup(tvar, context)) is not None - def find_type_variables_in_type(self, type: Type, *, include_callables: bool) -> List[Tuple[str, TypeVarExpr]]: - ret = type.accept(TypeVariableQuery(self.lookup, self.tvar_scope, include_callables)) - return ret - def anal_array(self, a: List[Type], nested: bool = True) -> List[Type]: res = [] # type: List[Type] for t in a: @@ -619,8 +615,8 @@ class TypeVariableQuery(TypeQuery[TypeVarList]): def __init__(self, lookup: Callable[[str, Context], SymbolTableNode], scope: 'TypeVarScope', - include_callables: bool, *, + include_callables: bool = True, include_bound: bool = False): self.include_callables = include_callables self.lookup = lookup From ea21337464178f9e210b7a8dc31a071e3f3bc634 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 01:45:12 -0700 Subject: [PATCH 24/37] fix some types --- mypy/semanal.py | 2 +- mypy/tvar_scope.py | 12 ++++++------ mypy/typeanal.py | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1066dff90901..54c49af4c639 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -509,7 +509,7 @@ def analyze_function(self, defn: FuncItem) -> None: # Bind the type variables again to visit the body. if defn.type: a = self.type_analyzer() - a.bind_function_type_variables(defn.type, defn) + a.bind_function_type_variables(cast(CallableType, defn.type), defn) self.function_stack.append(defn) self.enter() for arg in defn.arguments: diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 0f62a23bf0b3..cad6c7f0f5a6 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -6,7 +6,7 @@ class TypeVarScope: - def __init__(self, parent: Optional['TypeVarScope'] = None): + def __init__(self, parent: Optional['TypeVarScope'] = None) -> None: self.scope = {} # type: Dict[str, TypeVarDef] self.parent = parent self.func_id = 0 @@ -15,22 +15,22 @@ def __init__(self, parent: Optional['TypeVarScope'] = None): self.func_id = parent.func_id self.class_id = parent.class_id - def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr): + def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr) -> None: self.func_id -= 1 self._bind(name, tvar_expr, self.func_id) - def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr): + def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr) -> None: self.class_id += 1 self._bind(name, tvar_expr, self.class_id) - def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int): + def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int) -> None: tvar_def = TypeVarDef( name, i, values=tvar_expr.values, upper_bound=tvar_expr.upper_bound, variance=tvar_expr.variance, line=tvar_expr.line, column=tvar_expr.column) self.scope[tvar_expr.fullname()] = tvar_def - def get_binding(self, item: Union[str, SymbolTableNode]): + def get_binding(self, item: Union[str, SymbolTableNode]) -> Optional[TypeVarDef]: fullname = item.fullname if isinstance(item, SymbolTableNode) else item if fullname in self.scope: return self.scope[fullname] @@ -39,7 +39,7 @@ def get_binding(self, item: Union[str, SymbolTableNode]): else: return None - def __str__(self): + def __str__(self) -> str: me = ", ".join('{}: {}`{}'.format(k, v.name, v.id) for k, v in self.scope.items()) if self.parent is None: return me diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 9d79502d3277..1e86346c1b00 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -421,12 +421,15 @@ def infer_type_variables(self, tvars.append(tvar_expr) return list(zip(names, tvars)) - def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> None: + def bind_function_type_variables(self, + fun_type: CallableType, defn: Context) -> List[TypeVarDef]: """Find the type variables of the function type and bind them in our tvar_scope. """ if fun_type.variables: for var in fun_type.variables: - self.tvar_scope.bind_fun_tvar(var.name, self.lookup(var.name, var).node) + var_expr = self.lookup(var.name, var).node + assert isinstance(var_expr, TypeVarExpr) + self.tvar_scope.bind_fun_tvar(var.name, var_expr) return fun_type.variables typevars = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. @@ -617,7 +620,7 @@ def __init__(self, scope: 'TypeVarScope', *, include_callables: bool = True, - include_bound: bool = False): + include_bound: bool = False) -> None: self.include_callables = include_callables self.lookup = lookup self.scope = scope From 80d62a7b251fffcf53a239c0cef6b1762821cb22 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 10:34:17 -0700 Subject: [PATCH 25/37] Use same type alias throughout --- mypy/semanal.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 54c49af4c639..bf287c934055 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -85,7 +85,7 @@ from mypy.nodes import implicit_module_attrs from mypy.typeanal import ( TypeAnalyser, TypeAnalyserPass3, analyze_type_alias, no_subscript_builtin_alias, - TypeVariableQuery, + TypeVariableQuery, TypeVarList, ) from mypy.exprtotype import expr_to_unanalyzed_type, TypeTranslationError from mypy.sametypes import is_same_type @@ -670,7 +670,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: Note that this is performed *before* semantic analysis. """ removed = [] # type: List[int] - declared_tvars = [] # type: List[Tuple[str, TypeVarExpr]] + declared_tvars = [] # type: TypeVarList type_vars = [] # type: List[TypeVarDef] for i, base_expr in enumerate(defn.base_type_exprs): try: @@ -707,7 +707,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: tvar_defs.append(self.tvar_scope.get_binding(tvar_expr.fullname())) defn.type_vars = tvar_defs - def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeVarExpr]]]: + def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]: if not isinstance(t, UnboundType): return None unbound = t @@ -715,7 +715,7 @@ def analyze_typevar_declaration(self, t: Type) -> Optional[List[Tuple[str, TypeV if sym is None or sym.node is None: return None if sym.node.fullname() == 'typing.Generic': - tvars = [] # type: List[Tuple[str, TypeVarExpr]] + tvars = [] # type: TypeVarList for arg in unbound.args: tvar = self.analyze_unbound_tvar(arg) if tvar: @@ -736,9 +736,8 @@ def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: return unbound.name, sym.node return None - def get_all_bases_tvars(self, defn: ClassDef, - removed: List[int]) -> List[Tuple[str, TypeVarExpr]]: - tvars = [] # type: List[Tuple[str, TypeVarExpr]] + def get_all_bases_tvars(self, defn: ClassDef, removed: List[int]) -> TypeVarList: + tvars = [] # type: TypeVarList for i, base_expr in enumerate(defn.base_type_exprs): if i not in removed: try: @@ -746,8 +745,8 @@ def get_all_bases_tvars(self, defn: ClassDef, except TypeTranslationError: # This error will be caught later. continue - tvars.extend(base.accept(TypeVariableQuery( - self.lookup_qualified, self.tvar_scope))) + base_tvars = base.accept(TypeVariableQuery(self.lookup_qualified, self.tvar_scope)) + tvars.extend(base_tvars) return self.remove_dups(tvars) def remove_dups(self, tvars: List[T]) -> List[T]: From eaf8c0de55fbd447af2f606d015d33d3d85ff500 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 13:05:23 -0700 Subject: [PATCH 26/37] Make semanal tests pass. Delete one I think is wrong --- mypy/semanal.py | 6 +++--- mypy/typeanal.py | 8 ++++---- test-data/unit/semanal-errors.test | 8 -------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index bf287c934055..0d5f5d6bde45 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -498,7 +498,7 @@ def analyze_function(self, defn: FuncItem) -> None: assert isinstance(defn.type, CallableType) # Signature must be analyzed in the surrounding scope so that # class-level imported names and type variables are in scope. - defn.type = self.anal_type(defn.type) + defn.type = self.type_analyzer().visit_callable_type(defn.type, nested=False) self.check_function_signature(defn) if isinstance(defn, FuncDef): defn.type = set_callable_name(defn.type, defn) @@ -2972,14 +2972,14 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: # @contextmanager - def new_tvar_scope(self): + def new_tvar_scope(self) -> None: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope() yield self.tvar_scope = old_scope @contextmanager - def tvar_scope_frame(self): + def tvar_scope_frame(self) -> None: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope(self.tvar_scope) yield diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 1e86346c1b00..5a859c81041d 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -307,14 +307,14 @@ def visit_instance(self, t: Instance) -> Type: def visit_type_var(self, t: TypeVarType) -> Type: return t - def visit_callable_type(self, t: CallableType) -> Type: + def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: with self.tvar_scope_frame(): if self.aliasing: variables = t.variables else: variables = self.bind_function_type_variables(t, t) - ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=False), - ret_type=self.anal_type(t.ret_type, nested=False), + ret = t.copy_modified(arg_types=self.anal_array(t.arg_types, nested=nested), + ret_type=self.anal_type(t.ret_type, nested=nested), fallback=t.fallback or self.builtin_type('builtins.function'), variables=self.anal_var_defs(variables)) return ret @@ -398,7 +398,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: return ret.accept(self) @contextmanager - def tvar_scope_frame(self): + def tvar_scope_frame(self) -> None: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope(self.tvar_scope) yield diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 99584e77a977..f7af185b607a 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -940,14 +940,6 @@ from typing import Generic class A(Generic[int]): pass # E: Free type variable expected in Generic[...] [out] -[case testInvalidTypeWithinNestedGenericClass] -from typing import Generic, TypeVar -T = TypeVar('T') -class A(Generic[T]): - class B(Generic[T]): pass \ - # E: Free type variable expected in Generic[...] -[out] - [case testIncludingGenericTwiceInBaseClassList] from typing import Generic, TypeVar T = TypeVar('T') From 5009a2bae0e924add0cf51074b6aa4852cdb2c21 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 16:22:18 -0700 Subject: [PATCH 27/37] Generator types for generators --- mypy/semanal.py | 6 +++--- mypy/typeanal.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 0d5f5d6bde45..f18d388ccd96 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -47,7 +47,7 @@ from contextlib import contextmanager from typing import ( - List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable + List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable, Generator ) from mypy.nodes import ( @@ -2972,14 +2972,14 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: # @contextmanager - def new_tvar_scope(self) -> None: + def new_tvar_scope(self) -> Generator: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope() yield self.tvar_scope = old_scope @contextmanager - def tvar_scope_frame(self) -> None: + def tvar_scope_frame(self) -> Generator: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope(self.tvar_scope) yield diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 5a859c81041d..6914cb0dc011 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,7 +1,7 @@ """Semantic analysis of types""" from collections import OrderedDict -from typing import Callable, List, Optional, Set, Tuple +from typing import Callable, List, Optional, Set, Tuple, Generator from contextlib import contextmanager @@ -398,7 +398,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: return ret.accept(self) @contextmanager - def tvar_scope_frame(self) -> None: + def tvar_scope_frame(self) -> Generator: old_scope = self.tvar_scope self.tvar_scope = TypeVarScope(self.tvar_scope) yield From 7c64464ef86304c36ade884332f9c8185c824aab Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Sun, 2 Apr 2017 17:29:22 -0700 Subject: [PATCH 28/37] Lint and cleanups --- mypy/semanal.py | 4 +--- mypy/tvar_scope.py | 17 ++++++++++------- mypy/typeanal.py | 25 ++++++++++++++----------- typeshed | 2 +- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index f18d388ccd96..3d4f558e6fd8 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -671,7 +671,6 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: """ removed = [] # type: List[int] declared_tvars = [] # type: TypeVarList - type_vars = [] # type: List[TypeVarDef] for i, base_expr in enumerate(defn.base_type_exprs): try: base = expr_to_unanalyzed_type(base_expr) @@ -703,8 +702,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: del defn.base_type_exprs[i] tvar_defs = [] # type: List[TypeVarDef] for name, tvar_expr in declared_tvars: - self.tvar_scope.bind_class_tvar(name, tvar_expr) - tvar_defs.append(self.tvar_scope.get_binding(tvar_expr.fullname())) + tvar_defs.append(self.tvar_scope.bind_class_tvar(name, tvar_expr)) defn.type_vars = tvar_defs def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]: diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index cad6c7f0f5a6..5b4263352c50 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -1,10 +1,12 @@ from typing import Optional, Dict, Union - from mypy.types import TypeVarDef, TypeVarId - from mypy.nodes import TypeVarExpr, SymbolTableNode + class TypeVarScope: + """Scope that holds bindings for type variables. Node fullname -> TypeVarDef. + + Provide a parent on intitalization when you want to make a child scope.""" def __init__(self, parent: Optional['TypeVarScope'] = None) -> None: self.scope = {} # type: Dict[str, TypeVarDef] @@ -15,20 +17,21 @@ def __init__(self, parent: Optional['TypeVarScope'] = None) -> None: self.func_id = parent.func_id self.class_id = parent.class_id - def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr) -> None: + def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: self.func_id -= 1 - self._bind(name, tvar_expr, self.func_id) + return self._bind(name, tvar_expr, self.func_id) - def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr) -> None: + def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: self.class_id += 1 - self._bind(name, tvar_expr, self.class_id) + return self._bind(name, tvar_expr, self.class_id) - def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int) -> None: + def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int) -> TypeVarDef: tvar_def = TypeVarDef( name, i, values=tvar_expr.values, upper_bound=tvar_expr.upper_bound, variance=tvar_expr.variance, line=tvar_expr.line, column=tvar_expr.column) self.scope[tvar_expr.fullname()] = tvar_def + return tvar_def def get_binding(self, item: Union[str, SymbolTableNode]) -> Optional[TypeVarDef]: fullname = item.fullname if isinstance(item, SymbolTableNode) else item diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 6914cb0dc011..6751096faeb0 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -214,7 +214,7 @@ def visit_unbound_type(self, t: UnboundType) -> Type: return AnyType() # Allow unbound type variables when defining an alias if not (self.aliasing and sym.kind == TVAR and - self.tvar_scope.get_binding(sym) == None): + self.tvar_scope.get_binding(sym) is None): self.fail('Invalid type "{}"'.format(name), t) return t info = sym.node # type: TypeInfo @@ -365,11 +365,11 @@ def analyze_callable_type(self, t: UnboundType) -> Type: if len(t.args) == 0: # Callable (bare). Treat as Callable[..., Any]. ret = CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=AnyType(), - fallback=fallback, - is_ellipsis_args=True) + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=AnyType(), + fallback=fallback, + is_ellipsis_args=True) elif len(t.args) == 2: ret_type = t.args[1] if isinstance(t.args[0], TypeList): @@ -383,11 +383,11 @@ def analyze_callable_type(self, t: UnboundType) -> Type: elif isinstance(t.args[0], EllipsisType): # Callable[..., RET] (with literal ellipsis; accept arbitrary arguments) ret = CallableType([AnyType(), AnyType()], - [nodes.ARG_STAR, nodes.ARG_STAR2], - [None, None], - ret_type=ret_type, - fallback=fallback, - is_ellipsis_args=True) + [nodes.ARG_STAR, nodes.ARG_STAR2], + [None, None], + ret_type=ret_type, + fallback=fallback, + is_ellipsis_args=True) else: self.fail('The first argument to Callable must be a list of types or "..."', t) return AnyType() @@ -608,11 +608,14 @@ def visit_partial_type(self, t: PartialType) -> None: def visit_type_type(self, t: TypeType) -> None: pass + TypeVarList = List[Tuple[str, TypeVarExpr]] + def _concat_type_var_lists(a: TypeVarList, b: TypeVarList) -> TypeVarList: return b + [v for v in a if v not in b] + class TypeVariableQuery(TypeQuery[TypeVarList]): def __init__(self, diff --git a/typeshed b/typeshed index 48920fea7ee6..8b50522273d1 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 48920fea7ee6c1d413fba12fe709c652da065393 +Subproject commit 8b50522273d1e29d79e1d2fedddf32bcea0e09ef From a866d0fe214fd5c1b99ed50def4f6deb046decb5 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 11 Apr 2017 16:43:59 -0700 Subject: [PATCH 29/37] Test for using generic function return as a decorator --- test-data/unit/check-functions.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index 30e63a73b224..e82c0719bb7f 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1731,6 +1731,19 @@ reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> T`-1' i = f(3) reveal_type(i) # E: Revealed type is 'builtins.int*' +[case testGenericFunctionReturnAsDecorator] +from typing import Callable, TypeVar + +T = TypeVar('T') +def deco(__i: int) -> Callable[[T], T]: pass + +@deco(3) +def lol(x: int) -> str: ... + +reveal_type(lol) # E: Revealed type is 'def (x: builtins.int) -> builtins.str' +s = lol(4) +reveal_type(s) # E: Revealed type is 'builtins.str' + [case testGenericFunctionOnReturnTypeOnly] from typing import TypeVar, List From 4683d709b575d52d4c1457bbf3f225edd7a6842d Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 11 Apr 2017 18:29:24 -0700 Subject: [PATCH 30/37] Merge lint --- mypy/typeanal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index e270af00ef50..d2d002f644c2 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -658,6 +658,8 @@ def visit_callable_type(self, t: CallableType) -> TypeVarList: return super().visit_callable_type(t) else: return self.default + + def make_optional_type(t: Type) -> Type: """Return the type corresponding to Optional[t]. From da0d9367b6c3e9b5c3051064be06ac1ab19d861e Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 12 Apr 2017 13:26:27 -0700 Subject: [PATCH 31/37] More tests for nested and multi-type-var situations --- test-data/unit/check-functions.test | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test-data/unit/check-functions.test b/test-data/unit/check-functions.test index e82c0719bb7f..33fc51d3b046 100644 --- a/test-data/unit/check-functions.test +++ b/test-data/unit/check-functions.test @@ -1731,6 +1731,32 @@ reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> T`-1' i = f(3) reveal_type(i) # E: Revealed type is 'builtins.int*' +[case testFunctionReturningGenericFunctionPartialBinding] +from typing import Callable, TypeVar + +T = TypeVar('T') +U = TypeVar('U') + +def deco(x: U) -> Callable[[T, U], T]: pass +reveal_type(deco) # E: Revealed type is 'def [U] (x: U`-1) -> def [T] (T`-2, U`-1) -> T`-2' +f = deco("foo") +reveal_type(f) # E: Revealed type is 'def [T] (T`-2, builtins.str*) -> T`-2' +i = f(3, "eggs") +reveal_type(i) # E: Revealed type is 'builtins.int*' + +[case testFunctionReturningGenericFunctionTwoLevelBinding] +from typing import Callable, TypeVar + +T = TypeVar('T') +R = TypeVar('R') +def deco() -> Callable[[T], Callable[[T, R], R]]: pass +f = deco() +reveal_type(f) # E: Revealed type is 'def [T] (T`-1) -> def [R] (T`-1, R`-2) -> R`-2' +g = f(3) +reveal_type(g) # E: Revealed type is 'def [R] (builtins.int*, R`-2) -> R`-2' +s = g(4, "foo") +reveal_type(s) # E: Revealed type is 'builtins.str*' + [case testGenericFunctionReturnAsDecorator] from typing import Callable, TypeVar From edbecb08764356e056231dca695e9e78272fed4e Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 13 Apr 2017 18:35:40 -0700 Subject: [PATCH 32/37] Fixed type variable scoping rules to conform to PEP484 better --- mypy/semanal.py | 130 ++++++++++++++++++----------- test-data/unit/semanal-errors.test | 8 ++ 2 files changed, 91 insertions(+), 47 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 059a8a5e56be..e6752fef62e9 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -187,8 +187,12 @@ class SemanticAnalyzer(NodeVisitor): type_stack = None # type: List[TypeInfo] # Type variables that are bound by the directly enclosing class bound_tvars = None # type: List[SymbolTableNode] - # Stack of type variables that were bound by outer classess + # Type variables bound by the current scope, be it class or function tvar_scope = None # type: TypeVarScope + # Type variables bound by current function/method + function_tvar_scope = None # type: TypeVarScope + # Stack of type variable scopes bound by current class. Only the last is active. + classdef_tvar_scopes = None # type: List[TypeVarScope] # Per-module options options = None # type: Options @@ -223,6 +227,8 @@ def __init__(self, self.type = None self.type_stack = [] self.tvar_scope = TypeVarScope() + self.function_tvar_scope = self.tvar_scope + self.classdef_tvar_scopes = [] self.function_stack = [] self.block_depth = [0] self.loop_depth = 0 @@ -417,14 +423,15 @@ def set_original_def(self, previous: Node, new: FuncDef) -> bool: else: return False - def update_function_type_variables(self, fun_type: CallableType, defn: Context) -> None: + def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) -> None: """Make any type variables in the signature of defn explicit. Update the signature of defn to contain type variable definitions if defn is generic. """ - a = self.type_analyzer(tvar_scope=TypeVarScope(self.tvar_scope)) - fun_type.variables = a.bind_function_type_variables(fun_type, defn) + with self.tvar_scope_frame(): + a = self.type_analyzer() + fun_type.variables = a.bind_function_type_variables(fun_type, defn) def visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None: # OverloadedFuncDef refers to any legitimate situation where you have @@ -540,19 +547,19 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - def analyze_function(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - if defn.type: - self.check_classvar_in_signature(defn.type) - assert isinstance(defn.type, CallableType) - # Signature must be analyzed in the surrounding scope so that - # class-level imported names and type variables are in scope. - defn.type = self.type_analyzer().visit_callable_type(defn.type, nested=False) - self.check_function_signature(defn) - if isinstance(defn, FuncDef): - defn.type = set_callable_name(defn.type, defn) - for arg in defn.arguments: - if arg.initializer: - arg.initializer.accept(self) with self.tvar_scope_frame(): + if defn.type: + self.check_classvar_in_signature(defn.type) + assert isinstance(defn.type, CallableType) + # Signature must be analyzed in the surrounding scope so that + # class-level imported names and type variables are in scope. + defn.type = self.type_analyzer().visit_callable_type(defn.type, nested=False) + self.check_function_signature(defn) + if isinstance(defn, FuncDef): + defn.type = set_callable_name(defn.type, defn) + for arg in defn.arguments: + if arg.initializer: + arg.initializer.accept(self) # Bind the type variables again to visit the body. if defn.type: a = self.type_analyzer() @@ -620,34 +627,31 @@ def visit_class_def(self, defn: ClassDef) -> None: @contextmanager def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: - old_scope = self.tvar_scope - self.tvar_scope = TypeVarScope() - self.clean_up_bases_and_infer_type_variables(defn) - if self.analyze_typeddict_classdef(defn): - yield False - return - if self.analyze_namedtuple_classdef(defn): - # just analyze the class body so we catch type errors in default values - self.enter_class(defn) - yield True - self.leave_class() - else: - self.setup_class_def_analysis(defn) - self.analyze_base_classes(defn) - self.analyze_metaclass(defn) - - for decorator in defn.decorators: - self.analyze_class_decorator(defn, decorator) + with self.classdef_tvar_scope_frame(): + self.clean_up_bases_and_infer_type_variables(defn) + if self.analyze_typeddict_classdef(defn): + yield False + return + if self.analyze_namedtuple_classdef(defn): + # just analyze the class body so we catch type errors in default values + self.enter_class(defn) + yield True + self.leave_class() + else: + self.setup_class_def_analysis(defn) + self.analyze_base_classes(defn) + self.analyze_metaclass(defn) - self.enter_class(defn) - yield True + for decorator in defn.decorators: + self.analyze_class_decorator(defn, decorator) - self.calculate_abstract_status(defn.info) - self.setup_type_promotion(defn) + self.enter_class(defn) + yield True - self.leave_class() + self.calculate_abstract_status(defn.info) + self.setup_type_promotion(defn) - self.tvar_scope = old_scope + self.leave_class() def enter_class(self, defn: ClassDef) -> None: # Remember previous active class @@ -785,10 +789,17 @@ def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: return None unbound = t sym = self.lookup_qualified(unbound.name, unbound) - if sym is not None and sym.kind == TVAR and self.tvar_scope.get_binding(sym) is None: + if sym is None or sym.kind != TVAR: + return None + elif self.tvar_scope.get_binding(sym): + # It's bound by our type variable scope + return None + elif any(s.get_binding(sym) for s in self.classdef_tvar_scopes): + # It's bound by some outer class's type variable scope + return None + else: assert isinstance(sym.node, TypeVarExpr) return unbound.name, sym.node - return None def get_all_bases_tvars(self, defn: ClassDef, removed: List[int]) -> TypeVarList: tvars = [] # type: TypeVarList @@ -1363,7 +1374,7 @@ def visit_block_maybe(self, b: Block) -> None: if b: self.visit_block(b) - def type_analyzer(self, + def type_analyzer(self, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, aliasing: bool = False) -> TypeAnalyser: @@ -1377,10 +1388,13 @@ def type_analyzer(self, allow_tuple_literal=allow_tuple_literal, allow_unnormalized=self.is_stub_file) - def anal_type(self, t: Type, allow_tuple_literal: bool = False, + def anal_type(self, t: Type, *, + tvar_scope: Optional[TypeVarScope] = None, + allow_tuple_literal: bool = False, aliasing: bool = False) -> Type: if t: a = self.type_analyzer( + tvar_scope=tvar_scope, aliasing=aliasing, allow_tuple_literal=allow_tuple_literal) return t.accept(a) @@ -1395,7 +1409,7 @@ def visit_assignment_stmt(self, s: AssignmentStmt) -> None: s.rvalue.accept(self) if s.type: allow_tuple_literal = isinstance(s.lvalues[-1], (TupleExpr, ListExpr)) - s.type = self.anal_type(s.type, allow_tuple_literal) + s.type = self.anal_type(s.type, allow_tuple_literal=allow_tuple_literal) else: # For simple assignments, allow binding type aliases. # Also set the type if the rvalue is a simple literal. @@ -2498,7 +2512,7 @@ def visit_for_stmt(self, s: ForStmt) -> None: if self.is_classvar(s.index_type): self.fail_invalid_classvar(s.index) allow_tuple_literal = isinstance(s.index, (TupleExpr, ListExpr)) - s.index_type = self.anal_type(s.index_type, allow_tuple_literal) + s.index_type = self.anal_type(s.index_type, allow_tuple_literal=allow_tuple_literal) self.store_declared_types(s.index, s.index_type) self.loop_depth += 1 @@ -2575,7 +2589,7 @@ def visit_with_stmt(self, s: WithStmt) -> None: if self.is_classvar(t): self.fail_invalid_classvar(n) allow_tuple_literal = isinstance(n, (TupleExpr, ListExpr)) - t = self.anal_type(t, allow_tuple_literal) + t = self.anal_type(t, allow_tuple_literal=allow_tuple_literal) new_types.append(t) self.store_declared_types(n, t) @@ -3019,10 +3033,32 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: @contextmanager def tvar_scope_frame(self) -> Generator: + """Prepare our TypeVarScope for a function or method""" old_scope = self.tvar_scope + old_function_scope = self.function_tvar_scope self.tvar_scope = TypeVarScope(self.tvar_scope) + self.function_tvar_scope = self.tvar_scope + yield + self.function_tvar_scope = old_function_scope + self.tvar_scope = old_scope + + @contextmanager + def classdef_tvar_scope_frame(self) -> Generator: + """Prepare our TypeVarScope stack for a newly-declared class. + + The scope for an inner class is based on the surrounding function + TypeVarScope, not the surrounding class's -- but it's still an error to + try and bind the same variable as the surrounding class, so we keep a + stack of those around. + """ + old_class_scope = self.classdef_tvar_scopes + old_scope = self.tvar_scope + new_scope = TypeVarScope(self.function_tvar_scope) + self.classdef_tvar_scopes = old_class_scope + [new_scope] + self.tvar_scope = new_scope yield self.tvar_scope = old_scope + self.classdef_tvar_scopes = old_class_scope def lookup(self, name: str, ctx: Context) -> SymbolTableNode: """Look up an unqualified name in all active namespaces.""" diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index f7af185b607a..99584e77a977 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -940,6 +940,14 @@ from typing import Generic class A(Generic[int]): pass # E: Free type variable expected in Generic[...] [out] +[case testInvalidTypeWithinNestedGenericClass] +from typing import Generic, TypeVar +T = TypeVar('T') +class A(Generic[T]): + class B(Generic[T]): pass \ + # E: Free type variable expected in Generic[...] +[out] + [case testIncludingGenericTwiceInBaseClassList] from typing import Generic, TypeVar T = TypeVar('T') From 21d8b138dc0890e5ea29ad77b5aab6b6b688fdea Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 17 Apr 2017 16:23:37 -0700 Subject: [PATCH 33/37] Move the typevars that are prohibited due to being bound by the class into TypeVarScope This allows us to be more consistent about which type variables we allow and which we don't. --- mypy/semanal.py | 45 +++++---------------------- mypy/tvar_scope.py | 50 ++++++++++++++++++++++++------ mypy/typeanal.py | 9 ++++-- test-data/unit/check-generics.test | 12 ++++++- 4 files changed, 64 insertions(+), 52 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index e6752fef62e9..1363aeafb83a 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -189,10 +189,6 @@ class SemanticAnalyzer(NodeVisitor): bound_tvars = None # type: List[SymbolTableNode] # Type variables bound by the current scope, be it class or function tvar_scope = None # type: TypeVarScope - # Type variables bound by current function/method - function_tvar_scope = None # type: TypeVarScope - # Stack of type variable scopes bound by current class. Only the last is active. - classdef_tvar_scopes = None # type: List[TypeVarScope] # Per-module options options = None # type: Options @@ -227,8 +223,6 @@ def __init__(self, self.type = None self.type_stack = [] self.tvar_scope = TypeVarScope() - self.function_tvar_scope = self.tvar_scope - self.classdef_tvar_scopes = [] self.function_stack = [] self.block_depth = [0] self.loop_depth = 0 @@ -429,7 +423,7 @@ def update_function_type_variables(self, fun_type: CallableType, defn: FuncItem) Update the signature of defn to contain type variable definitions if defn is generic. """ - with self.tvar_scope_frame(): + with self.tvar_scope_frame(self.tvar_scope.method_frame()): a = self.type_analyzer() fun_type.variables = a.bind_function_type_variables(fun_type, defn) @@ -547,7 +541,7 @@ def analyze_property_with_multi_part_definition(self, defn: OverloadedFuncDef) - def analyze_function(self, defn: FuncItem) -> None: is_method = self.is_class_scope() - with self.tvar_scope_frame(): + with self.tvar_scope_frame(self.tvar_scope.method_frame()): if defn.type: self.check_classvar_in_signature(defn.type) assert isinstance(defn.type, CallableType) @@ -627,7 +621,7 @@ def visit_class_def(self, defn: ClassDef) -> None: @contextmanager def analyze_class_body(self, defn: ClassDef) -> Iterator[bool]: - with self.classdef_tvar_scope_frame(): + with self.tvar_scope_frame(self.tvar_scope.class_frame()): self.clean_up_bases_and_infer_type_variables(defn) if self.analyze_typeddict_classdef(defn): yield False @@ -762,7 +756,7 @@ def clean_up_bases_and_infer_type_variables(self, defn: ClassDef) -> None: del defn.base_type_exprs[i] tvar_defs = [] # type: List[TypeVarDef] for name, tvar_expr in declared_tvars: - tvar_defs.append(self.tvar_scope.bind_class_tvar(name, tvar_expr)) + tvar_defs.append(self.tvar_scope.bind(name, tvar_expr)) defn.type_vars = tvar_defs def analyze_typevar_declaration(self, t: Type) -> Optional[TypeVarList]: @@ -791,12 +785,9 @@ def analyze_unbound_tvar(self, t: Type) -> Tuple[str, TypeVarExpr]: sym = self.lookup_qualified(unbound.name, unbound) if sym is None or sym.kind != TVAR: return None - elif self.tvar_scope.get_binding(sym): + elif not self.tvar_scope.allow_binding(sym.fullname): # It's bound by our type variable scope return None - elif any(s.get_binding(sym) for s in self.classdef_tvar_scopes): - # It's bound by some outer class's type variable scope - return None else: assert isinstance(sym.node, TypeVarExpr) return unbound.name, sym.node @@ -3032,33 +3023,11 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: # @contextmanager - def tvar_scope_frame(self) -> Generator: - """Prepare our TypeVarScope for a function or method""" - old_scope = self.tvar_scope - old_function_scope = self.function_tvar_scope - self.tvar_scope = TypeVarScope(self.tvar_scope) - self.function_tvar_scope = self.tvar_scope - yield - self.function_tvar_scope = old_function_scope - self.tvar_scope = old_scope - - @contextmanager - def classdef_tvar_scope_frame(self) -> Generator: - """Prepare our TypeVarScope stack for a newly-declared class. - - The scope for an inner class is based on the surrounding function - TypeVarScope, not the surrounding class's -- but it's still an error to - try and bind the same variable as the surrounding class, so we keep a - stack of those around. - """ - old_class_scope = self.classdef_tvar_scopes + def tvar_scope_frame(self, frame: TypeVarScope) -> Generator: old_scope = self.tvar_scope - new_scope = TypeVarScope(self.function_tvar_scope) - self.classdef_tvar_scopes = old_class_scope + [new_scope] - self.tvar_scope = new_scope + self.tvar_scope = frame yield self.tvar_scope = old_scope - self.classdef_tvar_scopes = old_class_scope def lookup(self, name: str, ctx: Context) -> SymbolTableNode: """Look up an unqualified name in all active namespaces.""" diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 5b4263352c50..0d7d6443eda1 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -4,28 +4,58 @@ class TypeVarScope: - """Scope that holds bindings for type variables. Node fullname -> TypeVarDef. + """Scope that holds bindings for type variables. Node fullname -> TypeVarDef.""" - Provide a parent on intitalization when you want to make a child scope.""" + def __init__(self, + parent: Optional['TypeVarScope'] = None, + is_class_scope: bool = False, + prohibited: Optional['TypeVarScope'] = None) -> None: + """Initializer for TypeVarScope - def __init__(self, parent: Optional['TypeVarScope'] = None) -> None: + Parameters: + parent: the outer scope for this scope + is_class_scope: True if this represents a generic class + prohibited: Type variables that aren't strictly in scope exactly, + but can't be bound because they're part of an outer class's scope. + """ self.scope = {} # type: Dict[str, TypeVarDef] self.parent = parent self.func_id = 0 self.class_id = 0 + self.is_class_scope = is_class_scope + self.prohibited = prohibited if parent is not None: self.func_id = parent.func_id self.class_id = parent.class_id - def bind_fun_tvar(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: - self.func_id -= 1 - return self._bind(name, tvar_expr, self.func_id) + def get_function_scope(self) -> Optional['TypeVarScope']: + it = self + while it.is_class_scope: + it = it.parent + return it - def bind_class_tvar(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: - self.class_id += 1 - return self._bind(name, tvar_expr, self.class_id) + def allow_binding(self, fullname: str) -> bool: + if fullname in self.scope: + return False + elif self.parent and not self.parent.allow_binding(fullname): + return False + elif self.prohibited and not self.prohibited.allow_binding(fullname): + return False + return True + + def method_frame(self) -> 'TypeVarScope': + return TypeVarScope(self, False, None) - def _bind(self, name: str, tvar_expr: TypeVarExpr, i: int) -> TypeVarDef: + def class_frame(self) -> 'TypeVarScope': + return TypeVarScope(self.get_function_scope(), True, self) + + def bind(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: + if self.is_class_scope: + self.class_id += 1 + i = self.class_id + else: + self.func_id -= 1 + i = self.func_id tvar_def = TypeVarDef( name, i, values=tvar_expr.values, upper_bound=tvar_expr.upper_bound, variance=tvar_expr.variance, diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 46838bf7b050..dc477ff2d78e 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -408,7 +408,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: @contextmanager def tvar_scope_frame(self) -> Generator: old_scope = self.tvar_scope - self.tvar_scope = TypeVarScope(self.tvar_scope) + self.tvar_scope = self.tvar_scope.method_frame() yield self.tvar_scope = old_scope @@ -437,7 +437,7 @@ def bind_function_type_variables(self, for var in fun_type.variables: var_expr = self.lookup(var.name, var).node assert isinstance(var_expr, TypeVarExpr) - self.tvar_scope.bind_fun_tvar(var.name, var_expr) + self.tvar_scope.bind(var.name, var_expr) return fun_type.variables typevars = self.infer_type_variables(fun_type) # Do not define a new type variable if already defined in scope. @@ -445,8 +445,11 @@ def bind_function_type_variables(self, if not self.is_defined_type_var(name, defn)] defs = [] # type: List[TypeVarDef] for name, tvar in typevars: - self.tvar_scope.bind_fun_tvar(name, tvar) + if not self.tvar_scope.allow_binding(tvar.fullname()): + self.fail("Type variable '{}' is bound by an outer class".format(name), defn) + self.tvar_scope.bind(name, tvar) defs.append(self.tvar_scope.get_binding(tvar.fullname())) + return defs def is_defined_type_var(self, tvar: str, context: Context) -> bool: diff --git a/test-data/unit/check-generics.test b/test-data/unit/check-generics.test index 9d1a0ae8899e..4aa4bf49c2c1 100644 --- a/test-data/unit/check-generics.test +++ b/test-data/unit/check-generics.test @@ -1347,7 +1347,17 @@ class A(Generic[T]): self.a = a g(self.a) g(n) # E: Argument 1 to "g" has incompatible type "int"; expected "T" -[out] + +[case testFunctionInGenericInnerClassTypeVariable] +from typing import TypeVar, Generic + +T = TypeVar('T') +class Outer(Generic[T]): + class Inner: + x: T # E: Invalid type "__main__.T" + def f(self, x: T) -> T: ... # E: Type variable 'T' is bound by an outer class + def g(self) -> None: + y: T # E: Invalid type "__main__.T" -- Callable subtyping with generic functions From 214aebc781b8183b053923ebb230cc5870e59d1f Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 17 Apr 2017 16:25:58 -0700 Subject: [PATCH 34/37] More doc --- mypy/tvar_scope.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 0d7d6443eda1..29ca40b91ac2 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -29,6 +29,7 @@ def __init__(self, self.class_id = parent.class_id def get_function_scope(self) -> Optional['TypeVarScope']: + """Get the nearest parent that's a function scope, not a class scope""" it = self while it.is_class_scope: it = it.parent @@ -44,9 +45,11 @@ def allow_binding(self, fullname: str) -> bool: return True def method_frame(self) -> 'TypeVarScope': + """A new scope frame for binding a method""" return TypeVarScope(self, False, None) def class_frame(self) -> 'TypeVarScope': + """A new scope frame for binding a class. Prohibits *this* class's tvars""" return TypeVarScope(self.get_function_scope(), True, self) def bind(self, name: str, tvar_expr: TypeVarExpr) -> TypeVarDef: From 54056465e04e05b5e3a1b7f891cadf6a2dcd0f9b Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 09:57:34 -0700 Subject: [PATCH 35/37] Jukka comments --- mypy/tvar_scope.py | 10 +++++----- mypy/typeanal.py | 18 +++++++++++------- typeshed | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/mypy/tvar_scope.py b/mypy/tvar_scope.py index 29ca40b91ac2..3cdb67bbf992 100644 --- a/mypy/tvar_scope.py +++ b/mypy/tvar_scope.py @@ -13,10 +13,10 @@ def __init__(self, """Initializer for TypeVarScope Parameters: - parent: the outer scope for this scope - is_class_scope: True if this represents a generic class - prohibited: Type variables that aren't strictly in scope exactly, - but can't be bound because they're part of an outer class's scope. + parent: the outer scope for this scope + is_class_scope: True if this represents a generic class + prohibited: Type variables that aren't strictly in scope exactly, + but can't be bound because they're part of an outer class's scope. """ self.scope = {} # type: Dict[str, TypeVarDef] self.parent = parent @@ -31,7 +31,7 @@ def __init__(self, def get_function_scope(self) -> Optional['TypeVarScope']: """Get the nearest parent that's a function scope, not a class scope""" it = self - while it.is_class_scope: + while it is not None and it.is_class_scope: it = it.parent return it diff --git a/mypy/typeanal.py b/mypy/typeanal.py index dc477ff2d78e..b1c5a12eb637 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -266,7 +266,7 @@ def get_type_var_names(self, tp: Type) -> List[str]: """ return [name for name, _ in tp.accept(TypeVariableQuery(self.lookup, self.tvar_scope, - include_callables=True, include_bound=True))] + include_callables=True, include_bound_tvars=True))] def get_tvar_name(self, t: Type) -> Optional[str]: if not isinstance(t, UnboundType): @@ -316,6 +316,7 @@ def visit_type_var(self, t: TypeVarType) -> Type: return t def visit_callable_type(self, t: CallableType, nested: bool = True) -> Type: + # Every Callable can bind its own type variables, if they're not in the outer scope with self.tvar_scope_frame(): if self.aliasing: variables = t.variables @@ -406,7 +407,7 @@ def analyze_callable_type(self, t: UnboundType) -> Type: return ret.accept(self) @contextmanager - def tvar_scope_frame(self) -> Generator: + def tvar_scope_frame(self) -> Iterator[None]: old_scope = self.tvar_scope self.tvar_scope = self.tvar_scope.method_frame() yield @@ -422,6 +423,10 @@ def infer_type_variables(self, if name not in names: names.append(name) tvars.append(tvar_expr) + # When finding type variables in the return type of a function, don't + # look inside Callable types. Type variables only appearing in + # functions in the return type belong to those functions, not the + # function we're currently analyzing. for name, tvar_expr in type.ret_type.accept( TypeVariableQuery(self.lookup, self.tvar_scope, include_callables=False)): if name not in names: @@ -431,8 +436,7 @@ def infer_type_variables(self, def bind_function_type_variables(self, fun_type: CallableType, defn: Context) -> List[TypeVarDef]: - """Find the type variables of the function type and bind them in our tvar_scope. - """ + """Find the type variables of the function type and bind them in our tvar_scope""" if fun_type.variables: for var in fun_type.variables: var_expr = self.lookup(var.name, var).node @@ -645,11 +649,11 @@ def __init__(self, scope: 'TypeVarScope', *, include_callables: bool = True, - include_bound: bool = False) -> None: + include_bound_tvars: bool = False) -> None: self.include_callables = include_callables self.lookup = lookup self.scope = scope - self.include_bound = include_bound + self.include_bound_tvars = include_bound_tvars super().__init__(flatten_tvars) def _seems_like_callable(self, type: UnboundType) -> bool: @@ -663,7 +667,7 @@ def visit_unbound_type(self, t: UnboundType) -> TypeVarList: name = t.name node = self.lookup(name, t) if node and node.kind == TVAR and ( - self.include_bound or self.scope.get_binding(node) is None): + self.include_bound_tvars or self.scope.get_binding(node) is None): assert isinstance(node.node, TypeVarExpr) return[(name, node.node)] elif not self.include_callables and self._seems_like_callable(t): diff --git a/typeshed b/typeshed index 26360e821b42..8b835f95001b 160000 --- a/typeshed +++ b/typeshed @@ -1 +1 @@ -Subproject commit 26360e821b427c29ca27117061a3ba806f63e12e +Subproject commit 8b835f95001b61734c6b147d3aa6eb4fbe7bce03 From a25e9262d5fb3468bad5dcaea7ea0c2e3f41b2ed Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 09:58:06 -0700 Subject: [PATCH 36/37] one more --- mypy/typeanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index b1c5a12eb637..79af5c51ce56 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -669,7 +669,7 @@ def visit_unbound_type(self, t: UnboundType) -> TypeVarList: if node and node.kind == TVAR and ( self.include_bound_tvars or self.scope.get_binding(node) is None): assert isinstance(node.node, TypeVarExpr) - return[(name, node.node)] + return [(name, node.node)] elif not self.include_callables and self._seems_like_callable(t): return [] else: From 4aaab6cc124ee8c1c8a8f4a456cf8f083adf15b6 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 20 Apr 2017 10:47:08 -0700 Subject: [PATCH 37/37] get imports right --- mypy/semanal.py | 4 ++-- mypy/typeanal.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 1363aeafb83a..dfcf15218900 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -47,7 +47,7 @@ from contextlib import contextmanager from typing import ( - List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable, Generator, Iterator, + List, Dict, Set, Tuple, cast, TypeVar, Union, Optional, Callable, Iterator, ) from mypy.nodes import ( @@ -3023,7 +3023,7 @@ def visit_await_expr(self, expr: AwaitExpr) -> None: # @contextmanager - def tvar_scope_frame(self, frame: TypeVarScope) -> Generator: + def tvar_scope_frame(self, frame: TypeVarScope) -> Iterator[None]: old_scope = self.tvar_scope self.tvar_scope = frame yield diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 79af5c51ce56..6e25f1b157cf 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -1,7 +1,7 @@ """Semantic analysis of types""" from collections import OrderedDict -from typing import Callable, List, Optional, Set, Tuple, Generator, TypeVar, Iterable +from typing import Callable, List, Optional, Set, Tuple, Iterator, TypeVar, Iterable from itertools import chain from contextlib import contextmanager