From 595760f2bdb3eac612cea1ecec5fcef3cba240b7 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sun, 30 Dec 2018 16:54:14 -0800 Subject: [PATCH 1/2] Remove RawLiteralType synthetic type This diff changes how we track raw literal types in the semantic analysis phase. It makes the following changes: 1. Removes the `RawLiteralType` synthetic type. 2. Adds a new `TypeOfAny`: `TypeOfAny.invalid_type` as suggested in https://github.com/python/mypy/issues/4030. 3. Modifies `AnyType` so it can optionally contain a new `RawLiteral` class. This class contains information about the underlying literal that produced that particular `TypeOfAny`. 4. Adjusts mypy to stop recommending using `Literal[...]` when doing `A = NewType('A', 4)` or `T = TypeVar('T', bound=4)`. (The former suggestion is a bad one: you can't create a NewType of a Literal[...] type. The latter suggestion is a valid but stupid one: `T = TypeVar('T', bound=Literal[4])` is basically the same thing as `T = Literal[4]`.) This resolves https://github.com/python/mypy/issues/5989. The net effect of this diff is that: 1. RawLiteralTypes no longer leak during fine-grained mode, which should partially help unblock https://github.com/python/mypy/pull/6075. 2. The way mypy handles literal expressions in types is "inverted". Previously, we by default assumed literal expressions would belong inside `Literal[...]` and tacked on some logic to make them convert into error `AnyTypes`. Now, we do the reverse: we start with an error `AnyType` and convert those into `Literal[...]`s as needed. This more closely mirrors the way mypy *used* to work before we started work on Literal types. It should also hopefully help reduce some of the cognitive burden of working on other parts of the semantic analysis code, since we no longer need to worry about the `RawLiteralType` synthetic type. 3. We now have more flexibility in how we choose to handle invalid types: since they're just `Anys`, we have more opportunities to intercept and customize the exact way in which we handle errors. Also see https://github.com/python/mypy/issues/4030 for additional context. (This diff lays out some of the foundation work for that diff). --- mypy/exprtotype.py | 40 ++++++-- mypy/fastparse.py | 47 ++++++++-- mypy/indirection.py | 3 - mypy/plugin.py | 1 + mypy/semanal.py | 15 ++- mypy/semanal_newtype.py | 5 +- mypy/semanal_shared.py | 1 + mypy/server/astdiff.py | 13 ++- mypy/server/astmerge.py | 4 - mypy/type_visitor.py | 9 +- mypy/typeanal.py | 141 ++++++++++++++++------------ mypy/types.py | 143 +++++++++++++++++------------ test-data/unit/check-newtype.test | 3 +- test-data/unit/semanal-errors.test | 2 +- 14 files changed, 266 insertions(+), 161 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 6528ed562508..01d602a0db6f 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -8,7 +8,7 @@ from mypy.fastparse import parse_type_string from mypy.types import ( Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, TypeOfAny, - RawLiteralType, + RawLiteral, ) @@ -39,9 +39,19 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No if isinstance(expr, NameExpr): name = expr.name if name == 'True': - return RawLiteralType(True, 'builtins.bool', line=expr.line, column=expr.column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(True, 'builtins.bool'), + line=expr.line, + column=expr.column, + ) elif name == 'False': - return RawLiteralType(False, 'builtins.bool', line=expr.line, column=expr.column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(False, 'builtins.bool'), + line=expr.line, + column=expr.column, + ) else: return UnboundType(name, line=expr.line, column=expr.column) elif isinstance(expr, MemberExpr): @@ -122,17 +132,27 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No assume_str_is_unicode=True) elif isinstance(expr, UnaryExpr): typ = expr_to_unanalyzed_type(expr.expr) - if isinstance(typ, RawLiteralType) and isinstance(typ.value, int) and expr.op == '-': - typ.value *= -1 - return typ - else: - raise TypeTranslationError() + if isinstance(typ, AnyType) and typ.raw_literal is not None: + if isinstance(typ.raw_literal.value, int) and expr.op == '-': + typ.raw_literal.value *= -1 + return typ + raise TypeTranslationError() elif isinstance(expr, IntExpr): - return RawLiteralType(expr.value, 'builtins.int', line=expr.line, column=expr.column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(expr.value, 'builtins.int'), + line=expr.line, + column=expr.column, + ) elif isinstance(expr, FloatExpr): # Floats are not valid parameters for RawLiteralType, so we just # pass in 'None' for now. We'll report the appropriate error at a later stage. - return RawLiteralType(None, 'builtins.float', line=expr.line, column=expr.column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(None, 'builtins.float'), + line=expr.line, + column=expr.column, + ) elif isinstance(expr, EllipsisExpr): return EllipsisType(expr.line) else: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7c10e14fc1ac..19bf7d5f64cd 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -31,7 +31,7 @@ ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument, - TypeOfAny, Instance, RawLiteralType, + TypeOfAny, Instance, RawLiteral, ) from mypy import defaults from mypy import messages @@ -184,11 +184,21 @@ def parse_type_string(expr_string: str, expr_fallback_name: str, node.original_str_fallback = expr_fallback_name return node else: - return RawLiteralType(expr_string, expr_fallback_name, line, column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(expr_string, expr_fallback_name), + line=line, + column=column, + ) except (SyntaxError, ValueError): # Note: the parser will raise a `ValueError` instead of a SyntaxError if # the string happens to contain things like \x00. - return RawLiteralType(expr_string, expr_fallback_name, line, column) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(expr_string, expr_fallback_name), + line=line, + column=column, + ) def is_no_type_check_decorator(expr: ast3.expr) -> bool: @@ -1183,7 +1193,11 @@ def visit_Name(self, n: Name) -> Type: def visit_NameConstant(self, n: NameConstant) -> Type: if isinstance(n.value, bool): - return RawLiteralType(n.value, 'builtins.bool', line=self.line) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(n.value, 'builtins.bool'), + line=self.line, + ) else: return UnboundType(str(n.value), line=self.line) @@ -1192,9 +1206,9 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type: # We support specifically Literal[-4] and nothing else. # For example, Literal[+4] or Literal[~6] is not supported. typ = self.visit(n.operand) - if isinstance(typ, RawLiteralType) and isinstance(n.op, USub): - if isinstance(typ.value, int): - typ.value *= -1 + if isinstance(typ, AnyType) and typ.raw_literal is not None: + if isinstance(typ.raw_literal.value, int) and isinstance(n.op, USub): + typ.raw_literal.value *= -1 return typ self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) return AnyType(TypeOfAny.from_error) @@ -1204,11 +1218,19 @@ def visit_Num(self, n: Num) -> Type: # Could be either float or int numeric_value = n.n if isinstance(numeric_value, int): - return RawLiteralType(numeric_value, 'builtins.int', line=self.line) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(numeric_value, 'builtins.int'), + line=self.line, + ) elif isinstance(numeric_value, float): # Floats and other numbers are not valid parameters for RawLiteralType, so we just # pass in 'None' for now. We'll report the appropriate error at a later stage. - return RawLiteralType(None, 'builtins.float', line=self.line) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(None, 'builtins.float'), + line=self.line, + ) else: self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) return AnyType(TypeOfAny.from_error) @@ -1230,7 +1252,12 @@ def visit_Str(self, n: Str) -> Type: # Bytes(bytes s) def visit_Bytes(self, n: Bytes) -> Type: contents = bytes_to_human_readable_repr(n.s) - return RawLiteralType(contents, 'builtins.bytes', self.line, column=n.col_offset) + return AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteral(contents, 'builtins.bytes'), + line=self.line, + column=n.col_offset, + ) # Subscript(expr value, slice slice, expr_context ctx) def visit_Subscript(self, n: ast3.Subscript) -> Type: diff --git a/mypy/indirection.py b/mypy/indirection.py index 4e3390a65e3c..2776277acaa7 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -90,9 +90,6 @@ def visit_tuple_type(self, t: types.TupleType) -> Set[str]: def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]: return self._visit(t.items.values()) | self._visit(t.fallback) - def visit_raw_literal_type(self, t: types.RawLiteralType) -> Set[str]: - assert False, "Unexpected RawLiteralType after semantic analysis phase" - def visit_literal_type(self, t: types.LiteralType) -> Set[str]: return self._visit(t.fallback) diff --git a/mypy/plugin.py b/mypy/plugin.py index 7238dd132877..39379394bf9c 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -180,6 +180,7 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: """Analyze an unbound type.""" raise NotImplementedError diff --git a/mypy/semanal.py b/mypy/semanal.py index 38277e18b600..16df577ba638 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1283,7 +1283,7 @@ def update_metaclass(self, defn: ClassDef) -> None: return defn.metaclass = metas.pop() - def expr_to_analyzed_type(self, expr: Expression) -> Type: + def expr_to_analyzed_type(self, expr: Expression, report_invalid_types: bool = True) -> Type: if isinstance(expr, CallExpr): expr.accept(self) info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope()) @@ -1295,7 +1295,7 @@ def expr_to_analyzed_type(self, expr: Expression) -> Type: fallback = Instance(info, []) return TupleType(info.tuple_type.items, fallback=fallback) typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) + return self.anal_type(typ, report_invalid_types=report_invalid_types) def verify_base_classes(self, defn: ClassDef) -> bool: info = defn.info @@ -1686,6 +1686,7 @@ def type_analyzer(self, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope @@ -1696,6 +1697,7 @@ def type_analyzer(self, *, self.is_typeshed_stub_file, allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, allow_unnormalized=self.is_stub_file, third_pass=third_pass) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) @@ -1706,10 +1708,12 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: a = self.type_analyzer(tvar_scope=tvar_scope, allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, third_pass=third_pass) typ = t.accept(a) self.add_type_alias_deps(a.aliases_used) @@ -2394,7 +2398,12 @@ def process_typevar_parameters(self, args: List[Expression], self.fail("TypeVar cannot have both values and an upper bound", context) return None try: - upper_bound = self.expr_to_analyzed_type(param_value) + upper_bound = self.expr_to_analyzed_type(param_value, + report_invalid_types=False) + if isinstance(upper_bound, AnyType) and upper_bound.from_invalid_type: + self.fail("TypeVar 'bound' must be a type", param_value) + # Note: we do not return 'None' here: we want to continue using the + # AnyType as the upper bound. except TypeTranslationError: self.fail("TypeVar 'bound' must be a type", param_value) return None diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index f4511e01bdee..dcaf25c4318a 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -114,11 +114,12 @@ def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Opt self.fail(msg, context) return None - old_type = self.api.anal_type(unanalyzed_type) + old_type = self.api.anal_type(unanalyzed_type, report_invalid_types=False) # The caller of this function assumes that if we return a Type, it's always # a valid one. So, we translate AnyTypes created from errors into None. - if isinstance(old_type, AnyType) and old_type.type_of_any == TypeOfAny.from_error: + bad_anys = (TypeOfAny.from_error, TypeOfAny.invalid_type) + if isinstance(old_type, AnyType) and old_type.type_of_any in bad_anys: self.fail(msg, context) return None diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index c88c9ff98195..4d8994e2084e 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -91,6 +91,7 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: raise NotImplementedError diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 8697358a4205..df14855f5b75 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -267,7 +267,18 @@ def visit_unbound_type(self, typ: UnboundType) -> SnapshotItem: snapshot_types(typ.args)) def visit_any(self, typ: AnyType) -> SnapshotItem: - return snapshot_simple_type(typ) + if typ.raw_literal: + return ( + 'Any', + typ.type_of_any, + typ.raw_literal.value, + typ.raw_literal.base_type_name, + ) + else: + return ( + 'Any', + typ.type_of_any, + ) def visit_none_type(self, typ: NoneTyp) -> SnapshotItem: return snapshot_simple_type(typ) diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index edfdc076e7d7..241cdc988112 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -59,7 +59,6 @@ Type, SyntheticTypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType, LiteralType, - RawLiteralType, ) from mypy.util import get_prefix, replace_object_state from mypy.typestate import TypeState @@ -392,9 +391,6 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None: value_type.accept(self) typ.fallback.accept(self) - def visit_raw_literal_type(self, t: RawLiteralType) -> None: - assert False, "Unexpected RawLiteralType after semantic analysis phase" - def visit_literal_type(self, typ: LiteralType) -> None: typ.fallback.accept(self) diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 64ae75e174ee..ef957d4ad87c 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -20,7 +20,7 @@ from mypy.types import ( Type, AnyType, CallableType, Overloaded, TupleType, TypedDictType, LiteralType, - RawLiteralType, Instance, NoneTyp, TypeType, + Instance, NoneTyp, TypeType, UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef, UnboundType, ErasedType, ForwardRef, StarType, EllipsisType, TypeList, CallableArgument, ) @@ -127,10 +127,6 @@ def visit_callable_argument(self, t: CallableArgument) -> T: def visit_ellipsis_type(self, t: EllipsisType) -> T: pass - @abstractmethod - def visit_raw_literal_type(self, t: RawLiteralType) -> T: - pass - @trait class TypeTranslator(TypeVisitor[Type]): @@ -282,9 +278,6 @@ def visit_tuple_type(self, t: TupleType) -> T: def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) - def visit_raw_literal_type(self, t: RawLiteralType) -> T: - return self.strategy([]) - def visit_literal_type(self, t: LiteralType) -> T: return self.strategy([]) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a37aa4c6c03b..044308c3f9de 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -16,7 +16,7 @@ CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded, - LiteralType, RawLiteralType, + LiteralType, ) from mypy.fastparse import TYPE_COMMENT_SYNTAX_ERROR @@ -158,6 +158,7 @@ def __init__(self, allow_tuple_literal: bool = False, allow_unnormalized: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> None: self.api = api self.lookup = api.lookup_qualified @@ -175,6 +176,11 @@ def __init__(self, self.allow_unnormalized = allow_unnormalized # Should we accept unbound type variables (always OK in aliases)? self.allow_unbound_tvars = allow_unbound_tvars or defining_alias + # Should we report an error whenever we encounter an Any corresponding to + # an invalid type? Normally, we want to report an error, but the caller + # may want to do more specialized error handling (or we may be in the + # middle of defining a Literal[...] type.) + self.report_invalid_types = report_invalid_types self.plugin = plugin self.options = options self.is_typeshed_stub = is_typeshed_stub @@ -416,6 +422,29 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl return t def visit_any(self, t: AnyType) -> Type: + if self.report_invalid_types and t.raw_literal is not None: + # Normally, we want to report an error message if we encounter an + # AnyType corresponding to an invalid type. The caller can decide + # to skip doing so if they want to generate a more customized + # error message (or if we're in the middle of generating a LiteralType). + # + # Note: if at some point in the distant future, we decide to + # make signatures like "foo(x: 20) -> None" legal, we can change + # this method so it generates and returns an actual LiteralType + # instead. + holder = t.raw_literal + if holder.base_type_name == 'builtins.int' or holder.base_type_name == 'builtins.bool': + # The only time it makes sense to use an int or bool is inside of + # a literal type. + msg = "Invalid type: try using Literal[{}] instead?".format(repr(holder.value)) + self.fail(msg, t) + elif holder.base_type_name == 'builtins.float': + self.fail("Invalid type: float literals cannot be used as a type", t) + else: + # For other types like strings, it's unclear if the user meant + # to construct a literal type or just misspelled a regular type. + # So, we leave just a generic "syntax error" error. + self.fail('Invalid type: ' + TYPE_COMMENT_SYNTAX_ERROR, t) return t def visit_none_type(self, t: NoneTyp) -> Type: @@ -489,30 +518,6 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: ]) return TypedDictType(items, set(t.required_keys), t.fallback) - def visit_raw_literal_type(self, t: RawLiteralType) -> Type: - # We should never see a bare Literal. We synthesize these raw literals - # in the earlier stages of semantic analysis, but those - # "fake literals" should always be wrapped in an UnboundType - # corresponding to 'Literal'. - # - # Note: if at some point in the distant future, we decide to - # make signatures like "foo(x: 20) -> None" legal, we can change - # this method so it generates and returns an actual LiteralType - # instead. - if t.base_type_name == 'builtins.int' or t.base_type_name == 'builtins.bool': - # The only time it makes sense to use an int or bool is inside of - # a literal type. - self.fail("Invalid type: try using Literal[{}] instead?".format(repr(t.value)), t) - elif t.base_type_name == 'builtins.float': - self.fail("Invalid type: float literals cannot be used as a type", t) - else: - # For other types like strings, it's unclear if the user meant - # to construct a literal type or just misspelled a regular type. - # So, we leave just a generic "syntax error" error. - self.fail('Invalid type: ' + TYPE_COMMENT_SYNTAX_ERROR, t) - - return AnyType(TypeOfAny.from_error) - def visit_literal_type(self, t: LiteralType) -> Type: return t @@ -620,14 +625,22 @@ def analyze_literal_type(self, t: UnboundType) -> Type: self.fail('Literal[...] must have at least one parameter', t) return AnyType(TypeOfAny.from_error) - output = [] # type: List[Type] - for i, arg in enumerate(t.args): - analyzed_types = self.analyze_literal_param(i + 1, arg, t) - if analyzed_types is None: - return AnyType(TypeOfAny.from_error) - else: - output.extend(analyzed_types) - return UnionType.make_union(output, line=t.line) + with self._set_report_invalid_types(report=False): + output = [] # type: List[Type] + for i, arg in enumerate(t.args): + analyzed_types = self.analyze_literal_param(i + 1, arg, t) + if analyzed_types is None: + return AnyType(TypeOfAny.from_error) + else: + output.extend(analyzed_types) + return UnionType.make_union(output, line=t.line) + + @contextmanager + def _set_report_invalid_types(self, report: bool = True) -> Iterator[None]: + old_value = self.report_invalid_types + self.report_invalid_types = report + yield + self.report_invalid_types = old_value def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[List[Type]]: # This UnboundType was originally defined as a string. @@ -645,36 +658,50 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L if isinstance(arg, UnboundType): arg = self.anal_type(arg) - # Literal[...] cannot contain Any. Give up and add an error message - # (if we haven't already). if isinstance(arg, AnyType): - # Note: We can encounter Literals containing 'Any' under three circumstances: + # Note: We encounter Literals containing 'Any' under four circumstances: # - # 1. If the user attempts use an explicit Any as a parameter - # 2. If the user is trying to use an enum value imported from a module with - # no type hints, giving it an an implicit type of 'Any' - # 3. If there's some other underlying problem with the parameter. + # 1. If the Any was generated by an invalid type -- some literal expression. + # 2. If the user attempts use an explicit Any as a parameter. + # 3. If the user is trying to use an enum value imported from a module with + # no type hints, giving it an an implicit type of 'Any'. + # 4. If there's some other underlying problem with the parameter. # - # We report an error in only the first two cases. In the third case, we assume - # some other region of the code has already reported a more relevant error. + # In case 1, we intercept the 'Any', extract the underlying raw literal + # expression that generated that Any, and synthesize and return LiteralType. # - # TODO: Once we start adding support for enums, make sure we reprt a custom - # error for case 2 as well. - if arg.type_of_any != TypeOfAny.from_error: + # In cases 2 and 3, we report an error: Literal[...] may not contain Anys. + # + # In case 4, we assume some other region of the code has already reported a + # more relevant error so just silently end. + # + # TODO: Once we start adding support for enums, make sure we report a custom + # error for case 3 as well. + if arg.raw_literal is not None: + # Case 1: The Any came from an "invalid type" -- e.g. a literal expression. + if arg.raw_literal.base_type_name == 'builtins.float': + self.fail( + 'Parameter {} of Literal[...] cannot be of type "float"'.format(idx), + ctx) + return None + + # Remap bytes and unicode into the appropriate type for the correct Python version + fallback = self.named_type_with_normalized_str(arg.raw_literal.base_type_name) + assert isinstance(fallback, Instance) + typ = LiteralType( + arg.raw_literal.value, + fallback, + line=arg.line, + column=arg.column, + ) + return [typ] + elif arg.type_of_any != TypeOfAny.from_error: + # Cases 2 and 3: We encountered an explicit Any of some kind. self.fail('Parameter {} of Literal[...] cannot be of type "Any"'.format(idx), ctx) - return None - elif isinstance(arg, RawLiteralType): - # A raw literal. Convert it directly into a literal. - if arg.base_type_name == 'builtins.float': - self.fail( - 'Parameter {} of Literal[...] cannot be of type "float"'.format(idx), - ctx) return None - - # Remap bytes and unicode into the appropriate type for the correct Python version - fallback = self.named_type_with_normalized_str(arg.base_type_name) - assert isinstance(fallback, Instance) - return [LiteralType(arg.value, fallback, line=arg.line, column=arg.column)] + else: + # Case 4: We assume somebody else reported an error already and silently end. + return None elif isinstance(arg, (NoneTyp, LiteralType)): # Types that we can just add directly to the literal/potential union of literals. return [arg] diff --git a/mypy/types.py b/mypy/types.py index d518ac092037..b0ef6adf5830 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -83,6 +83,10 @@ class TypeOfAny: from_another_any = 7 # type: Final # Does this Any come from an implementation limitation/bug? implementation_artifact = 8 # type: Final + # Does this Any come from some expression that is not a valid type? + # For example, if we have the type 'List[3]', we will represent that as 'List[Any]' + # where the inner Any has TypeOfAny.invalid_type. + invalid_type = 9 # type: Final def deserialize_type(data: Union[JsonDict, str]) -> 'Type': @@ -366,15 +370,76 @@ def serialize(self) -> JsonDict: _dummy = object() # type: Final[Any] +class RawLiteral: + """RawLiteral is a minimal class that represents any type that + could plausibly be something that lives inside of a literal. + + This class is used ONLY as an implementation detail of AnyType: whenever + we encounter expression that is normally an invalid type (but plausibly + could be a part of a Literal[...]) type, we represent that expression as + an AnyType containing TypeOfAny.invalid_type and this class. + + For example, suppose we have a type like 'Foo[3]'. When we parse this + during semantic analysis, we initially represent this type like so: + + UnboundType( + name='Foo', + args=[ + AnyType( + TypeOfAny.invalid_type, + raw_literal=RawLiteralHolder(3, 'builtins.int'), + ), + ] + ) + + If it turns out that 'Foo' is just a plain old Instance, the arg + remains an AnyType. But if it turns out that 'Foo' is an alias + for 'Literal', we intercept the Any and transform the entire + type into the following: + + LiteralType(value=1, fallback=int_instance_here). + + Note that this class does NOT represent a type or semantic type. + """ + def __init__(self, value: LiteralValue, base_type_name: str) -> None: + self.value = value + self.base_type_name = base_type_name + + def serialize(self) -> JsonDict: + return { + '.class': 'RawLiteral', + 'value': self.value, + 'base_type_name': self.base_type_name, + } + + @classmethod + def deserialize(cls, data: JsonDict) -> 'RawLiteral': + assert data['.class'] == 'RawLiteralHolder' + return RawLiteral( + value=data['value'], + base_type_name=data['base_type_name'], + ) + + def __hash__(self) -> int: + return hash((self.value, self.base_type_name)) + + def __eq__(self, other: object) -> bool: + if isinstance(other, RawLiteral): + return self.base_type_name == other.base_type_name and self.value == other.value + else: + return NotImplemented + + class AnyType(Type): """The type 'Any'.""" - __slots__ = ('type_of_any', 'source_any', 'missing_import_name') + __slots__ = ('type_of_any', 'source_any', 'missing_import_name', 'raw_literal') def __init__(self, type_of_any: int, source_any: Optional['AnyType'] = None, missing_import_name: Optional[str] = None, + raw_literal: Optional[RawLiteral] = None, line: int = -1, column: int = -1) -> None: super().__init__(line, column) @@ -390,6 +455,8 @@ def __init__(self, else: self.missing_import_name = source_any.missing_import_name + self.raw_literal = raw_literal + # Only unimported type anys and anys from other anys should have an import name assert (missing_import_name is None or type_of_any in (TypeOfAny.from_unimported_type, TypeOfAny.from_another_any)) @@ -397,6 +464,8 @@ def __init__(self, assert type_of_any != TypeOfAny.from_another_any or source_any is not None # We should not have chains of Anys. assert not self.source_any or self.source_any.type_of_any != TypeOfAny.from_another_any + # This 'Any' is an invalid type if and only if a raw_literal is provided. + assert (type_of_any == TypeOfAny.invalid_type) == (raw_literal is not None) def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_any(self) @@ -414,6 +483,10 @@ def copy_modified(self, missing_import_name=self.missing_import_name, line=self.line, column=self.column) + @property + def from_invalid_type(self) -> bool: + return self.type_of_any == TypeOfAny.invalid_type + def __hash__(self) -> int: return hash(AnyType) @@ -421,17 +494,22 @@ def __eq__(self, other: object) -> bool: return isinstance(other, AnyType) def serialize(self) -> JsonDict: - return {'.class': 'AnyType', 'type_of_any': self.type_of_any, - 'source_any': self.source_any.serialize() if self.source_any is not None else None, - 'missing_import_name': self.missing_import_name} + return { + '.class': 'AnyType', 'type_of_any': self.type_of_any, + 'source_any': self.source_any.serialize() if self.source_any is not None else None, + 'missing_import_name': self.missing_import_name, + 'raw_literal': self.raw_literal.serialize() if self.raw_literal is not None else None, + } @classmethod def deserialize(cls, data: JsonDict) -> 'AnyType': assert data['.class'] == 'AnyType' source = data['source_any'] + raw_literal = data['raw_literal'] return AnyType(data['type_of_any'], AnyType.deserialize(source) if source is not None else None, - data['missing_import_name']) + data['missing_import_name'], + RawLiteral.deserialize(raw_literal) if raw_literal is not None else None) class UninhabitedType(Type): @@ -1299,58 +1377,6 @@ def zipall(self, right: 'TypedDictType') \ yield (item_name, None, right_item_type) -class RawLiteralType(Type): - """A synthetic type representing any type that could plausibly be something - that lives inside of a literal. - - This synthetic type is only used at the beginning stages of semantic analysis - and should be completely removing during the process for mapping UnboundTypes to - actual types. - - For example, `Foo[1]` is initially represented as the following: - - UnboundType( - name='Foo', - args=[ - RawLiteralType(value=1, base_type_name='builtins.int'), - ], - ) - - As we perform semantic analysis, this type will transform into one of two - possible forms. - - If 'Foo' was an alias for 'Literal' all along, this type is transformed into: - - LiteralType(value=1, fallback=int_instance_here) - - Alternatively, if 'Foo' is an unrelated class, we report an error and instead - produce something like this: - - Instance(type=typeinfo_for_foo, args=[AnyType(TypeOfAny.from_error)) - """ - def __init__(self, value: LiteralValue, base_type_name: str, - line: int = -1, column: int = -1) -> None: - super().__init__(line, column) - self.value = value - self.base_type_name = base_type_name - - def accept(self, visitor: 'TypeVisitor[T]') -> T: - assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_raw_literal_type(self) - - def serialize(self) -> JsonDict: - assert False, "Synthetic types don't serialize" - - def __hash__(self) -> int: - return hash((self.value, self.base_type_name)) - - def __eq__(self, other: object) -> bool: - if isinstance(other, RawLiteralType): - return self.base_type_name == other.base_type_name and self.value == other.value - else: - return NotImplemented - - class LiteralType(Type): """The type of a Literal instance. Literal[Value] @@ -1872,9 +1898,6 @@ def item_str(name: str, typ: str) -> str: prefix = repr(t.fallback.type.fullname()) + ', ' return 'TypedDict({}{})'.format(prefix, s) - def visit_raw_literal_type(self, t: RawLiteralType) -> str: - return repr(t.value) - def visit_literal_type(self, t: LiteralType) -> str: return 'Literal[{}]'.format(t.value_repr()) diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 6ea396865035..076821fb9b19 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -269,8 +269,7 @@ tmp/m.py:14: error: Revealed type is 'builtins.int' from typing import NewType a = NewType('b', int) # E: String argument 1 'b' to NewType(...) does not match variable name 'a' -b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type \ - # E: Invalid type: try using Literal[3] instead? +b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type c = NewType(2, int) # E: Argument 1 to NewType(...) must be a string literal foo = "d" d = NewType(foo, int) # E: Argument 1 to NewType(...) must be a string literal diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 5ad34c274a47..b7b7ff9ea9a5 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -980,7 +980,7 @@ e = TypeVar('e', int, str, x=1) # E: Unexpected argument to TypeVar(): x f = TypeVar('f', (int, str), int) # E: Type expected g = TypeVar('g', int) # E: TypeVar cannot have only a single constraint h = TypeVar('h', x=(int, str)) # E: Unexpected argument to TypeVar(): x -i = TypeVar('i', bound=1) # E: Invalid type: try using Literal[1] instead? +i = TypeVar('i', bound=1) # E: TypeVar 'bound' must be a type. [out] [case testMoreInvalidTypevarArguments] From 8f3e5ad246e4ded3c470ab3ced02083e69f10474 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Sun, 30 Dec 2018 17:53:30 -0800 Subject: [PATCH 2/2] Fix typo --- test-data/unit/semanal-errors.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index b7b7ff9ea9a5..6b78901a0567 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -980,7 +980,7 @@ e = TypeVar('e', int, str, x=1) # E: Unexpected argument to TypeVar(): x f = TypeVar('f', (int, str), int) # E: Type expected g = TypeVar('g', int) # E: TypeVar cannot have only a single constraint h = TypeVar('h', x=(int, str)) # E: Unexpected argument to TypeVar(): x -i = TypeVar('i', bound=1) # E: TypeVar 'bound' must be a type. +i = TypeVar('i', bound=1) # E: TypeVar 'bound' must be a type [out] [case testMoreInvalidTypevarArguments]