From fd048ab1e2d8a58637803df809d2fea133945301 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Mon, 7 Jan 2019 10:07:03 -0800 Subject: [PATCH] Improve error messages related to literal types (#6149) This pull request improves how we handle error messages with Literal types in three different ways: 1. When the user tries constructing types like `Literal[3 + a]`, we now report an error message that says 'Invalid type: Literal[...] cannot contain arbitrary expressions'. 2. We no longer recommend using Literal[...] when doing `A = NewType('A', 4)` or `T = TypeVar('T', bound=4)`. This resolves https://github.com/python/mypy/issues/5989. (The former suggestion is a bad one: you can't create a NewType of a Literal[...] type. The latter suggestion is a valid but stupid one: `T = TypeVar('T', bound=Literal[4])` is basically the same thing as `T = Literal[4]`.) 3. When the user tries using complex numbers inside Literals (e.g. `Literal[3j]`), we now report an error message that says 'Parameter 1 of Literal[...] cannot be of type "complex"'. This is the same kind of error message we previously used to report when the user tried using floats inside of literals. In order to accomplish bullet point 1, moved the "invalid type comment or annotation" checks from the parsing layer to the semantic analysis layer. This lets us customize which error message we report depending on whether or not the invalid type appears in the context of a Literal[...] type. In order to accomplish this, I repurposed RawLiteralType so it can represent any arbitrary expression that does not convert directly into a type (and renamed 'RawLiteralType' to 'RawExpressionType' to better reflect this new usage). I also added an optional "note" field to that class: this lets the parsing layer attach some extra context that would be difficult to obtain up in the semantic analysis layer. In order to accomplish bullet point 2, I modified the type analyzer so that the caller can optionally suppress the error messages that would otherwise be generated when a RawExpressionType appears outside of a Literal context. Bullet point 3 only required a minor tweak to the parsing and error handling code. --- mypy/exprtotype.py | 27 +++++----- mypy/fastparse.py | 79 +++++++++++++++++------------ mypy/indirection.py | 4 +- mypy/plugin.py | 1 + mypy/semanal.py | 17 +++++-- mypy/semanal_newtype.py | 6 ++- mypy/semanal_shared.py | 1 + mypy/server/astmerge.py | 6 +-- mypy/type_visitor.py | 6 +-- mypy/typeanal.py | 63 +++++++++++++++-------- mypy/types.py | 64 +++++++++++++++++------ test-data/unit/check-fastparse.test | 75 +++++++++++++-------------- test-data/unit/check-literal.test | 55 +++++++++----------- test-data/unit/check-newtype.test | 3 +- test-data/unit/semanal-errors.test | 8 +-- 15 files changed, 247 insertions(+), 168 deletions(-) diff --git a/mypy/exprtotype.py b/mypy/exprtotype.py index 6528ed562508..18166875b8e5 100644 --- a/mypy/exprtotype.py +++ b/mypy/exprtotype.py @@ -2,13 +2,13 @@ from mypy.nodes import ( Expression, NameExpr, MemberExpr, IndexExpr, TupleExpr, IntExpr, FloatExpr, UnaryExpr, - ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, + ComplexExpr, ListExpr, StrExpr, BytesExpr, UnicodeExpr, EllipsisExpr, CallExpr, get_member_expr_fullname ) from mypy.fastparse import parse_type_string from mypy.types import ( Type, UnboundType, TypeList, EllipsisType, AnyType, Optional, CallableArgument, TypeOfAny, - RawLiteralType, + RawExpressionType, ) @@ -39,9 +39,9 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No if isinstance(expr, NameExpr): name = expr.name if name == 'True': - return RawLiteralType(True, 'builtins.bool', line=expr.line, column=expr.column) + return RawExpressionType(True, 'builtins.bool', line=expr.line, column=expr.column) elif name == 'False': - return RawLiteralType(False, 'builtins.bool', line=expr.line, column=expr.column) + return RawExpressionType(False, 'builtins.bool', line=expr.line, column=expr.column) else: return UnboundType(name, line=expr.line, column=expr.column) elif isinstance(expr, MemberExpr): @@ -122,17 +122,20 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No assume_str_is_unicode=True) elif isinstance(expr, UnaryExpr): typ = expr_to_unanalyzed_type(expr.expr) - if isinstance(typ, RawLiteralType) and isinstance(typ.value, int) and expr.op == '-': - typ.value *= -1 - return typ - else: - raise TypeTranslationError() + if isinstance(typ, RawExpressionType): + if isinstance(typ.literal_value, int) and expr.op == '-': + typ.literal_value *= -1 + return typ + raise TypeTranslationError() elif isinstance(expr, IntExpr): - return RawLiteralType(expr.value, 'builtins.int', line=expr.line, column=expr.column) + return RawExpressionType(expr.value, 'builtins.int', line=expr.line, column=expr.column) elif isinstance(expr, FloatExpr): - # Floats are not valid parameters for RawLiteralType, so we just + # Floats are not valid parameters for RawExpressionType , so we just # pass in 'None' for now. We'll report the appropriate error at a later stage. - return RawLiteralType(None, 'builtins.float', line=expr.line, column=expr.column) + return RawExpressionType(None, 'builtins.float', line=expr.line, column=expr.column) + elif isinstance(expr, ComplexExpr): + # Same thing as above with complex numbers. + return RawExpressionType(None, 'builtins.complex', line=expr.line, column=expr.column) elif isinstance(expr, EllipsisExpr): return EllipsisType(expr.line) else: diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 7c10e14fc1ac..15122012a0dc 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -31,7 +31,7 @@ ) from mypy.types import ( Type, CallableType, AnyType, UnboundType, TupleType, TypeList, EllipsisType, CallableArgument, - TypeOfAny, Instance, RawLiteralType, + TypeOfAny, Instance, RawExpressionType, ) from mypy import defaults from mypy import messages @@ -83,7 +83,6 @@ _dummy_fallback = Instance(MISSING_FALLBACK, [], -1) # type: Final TYPE_COMMENT_SYNTAX_ERROR = 'syntax error in type comment' # type: Final -TYPE_COMMENT_AST_ERROR = 'invalid type comment or annotation' # type: Final # Older versions of typing don't allow using overload outside stubs, @@ -184,11 +183,11 @@ def parse_type_string(expr_string: str, expr_fallback_name: str, node.original_str_fallback = expr_fallback_name return node else: - return RawLiteralType(expr_string, expr_fallback_name, line, column) + return RawExpressionType(expr_string, expr_fallback_name, line, column) except (SyntaxError, ValueError): # Note: the parser will raise a `ValueError` instead of a SyntaxError if # the string happens to contain things like \x00. - return RawLiteralType(expr_string, expr_fallback_name, line, column) + return RawExpressionType(expr_string, expr_fallback_name, line, column) def is_no_type_check_decorator(expr: ast3.expr) -> bool: @@ -1069,6 +1068,24 @@ def __init__(self, self.node_stack = [] # type: List[AST] self.assume_str_is_unicode = assume_str_is_unicode + def invalid_type(self, node: AST, note: Optional[str] = None) -> RawExpressionType: + """Constructs a type representing some expression that normally forms an invalid type. + For example, if we see a type hint that says "3 + 4", we would transform that + expression into a RawExpressionType. + + The semantic analysis layer will report an "Invalid type" error when it + encounters this type, along with the given note if one is provided. + + See RawExpressionType's docstring for more details on how it's used. + """ + return RawExpressionType( + None, + 'typing.Any', + line=self.line, + column=getattr(node, 'col_offset', -1), + note=note, + ) + @overload def visit(self, node: ast3.expr) -> Type: ... @@ -1086,8 +1103,7 @@ def visit(self, node: Optional[AST]) -> Optional[Type]: # noqa if visitor is not None: return visitor(node) else: - self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(node, 'col_offset', -1)) - return AnyType(TypeOfAny.from_error) + return self.invalid_type(node) finally: self.node_stack.pop() @@ -1124,12 +1140,10 @@ def visit_Call(self, e: Call) -> Type: constructor = stringify_name(f) if not isinstance(self.parent(), ast3.List): - self.fail(TYPE_COMMENT_AST_ERROR, self.line, e.col_offset) + note = None if constructor: - self.note("Suggestion: use {}[...] instead of {}(...)".format( - constructor, constructor), - self.line, e.col_offset) - return AnyType(TypeOfAny.from_error) + note = "Suggestion: use {0}[...] instead of {0}(...)".format(constructor) + return self.invalid_type(e, note=note) if not constructor: self.fail("Expected arg constructor name", e.lineno, e.col_offset) @@ -1183,7 +1197,7 @@ def visit_Name(self, n: Name) -> Type: def visit_NameConstant(self, n: NameConstant) -> Type: if isinstance(n.value, bool): - return RawLiteralType(n.value, 'builtins.bool', line=self.line) + return RawExpressionType(n.value, 'builtins.bool', line=self.line) else: return UnboundType(str(n.value), line=self.line) @@ -1192,26 +1206,29 @@ def visit_UnaryOp(self, n: UnaryOp) -> Type: # We support specifically Literal[-4] and nothing else. # For example, Literal[+4] or Literal[~6] is not supported. typ = self.visit(n.operand) - if isinstance(typ, RawLiteralType) and isinstance(n.op, USub): - if isinstance(typ.value, int): - typ.value *= -1 + if isinstance(typ, RawExpressionType) and isinstance(n.op, USub): + if isinstance(typ.literal_value, int): + typ.literal_value *= -1 return typ - self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) - return AnyType(TypeOfAny.from_error) + return self.invalid_type(n) # Num(number n) def visit_Num(self, n: Num) -> Type: - # Could be either float or int - numeric_value = n.n - if isinstance(numeric_value, int): - return RawLiteralType(numeric_value, 'builtins.int', line=self.line) - elif isinstance(numeric_value, float): - # Floats and other numbers are not valid parameters for RawLiteralType, so we just - # pass in 'None' for now. We'll report the appropriate error at a later stage. - return RawLiteralType(None, 'builtins.float', line=self.line) + if isinstance(n.n, int): + numeric_value = n.n + type_name = 'builtins.int' else: - self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) - return AnyType(TypeOfAny.from_error) + # Other kinds of numbers (floats, complex) are not valid parameters for + # RawExpressionType so we just pass in 'None' for now. We'll report the + # appropriate error at a later stage. + numeric_value = None + type_name = 'builtins.{}'.format(type(n.n).__name__) + return RawExpressionType( + numeric_value, + type_name, + line=self.line, + column=getattr(n, 'col_offset', -1), + ) # Str(string s) def visit_Str(self, n: Str) -> Type: @@ -1230,7 +1247,7 @@ def visit_Str(self, n: Str) -> Type: # Bytes(bytes s) def visit_Bytes(self, n: Bytes) -> Type: contents = bytes_to_human_readable_repr(n.s) - return RawLiteralType(contents, 'builtins.bytes', self.line, column=n.col_offset) + return RawExpressionType(contents, 'builtins.bytes', self.line, column=n.col_offset) # Subscript(expr value, slice slice, expr_context ctx) def visit_Subscript(self, n: ast3.Subscript) -> Type: @@ -1251,8 +1268,7 @@ def visit_Subscript(self, n: ast3.Subscript) -> Type: return UnboundType(value.name, params, line=self.line, empty_tuple_index=empty_tuple_index) else: - self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) - return AnyType(TypeOfAny.from_error) + return self.invalid_type(n) def visit_Tuple(self, n: ast3.Tuple) -> Type: return TupleType(self.translate_expr_list(n.elts), _dummy_fallback, @@ -1265,8 +1281,7 @@ def visit_Attribute(self, n: Attribute) -> Type: if isinstance(before_dot, UnboundType) and not before_dot.args: return UnboundType("{}.{}".format(before_dot.name, n.attr), line=self.line) else: - self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(n, 'col_offset', -1)) - return AnyType(TypeOfAny.from_error) + return self.invalid_type(n) # Ellipsis def visit_Ellipsis(self, n: ast3_Ellipsis) -> Type: diff --git a/mypy/indirection.py b/mypy/indirection.py index 4e3390a65e3c..66b6d44f7c46 100644 --- a/mypy/indirection.py +++ b/mypy/indirection.py @@ -90,8 +90,8 @@ def visit_tuple_type(self, t: types.TupleType) -> Set[str]: def visit_typeddict_type(self, t: types.TypedDictType) -> Set[str]: return self._visit(t.items.values()) | self._visit(t.fallback) - def visit_raw_literal_type(self, t: types.RawLiteralType) -> Set[str]: - assert False, "Unexpected RawLiteralType after semantic analysis phase" + def visit_raw_expression_type(self, t: types.RawExpressionType) -> Set[str]: + assert False, "Unexpected RawExpressionType after semantic analysis phase" def visit_literal_type(self, t: types.LiteralType) -> Set[str]: return self._visit(t.fallback) diff --git a/mypy/plugin.py b/mypy/plugin.py index 7e5ae4e86fae..40783ddcc5d3 100644 --- a/mypy/plugin.py +++ b/mypy/plugin.py @@ -180,6 +180,7 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: """Analyze an unbound type.""" raise NotImplementedError diff --git a/mypy/semanal.py b/mypy/semanal.py index 7011960639b8..b2537a551f1f 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -1283,7 +1283,7 @@ def update_metaclass(self, defn: ClassDef) -> None: return defn.metaclass = metas.pop() - def expr_to_analyzed_type(self, expr: Expression) -> Type: + def expr_to_analyzed_type(self, expr: Expression, report_invalid_types: bool = True) -> Type: if isinstance(expr, CallExpr): expr.accept(self) info = self.named_tuple_analyzer.check_namedtuple(expr, None, self.is_func_scope()) @@ -1295,7 +1295,7 @@ def expr_to_analyzed_type(self, expr: Expression) -> Type: fallback = Instance(info, []) return TupleType(info.tuple_type.items, fallback=fallback) typ = expr_to_unanalyzed_type(expr) - return self.anal_type(typ) + return self.anal_type(typ, report_invalid_types=report_invalid_types) def verify_base_classes(self, defn: ClassDef) -> bool: info = defn.info @@ -1686,6 +1686,7 @@ def type_analyzer(self, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> TypeAnalyser: if tvar_scope is None: tvar_scope = self.tvar_scope @@ -1696,6 +1697,7 @@ def type_analyzer(self, *, self.is_typeshed_stub_file, allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, allow_unnormalized=self.is_stub_file, third_pass=third_pass) tpan.in_dynamic_func = bool(self.function_stack and self.function_stack[-1].is_dynamic()) @@ -1706,10 +1708,12 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: a = self.type_analyzer(tvar_scope=tvar_scope, allow_unbound_tvars=allow_unbound_tvars, allow_tuple_literal=allow_tuple_literal, + report_invalid_types=report_invalid_types, third_pass=third_pass) typ = t.accept(a) self.add_type_alias_deps(a.aliases_used) @@ -2394,7 +2398,14 @@ def process_typevar_parameters(self, args: List[Expression], self.fail("TypeVar cannot have both values and an upper bound", context) return None try: - upper_bound = self.expr_to_analyzed_type(param_value) + # We want to use our custom error message below, so we suppress + # the default error message for invalid types here. + upper_bound = self.expr_to_analyzed_type(param_value, + report_invalid_types=False) + if isinstance(upper_bound, AnyType) and upper_bound.is_from_error: + self.fail("TypeVar 'bound' must be a type", param_value) + # Note: we do not return 'None' here -- we want to continue + # using the AnyType as the upper bound. except TypeTranslationError: self.fail("TypeVar 'bound' must be a type", param_value) return None diff --git a/mypy/semanal_newtype.py b/mypy/semanal_newtype.py index f4511e01bdee..034dea1f6403 100644 --- a/mypy/semanal_newtype.py +++ b/mypy/semanal_newtype.py @@ -114,11 +114,13 @@ def check_newtype_args(self, name: str, call: CallExpr, context: Context) -> Opt self.fail(msg, context) return None - old_type = self.api.anal_type(unanalyzed_type) + # We want to use our custom error message (see above), so we suppress + # the default error message for invalid types here. + old_type = self.api.anal_type(unanalyzed_type, report_invalid_types=False) # The caller of this function assumes that if we return a Type, it's always # a valid one. So, we translate AnyTypes created from errors into None. - if isinstance(old_type, AnyType) and old_type.type_of_any == TypeOfAny.from_error: + if isinstance(old_type, AnyType) and old_type.is_from_error: self.fail(msg, context) return None diff --git a/mypy/semanal_shared.py b/mypy/semanal_shared.py index c88c9ff98195..4d8994e2084e 100644 --- a/mypy/semanal_shared.py +++ b/mypy/semanal_shared.py @@ -91,6 +91,7 @@ def anal_type(self, t: Type, *, tvar_scope: Optional[TypeVarScope] = None, allow_tuple_literal: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> Type: raise NotImplementedError diff --git a/mypy/server/astmerge.py b/mypy/server/astmerge.py index 32dd7157ccc8..81319bedb480 100644 --- a/mypy/server/astmerge.py +++ b/mypy/server/astmerge.py @@ -59,7 +59,7 @@ Type, SyntheticTypeVisitor, Instance, AnyType, NoneTyp, CallableType, DeletedType, PartialType, TupleType, TypeType, TypeVarType, TypedDictType, UnboundType, UninhabitedType, UnionType, Overloaded, TypeVarDef, TypeList, CallableArgument, EllipsisType, StarType, LiteralType, - RawLiteralType, + RawExpressionType, ) from mypy.util import get_prefix, replace_object_state from mypy.typestate import TypeState @@ -331,7 +331,7 @@ class TypeReplaceVisitor(SyntheticTypeVisitor[None]): """Similar to NodeReplaceVisitor, but for type objects. Note: this visitor may sometimes visit unanalyzed types - such as 'UnboundType' and 'RawLiteralType' For example, see + such as 'UnboundType' and 'RawExpressionType' For example, see NodeReplaceVisitor.process_base_func. """ @@ -397,7 +397,7 @@ def visit_typeddict_type(self, typ: TypedDictType) -> None: value_type.accept(self) typ.fallback.accept(self) - def visit_raw_literal_type(self, t: RawLiteralType) -> None: + def visit_raw_expression_type(self, t: RawExpressionType) -> None: pass def visit_literal_type(self, typ: LiteralType) -> None: diff --git a/mypy/type_visitor.py b/mypy/type_visitor.py index 64ae75e174ee..79318f56baa2 100644 --- a/mypy/type_visitor.py +++ b/mypy/type_visitor.py @@ -20,7 +20,7 @@ from mypy.types import ( Type, AnyType, CallableType, Overloaded, TupleType, TypedDictType, LiteralType, - RawLiteralType, Instance, NoneTyp, TypeType, + RawExpressionType, Instance, NoneTyp, TypeType, UnionType, TypeVarType, PartialType, DeletedType, UninhabitedType, TypeVarDef, UnboundType, ErasedType, ForwardRef, StarType, EllipsisType, TypeList, CallableArgument, ) @@ -128,7 +128,7 @@ def visit_ellipsis_type(self, t: EllipsisType) -> T: pass @abstractmethod - def visit_raw_literal_type(self, t: RawLiteralType) -> T: + def visit_raw_expression_type(self, t: RawExpressionType) -> T: pass @@ -282,7 +282,7 @@ def visit_tuple_type(self, t: TupleType) -> T: def visit_typeddict_type(self, t: TypedDictType) -> T: return self.query_types(t.items.values()) - def visit_raw_literal_type(self, t: RawLiteralType) -> T: + def visit_raw_expression_type(self, t: RawExpressionType) -> T: return self.strategy([]) def visit_literal_type(self, t: LiteralType) -> T: diff --git a/mypy/typeanal.py b/mypy/typeanal.py index a37aa4c6c03b..5c8d6770d95a 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -16,9 +16,8 @@ CallableType, NoneTyp, DeletedType, TypeList, TypeVarDef, TypeVisitor, SyntheticTypeVisitor, StarType, PartialType, EllipsisType, UninhabitedType, TypeType, get_typ_args, set_typ_args, CallableArgument, get_type_vars, TypeQuery, union_items, TypeOfAny, ForwardRef, Overloaded, - LiteralType, RawLiteralType, + LiteralType, RawExpressionType, ) -from mypy.fastparse import TYPE_COMMENT_SYNTAX_ERROR from mypy.nodes import ( TVAR, MODULE_REF, UNBOUND_IMPORTED, TypeInfo, Context, SymbolTableNode, Var, Expression, @@ -158,6 +157,7 @@ def __init__(self, allow_tuple_literal: bool = False, allow_unnormalized: bool = False, allow_unbound_tvars: bool = False, + report_invalid_types: bool = True, third_pass: bool = False) -> None: self.api = api self.lookup = api.lookup_qualified @@ -175,6 +175,11 @@ def __init__(self, self.allow_unnormalized = allow_unnormalized # Should we accept unbound type variables (always OK in aliases)? self.allow_unbound_tvars = allow_unbound_tvars or defining_alias + # Should we report an error whenever we encounter a RawExpressionType outside + # of a Literal context: e.g. whenever we encounter an invalid type? Normally, + # we want to report an error, but the caller may want to do more specialized + # error handling. + self.report_invalid_types = report_invalid_types self.plugin = plugin self.options = options self.is_typeshed_stub = is_typeshed_stub @@ -489,7 +494,7 @@ def visit_typeddict_type(self, t: TypedDictType) -> Type: ]) return TypedDictType(items, set(t.required_keys), t.fallback) - def visit_raw_literal_type(self, t: RawLiteralType) -> Type: + def visit_raw_expression_type(self, t: RawExpressionType) -> Type: # We should never see a bare Literal. We synthesize these raw literals # in the earlier stages of semantic analysis, but those # "fake literals" should always be wrapped in an UnboundType @@ -499,19 +504,28 @@ def visit_raw_literal_type(self, t: RawLiteralType) -> Type: # make signatures like "foo(x: 20) -> None" legal, we can change # this method so it generates and returns an actual LiteralType # instead. - if t.base_type_name == 'builtins.int' or t.base_type_name == 'builtins.bool': - # The only time it makes sense to use an int or bool is inside of - # a literal type. - self.fail("Invalid type: try using Literal[{}] instead?".format(repr(t.value)), t) - elif t.base_type_name == 'builtins.float': - self.fail("Invalid type: float literals cannot be used as a type", t) - else: - # For other types like strings, it's unclear if the user meant - # to construct a literal type or just misspelled a regular type. - # So, we leave just a generic "syntax error" error. - self.fail('Invalid type: ' + TYPE_COMMENT_SYNTAX_ERROR, t) - return AnyType(TypeOfAny.from_error) + if self.report_invalid_types: + if t.base_type_name in ('builtins.int', 'builtins.bool'): + # The only time it makes sense to use an int or bool is inside of + # a literal type. + msg = "Invalid type: try using Literal[{}] instead?".format(repr(t.literal_value)) + elif t.base_type_name in ('builtins.float', 'builtins.complex'): + # We special-case warnings for floats and complex numbers. + msg = "Invalid type: {} literals cannot be used as a type".format(t.simple_name()) + else: + # And in all other cases, we default to a generic error message. + # Note: the reason why we use a generic error message for strings + # but not ints or bools is because whenever we see an out-of-place + # string, it's unclear if the user meant to construct a literal type + # or just misspelled a regular type. So we avoid guessing. + msg = 'Invalid type comment or annotation' + + self.fail(msg, t) + if t.note is not None: + self.note_func(t.note, t) + + return AnyType(TypeOfAny.from_error, line=t.line, column=t.column) def visit_literal_type(self, t: LiteralType) -> Type: return t @@ -663,18 +677,23 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L if arg.type_of_any != TypeOfAny.from_error: self.fail('Parameter {} of Literal[...] cannot be of type "Any"'.format(idx), ctx) return None - elif isinstance(arg, RawLiteralType): - # A raw literal. Convert it directly into a literal. - if arg.base_type_name == 'builtins.float': - self.fail( - 'Parameter {} of Literal[...] cannot be of type "float"'.format(idx), - ctx) + elif isinstance(arg, RawExpressionType): + # A raw literal. Convert it directly into a literal if we can. + if arg.literal_value is None: + name = arg.simple_name() + if name in ('float', 'complex'): + msg = 'Parameter {} of Literal[...] cannot be of type "{}"'.format(idx, name) + else: + msg = 'Invalid type: Literal[...] cannot contain arbitrary expressions' + self.fail(msg, ctx) + # Note: we deliberately ignore arg.note here: the extra info might normally be + # helpful, but it generally won't make sense in the context of a Literal[...]. return None # Remap bytes and unicode into the appropriate type for the correct Python version fallback = self.named_type_with_normalized_str(arg.base_type_name) assert isinstance(fallback, Instance) - return [LiteralType(arg.value, fallback, line=arg.line, column=arg.column)] + return [LiteralType(arg.literal_value, fallback, line=arg.line, column=arg.column)] elif isinstance(arg, (NoneTyp, LiteralType)): # Types that we can just add directly to the literal/potential union of literals. return [arg] diff --git a/mypy/types.py b/mypy/types.py index d21eabdd77f5..776fee220b7a 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -50,7 +50,11 @@ # 3. server.astdiff.SnapshotTypeVisitor's visit_literal_type_method: this # method assumes that the following types supports equality checks and # hashability. -LiteralValue = Union[int, str, bool, None] +# +# Note: Although "Literal[None]" is a valid type, we internally always convert +# such a type directly into "None". So, "None" is not a valid parameter of +# LiteralType and is omitted from this list. +LiteralValue = Union[int, str, bool] # If we only import type_visitor in the middle of the file, mypy @@ -398,6 +402,10 @@ def __init__(self, # We should not have chains of Anys. assert not self.source_any or self.source_any.type_of_any != TypeOfAny.from_another_any + @property + def is_from_error(self) -> bool: + return self.type_of_any == TypeOfAny.from_error + def accept(self, visitor: 'TypeVisitor[T]') -> T: return visitor.visit_any(self) @@ -1299,20 +1307,20 @@ def zipall(self, right: 'TypedDictType') \ yield (item_name, None, right_item_type) -class RawLiteralType(Type): - """A synthetic type representing any type that could plausibly be something - that lives inside of a literal. +class RawExpressionType(Type): + """A synthetic type representing some arbitrary expression that does not cleanly + translate into a type. This synthetic type is only used at the beginning stages of semantic analysis and should be completely removing during the process for mapping UnboundTypes to - actual types. + actual types: we either turn it into a LiteralType or an AnyType. - For example, `Foo[1]` is initially represented as the following: + For example, suppose `Foo[1]` is initially represented as the following: UnboundType( name='Foo', args=[ - RawLiteralType(value=1, base_type_name='builtins.int'), + RawExpressionType(value=1, base_type_name='builtins.int'), ], ) @@ -1327,26 +1335,50 @@ class RawLiteralType(Type): produce something like this: Instance(type=typeinfo_for_foo, args=[AnyType(TypeOfAny.from_error)) + + If the "note" field is not None, the provided note will be reported alongside the + error at this point. + + Note: if "literal_value" is None, that means this object is representing some + expression that cannot possibly be a parameter of Literal[...]. For example, + "Foo[3j]" would be represented as: + + UnboundType( + name='Foo', + args=[ + RawExpressionType(value=None, base_type_name='builtins.complex'), + ], + ) """ - def __init__(self, value: LiteralValue, base_type_name: str, - line: int = -1, column: int = -1) -> None: + def __init__(self, + literal_value: Optional[LiteralValue], + base_type_name: str, + line: int = -1, + column: int = -1, + note: Optional[str] = None, + ) -> None: super().__init__(line, column) - self.value = value + self.literal_value = literal_value self.base_type_name = base_type_name + self.note = note + + def simple_name(self) -> str: + return self.base_type_name.replace("builtins.", "") def accept(self, visitor: 'TypeVisitor[T]') -> T: assert isinstance(visitor, SyntheticTypeVisitor) - return visitor.visit_raw_literal_type(self) + return visitor.visit_raw_expression_type(self) def serialize(self) -> JsonDict: assert False, "Synthetic types don't serialize" def __hash__(self) -> int: - return hash((self.value, self.base_type_name)) + return hash((self.literal_value, self.base_type_name)) def __eq__(self, other: object) -> bool: - if isinstance(other, RawLiteralType): - return self.base_type_name == other.base_type_name and self.value == other.value + if isinstance(other, RawExpressionType): + return (self.base_type_name == other.base_type_name + and self.literal_value == other.literal_value) else: return NotImplemented @@ -1872,8 +1904,8 @@ def item_str(name: str, typ: str) -> str: prefix = repr(t.fallback.type.fullname()) + ', ' return 'TypedDict({}{})'.format(prefix, s) - def visit_raw_literal_type(self, t: RawLiteralType) -> str: - return repr(t.value) + def visit_raw_expression_type(self, t: RawExpressionType) -> str: + return repr(t.literal_value) def visit_literal_type(self, t: LiteralType) -> str: return 'Literal[{}]'.format(t.value_repr()) diff --git a/test-data/unit/check-fastparse.test b/test-data/unit/check-fastparse.test index 50a8c0c2263b..997377ea54b5 100644 --- a/test-data/unit/check-fastparse.test +++ b/test-data/unit/check-fastparse.test @@ -8,7 +8,7 @@ x = None # type: a : b # E: syntax error in type comment [case testFastParseInvalidTypeComment] -x = None # type: a + b # E: invalid type comment or annotation +x = None # type: a + b # E: Invalid type comment or annotation -- Function type comments are attributed to the function def line. -- This happens in both parsers. @@ -26,7 +26,7 @@ def f(): # E: syntax error in type comment # N: Suggestion: wrap argument types [case testFastParseInvalidFunctionAnnotation] -def f(x): # E: invalid type comment or annotation +def f(x): # E: Invalid type comment or annotation # type: (a + b) -> None pass @@ -35,29 +35,29 @@ def f(x): # E: invalid type comment or annotation # All of these should not crash from typing import Callable, Tuple, Iterable -x = None # type: Tuple[int, str].x # E: invalid type comment or annotation -x = None # type: Iterable[x].x # E: invalid type comment or annotation -x = None # type: Tuple[x][x] # E: invalid type comment or annotation -x = None # type: Iterable[x][x] # E: invalid type comment or annotation -x = None # type: Callable[..., int][x] # E: invalid type comment or annotation -x = None # type: Callable[..., int].x # E: invalid type comment or annotation +x = None # type: Tuple[int, str].x # E: Invalid type comment or annotation +a = None # type: Iterable[x].x # E: Invalid type comment or annotation +b = None # type: Tuple[x][x] # E: Invalid type comment or annotation +c = None # type: Iterable[x][x] # E: Invalid type comment or annotation +d = None # type: Callable[..., int][x] # E: Invalid type comment or annotation +e = None # type: Callable[..., int].x # E: Invalid type comment or annotation -def f1(x): # E: invalid type comment or annotation +def f1(x): # E: Invalid type comment or annotation # type: (Tuple[int, str].x) -> None pass -def f2(x): # E: invalid type comment or annotation +def f2(x): # E: Invalid type comment or annotation # type: (Iterable[x].x) -> None pass -def f3(x): # E: invalid type comment or annotation +def f3(x): # E: Invalid type comment or annotation # type: (Tuple[x][x]) -> None pass -def f4(x): # E: invalid type comment or annotation +def f4(x): # E: Invalid type comment or annotation # type: (Iterable[x][x]) -> None pass -def f5(x): # E: invalid type comment or annotation +def f5(x): # E: Invalid type comment or annotation # type: (Callable[..., int][x]) -> None pass -def f6(x): # E: invalid type comment or annotation +def f6(x): # E: Invalid type comment or annotation # type: (Callable[..., int].x) -> None pass @@ -67,26 +67,26 @@ def f6(x): # E: invalid type comment or annotation # All of these should not crash from typing import Callable, Tuple, Iterable -x: Tuple[int, str].x # E: invalid type comment or annotation -x: Iterable[x].x # E: invalid type comment or annotation -x: Tuple[x][x] # E: invalid type comment or annotation -x: Iterable[x][x] # E: invalid type comment or annotation -x: Callable[..., int][x] # E: invalid type comment or annotation -x: Callable[..., int].x # E: invalid type comment or annotation - -x = None # type: Tuple[int, str].x # E: invalid type comment or annotation -x = None # type: Iterable[x].x # E: invalid type comment or annotation -x = None # type: Tuple[x][x] # E: invalid type comment or annotation -x = None # type: Iterable[x][x] # E: invalid type comment or annotation -x = None # type: Callable[..., int][x] # E: invalid type comment or annotation -x = None # type: Callable[..., int].x # E: invalid type comment or annotation - -def f1(x: Tuple[int, str].x) -> None: pass # E: invalid type comment or annotation -def f2(x: Iterable[x].x) -> None: pass # E: invalid type comment or annotation -def f3(x: Tuple[x][x]) -> None: pass # E: invalid type comment or annotation -def f4(x: Iterable[x][x]) -> None: pass # E: invalid type comment or annotation -def f5(x: Callable[..., int][x]) -> None: pass # E: invalid type comment or annotation -def f6(x: Callable[..., int].x) -> None: pass # E: invalid type comment or annotation +x: Tuple[int, str].x # E: Invalid type comment or annotation +a: Iterable[x].x # E: Invalid type comment or annotation +b: Tuple[x][x] # E: Invalid type comment or annotation +c: Iterable[x][x] # E: Invalid type comment or annotation +d: Callable[..., int][x] # E: Invalid type comment or annotation +e: Callable[..., int].x # E: Invalid type comment or annotation + +f = None # type: Tuple[int, str].x # E: Invalid type comment or annotation +g = None # type: Iterable[x].x # E: Invalid type comment or annotation +h = None # type: Tuple[x][x] # E: Invalid type comment or annotation +i = None # type: Iterable[x][x] # E: Invalid type comment or annotation +j = None # type: Callable[..., int][x] # E: Invalid type comment or annotation +k = None # type: Callable[..., int].x # E: Invalid type comment or annotation + +def f1(x: Tuple[int, str].x) -> None: pass # E: Invalid type comment or annotation +def f2(x: Iterable[x].x) -> None: pass # E: Invalid type comment or annotation +def f3(x: Tuple[x][x]) -> None: pass # E: Invalid type comment or annotation +def f4(x: Iterable[x][x]) -> None: pass # E: Invalid type comment or annotation +def f5(x: Callable[..., int][x]) -> None: pass # E: Invalid type comment or annotation +def f6(x: Callable[..., int].x) -> None: pass # E: Invalid type comment or annotation [case testFastParseProperty] @@ -233,7 +233,7 @@ def f(a): # type: (Tuple(int, int)) -> int pass [out] -main:3: error: invalid type comment or annotation +main:3: error: Invalid type comment or annotation main:3: note: Suggestion: use Tuple[...] instead of Tuple(...) [case testFasterParseTypeErrorList_python2] @@ -242,8 +242,9 @@ from typing import List def f(a): # type: (List(int)) -> int pass +[builtins_py2 fixtures/floatdict_python2.pyi] [out] -main:3: error: invalid type comment or annotation +main:3: error: Invalid type comment or annotation main:3: note: Suggestion: use List[...] instead of List(...) [case testFasterParseTypeErrorCustom] @@ -256,7 +257,7 @@ class Foo(Generic[T]): def f(a: Foo(int)) -> int: pass [out] -main:7: error: invalid type comment or annotation +main:7: error: Invalid type comment or annotation main:7: note: Suggestion: use Foo[...] instead of Foo(...) [case testFastParseMatMul] diff --git a/test-data/unit/check-literal.test b/test-data/unit/check-literal.test index c7b7869f2bb3..0168c691e514 100644 --- a/test-data/unit/check-literal.test +++ b/test-data/unit/check-literal.test @@ -5,12 +5,12 @@ [case testLiteralInvalidString] from typing_extensions import Literal -def f1(x: 'A[') -> None: pass # E: Invalid type: syntax error in type comment +def f1(x: 'A[') -> None: pass # E: Invalid type comment or annotation def g1(x: Literal['A[']) -> None: pass reveal_type(f1) # E: Revealed type is 'def (x: Any)' reveal_type(g1) # E: Revealed type is 'def (x: Literal['A['])' -def f2(x: 'A B') -> None: pass # E: Invalid type: syntax error in type comment +def f2(x: 'A B') -> None: pass # E: Invalid type comment or annotation def g2(x: Literal['A B']) -> None: pass reveal_type(f2) # E: Revealed type is 'def (x: Any)' reveal_type(g2) # E: Revealed type is 'def (x: Literal['A B'])' @@ -24,7 +24,7 @@ def f(x): # E: syntax error in type comment [case testLiteralInvalidTypeComment2] from typing_extensions import Literal -def f(x): # E: Invalid type: syntax error in type comment +def f(x): # E: Invalid type comment or annotation # type: ("A[") -> None pass @@ -41,7 +41,7 @@ reveal_type(g) # E: Revealed type is 'def (x: Literal['A['])' from typing import Optional from typing_extensions import Literal -def f(x): # E: Invalid type: syntax error in type comment +def f(x): # E: Invalid type comment or annotation # type: ("A[") -> None pass @@ -402,9 +402,9 @@ b_str_wrapper: "Literal['foo']" c_str_wrapper: "Literal[b'foo']" # In Python 3, forward references MUST be str, not bytes -a_bytes_wrapper: b"Literal[u'foo']" # E: Invalid type: syntax error in type comment -b_bytes_wrapper: b"Literal['foo']" # E: Invalid type: syntax error in type comment -c_bytes_wrapper: b"Literal[b'foo']" # E: Invalid type: syntax error in type comment +a_bytes_wrapper: b"Literal[u'foo']" # E: Invalid type comment or annotation +b_bytes_wrapper: b"Literal['foo']" # E: Invalid type comment or annotation +c_bytes_wrapper: b"Literal[b'foo']" # E: Invalid type comment or annotation reveal_type(a_unicode_wrapper) # E: Revealed type is 'Literal['foo']' reveal_type(b_unicode_wrapper) # E: Revealed type is 'Literal['foo']' @@ -855,48 +855,43 @@ reveal_type(d) # E: Revealed type is 'Any' [builtins fixtures/primitives.pyi] [out] -[case testLiteralDisallowFloats] +[case testLiteralDisallowFloatsAndComplex] from typing_extensions import Literal a1: Literal[3.14] # E: Parameter 1 of Literal[...] cannot be of type "float" b1: 3.14 # E: Invalid type: float literals cannot be used as a type +c1: Literal[3j] # E: Parameter 1 of Literal[...] cannot be of type "complex" +d1: 3j # E: Invalid type: complex literals cannot be used as a type a2t = Literal[3.14] # E: Parameter 1 of Literal[...] cannot be of type "float" b2t = 3.14 +c2t = Literal[3j] # E: Parameter 1 of Literal[...] cannot be of type "complex" +d2t = 3j a2: a2t reveal_type(a2) # E: Revealed type is 'Any' b2: b2t # E: Invalid type "__main__.b2t" - -[out] - -[case testLiteralDisallowComplexNumbers] -from typing_extensions import Literal -a: Literal[3j] # E: invalid type comment or annotation -b: Literal[3j + 2] # E: invalid type comment or annotation -c: 3j # E: invalid type comment or annotation -d: 3j + 2 # E: invalid type comment or annotation - -[case testLiteralDisallowComplexNumbersTypeAlias] -from typing_extensions import Literal -at = Literal[3j] # E: Invalid type alias -a: at # E: Invalid type "__main__.at" +c2: c2t +reveal_type(c2) # E: Revealed type is 'Any' +d2: d2t # E: Invalid type "__main__.d2t" [builtins fixtures/complex.pyi] [out] [case testLiteralDisallowComplexExpressions] from typing_extensions import Literal -a: Literal[3 + 4] # E: invalid type comment or annotation -b: Literal[" foo ".trim()] # E: invalid type comment or annotation -c: Literal[+42] # E: invalid type comment or annotation -d: Literal[~12] # E: invalid type comment or annotation +def dummy() -> int: return 3 +a: Literal[3 + 4] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +b: Literal[" foo ".trim()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +c: Literal[+42] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +d: Literal[~12] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +e: Literal[dummy()] # E: Invalid type: Literal[...] cannot contain arbitrary expressions [out] [case testLiteralDisallowCollections] from typing_extensions import Literal -a: Literal[{"a": 1, "b": 2}] # E: invalid type comment or annotation -b: literal[{1, 2, 3}] # E: invalid type comment or annotation -c: {"a": 1, "b": 2} # E: invalid type comment or annotation -d: {1, 2, 3} # E: invalid type comment or annotation +a: Literal[{"a": 1, "b": 2}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +b: Literal[{1, 2, 3}] # E: Invalid type: Literal[...] cannot contain arbitrary expressions +c: {"a": 1, "b": 2} # E: Invalid type comment or annotation +d: {1, 2, 3} # E: Invalid type comment or annotation [case testLiteralDisallowCollections2] from typing_extensions import Literal diff --git a/test-data/unit/check-newtype.test b/test-data/unit/check-newtype.test index 6ea396865035..076821fb9b19 100644 --- a/test-data/unit/check-newtype.test +++ b/test-data/unit/check-newtype.test @@ -269,8 +269,7 @@ tmp/m.py:14: error: Revealed type is 'builtins.int' from typing import NewType a = NewType('b', int) # E: String argument 1 'b' to NewType(...) does not match variable name 'a' -b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type \ - # E: Invalid type: try using Literal[3] instead? +b = NewType('b', 3) # E: Argument 2 to NewType(...) must be a valid type c = NewType(2, int) # E: Argument 1 to NewType(...) must be a string literal foo = "d" d = NewType(foo, int) # E: Argument 1 to NewType(...) must be a string literal diff --git a/test-data/unit/semanal-errors.test b/test-data/unit/semanal-errors.test index 5ad34c274a47..e32b64db3ddb 100644 --- a/test-data/unit/semanal-errors.test +++ b/test-data/unit/semanal-errors.test @@ -980,7 +980,7 @@ e = TypeVar('e', int, str, x=1) # E: Unexpected argument to TypeVar(): x f = TypeVar('f', (int, str), int) # E: Type expected g = TypeVar('g', int) # E: TypeVar cannot have only a single constraint h = TypeVar('h', x=(int, str)) # E: Unexpected argument to TypeVar(): x -i = TypeVar('i', bound=1) # E: Invalid type: try using Literal[1] instead? +i = TypeVar('i', bound=1) # E: TypeVar 'bound' must be a type [out] [case testMoreInvalidTypevarArguments] @@ -1057,15 +1057,15 @@ def f(x: 'foo'): pass # E: Name 'foo' is not defined [out] [case testInvalidStrLiteralStrayBrace] -def f(x: 'int['): pass # E: Invalid type: syntax error in type comment +def f(x: 'int['): pass # E: Invalid type comment or annotation [out] [case testInvalidStrLiteralSpaces] -def f(x: 'A B'): pass # E: Invalid type: syntax error in type comment +def f(x: 'A B'): pass # E: Invalid type comment or annotation [out] [case testInvalidMultilineLiteralType] -def f() -> "A\nB": pass # E: Invalid type: syntax error in type comment +def f() -> "A\nB": pass # E: Invalid type comment or annotation [out] [case testInconsistentOverload]