diff --git a/mypy/fastparse.py b/mypy/fastparse.py index 37f2661aa188..0bfb0b4265c2 100644 --- a/mypy/fastparse.py +++ b/mypy/fastparse.py @@ -1821,15 +1821,6 @@ def note(self, msg: str, line: int, column: int) -> None: def translate_expr_list(self, l: Sequence[ast3.expr]) -> List[Type]: return [self.visit(e) for e in l] - def visit_raw_str(self, s: str) -> Type: - # An escape hatch that allows the AST walker in fastparse2 to - # directly hook into the Python 3 type converter in some cases - # without needing to create an intermediary `Str` object. - _, typ = parse_type_comment( - s.strip(), self.line, -1, self.errors, self.assume_str_is_unicode - ) - return typ or AnyType(TypeOfAny.from_error) - def visit_Call(self, e: Call) -> Type: # Parse the arg constructor f = e.func diff --git a/mypy/fastparse2.py b/mypy/fastparse2.py deleted file mode 100644 index 7fd418252cd0..000000000000 --- a/mypy/fastparse2.py +++ /dev/null @@ -1,1172 +0,0 @@ -""" -This file is nearly identical to `fastparse.py`, except that it works with a Python 2 -AST instead of a Python 3 AST. - -Previously, how we handled Python 2 code was by first obtaining the Python 2 AST via -typed_ast, converting it into a Python 3 AST by using typed_ast.conversion, then -running it through mypy.fastparse. - -While this worked, it did add some overhead, especially in larger Python 2 codebases. -This module allows us to skip the conversion step, saving us some time. - -The reason why this file is not easily merged with mypy.fastparse despite the large amount -of redundancy is because the Python 2 AST and the Python 3 AST nodes belong to two completely -different class hierarchies, which made it difficult to write a shared visitor between the -two in a typesafe way. -""" -import sys -import typing # for typing.Type, which conflicts with types.Type -import warnings -from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union, cast - -from typing_extensions import Final, Literal - -from mypy import errorcodes as codes, message_registry -from mypy.errors import Errors -from mypy.fastparse import ( - INVALID_TYPE_IGNORE, - TYPE_IGNORE_PATTERN, - TypeConverter, - parse_type_comment, - parse_type_ignore_tag, -) -from mypy.nodes import ( - ARG_NAMED, - ARG_OPT, - ARG_POS, - ARG_STAR, - ARG_STAR2, - ArgKind, - Argument, - AssertStmt, - AssignmentStmt, - BackquoteExpr, - Block, - BreakStmt, - CallExpr, - ClassDef, - ComparisonExpr, - ComplexExpr, - ConditionalExpr, - ContinueStmt, - Decorator, - DelStmt, - DictExpr, - DictionaryComprehension, - EllipsisExpr, - ExecStmt, - Expression, - ExpressionStmt, - FakeInfo, - FloatExpr, - ForStmt, - FuncDef, - GeneratorExpr, - GlobalDecl, - IfStmt, - Import, - ImportAll, - ImportBase, - ImportFrom, - IndexExpr, - IntExpr, - LambdaExpr, - ListComprehension, - ListExpr, - MemberExpr, - MypyFile, - NameExpr, - Node, - OperatorAssignmentStmt, - OpExpr, - OverloadedFuncDef, - OverloadPart, - PassStmt, - PrintStmt, - RaiseStmt, - ReturnStmt, - SetComprehension, - SetExpr, - SliceExpr, - Statement, - StrExpr, - SuperExpr, - TryStmt, - TupleExpr, - UnaryExpr, - UnicodeExpr, - Var, - WhileStmt, - WithStmt, - YieldExpr, - check_arg_names, -) -from mypy.options import Options -from mypy.reachability import mark_block_unreachable -from mypy.sharedparse import argument_elide_name, special_function_elide_names -from mypy.types import ( - AnyType, - CallableType, - EllipsisType, - Instance, - ProperType, - Type, - TypeOfAny, - UnboundType, -) -from mypy.util import bytes_to_human_readable_repr, unnamed_function - -try: - from typed_ast import ast27 - from typed_ast.ast27 import AST, Attribute, Call, Name, Tuple as ast27_Tuple - - # Import ast3 from fastparse, which has special case for Python 3.8 - from mypy.fastparse import ast3, ast3_parse -except ImportError: - try: - from typed_ast import ast35 # type: ignore[attr-defined] # noqa: F401 - except ImportError: - print( - "The typed_ast package is not installed.\n" - 'For Python 2 support, install mypy using `python3 -m pip install "mypy[python2]"`' - "Alternatively, you can install typed_ast with `python3 -m pip install typed-ast`.", - file=sys.stderr, - ) - else: - print( - "You need a more recent version of the typed_ast package.\n" - "You can update to the latest version with " - "`python3 -m pip install -U typed-ast`.", - file=sys.stderr, - ) - sys.exit(1) - -N = TypeVar("N", bound=Node) - -# There is no way to create reasonable fallbacks at this stage, -# they must be patched later. -MISSING_FALLBACK: Final = FakeInfo("fallback can't be filled out until semanal") -_dummy_fallback: Final = Instance(MISSING_FALLBACK, [], -1) - -TYPE_COMMENT_SYNTAX_ERROR: Final = "syntax error in type comment" -TYPE_COMMENT_AST_ERROR: Final = "invalid type comment" - - -def parse( - source: Union[str, bytes], - fnam: str, - module: Optional[str], - errors: Optional[Errors] = None, - options: Optional[Options] = None, -) -> MypyFile: - """Parse a source file, without doing any semantic analysis. - - Return the parse tree. If errors is not provided, raise ParseError - on failure. Otherwise, use the errors object to report parse errors. - """ - raise_on_error = False - if errors is None: - errors = Errors() - raise_on_error = True - if options is None: - options = Options() - errors.set_file(fnam, module) - is_stub_file = fnam.endswith(".pyi") - try: - assert options.python_version[0] < 3 and not is_stub_file - # Disable deprecation warnings about <>. - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - ast = ast27.parse(source, fnam, "exec") - tree = ASTConverter(options=options, errors=errors).visit(ast) - assert isinstance(tree, MypyFile) - tree.path = fnam - tree.is_stub = is_stub_file - except SyntaxError as e: - errors.report( - e.lineno if e.lineno is not None else -1, - e.offset, - e.msg, - blocker=True, - code=codes.SYNTAX, - ) - tree = MypyFile([], [], False, {}) - - if raise_on_error and errors.is_errors(): - errors.raise_error() - - return tree - - -def is_no_type_check_decorator(expr: ast27.expr) -> bool: - if isinstance(expr, Name): - return expr.id == "no_type_check" - elif isinstance(expr, Attribute): - if isinstance(expr.value, Name): - return expr.value.id == "typing" and expr.attr == "no_type_check" - return False - - -class ASTConverter: - def __init__(self, options: Options, errors: Errors) -> None: - # 'C' for class, 'F' for function - self.class_and_function_stack: List[Literal["C", "F"]] = [] - self.imports: List[ImportBase] = [] - - self.options = options - self.errors = errors - - # Indicates whether this file is being parsed with unicode_literals enabled. - # Note: typed_ast already naturally takes unicode_literals into account when - # parsing so we don't have to worry when analyzing strings within this class. - # - # The only place where we use this field is when we call fastparse's TypeConverter - # and any related methods. That class accepts a Python 3 AST instead of a Python 2 - # AST: as a result, it don't special-case the `unicode_literals` import and won't know - # exactly whether to parse some string as bytes or unicode. - # - # This distinction is relevant mostly when handling Literal types -- Literal[u"foo"] - # is not the same type as Literal[b"foo"], and Literal["foo"] could mean either the - # former or the latter based on context. - # - # This field is set in the 'visit_ImportFrom' method: it's ok to delay computing it - # because any `from __future__ import blah` import must be located at the top of the - # file, with the exception of the docstring. This means we're guaranteed to correctly - # set this field before we encounter any type hints. - self.unicode_literals = False - - # Cache of visit_X methods keyed by type of visited object - self.visitor_cache: Dict[type, Callable[[Optional[AST]], Any]] = {} - - self.type_ignores: Dict[int, List[str]] = {} - - def fail(self, msg: str, line: int, column: int, blocker: bool = True) -> None: - if blocker or not self.options.ignore_errors: - self.errors.report(line, column, msg, blocker=blocker, code=codes.SYNTAX) - - def visit(self, node: Optional[AST]) -> Any: # same as in typed_ast stub - if node is None: - return None - typeobj = type(node) - visitor = self.visitor_cache.get(typeobj) - if visitor is None: - method = "visit_" + node.__class__.__name__ - visitor = getattr(self, method) - self.visitor_cache[typeobj] = visitor - return visitor(node) - - def set_line(self, node: N, n: Union[ast27.expr, ast27.stmt, ast27.ExceptHandler]) -> N: - node.line = n.lineno - node.column = n.col_offset - return node - - def translate_expr_list(self, l: Sequence[AST]) -> List[Expression]: - res: List[Expression] = [] - for e in l: - exp = self.visit(e) - assert isinstance(exp, Expression) - res.append(exp) - return res - - def get_lineno(self, node: Union[ast27.expr, ast27.stmt]) -> int: - if isinstance(node, (ast27.ClassDef, ast27.FunctionDef)) and node.decorator_list: - return node.decorator_list[0].lineno - return node.lineno - - def translate_stmt_list( - self, stmts: Sequence[ast27.stmt], module: bool = False - ) -> List[Statement]: - # A "# type: ignore" comment before the first statement of a module - # ignores the whole module: - if ( - module - and stmts - and self.type_ignores - and min(self.type_ignores) < self.get_lineno(stmts[0]) - ): - self.errors.used_ignored_lines[self.errors.file][min(self.type_ignores)].append( - codes.FILE.code - ) - block = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) - mark_block_unreachable(block) - return [block] - - res: List[Statement] = [] - for stmt in stmts: - node = self.visit(stmt) - assert isinstance(node, Statement) - res.append(node) - return res - - def translate_type_comment( - self, n: ast27.stmt, type_comment: Optional[str] - ) -> Optional[ProperType]: - if type_comment is None: - return None - else: - lineno = n.lineno - extra_ignore, typ = parse_type_comment( - type_comment, - lineno, - n.col_offset, - self.errors, - assume_str_is_unicode=self.unicode_literals, - ) - if extra_ignore is not None: - self.type_ignores[lineno] = extra_ignore - return typ - - op_map: Final[Dict[typing.Type[AST], str]] = { - ast27.Add: "+", - ast27.Sub: "-", - ast27.Mult: "*", - ast27.Div: "/", - ast27.Mod: "%", - ast27.Pow: "**", - ast27.LShift: "<<", - ast27.RShift: ">>", - ast27.BitOr: "|", - ast27.BitXor: "^", - ast27.BitAnd: "&", - ast27.FloorDiv: "//", - } - - def from_operator(self, op: ast27.operator) -> str: - op_name = ASTConverter.op_map.get(type(op)) - if op_name is None: - raise RuntimeError("Unknown operator " + str(type(op))) - elif op_name == "@": - raise RuntimeError("mypy does not support the MatMult operator") - else: - return op_name - - comp_op_map: Final[Dict[typing.Type[AST], str]] = { - ast27.Gt: ">", - ast27.Lt: "<", - ast27.Eq: "==", - ast27.GtE: ">=", - ast27.LtE: "<=", - ast27.NotEq: "!=", - ast27.Is: "is", - ast27.IsNot: "is not", - ast27.In: "in", - ast27.NotIn: "not in", - } - - def from_comp_operator(self, op: ast27.cmpop) -> str: - op_name = ASTConverter.comp_op_map.get(type(op)) - if op_name is None: - raise RuntimeError("Unknown comparison operator " + str(type(op))) - else: - return op_name - - def as_block(self, stmts: List[ast27.stmt], lineno: int) -> Optional[Block]: - b = None - if stmts: - b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) - b.set_line(lineno) - return b - - def as_required_block(self, stmts: List[ast27.stmt], lineno: int) -> Block: - assert stmts # must be non-empty - b = Block(self.fix_function_overloads(self.translate_stmt_list(stmts))) - b.set_line(lineno) - return b - - def fix_function_overloads(self, stmts: List[Statement]) -> List[Statement]: - ret: List[Statement] = [] - current_overload: List[OverloadPart] = [] - current_overload_name: Optional[str] = None - for stmt in stmts: - if ( - current_overload_name is not None - and isinstance(stmt, (Decorator, FuncDef)) - and stmt.name == current_overload_name - ): - current_overload.append(stmt) - else: - if len(current_overload) == 1: - ret.append(current_overload[0]) - elif len(current_overload) > 1: - ret.append(OverloadedFuncDef(current_overload)) - - if isinstance(stmt, Decorator) and not unnamed_function(stmt.name): - current_overload = [stmt] - current_overload_name = stmt.name - else: - current_overload = [] - current_overload_name = None - ret.append(stmt) - - if len(current_overload) == 1: - ret.append(current_overload[0]) - elif len(current_overload) > 1: - ret.append(OverloadedFuncDef(current_overload)) - return ret - - def in_method_scope(self) -> bool: - return self.class_and_function_stack[-2:] == ["C", "F"] - - def translate_module_id(self, id: str) -> str: - """Return the actual, internal module id for a source text id. - - For example, translate '__builtin__' in Python 2 to 'builtins'. - """ - if id == self.options.custom_typing_module: - return "typing" - elif id == "__builtin__": - # HACK: __builtin__ in Python 2 is aliases to builtins. However, the implementation - # is named __builtin__.py (there is another layer of translation elsewhere). - return "builtins" - return id - - def visit_Module(self, mod: ast27.Module) -> MypyFile: - self.type_ignores = {} - for ti in mod.type_ignores: - parsed = parse_type_ignore_tag(ti.tag) # type: ignore[attr-defined] - if parsed is not None: - self.type_ignores[ti.lineno] = parsed - else: - self.fail(INVALID_TYPE_IGNORE, ti.lineno, -1) - body = self.fix_function_overloads(self.translate_stmt_list(mod.body, module=True)) - return MypyFile(body, self.imports, False, self.type_ignores) - - # --- stmt --- - # FunctionDef(identifier name, arguments args, - # stmt* body, expr* decorator_list, expr? returns, string? type_comment) - # arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, - # arg? kwarg, expr* defaults) - def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement: - self.class_and_function_stack.append("F") - lineno = n.lineno - converter = TypeConverter( - self.errors, - line=lineno, - override_column=n.col_offset, - assume_str_is_unicode=self.unicode_literals, - ) - args, decompose_stmts = self.transform_args(n.args, lineno) - if special_function_elide_names(n.name): - for arg in args: - arg.pos_only = True - - arg_kinds = [arg.kind for arg in args] - arg_names = [None if arg.pos_only else arg.variable.name for arg in args] - - arg_types: List[Optional[Type]] = [] - type_comment = n.type_comment - if n.decorator_list and any(is_no_type_check_decorator(d) for d in n.decorator_list): - arg_types = [None] * len(args) - return_type = None - elif type_comment is not None and len(type_comment) > 0: - try: - func_type_ast = ast3_parse(type_comment, "", "func_type") - assert isinstance(func_type_ast, ast3.FunctionType) - # for ellipsis arg - if len(func_type_ast.argtypes) == 1 and isinstance( - func_type_ast.argtypes[0], ast3.Ellipsis - ): - arg_types = [ - a.type_annotation - if a.type_annotation is not None - else AnyType(TypeOfAny.unannotated) - for a in args - ] - else: - # PEP 484 disallows both type annotations and type comments - if any(a.type_annotation is not None for a in args): - self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES, lineno, n.col_offset) - arg_types = [ - a if a is not None else AnyType(TypeOfAny.unannotated) - for a in converter.translate_expr_list(func_type_ast.argtypes) - ] - return_type = converter.visit(func_type_ast.returns) - - # add implicit self type - if self.in_method_scope() and len(arg_types) < len(args): - arg_types.insert(0, AnyType(TypeOfAny.special_form)) - except SyntaxError: - stripped_type = type_comment.split("#", 2)[0].strip() - err_msg = f'{TYPE_COMMENT_SYNTAX_ERROR} "{stripped_type}"' - self.fail(err_msg, lineno, n.col_offset) - arg_types = [AnyType(TypeOfAny.from_error)] * len(args) - return_type = AnyType(TypeOfAny.from_error) - else: - arg_types = [a.type_annotation for a in args] - return_type = converter.visit(None) - - for arg, arg_type in zip(args, arg_types): - self.set_type_optional(arg_type, arg.initializer) - - func_type = None - if any(arg_types) or return_type: - if len(arg_types) != 1 and any(isinstance(t, EllipsisType) for t in arg_types): - self.fail( - "Ellipses cannot accompany other argument types " "in function type signature", - lineno, - n.col_offset, - ) - elif len(arg_types) > len(arg_kinds): - self.fail( - "Type signature has too many arguments", lineno, n.col_offset, blocker=False - ) - elif len(arg_types) < len(arg_kinds): - self.fail( - "Type signature has too few arguments", lineno, n.col_offset, blocker=False - ) - else: - any_type = AnyType(TypeOfAny.unannotated) - func_type = CallableType( - [a if a is not None else any_type for a in arg_types], - arg_kinds, - arg_names, - return_type if return_type is not None else any_type, - _dummy_fallback, - ) - - body = self.as_required_block(n.body, lineno) - if decompose_stmts: - body.body = decompose_stmts + body.body - func_def = FuncDef(n.name, args, body, func_type) - if isinstance(func_def.type, CallableType): - # semanal.py does some in-place modifications we want to avoid - func_def.unanalyzed_type = func_def.type.copy_modified() - if func_type is not None: - func_type.definition = func_def - func_type.line = lineno - - if n.decorator_list: - var = Var(func_def.name) - var.is_ready = False - var.set_line(n.decorator_list[0].lineno) - - func_def.is_decorated = True - func_def.set_line(lineno + len(n.decorator_list)) - func_def.body.set_line(func_def.get_line()) - dec = Decorator(func_def, self.translate_expr_list(n.decorator_list), var) - dec.set_line(lineno, n.col_offset) - retval: Statement = dec - else: - # Overrides set_line -- can't use self.set_line - func_def.set_line(lineno, n.col_offset) - retval = func_def - self.class_and_function_stack.pop() - return retval - - def set_type_optional(self, type: Optional[Type], initializer: Optional[Expression]) -> None: - if self.options.no_implicit_optional: - return - # Indicate that type should be wrapped in an Optional if arg is initialized to None. - optional = isinstance(initializer, NameExpr) and initializer.name == "None" - if isinstance(type, UnboundType): - type.optional = optional - - def transform_args( - self, n: ast27.arguments, line: int - ) -> Tuple[List[Argument], List[Statement]]: - type_comments: Sequence[Optional[str]] = n.type_comments - converter = TypeConverter( - self.errors, line=line, assume_str_is_unicode=self.unicode_literals - ) - decompose_stmts: List[Statement] = [] - - n_args = n.args - args = [ - ( - self.convert_arg(i, arg, line, decompose_stmts), - self.get_type(i, type_comments, converter), - ) - for i, arg in enumerate(n_args) - ] - defaults = self.translate_expr_list(n.defaults) - names: List[str] = [name for arg in n_args for name in self.extract_names(arg)] - - new_args: List[Argument] = [] - num_no_defaults = len(args) - len(defaults) - # positional arguments without defaults - for a, annotation in args[:num_no_defaults]: - new_args.append(Argument(a, annotation, None, ARG_POS)) - - # positional arguments with defaults - for (a, annotation), d in zip(args[num_no_defaults:], defaults): - new_args.append(Argument(a, annotation, d, ARG_OPT)) - - # *arg - if n.vararg is not None: - new_args.append( - Argument( - Var(n.vararg), - self.get_type(len(args), type_comments, converter), - None, - ARG_STAR, - ) - ) - names.append(n.vararg) - - # **kwarg - if n.kwarg is not None: - typ = self.get_type( - len(args) + (0 if n.vararg is None else 1), type_comments, converter - ) - new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2)) - names.append(n.kwarg) - - for arg in new_args: - if argument_elide_name(arg.variable.name): - arg.pos_only = True - - # We don't have any context object to give, but we have closed around the line num - def fail_arg(msg: str, arg: None) -> None: - self.fail(msg, line, 0) - - check_arg_names(names, [None] * len(names), fail_arg) - - return new_args, decompose_stmts - - def extract_names(self, arg: ast27.expr) -> List[str]: - if isinstance(arg, Name): - return [arg.id] - elif isinstance(arg, ast27_Tuple): - return [name for elt in arg.elts for name in self.extract_names(elt)] - else: - return [] - - def convert_arg( - self, index: int, arg: ast27.expr, line: int, decompose_stmts: List[Statement] - ) -> Var: - if isinstance(arg, Name): - v = arg.id - elif isinstance(arg, ast27_Tuple): - v = f"__tuple_arg_{index + 1}" - rvalue = NameExpr(v) - rvalue.set_line(line) - assignment = AssignmentStmt([self.visit(arg)], rvalue) - assignment.set_line(line) - decompose_stmts.append(assignment) - else: - raise RuntimeError(f"'{ast27.dump(arg)}' is not a valid argument.") - return Var(v) - - def get_type( - self, i: int, type_comments: Sequence[Optional[str]], converter: TypeConverter - ) -> Optional[Type]: - if i < len(type_comments): - comment = type_comments[i] - if comment is not None: - typ = converter.visit_raw_str(comment) - extra_ignore = TYPE_IGNORE_PATTERN.match(comment) - if extra_ignore: - tag: Optional[str] = cast(Any, extra_ignore).group(1) - ignored = parse_type_ignore_tag(tag) - if ignored is None: - self.fail(INVALID_TYPE_IGNORE, converter.line, -1) - else: - self.type_ignores[converter.line] = ignored - return typ - return None - - def stringify_name(self, n: AST) -> str: - if isinstance(n, Name): - return n.id - elif isinstance(n, Attribute): - return f"{self.stringify_name(n.value)}.{n.attr}" - else: - assert False, "can't stringify " + str(type(n)) - - # ClassDef(identifier name, - # expr* bases, - # keyword* keywords, - # stmt* body, - # expr* decorator_list) - def visit_ClassDef(self, n: ast27.ClassDef) -> ClassDef: - self.class_and_function_stack.append("C") - - cdef = ClassDef( - n.name, - self.as_required_block(n.body, n.lineno), - None, - self.translate_expr_list(n.bases), - metaclass=None, - ) - cdef.decorators = self.translate_expr_list(n.decorator_list) - cdef.line = n.lineno + len(n.decorator_list) - cdef.column = n.col_offset - cdef.end_line = n.lineno - cdef.end_column = None - self.class_and_function_stack.pop() - return cdef - - # Return(expr? value) - def visit_Return(self, n: ast27.Return) -> ReturnStmt: - stmt = ReturnStmt(self.visit(n.value)) - return self.set_line(stmt, n) - - # Delete(expr* targets) - def visit_Delete(self, n: ast27.Delete) -> DelStmt: - if len(n.targets) > 1: - tup = TupleExpr(self.translate_expr_list(n.targets)) - tup.set_line(n.lineno) - stmt = DelStmt(tup) - else: - stmt = DelStmt(self.visit(n.targets[0])) - return self.set_line(stmt, n) - - # Assign(expr* targets, expr value, string? type_comment) - def visit_Assign(self, n: ast27.Assign) -> AssignmentStmt: - typ = self.translate_type_comment(n, n.type_comment) - stmt = AssignmentStmt(self.translate_expr_list(n.targets), self.visit(n.value), type=typ) - return self.set_line(stmt, n) - - # AugAssign(expr target, operator op, expr value) - def visit_AugAssign(self, n: ast27.AugAssign) -> OperatorAssignmentStmt: - stmt = OperatorAssignmentStmt( - self.from_operator(n.op), self.visit(n.target), self.visit(n.value) - ) - return self.set_line(stmt, n) - - # For(expr target, expr iter, stmt* body, stmt* orelse, string? type_comment) - def visit_For(self, n: ast27.For) -> ForStmt: - typ = self.translate_type_comment(n, n.type_comment) - stmt = ForStmt( - self.visit(n.target), - self.visit(n.iter), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno), - typ, - ) - return self.set_line(stmt, n) - - # While(expr test, stmt* body, stmt* orelse) - def visit_While(self, n: ast27.While) -> WhileStmt: - stmt = WhileStmt( - self.visit(n.test), - self.as_required_block(n.body, n.lineno), - self.as_block(n.orelse, n.lineno), - ) - return self.set_line(stmt, n) - - # If(expr test, stmt* body, stmt* orelse) - def visit_If(self, n: ast27.If) -> IfStmt: - stmt = IfStmt( - [self.visit(n.test)], - [self.as_required_block(n.body, n.lineno)], - self.as_block(n.orelse, n.lineno), - ) - return self.set_line(stmt, n) - - # With(withitem* items, stmt* body, string? type_comment) - def visit_With(self, n: ast27.With) -> WithStmt: - typ = self.translate_type_comment(n, n.type_comment) - stmt = WithStmt( - [self.visit(n.context_expr)], - [self.visit(n.optional_vars)], - self.as_required_block(n.body, n.lineno), - typ, - ) - return self.set_line(stmt, n) - - # 'raise' [test [',' test [',' test]]] - def visit_Raise(self, n: ast27.Raise) -> RaiseStmt: - if n.type is None: - e = None - else: - if n.inst is None: - e = self.visit(n.type) - else: - if n.tback is None: - e = TupleExpr([self.visit(n.type), self.visit(n.inst)]) - else: - e = TupleExpr([self.visit(n.type), self.visit(n.inst), self.visit(n.tback)]) - - stmt = RaiseStmt(e, None) - return self.set_line(stmt, n) - - # TryExcept(stmt* body, excepthandler* handlers, stmt* orelse) - def visit_TryExcept(self, n: ast27.TryExcept) -> TryStmt: - stmt = self.try_handler(n.body, n.handlers, n.orelse, [], n.lineno) - return self.set_line(stmt, n) - - def visit_TryFinally(self, n: ast27.TryFinally) -> TryStmt: - if len(n.body) == 1 and isinstance(n.body[0], ast27.TryExcept): - stmt = self.try_handler([n.body[0]], [], [], n.finalbody, n.lineno) - else: - stmt = self.try_handler(n.body, [], [], n.finalbody, n.lineno) - return self.set_line(stmt, n) - - def try_handler( - self, - body: List[ast27.stmt], - handlers: List[ast27.ExceptHandler], - orelse: List[ast27.stmt], - finalbody: List[ast27.stmt], - lineno: int, - ) -> TryStmt: - vs: List[Optional[NameExpr]] = [] - for item in handlers: - if item.name is None: - vs.append(None) - elif isinstance(item.name, Name): - vs.append(self.set_line(NameExpr(item.name.id), item)) - else: - self.fail( - 'Sorry, "except , " is not supported', - item.lineno, - item.col_offset, - ) - vs.append(None) - types = [self.visit(h.type) for h in handlers] - handlers_ = [self.as_required_block(h.body, h.lineno) for h in handlers] - - return TryStmt( - self.as_required_block(body, lineno), - vs, - types, - handlers_, - self.as_block(orelse, lineno), - self.as_block(finalbody, lineno), - ) - - def visit_Print(self, n: ast27.Print) -> PrintStmt: - stmt = PrintStmt(self.translate_expr_list(n.values), n.nl, self.visit(n.dest)) - return self.set_line(stmt, n) - - def visit_Exec(self, n: ast27.Exec) -> ExecStmt: - stmt = ExecStmt(self.visit(n.body), self.visit(n.globals), self.visit(n.locals)) - return self.set_line(stmt, n) - - def visit_Repr(self, n: ast27.Repr) -> BackquoteExpr: - stmt = BackquoteExpr(self.visit(n.value)) - return self.set_line(stmt, n) - - # Assert(expr test, expr? msg) - def visit_Assert(self, n: ast27.Assert) -> AssertStmt: - stmt = AssertStmt(self.visit(n.test), self.visit(n.msg)) - return self.set_line(stmt, n) - - # Import(alias* names) - def visit_Import(self, n: ast27.Import) -> Import: - names: List[Tuple[str, Optional[str]]] = [] - for alias in n.names: - name = self.translate_module_id(alias.name) - asname = alias.asname - if asname is None and name != alias.name: - # if the module name has been translated (and it's not already - # an explicit import-as), make it an implicit import-as the - # original name - asname = alias.name - names.append((name, asname)) - i = Import(names) - self.imports.append(i) - return self.set_line(i, n) - - # ImportFrom(identifier? module, alias* names, int? level) - def visit_ImportFrom(self, n: ast27.ImportFrom) -> ImportBase: - assert n.level is not None - if len(n.names) == 1 and n.names[0].name == "*": - mod = n.module if n.module is not None else "" - i: ImportBase = ImportAll(mod, n.level) - else: - module_id = self.translate_module_id(n.module) if n.module is not None else "" - i = ImportFrom(module_id, n.level, [(a.name, a.asname) for a in n.names]) - - # See comments in the constructor for more information about this field. - if module_id == "__future__" and any(a.name == "unicode_literals" for a in n.names): - self.unicode_literals = True - self.imports.append(i) - return self.set_line(i, n) - - # Global(identifier* names) - def visit_Global(self, n: ast27.Global) -> GlobalDecl: - stmt = GlobalDecl(n.names) - return self.set_line(stmt, n) - - # Expr(expr value) - def visit_Expr(self, n: ast27.Expr) -> ExpressionStmt: - value = self.visit(n.value) - stmt = ExpressionStmt(value) - return self.set_line(stmt, n) - - # Pass - def visit_Pass(self, n: ast27.Pass) -> PassStmt: - stmt = PassStmt() - return self.set_line(stmt, n) - - # Break - def visit_Break(self, n: ast27.Break) -> BreakStmt: - stmt = BreakStmt() - return self.set_line(stmt, n) - - # Continue - def visit_Continue(self, n: ast27.Continue) -> ContinueStmt: - stmt = ContinueStmt() - return self.set_line(stmt, n) - - # --- expr --- - - # BoolOp(boolop op, expr* values) - def visit_BoolOp(self, n: ast27.BoolOp) -> OpExpr: - # mypy translates (1 and 2 and 3) as (1 and (2 and 3)) - assert len(n.values) >= 2 - if isinstance(n.op, ast27.And): - op = "and" - elif isinstance(n.op, ast27.Or): - op = "or" - else: - raise RuntimeError("unknown BoolOp " + str(type(n))) - - # potentially inefficient! - e = self.group(self.translate_expr_list(n.values), op) - return self.set_line(e, n) - - def group(self, vals: List[Expression], op: str) -> OpExpr: - if len(vals) == 2: - return OpExpr(op, vals[0], vals[1]) - else: - return OpExpr(op, vals[0], self.group(vals[1:], op)) - - # BinOp(expr left, operator op, expr right) - def visit_BinOp(self, n: ast27.BinOp) -> OpExpr: - op = self.from_operator(n.op) - - if op is None: - raise RuntimeError("cannot translate BinOp " + str(type(n.op))) - - e = OpExpr(op, self.visit(n.left), self.visit(n.right)) - return self.set_line(e, n) - - # UnaryOp(unaryop op, expr operand) - def visit_UnaryOp(self, n: ast27.UnaryOp) -> UnaryExpr: - op = None - if isinstance(n.op, ast27.Invert): - op = "~" - elif isinstance(n.op, ast27.Not): - op = "not" - elif isinstance(n.op, ast27.UAdd): - op = "+" - elif isinstance(n.op, ast27.USub): - op = "-" - - if op is None: - raise RuntimeError("cannot translate UnaryOp " + str(type(n.op))) - - e = UnaryExpr(op, self.visit(n.operand)) - return self.set_line(e, n) - - # Lambda(arguments args, expr body) - def visit_Lambda(self, n: ast27.Lambda) -> LambdaExpr: - args, decompose_stmts = self.transform_args(n.args, n.lineno) - - n_body = ast27.Return(n.body) - n_body.lineno = n.body.lineno - n_body.col_offset = n.body.col_offset - body = self.as_required_block([n_body], n.lineno) - if decompose_stmts: - body.body = decompose_stmts + body.body - - e = LambdaExpr(args, body) - e.set_line(n.lineno, n.col_offset) # Overrides set_line -- can't use self.set_line - return e - - # IfExp(expr test, expr body, expr orelse) - def visit_IfExp(self, n: ast27.IfExp) -> ConditionalExpr: - e = ConditionalExpr(self.visit(n.test), self.visit(n.body), self.visit(n.orelse)) - return self.set_line(e, n) - - # Dict(expr* keys, expr* values) - def visit_Dict(self, n: ast27.Dict) -> DictExpr: - e = DictExpr( - list(zip(self.translate_expr_list(n.keys), self.translate_expr_list(n.values))) - ) - return self.set_line(e, n) - - # Set(expr* elts) - def visit_Set(self, n: ast27.Set) -> SetExpr: - e = SetExpr(self.translate_expr_list(n.elts)) - return self.set_line(e, n) - - # ListComp(expr elt, comprehension* generators) - def visit_ListComp(self, n: ast27.ListComp) -> ListComprehension: - e = ListComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) - return self.set_line(e, n) - - # SetComp(expr elt, comprehension* generators) - def visit_SetComp(self, n: ast27.SetComp) -> SetComprehension: - e = SetComprehension(self.visit_GeneratorExp(cast(ast27.GeneratorExp, n))) - return self.set_line(e, n) - - # DictComp(expr key, expr value, comprehension* generators) - def visit_DictComp(self, n: ast27.DictComp) -> DictionaryComprehension: - targets = [self.visit(c.target) for c in n.generators] - iters = [self.visit(c.iter) for c in n.generators] - ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] - e = DictionaryComprehension( - self.visit(n.key), - self.visit(n.value), - targets, - iters, - ifs_list, - [False for _ in n.generators], - ) - return self.set_line(e, n) - - # GeneratorExp(expr elt, comprehension* generators) - def visit_GeneratorExp(self, n: ast27.GeneratorExp) -> GeneratorExpr: - targets = [self.visit(c.target) for c in n.generators] - iters = [self.visit(c.iter) for c in n.generators] - ifs_list = [self.translate_expr_list(c.ifs) for c in n.generators] - e = GeneratorExpr( - self.visit(n.elt), targets, iters, ifs_list, [False for _ in n.generators] - ) - return self.set_line(e, n) - - # Yield(expr? value) - def visit_Yield(self, n: ast27.Yield) -> YieldExpr: - e = YieldExpr(self.visit(n.value)) - return self.set_line(e, n) - - # Compare(expr left, cmpop* ops, expr* comparators) - def visit_Compare(self, n: ast27.Compare) -> ComparisonExpr: - operators = [self.from_comp_operator(o) for o in n.ops] - operands = self.translate_expr_list([n.left] + n.comparators) - e = ComparisonExpr(operators, operands) - return self.set_line(e, n) - - # Call(expr func, expr* args, keyword* keywords) - # keyword = (identifier? arg, expr value) - def visit_Call(self, n: Call) -> CallExpr: - arg_types: List[ast27.expr] = [] - arg_kinds: List[ArgKind] = [] - signature: List[Optional[str]] = [] - - args = n.args - arg_types.extend(args) - arg_kinds.extend(ARG_POS for a in args) - signature.extend(None for a in args) - - if n.starargs is not None: - arg_types.append(n.starargs) - arg_kinds.append(ARG_STAR) - signature.append(None) - - keywords = n.keywords - arg_types.extend(k.value for k in keywords) - arg_kinds.extend(ARG_NAMED for k in keywords) - signature.extend(k.arg for k in keywords) - - if n.kwargs is not None: - arg_types.append(n.kwargs) - arg_kinds.append(ARG_STAR2) - signature.append(None) - - e = CallExpr(self.visit(n.func), self.translate_expr_list(arg_types), arg_kinds, signature) - return self.set_line(e, n) - - # Num(object n) -- a number as a PyObject. - def visit_Num(self, n: ast27.Num) -> Expression: - # The n field has the type complex, but complex isn't *really* - # a parent of int and float, and this causes isinstance below - # to think that the complex branch is always picked. Avoid - # this by throwing away the type. - value: object = n.n - is_inverse = False - if str(n.n).startswith("-"): # Hackish because of complex. - value = -n.n - is_inverse = True - - if isinstance(value, int): - expr: Expression = IntExpr(value) - elif isinstance(value, float): - expr = FloatExpr(value) - elif isinstance(value, complex): - expr = ComplexExpr(value) - else: - raise RuntimeError("num not implemented for " + str(type(n.n))) - - if is_inverse: - expr = UnaryExpr("-", expr) - - return self.set_line(expr, n) - - # Str(string s) - def visit_Str(self, n: ast27.Str) -> Expression: - # Note: typed_ast.ast27 will handled unicode_literals for us. If - # n.s is of type 'bytes', we know unicode_literals was not enabled; - # otherwise we know it was. - # - # Note that the following code is NOT run when parsing Python 2.7 stubs: - # we always parse stub files (no matter what version) using the Python 3 - # parser. This is also why string literals in Python 2.7 stubs are assumed - # to be unicode. - if isinstance(n.s, bytes): - contents = bytes_to_human_readable_repr(n.s) - e: Union[StrExpr, UnicodeExpr] = StrExpr(contents, from_python_3=False) - return self.set_line(e, n) - else: - e = UnicodeExpr(n.s) - return self.set_line(e, n) - - # Ellipsis - def visit_Ellipsis(self, n: ast27.Ellipsis) -> EllipsisExpr: - return EllipsisExpr() - - # Attribute(expr value, identifier attr, expr_context ctx) - def visit_Attribute(self, n: Attribute) -> Expression: - # First create MemberExpr and then potentially replace with a SuperExpr - # to improve performance when compiled. The check for "super()" will be - # faster with native AST nodes. Note also that super expressions are - # less common than normal member expressions. - member_expr = MemberExpr(self.visit(n.value), n.attr) - obj = member_expr.expr - if ( - isinstance(obj, CallExpr) - and isinstance(obj.callee, NameExpr) - and obj.callee.name == "super" - ): - e: Expression = SuperExpr(member_expr.name, obj) - else: - e = member_expr - return self.set_line(e, n) - - # Subscript(expr value, slice slice, expr_context ctx) - def visit_Subscript(self, n: ast27.Subscript) -> IndexExpr: - e = IndexExpr(self.visit(n.value), self.visit(n.slice)) - self.set_line(e, n) - if isinstance(e.index, SliceExpr): - # Slice has no line/column in the raw ast. - e.index.line = e.line - e.index.column = e.column - return e - - # Name(identifier id, expr_context ctx) - def visit_Name(self, n: Name) -> NameExpr: - e = NameExpr(n.id) - return self.set_line(e, n) - - # List(expr* elts, expr_context ctx) - def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]: - expr_list: List[Expression] = [self.visit(e) for e in n.elts] - if isinstance(n.ctx, ast27.Store): - # [x, y] = z and (x, y) = z means exactly the same thing - e: Union[ListExpr, TupleExpr] = TupleExpr(expr_list) - else: - e = ListExpr(expr_list) - return self.set_line(e, n) - - # Tuple(expr* elts, expr_context ctx) - def visit_Tuple(self, n: ast27_Tuple) -> TupleExpr: - e = TupleExpr([self.visit(e) for e in n.elts]) - return self.set_line(e, n) - - # --- slice --- - - # Slice(expr? lower, expr? upper, expr? step) - def visit_Slice(self, n: ast27.Slice) -> SliceExpr: - return SliceExpr(self.visit(n.lower), self.visit(n.upper), self.visit(n.step)) - - # ExtSlice(slice* dims) - def visit_ExtSlice(self, n: ast27.ExtSlice) -> TupleExpr: - return TupleExpr(self.translate_expr_list(n.dims)) - - # Index(expr value) - def visit_Index(self, n: ast27.Index) -> Expression: - return self.visit(n.value) diff --git a/mypy/parse.py b/mypy/parse.py index 4c18fe22a1dc..d078a742562d 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -19,18 +19,8 @@ def parse( The python_version (major, minor) option determines the Python syntax variant. """ - is_stub_file = fnam.endswith(".pyi") if options.transform_source is not None: source = options.transform_source(source) - if options.python_version[0] >= 3 or is_stub_file: - import mypy.fastparse + import mypy.fastparse - return mypy.fastparse.parse( - source, fnam=fnam, module=module, errors=errors, options=options - ) - else: - import mypy.fastparse2 - - return mypy.fastparse2.parse( - source, fnam=fnam, module=module, errors=errors, options=options - ) + return mypy.fastparse.parse(source, fnam=fnam, module=module, errors=errors, options=options)