Skip to content

Commit

Permalink
bpo-43224: Implement PEP 646 grammar changes (GH-31018)
Browse files Browse the repository at this point in the history 
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com>
  • Loading branch information
@mrahtz @JelleZijlstra
mrahtz and JelleZijlstra authored Mar 26, 2022
1 parent 26cca80 commit e8e737b
Show file tree
Hide file tree
Showing 12 changed files with 3,056 additions and 2,128 deletions.
9 changes: 8 additions & 1 deletion Grammar/python.gram
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ star_etc[StarEtc*]:
| invalid_star_etc
| '*' a=param_no_default b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' ',' b=param_maybe_default+ c=[kwds] {
_PyPegen_star_etc(p, NULL, b, c) }
| a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
Expand All @@ -333,14 +335,19 @@ kwds[arg_ty]:
param_no_default[arg_ty]:
| a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_no_default_star_annotation[arg_ty]:
| a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_with_default[NameDefaultPair*]:
| a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param_maybe_default[NameDefaultPair*]:
| a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
annotation[expr_ty]: ':' a=expression { a }
star_annotation[expr_ty]: ':' a=star_expression { a }
default[expr_ty]: '=' a=expression { a } | invalid_default

# If statement
Expand Down Expand Up @@ -782,7 +789,7 @@ primary[expr_ty]:

slices[expr_ty]:
| a=slice !',' { a }
| a[asdl_expr_seq*]=','.slice+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
| a[asdl_expr_seq*]=','.(slice | starred_expression)+ [','] { _PyAST_Tuple(a, Load, EXTRA) }

slice[expr_ty]:
| a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _PyAST_Slice(a, b, c, EXTRA) }
Expand Down
9 changes: 3 additions & 6 deletions Lib/ast.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1476,20 +1476,17 @@ def visit_Call(self, node):
self.traverse(e)

def visit_Subscript(self, node):
def is_simple_tuple(slice_value):
# when unparsing a non-empty tuple, the parentheses can be safely
# omitted if there aren't any elements that explicitly requires
# parentheses (such as starred expressions).
def is_non_empty_tuple(slice_value):
return (
isinstance(slice_value, Tuple)
and slice_value.elts
and not any(isinstance(elt, Starred) for elt in slice_value.elts)
)

self.set_precedence(_Precedence.ATOM, node.value)
self.traverse(node.value)
with self.delimit("[", "]"):
if is_simple_tuple(node.slice):
if is_non_empty_tuple(node.slice):
# parentheses can be omitted if the tuple isn't empty
self.items_view(self.traverse, node.slice.elts)
else:
self.traverse(node.slice)
Expand Down
25 changes: 24 additions & 1 deletion Lib/test/test_ast.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from test import support

def to_tuple(t):
if t is None or isinstance(t, (str, int, complex)):
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
return t
elif isinstance(t, list):
return [to_tuple(e) for e in t]
Expand Down Expand Up @@ -46,10 +46,20 @@ def to_tuple(t):
"def f(a=0): pass",
# FunctionDef with varargs
"def f(*args): pass",
# FunctionDef with varargs as TypeVarTuple
"def f(*args: *Ts): pass",
# FunctionDef with varargs as unpacked Tuple
"def f(*args: *tuple[int, ...]): pass",
# FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple
"def f(*args: *tuple[int, *Ts]): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# FunctionDef with type annotation on return involving unpacking
"def f() -> tuple[*Ts]: pass",
"def f() -> tuple[int, *Ts]: pass",
"def f() -> tuple[int, *tuple[int, ...]]: pass",
# ClassDef
"class C:pass",
# ClassDef with docstring
Expand All @@ -65,6 +75,10 @@ def to_tuple(t):
"a,b = c",
"(a,b) = c",
"[a,b] = c",
# AnnAssign with unpacked types
"x: tuple[*Ts]",
"x: tuple[int, *Ts]",
"x: tuple[int, *tuple[str, ...]]",
# AugAssign
"v += 1",
# For
Expand Down Expand Up @@ -2315,8 +2329,14 @@ def main():
('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []),
Expand All @@ -2326,6 +2346,9 @@ def main():
('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []),
('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []),
('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []),
Expand Down
41 changes: 38 additions & 3 deletions Lib/test/test_future.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def _exec_future(self, code):
scope = {}
exec(
"from __future__ import annotations\n"
+ code, {}, scope
+ code, scope
)
return scope

Expand Down Expand Up @@ -287,10 +287,11 @@ def test_annotations(self):
eq("list[str]")
eq("dict[str, int]")
eq("set[str,]")
eq("tuple[()]")
eq("tuple[str, ...]")
eq("tuple[(str, *types)]")
eq("tuple[str, *types]")
eq("tuple[str, int, (str, int)]")
eq("tuple[(*int, str, str, (str, int))]")
eq("tuple[*int, str, str, (str, int)]")
eq("tuple[str, int, float, dict[str, int]]")
eq("slice[0]")
eq("slice[0:1]")
Expand All @@ -305,6 +306,21 @@ def test_annotations(self):
eq("slice[1:2, 1]")
eq("slice[1:2, 2, 3]")
eq("slice[()]")
# Note that `slice[*Ts]`, `slice[*Ts,]`, and `slice[(*Ts,)]` all have
# the same AST, but only `slice[*Ts,]` passes this test, because that's
# what the unparser produces.
eq("slice[*Ts,]")
eq("slice[1, *Ts]")
eq("slice[*Ts, 2]")
eq("slice[1, *Ts, 2]")
eq("slice[*Ts, *Ts]")
eq("slice[1, *Ts, *Ts]")
eq("slice[*Ts, 1, *Ts]")
eq("slice[*Ts, *Ts, 1]")
eq("slice[1, *Ts, *Ts, 2]")
eq("slice[1:2, *Ts]")
eq("slice[*Ts, 1:2]")
eq("slice[1:2, *Ts, 3:4]")
eq("slice[a, b:c, d:e:f]")
eq("slice[(x for x in a)]")
eq('str or None if sys.version_info[0] > (3,) else str or bytes or None')
Expand Down Expand Up @@ -403,6 +419,25 @@ def foo():
def bar(arg: (yield)): pass
"""))

def test_get_type_hints_on_func_with_variadic_arg(self):
# `typing.get_type_hints` might break on a function with a variadic
# annotation (e.g. `f(*args: *Ts)`) if `from __future__ import
# annotations`, because it could try to evaluate `*Ts` as an expression,
# which on its own isn't value syntax.
namespace = self._exec_future(dedent("""\
class StarredC: pass
class C:
def __iter__(self):
yield StarredC()
c = C()
def f(*args: *c): pass
import typing
hints = typing.get_type_hints(f)
"""))

hints = namespace.pop('hints')
self.assertIsInstance(hints['args'], namespace['StarredC'])


if __name__ == "__main__":
unittest.main()
Loading

0 comments on commit e8e737b

Please sign in to comment.