Support PEP 492, Coroutines with async and await syntax

baron/formatting_grouper.py

@@ -79,6 +79,8 @@ class UnExpectedSpaceToken(BaronError):
 GROUP_SPACE_AFTER = BOTH + (
  "FROM",
  "TILDE",
+ "ASYNC",
+ "AWAIT",
  "RETURN",
  "YIELD",
  "WITH",

baron/grammator.py

@@ -140,10 +140,37 @@ def simple_stmt_semicolon(pack):
  @pg.production("compound_stmt : classdef")
  @pg.production("compound_stmt : with_stmt")
  @pg.production("compound_stmt : decorated")
+ @pg.production("compound_stmt : async_stmt")
  def small_and_compound_stmt(pack):
  (statement,) = pack
  return statement
 
+
+ @pg.production("async_maybe : ")
+ def async__(pack):
+ return {}
+
+
+ @pg.production("async_maybe : ASYNC")
+ @pg.production("async : ASYNC")
+ def async__(pack):
+ (async_, ) = pack
+ return {
+ "type": "async",
+ "formatting": async_.hidden_tokens_after
+
+ }
+
+
+ @pg.production("async_stmt : async with_stmt")
+ @pg.production("async_stmt : async for_stmt")
+ @pg.production("async_stmt : async funcdef")
+ def async_stmt(pack):
+ (async_, statement,) = pack
+ statement[0]["async"] = async_
+ return statement
+
+
  if not print_function:
  @pg.production("small_stmt : print_stmt")
  def print_statement(pack):
@@ -179,6 +206,7 @@ def with_stmt(pack):
  (with_, with_items, colon, suite) = pack
  return [{
  "type": "with",
+ "async": {},
  "value": suite,
  "first_formatting": with_.hidden_tokens_after,
  "second_formatting": colon.hidden_tokens_before,
@@ -364,12 +392,12 @@ def decorator_call(pack):
  "formatting": at.hidden_tokens_after,
  }] + endl
 
-
- @pg.production("funcdef : DEF NAME LEFT_PARENTHESIS parameters RIGHT_PARENTHESIS COLON suite")
+ @pg.production("funcdef : async_maybe DEF NAME LEFT_PARENTHESIS parameters RIGHT_PARENTHESIS COLON suite")
  def function_definition(pack):
- (def_, name, left_parenthesis, parameters, right_parenthesis, colon, suite) = pack
+ (async_maybe, def_, name, left_parenthesis, parameters, right_parenthesis, colon, suite) = pack
  return [{
  "type": "def",
+ "async": async_maybe,
  "decorators": [],
  "name": name.value,
  "first_formatting": def_.hidden_tokens_after,

baron/grammator_control_structures.py

@@ -163,6 +163,7 @@ def for_stmt(pack,):
  (for_, exprlist, in_, testlist, colon, suite) = pack
  return [{
  "type": "for",
+ "async": {},
  "value": suite,
  "iterator": exprlist,
  "target": testlist,

baron/grammator_operators.py

@@ -160,6 +160,7 @@ def binary_operator_node(pack):
  "second_formatting": operator.hidden_tokens_after
  }
 
+
  @pg.production("factor : PLUS factor")
  @pg.production("factor : MINUS factor")
  @pg.production("factor : TILDE factor")
@@ -196,6 +197,18 @@ def power_atomtrailers(pack):
  "value": atomtrailers
  }
 
+ @pg.production("power : AWAIT atomtrailers")
+ def power_atomtrailers_await(pack):
+ (await_, atomtrailers,) = pack
+
+ return [{
+ "type": "await",
+ "formatting": await_.hidden_tokens_after
+ }, {
+ "type": "atomtrailers",
+ "value": atomtrailers
+ }]
+
  @pg.production("atomtrailers : atom")
  def atomtrailers_atom(pack):
  (atom,) = pack

baron/inner_formatting_grouper.py

@@ -51,6 +51,8 @@ class GroupingError(BaronError):
  "DOUBLE_SLASH",
  "PLUS",
  "MINUS",
+ "ASYNC",
+ "AWAIT",
  "LEFT_SHIFT",
  "RIGHT_SHIFT",
  "AMPER",

baron/render.py

@@ -289,7 +289,18 @@ def child_by_key(node, key):
  ("bool", "with_parenthesis", False),
  ],
 
+ "async": [
+ ("constant", "async", True),
+ ("formatting", "formatting", True),
+
+ ],
+ "await": [
+ ("constant", "await", True),
+ ("formatting", "formatting", True),
+
+ ],
  "def": [
+ ("key", "async", "async"),
  ("list", "decorators", True),
  ("constant", "def", True),
  ("formatting", "first_formatting", True),
@@ -489,6 +500,7 @@ def child_by_key(node, key):
  ],
 
  "with": [
+ ("key", "async", "async"),
  ("constant", "with", True),
  ("formatting", "first_formatting", True),
  ("list", "contexts", True),
@@ -555,6 +567,7 @@ def child_by_key(node, key):
  ("key", "else", "else"),
  ],
  "for": [
+ ("key", "async", "async"),
  ("constant", "for", True),
  ("formatting", "first_formatting", True),
  ("key", "iterator", True),

baron/tokenizer.py

@@ -5,7 +5,7 @@
 class UnknowItem(BaronError):
  pass
 
-KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
+KEYWORDS = ("and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
 
 TOKENS = (
  (r'[a-zA-Z_]\w*', 'NAME'),

tests/test_grammator.py

@@ -401,6 +401,7 @@ def a () :
  "type": "def",
  "name": "a",
  "decorators": [],
+ "async": {},
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [{"type": "space", "value": " "}],
  "third_formatting": [],
@@ -428,6 +429,61 @@ def a () :
  }
  ])
 
+
+def test_funcdef_stmt_async():
+ """
+ async def a():
+ pass
+ """
+ parse_multi([
+ ('ASYNC', 'async', [], [('SPACE', ' ')]),
+ ('DEF', 'def', [], [('SPACE', ' ')]),
+ ('NAME', 'a'),
+ ('LEFT_PARENTHESIS', '(', [('SPACE', ' ')]),
+ ('RIGHT_PARENTHESIS', ')'),
+ ('COLON', ':', [], [('SPACE', ' ')]),
+ ('ENDL', '\n', [], [('SPACE', ' ')]),
+ ('INDENT', ''),
+ ('PASS', 'pass'),
+ ('ENDL', '\n'),
+ ('DEDENT', ''),
+ ], [
+ {
+ "type": "def",
+ "name": "a",
+ "decorators": [],
+ "async": {
+ "type": "async",
+ "formatting": [{"type": "space", "value": " "}],
+ },
+ "first_formatting": [{"type": "space", "value": " "}],
+ "second_formatting": [{"type": "space", "value": " "}],
+ "third_formatting": [],
+ "fourth_formatting": [],
+ "fifth_formatting": [],
+ "sixth_formatting": [{"type": "space", "value": " "}],
+ "arguments": [],
+ "value": [
+ {
+ "type": "endl",
+ "value": "\n",
+ "formatting": [],
+ "indent": " "
+ },
+ {
+ "type": "pass",
+ },
+ {
+ "type": "endl",
+ "formatting": [],
+ "indent": "",
+ "value": "\n"
+ }
+ ],
+ }
+ ])
+
+
 def test_funcdef_stmt_one_parameter_indent():
  """
  def a ( x ) :
@@ -450,6 +506,7 @@ def a ( x ) :
  "type": "def",
  "name": "a",
  "decorators": [],
+ "async": {},
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [{"type": "space", "value": " "}],
  "third_formatting": [{"type": "space", "value": " "}],
@@ -510,6 +567,7 @@ def a ( x , ) :
  {
  "type": "def",
  "decorators": [],
+ "async": {},
  "name": "a",
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [{"type": "space", "value": " "}],
@@ -578,6 +636,7 @@ def a ( x=1 , ) :
  {
  "type": "def",
  "decorators": [],
+ "async": {},
  "name": "a",
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [{"type": "space", "value": " "}],
@@ -800,6 +859,7 @@ def a (*b):
  "type": "def",
  "name": "a",
  "decorators": [],
+ "async": {},
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [],
  "third_formatting": [],
@@ -859,6 +919,7 @@ def a (**b):
  "type": "def",
  "name": "a",
  "decorators": [],
+ "async": {},
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [],
  "third_formatting": [],
@@ -955,6 +1016,7 @@ def test_with_a():
  "second_formatting": [],
  "third_formatting": [{"type": "space", "value": " "}],
  "type": "with",
+ "async": {},
  "contexts": [
  {
  "type": "with_context_item",
@@ -1000,6 +1062,7 @@ def test_with_a_as_b():
  "second_formatting": [],
  "third_formatting": [{"type": "space", "value": " "}],
  "type": "with",
+ "async": {},
  "contexts": [
  {
  "type": "with_context_item",
@@ -1050,6 +1113,7 @@ def test_with_a_as_b_c():
  "second_formatting": [],
  "third_formatting": [{"type": "space", "value": " "}],
  "type": "with",
+ "async": {},
  "contexts": [
  {
  "type": "with_context_item",
@@ -1122,6 +1186,7 @@ def b(): pass
  "type": "def",
  "arguments": [],
  "name": "b",
+ "async": {},
  "decorators": [
  {
  "type": "decorator",
@@ -1188,6 +1253,7 @@ def b(): pass
  "type": "def",
  "arguments": [],
  "name": "b",
+ "async": {},
  "decorators": [
  {
  "type": "decorator",
@@ -1262,6 +1328,7 @@ def b(): pass
  "type": "def",
  "arguments": [],
  "name": "b",
+ "async": {},
  "decorators": [
  {
  "type": "decorator",
@@ -1348,6 +1415,7 @@ def b(): pass
  "type": "def",
  "arguments": [],
  "name": "b",
+ "async": {},
  "decorators": [
  {
  "type": "decorator",
@@ -1613,6 +1681,7 @@ def a((b,)): pass
  ],
  "name": "a",
  "decorators": [],
+ "async": {},
  "value": [
  {
  "type": "pass",
@@ -1687,6 +1756,7 @@ def a((b,c)): pass
  ],
  "name": "a",
  "decorators": [],
+ "async": {},
  "value": [
  {
  "type": "pass",
@@ -1747,6 +1817,7 @@ def a((b)): pass
  ],
  "name": "a",
  "decorators": [],
+ "async": {},
  "value": [
  {
  "type": "pass",
@@ -1834,6 +1905,7 @@ def function_name((a,b)=c):\n pass\n
  }
  ],
  'decorators': [],
+ 'async': {},
  'fifth_formatting': [],
  'first_formatting': [
  {
@@ -1951,6 +2023,7 @@ def function_name((a,(x,y))=c):\n pass\n
  }
  ],
  'decorators': [],
+ 'async': {},
  'fifth_formatting': [],
  'first_formatting': [
  {

tests/test_grammator_control_structures.py

@@ -756,6 +756,7 @@ def test_for_stmt_indent():
  ], [
  {
  "type": "for",
+ "async": {},
  "first_formatting": [{"type": "space", "value": " "}],
  "second_formatting": [{"type": "space", "value": " "}],
  "third_formatting": [{"type": "space", "value": " "}],

tests/test_grammator_operators.py

@@ -8489,6 +8489,17 @@ def test_expr_comma_list_3_items():
  ])
 
 
+def test_await_a():
+ "await a"
+ parse_simple([
+ ('AWAIT', 'await', [], [('SPACE', ' ')]),
+ ('NAME', 'a'),
+ ], [[
+ {'formatting': [{'type': 'space', 'value': ' '}], 'type': 'await'},
+ {'type': 'atomtrailers', 'value': [{'type': 'name', 'value': 'a'}]},
+ ]])
+
+
 def test_implicit_tuple_space():
  "a, b , c"
  parse_simple([