Merge branch 'master' into python3-keyword-only-marker

CHANGELOG

@@ -1,6 +1,21 @@
 Changelog
 =========
 
+0.7 (unreleased)
+------------------
+
+- fix line continuation https://github.com/PyCQA/baron/pull/92 by ibizaman
+- handle corrupt cache file situation https://github.com/PyCQA/baron/pull/76 by ryu2
+- fix special crashing edge case in indentation marker https://github.com/PyCQA/bar by Ahuge
+- fixed incorrect tokenization case "d*e-1". Fixes #85 https://github.com/PyCQA/baron/pull/107 by boxed
+
+Python 3 parsing:
+- support ellipsis https://github.com/PyCQA/baron/pull/121 by odcinek
+- support matrix operator https://github.com/PyCQA/baron/pull/117 by odcinek
+- support f-strings https://github.com/PyCQA/baron/pull/110 by odcinek
+- support numeric literals https://github.com/PyCQA/baron/pull/111 by odcinek
+- support nonlocal statement https://github.com/PyCQA/baron/pull/112 by odcinek
+
 0.6.6 (2017-06-12)
 ------------------
 

baron/baron.py

@@ -67,4 +67,10 @@ def parse(source_code, print_function=None):
 
 
 def tokenize(pouet, print_function=False):
-    return mark_indentation(inner_group(space_group(_tokenize(group(split(pouet)), print_function))))
+    splitted = split(pouet)
+    grouped = group(splitted)
+    print_tokenized = _tokenize(grouped, print_function)
+    space_grouped = space_group(print_tokenized)
+    inner_grouped = inner_group(space_grouped)
+    indentation_marked = mark_indentation(inner_grouped)
+    return indentation_marked

baron/formatting_grouper.py

@@ -24,6 +24,7 @@ class UnExpectedSpaceToken(BaronError):
     "DOUBLE_SLASH",
     "PLUS",
     "MINUS",
+    "AT",
     "LEFT_SHIFT",
     "RIGHT_SHIFT",
     "AMPER",
@@ -46,6 +47,7 @@ class UnExpectedSpaceToken(BaronError):
     "PLUS_EQUAL",
     "MINUS_EQUAL",
     "STAR_EQUAL",
+    "AT_EQUAL",
     "SLASH_EQUAL",
     "PERCENT_EQUAL",
     "AMPER_EQUAL",
@@ -65,6 +67,8 @@ class UnExpectedSpaceToken(BaronError):
 STRING = (
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "UNICODE_STRING",
     "UNICODE_RAW_STRING",
     "BINARY_STRING",
@@ -87,6 +91,7 @@ class UnExpectedSpaceToken(BaronError):
     "RAISE",
     "EXEC",
     "GLOBAL",
+    "NONLOCAL",
     "PRINT",
     "INDENT",
     "WHILE",

baron/grammator.py

@@ -132,6 +132,7 @@ def simple_stmt_semicolon(pack):
     @pg.production("small_stmt : assert_stmt")
     @pg.production("small_stmt : raise_stmt")
     @pg.production("small_stmt : global_stmt")
+    @pg.production("small_stmt : nonlocal_stmt")
     @pg.production("compound_stmt : if_stmt")
     @pg.production("compound_stmt : while_stmt")
     @pg.production("compound_stmt : for_stmt")
@@ -664,10 +665,12 @@ def strings_string(pack):
     # TODO tests those other kind of strings
     @pg.production("string : STRING")
     @pg.production("string : RAW_STRING")
+    @pg.production("string : INTERPOLATED_STRING")
     @pg.production("string : UNICODE_STRING")
     @pg.production("string : BINARY_STRING")
     @pg.production("string : UNICODE_RAW_STRING")
     @pg.production("string : BINARY_RAW_STRING")
+    @pg.production("string : INTERPOLATED_RAW_STRING")
     def string(pack):
         (string_,) = pack
         return [{

baron/grammator_operators.py

@@ -34,6 +34,7 @@ def augmented_assignment_node(pack):
     @pg.production("augassign_operator : SLASH_EQUAL")
     @pg.production("augassign_operator : PERCENT_EQUAL")
     @pg.production("augassign_operator : AMPER_EQUAL")
+    @pg.production("augassign_operator : AT_EQUAL")
     @pg.production("augassign_operator : VBAR_EQUAL")
     @pg.production("augassign_operator : CIRCUMFLEX_EQUAL")
     @pg.production("augassign_operator : LEFT_SHIFT_EQUAL")
@@ -147,6 +148,7 @@ def comparison_advanced_node(pack):
     @pg.production("term : factor SLASH term")
     @pg.production("term : factor PERCENT term")
     @pg.production("term : factor DOUBLE_SLASH term")
+    @pg.production("term : factor AT term")
     @pg.production("power : atom DOUBLE_STAR factor")
     @pg.production("power : atom DOUBLE_STAR power")
     def binary_operator_node(pack):
@@ -254,6 +256,7 @@ def trailer_getitem_ellipsis(pack):
         }]
 
     @pg.production("subscript : DOT DOT DOT")
+    @pg.production("atom : DOT DOT DOT")
     def subscript_ellipsis(pack):
         (dot1, dot2, dot3) = pack
         return {

baron/grammator_primitives.py

@@ -235,6 +235,16 @@ def global_stmt(pack):
         }
 
 
+    @pg.production("nonlocal_stmt : NONLOCAL names")
+    def nonlocal_stmt(pack):
+        (token, names) = pack
+        return {
+            "type": "nonlocal",
+            "formatting": token.hidden_tokens_after,
+            "value": names,
+        }
+
+
     @pg.production("names : NAME")
     def names_name(pack):
         (name,) = pack

baron/grouper.py

@@ -12,6 +12,7 @@
     ("&", "="),
     ("|", "="),
     ("^", "="),
+    ("@", "="),
     ("/", "/"),
     ("*", "*"),
     ("<", "<"),
@@ -51,9 +52,9 @@ def group_generator(sequence):
             current += next(iterator)
         if current in to_group_keys and matching_found(to_group, current, iterator.show_next()):
             current += next(iterator)
-        if current in list('uUrRbB') and str(iterator.show_next()).startswith(('"', "'")):
+        if current in list('uUfFrRbB') and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
-        if str(current).lower() in ["ur", "br"] and str(iterator.show_next()).startswith(('"', "'")):
+        if str(current).lower() in ["ur", "br", "fr", "rf"] and str(iterator.show_next()).startswith(('"', "'")):
             current += next(iterator)
         if any([re.match(x, current) for x in (r'^\d+[eE]$', r'^\d+\.\d*[eE]$', r'^\.\d+[eE]$')]):
             current += next(iterator)
@@ -95,7 +96,7 @@ def group_generator(sequence):
         if re.match(r'^\d+\.?[eE]$', current) and match_on_next(r'^\d+$', iterator):
             current += next(iterator)
 
-        if re.match(r'^\d*\.?\d*[eE]$', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
+        if re.match(r'^\d*\.?\d*[eE]$', current) and not re.match('[eE]', current) and match_on_next(r'^[-+]$', iterator) and iterator.show_next(2) and re.match(r'^\d+$', iterator.show_next(2)):
             current += next(iterator)
             current += next(iterator)
 

baron/indentation_marker.py

@@ -35,7 +35,7 @@ def get_space(node):
     a ('ENDL', '\n') node - then we return None as a flag value. This is
     maybe not the best behavior but it seems to work for now.
     """
-    if len(node) < 3 or len(node[3]) == 0:
+    if len(node) < 4 or len(node[3]) == 0:
         return None
     return transform_tabs_to_spaces(node[3][0][1])
 

baron/inner_formatting_grouper.py

@@ -35,6 +35,8 @@ class GroupingError(BaronError):
     # TODO test everything bellow
     "STRING",
     "RAW_STRING",
+    "INTERPOLATED_STRING",
+    "INTERPOLATED_RAW_STRING",
     "BINARY_STRING",
     "BINARY_RAW_STRING",
     "UNICODE_STRING",
@@ -67,11 +69,13 @@ class GroupingError(BaronError):
     "NOT",
     "AND",
     "OR",
+    "AT",
     "IF",
     "ELSE",
     "EQUAL",
     "PLUS_EQUAL",
     "MINUS_EQUAL",
+    "AT_EQUAL",
     "STAR_EQUAL",
     "SLASH_EQUAL",
     "PERCENT_EQUAL",
@@ -178,15 +182,4 @@ def group_generator(sequence):
                     debug_file_content += _append_to_debug_file_content(iterator.show_next())
                     current = append_to_token_after(current, [next(iterator)])
 
-
-        if current[0] == "SPACE":
-            debug_file_content = debug_file_content.split("\n")
-            debug_file_content = list(zip(range(1, len(debug_file_content) + 1), debug_file_content))
-            debug_file_content = debug_file_content[-3:]
-            debug_file_content = "\n".join(["%4s %s" % (x[0], x[1]) for x in debug_file_content])
-            debug_file_content += "<--- here"
-            debug_text = "Unexpected '%s' token:\n\n" % current[0].lower() + debug_file_content + "\n\n"
-            debug_text += "Should have been grouped on either %s (before) or %s (after) token." % (debug_previous_token, iterator.show_next())
-            raise UnExpectedFormattingToken(debug_text)
-
         yield current

baron/parser.py

@@ -63,16 +63,23 @@ def build(self):
         table = None
         if os.path.exists(cache_file):
             with open(cache_file) as f:
-                data = json.load(f)
-                stat_result = os.fstat(f.fileno())
-            if (
-                os.name == "nt" or (
-                    stat_result.st_uid == os.getuid() and
-                    stat.S_IMODE(stat_result.st_mode) == 0o0600
-                )
-            ):
-                if self.data_is_valid(g, data):
-                    table = LRTable.from_cache(g, data)
+                try:
+                    data = json.load(f)
+                except:
+                    os.remove(cache_file)
+                    data = None
+
+                if data is not None:
+                    stat_result = os.fstat(f.fileno())
+                    if (
+                        os.name == "nt" or (
+                            stat_result.st_uid == os.getuid() and
+                            stat.S_IMODE(stat_result.st_mode) == 0o0600
+                        )
+                    ):
+                        if self.data_is_valid(g, data):
+                            table = LRTable.from_cache(g, data)
+
         if table is None:
             table = LRTable.from_grammar(g)
             try:

baron/render.py

@@ -508,7 +508,11 @@ def child_by_key(node, key):
             ("formatting", "second_formatting", "as"),
             ("key",        "as",                "as"),
         ],
-
+        "nonlocal": [
+            ("constant",    "nonlocal",         True),
+            ("formatting", "formatting",        True),
+            ("list",        "value",            True),
+        ],
         "del": [
             ("constant",   "del",               True),
             ("formatting", "formatting",        True),

baron/tokenizer.py

@@ -5,7 +5,7 @@
 class UnknowItem(BaronError):
     pass
 
-KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
+KEYWORDS = ("and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "nonlocal", "if", "import", "in", "is", "lambda", "not", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield")
 
 TOKENS = (
     (r'[a-zA-Z_]\w*', 'NAME'),
@@ -20,14 +20,14 @@ class UnknowItem(BaronError):
     (r'\d+\.[jJ]', 'COMPLEX'),
     (r'\d+[jJ]', 'COMPLEX'),
     (r'\d+\.', 'FLOAT'),
-    (r'\d*\.\d+[lL]?', 'FLOAT'),
-    (r'\d+\.\d*[lL]?', 'FLOAT'),
+    (r'\d*[_\d]*\.[_\d]+[lL]?', 'FLOAT'),
+    (r'\d+[_\d]+\.[_\d]*[lL]?', 'FLOAT'),
     (r'\.', 'DOT'),
-    (r'[1-9]+\d*[lL]', 'LONG'),
-    (r'[1-9]+\d*', 'INT'),
-    (r'0[xX][\da-fA-F]+[lL]?', 'HEXA'),
-    (r'(0[oO][0-7]+)|(0[0-7]*)[lL]?', 'OCTA'),
-    (r'0[bB][01]+[lL]?', 'BINARY'),
+    (r'[1-9]+[_\d]*[lL]', 'LONG'),
+    (r'[1-9]+[_\d]*', 'INT'),
+    (r'0[xX][\d_a-fA-F]+[lL]?', 'HEXA'),
+    (r'(0[oO][0-7]+)|(0[0-7_]*)[lL]?', 'OCTA'),
+    (r'0[bB][01_]+[lL]?', 'BINARY'),
     (r'\(', 'LEFT_PARENTHESIS'),
     (r'\)', 'RIGHT_PARENTHESIS'),
     (r':', 'COLON'),
@@ -40,6 +40,7 @@ class UnknowItem(BaronError):
     (r'/', 'SLASH'),
     (r'\|', 'VBAR'),
     (r'&', 'AMPER'),
+    (r'@', 'AT'),
     (r'<', 'LESS'),
     (r'>', 'GREATER'),
     (r'=', 'EQUAL'),
@@ -61,6 +62,7 @@ class UnknowItem(BaronError):
     (r'\*\*', 'DOUBLE_STAR'),
     (r'\+=', 'PLUS_EQUAL'),
     (r'-=', 'MINUS_EQUAL'),
+    (r'@=', 'AT_EQUAL'),
     (r'\*=', 'STAR_EQUAL'),
     (r'/=', 'SLASH_EQUAL'),
     (r'%=', 'PERCENT_EQUAL'),
@@ -78,10 +80,13 @@ class UnknowItem(BaronError):
     (r'(\s|\\\n|\\\r\n)+', 'SPACE'),
     (r'["\'](.|\n|\r)*["\']', 'STRING'),
     (r'[uU]["\'](.|\n|\r)*["\']', 'UNICODE_STRING'),
+    (r'[fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_STRING'),
     (r'[rR]["\'](.|\n|\r)*["\']', 'RAW_STRING'),
     (r'[bB]["\'](.|\n|\r)*["\']', 'BINARY_STRING'),
     (r'[uU][rR]["\'](.|\n|\r)*["\']', 'UNICODE_RAW_STRING'),
     (r'[bB][rR]["\'](.|\n|\r)*["\']', 'BINARY_RAW_STRING'),
+    (r'[fF][rR]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
+    (r'[rR][fF]["\'](.|\n|\r)*["\']', 'INTERPOLATED_RAW_STRING'),
 )