From 1cfbe068fdf3171455cd6a21b178575cbb2856ab Mon Sep 17 00:00:00 2001 From: Evan Hubinger Date: Fri, 10 Nov 2023 02:02:14 -0800 Subject: [PATCH] Use fast grammar methods --- coconut/command/command.py | 9 +- coconut/compiler/compiler.py | 34 +- coconut/compiler/grammar.py | 3600 +++++++++-------- coconut/compiler/util.py | 71 +- coconut/constants.py | 4 +- coconut/terminal.py | 21 +- .../src/cocotest/agnostic/primary_2.coco | 1 + coconut/util.py | 19 + 8 files changed, 1928 insertions(+), 1831 deletions(-) diff --git a/coconut/command/command.py b/coconut/command/command.py index 69e713641..49ac9c4e6 100644 --- a/coconut/command/command.py +++ b/coconut/command/command.py @@ -83,6 +83,7 @@ get_clock_time, first_import_time, ensure_dir, + assert_remove_prefix, ) from coconut.command.util import ( writefile, @@ -324,7 +325,13 @@ def execute_args(self, args, interact=True, original_args=None): # process mypy args and print timing info (must come after compiler setup) if args.mypy is not None: self.set_mypy_args(args.mypy) - logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + if logger.verbose: + logger.log("Grammar init time: " + str(self.comp.grammar_init_time) + " secs / Total init time: " + str(get_clock_time() - first_import_time) + " secs") + for stat_name, (no_copy, yes_copy) in logger.recorded_stats.items(): + if not stat_name.startswith("maybe_copy_"): + continue + name = assert_remove_prefix(stat_name, "maybe_copy_") + logger.printlog("\tGrammar copying stats (" + name + "):", no_copy, "not copied;", yes_copy, "copied") # do compilation, keeping track of compiled filepaths filepaths = [] diff --git a/coconut/compiler/compiler.py b/coconut/compiler/compiler.py index 551462807..1f1ce3c39 100644 --- a/coconut/compiler/compiler.py +++ b/coconut/compiler/compiler.py @@ -2936,13 +2936,18 @@ def item_handle(self, original, loc, tokens): out = "_coconut_iter_getitem(" + out + ", " + trailer[1] + ")" elif trailer[0] == "$(?": pos_args, star_args, base_kwd_args, dubstar_args = self.split_function_call(trailer[1], loc) + has_question_mark = False + needs_complex_partial = False argdict_pairs = [] + last_pos_i = -1 for i, arg in enumerate(pos_args): if arg == "?": has_question_mark = True else: + if last_pos_i != i - 1: + needs_complex_partial = True argdict_pairs.append(str(i) + ": " + arg) pos_kwargs = [] @@ -2950,25 +2955,36 @@ def item_handle(self, original, loc, tokens): for i, arg in enumerate(base_kwd_args): if arg.endswith("=?"): has_question_mark = True + needs_complex_partial = True pos_kwargs.append(arg[:-2]) else: kwd_args.append(arg) - extra_args_str = join_args(star_args, kwd_args, dubstar_args) if not has_question_mark: raise CoconutInternalException("no question mark in question mark partial", trailer[1]) - elif argdict_pairs or pos_kwargs or extra_args_str: + + if needs_complex_partial: + extra_args_str = join_args(star_args, kwd_args, dubstar_args) + if argdict_pairs or pos_kwargs or extra_args_str: + out = ( + "_coconut_complex_partial(" + + out + + ", {" + ", ".join(argdict_pairs) + "}" + + ", " + str(len(pos_args)) + + ", " + tuple_str_of(pos_kwargs, add_quotes=True) + + (", " if extra_args_str else "") + extra_args_str + + ")" + ) + else: + raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) + else: out = ( - "_coconut_complex_partial(" + "_coconut_partial(" + out - + ", {" + ", ".join(argdict_pairs) + "}" - + ", " + str(len(pos_args)) - + ", " + tuple_str_of(pos_kwargs, add_quotes=True) - + (", " if extra_args_str else "") + extra_args_str + + ", " + + join_args([arg for arg in pos_args if arg != "?"], star_args, kwd_args, dubstar_args) + ")" ) - else: - raise CoconutDeferredSyntaxError("a non-? partial application argument is required", loc) else: raise CoconutInternalException("invalid special trailer", trailer[0]) diff --git a/coconut/compiler/grammar.py b/coconut/compiler/grammar.py index cc02add8c..e6f3b6bdb 100644 --- a/coconut/compiler/grammar.py +++ b/coconut/compiler/grammar.py @@ -116,6 +116,7 @@ compile_regex, always_match, caseless_literal, + using_fast_grammar_methods, ) @@ -614,1941 +615,1942 @@ def typedef_op_item_handle(loc, tokens): class Grammar(object): """Coconut grammar specification.""" grammar_init_time = get_clock_time() + with using_fast_grammar_methods(): + + comma = Literal(",") + dubstar = Literal("**") + star = ~dubstar + Literal("*") + at = Literal("@") + arrow = Literal("->") | fixto(Literal("\u2192"), "->") + unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") + colon_eq = Literal(":=") + unsafe_dubcolon = Literal("::") + unsafe_colon = Literal(":") + colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon + lt_colon = Literal("<:") + semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) + multisemicolon = combine(OneOrMore(semicolon)) + eq = Literal("==") + equals = ~eq + ~Literal("=>") + Literal("=") + lbrack = Literal("[") + rbrack = Literal("]") + lbrace = Literal("{") + rbrace = Literal("}") + lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") + rbanana = Literal("|)") + lparen = ~lbanana + Literal("(") + rparen = Literal(")") + unsafe_dot = Literal(".") + dot = ~Literal("..") + unsafe_dot + plus = Literal("+") + minus = ~Literal("->") + Literal("-") + dubslash = Literal("//") + slash = ~dubslash + Literal("/") + pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") + star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") + dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") + back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") + back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") + back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") + none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") + none_star_pipe = ( + Literal("|?*>") + | fixto(Literal("?*\u21a6"), "|?*>") + | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") + ) + none_dubstar_pipe = ( + Literal("|?**>") + | fixto(Literal("?**\u21a6"), "|?**>") + | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") + ) + back_none_pipe = Literal("") + ~Literal("..*") + ~Literal("..?") + Literal("..") + | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") + ) + comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") + comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") + comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") + comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") + comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") + comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") + comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") + comp_back_none_pipe = Literal("") + | fixto(Literal("\u2218?*>"), "..?*>") + | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") + ) + comp_back_none_star_pipe = ( + Literal("<*?..") + | fixto(Literal("<*?\u2218"), "<*?..") + | invalid_syntax("") + | fixto(Literal("\u2218?**>"), "..?**>") + | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") + ) + comp_back_none_dubstar_pipe = ( + Literal("<**?..") + | fixto(Literal("<**?\u2218"), "<**?..") + | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") + bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) + percent = Literal("%") + dollar = Literal("$") + lshift = Literal("<<") | fixto(Literal("\xab"), "<<") + rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") + tilde = Literal("~") + underscore = Literal("_") + pound = Literal("#") + unsafe_backtick = Literal("`") + dubbackslash = Literal("\\\\") + backslash = ~dubbackslash + Literal("\\") + dubquestion = Literal("??") + questionmark = ~dubquestion + Literal("?") + bang = ~Literal("!=") + Literal("!") + + kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) + keyword = kwds.__getitem__ + + except_star_kwd = combine(keyword("except") + star) + kwds["except"] = ~except_star_kwd + keyword("except") + kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") + kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) + + ellipsis = Forward() + ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") + + lt = ( + ~Literal("<<") + + ~Literal("<=") + + ~Literal("<|") + + ~Literal("<..") + + ~Literal("<*") + + ~Literal("<:") + + Literal("<") + | fixto(Literal("\u228a"), "<") + ) + gt = ( + ~Literal(">>") + + ~Literal(">=") + + Literal(">") + | fixto(Literal("\u228b"), ">") + ) + le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") + ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") + ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") + + mul_star = star | fixto(Literal("\xd7"), "*") + exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") + neg_minus = ( + minus + | fixto(Literal("\u207b"), "-") + ) + sub_minus = ( + minus + | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") + ) + div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") + div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") + matrix_at = at + + test = Forward() + test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) + test_no_infix, backtick = disable_inside(test, unsafe_backtick) + + base_name_regex = r"" + for no_kwd in keyword_vars + const_vars: + base_name_regex += r"(?!" + no_kwd + r"\b)" + # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} + base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" + base_name = regex_item(base_name_regex) + + refname = Forward() + setname = Forward() + classname = Forward() + name_ref = combine(Optional(backslash) + base_name) + unsafe_name = combine(Optional(backslash.suppress()) + base_name) + + # use unsafe_name for dotted components since name should only be used for base names + dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) + dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) + unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) + must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) + + integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) + binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) + octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) + hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) + + imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") + basenum = combine( + integer + dot + Optional(integer) + | Optional(integer) + dot + integer + ) | integer + sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) + numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) + imag_num = combine(numitem + imag_j) + bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) + oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) + hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) + number = ( + bin_num + | oct_num + | hex_num + | imag_num + | numitem + ) + # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError + num_atom = addspace(number + Optional(condense(dot + unsafe_name))) + + moduledoc_item = Forward() + unwrap = Literal(unwrapper) + comment = Forward() + comment_tokens = combine(pound + integer + unwrap) + string_item = ( + combine(Literal(strwrapper) + integer + unwrap) + | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) + ) - comma = Literal(",") - dubstar = Literal("**") - star = ~dubstar + Literal("*") - at = Literal("@") - arrow = Literal("->") | fixto(Literal("\u2192"), "->") - unsafe_fat_arrow = Literal("=>") | fixto(Literal("\u21d2"), "=>") - colon_eq = Literal(":=") - unsafe_dubcolon = Literal("::") - unsafe_colon = Literal(":") - colon = ~unsafe_dubcolon + ~colon_eq + unsafe_colon - lt_colon = Literal("<:") - semicolon = Literal(";") | invalid_syntax("\u037e", "invalid Greek question mark instead of semicolon", greedy=True) - multisemicolon = combine(OneOrMore(semicolon)) - eq = Literal("==") - equals = ~eq + ~Literal("=>") + Literal("=") - lbrack = Literal("[") - rbrack = Literal("]") - lbrace = Literal("{") - rbrace = Literal("}") - lbanana = ~Literal("(|)") + ~Literal("(|>") + ~Literal("(|*") + ~Literal("(|?") + Literal("(|") - rbanana = Literal("|)") - lparen = ~lbanana + Literal("(") - rparen = Literal(")") - unsafe_dot = Literal(".") - dot = ~Literal("..") + unsafe_dot - plus = Literal("+") - minus = ~Literal("->") + Literal("-") - dubslash = Literal("//") - slash = ~dubslash + Literal("/") - pipe = Literal("|>") | fixto(Literal("\u21a6"), "|>") - star_pipe = Literal("|*>") | fixto(Literal("*\u21a6"), "|*>") - dubstar_pipe = Literal("|**>") | fixto(Literal("**\u21a6"), "|**>") - back_pipe = Literal("<|") | fixto(Literal("\u21a4"), "<|") - back_star_pipe = Literal("<*|") | ~Literal("\u21a4**") + fixto(Literal("\u21a4*"), "<*|") - back_dubstar_pipe = Literal("<**|") | fixto(Literal("\u21a4**"), "<**|") - none_pipe = Literal("|?>") | fixto(Literal("?\u21a6"), "|?>") - none_star_pipe = ( - Literal("|?*>") - | fixto(Literal("?*\u21a6"), "|?*>") - | invalid_syntax("|*?>", "Coconut's None-aware forward multi-arg pipe is '|?*>', not '|*?>'") - ) - none_dubstar_pipe = ( - Literal("|?**>") - | fixto(Literal("?**\u21a6"), "|?**>") - | invalid_syntax("|**?>", "Coconut's None-aware forward keyword pipe is '|?**>', not '|**?>'") - ) - back_none_pipe = Literal("") + ~Literal("..*") + ~Literal("..?") + Literal("..") - | ~Literal("\u2218>") + ~Literal("\u2218*") + ~Literal("\u2218?") + fixto(Literal("\u2218"), "..") - ) - comp_pipe = Literal("..>") | fixto(Literal("\u2218>"), "..>") - comp_back_pipe = Literal("<..") | fixto(Literal("<\u2218"), "<..") - comp_star_pipe = Literal("..*>") | fixto(Literal("\u2218*>"), "..*>") - comp_back_star_pipe = Literal("<*..") | fixto(Literal("<*\u2218"), "<*..") - comp_dubstar_pipe = Literal("..**>") | fixto(Literal("\u2218**>"), "..**>") - comp_back_dubstar_pipe = Literal("<**..") | fixto(Literal("<**\u2218"), "<**..") - comp_none_pipe = Literal("..?>") | fixto(Literal("\u2218?>"), "..?>") - comp_back_none_pipe = Literal("") - | fixto(Literal("\u2218?*>"), "..?*>") - | invalid_syntax("..*?>", "Coconut's None-aware forward multi-arg composition pipe is '..?*>', not '..*?>'") - ) - comp_back_none_star_pipe = ( - Literal("<*?..") - | fixto(Literal("<*?\u2218"), "<*?..") - | invalid_syntax("") - | fixto(Literal("\u2218?**>"), "..?**>") - | invalid_syntax("..**?>", "Coconut's None-aware forward keyword composition pipe is '..?**>', not '..**?>'") - ) - comp_back_none_dubstar_pipe = ( - Literal("<**?..") - | fixto(Literal("<**?\u2218"), "<**?..") - | invalid_syntax("") + ~Literal("|*") + Literal("|") | fixto(Literal("\u222a"), "|") - bar = ~rbanana + unsafe_bar | invalid_syntax("\xa6", "invalid broken bar character", greedy=True) - percent = Literal("%") - dollar = Literal("$") - lshift = Literal("<<") | fixto(Literal("\xab"), "<<") - rshift = Literal(">>") | fixto(Literal("\xbb"), ">>") - tilde = Literal("~") - underscore = Literal("_") - pound = Literal("#") - unsafe_backtick = Literal("`") - dubbackslash = Literal("\\\\") - backslash = ~dubbackslash + Literal("\\") - dubquestion = Literal("??") - questionmark = ~dubquestion + Literal("?") - bang = ~Literal("!=") + Literal("!") - - kwds = keydefaultdict(partial(base_keyword, explicit_prefix=colon)) - keyword = kwds.__getitem__ - - except_star_kwd = combine(keyword("except") + star) - kwds["except"] = ~except_star_kwd + keyword("except") - kwds["lambda"] = keyword("lambda") | fixto(keyword("\u03bb"), "lambda") - kwds["operator"] = base_keyword("operator", explicit_prefix=colon, require_whitespace=True) - - ellipsis = Forward() - ellipsis_tokens = Literal("...") | fixto(Literal("\u2026"), "...") - - lt = ( - ~Literal("<<") - + ~Literal("<=") - + ~Literal("<|") - + ~Literal("<..") - + ~Literal("<*") - + ~Literal("<:") - + Literal("<") - | fixto(Literal("\u228a"), "<") - ) - gt = ( - ~Literal(">>") - + ~Literal(">=") - + Literal(">") - | fixto(Literal("\u228b"), ">") - ) - le = Literal("<=") | fixto(Literal("\u2264") | Literal("\u2286"), "<=") - ge = Literal(">=") | fixto(Literal("\u2265") | Literal("\u2287"), ">=") - ne = Literal("!=") | fixto(Literal("\xac=") | Literal("\u2260"), "!=") - - mul_star = star | fixto(Literal("\xd7"), "*") - exp_dubstar = dubstar | fixto(Literal("\u2191"), "**") - neg_minus = ( - minus - | fixto(Literal("\u207b"), "-") - ) - sub_minus = ( - minus - | invalid_syntax("\u207b", "U+207b is only for negation, not subtraction") - ) - div_slash = slash | fixto(Literal("\xf7") + ~slash, "/") - div_dubslash = dubslash | fixto(combine(Literal("\xf7") + slash), "//") - matrix_at = at - - test = Forward() - test_no_chain, dubcolon = disable_inside(test, unsafe_dubcolon) - test_no_infix, backtick = disable_inside(test, unsafe_backtick) - - base_name_regex = r"" - for no_kwd in keyword_vars + const_vars: - base_name_regex += r"(?!" + no_kwd + r"\b)" - # we disallow ['"{] after to not match the "b" in b"" or the "s" in s{} - base_name_regex += r"(?![0-9])\w+\b(?![{" + strwrapper + r"])" - base_name = regex_item(base_name_regex) - - refname = Forward() - setname = Forward() - classname = Forward() - name_ref = combine(Optional(backslash) + base_name) - unsafe_name = combine(Optional(backslash.suppress()) + base_name) - - # use unsafe_name for dotted components since name should only be used for base names - dotted_refname = condense(refname + ZeroOrMore(dot + unsafe_name)) - dotted_setname = condense(setname + ZeroOrMore(dot + unsafe_name)) - unsafe_dotted_name = condense(unsafe_name + ZeroOrMore(dot + unsafe_name)) - must_be_dotted_name = condense(refname + OneOrMore(dot + unsafe_name)) - - integer = combine(Word(nums) + ZeroOrMore(underscore.suppress() + Word(nums))) - binint = combine(Word("01") + ZeroOrMore(underscore.suppress() + Word("01"))) - octint = combine(Word("01234567") + ZeroOrMore(underscore.suppress() + Word("01234567"))) - hexint = combine(Word(hexnums) + ZeroOrMore(underscore.suppress() + Word(hexnums))) - - imag_j = caseless_literal("j") | fixto(caseless_literal("i", suppress=True), "j") - basenum = combine( - integer + dot + Optional(integer) - | Optional(integer) + dot + integer - ) | integer - sci_e = combine((caseless_literal("e") | fixto(Literal("\u23e8"), "e")) + Optional(plus | neg_minus)) - numitem = ~(Literal("0") + Word(nums + "_", exact=1)) + combine(basenum + Optional(sci_e + integer)) - imag_num = combine(numitem + imag_j) - bin_num = combine(caseless_literal("0b") + Optional(underscore.suppress()) + binint) - oct_num = combine(caseless_literal("0o") + Optional(underscore.suppress()) + octint) - hex_num = combine(caseless_literal("0x") + Optional(underscore.suppress()) + hexint) - number = ( - bin_num - | oct_num - | hex_num - | imag_num - | numitem - ) - # make sure that this gets addspaced not condensed so it doesn't produce a SyntaxError - num_atom = addspace(number + Optional(condense(dot + unsafe_name))) - - moduledoc_item = Forward() - unwrap = Literal(unwrapper) - comment = Forward() - comment_tokens = combine(pound + integer + unwrap) - string_item = ( - combine(Literal(strwrapper) + integer + unwrap) - | invalid_syntax(("\u201c", "\u201d", "\u2018", "\u2019"), "invalid unicode quotation mark; strings must use \" or '", greedy=True) - ) - - xonsh_command = Forward() - passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command - passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) - - endline = Forward() - endline_ref = condense(OneOrMore(Literal("\n"))) - lineitem = ZeroOrMore(comment) + endline - newline = condense(OneOrMore(lineitem)) - # rparen handles simple stmts ending parenthesized stmt lambdas - end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) - - start_marker = StringStart() - moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) - end_marker = StringEnd() - indent = Literal(openindent) - dedent = Literal(closeindent) - - u_string = Forward() - f_string = Forward() - - bit_b = caseless_literal("b") - raw_r = caseless_literal("r") - unicode_u = caseless_literal("u", suppress=True) - format_f = caseless_literal("f", suppress=True) - - string = combine(Optional(raw_r) + string_item) - # Python 2 only supports br"..." not rb"..." - b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) - # ur"..."/ru"..." strings are not suppored in Python 3 - u_string_ref = combine(unicode_u + string_item) - f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) - nonbf_string = string | u_string - nonb_string = nonbf_string | f_string - any_string = nonb_string | b_string - moduledoc = any_string + newline - docstring = condense(moduledoc) - - pipe_augassign = ( - combine(pipe + equals) - | combine(star_pipe + equals) - | combine(dubstar_pipe + equals) - | combine(back_pipe + equals) - | combine(back_star_pipe + equals) - | combine(back_dubstar_pipe + equals) - | combine(none_pipe + equals) - | combine(none_star_pipe + equals) - | combine(none_dubstar_pipe + equals) - | combine(back_none_pipe + equals) - | combine(back_none_star_pipe + equals) - | combine(back_none_dubstar_pipe + equals) - ) - augassign = ( - pipe_augassign - | combine(comp_pipe + equals) - | combine(dotdot + equals) - | combine(comp_back_pipe + equals) - | combine(comp_star_pipe + equals) - | combine(comp_back_star_pipe + equals) - | combine(comp_dubstar_pipe + equals) - | combine(comp_back_dubstar_pipe + equals) - | combine(comp_none_pipe + equals) - | combine(comp_back_none_pipe + equals) - | combine(comp_none_star_pipe + equals) - | combine(comp_back_none_star_pipe + equals) - | combine(comp_none_dubstar_pipe + equals) - | combine(comp_back_none_dubstar_pipe + equals) - | combine(unsafe_dubcolon + equals) - | combine(div_dubslash + equals) - | combine(div_slash + equals) - | combine(exp_dubstar + equals) - | combine(mul_star + equals) - | combine(plus + equals) - | combine(sub_minus + equals) - | combine(percent + equals) - | combine(amp + equals) - | combine(bar + equals) - | combine(caret + equals) - | combine(lshift + equals) - | combine(rshift + equals) - | combine(matrix_at + equals) - | combine(dubquestion + equals) - ) + xonsh_command = Forward() + passthrough_item = combine((backslash | Literal(early_passthrough_wrapper)) + integer + unwrap) | xonsh_command + passthrough_block = combine(fixto(dubbackslash, "\\") + integer + unwrap) + + endline = Forward() + endline_ref = condense(OneOrMore(Literal("\n"))) + lineitem = ZeroOrMore(comment) + endline + newline = condense(OneOrMore(lineitem)) + # rparen handles simple stmts ending parenthesized stmt lambdas + end_simple_stmt_item = FollowedBy(semicolon | newline | rparen) + + start_marker = StringStart() + moduledoc_marker = condense(ZeroOrMore(lineitem) - Optional(moduledoc_item)) + end_marker = StringEnd() + indent = Literal(openindent) + dedent = Literal(closeindent) + + u_string = Forward() + f_string = Forward() + + bit_b = caseless_literal("b") + raw_r = caseless_literal("r") + unicode_u = caseless_literal("u", suppress=True) + format_f = caseless_literal("f", suppress=True) + + string = combine(Optional(raw_r) + string_item) + # Python 2 only supports br"..." not rb"..." + b_string = combine((bit_b + Optional(raw_r) | fixto(raw_r + bit_b, "br")) + string_item) + # ur"..."/ru"..." strings are not suppored in Python 3 + u_string_ref = combine(unicode_u + string_item) + f_string_tokens = combine((format_f + Optional(raw_r) | raw_r + format_f) + string_item) + nonbf_string = string | u_string + nonb_string = nonbf_string | f_string + any_string = nonb_string | b_string + moduledoc = any_string + newline + docstring = condense(moduledoc) + + pipe_augassign = ( + combine(pipe + equals) + | combine(star_pipe + equals) + | combine(dubstar_pipe + equals) + | combine(back_pipe + equals) + | combine(back_star_pipe + equals) + | combine(back_dubstar_pipe + equals) + | combine(none_pipe + equals) + | combine(none_star_pipe + equals) + | combine(none_dubstar_pipe + equals) + | combine(back_none_pipe + equals) + | combine(back_none_star_pipe + equals) + | combine(back_none_dubstar_pipe + equals) + ) + augassign = ( + pipe_augassign + | combine(comp_pipe + equals) + | combine(dotdot + equals) + | combine(comp_back_pipe + equals) + | combine(comp_star_pipe + equals) + | combine(comp_back_star_pipe + equals) + | combine(comp_dubstar_pipe + equals) + | combine(comp_back_dubstar_pipe + equals) + | combine(comp_none_pipe + equals) + | combine(comp_back_none_pipe + equals) + | combine(comp_none_star_pipe + equals) + | combine(comp_back_none_star_pipe + equals) + | combine(comp_none_dubstar_pipe + equals) + | combine(comp_back_none_dubstar_pipe + equals) + | combine(unsafe_dubcolon + equals) + | combine(div_dubslash + equals) + | combine(div_slash + equals) + | combine(exp_dubstar + equals) + | combine(mul_star + equals) + | combine(plus + equals) + | combine(sub_minus + equals) + | combine(percent + equals) + | combine(amp + equals) + | combine(bar + equals) + | combine(caret + equals) + | combine(lshift + equals) + | combine(rshift + equals) + | combine(matrix_at + equals) + | combine(dubquestion + equals) + ) - comp_op = ( - le | ge | ne | lt | gt | eq - | addspace(keyword("not") + keyword("in")) - | keyword("in") - | addspace(keyword("is") + keyword("not")) - | keyword("is") - ) + comp_op = ( + le | ge | ne | lt | gt | eq + | addspace(keyword("not") + keyword("in")) + | keyword("in") + | addspace(keyword("is") + keyword("not")) + | keyword("is") + ) - atom_item = Forward() - expr = Forward() - star_expr = Forward() - dubstar_expr = Forward() - comp_for = Forward() - test_no_cond = Forward() - infix_op = Forward() - namedexpr_test = Forward() - # for namedexpr locations only supported in Python 3.10 - new_namedexpr_test = Forward() - lambdef = Forward() - - typedef = Forward() - typedef_default = Forward() - unsafe_typedef_default = Forward() - typedef_test = Forward() - typedef_tuple = Forward() - typedef_ellipsis = Forward() - typedef_op_item = Forward() - - negable_atom_item = condense(Optional(neg_minus) + atom_item) - - testlist = itemlist(test, comma, suppress_trailing=False) - testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) - new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) - - testlist_star_expr = Forward() - testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) - testlist_star_namedexpr = Forward() - testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) - # for testlist_star_expr locations only supported in Python 3.9 - new_testlist_star_expr = Forward() - new_testlist_star_expr_ref = testlist_star_expr - - yield_from = Forward() - dict_comp = Forward() - dict_literal = Forward() - yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) - yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test - yield_expr = yield_from | yield_classic - dict_comp_ref = lbrace.suppress() + ( - test + colon.suppress() + test + comp_for - | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") - ) + rbrace.suppress() - dict_literal_ref = ( - lbrace.suppress() - + Optional(tokenlist( - Group(test + colon + test) - | dubstar_expr, - comma, - )) - + rbrace.suppress() - ) - test_expr = yield_expr | testlist_star_expr - - base_op_item = ( - # must go dubstar then star then no star - fixto(dubstar_pipe, "_coconut_dubstar_pipe") - | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") - | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") - | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") - | fixto(star_pipe, "_coconut_star_pipe") - | fixto(back_star_pipe, "_coconut_back_star_pipe") - | fixto(none_star_pipe, "_coconut_none_star_pipe") - | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") - | fixto(pipe, "_coconut_pipe") - | fixto(back_pipe, "_coconut_back_pipe") - | fixto(none_pipe, "_coconut_none_pipe") - | fixto(back_none_pipe, "_coconut_back_none_pipe") - - # must go dubstar then star then no star - | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") - | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") - | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") - | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") - | fixto(comp_star_pipe, "_coconut_forward_star_compose") - | fixto(comp_back_star_pipe, "_coconut_back_star_compose") - | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") - | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") - | fixto(comp_pipe, "_coconut_forward_compose") - | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") - | fixto(comp_none_pipe, "_coconut_forward_none_compose") - | fixto(comp_back_none_pipe, "_coconut_back_none_compose") - - # neg_minus must come after minus - | fixto(minus, "_coconut_minus") - | fixto(neg_minus, "_coconut.operator.neg") - - | fixto(keyword("assert"), "_coconut_assert") - | fixto(keyword("raise"), "_coconut_raise") - | fixto(keyword("and"), "_coconut_bool_and") - | fixto(keyword("or"), "_coconut_bool_or") - | fixto(comma, "_coconut_comma_op") - | fixto(dubquestion, "_coconut_none_coalesce") - | fixto(dot, "_coconut.getattr") - | fixto(unsafe_dubcolon, "_coconut.itertools.chain") - | fixto(dollar, "_coconut_partial") - | fixto(exp_dubstar, "_coconut.operator.pow") - | fixto(mul_star, "_coconut.operator.mul") - | fixto(div_dubslash, "_coconut.operator.floordiv") - | fixto(div_slash, "_coconut.operator.truediv") - | fixto(percent, "_coconut.operator.mod") - | fixto(plus, "_coconut.operator.add") - | fixto(amp, "_coconut.operator.and_") - | fixto(caret, "_coconut.operator.xor") - | fixto(unsafe_bar, "_coconut.operator.or_") - | fixto(lshift, "_coconut.operator.lshift") - | fixto(rshift, "_coconut.operator.rshift") - | fixto(lt, "_coconut.operator.lt") - | fixto(gt, "_coconut.operator.gt") - | fixto(eq, "_coconut.operator.eq") - | fixto(le, "_coconut.operator.le") - | fixto(ge, "_coconut.operator.ge") - | fixto(ne, "_coconut.operator.ne") - | fixto(tilde, "_coconut.operator.inv") - | fixto(matrix_at, "_coconut_matmul") - | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") - | fixto(keyword("not") + keyword("in"), "_coconut_not_in") - - # must come after is not / not in - | fixto(keyword("not"), "_coconut.operator.not_") - | fixto(keyword("is"), "_coconut.operator.is_") - | fixto(keyword("in"), "_coconut_in") - ) - partialable_op = base_op_item | infix_op - partial_op_item_tokens = ( - labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") - | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") - ) - partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) - op_item = ( - typedef_op_item - | partial_op_item - | base_op_item - ) + atom_item = Forward() + expr = Forward() + star_expr = Forward() + dubstar_expr = Forward() + comp_for = Forward() + test_no_cond = Forward() + infix_op = Forward() + namedexpr_test = Forward() + # for namedexpr locations only supported in Python 3.10 + new_namedexpr_test = Forward() + lambdef = Forward() + + typedef = Forward() + typedef_default = Forward() + unsafe_typedef_default = Forward() + typedef_test = Forward() + typedef_tuple = Forward() + typedef_ellipsis = Forward() + typedef_op_item = Forward() + + negable_atom_item = condense(Optional(neg_minus) + atom_item) + + testlist = itemlist(test, comma, suppress_trailing=False) + testlist_has_comma = addspace(OneOrMore(condense(test + comma)) + Optional(test)) + new_namedexpr_testlist_has_comma = addspace(OneOrMore(condense(new_namedexpr_test + comma)) + Optional(test)) + + testlist_star_expr = Forward() + testlist_star_expr_ref = tokenlist(Group(test) | star_expr, comma, suppress=False) + testlist_star_namedexpr = Forward() + testlist_star_namedexpr_tokens = tokenlist(Group(namedexpr_test) | star_expr, comma, suppress=False) + # for testlist_star_expr locations only supported in Python 3.9 + new_testlist_star_expr = Forward() + new_testlist_star_expr_ref = testlist_star_expr + + yield_from = Forward() + dict_comp = Forward() + dict_literal = Forward() + yield_classic = addspace(keyword("yield") + Optional(new_testlist_star_expr)) + yield_from_ref = keyword("yield").suppress() + keyword("from").suppress() + test + yield_expr = yield_from | yield_classic + dict_comp_ref = lbrace.suppress() + ( + test + colon.suppress() + test + comp_for + | invalid_syntax(dubstar_expr + comp_for, "dict unpacking cannot be used in dict comprehension") + ) + rbrace.suppress() + dict_literal_ref = ( + lbrace.suppress() + + Optional(tokenlist( + Group(test + colon + test) + | dubstar_expr, + comma, + )) + + rbrace.suppress() + ) + test_expr = yield_expr | testlist_star_expr + + base_op_item = ( + # must go dubstar then star then no star + fixto(dubstar_pipe, "_coconut_dubstar_pipe") + | fixto(back_dubstar_pipe, "_coconut_back_dubstar_pipe") + | fixto(none_dubstar_pipe, "_coconut_none_dubstar_pipe") + | fixto(back_none_dubstar_pipe, "_coconut_back_none_dubstar_pipe") + | fixto(star_pipe, "_coconut_star_pipe") + | fixto(back_star_pipe, "_coconut_back_star_pipe") + | fixto(none_star_pipe, "_coconut_none_star_pipe") + | fixto(back_none_star_pipe, "_coconut_back_none_star_pipe") + | fixto(pipe, "_coconut_pipe") + | fixto(back_pipe, "_coconut_back_pipe") + | fixto(none_pipe, "_coconut_none_pipe") + | fixto(back_none_pipe, "_coconut_back_none_pipe") + + # must go dubstar then star then no star + | fixto(comp_dubstar_pipe, "_coconut_forward_dubstar_compose") + | fixto(comp_back_dubstar_pipe, "_coconut_back_dubstar_compose") + | fixto(comp_none_dubstar_pipe, "_coconut_forward_none_dubstar_compose") + | fixto(comp_back_none_dubstar_pipe, "_coconut_back_none_dubstar_compose") + | fixto(comp_star_pipe, "_coconut_forward_star_compose") + | fixto(comp_back_star_pipe, "_coconut_back_star_compose") + | fixto(comp_none_star_pipe, "_coconut_forward_none_star_compose") + | fixto(comp_back_none_star_pipe, "_coconut_back_none_star_compose") + | fixto(comp_pipe, "_coconut_forward_compose") + | fixto(dotdot | comp_back_pipe, "_coconut_back_compose") + | fixto(comp_none_pipe, "_coconut_forward_none_compose") + | fixto(comp_back_none_pipe, "_coconut_back_none_compose") + + # neg_minus must come after minus + | fixto(minus, "_coconut_minus") + | fixto(neg_minus, "_coconut.operator.neg") + + | fixto(keyword("assert"), "_coconut_assert") + | fixto(keyword("raise"), "_coconut_raise") + | fixto(keyword("and"), "_coconut_bool_and") + | fixto(keyword("or"), "_coconut_bool_or") + | fixto(comma, "_coconut_comma_op") + | fixto(dubquestion, "_coconut_none_coalesce") + | fixto(dot, "_coconut.getattr") + | fixto(unsafe_dubcolon, "_coconut.itertools.chain") + | fixto(dollar, "_coconut_partial") + | fixto(exp_dubstar, "_coconut.operator.pow") + | fixto(mul_star, "_coconut.operator.mul") + | fixto(div_dubslash, "_coconut.operator.floordiv") + | fixto(div_slash, "_coconut.operator.truediv") + | fixto(percent, "_coconut.operator.mod") + | fixto(plus, "_coconut.operator.add") + | fixto(amp, "_coconut.operator.and_") + | fixto(caret, "_coconut.operator.xor") + | fixto(unsafe_bar, "_coconut.operator.or_") + | fixto(lshift, "_coconut.operator.lshift") + | fixto(rshift, "_coconut.operator.rshift") + | fixto(lt, "_coconut.operator.lt") + | fixto(gt, "_coconut.operator.gt") + | fixto(eq, "_coconut.operator.eq") + | fixto(le, "_coconut.operator.le") + | fixto(ge, "_coconut.operator.ge") + | fixto(ne, "_coconut.operator.ne") + | fixto(tilde, "_coconut.operator.inv") + | fixto(matrix_at, "_coconut_matmul") + | fixto(keyword("is") + keyword("not"), "_coconut.operator.is_not") + | fixto(keyword("not") + keyword("in"), "_coconut_not_in") + + # must come after is not / not in + | fixto(keyword("not"), "_coconut.operator.not_") + | fixto(keyword("is"), "_coconut.operator.is_") + | fixto(keyword("in"), "_coconut_in") + ) + partialable_op = base_op_item | infix_op + partial_op_item_tokens = ( + labeled_group(dot.suppress() + partialable_op + test_no_infix, "right partial") + | labeled_group(test_no_infix + partialable_op + dot.suppress(), "left partial") + ) + partial_op_item = attach(partial_op_item_tokens, partial_op_item_handle) + op_item = ( + typedef_op_item + | partial_op_item + | base_op_item + ) - partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() - - # we include (var)arg_comma to ensure the pattern matches the whole arg - arg_comma = comma | fixto(FollowedBy(rparen), "") - setarg_comma = arg_comma | fixto(FollowedBy(colon), "") - typedef_ref = setname + colon.suppress() + typedef_test + arg_comma - default = condense(equals + test) - unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) - typedef_default_ref = unsafe_typedef_default_ref + arg_comma - tfpdef = typedef | condense(setname + arg_comma) - tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) - - star_sep_arg = Forward() - star_sep_arg_ref = condense(star + arg_comma) - star_sep_setarg = Forward() - star_sep_setarg_ref = condense(star + setarg_comma) - - slash_sep_arg = Forward() - slash_sep_arg_ref = condense(slash + arg_comma) - slash_sep_setarg = Forward() - slash_sep_setarg_ref = condense(slash + setarg_comma) - - just_star = star + rparen - just_slash = slash + rparen - just_op = just_star | just_slash - - match = Forward() - args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with arg_comma - (star | dubstar) + tfpdef - | star_sep_arg - | slash_sep_arg - | tfpdef_default + partial_op_atom_tokens = lparen.suppress() + partial_op_item_tokens + rparen.suppress() + + # we include (var)arg_comma to ensure the pattern matches the whole arg + arg_comma = comma | fixto(FollowedBy(rparen), "") + setarg_comma = arg_comma | fixto(FollowedBy(colon), "") + typedef_ref = setname + colon.suppress() + typedef_test + arg_comma + default = condense(equals + test) + unsafe_typedef_default_ref = setname + colon.suppress() + typedef_test + Optional(default) + typedef_default_ref = unsafe_typedef_default_ref + arg_comma + tfpdef = typedef | condense(setname + arg_comma) + tfpdef_default = typedef_default | condense(setname + Optional(default) + arg_comma) + + star_sep_arg = Forward() + star_sep_arg_ref = condense(star + arg_comma) + star_sep_setarg = Forward() + star_sep_setarg_ref = condense(star + setarg_comma) + + slash_sep_arg = Forward() + slash_sep_arg_ref = condense(slash + arg_comma) + slash_sep_setarg = Forward() + slash_sep_setarg_ref = condense(slash + setarg_comma) + + just_star = star + rparen + just_slash = slash + rparen + just_op = just_star | just_slash + + match = Forward() + args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with arg_comma + (star | dubstar) + tfpdef + | star_sep_arg + | slash_sep_arg + | tfpdef_default + ) ) ) ) - ) - parameters = condense(lparen + args_list + rparen) - set_args_list = ( - ~just_op - + addspace( - ZeroOrMore( - condense( - # everything here must end with setarg_comma - (star | dubstar) + setname + setarg_comma - | star_sep_setarg - | slash_sep_setarg - | setname + Optional(default) + setarg_comma + parameters = condense(lparen + args_list + rparen) + set_args_list = ( + ~just_op + + addspace( + ZeroOrMore( + condense( + # everything here must end with setarg_comma + (star | dubstar) + setname + setarg_comma + | star_sep_setarg + | slash_sep_setarg + | setname + Optional(default) + setarg_comma + ) ) ) ) - ) - match_args_list = Group(Optional( - tokenlist( - Group( - (star | dubstar) + match - | star # not star_sep because pattern-matching can handle star separators on any Python version - | slash # not slash_sep as above - | match + Optional(equals.suppress() + test) - ), - comma, + match_args_list = Group(Optional( + tokenlist( + Group( + (star | dubstar) + match + | star # not star_sep because pattern-matching can handle star separators on any Python version + | slash # not slash_sep as above + | match + Optional(equals.suppress() + test) + ), + comma, + ) + )) + + call_item = ( + unsafe_name + default + | dubstar + test + | star + test + | ellipsis_tokens + equals.suppress() + refname + | namedexpr_test + ) + function_call_tokens = lparen.suppress() + ( + # everything here must end with rparen + rparen.suppress() + | tokenlist(Group(call_item), comma) + rparen.suppress() + | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() + | Group(op_item) + rparen.suppress() + ) + function_call = Forward() + questionmark_call_tokens = Group( + tokenlist( + Group( + questionmark + | unsafe_name + condense(equals + questionmark) + | call_item + ), + comma, + ) + ) + methodcaller_args = ( + itemlist(condense(call_item), comma) + | op_item ) - )) - call_item = ( - unsafe_name + default - | dubstar + test - | star + test - | ellipsis_tokens + equals.suppress() + refname - | namedexpr_test - ) - function_call_tokens = lparen.suppress() + ( - # everything here must end with rparen - rparen.suppress() - | tokenlist(Group(call_item), comma) + rparen.suppress() - | Group(attach(addspace(test + comp_for), add_parens_handle)) + rparen.suppress() - | Group(op_item) + rparen.suppress() - ) - function_call = Forward() - questionmark_call_tokens = Group( - tokenlist( + subscript_star = Forward() + subscript_star_ref = star + slicetest = Optional(test_no_chain) + sliceop = condense(unsafe_colon + slicetest) + subscript = condense( + slicetest + sliceop + Optional(sliceop) + | Optional(subscript_star) + test + ) + subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test + + slicetestgroup = Optional(test_no_chain, default="") + sliceopgroup = unsafe_colon.suppress() + slicetestgroup + subscriptgroup = attach( + slicetestgroup + sliceopgroup + Optional(sliceopgroup) + | test, + subscriptgroup_handle, + ) + subscriptgrouplist = itemlist(subscriptgroup, comma) + + anon_namedtuple = Forward() + maybe_typedef = Optional(colon.suppress() + typedef_test) + anon_namedtuple_ref = tokenlist( Group( - questionmark - | unsafe_name + condense(equals + questionmark) - | call_item + unsafe_name + maybe_typedef + equals.suppress() + test + | ellipsis_tokens + maybe_typedef + equals.suppress() + refname ), comma, ) - ) - methodcaller_args = ( - itemlist(condense(call_item), comma) - | op_item - ) - - subscript_star = Forward() - subscript_star_ref = star - slicetest = Optional(test_no_chain) - sliceop = condense(unsafe_colon + slicetest) - subscript = condense( - slicetest + sliceop + Optional(sliceop) - | Optional(subscript_star) + test - ) - subscriptlist = itemlist(subscript, comma, suppress_trailing=False) | new_namedexpr_test - - slicetestgroup = Optional(test_no_chain, default="") - sliceopgroup = unsafe_colon.suppress() + slicetestgroup - subscriptgroup = attach( - slicetestgroup + sliceopgroup + Optional(sliceopgroup) - | test, - subscriptgroup_handle, - ) - subscriptgrouplist = itemlist(subscriptgroup, comma) - - anon_namedtuple = Forward() - maybe_typedef = Optional(colon.suppress() + typedef_test) - anon_namedtuple_ref = tokenlist( - Group( - unsafe_name + maybe_typedef + equals.suppress() + test - | ellipsis_tokens + maybe_typedef + equals.suppress() + refname - ), - comma, - ) - comprehension_expr = ( - addspace(namedexpr_test + comp_for) - | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") - ) - paren_atom = condense( - lparen + ( - # everything here must end with rparen - rparen - | yield_expr + rparen - | comprehension_expr + rparen - | testlist_star_namedexpr + rparen - | op_item + rparen - | anon_namedtuple + rparen - ) | ( - lparen.suppress() - + typedef_tuple - + rparen.suppress() + comprehension_expr = ( + addspace(namedexpr_test + comp_for) + | invalid_syntax(star_expr + comp_for, "iterable unpacking cannot be used in comprehension") + ) + paren_atom = condense( + lparen + ( + # everything here must end with rparen + rparen + | yield_expr + rparen + | comprehension_expr + rparen + | testlist_star_namedexpr + rparen + | op_item + rparen + | anon_namedtuple + rparen + ) | ( + lparen.suppress() + + typedef_tuple + + rparen.suppress() + ) ) - ) - list_expr = Forward() - list_expr_ref = testlist_star_namedexpr_tokens - array_literal = attach( - lbrack.suppress() + OneOrMore( - multisemicolon - | attach(comprehension_expr, add_bracks_handle) - | namedexpr_test + ~comma - | list_expr - ) + rbrack.suppress(), - array_literal_handle, - ) - list_item = ( - condense(lbrack + Optional(comprehension_expr) + rbrack) - | lbrack.suppress() + list_expr + rbrack.suppress() - | array_literal - ) + list_expr = Forward() + list_expr_ref = testlist_star_namedexpr_tokens + array_literal = attach( + lbrack.suppress() + OneOrMore( + multisemicolon + | attach(comprehension_expr, add_bracks_handle) + | namedexpr_test + ~comma + | list_expr + ) + rbrack.suppress(), + array_literal_handle, + ) + list_item = ( + condense(lbrack + Optional(comprehension_expr) + rbrack) + | lbrack.suppress() + list_expr + rbrack.suppress() + | array_literal + ) - string_atom = Forward() - string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) - fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) - f_string_atom = Forward() - f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) - - keyword_atom = any_keyword_in(const_vars) - passthrough_atom = addspace(OneOrMore(passthrough_item)) - - set_literal = Forward() - set_letter_literal = Forward() - set_s = caseless_literal("s") - set_f = caseless_literal("f") - set_m = caseless_literal("m") - set_letter = set_s | set_f | set_m - setmaker = Group( - (new_namedexpr_test + FollowedBy(rbrace))("test") - | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") - | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") - | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") - ) - set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() - set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() - - lazy_items = Optional(tokenlist(test, comma)) - lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) - - known_atom = ( - keyword_atom - | string_atom - | num_atom - | list_item - | dict_comp - | dict_literal - | set_literal - | set_letter_literal - | lazy_list - | typedef_ellipsis - | ellipsis - ) - atom = ( - # known_atom must come before name to properly parse string prefixes - known_atom - | refname - | paren_atom - | passthrough_atom - ) + string_atom = Forward() + string_atom_ref = OneOrMore(nonb_string) | OneOrMore(b_string) + fixed_len_string_tokens = OneOrMore(nonbf_string) | OneOrMore(b_string) + f_string_atom = Forward() + f_string_atom_ref = ZeroOrMore(nonbf_string) + f_string + ZeroOrMore(nonb_string) + + keyword_atom = any_keyword_in(const_vars) + passthrough_atom = addspace(OneOrMore(passthrough_item)) + + set_literal = Forward() + set_letter_literal = Forward() + set_s = caseless_literal("s") + set_f = caseless_literal("f") + set_m = caseless_literal("m") + set_letter = set_s | set_f | set_m + setmaker = Group( + (new_namedexpr_test + FollowedBy(rbrace))("test") + | (new_namedexpr_testlist_has_comma + FollowedBy(rbrace))("list") + | addspace(new_namedexpr_test + comp_for + FollowedBy(rbrace))("comp") + | (testlist_star_namedexpr + FollowedBy(rbrace))("testlist_star_expr") + ) + set_literal_ref = lbrace.suppress() + setmaker + rbrace.suppress() + set_letter_literal_ref = combine(set_letter + lbrace.suppress()) + Optional(setmaker) + rbrace.suppress() + + lazy_items = Optional(tokenlist(test, comma)) + lazy_list = attach(lbanana.suppress() + lazy_items + rbanana.suppress(), lazy_list_handle) + + known_atom = ( + keyword_atom + | string_atom + | num_atom + | list_item + | dict_comp + | dict_literal + | set_literal + | set_letter_literal + | lazy_list + | typedef_ellipsis + | ellipsis + ) + atom = ( + # known_atom must come before name to properly parse string prefixes + known_atom + | refname + | paren_atom + | passthrough_atom + ) - typedef_trailer = Forward() - typedef_or_expr = Forward() + typedef_trailer = Forward() + typedef_or_expr = Forward() - simple_trailer = ( - condense(dot + unsafe_name) - | condense(lbrack + subscriptlist + rbrack) - ) - call_trailer = ( - function_call - | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") - | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle - ) - known_trailer = typedef_trailer | ( - Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ - | Group(condense(dollar + lbrack + rbrack)) # $[] - | Group(condense(lbrack + rbrack)) # [] - | Group(questionmark) # ? - | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . - ) + ~questionmark - partial_trailer = ( - Group(fixto(dollar, "$(") + function_call) # $( - | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? - ) + ~questionmark - partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) - - no_call_trailer = simple_trailer | known_trailer | partial_trailer - - no_partial_complex_trailer = call_trailer | known_trailer - no_partial_trailer = simple_trailer | no_partial_complex_trailer - - complex_trailer = no_partial_complex_trailer | partial_trailer - trailer = simple_trailer | complex_trailer - - attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( - lparen + Optional(methodcaller_args) + rparen.suppress() - ) - attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) - - itemgetter_atom_tokens = ( - dot.suppress() - + Optional(unsafe_dotted_name) - + Group(OneOrMore(Group( - condense(Optional(dollar) + lbrack) - + subscriptgrouplist - + rbrack.suppress() - ))) - ) - itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) + simple_trailer = ( + condense(dot + unsafe_name) + | condense(lbrack + subscriptlist + rbrack) + ) + call_trailer = ( + function_call + | invalid_syntax(dollar + questionmark, "'?' must come before '$' in None-coalescing partial application") + | Group(dollar + ~lparen + ~lbrack) # keep $ for item_handle + ) + known_trailer = typedef_trailer | ( + Group(condense(dollar + lbrack) + subscriptgroup + rbrack.suppress()) # $[ + | Group(condense(dollar + lbrack + rbrack)) # $[] + | Group(condense(lbrack + rbrack)) # [] + | Group(questionmark) # ? + | Group(dot + ~unsafe_name + ~lbrack + ~dot) # . + ) + ~questionmark + partial_trailer = ( + Group(fixto(dollar, "$(") + function_call) # $( + | Group(fixto(dollar + lparen, "$(?") + questionmark_call_tokens) + rparen.suppress() # $(? + ) + ~questionmark + partial_trailer_tokens = Group(dollar.suppress() + function_call_tokens) + + no_call_trailer = simple_trailer | known_trailer | partial_trailer + + no_partial_complex_trailer = call_trailer | known_trailer + no_partial_trailer = simple_trailer | no_partial_complex_trailer + + complex_trailer = no_partial_complex_trailer | partial_trailer + trailer = simple_trailer | complex_trailer + + attrgetter_atom_tokens = dot.suppress() + unsafe_dotted_name + Optional( + lparen + Optional(methodcaller_args) + rparen.suppress() + ) + attrgetter_atom = attach(attrgetter_atom_tokens, attrgetter_atom_handle) + + itemgetter_atom_tokens = ( + dot.suppress() + + Optional(unsafe_dotted_name) + + Group(OneOrMore(Group( + condense(Optional(dollar) + lbrack) + + subscriptgrouplist + + rbrack.suppress() + ))) + ) + itemgetter_atom = attach(itemgetter_atom_tokens, itemgetter_handle) - implicit_partial_atom = ( - itemgetter_atom - | attrgetter_atom - | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") - | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") - ) + implicit_partial_atom = ( + itemgetter_atom + | attrgetter_atom + | fixto(dot + lbrack + rbrack, "_coconut.operator.getitem") + | fixto(dot + dollar + lbrack + rbrack, "_coconut_iter_getitem") + ) - trailer_atom = Forward() - trailer_atom_ref = atom + ZeroOrMore(trailer) - atom_item <<= ( - trailer_atom - | implicit_partial_atom - ) + trailer_atom = Forward() + trailer_atom_ref = atom + ZeroOrMore(trailer) + atom_item <<= ( + trailer_atom + | implicit_partial_atom + ) - no_partial_trailer_atom = Forward() - no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) - partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens + no_partial_trailer_atom = Forward() + no_partial_trailer_atom_ref = atom + ZeroOrMore(no_partial_trailer) + partial_atom_tokens = no_partial_trailer_atom + partial_trailer_tokens - simple_assign = Forward() - simple_assign_ref = maybeparens( - lparen, - (setname | passthrough_atom) - + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), - rparen, - ) - simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) - - assignlist = Forward() - star_assign_item = Forward() - base_assign_item = condense( - simple_assign - | lparen + assignlist + rparen - | lbrack + assignlist + rbrack - ) - star_assign_item_ref = condense(star + base_assign_item) - assign_item = star_assign_item | base_assign_item - assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) - - typed_assign_stmt = Forward() - typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) - basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) - - type_param = Forward() - type_param_bound_op = lt_colon | colon | le - type_var_name = stores_loc_item + setname - type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() - type_param_ref = ( - (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") - | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") - | (star.suppress() + type_var_name)("TypeVarTuple") - | (dubstar.suppress() + type_var_name)("ParamSpec") - ) - type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) - - type_alias_stmt = Forward() - type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test - - await_expr = Forward() - await_expr_ref = keyword("await").suppress() + atom_item - await_item = await_expr | atom_item - - factor = Forward() - unary = plus | neg_minus | tilde - - power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) - power_in_impl_call = Forward() - - impl_call_arg = condense(( - keyword_atom - | number - | disallow_keywords(reserved_vars) + dotted_refname - ) + Optional(power_in_impl_call)) - impl_call_item = condense( - disallow_keywords(reserved_vars) - + ~any_string - + atom_item - + Optional(power_in_impl_call) - ) - impl_call = Forward() - # we need to disable this inside the xonsh parser - impl_call_ref = Forward() - unsafe_impl_call_ref = ( - impl_call_item + OneOrMore(impl_call_arg) - ) + simple_assign = Forward() + simple_assign_ref = maybeparens( + lparen, + (setname | passthrough_atom) + + ZeroOrMore(ZeroOrMore(complex_trailer) + OneOrMore(simple_trailer)), + rparen, + ) + simple_assignlist = maybeparens(lparen, itemlist(simple_assign, comma, suppress_trailing=False), rparen) + + assignlist = Forward() + star_assign_item = Forward() + base_assign_item = condense( + simple_assign + | lparen + assignlist + rparen + | lbrack + assignlist + rbrack + ) + star_assign_item_ref = condense(star + base_assign_item) + assign_item = star_assign_item | base_assign_item + assignlist <<= itemlist(assign_item, comma, suppress_trailing=False) + + typed_assign_stmt = Forward() + typed_assign_stmt_ref = simple_assign + colon.suppress() + typedef_test + Optional(equals.suppress() + test_expr) + basic_stmt = addspace(ZeroOrMore(assignlist + equals) + test_expr) + + type_param = Forward() + type_param_bound_op = lt_colon | colon | le + type_var_name = stores_loc_item + setname + type_param_constraint = lparen.suppress() + Group(tokenlist(typedef_test, comma, require_sep=True)) + rparen.suppress() + type_param_ref = ( + (type_var_name + type_param_bound_op + type_param_constraint)("TypeVar constraint") + | (type_var_name + Optional(type_param_bound_op + typedef_test))("TypeVar") + | (star.suppress() + type_var_name)("TypeVarTuple") + | (dubstar.suppress() + type_var_name)("ParamSpec") + ) + type_params = Group(lbrack.suppress() + tokenlist(type_param, comma) + rbrack.suppress()) + + type_alias_stmt = Forward() + type_alias_stmt_ref = keyword("type").suppress() + setname + Optional(type_params) + equals.suppress() + typedef_test + + await_expr = Forward() + await_expr_ref = keyword("await").suppress() + atom_item + await_item = await_expr | atom_item + + factor = Forward() + unary = plus | neg_minus | tilde + + power = condense(exp_dubstar + ZeroOrMore(unary) + await_item) + power_in_impl_call = Forward() + + impl_call_arg = condense(( + keyword_atom + | number + | disallow_keywords(reserved_vars) + dotted_refname + ) + Optional(power_in_impl_call)) + impl_call_item = condense( + disallow_keywords(reserved_vars) + + ~any_string + + atom_item + + Optional(power_in_impl_call) + ) + impl_call = Forward() + # we need to disable this inside the xonsh parser + impl_call_ref = Forward() + unsafe_impl_call_ref = ( + impl_call_item + OneOrMore(impl_call_arg) + ) - factor <<= condense( - ZeroOrMore(unary) + ( - impl_call - | await_item + Optional(power) + factor <<= condense( + ZeroOrMore(unary) + ( + impl_call + | await_item + Optional(power) + ) ) - ) - mulop = mul_star | div_slash | div_dubslash | percent | matrix_at - addop = plus | sub_minus - shift = lshift | rshift - - term = Forward() - term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) - - # we condense all of these down, since Python handles the precedence, not Coconut - # arith_expr = exprlist(term, addop) - # shift_expr = exprlist(arith_expr, shift) - # and_expr = exprlist(shift_expr, amp) - and_expr = exprlist( - term, - addop - | shift - | amp, - ) + mulop = mul_star | div_slash | div_dubslash | percent | matrix_at + addop = plus | sub_minus + shift = lshift | rshift + + term = Forward() + term_ref = tokenlist(factor, mulop, allow_trailing=False, suppress=False) + + # we condense all of these down, since Python handles the precedence, not Coconut + # arith_expr = exprlist(term, addop) + # shift_expr = exprlist(arith_expr, shift) + # and_expr = exprlist(shift_expr, amp) + and_expr = exprlist( + term, + addop + | shift + | amp, + ) - protocol_intersect_expr = Forward() - protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) + protocol_intersect_expr = Forward() + protocol_intersect_expr_ref = tokenlist(and_expr, amp_colon, allow_trailing=False) - xor_expr = exprlist(protocol_intersect_expr, caret) + xor_expr = exprlist(protocol_intersect_expr, caret) - or_expr = typedef_or_expr | exprlist(xor_expr, bar) + or_expr = typedef_or_expr | exprlist(xor_expr, bar) - chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) + chain_expr = attach(tokenlist(or_expr, dubcolon, allow_trailing=False), chain_handle) - compose_expr = attach( - tokenlist( - chain_expr, - dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), - allow_trailing=False, - ), compose_expr_handle, - ) + compose_expr = attach( + tokenlist( + chain_expr, + dotdot + Optional(invalid_syntax(lambdef, "lambdas only allowed after composition pipe operators '..>' and '<..', not '..' (replace '..' with '<..' to fix)")), + allow_trailing=False, + ), compose_expr_handle, + ) - infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() - infix_expr = Forward() - infix_item = attach( - Group(Optional(compose_expr)) - + OneOrMore( - infix_op + Group(Optional(lambdef | compose_expr)) - ), - infix_handle, - ) - infix_expr <<= ( - compose_expr + ~backtick - | infix_item - ) + infix_op <<= backtick.suppress() + test_no_infix + backtick.suppress() + infix_expr = Forward() + infix_item = attach( + Group(Optional(compose_expr)) + + OneOrMore( + infix_op + Group(Optional(lambdef | compose_expr)) + ), + infix_handle, + ) + infix_expr <<= ( + compose_expr + ~backtick + | infix_item + ) - none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) - - comp_pipe_op = ( - comp_pipe - | comp_star_pipe - | comp_back_pipe - | comp_back_star_pipe - | comp_dubstar_pipe - | comp_back_dubstar_pipe - | comp_none_dubstar_pipe - | comp_back_none_dubstar_pipe - | comp_none_star_pipe - | comp_back_none_star_pipe - | comp_none_pipe - | comp_back_none_pipe - ) - comp_pipe_item = attach( - OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), - comp_pipe_handle, - ) - comp_pipe_expr = ( - none_coalesce_expr + ~comp_pipe_op - | comp_pipe_item - ) + none_coalesce_expr = attach(tokenlist(infix_expr, dubquestion, allow_trailing=False), none_coalesce_handle) + + comp_pipe_op = ( + comp_pipe + | comp_star_pipe + | comp_back_pipe + | comp_back_star_pipe + | comp_dubstar_pipe + | comp_back_dubstar_pipe + | comp_none_dubstar_pipe + | comp_back_none_dubstar_pipe + | comp_none_star_pipe + | comp_back_none_star_pipe + | comp_none_pipe + | comp_back_none_pipe + ) + comp_pipe_item = attach( + OneOrMore(none_coalesce_expr + comp_pipe_op) + (lambdef | none_coalesce_expr), + comp_pipe_handle, + ) + comp_pipe_expr = ( + none_coalesce_expr + ~comp_pipe_op + | comp_pipe_item + ) - pipe_op = ( - pipe - | star_pipe - | dubstar_pipe - | back_pipe - | back_star_pipe - | back_dubstar_pipe - | none_pipe - | none_star_pipe - | none_dubstar_pipe - | back_none_pipe - | back_none_star_pipe - | back_none_dubstar_pipe - ) - pipe_item = ( - # we need the pipe_op since any of the atoms could otherwise be the start of an expression - labeled_group(keyword("await"), "await") + pipe_op - | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op - | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op - | labeled_group(partial_atom_tokens, "partial") + pipe_op - | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op - # expr must come at end - | labeled_group(comp_pipe_expr, "expr") + pipe_op - ) - pipe_augassign_item = ( - # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr - labeled_group(keyword("await"), "await") + end_simple_stmt_item - | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item - | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item - | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item - | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item - ) - last_pipe_item = Group( - lambdef("expr") - # we need longest here because there's no following pipe_op we can use as above - | longest( - keyword("await")("await"), - itemgetter_atom_tokens("itemgetter"), - attrgetter_atom_tokens("attrgetter"), - partial_atom_tokens("partial"), - partial_op_atom_tokens("op partial"), - comp_pipe_expr("expr"), + pipe_op = ( + pipe + | star_pipe + | dubstar_pipe + | back_pipe + | back_star_pipe + | back_dubstar_pipe + | none_pipe + | none_star_pipe + | none_dubstar_pipe + | back_none_pipe + | back_none_star_pipe + | back_none_dubstar_pipe ) - ) - normal_pipe_expr = Forward() - normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item + pipe_item = ( + # we need the pipe_op since any of the atoms could otherwise be the start of an expression + labeled_group(keyword("await"), "await") + pipe_op + | labeled_group(itemgetter_atom_tokens, "itemgetter") + pipe_op + | labeled_group(attrgetter_atom_tokens, "attrgetter") + pipe_op + | labeled_group(partial_atom_tokens, "partial") + pipe_op + | labeled_group(partial_op_atom_tokens, "op partial") + pipe_op + # expr must come at end + | labeled_group(comp_pipe_expr, "expr") + pipe_op + ) + pipe_augassign_item = ( + # should match pipe_item but with pipe_op -> end_simple_stmt_item and no expr + labeled_group(keyword("await"), "await") + end_simple_stmt_item + | labeled_group(itemgetter_atom_tokens, "itemgetter") + end_simple_stmt_item + | labeled_group(attrgetter_atom_tokens, "attrgetter") + end_simple_stmt_item + | labeled_group(partial_atom_tokens, "partial") + end_simple_stmt_item + | labeled_group(partial_op_atom_tokens, "op partial") + end_simple_stmt_item + ) + last_pipe_item = Group( + lambdef("expr") + # we need longest here because there's no following pipe_op we can use as above + | longest( + keyword("await")("await"), + itemgetter_atom_tokens("itemgetter"), + attrgetter_atom_tokens("attrgetter"), + partial_atom_tokens("partial"), + partial_op_atom_tokens("op partial"), + comp_pipe_expr("expr"), + ) + ) + normal_pipe_expr = Forward() + normal_pipe_expr_tokens = OneOrMore(pipe_item) + last_pipe_item - pipe_expr = ( - comp_pipe_expr + ~pipe_op - | normal_pipe_expr - ) + pipe_expr = ( + comp_pipe_expr + ~pipe_op + | normal_pipe_expr + ) - expr <<= pipe_expr - - # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later - star_expr <<= Group(star + expr) - dubstar_expr <<= Group(dubstar + expr) - - comparison = exprlist(expr, comp_op) - not_test = addspace(ZeroOrMore(keyword("not")) + comparison) - # we condense "and" and "or" into one, since Python handles the precedence, not Coconut - # and_test = exprlist(not_test, keyword("and")) - # test_item = exprlist(and_test, keyword("or")) - test_item = exprlist(not_test, keyword("and") | keyword("or")) - - simple_stmt_item = Forward() - unsafe_simple_stmt_item = Forward() - simple_stmt = Forward() - stmt = Forward() - suite = Forward() - nocolon_suite = Forward() - base_suite = Forward() - - fat_arrow = Forward() - lambda_arrow = Forward() - unsafe_lambda_arrow = fat_arrow | arrow - - keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) - arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname - - keyword_lambdef = Forward() - keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) - arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) - implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") - lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef - - stmt_lambdef = Forward() - match_guard = Optional(keyword("if").suppress() + namedexpr_test) - closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) - stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) - stmt_lambdef_params = Optional( - attach(setname, add_parens_handle) - | parameters - | stmt_lambdef_match_params, - default="(_=None)", - ) - stmt_lambdef_body = Group( - Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) - | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, - ) + expr <<= pipe_expr + + # though 3.9 allows tests in the grammar here, they still raise a SyntaxError later + star_expr <<= Group(star + expr) + dubstar_expr <<= Group(dubstar + expr) + + comparison = exprlist(expr, comp_op) + not_test = addspace(ZeroOrMore(keyword("not")) + comparison) + # we condense "and" and "or" into one, since Python handles the precedence, not Coconut + # and_test = exprlist(not_test, keyword("and")) + # test_item = exprlist(and_test, keyword("or")) + test_item = exprlist(not_test, keyword("and") | keyword("or")) + + simple_stmt_item = Forward() + unsafe_simple_stmt_item = Forward() + simple_stmt = Forward() + stmt = Forward() + suite = Forward() + nocolon_suite = Forward() + base_suite = Forward() + + fat_arrow = Forward() + lambda_arrow = Forward() + unsafe_lambda_arrow = fat_arrow | arrow + + keyword_lambdef_params = maybeparens(lparen, set_args_list, rparen) + arrow_lambdef_params = lparen.suppress() + set_args_list + rparen.suppress() | setname + + keyword_lambdef = Forward() + keyword_lambdef_ref = addspace(keyword("lambda") + condense(keyword_lambdef_params + colon)) + arrow_lambdef = attach(arrow_lambdef_params + lambda_arrow.suppress(), lambdef_handle) + implicit_lambdef = fixto(lambda_arrow, "lambda _=None:") + lambdef_base = keyword_lambdef | arrow_lambdef | implicit_lambdef + + stmt_lambdef = Forward() + match_guard = Optional(keyword("if").suppress() + namedexpr_test) + closing_stmt = longest(new_testlist_star_expr("tests"), unsafe_simple_stmt_item) + stmt_lambdef_match_params = Group(lparen.suppress() + match_args_list + match_guard + rparen.suppress()) + stmt_lambdef_params = Optional( + attach(setname, add_parens_handle) + | parameters + | stmt_lambdef_match_params, + default="(_=None)", + ) + stmt_lambdef_body = Group( + Group(OneOrMore(simple_stmt_item + semicolon.suppress())) + Optional(closing_stmt) + | Group(ZeroOrMore(simple_stmt_item + semicolon.suppress())) + closing_stmt, + ) - no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) - fat_arrow <<= _fat_arrow - stmt_lambdef_suite = ( - arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow - | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body - ) + no_fat_arrow_stmt_lambdef_body, _fat_arrow = disable_inside(stmt_lambdef_body, unsafe_fat_arrow) + fat_arrow <<= _fat_arrow + stmt_lambdef_suite = ( + arrow.suppress() + no_fat_arrow_stmt_lambdef_body + ~fat_arrow + | Optional(arrow.suppress() + typedef_test) + fat_arrow.suppress() + stmt_lambdef_body + ) - general_stmt_lambdef = ( - Group(any_len_perm( - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_params - + stmt_lambdef_suite - ) - match_stmt_lambdef = ( - Group(any_len_perm( - keyword("match").suppress(), - keyword("async"), - keyword("copyclosure"), - )) + keyword("def").suppress() - + stmt_lambdef_match_params - + stmt_lambdef_suite - ) - stmt_lambdef_ref = trace( - general_stmt_lambdef - | match_stmt_lambdef - ) + ( - fixto(FollowedBy(comma), ",") - | fixto(always_match, "") - ) + general_stmt_lambdef = ( + Group(any_len_perm( + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_params + + stmt_lambdef_suite + ) + match_stmt_lambdef = ( + Group(any_len_perm( + keyword("match").suppress(), + keyword("async"), + keyword("copyclosure"), + )) + keyword("def").suppress() + + stmt_lambdef_match_params + + stmt_lambdef_suite + ) + stmt_lambdef_ref = trace( + general_stmt_lambdef + | match_stmt_lambdef + ) + ( + fixto(FollowedBy(comma), ",") + | fixto(always_match, "") + ) - lambdef <<= addspace(lambdef_base + test) | stmt_lambdef - lambdef_no_cond = addspace(lambdef_base + test_no_cond) + lambdef <<= addspace(lambdef_base + test) | stmt_lambdef + lambdef_no_cond = addspace(lambdef_base + test_no_cond) - typedef_callable_arg = Group( - test("arg") - | (dubstar.suppress() + refname)("paramspec") - ) - typedef_callable_params = Optional(Group( - labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") - | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() - | labeled_group(negable_atom_item, "arg") - )) - unsafe_typedef_callable = attach( - Optional(keyword("async"), default="") - + typedef_callable_params - + arrow.suppress() - + typedef_test, - typedef_callable_handle, - ) + typedef_callable_arg = Group( + test("arg") + | (dubstar.suppress() + refname)("paramspec") + ) + typedef_callable_params = Optional(Group( + labeled_group(maybeparens(lparen, ellipsis_tokens, rparen), "ellipsis") + | lparen.suppress() + Optional(tokenlist(typedef_callable_arg, comma)) + rparen.suppress() + | labeled_group(negable_atom_item, "arg") + )) + unsafe_typedef_callable = attach( + Optional(keyword("async"), default="") + + typedef_callable_params + + arrow.suppress() + + typedef_test, + typedef_callable_handle, + ) - unsafe_typedef_trailer = ( # use special type signifier for item_handle - Group(fixto(lbrack + rbrack, "type:[]")) - | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) - | Group(fixto(questionmark + ~questionmark, "type:?")) - ) + unsafe_typedef_trailer = ( # use special type signifier for item_handle + Group(fixto(lbrack + rbrack, "type:[]")) + | Group(fixto(dollar + lbrack + rbrack, "type:$[]")) + | Group(fixto(questionmark + ~questionmark, "type:?")) + ) - unsafe_typedef_or_expr = Forward() - unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) + unsafe_typedef_or_expr = Forward() + unsafe_typedef_or_expr_ref = tokenlist(xor_expr, bar, allow_trailing=False, at_least_two=True) - unsafe_typedef_tuple = Forward() - # should mimic testlist_star_namedexpr but with require_sep=True - unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) + unsafe_typedef_tuple = Forward() + # should mimic testlist_star_namedexpr but with require_sep=True + unsafe_typedef_tuple_ref = tokenlist(Group(namedexpr_test) | star_expr, fixto(semicolon, ","), suppress=False, require_sep=True) - unsafe_typedef_ellipsis = ellipsis_tokens + unsafe_typedef_ellipsis = ellipsis_tokens - unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) + unsafe_typedef_op_item = attach(base_op_item, typedef_op_item_handle) - unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( - test, - unsafe_typedef_callable, - unsafe_typedef_trailer, - unsafe_typedef_or_expr, - unsafe_typedef_tuple, - unsafe_typedef_ellipsis, - unsafe_typedef_op_item, - ) - typedef_trailer <<= _typedef_trailer - typedef_or_expr <<= _typedef_or_expr - typedef_tuple <<= _typedef_tuple - typedef_ellipsis <<= _typedef_ellipsis - typedef_op_item <<= _typedef_op_item - - _typedef_test, _lambda_arrow = disable_inside( - unsafe_typedef_test, - unsafe_lambda_arrow, - ) - typedef_test <<= _typedef_test - lambda_arrow <<= _lambda_arrow - - alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) - test <<= ( - typedef_callable - | lambdef - | alt_ternary_expr - | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item - ) - test_no_cond <<= lambdef_no_cond | test_item + unsafe_typedef_test, typedef_callable, _typedef_trailer, _typedef_or_expr, _typedef_tuple, _typedef_ellipsis, _typedef_op_item = disable_outside( + test, + unsafe_typedef_callable, + unsafe_typedef_trailer, + unsafe_typedef_or_expr, + unsafe_typedef_tuple, + unsafe_typedef_ellipsis, + unsafe_typedef_op_item, + ) + typedef_trailer <<= _typedef_trailer + typedef_or_expr <<= _typedef_or_expr + typedef_tuple <<= _typedef_tuple + typedef_ellipsis <<= _typedef_ellipsis + typedef_op_item <<= _typedef_op_item + + _typedef_test, _lambda_arrow = disable_inside( + unsafe_typedef_test, + unsafe_lambda_arrow, + ) + typedef_test <<= _typedef_test + lambda_arrow <<= _lambda_arrow + + alt_ternary_expr = attach(keyword("if").suppress() + test_item + keyword("then").suppress() + test_item + keyword("else").suppress() + test, alt_ternary_handle) + test <<= ( + typedef_callable + | lambdef + | alt_ternary_expr + | addspace(test_item + Optional(keyword("if") + test_item + keyword("else") + test)) # must come last since it includes plain test_item + ) + test_no_cond <<= lambdef_no_cond | test_item - namedexpr = Forward() - namedexpr_ref = addspace( - setname + colon_eq + ( + namedexpr = Forward() + namedexpr_ref = addspace( + setname + colon_eq + ( + test + ~colon_eq + | attach(namedexpr, add_parens_handle) + ) + ) + namedexpr_test <<= ( test + ~colon_eq - | attach(namedexpr, add_parens_handle) + | namedexpr ) - ) - namedexpr_test <<= ( - test + ~colon_eq - | namedexpr - ) - new_namedexpr = Forward() - new_namedexpr_ref = namedexpr_ref - new_namedexpr_test <<= ( - test + ~colon_eq - | new_namedexpr - ) - - classdef = Forward() - decorators = Forward() - classlist = Group( - Optional(function_call_tokens) - + ~equals, # don't match class destructuring assignment - ) - class_suite = suite | attach(newline, class_suite_handle) - classdef_ref = ( - Optional(decorators, default="") - + keyword("class").suppress() - + classname - + Optional(type_params, default=()) - + classlist - + class_suite - ) - - async_comp_for = Forward() - comp_iter = Forward() - comp_it_item = ( - invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") - | test_item - ) - base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) - async_comp_for_ref = addspace(keyword("async") + base_comp_for) - comp_for <<= async_comp_for | base_comp_for - comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) - comp_iter <<= comp_for | comp_if - - return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) - - complex_raise_stmt = Forward() - pass_stmt = keyword("pass") - break_stmt = keyword("break") - continue_stmt = keyword("continue") - simple_raise_stmt = addspace(keyword("raise") + Optional(test)) - complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test - raise_stmt = complex_raise_stmt | simple_raise_stmt - flow_stmt = ( - return_stmt - | raise_stmt - | break_stmt - | yield_expr - | continue_stmt - ) + new_namedexpr = Forward() + new_namedexpr_ref = namedexpr_ref + new_namedexpr_test <<= ( + test + ~colon_eq + | new_namedexpr + ) - imp_name = ( - # maybeparens allows for using custom operator names here - maybeparens(lparen, setname, rparen) - | passthrough_item - ) - unsafe_imp_name = ( - # should match imp_name except with unsafe_name instead of setname - maybeparens(lparen, unsafe_name, rparen) - | passthrough_item - ) - dotted_imp_name = ( - dotted_setname - | passthrough_item - ) - unsafe_dotted_imp_name = ( - # should match dotted_imp_name except with unsafe_dotted_name - unsafe_dotted_name - | passthrough_item - ) - imp_as = keyword("as").suppress() - imp_name - import_item = Group( - unsafe_dotted_imp_name + imp_as - | dotted_imp_name - ) - from_import_item = Group( - unsafe_imp_name + imp_as - | imp_name - ) + classdef = Forward() + decorators = Forward() + classlist = Group( + Optional(function_call_tokens) + + ~equals, # don't match class destructuring assignment + ) + class_suite = suite | attach(newline, class_suite_handle) + classdef_ref = ( + Optional(decorators, default="") + + keyword("class").suppress() + + classname + + Optional(type_params, default=()) + + classlist + + class_suite + ) - import_names = Group( - maybeparens(lparen, tokenlist(import_item, comma), rparen) - | star - ) - from_import_names = Group( - maybeparens(lparen, tokenlist(from_import_item, comma), rparen) - | star - ) - basic_import = keyword("import").suppress() - import_names - import_from_name = condense( - ZeroOrMore(unsafe_dot) + unsafe_dotted_name - | OneOrMore(unsafe_dot) - | star - ) - from_import = ( - keyword("from").suppress() - - import_from_name - - keyword("import").suppress() - from_import_names - ) - import_stmt = Forward() - import_stmt_ref = from_import | basic_import + async_comp_for = Forward() + comp_iter = Forward() + comp_it_item = ( + invalid_syntax(maybeparens(lparen, namedexpr, rparen), "PEP 572 disallows assignment expressions in comprehension iterable expressions") + | test_item + ) + base_comp_for = addspace(keyword("for") + assignlist + keyword("in") + comp_it_item + Optional(comp_iter)) + async_comp_for_ref = addspace(keyword("async") + base_comp_for) + comp_for <<= async_comp_for | base_comp_for + comp_if = addspace(keyword("if") + test_no_cond + Optional(comp_iter)) + comp_iter <<= comp_for | comp_if + + return_stmt = addspace(keyword("return") - Optional(new_testlist_star_expr)) + + complex_raise_stmt = Forward() + pass_stmt = keyword("pass") + break_stmt = keyword("break") + continue_stmt = keyword("continue") + simple_raise_stmt = addspace(keyword("raise") + Optional(test)) + complex_raise_stmt_ref = keyword("raise").suppress() + test + keyword("from").suppress() - test + raise_stmt = complex_raise_stmt | simple_raise_stmt + flow_stmt = ( + return_stmt + | raise_stmt + | break_stmt + | yield_expr + | continue_stmt + ) - augassign_stmt = Forward() - augassign_rhs = ( - labeled_group(pipe_augassign + pipe_augassign_item, "pipe") - | labeled_group(augassign + test_expr, "simple") - ) - augassign_stmt_ref = simple_assign + augassign_rhs + imp_name = ( + # maybeparens allows for using custom operator names here + maybeparens(lparen, setname, rparen) + | passthrough_item + ) + unsafe_imp_name = ( + # should match imp_name except with unsafe_name instead of setname + maybeparens(lparen, unsafe_name, rparen) + | passthrough_item + ) + dotted_imp_name = ( + dotted_setname + | passthrough_item + ) + unsafe_dotted_imp_name = ( + # should match dotted_imp_name except with unsafe_dotted_name + unsafe_dotted_name + | passthrough_item + ) + imp_as = keyword("as").suppress() - imp_name + import_item = Group( + unsafe_dotted_imp_name + imp_as + | dotted_imp_name + ) + from_import_item = Group( + unsafe_imp_name + imp_as + | imp_name + ) - simple_kwd_assign = attach( - maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), - simple_kwd_assign_handle, - ) - kwd_augassign = Forward() - kwd_augassign_ref = setname + augassign_rhs - kwd_assign = ( - kwd_augassign - | simple_kwd_assign - ) - global_stmt = addspace(keyword("global") - kwd_assign) - nonlocal_stmt = Forward() - nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) - - del_stmt = addspace(keyword("del") - simple_assignlist) - - matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) - matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) - - match_check_equals = Forward() - match_check_equals_ref = equals - - match_dotted_name_const = Forward() - complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) - match_const = condense( - (eq | match_check_equals).suppress() + negable_atom_item - | string_atom - | complex_number - | Optional(neg_minus) + number - | match_dotted_name_const - ) - empty_const = fixto( - lparen + rparen - | lbrack + rbrack - | set_letter + lbrace + rbrace, - "()", - ) + import_names = Group( + maybeparens(lparen, tokenlist(import_item, comma), rparen) + | star + ) + from_import_names = Group( + maybeparens(lparen, tokenlist(from_import_item, comma), rparen) + | star + ) + basic_import = keyword("import").suppress() - import_names + import_from_name = condense( + ZeroOrMore(unsafe_dot) + unsafe_dotted_name + | OneOrMore(unsafe_dot) + | star + ) + from_import = ( + keyword("from").suppress() + - import_from_name + - keyword("import").suppress() - from_import_names + ) + import_stmt = Forward() + import_stmt_ref = from_import | basic_import - match_pair = Group(match_const + colon.suppress() + match) - matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) - set_star = star.suppress() + (keyword(wildcard) | empty_const) + augassign_stmt = Forward() + augassign_rhs = ( + labeled_group(pipe_augassign + pipe_augassign_item, "pipe") + | labeled_group(augassign + test_expr, "simple") + ) + augassign_stmt_ref = simple_assign + augassign_rhs - matchlist_tuple_items = ( - match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) - | match + comma.suppress() - ) - matchlist_tuple = Group(Optional(matchlist_tuple_items)) - matchlist_list = Group(Optional(tokenlist(match, comma))) - match_list = Group(lbrack + matchlist_list + rbrack.suppress()) - match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) - match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) - - interior_name_match = labeled_group(setname, "var") - match_string = interleaved_tokenlist( - # f_string_atom must come first - f_string_atom("f_string") | fixed_len_string_tokens("string"), - interior_name_match("capture"), - plus, - at_least_two=True, - )("string_sequence") - sequence_match = interleaved_tokenlist( - (match_list | match_tuple)("literal"), - interior_name_match("capture"), - plus, - )("sequence") - iter_match = interleaved_tokenlist( - (match_list | match_tuple | match_lazy)("literal"), - interior_name_match("capture"), - unsafe_dubcolon, - at_least_two=True, - )("iter") - matchlist_star = interleaved_tokenlist( - star.suppress() + match("capture"), - match("elem"), - comma, - allow_trailing=True, - ) - star_match = ( - lbrack.suppress() + matchlist_star + rbrack.suppress() - | lparen.suppress() + matchlist_star + rparen.suppress() - )("star") - - base_match = Group( - (negable_atom_item + arrow.suppress() + match)("view") - | match_string - | match_const("const") - | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") - | (keyword("in").suppress() + negable_atom_item)("in") - | iter_match - | match_lazy("lazy") - | sequence_match - | star_match - | (lparen.suppress() + match + rparen.suppress())("paren") - | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") - | ( - Group(Optional(set_letter)) - + lbrace.suppress() - + ( - Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) - | Group(always_match) + set_star + Optional(comma.suppress()) - | Group(Optional(tokenlist(match_const, comma))) - ) + rbrace.suppress() - )("set") - | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") - | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") - | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") - | Optional(keyword("as").suppress()) + setname("var") - ) + simple_kwd_assign = attach( + maybeparens(lparen, itemlist(setname, comma), rparen) + Optional(equals.suppress() - test_expr), + simple_kwd_assign_handle, + ) + kwd_augassign = Forward() + kwd_augassign_ref = setname + augassign_rhs + kwd_assign = ( + kwd_augassign + | simple_kwd_assign + ) + global_stmt = addspace(keyword("global") - kwd_assign) + nonlocal_stmt = Forward() + nonlocal_stmt_ref = addspace(keyword("nonlocal") - kwd_assign) + + del_stmt = addspace(keyword("del") - simple_assignlist) + + matchlist_data_item = Group(Optional(star | Optional(dot) + unsafe_name + equals) + match) + matchlist_data = Group(Optional(tokenlist(matchlist_data_item, comma))) + + match_check_equals = Forward() + match_check_equals_ref = equals + + match_dotted_name_const = Forward() + complex_number = condense(Optional(neg_minus) + number + (plus | sub_minus) + Optional(neg_minus) + imag_num) + match_const = condense( + (eq | match_check_equals).suppress() + negable_atom_item + | string_atom + | complex_number + | Optional(neg_minus) + number + | match_dotted_name_const + ) + empty_const = fixto( + lparen + rparen + | lbrack + rbrack + | set_letter + lbrace + rbrace, + "()", + ) - matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) - isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match + match_pair = Group(match_const + colon.suppress() + match) + matchlist_dict = Group(Optional(tokenlist(match_pair, comma))) + set_star = star.suppress() + (keyword(wildcard) | empty_const) - matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) - bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match + matchlist_tuple_items = ( + match + OneOrMore(comma.suppress() + match) + Optional(comma.suppress()) + | match + comma.suppress() + ) + matchlist_tuple = Group(Optional(matchlist_tuple_items)) + matchlist_list = Group(Optional(tokenlist(match, comma))) + match_list = Group(lbrack + matchlist_list + rbrack.suppress()) + match_tuple = Group(lparen + matchlist_tuple + rparen.suppress()) + match_lazy = Group(lbanana + matchlist_list + rbanana.suppress()) + + interior_name_match = labeled_group(setname, "var") + match_string = interleaved_tokenlist( + # f_string_atom must come first + f_string_atom("f_string") | fixed_len_string_tokens("string"), + interior_name_match("capture"), + plus, + at_least_two=True, + )("string_sequence") + sequence_match = interleaved_tokenlist( + (match_list | match_tuple)("literal"), + interior_name_match("capture"), + plus, + )("sequence") + iter_match = interleaved_tokenlist( + (match_list | match_tuple | match_lazy)("literal"), + interior_name_match("capture"), + unsafe_dubcolon, + at_least_two=True, + )("iter") + matchlist_star = interleaved_tokenlist( + star.suppress() + match("capture"), + match("elem"), + comma, + allow_trailing=True, + ) + star_match = ( + lbrack.suppress() + matchlist_star + rbrack.suppress() + | lparen.suppress() + matchlist_star + rparen.suppress() + )("star") + + base_match = Group( + (negable_atom_item + arrow.suppress() + match)("view") + | match_string + | match_const("const") + | (keyword_atom | keyword("is").suppress() + negable_atom_item)("is") + | (keyword("in").suppress() + negable_atom_item)("in") + | iter_match + | match_lazy("lazy") + | sequence_match + | star_match + | (lparen.suppress() + match + rparen.suppress())("paren") + | (lbrace.suppress() + matchlist_dict + Optional(dubstar.suppress() + (setname | condense(lbrace + rbrace)) + Optional(comma.suppress())) + rbrace.suppress())("dict") + | ( + Group(Optional(set_letter)) + + lbrace.suppress() + + ( + Group(tokenlist(match_const, comma, allow_trailing=False)) + Optional(comma.suppress() + set_star + Optional(comma.suppress())) + | Group(always_match) + set_star + Optional(comma.suppress()) + | Group(Optional(tokenlist(match_const, comma))) + ) + rbrace.suppress() + )("set") + | (keyword("data").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data") + | (keyword("class").suppress() + dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("class") + | (dotted_refname + lparen.suppress() + matchlist_data + rparen.suppress())("data_or_class") + | Optional(keyword("as").suppress()) + setname("var") + ) - matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) - infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match + matchlist_isinstance = base_match + OneOrMore(keyword("is").suppress() + negable_atom_item) + isinstance_match = labeled_group(matchlist_isinstance, "isinstance_is") | base_match - matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) - as_match = labeled_group(matchlist_as, "as") | infix_match + matchlist_bar_or = isinstance_match + OneOrMore(bar.suppress() + isinstance_match) + bar_or_match = labeled_group(matchlist_bar_or, "or") | isinstance_match - matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) - and_match = labeled_group(matchlist_and, "and") | as_match + matchlist_infix = bar_or_match + OneOrMore(Group(infix_op + Optional(negable_atom_item))) + infix_match = labeled_group(matchlist_infix, "infix") | bar_or_match - matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) - kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match + matchlist_as = infix_match + OneOrMore(keyword("as").suppress() + setname) + as_match = labeled_group(matchlist_as, "as") | infix_match - match <<= kwd_or_match + matchlist_and = as_match + OneOrMore(keyword("and").suppress() + as_match) + and_match = labeled_group(matchlist_and, "and") | as_match - many_match = ( - labeled_group(matchlist_star, "star") - | labeled_group(matchlist_tuple_items, "implicit_tuple") - | match - ) + matchlist_kwd_or = and_match + OneOrMore(keyword("or").suppress() + and_match) + kwd_or_match = labeled_group(matchlist_kwd_or, "or") | and_match - else_stmt = condense(keyword("else") - suite) - full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) - full_match = Forward() - full_match_ref = ( - keyword("match").suppress() - + many_match - + addspace(Optional(keyword("not")) + keyword("in")) - + testlist_star_namedexpr - + match_guard - # avoid match match-case blocks - + ~FollowedBy(colon + newline + indent + keyword("case")) - - full_suite - ) - match_stmt = condense(full_match - Optional(else_stmt)) - - destructuring_stmt = Forward() - base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr - destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) - - # both syntaxes here must be kept the same except for the keywords - case_match_co_syntax = Group( - (keyword("match") | keyword("case")).suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_co_syntax = ( - (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() + suite) - ) - case_match_py_syntax = Group( - keyword("case").suppress() - + stores_loc_item - + many_match - + Optional(keyword("if").suppress() + namedexpr_test) - - full_suite - ) - cases_stmt_py_syntax = ( - keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() - + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) - + dedent.suppress() + Optional(keyword("else").suppress() - suite) - ) - cases_stmt = Forward() - cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + match <<= kwd_or_match - assert_stmt = addspace( - keyword("assert") - - ( - lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item - | testlist + many_match = ( + labeled_group(matchlist_star, "star") + | labeled_group(matchlist_tuple_items, "implicit_tuple") + | match ) - ) - if_stmt = condense( - addspace(keyword("if") + condense(namedexpr_test + suite)) - - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) - - Optional(else_stmt) - ) - while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) - for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) + else_stmt = condense(keyword("else") - suite) + full_suite = colon.suppress() - Group((newline.suppress() - indent.suppress() - OneOrMore(stmt) - dedent.suppress()) | simple_stmt) + full_match = Forward() + full_match_ref = ( + keyword("match").suppress() + + many_match + + addspace(Optional(keyword("not")) + keyword("in")) + + testlist_star_namedexpr + + match_guard + # avoid match match-case blocks + + ~FollowedBy(colon + newline + indent + keyword("case")) + - full_suite + ) + match_stmt = condense(full_match - Optional(else_stmt)) + + destructuring_stmt = Forward() + base_destructuring_stmt = Optional(keyword("match").suppress()) + many_match + equals.suppress() + test_expr + destructuring_stmt_ref, match_dotted_name_const_ref = disable_inside(base_destructuring_stmt, must_be_dotted_name + ~lparen) + + # both syntaxes here must be kept the same except for the keywords + case_match_co_syntax = Group( + (keyword("match") | keyword("case")).suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_co_syntax = ( + (keyword("cases") | keyword("case")) + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_co_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() + suite) + ) + case_match_py_syntax = Group( + keyword("case").suppress() + + stores_loc_item + + many_match + + Optional(keyword("if").suppress() + namedexpr_test) + - full_suite + ) + cases_stmt_py_syntax = ( + keyword("match") + testlist_star_namedexpr + colon.suppress() + newline.suppress() + + indent.suppress() + Group(OneOrMore(case_match_py_syntax)) + + dedent.suppress() + Optional(keyword("else").suppress() - suite) + ) + cases_stmt = Forward() + cases_stmt_ref = cases_stmt_co_syntax | cases_stmt_py_syntax + + assert_stmt = addspace( + keyword("assert") + - ( + lparen.suppress() + testlist + rparen.suppress() + end_simple_stmt_item + | testlist + ) + ) + if_stmt = condense( + addspace(keyword("if") + condense(namedexpr_test + suite)) + - ZeroOrMore(addspace(keyword("elif") - condense(namedexpr_test - suite))) + - Optional(else_stmt) + ) + while_stmt = addspace(keyword("while") - condense(namedexpr_test - suite - Optional(else_stmt))) - suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) + for_stmt = addspace(keyword("for") + assignlist + keyword("in") - condense(new_testlist_star_expr - suite - Optional(else_stmt))) - base_match_for_stmt = Forward() - base_match_for_stmt_ref = ( - keyword("for").suppress() - + many_match - + keyword("in").suppress() - - new_testlist_star_expr - - suite_with_else_tokens - ) - match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt + suite_with_else_tokens = colon.suppress() + condense(nocolon_suite + Optional(else_stmt)) - except_item = ( - testlist_has_comma("list") - | test("test") - ) - Optional( - keyword("as").suppress() - setname - ) - except_clause = attach(keyword("except") + except_item, except_handle) - except_star_clause = Forward() - except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) - try_stmt = condense( - keyword("try") - suite + ( - keyword("finally") - suite - | ( - OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) - | keyword("except") - suite - | OneOrMore(except_star_clause - suite) - ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + base_match_for_stmt = Forward() + base_match_for_stmt_ref = ( + keyword("for").suppress() + + many_match + + keyword("in").suppress() + - new_testlist_star_expr + - suite_with_else_tokens ) - ) - - with_item = addspace(test + Optional(keyword("as") + base_assign_item)) - with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) - with_stmt_ref = keyword("with").suppress() - with_item_list - suite - with_stmt = Forward() - - funcname_typeparams = Forward() - funcname_typeparams_ref = dotted_setname + Optional(type_params) - name_funcdef = condense(funcname_typeparams + parameters) - op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) - op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) - op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() - op_funcdef = attach( - Group(Optional(op_funcdef_arg)) - + op_funcdef_name - + Group(Optional(op_funcdef_arg)), - op_funcdef_handle, - ) + match_for_stmt = Optional(keyword("match").suppress()) + base_match_for_stmt - return_typedef = Forward() - return_typedef_ref = arrow.suppress() + typedef_test - end_func_colon = return_typedef + colon.suppress() | colon - base_funcdef = op_funcdef | name_funcdef - funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) - - name_match_funcdef = Forward() - op_match_funcdef = Forward() - op_match_funcdef_arg = Group(Optional( - Group( - ( - lparen.suppress() - + match - + Optional(equals.suppress() + test) - + rparen.suppress() - ) | interior_name_match - ) - )) - name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() - op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard - base_match_funcdef = op_match_funcdef | name_match_funcdef - func_suite = ( - attach(simple_stmt, make_suite_handle) - | ( - newline.suppress() - - indent.suppress() - - Optional(docstring) - - attach(condense(OneOrMore(stmt)), make_suite_handle) - - dedent.suppress() + except_item = ( + testlist_has_comma("list") + | test("test") + ) - Optional( + keyword("as").suppress() - setname + ) + except_clause = attach(keyword("except") + except_item, except_handle) + except_star_clause = Forward() + except_star_clause_ref = attach(except_star_kwd + except_item, except_handle) + try_stmt = condense( + keyword("try") - suite + ( + keyword("finally") - suite + | ( + OneOrMore(except_clause - suite) - Optional(keyword("except") - suite) + | keyword("except") - suite + | OneOrMore(except_star_clause - suite) + ) - Optional(else_stmt) - Optional(keyword("finally") - suite) + ) ) - ) - def_match_funcdef = attach( - base_match_funcdef - + end_func_colon - - func_suite, - join_match_funcdef, - ) - match_def_modifiers = any_len_perm( - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - ) - match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - - where_suite = keyword("where").suppress() - full_suite - where_stmt = Forward() - where_item = Forward() - where_item_ref = unsafe_simple_stmt_item - where_stmt_ref = where_item + where_suite + with_item = addspace(test + Optional(keyword("as") + base_assign_item)) + with_item_list = Group(maybeparens(lparen, tokenlist(with_item, comma), rparen)) + with_stmt_ref = keyword("with").suppress() - with_item_list - suite + with_stmt = Forward() + + funcname_typeparams = Forward() + funcname_typeparams_ref = dotted_setname + Optional(type_params) + name_funcdef = condense(funcname_typeparams + parameters) + op_tfpdef = unsafe_typedef_default | condense(setname + Optional(default)) + op_funcdef_arg = setname | condense(lparen.suppress() + op_tfpdef + rparen.suppress()) + op_funcdef_name = unsafe_backtick.suppress() + funcname_typeparams + unsafe_backtick.suppress() + op_funcdef = attach( + Group(Optional(op_funcdef_arg)) + + op_funcdef_name + + Group(Optional(op_funcdef_arg)), + op_funcdef_handle, + ) - implicit_return = ( - invalid_syntax(return_stmt, "expected expression but got return statement") - | attach(new_testlist_star_expr, implicit_return_handle) - ) - implicit_return_where = Forward() - implicit_return_where_item = Forward() - implicit_return_where_item_ref = implicit_return - implicit_return_where_ref = implicit_return_where_item + where_suite - implicit_return_stmt = ( - condense(implicit_return + newline) - | implicit_return_where - ) + return_typedef = Forward() + return_typedef_ref = arrow.suppress() + typedef_test + end_func_colon = return_typedef + colon.suppress() | colon + base_funcdef = op_funcdef | name_funcdef + funcdef = addspace(keyword("def") + condense(base_funcdef + end_func_colon + nocolon_suite)) - math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) - math_funcdef_suite = ( - attach(implicit_return_stmt, make_suite_handle) - | condense(newline - indent - math_funcdef_body - dedent) - ) - end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") - math_funcdef = attach( - condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, - math_funcdef_handle, - ) - math_match_funcdef = addspace( - match_def_modifiers - + attach( + name_match_funcdef = Forward() + op_match_funcdef = Forward() + op_match_funcdef_arg = Group(Optional( + Group( + ( + lparen.suppress() + + match + + Optional(equals.suppress() + test) + + rparen.suppress() + ) | interior_name_match + ) + )) + name_match_funcdef_ref = keyword("def").suppress() + funcname_typeparams + lparen.suppress() + match_args_list + match_guard + rparen.suppress() + op_match_funcdef_ref = keyword("def").suppress() + op_match_funcdef_arg + op_funcdef_name + op_match_funcdef_arg + match_guard + base_match_funcdef = op_match_funcdef | name_match_funcdef + func_suite = ( + attach(simple_stmt, make_suite_handle) + | ( + newline.suppress() + - indent.suppress() + - Optional(docstring) + - attach(condense(OneOrMore(stmt)), make_suite_handle) + - dedent.suppress() + ) + ) + def_match_funcdef = attach( base_match_funcdef - + end_func_equals - + ( - attach(implicit_return_stmt, make_suite_handle) - | ( - newline.suppress() - indent.suppress() - + Optional(docstring) - + attach(math_funcdef_body, make_suite_handle) - + dedent.suppress() - ) - ), + + end_func_colon + - func_suite, join_match_funcdef, ) - ) - - async_stmt = Forward() - async_with_for_stmt = Forward() - async_with_for_stmt_ref = ( - labeled_group( - (keyword("async") + keyword("with") + keyword("for")).suppress() - + assignlist + keyword("in").suppress() - - test - - suite_with_else_tokens, - "normal", - ) - | labeled_group( - (any_len_perm( - keyword("match"), - required=(keyword("async"), keyword("with")), - ) + keyword("for")).suppress() - + many_match + keyword("in").suppress() - - test - - suite_with_else_tokens, - "match", - ) - ) - async_stmt_ref = addspace( - keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for - | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for - | async_with_for_stmt - ) - - async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) - async_match_funcdef = addspace( - any_len_perm( + match_def_modifiers = any_len_perm( keyword("match").suppress(), # addpattern is detected later keyword("addpattern"), - required=(keyword("async").suppress(),), - ) + (def_match_funcdef | math_match_funcdef), - ) + ) + match_funcdef = addspace(match_def_modifiers + def_match_funcdef) - async_keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - required=(keyword("async").suppress(),), - ) - ) + (funcdef | math_funcdef) - async_keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), - required=(keyword("async").suppress(),), + where_suite = keyword("where").suppress() - full_suite + + where_stmt = Forward() + where_item = Forward() + where_item_ref = unsafe_simple_stmt_item + where_stmt_ref = where_item + where_suite + + implicit_return = ( + invalid_syntax(return_stmt, "expected expression but got return statement") + | attach(new_testlist_star_expr, implicit_return_handle) + ) + implicit_return_where = Forward() + implicit_return_where_item = Forward() + implicit_return_where_item_ref = implicit_return + implicit_return_where_ref = implicit_return_where_item + where_suite + implicit_return_stmt = ( + condense(implicit_return + newline) + | implicit_return_where ) - ) + (def_match_funcdef | math_match_funcdef) - async_keyword_funcdef = Forward() - async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef - async_funcdef_stmt = ( - async_funcdef - | async_match_funcdef - | async_keyword_funcdef - ) + math_funcdef_body = condense(ZeroOrMore(~(implicit_return_stmt + dedent) + stmt) - implicit_return_stmt) + math_funcdef_suite = ( + attach(implicit_return_stmt, make_suite_handle) + | condense(newline - indent - math_funcdef_body - dedent) + ) + end_func_equals = return_typedef + equals.suppress() | fixto(equals, ":") + math_funcdef = attach( + condense(addspace(keyword("def") + base_funcdef) + end_func_equals) - math_funcdef_suite, + math_funcdef_handle, + ) + math_match_funcdef = addspace( + match_def_modifiers + + attach( + base_match_funcdef + + end_func_equals + + ( + attach(implicit_return_stmt, make_suite_handle) + | ( + newline.suppress() - indent.suppress() + + Optional(docstring) + + attach(math_funcdef_body, make_suite_handle) + + dedent.suppress() + ) + ), + join_match_funcdef, + ) + ) - keyword_normal_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), + async_stmt = Forward() + async_with_for_stmt = Forward() + async_with_for_stmt_ref = ( + labeled_group( + (keyword("async") + keyword("with") + keyword("for")).suppress() + + assignlist + keyword("in").suppress() + - test + - suite_with_else_tokens, + "normal", + ) + | labeled_group( + (any_len_perm( + keyword("match"), + required=(keyword("async"), keyword("with")), + ) + keyword("for")).suppress() + + many_match + keyword("in").suppress() + - test + - suite_with_else_tokens, + "match", + ) ) - ) + (funcdef | math_funcdef) - keyword_match_funcdef = Group( - any_len_perm_at_least_one( - keyword("yield"), - keyword("copyclosure"), - keyword("match").suppress(), - # addpattern is detected later - keyword("addpattern"), + async_stmt_ref = addspace( + keyword("async") + (with_stmt | for_stmt | match_for_stmt) # handles async [match] for + | keyword("match").suppress() + keyword("async") + base_match_for_stmt # handles match async for + | async_with_for_stmt ) - ) + (def_match_funcdef | math_match_funcdef) - keyword_funcdef = Forward() - keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef - normal_funcdef_stmt = ( - funcdef - | math_funcdef - | math_match_funcdef - | match_funcdef - | keyword_funcdef - ) + async_funcdef = keyword("async").suppress() + (funcdef | math_funcdef) + async_match_funcdef = addspace( + any_len_perm( + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + (def_match_funcdef | math_match_funcdef), + ) - datadef = Forward() - data_args = Group(Optional( - lparen.suppress() + ZeroOrMore(Group( - # everything here must end with arg_comma - (unsafe_name + arg_comma.suppress())("name") - | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") - | (star.suppress() + unsafe_name + arg_comma.suppress())("star") - | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") - | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") - )) + rparen.suppress() - )) - data_inherit = Optional(keyword("from").suppress() + testlist) - data_suite = Group( - colon.suppress() - ( - (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") - | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") - | simple_stmt("simple") - ) | newline("empty") - ) - datadef_ref = ( - Optional(decorators, default="") - + keyword("data").suppress() - + classname - + Optional(type_params, default=()) - + data_args - + data_inherit - + data_suite - ) + async_keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + required=(keyword("async").suppress(),), + ) + ) + (funcdef | math_funcdef) + async_keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + required=(keyword("async").suppress(),), + ) + ) + (def_match_funcdef | math_match_funcdef) + async_keyword_funcdef = Forward() + async_keyword_funcdef_ref = async_keyword_normal_funcdef | async_keyword_match_funcdef + + async_funcdef_stmt = ( + async_funcdef + | async_match_funcdef + | async_keyword_funcdef + ) - match_datadef = Forward() - match_data_args = lparen.suppress() + Group( - match_args_list + match_guard - ) + rparen.suppress() - # we don't support type_params here since we don't support types - match_datadef_ref = ( - Optional(decorators, default="") - + Optional(keyword("match").suppress()) - + keyword("data").suppress() - + classname - + match_data_args - + data_inherit - + data_suite - ) + keyword_normal_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + ) + ) + (funcdef | math_funcdef) + keyword_match_funcdef = Group( + any_len_perm_at_least_one( + keyword("yield"), + keyword("copyclosure"), + keyword("match").suppress(), + # addpattern is detected later + keyword("addpattern"), + ) + ) + (def_match_funcdef | math_match_funcdef) + keyword_funcdef = Forward() + keyword_funcdef_ref = keyword_normal_funcdef | keyword_match_funcdef + + normal_funcdef_stmt = ( + funcdef + | math_funcdef + | math_match_funcdef + | match_funcdef + | keyword_funcdef + ) - simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") - complex_decorator = condense(namedexpr_test + newline)("complex") - decorators_ref = OneOrMore( - at.suppress() - - Group( - simple_decorator - | complex_decorator + datadef = Forward() + data_args = Group(Optional( + lparen.suppress() + ZeroOrMore(Group( + # everything here must end with arg_comma + (unsafe_name + arg_comma.suppress())("name") + | (unsafe_name + equals.suppress() + test + arg_comma.suppress())("default") + | (star.suppress() + unsafe_name + arg_comma.suppress())("star") + | (unsafe_name + colon.suppress() + typedef_test + equals.suppress() + test + arg_comma.suppress())("type default") + | (unsafe_name + colon.suppress() + typedef_test + arg_comma.suppress())("type") + )) + rparen.suppress() + )) + data_inherit = Optional(keyword("from").suppress() + testlist) + data_suite = Group( + colon.suppress() - ( + (newline.suppress() + indent.suppress() + Optional(docstring) + Group(OneOrMore(stmt)) - dedent.suppress())("complex") + | (newline.suppress() + indent.suppress() + docstring - dedent.suppress() | docstring)("docstring") + | simple_stmt("simple") + ) | newline("empty") + ) + datadef_ref = ( + Optional(decorators, default="") + + keyword("data").suppress() + + classname + + Optional(type_params, default=()) + + data_args + + data_inherit + + data_suite ) - ) - decoratable_normal_funcdef_stmt = Forward() - decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt + match_datadef = Forward() + match_data_args = lparen.suppress() + Group( + match_args_list + match_guard + ) + rparen.suppress() + # we don't support type_params here since we don't support types + match_datadef_ref = ( + Optional(decorators, default="") + + Optional(keyword("match").suppress()) + + keyword("data").suppress() + + classname + + match_data_args + + data_inherit + + data_suite + ) - decoratable_async_funcdef_stmt = Forward() - decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt + simple_decorator = condense(dotted_refname + Optional(function_call) + newline)("simple") + complex_decorator = condense(namedexpr_test + newline)("complex") + decorators_ref = OneOrMore( + at.suppress() + - Group( + simple_decorator + | complex_decorator + ) + ) - decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt + decoratable_normal_funcdef_stmt = Forward() + decoratable_normal_funcdef_stmt_ref = Optional(decorators) + normal_funcdef_stmt - # decorators are integrated into the definitions of each item here - decoratable_class_stmt = classdef | datadef | match_datadef + decoratable_async_funcdef_stmt = Forward() + decoratable_async_funcdef_stmt_ref = Optional(decorators) + async_funcdef_stmt - passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + decoratable_func_stmt = decoratable_normal_funcdef_stmt | decoratable_async_funcdef_stmt - simple_compound_stmt = ( - if_stmt - | try_stmt - | match_stmt - | passthrough_stmt - ) - compound_stmt = ( - decoratable_class_stmt - | decoratable_func_stmt - | while_stmt - | for_stmt - | with_stmt - | async_stmt - | match_for_stmt - | simple_compound_stmt - | where_stmt - ) - endline_semicolon = Forward() - endline_semicolon_ref = semicolon.suppress() + newline - keyword_stmt = ( - flow_stmt - | import_stmt - | assert_stmt - | pass_stmt - | del_stmt - | global_stmt - | nonlocal_stmt - ) - special_stmt = ( - keyword_stmt - | augassign_stmt - | typed_assign_stmt - | type_alias_stmt - ) - unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) - simple_stmt_item <<= ( - special_stmt - | basic_stmt + end_simple_stmt_item - | destructuring_stmt + end_simple_stmt_item - ) - simple_stmt <<= condense( - simple_stmt_item - + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) - + (newline | endline_semicolon) - ) - anything_stmt = Forward() - stmt <<= final( - compound_stmt - | simple_stmt - # must be after destructuring due to ambiguity - | cases_stmt - # at the very end as a fallback case for the anything parser - | anything_stmt - ) - base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) - simple_suite = attach(stmt, make_suite_handle) - nocolon_suite <<= base_suite | simple_suite - suite <<= condense(colon + nocolon_suite) - line = newline | stmt - - single_input = condense(Optional(line) - ZeroOrMore(newline)) - file_input = condense(moduledoc_marker - ZeroOrMore(line)) - eval_input = condense(testlist - ZeroOrMore(newline)) - - single_parser = start_marker - single_input - end_marker - file_parser = start_marker - file_input - end_marker - eval_parser = start_marker - eval_input - end_marker - some_eval_parser = start_marker + eval_input - - parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) - brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) - braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) - - unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) - unsafe_xonsh_command = originalTextFor( - (Optional(at) + dollar | bang) - + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) - + (parens | brackets | braces | unsafe_name) - ) - unsafe_xonsh_parser, _impl_call_ref = disable_inside( - single_parser, - unsafe_impl_call_ref, - ) - impl_call_ref <<= _impl_call_ref - xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( - unsafe_xonsh_parser, - unsafe_anything_stmt, - unsafe_xonsh_command, - ) - anything_stmt <<= _anything_stmt - xonsh_command <<= _xonsh_command + # decorators are integrated into the definitions of each item here + decoratable_class_stmt = classdef | datadef | match_datadef + + passthrough_stmt = condense(passthrough_block - (base_suite | newline)) + + simple_compound_stmt = ( + if_stmt + | try_stmt + | match_stmt + | passthrough_stmt + ) + compound_stmt = ( + decoratable_class_stmt + | decoratable_func_stmt + | while_stmt + | for_stmt + | with_stmt + | async_stmt + | match_for_stmt + | simple_compound_stmt + | where_stmt + ) + endline_semicolon = Forward() + endline_semicolon_ref = semicolon.suppress() + newline + keyword_stmt = ( + flow_stmt + | import_stmt + | assert_stmt + | pass_stmt + | del_stmt + | global_stmt + | nonlocal_stmt + ) + special_stmt = ( + keyword_stmt + | augassign_stmt + | typed_assign_stmt + | type_alias_stmt + ) + unsafe_simple_stmt_item <<= special_stmt | longest(basic_stmt, destructuring_stmt) + simple_stmt_item <<= ( + special_stmt + | basic_stmt + end_simple_stmt_item + | destructuring_stmt + end_simple_stmt_item + ) + simple_stmt <<= condense( + simple_stmt_item + + ZeroOrMore(fixto(semicolon, "\n") + simple_stmt_item) + + (newline | endline_semicolon) + ) + anything_stmt = Forward() + stmt <<= final( + compound_stmt + | simple_stmt + # must be after destructuring due to ambiguity + | cases_stmt + # at the very end as a fallback case for the anything parser + | anything_stmt + ) + base_suite <<= condense(newline + indent - OneOrMore(stmt) - dedent) + simple_suite = attach(stmt, make_suite_handle) + nocolon_suite <<= base_suite | simple_suite + suite <<= condense(colon + nocolon_suite) + line = newline | stmt + + single_input = condense(Optional(line) - ZeroOrMore(newline)) + file_input = condense(moduledoc_marker - ZeroOrMore(line)) + eval_input = condense(testlist - ZeroOrMore(newline)) + + single_parser = start_marker - single_input - end_marker + file_parser = start_marker - file_input - end_marker + eval_parser = start_marker - eval_input - end_marker + some_eval_parser = start_marker + eval_input + + parens = originalTextFor(nestedExpr("(", ")", ignoreExpr=None)) + brackets = originalTextFor(nestedExpr("[", "]", ignoreExpr=None)) + braces = originalTextFor(nestedExpr("{", "}", ignoreExpr=None)) + + unsafe_anything_stmt = originalTextFor(regex_item("[^\n]+\n+")) + unsafe_xonsh_command = originalTextFor( + (Optional(at) + dollar | bang) + + ~(lparen + rparen | lbrack + rbrack | lbrace + rbrace) + + (parens | brackets | braces | unsafe_name) + ) + unsafe_xonsh_parser, _impl_call_ref = disable_inside( + single_parser, + unsafe_impl_call_ref, + ) + impl_call_ref <<= _impl_call_ref + xonsh_parser, _anything_stmt, _xonsh_command = disable_outside( + unsafe_xonsh_parser, + unsafe_anything_stmt, + unsafe_xonsh_command, + ) + anything_stmt <<= _anything_stmt + xonsh_command <<= _xonsh_command # end: MAIN GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- # EXTRA GRAMMAR: # ----------------------------------------------------------------------------------------------------------------------- - # we don't need to include opens/closes here because those are explicitly disallowed - existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") + # we don't need to include opens/closes here because those are explicitly disallowed + existing_operator_regex = compile_regex(r"([.;\\]|([+-=@%^&|*:,/<>~]|\*\*|//|>>|<<)=?|!=|" + r"|".join(new_operators) + r")$") - whitespace_regex = compile_regex(r"\s") + whitespace_regex = compile_regex(r"\s") - def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") - yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") - yield_from_regex = compile_regex(r"\byield\s+from\b") + def_regex = compile_regex(r"\b((async|addpattern|copyclosure)\s+)*def\b") + yield_regex = compile_regex(r"\byield(?!\s+_coconut\.asyncio\.From)\b") + yield_from_regex = compile_regex(r"\byield\s+from\b") - tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") - return_regex = compile_regex(r"\breturn\b") + tco_disable_regex = compile_regex(r"\b(try\b|(async\s+)?(with\b|for\b)|while\b)") + return_regex = compile_regex(r"\breturn\b") - noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") + noqa_regex = compile_regex(r"\b[Nn][Oo][Qq][Aa]\b") - just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker + just_non_none_atom = start_marker + ~keyword("None") + known_atom + end_marker - original_function_call_tokens = ( - lparen.suppress() + rparen.suppress() - # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not - | condense(lparen + originalTextFor(test + comp_for) + rparen) - | attach(parens, strip_parens_handle) - ) + original_function_call_tokens = ( + lparen.suppress() + rparen.suppress() + # we need to keep the parens here, since f(x for x in y) is fine but tail_call(f, x for x in y) is not + | condense(lparen + originalTextFor(test + comp_for) + rparen) + | attach(parens, strip_parens_handle) + ) - tre_func_name = Forward() - tre_return = ( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - tre_func_name + original_function_call_tokens, - rparen, - ) + end_marker - ) + tre_func_name = Forward() + tre_return = ( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + tre_func_name + original_function_call_tokens, + rparen, + ) + end_marker + ) - tco_return = attach( - start_marker - + keyword("return").suppress() - + maybeparens( - lparen, - disallow_keywords(untcoable_funcs, with_suffix="(") - + condense( - (unsafe_name | parens | brackets | braces | string_atom) - + ZeroOrMore( - dot + unsafe_name - | brackets - # don't match the last set of parentheses - | parens + ~end_marker + ~rparen - ), - ) - + original_function_call_tokens, - rparen, - ) + end_marker, - tco_return_handle, - # this is the root in what it's used for, so might as well evaluate greedily - greedy=True, - ) + tco_return = attach( + start_marker + + keyword("return").suppress() + + maybeparens( + lparen, + disallow_keywords(untcoable_funcs, with_suffix="(") + + condense( + (unsafe_name | parens | brackets | braces | string_atom) + + ZeroOrMore( + dot + unsafe_name + | brackets + # don't match the last set of parentheses + | parens + ~end_marker + ~rparen + ), + ) + + original_function_call_tokens, + rparen, + ) + end_marker, + tco_return_handle, + # this is the root in what it's used for, so might as well evaluate greedily + greedy=True, + ) - rest_of_lambda = Forward() - lambdas = keyword("lambda") - rest_of_lambda - colon - rest_of_lambda <<= ZeroOrMore( - # handle anything that could capture colon - parens - | brackets - | braces - | lambdas - | ~colon + any_char - ) - rest_of_tfpdef = originalTextFor( - ZeroOrMore( - # handle anything that could capture comma, rparen, or equals + rest_of_lambda = Forward() + lambdas = keyword("lambda") - rest_of_lambda - colon + rest_of_lambda <<= ZeroOrMore( + # handle anything that could capture colon parens | brackets | braces | lambdas - | ~comma + ~rparen + ~equals + any_char + | ~colon + any_char + ) + rest_of_tfpdef = originalTextFor( + ZeroOrMore( + # handle anything that could capture comma, rparen, or equals + parens + | brackets + | braces + | lambdas + | ~comma + ~rparen + ~equals + any_char + ) + ) + tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() + tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) + type_comment = Optional( + comment_tokens + | passthrough_item + ).suppress() + parameters_tokens = Group( + Optional(tokenlist( + Group( + dubstar - tfpdef_tokens + | star - Optional(tfpdef_tokens) + | slash + | tfpdef_default_tokens + ) + type_comment, + comma + type_comment, + )) ) - ) - tfpdef_tokens = unsafe_name - Optional(colon - rest_of_tfpdef).suppress() - tfpdef_default_tokens = tfpdef_tokens - Optional(equals - rest_of_tfpdef) - type_comment = Optional( - comment_tokens - | passthrough_item - ).suppress() - parameters_tokens = Group( - Optional(tokenlist( - Group( - dubstar - tfpdef_tokens - | star - Optional(tfpdef_tokens) - | slash - | tfpdef_default_tokens - ) + type_comment, - comma + type_comment, - )) - ) - split_func = ( - start_marker - - keyword("def").suppress() - - unsafe_dotted_name - - Optional(brackets).suppress() - - lparen.suppress() - parameters_tokens - rparen.suppress() - ) + split_func = ( + start_marker + - keyword("def").suppress() + - unsafe_dotted_name + - Optional(brackets).suppress() + - lparen.suppress() - parameters_tokens - rparen.suppress() + ) - stores_scope = boundary + ( - keyword("lambda") - # match comprehensions but not for loops - | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") - ) + stores_scope = boundary + ( + keyword("lambda") + # match comprehensions but not for loops + | ~indent + ~dedent + any_char + keyword("for") + unsafe_name + keyword("in") + ) - just_a_string = start_marker + string_atom + end_marker + just_a_string = start_marker + string_atom + end_marker - end_of_line = end_marker | Literal("\n") | pound + end_of_line = end_marker | Literal("\n") | pound - unsafe_equals = Literal("=") + unsafe_equals = Literal("=") - kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) - parse_err_msg = ( - start_marker + ( - fixto(end_of_line, "misplaced newline (maybe missing ':')") - | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") - | kwd_err_msg - ) - | fixto( - questionmark - + ~dollar - + ~lparen - + ~lbrack - + ~dot, - "misplaced '?' (naked '?' is only supported inside partial application arguments)", + kwd_err_msg = attach(any_keyword_in(keyword_vars + reserved_vars), kwd_err_msg_handle) + parse_err_msg = ( + start_marker + ( + fixto(end_of_line, "misplaced newline (maybe missing ':')") + | fixto(Optional(keyword("if") + skip_to_in_line(unsafe_equals)) + equals, "misplaced assignment (maybe should be '==')") + | kwd_err_msg + ) + | fixto( + questionmark + + ~dollar + + ~lparen + + ~lbrack + + ~dot, + "misplaced '?' (naked '?' is only supported inside partial application arguments)", + ) ) - ) - end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) + end_f_str_expr = combine(start_marker + (bang | colon | rbrace)) - string_start = start_marker + python_quoted_string + string_start = start_marker + python_quoted_string - no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker + no_unquoted_newlines = start_marker + ZeroOrMore(python_quoted_string | ~Literal("\n") + any_char) + end_marker - operator_stmt = ( - start_marker - + keyword("operator").suppress() - + restOfLine - ) + operator_stmt = ( + start_marker + + keyword("operator").suppress() + + restOfLine + ) - unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) - from_import_operator = ( - start_marker - + keyword("from").suppress() - + unsafe_import_from_name - + keyword("import").suppress() - + keyword("operator").suppress() - + restOfLine - ) + unsafe_import_from_name = condense(ZeroOrMore(unsafe_dot) + unsafe_dotted_name | OneOrMore(unsafe_dot)) + from_import_operator = ( + start_marker + + keyword("from").suppress() + + unsafe_import_from_name + + keyword("import").suppress() + + keyword("operator").suppress() + + restOfLine + ) # end: EXTRA GRAMMAR # ----------------------------------------------------------------------------------------------------------------------- diff --git a/coconut/compiler/util.py b/coconut/compiler/util.py index db97100e8..4e8a138ee 100644 --- a/coconut/compiler/util.py +++ b/coconut/compiler/util.py @@ -45,6 +45,7 @@ import cPickle as pickle from coconut._pyparsing import ( + MODERN_PYPARSING, USE_COMPUTATION_GRAPH, SUPPORTS_INCREMENTAL, USE_ADAPTIVE, @@ -66,6 +67,7 @@ Group, ParserElement, MatchFirst, + And, _trim_arity, _ParseResultsWithOffset, all_parse_elements, @@ -324,13 +326,65 @@ def postParse(self, original, loc, tokens): combine = Combine +def maybe_copy_elem(item, name): + """Copy the given grammar element if it's referenced somewhere else.""" + item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") + internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) + if item_ref_count <= temp_grammar_item_ref_count: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, False) + return item + else: + if DEVELOP: + logger.record_stat("maybe_copy_" + name, True) + return item.copy() + + +def hasaction(elem): + """Determine if the given grammar element has any actions associated with it.""" + return ( + MODERN_PYPARSING + or elem.parseAction + or elem.resultsName is not None + or elem.debug + ) + + +@contextmanager +def using_fast_grammar_methods(): + """Enables grammar methods that modify their operands when they aren't referenced elsewhere.""" + if MODERN_PYPARSING: + yield + return + + def fast_add(self, other): + if hasaction(self): + return old_add(self, other) + self = maybe_copy_elem(self, "add") + self += other + return self + old_add, And.__add__ = And.__add__, fast_add + + def fast_or(self, other): + if hasaction(self): + return old_or(self, other) + self = maybe_copy_elem(self, "or") + self |= other + return self + old_or, MatchFirst.__or__ = MatchFirst.__or__, fast_or + + try: + yield + finally: + And.__add__ = old_add + MatchFirst.__or__ = old_or + + def add_action(item, action, make_copy=None): """Add a parse action to the given item.""" if make_copy is None: - item_ref_count = sys.getrefcount(item) if CPYTHON and not on_new_python else float("inf") - internal_assert(lambda: item_ref_count >= temp_grammar_item_ref_count, "add_action got item with too low ref count", (item, type(item), item_ref_count)) - make_copy = item_ref_count > temp_grammar_item_ref_count - if make_copy: + item = maybe_copy_elem(item, "attach") + elif make_copy: item = item.copy() return item.addParseAction(action) @@ -386,10 +440,10 @@ def adaptive_manager(item, original, loc, reparse=False): except Exception as exc: if DEVELOP: logger.log("reparsing due to:", exc) - logger.record_adaptive_stat(False) + logger.record_stat("adaptive", False) else: if DEVELOP: - logger.record_adaptive_stat(True) + logger.record_stat("adaptive", True) finally: MatchFirst.setAdaptiveMode(False) @@ -783,10 +837,9 @@ class MatchAny(MatchFirst): adaptive_mode = True -def any_of(match_first): +def any_of(*exprs): """Build a MatchAny of the given MatchFirst.""" - internal_assert(isinstance(match_first, MatchFirst), "invalid any_of target", match_first) - return MatchAny(match_first.exprs) + return MatchAny(exprs) class Wrap(ParseElementEnhance): diff --git a/coconut/constants.py b/coconut/constants.py index e9379f00a..7c0a8c11c 100644 --- a/coconut/constants.py +++ b/coconut/constants.py @@ -153,8 +153,8 @@ def get_path_env_var(env_var, default): embed_on_internal_exc = False assert not embed_on_internal_exc or DEVELOP, "embed_on_internal_exc should never be enabled on non-develop build" -# should be the minimal ref count observed by attach -temp_grammar_item_ref_count = 3 if PY311 else 5 +# should be the minimal ref count observed by maybe_copy_elem +temp_grammar_item_ref_count = 4 if PY311 else 5 minimum_recursion_limit = 128 # shouldn't be raised any higher to avoid stack overflows diff --git a/coconut/terminal.py b/coconut/terminal.py index c0fcf1809..2833558ce 100644 --- a/coconut/terminal.py +++ b/coconut/terminal.py @@ -183,14 +183,17 @@ def logging(self): class Logger(object): """Container object for various logger functions and variables.""" force_verbose = force_verbose_logger + colors_enabled = False + verbose = force_verbose quiet = False path = None name = None - colors_enabled = False tracing = False trace_ind = 0 + recorded_stats = defaultdict(lambda: [0, 0]) + def __init__(self, other=None): """Create a logger, optionally from another logger.""" if other is not None: @@ -522,19 +525,15 @@ def trace(self, item): item.debug = True return item - adaptive_stats = None - - def record_adaptive_stat(self, success): - if self.verbose: - if self.adaptive_stats is None: - self.adaptive_stats = [0, 0] - self.adaptive_stats[success] += 1 + def record_stat(self, stat_name, stat_bool): + """Record the given boolean statistic for the given stat_name.""" + self.recorded_stats[stat_name][stat_bool] += 1 @contextmanager def gather_parsing_stats(self): """Times parsing if --verbose.""" if self.verbose: - self.adaptive_stats = None + self.recorded_stats.pop("adaptive", None) start_time = get_clock_time() try: yield @@ -547,8 +546,8 @@ def gather_parsing_stats(self): # reset stats after printing if in incremental mode if ParserElement._incrementalEnabled: ParserElement.packrat_cache_stats[:] = [0] * len(ParserElement.packrat_cache_stats) - if self.adaptive_stats: - failures, successes = self.adaptive_stats + if "adaptive" in self.recorded_stats: + failures, successes = self.recorded_stats["adaptive"] self.printlog("\tAdaptive parsing stats:", successes, "successes;", failures, "failures") else: yield diff --git a/coconut/tests/src/cocotest/agnostic/primary_2.coco b/coconut/tests/src/cocotest/agnostic/primary_2.coco index 3f9d63a65..0bc92d989 100644 --- a/coconut/tests/src/cocotest/agnostic/primary_2.coco +++ b/coconut/tests/src/cocotest/agnostic/primary_2.coco @@ -408,6 +408,7 @@ def primary_test_2() -> bool: assert_raises(-> collectby(.[0], [(0, 1), (0, 2)], reduce_func=False), ValueError) # type: ignore assert ident$(x=?).__name__ == "ident" == ident$(1).__name__ # type: ignore assert collectby(.[0], [(0, 1), (0, 2)], value_func=.[1], reduce_func=(+), reduce_func_init=1) == {0: 4} + assert ident$(1, ?) |> type == ident$(1) |> type with process_map.multiple_sequential_calls(): # type: ignore assert map((+), range(3), range(4)$[:-1], strict=True) |> list == [0, 2, 4] == process_map((+), range(3), range(4)$[:-1], strict=True) |> list # type: ignore diff --git a/coconut/util.py b/coconut/util.py index a5d68f39c..4b4338a15 100644 --- a/coconut/util.py +++ b/coconut/util.py @@ -92,6 +92,20 @@ def __reduce_ex__(self, _): return self.__reduce__() +class const(pickleable_obj): + """Implementaiton of Coconut's const for use within Coconut.""" + __slots__ = ("value",) + + def __init__(self, value): + self.value = value + + def __reduce__(self): + return (self.__class__, (self.value,)) + + def __call__(self, *args, **kwargs): + return self.value + + class override(pickleable_obj): """Implementation of Coconut's @override for use within Coconut.""" __slots__ = ("func",) @@ -273,6 +287,11 @@ def ensure_dir(dirpath): os.makedirs(dirpath) +def without_keys(inputdict, rem_keys): + """Get a copy of inputdict without rem_keys.""" + return {k: v for k, v in inputdict.items() if k not in rem_keys} + + # ----------------------------------------------------------------------------------------------------------------------- # VERSIONING: # -----------------------------------------------------------------------------------------------------------------------