From e24d7721ddd8e708735f7e3b16c50b81cbfb4a89 Mon Sep 17 00:00:00 2001 From: Valentin Berlier Date: Sun, 17 Jul 2022 05:36:39 +0200 Subject: [PATCH] feat: add raw strings --- bolt/parse.py | 12 +- bolt/pattern.py | 2 +- .../src/data/demo/functions/foo.mcfunction | 13 +- .../src/data/demo/functions/util.mcfunction | 17 ++ tests/resources/bolt_examples.mcfunction | 13 ++ tests/snapshots/bolt__parse_241__0.txt | 26 +++ tests/snapshots/bolt__parse_241__1.txt | 16 ++ tests/snapshots/bolt__parse_242__0.txt | 5 + tests/snapshots/bolt__parse_243__0.txt | 26 +++ tests/snapshots/bolt__parse_243__1.txt | 16 ++ tests/snapshots/bolt__parse_244__0.txt | 26 +++ tests/snapshots/bolt__parse_244__1.txt | 16 ++ tests/snapshots/bolt__parse_245__0.txt | 160 ++++++++++++++++++ tests/snapshots/bolt__parse_245__1.txt | 32 ++++ ...examples__build_bolt_proc_macro__0.pack.md | 5 + 15 files changed, 381 insertions(+), 4 deletions(-) create mode 100644 tests/snapshots/bolt__parse_241__0.txt create mode 100644 tests/snapshots/bolt__parse_241__1.txt create mode 100644 tests/snapshots/bolt__parse_242__0.txt create mode 100644 tests/snapshots/bolt__parse_243__0.txt create mode 100644 tests/snapshots/bolt__parse_243__1.txt create mode 100644 tests/snapshots/bolt__parse_244__0.txt create mode 100644 tests/snapshots/bolt__parse_244__1.txt create mode 100644 tests/snapshots/bolt__parse_245__0.txt create mode 100644 tests/snapshots/bolt__parse_245__1.txt diff --git a/bolt/parse.py b/bolt/parse.py index 9224e50..f3b47c4 100644 --- a/bolt/parse.py +++ b/bolt/parse.py @@ -1863,7 +1863,11 @@ def __call__(self, stream: TokenStream) -> Any: continue if string: - value = self.quote_helper.unquote_string(string) + value = ( + string.value[2:-1] + if string.value.startswith("r") + else self.quote_helper.unquote_string(string) + ) elif number: value = int(number.value) @@ -2034,7 +2038,11 @@ def __call__(self, stream: TokenStream) -> Any: elif null: value = None elif string: - value = self.quote_helper.unquote_string(string) + value = ( + string.value[2:-1] + if string.value.startswith("r") + else self.quote_helper.unquote_string(string) + ) elif resource: if resource.value.startswith(("./", "../")): value = ":".join( diff --git a/bolt/pattern.py b/bolt/pattern.py index d126682..3ff94d8 100644 --- a/bolt/pattern.py +++ b/bolt/pattern.py @@ -20,6 +20,6 @@ NULL_PATTERN: str = r"\b(?:null|None)\b" IDENTIFIER_PATTERN: str = rf"(?!_bolt_|{KEYWORD_PATTERN})[a-zA-Z_][a-zA-Z0-9_]*\b" MODULE_PATTERN: str = rf"{IDENTIFIER_PATTERN}(?:\.{IDENTIFIER_PATTERN})*" -STRING_PATTERN: str = r'"(?:\\.|[^\\\n])*?"' "|" r"'(?:\\.|[^\\\n])*?'" +STRING_PATTERN: str = r'r?"(?:\\.|[^\\\n])*?"' "|" r"r?'(?:\\.|[^\\\n])*?'" NUMBER_PATTERN: str = r"(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?\b" RESOURCE_LOCATION_PATTERN: str = r"(?:\.\./|\./|[0-9a-z_\-\.]+:)[0-9a-z_./-]+" diff --git a/examples/bolt_proc_macro/src/data/demo/functions/foo.mcfunction b/examples/bolt_proc_macro/src/data/demo/functions/foo.mcfunction index 690f0a2..84c46d3 100644 --- a/examples/bolt_proc_macro/src/data/demo/functions/foo.mcfunction +++ b/examples/bolt_proc_macro/src/data/demo/functions/foo.mcfunction @@ -1,4 +1,4 @@ -from ./util import repeat +from ./util import repeat, lisp value = 3 @@ -17,3 +17,14 @@ repeat until value < 5: repeat until @a[scores={counter=10}]: scoreboard players add @r counter 1 + +lisp foo +lisp 123 +lisp (foo 123) +lisp (foo 123 () (bar)) +lisp ( + (defun do_math (x y) + (add x y)) + (defun do_more_math (x y) + (mul (do_math x y) y)) +) diff --git a/examples/bolt_proc_macro/src/data/demo/functions/util.mcfunction b/examples/bolt_proc_macro/src/data/demo/functions/util.mcfunction index d998e31..c298a52 100644 --- a/examples/bolt_proc_macro/src/data/demo/functions/util.mcfunction +++ b/examples/bolt_proc_macro/src/data/demo/functions/util.mcfunction @@ -25,3 +25,20 @@ macro repeat until(stream): execute function path: yield body unless entity entity function path + +def parse_sexp(stream): + with stream.syntax(brace=r"\(|\)", number=r"\d+", name=r"\w+"): + brace, number, name = stream.expect(("brace", "("), "number", "name") + if brace: + with stream.ignore("newline"): + result = [] + while not stream.get(("brace", ")")): + result.append(parse_sexp(stream)) + return result + elif number: + return int(number.value) + elif name: + return name.value + +macro lisp(stream): + say parse_sexp(stream) diff --git a/tests/resources/bolt_examples.mcfunction b/tests/resources/bolt_examples.mcfunction index 4b7c4e6..c6f2e31 100644 --- a/tests/resources/bolt_examples.mcfunction +++ b/tests/resources/bolt_examples.mcfunction @@ -895,3 +895,16 @@ foo = { a: 1, "b" + "c": 2 } +### +foo = r"" +### +foo = r"\" +### +foo = r"\\" +### +foo = r"\"" +### +macro foo(stream): + with stream.syntax(number=r"\d+", name=r"\w+"): + stream.expect("number") + stream.expect("name") diff --git a/tests/snapshots/bolt__parse_241__0.txt b/tests/snapshots/bolt__parse_241__0.txt new file mode 100644 index 0000000..dc23e8f --- /dev/null +++ b/tests/snapshots/bolt__parse_241__0.txt @@ -0,0 +1,26 @@ +foo = r"" +--- + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=10, lineno=2, colno=1) + commands: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=9, lineno=1, colno=10) + identifier: 'statement' + arguments: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=9, lineno=1, colno=10) + operator: '=' + target: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=3, lineno=1, colno=4) + value: 'foo' + rebind: False + value: + + location: SourceLocation(pos=6, lineno=1, colno=7) + end_location: SourceLocation(pos=9, lineno=1, colno=10) + value: '' diff --git a/tests/snapshots/bolt__parse_241__1.txt b/tests/snapshots/bolt__parse_241__1.txt new file mode 100644 index 0000000..21d51a9 --- /dev/null +++ b/tests/snapshots/bolt__parse_241__1.txt @@ -0,0 +1,16 @@ +_bolt_lineno = [1], [1] +_bolt_helper_children = _bolt_runtime.helpers['children'] +_bolt_helper_replace = _bolt_runtime.helpers['replace'] +with _bolt_runtime.scope() as _bolt_var1: + _bolt_var0 = '' + foo = _bolt_var0 +_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1)) +--- +output = _bolt_var2 +--- +_bolt_refs[0] + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=10, lineno=2, colno=1) + commands: + diff --git a/tests/snapshots/bolt__parse_242__0.txt b/tests/snapshots/bolt__parse_242__0.txt new file mode 100644 index 0000000..4ead4f1 --- /dev/null +++ b/tests/snapshots/bolt__parse_242__0.txt @@ -0,0 +1,5 @@ +#>ERROR Identifier "r" is not defined. +# line 1, column 7 +# 1 | foo = r"\" +# : ^ +foo = r"\" diff --git a/tests/snapshots/bolt__parse_243__0.txt b/tests/snapshots/bolt__parse_243__0.txt new file mode 100644 index 0000000..314a55e --- /dev/null +++ b/tests/snapshots/bolt__parse_243__0.txt @@ -0,0 +1,26 @@ +foo = r"\\" +--- + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=12, lineno=2, colno=1) + commands: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + identifier: 'statement' + arguments: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + operator: '=' + target: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=3, lineno=1, colno=4) + value: 'foo' + rebind: False + value: + + location: SourceLocation(pos=6, lineno=1, colno=7) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + value: '\\\\' diff --git a/tests/snapshots/bolt__parse_243__1.txt b/tests/snapshots/bolt__parse_243__1.txt new file mode 100644 index 0000000..39ec0d2 --- /dev/null +++ b/tests/snapshots/bolt__parse_243__1.txt @@ -0,0 +1,16 @@ +_bolt_lineno = [1], [1] +_bolt_helper_children = _bolt_runtime.helpers['children'] +_bolt_helper_replace = _bolt_runtime.helpers['replace'] +with _bolt_runtime.scope() as _bolt_var1: + _bolt_var0 = '\\\\' + foo = _bolt_var0 +_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1)) +--- +output = _bolt_var2 +--- +_bolt_refs[0] + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=12, lineno=2, colno=1) + commands: + diff --git a/tests/snapshots/bolt__parse_244__0.txt b/tests/snapshots/bolt__parse_244__0.txt new file mode 100644 index 0000000..1575fe1 --- /dev/null +++ b/tests/snapshots/bolt__parse_244__0.txt @@ -0,0 +1,26 @@ +foo = r"\"" +--- + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=12, lineno=2, colno=1) + commands: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + identifier: 'statement' + arguments: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + operator: '=' + target: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=3, lineno=1, colno=4) + value: 'foo' + rebind: False + value: + + location: SourceLocation(pos=6, lineno=1, colno=7) + end_location: SourceLocation(pos=11, lineno=1, colno=12) + value: '\\"' diff --git a/tests/snapshots/bolt__parse_244__1.txt b/tests/snapshots/bolt__parse_244__1.txt new file mode 100644 index 0000000..132fabe --- /dev/null +++ b/tests/snapshots/bolt__parse_244__1.txt @@ -0,0 +1,16 @@ +_bolt_lineno = [1], [1] +_bolt_helper_children = _bolt_runtime.helpers['children'] +_bolt_helper_replace = _bolt_runtime.helpers['replace'] +with _bolt_runtime.scope() as _bolt_var1: + _bolt_var0 = '\\"' + foo = _bolt_var0 +_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1)) +--- +output = _bolt_var2 +--- +_bolt_refs[0] + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=12, lineno=2, colno=1) + commands: + diff --git a/tests/snapshots/bolt__parse_245__0.txt b/tests/snapshots/bolt__parse_245__0.txt new file mode 100644 index 0000000..783f176 --- /dev/null +++ b/tests/snapshots/bolt__parse_245__0.txt @@ -0,0 +1,160 @@ +macro foo(stream): + with stream.syntax(number=r"\d+", name=r"\w+"): + stream.expect("number") + stream.expect("name") +--- + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + commands: + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + identifier: 'foo:proc_macro_overload3' + arguments: + + location: SourceLocation(pos=6, lineno=1, colno=7) + end_location: SourceLocation(pos=9, lineno=1, colno=10) + value: 'foo' + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + match_identifier: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + value: 'proc_macro_overload3' + match_argument_parser: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + is_tag: False + namespace: 'bolt' + path: 'proc_macro' + match_argument_properties: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + entries: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + key: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + value: 'resource_location' + value: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + value: 'demo:test' + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + key: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + value: 'identifier' + value: + + location: SourceLocation(pos=-1, lineno=0, colno=0) + end_location: SourceLocation(pos=-1, lineno=0, colno=0) + value: 'foo:proc_macro_overload3' + + location: SourceLocation(pos=23, lineno=2, colno=5) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + commands: + + location: SourceLocation(pos=23, lineno=2, colno=5) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + identifier: 'with:subcommand' + arguments: + + location: SourceLocation(pos=28, lineno=2, colno=10) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + identifier: 'with:expression:body' + arguments: + + location: SourceLocation(pos=28, lineno=2, colno=10) + end_location: SourceLocation(pos=69, lineno=2, colno=51) + value: + + location: SourceLocation(pos=28, lineno=2, colno=10) + end_location: SourceLocation(pos=41, lineno=2, colno=23) + name: 'syntax' + value: + + location: SourceLocation(pos=28, lineno=2, colno=10) + end_location: SourceLocation(pos=34, lineno=2, colno=16) + value: 'stream' + arguments: + + location: SourceLocation(pos=42, lineno=2, colno=24) + end_location: SourceLocation(pos=55, lineno=2, colno=37) + name: 'number' + value: + + location: SourceLocation(pos=49, lineno=2, colno=31) + end_location: SourceLocation(pos=55, lineno=2, colno=37) + value: '\\d+' + + location: SourceLocation(pos=57, lineno=2, colno=39) + end_location: SourceLocation(pos=68, lineno=2, colno=50) + name: 'name' + value: + + location: SourceLocation(pos=62, lineno=2, colno=44) + end_location: SourceLocation(pos=68, lineno=2, colno=50) + value: '\\w+' + + location: SourceLocation(pos=79, lineno=3, colno=9) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + commands: + + location: SourceLocation(pos=79, lineno=3, colno=9) + end_location: SourceLocation(pos=102, lineno=3, colno=32) + identifier: 'statement' + arguments: + + location: SourceLocation(pos=79, lineno=3, colno=9) + end_location: SourceLocation(pos=102, lineno=3, colno=32) + value: + + location: SourceLocation(pos=79, lineno=3, colno=9) + end_location: SourceLocation(pos=92, lineno=3, colno=22) + name: 'expect' + value: + + location: SourceLocation(pos=79, lineno=3, colno=9) + end_location: SourceLocation(pos=85, lineno=3, colno=15) + value: 'stream' + arguments: + + location: SourceLocation(pos=93, lineno=3, colno=23) + end_location: SourceLocation(pos=101, lineno=3, colno=31) + value: 'number' + + location: SourceLocation(pos=111, lineno=4, colno=9) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + identifier: 'statement' + arguments: + + location: SourceLocation(pos=111, lineno=4, colno=9) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + value: + + location: SourceLocation(pos=111, lineno=4, colno=9) + end_location: SourceLocation(pos=124, lineno=4, colno=22) + name: 'expect' + value: + + location: SourceLocation(pos=111, lineno=4, colno=9) + end_location: SourceLocation(pos=117, lineno=4, colno=15) + value: 'stream' + arguments: + + location: SourceLocation(pos=125, lineno=4, colno=23) + end_location: SourceLocation(pos=131, lineno=4, colno=29) + value: 'name' diff --git a/tests/snapshots/bolt__parse_245__1.txt b/tests/snapshots/bolt__parse_245__1.txt new file mode 100644 index 0000000..5cadd6f --- /dev/null +++ b/tests/snapshots/bolt__parse_245__1.txt @@ -0,0 +1,32 @@ +_bolt_lineno = [1, 8, 14, 18], [1, 2, 3, 4] +_bolt_helper_get_attribute = _bolt_runtime.helpers['get_attribute'] +_bolt_helper_children = _bolt_runtime.helpers['children'] +_bolt_proc_macros = {} +_bolt_helper_replace = _bolt_runtime.helpers['replace'] +with _bolt_runtime.scope() as _bolt_var7: + def _bolt_macro0(stream): + _bolt_var0 = stream + _bolt_var0 = _bolt_helper_get_attribute(_bolt_var0, 'syntax') + _bolt_var1 = '\\d+' + _bolt_var2 = '\\w+' + _bolt_var0 = _bolt_var0(number=_bolt_var1, name=_bolt_var2) + with _bolt_var0: + _bolt_var3 = stream + _bolt_var3 = _bolt_helper_get_attribute(_bolt_var3, 'expect') + _bolt_var4 = 'number' + _bolt_var3 = _bolt_var3(_bolt_var4) + _bolt_var5 = stream + _bolt_var5 = _bolt_helper_get_attribute(_bolt_var5, 'expect') + _bolt_var6 = 'name' + _bolt_var5 = _bolt_var5(_bolt_var6) + _bolt_proc_macros['foo:proc_macro_overload3'] = _bolt_macro0 +_bolt_var8 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var7)) +--- +output = _bolt_var8 +--- +_bolt_refs[0] + + location: SourceLocation(pos=0, lineno=1, colno=1) + end_location: SourceLocation(pos=132, lineno=4, colno=30) + commands: + diff --git a/tests/snapshots/examples__build_bolt_proc_macro__0.pack.md b/tests/snapshots/examples__build_bolt_proc_macro__0.pack.md index 73b7656..0e24b78 100644 --- a/tests/snapshots/examples__build_bolt_proc_macro__0.pack.md +++ b/tests/snapshots/examples__build_bolt_proc_macro__0.pack.md @@ -31,6 +31,11 @@ say 7 say 6 say 5 function bolt_proc_macro:generated_0 +say foo +say 123 +say ['foo', 123] +say ['foo', 123, [], ['bar']] +say [['defun', 'do_math', ['x', 'y'], ['add', 'x', 'y']], ['defun', 'do_more_math', ['x', 'y'], ['mul', ['do_math', 'x', 'y'], 'y']]] ``` `@function(strip_final_newline) demo:util`