feat: add raw strings

bolt/parse.py

@@ -1863,7 +1863,11 @@ def __call__(self, stream: TokenStream) -> Any:
                         continue
 
                     if string:
-                        value = self.quote_helper.unquote_string(string)
+                        value = (
+                            string.value[2:-1]
+                            if string.value.startswith("r")
+                            else self.quote_helper.unquote_string(string)
+                        )
                     elif number:
                         value = int(number.value)
 
@@ -2034,7 +2038,11 @@ def __call__(self, stream: TokenStream) -> Any:
             elif null:
                 value = None
             elif string:
-                value = self.quote_helper.unquote_string(string)
+                value = (
+                    string.value[2:-1]
+                    if string.value.startswith("r")
+                    else self.quote_helper.unquote_string(string)
+                )
             elif resource:
                 if resource.value.startswith(("./", "../")):
                     value = ":".join(

bolt/pattern.py

@@ -20,6 +20,6 @@
 NULL_PATTERN: str = r"\b(?:null|None)\b"
 IDENTIFIER_PATTERN: str = rf"(?!_bolt_|{KEYWORD_PATTERN})[a-zA-Z_][a-zA-Z0-9_]*\b"
 MODULE_PATTERN: str = rf"{IDENTIFIER_PATTERN}(?:\.{IDENTIFIER_PATTERN})*"
-STRING_PATTERN: str = r'"(?:\\.|[^\\\n])*?"' "|" r"'(?:\\.|[^\\\n])*?'"
+STRING_PATTERN: str = r'r?"(?:\\.|[^\\\n])*?"' "|" r"r?'(?:\\.|[^\\\n])*?'"
 NUMBER_PATTERN: str = r"(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?\b"
 RESOURCE_LOCATION_PATTERN: str = r"(?:\.\./|\./|[0-9a-z_\-\.]+:)[0-9a-z_./-]+"

examples/bolt_proc_macro/src/data/demo/functions/foo.mcfunction

@@ -1,4 +1,4 @@
-from ./util import repeat
+from ./util import repeat, lisp
 
 value = 3
 
@@ -17,3 +17,14 @@ repeat until value < 5:
 
 repeat until @a[scores={counter=10}]:
     scoreboard players add @r counter 1
+
+lisp foo
+lisp 123
+lisp (foo 123)
+lisp (foo 123 () (bar))
+lisp (
+    (defun do_math (x y)
+        (add x y))
+    (defun do_more_math (x y)
+        (mul (do_math x y) y))
+)

examples/bolt_proc_macro/src/data/demo/functions/util.mcfunction

@@ -25,3 +25,20 @@ macro repeat until(stream):
     execute function path:
         yield body
         unless entity entity function path
+
+def parse_sexp(stream):
+    with stream.syntax(brace=r"\(|\)", number=r"\d+", name=r"\w+"):
+        brace, number, name = stream.expect(("brace", "("), "number", "name")
+        if brace:
+            with stream.ignore("newline"):
+                result = []
+                while not stream.get(("brace", ")")):
+                    result.append(parse_sexp(stream))
+                return result
+        elif number:
+            return int(number.value)
+        elif name:
+            return name.value
+
+macro lisp(stream):
+    say parse_sexp(stream)

tests/resources/bolt_examples.mcfunction

@@ -895,3 +895,16 @@ foo = {
     a: 1,
     "b" + "c": 2
 }
+###
+foo = r""
+###
+foo = r"\"
+###
+foo = r"\\"
+###
+foo = r"\""
+###
+macro foo(stream):
+    with stream.syntax(number=r"\d+", name=r"\w+"):
+        stream.expect("number")
+        stream.expect("name")

tests/snapshots/bolt__parse_241__0.txt

@@ -0,0 +1,26 @@
+foo = r""
+---
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=10, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>
+      location: SourceLocation(pos=0, lineno=1, colno=1)
+      end_location: SourceLocation(pos=9, lineno=1, colno=10)
+      identifier: 'statement'
+      arguments:
+        <class 'bolt.ast.AstAssignment'>
+          location: SourceLocation(pos=0, lineno=1, colno=1)
+          end_location: SourceLocation(pos=9, lineno=1, colno=10)
+          operator: '='
+          target:
+            <class 'bolt.ast.AstTargetIdentifier'>
+              location: SourceLocation(pos=0, lineno=1, colno=1)
+              end_location: SourceLocation(pos=3, lineno=1, colno=4)
+              value: 'foo'
+              rebind: False
+          value:
+            <class 'bolt.ast.AstValue'>
+              location: SourceLocation(pos=6, lineno=1, colno=7)
+              end_location: SourceLocation(pos=9, lineno=1, colno=10)
+              value: ''

tests/snapshots/bolt__parse_241__1.txt

@@ -0,0 +1,16 @@
+_bolt_lineno = [1], [1]
+_bolt_helper_children = _bolt_runtime.helpers['children']
+_bolt_helper_replace = _bolt_runtime.helpers['replace']
+with _bolt_runtime.scope() as _bolt_var1:
+    _bolt_var0 = ''
+    foo = _bolt_var0
+_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1))
+---
+output = _bolt_var2
+---
+_bolt_refs[0]
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=10, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>

tests/snapshots/bolt__parse_242__0.txt

@@ -0,0 +1,5 @@
+#>ERROR  Identifier "r" is not defined.
+# line 1, column 7
+#      1 |  foo = r"\"
+#        :        ^
+foo = r"\"

tests/snapshots/bolt__parse_243__0.txt

@@ -0,0 +1,26 @@
+foo = r"\\"
+---
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=12, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>
+      location: SourceLocation(pos=0, lineno=1, colno=1)
+      end_location: SourceLocation(pos=11, lineno=1, colno=12)
+      identifier: 'statement'
+      arguments:
+        <class 'bolt.ast.AstAssignment'>
+          location: SourceLocation(pos=0, lineno=1, colno=1)
+          end_location: SourceLocation(pos=11, lineno=1, colno=12)
+          operator: '='
+          target:
+            <class 'bolt.ast.AstTargetIdentifier'>
+              location: SourceLocation(pos=0, lineno=1, colno=1)
+              end_location: SourceLocation(pos=3, lineno=1, colno=4)
+              value: 'foo'
+              rebind: False
+          value:
+            <class 'bolt.ast.AstValue'>
+              location: SourceLocation(pos=6, lineno=1, colno=7)
+              end_location: SourceLocation(pos=11, lineno=1, colno=12)
+              value: '\\\\'

tests/snapshots/bolt__parse_243__1.txt

@@ -0,0 +1,16 @@
+_bolt_lineno = [1], [1]
+_bolt_helper_children = _bolt_runtime.helpers['children']
+_bolt_helper_replace = _bolt_runtime.helpers['replace']
+with _bolt_runtime.scope() as _bolt_var1:
+    _bolt_var0 = '\\\\'
+    foo = _bolt_var0
+_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1))
+---
+output = _bolt_var2
+---
+_bolt_refs[0]
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=12, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>

tests/snapshots/bolt__parse_244__0.txt

@@ -0,0 +1,26 @@
+foo = r"\""
+---
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=12, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>
+      location: SourceLocation(pos=0, lineno=1, colno=1)
+      end_location: SourceLocation(pos=11, lineno=1, colno=12)
+      identifier: 'statement'
+      arguments:
+        <class 'bolt.ast.AstAssignment'>
+          location: SourceLocation(pos=0, lineno=1, colno=1)
+          end_location: SourceLocation(pos=11, lineno=1, colno=12)
+          operator: '='
+          target:
+            <class 'bolt.ast.AstTargetIdentifier'>
+              location: SourceLocation(pos=0, lineno=1, colno=1)
+              end_location: SourceLocation(pos=3, lineno=1, colno=4)
+              value: 'foo'
+              rebind: False
+          value:
+            <class 'bolt.ast.AstValue'>
+              location: SourceLocation(pos=6, lineno=1, colno=7)
+              end_location: SourceLocation(pos=11, lineno=1, colno=12)
+              value: '\\"'

tests/snapshots/bolt__parse_244__1.txt

@@ -0,0 +1,16 @@
+_bolt_lineno = [1], [1]
+_bolt_helper_children = _bolt_runtime.helpers['children']
+_bolt_helper_replace = _bolt_runtime.helpers['replace']
+with _bolt_runtime.scope() as _bolt_var1:
+    _bolt_var0 = '\\"'
+    foo = _bolt_var0
+_bolt_var2 = _bolt_helper_replace(_bolt_refs[0], commands=_bolt_helper_children(_bolt_var1))
+---
+output = _bolt_var2
+---
+_bolt_refs[0]
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=12, lineno=2, colno=1)
+  commands:
+    <class 'mecha.ast.AstCommand'>

tests/snapshots/bolt__parse_245__0.txt

@@ -0,0 +1,160 @@
+macro foo(stream):
+    with stream.syntax(number=r"\d+", name=r"\w+"):
+        stream.expect("number")
+        stream.expect("name")
+---
+<class 'mecha.ast.AstRoot'>
+  location: SourceLocation(pos=0, lineno=1, colno=1)
+  end_location: SourceLocation(pos=132, lineno=4, colno=30)
+  commands:
+    <class 'bolt.ast.AstProcMacro'>
+      location: SourceLocation(pos=0, lineno=1, colno=1)
+      end_location: SourceLocation(pos=132, lineno=4, colno=30)
+      identifier: 'foo:proc_macro_overload3'
+      arguments:
+        <class 'bolt.ast.AstMacroLiteral'>
+          location: SourceLocation(pos=6, lineno=1, colno=7)
+          end_location: SourceLocation(pos=9, lineno=1, colno=10)
+          value: 'foo'
+        <class 'bolt.ast.AstMacroMatchArgument'>
+          location: SourceLocation(pos=-1, lineno=0, colno=0)
+          end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+          match_identifier:
+            <class 'bolt.ast.AstMacroArgument'>
+              location: SourceLocation(pos=-1, lineno=0, colno=0)
+              end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+              value: 'proc_macro_overload3'
+          match_argument_parser:
+            <class 'mecha.ast.AstResourceLocation'>
+              location: SourceLocation(pos=-1, lineno=0, colno=0)
+              end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+              is_tag: False
+              namespace: 'bolt'
+              path: 'proc_macro'
+          match_argument_properties:
+            <class 'mecha.ast.AstJsonObject'>
+              location: SourceLocation(pos=-1, lineno=0, colno=0)
+              end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+              entries:
+                <class 'mecha.ast.AstJsonObjectEntry'>
+                  location: SourceLocation(pos=-1, lineno=0, colno=0)
+                  end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                  key:
+                    <class 'mecha.ast.AstJsonObjectKey'>
+                      location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      value: 'resource_location'
+                  value:
+                    <class 'mecha.ast.AstJsonValue'>
+                      location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      value: 'demo:test'
+                <class 'mecha.ast.AstJsonObjectEntry'>
+                  location: SourceLocation(pos=-1, lineno=0, colno=0)
+                  end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                  key:
+                    <class 'mecha.ast.AstJsonObjectKey'>
+                      location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      value: 'identifier'
+                  value:
+                    <class 'mecha.ast.AstJsonValue'>
+                      location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      end_location: SourceLocation(pos=-1, lineno=0, colno=0)
+                      value: 'foo:proc_macro_overload3'
+        <class 'mecha.ast.AstRoot'>
+          location: SourceLocation(pos=23, lineno=2, colno=5)
+          end_location: SourceLocation(pos=132, lineno=4, colno=30)
+          commands:
+            <class 'mecha.ast.AstCommand'>
+              location: SourceLocation(pos=23, lineno=2, colno=5)
+              end_location: SourceLocation(pos=132, lineno=4, colno=30)
+              identifier: 'with:subcommand'
+              arguments:
+                <class 'mecha.ast.AstCommand'>
+                  location: SourceLocation(pos=28, lineno=2, colno=10)
+                  end_location: SourceLocation(pos=132, lineno=4, colno=30)
+                  identifier: 'with:expression:body'
+                  arguments:
+                    <class 'bolt.ast.AstCall'>
+                      location: SourceLocation(pos=28, lineno=2, colno=10)
+                      end_location: SourceLocation(pos=69, lineno=2, colno=51)
+                      value:
+                        <class 'bolt.ast.AstAttribute'>
+                          location: SourceLocation(pos=28, lineno=2, colno=10)
+                          end_location: SourceLocation(pos=41, lineno=2, colno=23)
+                          name: 'syntax'
+                          value:
+                            <class 'bolt.ast.AstIdentifier'>
+                              location: SourceLocation(pos=28, lineno=2, colno=10)
+                              end_location: SourceLocation(pos=34, lineno=2, colno=16)
+                              value: 'stream'
+                      arguments:
+                        <class 'bolt.ast.AstKeyword'>
+                          location: SourceLocation(pos=42, lineno=2, colno=24)
+                          end_location: SourceLocation(pos=55, lineno=2, colno=37)
+                          name: 'number'
+                          value:
+                            <class 'bolt.ast.AstValue'>
+                              location: SourceLocation(pos=49, lineno=2, colno=31)
+                              end_location: SourceLocation(pos=55, lineno=2, colno=37)
+                              value: '\\d+'
+                        <class 'bolt.ast.AstKeyword'>
+                          location: SourceLocation(pos=57, lineno=2, colno=39)
+                          end_location: SourceLocation(pos=68, lineno=2, colno=50)
+                          name: 'name'
+                          value:
+                            <class 'bolt.ast.AstValue'>
+                              location: SourceLocation(pos=62, lineno=2, colno=44)
+                              end_location: SourceLocation(pos=68, lineno=2, colno=50)
+                              value: '\\w+'
+                    <class 'mecha.ast.AstRoot'>
+                      location: SourceLocation(pos=79, lineno=3, colno=9)
+                      end_location: SourceLocation(pos=132, lineno=4, colno=30)
+                      commands:
+                        <class 'mecha.ast.AstCommand'>
+                          location: SourceLocation(pos=79, lineno=3, colno=9)
+                          end_location: SourceLocation(pos=102, lineno=3, colno=32)
+                          identifier: 'statement'
+                          arguments:
+                            <class 'bolt.ast.AstCall'>
+                              location: SourceLocation(pos=79, lineno=3, colno=9)
+                              end_location: SourceLocation(pos=102, lineno=3, colno=32)
+                              value:
+                                <class 'bolt.ast.AstAttribute'>
+                                  location: SourceLocation(pos=79, lineno=3, colno=9)
+                                  end_location: SourceLocation(pos=92, lineno=3, colno=22)
+                                  name: 'expect'
+                                  value:
+                                    <class 'bolt.ast.AstIdentifier'>
+                                      location: SourceLocation(pos=79, lineno=3, colno=9)
+                                      end_location: SourceLocation(pos=85, lineno=3, colno=15)
+                                      value: 'stream'
+                              arguments:
+                                <class 'bolt.ast.AstValue'>
+                                  location: SourceLocation(pos=93, lineno=3, colno=23)
+                                  end_location: SourceLocation(pos=101, lineno=3, colno=31)
+                                  value: 'number'
+                        <class 'mecha.ast.AstCommand'>
+                          location: SourceLocation(pos=111, lineno=4, colno=9)
+                          end_location: SourceLocation(pos=132, lineno=4, colno=30)
+                          identifier: 'statement'
+                          arguments:
+                            <class 'bolt.ast.AstCall'>
+                              location: SourceLocation(pos=111, lineno=4, colno=9)
+                              end_location: SourceLocation(pos=132, lineno=4, colno=30)
+                              value:
+                                <class 'bolt.ast.AstAttribute'>
+                                  location: SourceLocation(pos=111, lineno=4, colno=9)
+                                  end_location: SourceLocation(pos=124, lineno=4, colno=22)
+                                  name: 'expect'
+                                  value:
+                                    <class 'bolt.ast.AstIdentifier'>
+                                      location: SourceLocation(pos=111, lineno=4, colno=9)
+                                      end_location: SourceLocation(pos=117, lineno=4, colno=15)
+                                      value: 'stream'
+                              arguments:
+                                <class 'bolt.ast.AstValue'>
+                                  location: SourceLocation(pos=125, lineno=4, colno=23)
+                                  end_location: SourceLocation(pos=131, lineno=4, colno=29)
+                                  value: 'name'