Merge pull request #1 from pest-parser/master

one-or-more rule fix based on pest-parser#397 (pest-parser#878)
TheVeryDarkness · Jul 10, 2023 · 582cda9 · 582cda9
2 parents 8fe45d9 + d9bfdde
commit 582cda9
Show file tree

Hide file tree

Showing 14 changed files with 112 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -200,6 +200,7 @@ You can find more projects and ecosystem tools in the [awesome-pest](https://git
 * [caith](https://github.com/Geobert/caith) (a dice roller crate)
 * [Melody](https://github.com/yoav-lavi/melody)
 * [json5-nodes](https://github.com/jlyonsmith/json5-nodes)
+* [prisma](https://github.com/prisma/prisma)
 
 ## Minimum Supported Rust Version (MSRV)
 

diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_debugger"
 description = "pest grammar debugger"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>", "Tomas Tauber <me@tomtau.be>"]
 homepage = "https://pest.rs/"
@@ -14,9 +14,9 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_meta = { path = "../meta", version = "2.7.0" }
-pest_vm = { path = "../vm", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_meta = { path = "../meta", version = "2.7.1" }
+pest_vm = { path = "../vm", version = "2.7.1" }
 reqwest = { version = "= 0.11.13", default-features = false, features = ["blocking", "json", "default-tls"] }
 rustyline = "10"
 serde_json = "1"

diff --git a/derive/Cargo.toml b/derive/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_derive"
 description = "pest's derive macro"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -25,5 +25,5 @@ grammar-extras = ["pest_generator/grammar-extras"]
 
 [dependencies]
 # for tests, included transitively anyway
-pest = { path = "../pest", version = "2.7.0", default-features = false }
-pest_generator = { path = "../generator", version = "2.7.0", default-features = false }
+pest = { path = "../pest", version = "2.7.1", default-features = false }
+pest_generator = { path = "../generator", version = "2.7.1", default-features = false }
diff --git a/derive/tests/oneormore.pest b/derive/tests/oneormore.pest
@@ -0,0 +1,5 @@
+WHITESPACE = _{ " " | "\r" | "\n" | "\t" }
+
+
+identifier = { (ASCII_ALPHA | "_")+ }
+assign = { identifier ~ "<-" ~ identifier }
diff --git a/derive/tests/oneormore.rs b/derive/tests/oneormore.rs
@@ -0,0 +1,29 @@
+#![cfg_attr(not(feature = "std"), no_std)]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::{format, vec::Vec};
+
+#[cfg(feature = "grammar-extras")]
+use pest::Parser;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../tests/oneormore.pest"]
+pub struct OneOrMoreParser;
+
+#[test]
+#[cfg(feature = "grammar-extras")]
+pub fn test_one_or_more() {
+    let result = OneOrMoreParser::parse(Rule::assign, "k <- b\n");
+    assert!(result.is_ok());
+    let mut pairs = result.unwrap();
+    let pair = pairs.next().unwrap();
+    assert_eq!(pair.as_rule(), Rule::assign);
+    let mut inner = pair.into_inner();
+    let lhs = inner.next().unwrap();
+    assert_eq!(lhs.as_rule(), Rule::identifier);
+    assert_eq!(lhs.as_str(), "k");
+    let rhs = inner.next().unwrap();
+    assert_eq!(rhs.as_rule(), Rule::identifier);
+    assert_eq!(rhs.as_str(), "b");
+}
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_generator"
 description = "pest code generator"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -20,8 +20,8 @@ not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
 grammar-extras = ["pest_meta/grammar-extras"]
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0", default-features = false }
-pest_meta = { path = "../meta", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1", default-features = false }
+pest_meta = { path = "../meta", version = "2.7.1" }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
diff --git a/generator/src/generator.rs b/generator/src/generator.rs
@@ -537,6 +537,26 @@ fn generate_expr(expr: OptimizedExpr) -> TokenStream {
                 })
             }
         }
+        #[cfg(feature = "grammar-extras")]
+        OptimizedExpr::RepOnce(expr) => {
+            let expr = generate_expr(*expr);
+
+            quote! {
+                state.sequence(|state| {
+                    #expr.and_then(|state| {
+                        state.repeat(|state| {
+                            state.sequence(|state| {
+                                super::hidden::skip(
+                                    state
+                                ).and_then(|state| {
+                                    #expr
+                                })
+                            })
+                        })
+                    })
+                })
+            }
+        }
         OptimizedExpr::Skip(strings) => {
             quote! {
                 let strings = [#(#strings),*];
@@ -677,6 +697,22 @@ fn generate_expr_atomic(expr: OptimizedExpr) -> TokenStream {
                 })
             }
         }
+        #[cfg(feature = "grammar-extras")]
+        OptimizedExpr::RepOnce(expr) => {
+            let expr = generate_expr_atomic(*expr);
+
+            quote! {
+                state.sequence(|state| {
+                    #expr.and_then(|state| {
+                        state.repeat(|state| {
+                            state.sequence(|state| {
+                                #expr
+                            })
+                        })
+                    })
+                })
+            }
+        }
         OptimizedExpr::Skip(strings) => {
             quote! {
                 let strings = [#(#strings),*];

diff --git a/grammars/Cargo.toml b/grammars/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_grammars"
 description = "pest popular grammar implementations"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_derive = { path = "../derive", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_derive = { path = "../derive", version = "2.7.1" }
 
 [dev-dependencies]
 criterion = "0.5"

diff --git a/meta/Cargo.toml b/meta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_meta"
 description = "pest meta language parser and validator"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -16,7 +16,7 @@ include = ["Cargo.toml", "src/**/*", "src/grammar.rs", "_README.md", "LICENSE-*"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
 once_cell = "1.8.0"
 
 [build-dependencies]

diff --git a/meta/src/optimizer/mod.rs b/meta/src/optimizer/mod.rs
@@ -70,11 +70,13 @@ fn rule_to_optimized_rule(rule: Rule) -> OptimizedRule {
             Expr::Push(expr) => OptimizedExpr::Push(Box::new(to_optimized(*expr))),
             #[cfg(feature = "grammar-extras")]
             Expr::NodeTag(expr, tag) => OptimizedExpr::NodeTag(Box::new(to_optimized(*expr)), tag),
-            Expr::RepOnce(_)
-            | Expr::RepExact(..)
-            | Expr::RepMin(..)
-            | Expr::RepMax(..)
-            | Expr::RepMinMax(..) => unreachable!("No valid transformation to OptimizedRule"),
+            #[cfg(feature = "grammar-extras")]
+            Expr::RepOnce(expr) => OptimizedExpr::RepOnce(Box::new(to_optimized(*expr))),
+            #[cfg(not(feature = "grammar-extras"))]
+            Expr::RepOnce(_) => unreachable!("No valid transformation to OptimizedRule"),
+            Expr::RepExact(..) | Expr::RepMin(..) | Expr::RepMax(..) | Expr::RepMinMax(..) => {
+                unreachable!("No valid transformation to OptimizedRule")
+            }
         }
     }
 
@@ -135,6 +137,9 @@ pub enum OptimizedExpr {
     Opt(Box<OptimizedExpr>),
     /// Matches an expression zero or more times, e.g. `e*`
     Rep(Box<OptimizedExpr>),
+    /// Matches an expression one or more times, e.g. `e+`
+    #[cfg(feature = "grammar-extras")]
+    RepOnce(Box<OptimizedExpr>),
     /// Continues to match expressions until one of the strings in the `Vec` is found
     Skip(Vec<String>),
     /// Matches an expression and pushes it to the stack, e.g. `push(e)`

diff --git a/meta/src/optimizer/unroller.rs b/meta/src/optimizer/unroller.rs
@@ -15,6 +15,7 @@ pub fn unroll(rule: Rule) -> Rule {
         name,
         ty,
         expr: expr.map_bottom_up(|expr| match expr {
+            #[cfg(not(feature = "grammar-extras"))]
             Expr::RepOnce(expr) => Expr::Seq(expr.clone(), Box::new(Expr::Rep(expr))),
             Expr::RepExact(expr, num) => (1..num + 1)
                 .map(|_| *expr.clone())

diff --git a/pest/Cargo.toml b/pest/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest"
 description = "The Elegant Parser"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"

diff --git a/vm/Cargo.toml b/vm/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "pest_vm"
 description = "pest grammar virtual machine"
-version = "2.7.0"
+version = "2.7.1"
 edition = "2021"
 authors = ["Dragoș Tiselice <dragostiselice@gmail.com>"]
 homepage = "https://pest.rs/"
@@ -14,8 +14,8 @@ readme = "_README.md"
 rust-version = "1.60"
 
 [dependencies]
-pest = { path = "../pest", version = "2.7.0" }
-pest_meta = { path = "../meta", version = "2.7.0" }
+pest = { path = "../pest", version = "2.7.1" }
+pest_meta = { path = "../meta", version = "2.7.1" }
 
 [features]
 grammar-extras = ["pest_meta/grammar-extras"]
diff --git a/vm/src/lib.rs b/vm/src/lib.rs
@@ -224,6 +224,17 @@ impl Vm {
                     })
                 })
             }),
+            #[cfg(feature = "grammar-extras")]
+            OptimizedExpr::RepOnce(ref expr) => state.sequence(|state| {
+                self.parse_expr(expr, state).and_then(|state| {
+                    state.repeat(|state| {
+                        state.sequence(|state| {
+                            self.skip(state)
+                                .and_then(|state| self.parse_expr(expr, state))
+                        })
+                    })
+                })
+            }),
             OptimizedExpr::Push(ref expr) => state.stack_push(|state| self.parse_expr(expr, state)),
             OptimizedExpr::Skip(ref strings) => state.skip_until(
                 &strings