Extract the parser and validator from the pest_derive crate into pest_meta

Cargo.toml

@@ -2,5 +2,6 @@
 members = [
     "pest",
     "pest_derive",
-    "pest_grammars"
+    "pest_grammars",
+    "pest_meta"
 ]

pest_derive/Cargo.toml

@@ -18,6 +18,7 @@ proc-macro = true
 quote = "^0.3"
 syn = "^0.10"
 pest = { path = "../pest", version = "^1.0.0-beta" }
+pest_meta = { path = "../pest_meta", version = "^1.0.0-beta" }
 
 [badges]
 travis-ci = { repository = "pest-parser/pest" }

pest_derive/src/generator.rs

@@ -9,9 +9,9 @@ use std::collections::HashMap;
 
 use quote::{Ident, Tokens};
 
-use super::ast::*;
+use super::ast::{Rule, RuleType, Expr};
 
-pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
+pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<String>) -> Tokens {
     let mut predefined = HashMap::new();
     predefined.insert("any", quote! {
         fn any<I: ::pest::inputs::Input>(
@@ -70,7 +70,7 @@ pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
     let skip = generate_skip(&rules);
 
     let mut rules: Vec<_> = rules.into_iter().map(|rule| generate_rule(rule)).collect();
-    rules.extend(defaults.into_iter().map(|name| predefined.get(name.as_ref()).unwrap().clone()));
+    rules.extend(defaults.iter().map(|name| predefined.get(name.as_str()).unwrap().clone()));
 
 
     let parser_impl = quote! {
@@ -102,7 +102,7 @@ pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
 }
 
 fn generate_enum(rules: &Vec<Rule>) -> Tokens {
-    let rules = rules.iter().map(|rule| &rule.name);
+    let rules = rules.iter().map(|rule| Ident::new(rule.name.as_str()));
 
     quote! {
         #[allow(dead_code, non_camel_case_types)]
@@ -117,7 +117,7 @@ fn generate_patterns(rules: &Vec<Rule>) -> Tokens {
     let mut tokens = Tokens::new();
 
     let rules = rules.iter().map(|rule| {
-        let rule = &rule.name;
+        let rule = Ident::new(rule.name.as_str());
         quote! {
             Rule::#rule => rules::#rule(pos, &mut state)
         }
@@ -129,7 +129,7 @@ fn generate_patterns(rules: &Vec<Rule>) -> Tokens {
 }
 
 fn generate_rule(rule: Rule) -> Tokens {
-    let name = rule.name;
+    let name = Ident::new(rule.name.as_str());
     let expr = if {
         rule.ty == RuleType::Atomic ||
         rule.ty == RuleType::CompoundAtomic ||
@@ -205,8 +205,8 @@ fn generate_rule(rule: Rule) -> Tokens {
 }
 
 fn generate_skip(rules: &Vec<Rule>) -> Tokens {
-    let whitespace = rules.iter().any(|rule| rule.name.as_ref() == "whitespace");
-    let comment = rules.iter().any(|rule| rule.name.as_ref() == "comment");
+    let whitespace = rules.iter().any(|rule| rule.name.as_str() == "whitespace");
+    let comment = rules.iter().any(|rule| rule.name.as_str() == "comment");
 
     match (whitespace, comment) {
         (false, false) => quote! {
@@ -327,8 +327,9 @@ fn generate_expr(expr: Expr) -> Tokens {
 
             tokens
         },
-        Expr::Ident(ident) => quote! {
-            self::#ident(pos, state)
+        Expr::Ident(ident) => {
+            let ident = Ident::new(ident.as_str());
+            quote! { self::#ident(pos, state) }
         },
         Expr::PosPred(expr) => {
             let expr = generate_expr(*expr);
@@ -502,8 +503,9 @@ fn generate_expr_atomic(expr: Expr) -> Tokens {
 
             tokens
         },
-        Expr::Ident(ident) => quote! {
-            self::#ident(pos, state)
+        Expr::Ident(ident) => {
+            let ident = Ident::new(ident.as_str());
+            quote! { self::#ident(pos, state) }
         },
         Expr::PosPred(expr) => {
             let expr = generate_expr_atomic(*expr);
@@ -632,9 +634,9 @@ mod tests {
     fn rule_enum_simple() {
         let rules = vec![
             Rule {
-                name: Ident::new("f"),
+                name: "f".to_owned(),
                 ty: RuleType::Normal,
-                expr: Expr::Ident(Ident::new("g"))
+                expr: Expr::Ident("g".to_owned())
             }
         ];
 
@@ -759,7 +761,7 @@ mod tests {
     #[test]
     fn expr_complex() {
         let expr = Expr::Choice(
-            Box::new(Expr::Ident(Ident::new("a"))),
+            Box::new(Expr::Ident("a".to_owned())),
             Box::new(Expr::Seq(
                 Box::new(Expr::Range("'a'".to_owned(), "'b'".to_owned())),
                 Box::new(Expr::Seq(
@@ -854,7 +856,7 @@ mod tests {
     #[test]
     fn expr_complex_atomic() {
         let expr = Expr::Choice(
-            Box::new(Expr::Ident(Ident::new("a"))),
+            Box::new(Expr::Ident("a".to_owned())),
             Box::new(Expr::Seq(
                 Box::new(Expr::Range("'a'".to_owned(), "'b'".to_owned())),
                 Box::new(Expr::Seq(
@@ -912,11 +914,11 @@ mod tests {
     fn generate_complete() {
         let name = Ident::new("MyParser");
         let rules = vec![Rule {
-            name: Ident::new("a"),
+            name: "a".to_owned(),
             ty: RuleType::Silent,
             expr: Expr::Str("b".to_owned())
         }];
-        let defaults = vec![Ident::new("any")];
+        let defaults = vec!["any".to_owned()];
 
         assert_eq!(generate(name, rules, defaults), quote! {
             #[allow(dead_code, non_camel_case_types)]

pest_derive/src/lib.rs

@@ -226,14 +226,15 @@
 #![doc(html_root_url = "https://docs.rs/pest_derive")]
 #![recursion_limit="256"]
 
-#[macro_use]
 extern crate pest;
+extern crate pest_meta;
 extern crate proc_macro;
 #[macro_use]
 extern crate quote;
 extern crate syn;
 
 use std::env;
+use std::fmt::Display;
 use std::path::Path;
 use std::rc::Rc;
 
@@ -243,13 +244,11 @@ use proc_macro::TokenStream;
 use quote::Ident;
 use syn::{Attribute, Lit, MetaItem};
 
-mod ast;
 mod generator;
 mod optimizer;
-mod parser;
-mod validator;
 
-use parser::{GrammarParser, GrammarRule};
+use pest_meta::parser::{self, PestParser, PestRule};
+use pest_meta::{ast, validator};
 
 #[proc_macro_derive(Parser, attributes(grammar))]
 pub fn derive_parser(input: TokenStream) -> TokenStream {
@@ -268,39 +267,46 @@ pub fn derive_parser(input: TokenStream) -> TokenStream {
         Ok(input) => Rc::new(input),
         Err(error) => panic!("error opening {:?}: {}", file_name, error)
     };
-    let pairs = match GrammarParser::parse(GrammarRule::grammar_rules, input) {
+    let pairs = match PestParser::parse(PestRule::grammar_rules, input) {
         Ok(pairs) => pairs,
         Err(error) => panic!("error parsing {:?}\n\n{}", file_name, error.renamed_rules(|rule| {
             match *rule {
-                GrammarRule::grammar_rule => "rule".to_owned(),
-                GrammarRule::eoi => "end-of-input".to_owned(),
-                GrammarRule::assignment_operator => "`=`".to_owned(),
-                GrammarRule::silent_modifier => "`_`".to_owned(),
-                GrammarRule::atomic_modifier => "`@`".to_owned(),
-                GrammarRule::compound_atomic_modifier => "`$`".to_owned(),
-                GrammarRule::non_atomic_modifier => "`!`".to_owned(),
-                GrammarRule::opening_brace => "`{`".to_owned(),
-                GrammarRule::closing_brace => "`}`".to_owned(),
-                GrammarRule::opening_paren => "`(`".to_owned(),
-                GrammarRule::positive_predicate_operator => "`&`".to_owned(),
-                GrammarRule::negative_predicate_operator => "`!`".to_owned(),
-                GrammarRule::sequence_operator => "`&`".to_owned(),
-                GrammarRule::choice_operator => "`|`".to_owned(),
-                GrammarRule::optional_operator => "`?`".to_owned(),
-                GrammarRule::repeat_operator => "`*`".to_owned(),
-                GrammarRule::repeat_once_operator => "`+`".to_owned(),
-                GrammarRule::comma => "`,`".to_owned(),
-                GrammarRule::closing_paren => "`)`".to_owned(),
-                GrammarRule::quote => "`\"`".to_owned(),
-                GrammarRule::insensitive_string => "`^`".to_owned(),
-                GrammarRule::range_operator => "`..`".to_owned(),
-                GrammarRule::single_quote => "`'`".to_owned(),
+                PestRule::grammar_rule => "rule".to_owned(),
+                PestRule::assignment_operator => "`=`".to_owned(),
+                PestRule::silent_modifier => "`_`".to_owned(),
+                PestRule::atomic_modifier => "`@`".to_owned(),
+                PestRule::compound_atomic_modifier => "`$`".to_owned(),
+                PestRule::non_atomic_modifier => "`!`".to_owned(),
+                PestRule::opening_brace => "`{`".to_owned(),
+                PestRule::closing_brace => "`}`".to_owned(),
+                PestRule::opening_paren => "`(`".to_owned(),
+                PestRule::positive_predicate_operator => "`&`".to_owned(),
+                PestRule::negative_predicate_operator => "`!`".to_owned(),
+                PestRule::sequence_operator => "`&`".to_owned(),
+                PestRule::choice_operator => "`|`".to_owned(),
+                PestRule::optional_operator => "`?`".to_owned(),
+                PestRule::repeat_operator => "`*`".to_owned(),
+                PestRule::repeat_once_operator => "`+`".to_owned(),
+                PestRule::comma => "`,`".to_owned(),
+                PestRule::closing_paren => "`)`".to_owned(),
+                PestRule::quote => "`\"`".to_owned(),
+                PestRule::insensitive_string => "`^`".to_owned(),
+                PestRule::range_operator => "`..`".to_owned(),
+                PestRule::single_quote => "`'`".to_owned(),
                 other_rule => format!("{:?}", other_rule)
             }
         }))
     };
 
-    let (ast, defaults) = parser::consume_rules(pairs);
+    fn format_errors<Errors: IntoIterator>(errors: Errors) -> String where Errors::Item: Display {
+        errors.into_iter()
+              .map(|error| { format!("{}", error) })
+              .collect::<Vec<_>>()
+              .join("\n\n")
+    }
+
+    let defaults = validator::validate_pairs(pairs.clone()).unwrap_or_else(|e| panic!(format_errors(e)));
+    let ast = parser::consume_rules(pairs).unwrap_or_else(|e| panic!(format_errors(e)));
     let optimized = optimizer::optimize(ast);
     let generated = generator::generate(name, optimized, defaults);
 

pest_derive/src/optimizer.rs

@@ -139,13 +139,11 @@ pub fn optimize(rules: Vec<Rule>) -> Vec<Rule> {
 mod tests {
     use super::*;
 
-    use quote::Ident;
-
     #[test]
     fn concat_strings() {
         let rules = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Atomic,
                 expr: Expr::Seq(
                     Box::new(Expr::Seq(
@@ -161,7 +159,7 @@ mod tests {
         ];
         let concatenated = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Atomic,
                 expr: Expr::Str("abcd".to_owned())
             }
@@ -174,7 +172,7 @@ mod tests {
     fn concat_insensitive_strings() {
         let rules = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Atomic,
                 expr: Expr::Seq(
                     Box::new(Expr::Seq(
@@ -190,7 +188,7 @@ mod tests {
         ];
         let concatenated = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Atomic,
                 expr: Expr::Insens("abcd".to_owned())
             }
@@ -203,37 +201,37 @@ mod tests {
     fn long_common_sequence() {
         let rules = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Silent,
                 expr: Expr::Choice(
                     Box::new(Expr::Seq(
-                        Box::new(Expr::Ident(Ident::new("a"))),
+                        Box::new(Expr::Ident("a".to_owned())),
                         Box::new(Expr::Seq(
-                            Box::new(Expr::Ident(Ident::new("b"))),
-                            Box::new(Expr::Ident(Ident::new("c")))
+                            Box::new(Expr::Ident("b".to_owned())),
+                            Box::new(Expr::Ident("c".to_owned()))
                         ))
                     )),
                     Box::new(Expr::Seq(
                         Box::new(Expr::Seq(
-                            Box::new(Expr::Ident(Ident::new("a"))),
-                            Box::new(Expr::Ident(Ident::new("b")))
+                            Box::new(Expr::Ident("a".to_owned())),
+                            Box::new(Expr::Ident("b".to_owned()))
                         )),
-                        Box::new(Expr::Ident(Ident::new("d")))
+                        Box::new(Expr::Ident("d".to_owned()))
                     ))
                 )
             }
         ];
         let optimized = vec![
             Rule {
-                name: Ident::new("rule"),
+                name: "rule".to_owned(),
                 ty:   RuleType::Silent,
                 expr: Expr::Seq(
-                    Box::new(Expr::Ident(Ident::new("a"))),
+                    Box::new(Expr::Ident("a".to_owned())),
                     Box::new(Expr::Seq(
-                        Box::new(Expr::Ident(Ident::new("b"))),
+                        Box::new(Expr::Ident("b".to_owned())),
                         Box::new(Expr::Choice(
-                            Box::new(Expr::Ident(Ident::new("c"))),
-                            Box::new(Expr::Ident(Ident::new("d")))
+                            Box::new(Expr::Ident("c".to_owned())),
+                            Box::new(Expr::Ident("d".to_owned()))
                         ))
                     ))
                 )