Extract the parser and validator from the pest_derive crate into pest_meta

Cargo.toml

@@ -2,5 +2,6 @@
 members = [
     "pest",
     "pest_derive",
-    "pest_grammars"
+    "pest_grammars",
+    "pest_meta"
 ]

pest_derive/Cargo.toml

@@ -18,6 +18,7 @@ proc-macro = true
 quote = "^0.3"
 syn = "^0.11"
 pest = { path = "../pest", version = "^1.0" }
+pest_meta = { path = "../pest_meta", version = "0.1.0" }
 
 [badges]
 codecov = { repository = "pest-parser/pest" }

pest_derive/src/generator.rs

@@ -13,7 +13,7 @@ use quote::{Ident, Tokens};
 
 use ast::*;
 
-pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
+pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<&str>) -> Tokens {
     let mut predefined = HashMap::new();
     predefined.insert(
         "any",
@@ -96,7 +96,7 @@ pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
     rules.extend(
         defaults
             .into_iter()
-            .map(|name| predefined.get(name.as_ref()).unwrap().clone())
+            .map(|name| predefined.get(name).unwrap().clone())
     );
 
     let parser_impl = quote! {
@@ -131,7 +131,7 @@ pub fn generate(name: Ident, rules: Vec<Rule>, defaults: Vec<Ident>) -> Tokens {
 }
 
 fn generate_enum(rules: &Vec<Rule>) -> Tokens {
-    let rules = rules.iter().map(|rule| &rule.name);
+    let rules = rules.iter().map(|rule| Ident::new(rule.name));
 
     quote! {
         #[allow(dead_code, non_camel_case_types)]
@@ -146,7 +146,7 @@ fn generate_patterns(rules: &Vec<Rule>) -> Tokens {
     let mut tokens = Tokens::new();
 
     let rules = rules.iter().map(|rule| {
-        let rule = &rule.name;
+        let rule = Ident::new(rule.name);
         quote! {
             Rule::#rule => rules::#rule(pos, &mut state)
         }
@@ -158,11 +158,11 @@ fn generate_patterns(rules: &Vec<Rule>) -> Tokens {
 }
 
 fn generate_rule(rule: Rule) -> Tokens {
-    let name = rule.name;
+    let name = Ident::new(rule.name);
     let expr = if { rule.ty == RuleType::Atomic || rule.ty == RuleType::CompoundAtomic } {
         generate_expr_atomic(rule.expr)
     } else {
-        if &name == "whitespace" || &name == "comment" {
+        if name == "whitespace" || name == "comment" {
             let atomic = generate_expr_atomic(rule.expr);
 
             quote! {
@@ -244,8 +244,8 @@ fn generate_rule(rule: Rule) -> Tokens {
 }
 
 fn generate_skip(rules: &Vec<Rule>) -> Tokens {
-    let whitespace = rules.iter().any(|rule| rule.name.as_ref() == "whitespace");
-    let comment = rules.iter().any(|rule| rule.name.as_ref() == "comment");
+    let whitespace = rules.iter().any(|rule| rule.name == "whitespace");
+    let comment = rules.iter().any(|rule| rule.name == "comment");
 
     match (whitespace, comment) {
         (false, false) => quote! {
@@ -362,8 +362,9 @@ fn generate_expr(expr: Expr) -> Tokens {
 
             tokens
         }
-        Expr::Ident(ident) => quote! {
-            self::#ident(pos, state)
+        Expr::Ident(ident) => {
+            let ident = Ident::new(ident);
+            quote! { self::#ident(pos, state) }
         },
         Expr::PosPred(expr) => {
             let expr = generate_expr(*expr);
@@ -537,8 +538,9 @@ fn generate_expr_atomic(expr: Expr) -> Tokens {
 
             tokens
         }
-        Expr::Ident(ident) => quote! {
-            self::#ident(pos, state)
+        Expr::Ident(ident) => {
+            let ident = Ident::new(ident);
+            quote! { self::#ident(pos, state) }
         },
         Expr::PosPred(expr) => {
             let expr = generate_expr_atomic(*expr);
@@ -667,10 +669,10 @@ mod tests {
     fn rule_enum_simple() {
         let rules = vec![
             Rule {
-                name: Ident::new("f"),
+                name: "f",
                 ty: RuleType::Normal,
-                expr: Expr::Ident(Ident::new("g"))
-            },
+                expr: Expr::Ident("g")
+            }
         ];
 
         assert_eq!(
@@ -810,7 +812,7 @@ mod tests {
     #[test]
     fn expr_complex() {
         let expr = Expr::Choice(
-            Box::new(Expr::Ident(Ident::new("a"))),
+            Box::new(Expr::Ident("a")),
             Box::new(Expr::Seq(
                 Box::new(Expr::Range("'a'".to_owned(), "'b'".to_owned())),
                 Box::new(Expr::Seq(
@@ -908,7 +910,7 @@ mod tests {
     #[test]
     fn expr_complex_atomic() {
         let expr = Expr::Choice(
-            Box::new(Expr::Ident(Ident::new("a"))),
+            Box::new(Expr::Ident("a")),
             Box::new(Expr::Seq(
                 Box::new(Expr::Range("'a'".to_owned(), "'b'".to_owned())),
                 Box::new(Expr::Seq(
@@ -965,12 +967,12 @@ mod tests {
         let name = Ident::new("MyParser");
         let rules = vec![
             Rule {
-                name: Ident::new("a"),
+                name: "a",
                 ty: RuleType::Silent,
                 expr: Expr::Str("b".to_owned())
             },
         ];
-        let defaults = vec![Ident::new("any")];
+        let defaults = vec!["any"];
 
         assert_eq!(
             generate(name, rules, defaults),

pest_derive/src/lib.rs

@@ -230,11 +230,8 @@
 #![doc(html_root_url = "https://docs.rs/pest_derive")]
 #![recursion_limit = "256"]
 
-#[cfg(test)]
-#[macro_use]
-extern crate pest;
-#[cfg(not(test))]
 extern crate pest;
+extern crate pest_meta;
 
 extern crate proc_macro;
 #[macro_use]
@@ -246,18 +243,16 @@ use std::fs::File;
 use std::io::{self, Read};
 use std::path::Path;
 
-use pest::Parser;
 use proc_macro::TokenStream;
 use quote::Ident;
 use syn::{Attribute, Lit, MetaItem};
 
-mod ast;
 mod generator;
 mod optimizer;
-mod parser;
-mod validator;
 
-use parser::{GrammarParser, GrammarRule};
+use pest_meta::parser::{self, Rule};
+use pest_meta::{ast, validator, unwrap_or_report};
+
 
 #[proc_macro_derive(Parser, attributes(grammar))]
 pub fn derive_parser(input: TokenStream) -> TokenStream {
@@ -277,42 +272,42 @@ pub fn derive_parser(input: TokenStream) -> TokenStream {
         Err(error) => panic!("error opening {:?}: {}", file_name, error)
     };
 
-    let input = &data;
-    let pairs = match GrammarParser::parse(GrammarRule::grammar_rules, input) {
+    let pairs = match parser::parse(Rule::grammar_rules, &data) {
         Ok(pairs) => pairs,
         Err(error) => panic!(
             "error parsing {:?}\n\n{}",
             file_name,
-            error.renamed_rules(|rule| match *rule {
-                GrammarRule::grammar_rule => "rule".to_owned(),
-                GrammarRule::eoi => "end-of-input".to_owned(),
-                GrammarRule::assignment_operator => "`=`".to_owned(),
-                GrammarRule::silent_modifier => "`_`".to_owned(),
-                GrammarRule::atomic_modifier => "`@`".to_owned(),
-                GrammarRule::compound_atomic_modifier => "`$`".to_owned(),
-                GrammarRule::non_atomic_modifier => "`!`".to_owned(),
-                GrammarRule::opening_brace => "`{`".to_owned(),
-                GrammarRule::closing_brace => "`}`".to_owned(),
-                GrammarRule::opening_paren => "`(`".to_owned(),
-                GrammarRule::positive_predicate_operator => "`&`".to_owned(),
-                GrammarRule::negative_predicate_operator => "`!`".to_owned(),
-                GrammarRule::sequence_operator => "`&`".to_owned(),
-                GrammarRule::choice_operator => "`|`".to_owned(),
-                GrammarRule::optional_operator => "`?`".to_owned(),
-                GrammarRule::repeat_operator => "`*`".to_owned(),
-                GrammarRule::repeat_once_operator => "`+`".to_owned(),
-                GrammarRule::comma => "`,`".to_owned(),
-                GrammarRule::closing_paren => "`)`".to_owned(),
-                GrammarRule::quote => "`\"`".to_owned(),
-                GrammarRule::insensitive_string => "`^`".to_owned(),
-                GrammarRule::range_operator => "`..`".to_owned(),
-                GrammarRule::single_quote => "`'`".to_owned(),
+            error.renamed_rules(|rule| { match *rule {
+                Rule::grammar_rule => "rule".to_owned(),
+                Rule::assignment_operator => "`=`".to_owned(),
+                Rule::silent_modifier => "`_`".to_owned(),
+                Rule::atomic_modifier => "`@`".to_owned(),
+                Rule::compound_atomic_modifier => "`$`".to_owned(),
+                Rule::non_atomic_modifier => "`!`".to_owned(),
+                Rule::opening_brace => "`{`".to_owned(),
+                Rule::closing_brace => "`}`".to_owned(),
+                Rule::opening_paren => "`(`".to_owned(),
+                Rule::positive_predicate_operator => "`&`".to_owned(),
+                Rule::negative_predicate_operator => "`!`".to_owned(),
+                Rule::sequence_operator => "`&`".to_owned(),
+                Rule::choice_operator => "`|`".to_owned(),
+                Rule::optional_operator => "`?`".to_owned(),
+                Rule::repeat_operator => "`*`".to_owned(),
+                Rule::repeat_once_operator => "`+`".to_owned(),
+                Rule::comma => "`,`".to_owned(),
+                Rule::closing_paren => "`)`".to_owned(),
+                Rule::quote => "`\"`".to_owned(),
+                Rule::insensitive_string => "`^`".to_owned(),
+                Rule::range_operator => "`..`".to_owned(),
+                Rule::single_quote => "`'`".to_owned(),
                 other_rule => format!("{:?}", other_rule)
-            })
-        )
+            }
+        }))
     };
 
-    let (ast, defaults) = parser::consume_rules(pairs);
+
+    let defaults = unwrap_or_report(validator::validate_pairs(pairs.clone()));
+    let ast = unwrap_or_report(parser::consume_rules(pairs));
     let optimized = optimizer::optimize(ast);
     let generated = generator::generate(name, optimized, defaults);