Export generator and docs api in the pest_generator trait (#961)

* feat(generator): Export generator and docs api Signed-off-by: marcfir <72923599+marcfir@users.noreply.github.com> * feat(generator): Add feature guard for export and restructure modules * Make internal API public with the `export-internal` feature. * Move the parse_derive functions and types to a new `parse_derive module. Signed-off-by: marcfir <72923599+marcfir@users.noreply.github.com> --------- Signed-off-by: marcfir <72923599+marcfir@users.noreply.github.com>
pest-parser · Jan 6, 2024 · 199f594 · 199f594
1 parent d16266a
commit 199f594
Show file tree

Hide file tree

Showing 5 changed files with 204 additions and 158 deletions.
diff --git a/generator/Cargo.toml b/generator/Cargo.toml
@@ -18,6 +18,8 @@ default = ["std"]
 std = ["pest/std"]
 not-bootstrap-in-src = ["pest_meta/not-bootstrap-in-src"]
 grammar-extras = ["pest_meta/grammar-extras"]
+# Export internal API that is not intended to be stable
+export-internal = []
 
 [dependencies]
 pest = { path = "../pest", version = "2.7.6", default-features = false }

diff --git a/generator/src/docs.rs b/generator/src/docs.rs
@@ -1,9 +1,13 @@
+//! Type and helper to collect the gramamr and rule documentation.
+
 use pest::iterators::Pairs;
 use pest_meta::parser::Rule;
 use std::collections::HashMap;
 
+/// Abstraction for the grammer and rule doc.
 #[derive(Debug)]
-pub(crate) struct DocComment {
+pub struct DocComment {
+    /// The grammar documentation is defined at the beginning of a file with //!.
     pub grammar_doc: String,
 
     /// HashMap for store all doc_comments for rules.
@@ -33,7 +37,7 @@ pub(crate) struct DocComment {
 /// grammar_doc = "This is a grammar doc"
 /// line_docs = { "foo": "line doc 1\nline doc 2", "bar": "line doc 3" }
 /// ```
-pub(crate) fn consume(pairs: Pairs<'_, Rule>) -> DocComment {
+pub fn consume(pairs: Pairs<'_, Rule>) -> DocComment {
     let mut grammar_doc = String::new();
 
     let mut line_docs: HashMap<String, String> = HashMap::new();

diff --git a/generator/src/generator.rs b/generator/src/generator.rs
@@ -7,6 +7,8 @@
 // option. All files in the project carrying such notice may not be copied,
 // modified, or distributed except according to those terms.
 
+//! Helpers to generate the code for the Parser `derive``.
+
 use std::path::PathBuf;
 
 use proc_macro2::TokenStream;
@@ -18,9 +20,12 @@ use pest_meta::ast::*;
 use pest_meta::optimizer::*;
 
 use crate::docs::DocComment;
-use crate::ParsedDerive;
+use crate::parse_derive::ParsedDerive;
 
-pub(crate) fn generate(
+/// Generates the corresponding parser based based on the processed macro input. If `include_grammar`
+/// is set to true, it'll generate an explicit "include_str" statement (done in pest_derive, but
+/// turned off in the local bootstrap).
+pub fn generate(
     parsed_derive: ParsedDerive,
     paths: Vec<PathBuf>,
     rules: Vec<OptimizedRule>,

diff --git a/generator/src/lib.rs b/generator/src/lib.rs
@@ -26,14 +26,29 @@ use std::fs::File;
 use std::io::{self, Read};
 use std::path::Path;
 
+use generator::generate;
 use proc_macro2::TokenStream;
-use syn::{Attribute, DeriveInput, Expr, ExprLit, Generics, Ident, Lit, Meta};
+use syn::DeriveInput;
 
 #[macro_use]
 mod macros;
+
+#[cfg(feature = "export-internal")]
+pub mod docs;
+#[cfg(not(feature = "export-internal"))]
 mod docs;
+
+#[cfg(feature = "export-internal")]
+pub mod generator;
+#[cfg(not(feature = "export-internal"))]
 mod generator;
 
+#[cfg(feature = "export-internal")]
+pub mod parse_derive;
+#[cfg(not(feature = "export-internal"))]
+mod parse_derive;
+
+use crate::parse_derive::{parse_derive, GrammarSource};
 use pest_meta::parser::{self, rename_meta_rule, Rule};
 use pest_meta::{optimizer, unwrap_or_report, validator};
 
@@ -96,7 +111,7 @@ pub fn derive_parser(input: TokenStream, include_grammar: bool) -> TokenStream {
     let ast = unwrap_or_report(parser::consume_rules(pairs));
     let optimized = optimizer::optimize(ast);
 
-    generator::generate(
+    generate(
         parsed_derive,
         paths,
         optimized,
@@ -113,160 +128,8 @@ fn read_file<P: AsRef<Path>>(path: P) -> io::Result<String> {
     Ok(string)
 }
 
-#[derive(Debug, PartialEq)]
-enum GrammarSource {
-    File(String),
-    Inline(String),
-}
-
-struct ParsedDerive {
-    pub(crate) name: Ident,
-    pub(crate) generics: Generics,
-    pub(crate) non_exhaustive: bool,
-}
-
-fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec<GrammarSource>) {
-    let name = ast.ident;
-    let generics = ast.generics;
-
-    let grammar: Vec<&Attribute> = ast
-        .attrs
-        .iter()
-        .filter(|attr| {
-            let path = attr.meta.path();
-            path.is_ident("grammar") || path.is_ident("grammar_inline")
-        })
-        .collect();
-
-    if grammar.is_empty() {
-        panic!("a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute");
-    }
-
-    let mut grammar_sources = Vec::with_capacity(grammar.len());
-    for attr in grammar {
-        grammar_sources.push(get_attribute(attr))
-    }
-
-    let non_exhaustive = ast
-        .attrs
-        .iter()
-        .any(|attr| attr.meta.path().is_ident("non_exhaustive"));
-
-    (
-        ParsedDerive {
-            name,
-            generics,
-            non_exhaustive,
-        },
-        grammar_sources,
-    )
-}
-
-fn get_attribute(attr: &Attribute) -> GrammarSource {
-    match &attr.meta {
-        Meta::NameValue(name_value) => match &name_value.value {
-            Expr::Lit(ExprLit {
-                lit: Lit::Str(string),
-                ..
-            }) => {
-                if name_value.path.is_ident("grammar") {
-                    GrammarSource::File(string.value())
-                } else {
-                    GrammarSource::Inline(string.value())
-                }
-            }
-            _ => panic!("grammar attribute must be a string"),
-        },
-        _ => panic!("grammar attribute must be of the form `grammar = \"...\"`"),
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use super::parse_derive;
-    use super::GrammarSource;
-
-    #[test]
-    fn derive_inline_file() {
-        let definition = "
-            #[other_attr]
-            #[grammar_inline = \"GRAMMAR\"]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        let (_, filenames) = parse_derive(ast);
-        assert_eq!(filenames, [GrammarSource::Inline("GRAMMAR".to_string())]);
-    }
-
-    #[test]
-    fn derive_ok() {
-        let definition = "
-            #[other_attr]
-            #[grammar = \"myfile.pest\"]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        let (parsed_derive, filenames) = parse_derive(ast);
-        assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]);
-        assert!(!parsed_derive.non_exhaustive);
-    }
-
-    #[test]
-    fn derive_multiple_grammars() {
-        let definition = "
-            #[other_attr]
-            #[grammar = \"myfile1.pest\"]
-            #[grammar = \"myfile2.pest\"]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        let (_, filenames) = parse_derive(ast);
-        assert_eq!(
-            filenames,
-            [
-                GrammarSource::File("myfile1.pest".to_string()),
-                GrammarSource::File("myfile2.pest".to_string())
-            ]
-        );
-    }
-
-    #[test]
-    fn derive_nonexhaustive() {
-        let definition = "
-            #[non_exhaustive]
-            #[grammar = \"myfile.pest\"]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        let (parsed_derive, filenames) = parse_derive(ast);
-        assert_eq!(filenames, [GrammarSource::File("myfile.pest".to_string())]);
-        assert!(parsed_derive.non_exhaustive);
-    }
-
-    #[test]
-    #[should_panic(expected = "grammar attribute must be a string")]
-    fn derive_wrong_arg() {
-        let definition = "
-            #[other_attr]
-            #[grammar = 1]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        parse_derive(ast);
-    }
-
-    #[test]
-    #[should_panic(
-        expected = "a grammar file needs to be provided with the #[grammar = \"PATH\"] or #[grammar_inline = \"GRAMMAR CONTENTS\"] attribute"
-    )]
-    fn derive_no_grammar() {
-        let definition = "
-            #[other_attr]
-            pub struct MyParser<'a, T>;
-        ";
-        let ast = syn::parse_str(definition).unwrap();
-        parse_derive(ast);
-    }
 
     #[doc = "Matches dar\n\nMatch dar description\n"]
     #[test]