From 38d33c2ea1c663139dd33cbf8b2a14b00c73af88 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 21:39:20 +0800 Subject: [PATCH 1/8] add tracing support --- peg-macros/Cargo.toml | 3 ++- peg-macros/translate.rs | 46 +++++++++++++++++++++++++++++++++-------- peg-runtime/error.rs | 1 + 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/peg-macros/Cargo.toml b/peg-macros/Cargo.toml index c4a9e9d..1eaef25 100644 --- a/peg-macros/Cargo.toml +++ b/peg-macros/Cargo.toml @@ -11,9 +11,10 @@ edition = "2018" quote = "1.0" proc-macro2 = "1.0.24" peg-runtime = { version = "= 0.8.1", path = "../peg-runtime" } +tracing = { version = "0.1.37", optional = true, features = ["attributes"] } [features] -trace = [] +trace = ["dep:tracing"] [lib] proc-macro = true diff --git a/peg-macros/translate.rs b/peg-macros/translate.rs index 3afbe18..2bda766 100644 --- a/peg-macros/translate.rs +++ b/peg-macros/translate.rs @@ -168,8 +168,15 @@ fn make_parse_state(grammar: &Grammar) -> TokenStream { } } + let trace = if cfg!(feature = "trace") { + quote!( #[derive(Debug)] ) + } else { + quote!() + }; + quote_spanned! { span => #[allow(unused_parens)] + #trace struct ParseState<'input #(, #grammar_lifetime_params)*> { _phantom: ::core::marker::PhantomData<(&'input () #(, &#grammar_lifetime_params ())*)>, #(#cache_fields_def),* @@ -232,18 +239,17 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream { let body = compile_expr(&context, &rule.expr, rule.ret_type.is_some()); let wrapped_body = if cfg!(feature = "trace") { - let str_rule_name = rule.name.to_string(); quote_spanned! { span => { let loc = ::peg::Parse::position_repr(__input, __pos); - println!("[PEG_TRACE] Attempting to match rule `{}` at {}", #str_rule_name, loc); + tracing::trace!(loc = %loc, "Attempting"); let __peg_result: ::peg::RuleResult<#ret_ty> = {#body}; match __peg_result { ::peg::RuleResult::Matched(epos, _) => { let eloc = ::peg::Parse::position_repr(__input, epos); - println!("[PEG_TRACE] Matched rule `{}` at {} to {}", #str_rule_name, loc, eloc); + tracing::trace!(eloc = %eloc, "Matched"); } ::peg::RuleResult::Failed => { - println!("[PEG_TRACE] Failed to match rule `{}` at {}", #str_rule_name, loc); + tracing::trace!("Failed"); } } @@ -261,12 +267,11 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream { let cache_field = format_ident!("{}_cache", rule.name); let cache_trace = if cfg!(feature = "trace") { - let str_rule_name = rule.name.to_string(); quote_spanned! { span => let loc = ::peg::Parse::position_repr(__input, __pos); match &entry { - &::peg::RuleResult::Matched(..) => println!("[PEG_TRACE] Cached match of rule {} at {}", #str_rule_name, loc), - &Failed => println!("[PEG_TRACE] Cached fail of rule {} at {}", #str_rule_name, loc), + &::peg::RuleResult::Matched(..) => tracing::trace!(loc = %loc, "Cache matched"), + &::peg::RuleResult::Failed => tracing::trace!(loc = %loc, "Cache failed"), }; } } else { @@ -318,7 +323,16 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream { } }; + let trace = if cfg!(feature = "trace") { + let str_rule_name = rule.name.to_string(); + + quote!( #[tracing::instrument(fields(rule = #str_rule_name))] ) + } else { + quote!() + }; + quote_spanned! { span => + #trace fn #name<'input #(, #grammar_lifetime_params)* #(, #ty_params)*>(__input: #input_ty, __state: #parse_state_ty, __err_state: &mut ::peg::error::ErrorState, __pos: usize #extra_args_def #(, #rule_params)*) -> ::peg::RuleResult<#ret_ty> { #![allow(non_snake_case, unused, clippy::redundant_closure_call)] #fn_body @@ -363,11 +377,18 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream { quote_spanned! { span => ::peg::Parse::is_eof(__input, __pos) } }; + let trace = if cfg!(feature = "trace") { + quote!( #[tracing::instrument] ) + } else { + quote!() + }; + // Parse once. If it succeeds or throws an error, return that. // If it fails, parse again to determine the set of all tokens // that were expected at the failure position. quote_spanned! { span => + #trace #doc #visibility fn #name<'input #(, #grammar_lifetime_params)* #(, #ty_params)*>(__input: #input_ty #extra_args_def #(, #rule_params)*) -> ::core::result::Result<#ret_ty, ::peg::error::ParseError>> { #![allow(non_snake_case, unused)] @@ -921,17 +942,24 @@ fn compile_expr(context: &Context, e: &SpannedExpr, result_used: bool) -> TokenS let (enter, leave) = if cfg!(feature = "trace") { ( - quote_spanned! {span => println!("[PEG_TRACE] Entering level {}", min_prec);}, - quote_spanned! {span => println!("[PEG_TRACE] Leaving level {}", min_prec);}, + quote_spanned! {span => tracing::trace!("Entering");}, + quote_spanned! {span => tracing::trace!("Leaving");}, ) } else { (quote!(), quote!()) }; + let trace = if cfg!(feature = "trace") { + quote!( #[tracing::instrument] ) + } else { + quote!() + }; + // The closures below must be defined within the function call to which they are passed // due to https://github.com/rust-lang/rust/issues/41078 quote_spanned! { span => { + #trace fn __infix_parse( state: &mut S, err_state: &mut ::peg::error::ErrorState, diff --git a/peg-runtime/error.rs b/peg-runtime/error.rs index b092643..18ac137 100644 --- a/peg-runtime/error.rs +++ b/peg-runtime/error.rs @@ -71,6 +71,7 @@ impl ::std::error::Error for ParseError { } #[doc(hidden)] +#[derive(Debug)] pub struct ErrorState { /// Furthest failure we've hit so far. pub max_err_pos: usize, From 844ac41465fce348b88c283afba4faddc4be801e Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:19:48 +0800 Subject: [PATCH 2/8] update cargo lock --- Cargo.lock | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index bbf5526..1636306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "serde", ] +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "glob" version = "0.3.0" @@ -45,12 +51,19 @@ dependencies = [ "peg-runtime", "proc-macro2", "quote", + "tracing", ] [[package]] name = "peg-runtime" version = "0.8.1" +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + [[package]] name = "proc-macro2" version = "1.0.56" @@ -89,7 +102,7 @@ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.13", ] [[package]] @@ -103,6 +116,17 @@ dependencies = [ "serde", ] +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.13" @@ -123,6 +147,38 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + [[package]] name = "trybuild" version = "1.0.80" From e0133807a0bb6b04adb9452cd1865f4ff2a3d9b6 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:31:33 +0800 Subject: [PATCH 3/8] remove debug requirement for ParseState --- peg-macros/translate.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/peg-macros/translate.rs b/peg-macros/translate.rs index 2bda766..40b3327 100644 --- a/peg-macros/translate.rs +++ b/peg-macros/translate.rs @@ -168,15 +168,8 @@ fn make_parse_state(grammar: &Grammar) -> TokenStream { } } - let trace = if cfg!(feature = "trace") { - quote!( #[derive(Debug)] ) - } else { - quote!() - }; - quote_spanned! { span => #[allow(unused_parens)] - #trace struct ParseState<'input #(, #grammar_lifetime_params)*> { _phantom: ::core::marker::PhantomData<(&'input () #(, &#grammar_lifetime_params ())*)>, #(#cache_fields_def),* From 13b3996e1870c01598b11316ba24e6e0be104bd2 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:32:02 +0800 Subject: [PATCH 4/8] be conservative and pick only traces that are relevant --- peg-macros/translate.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/peg-macros/translate.rs b/peg-macros/translate.rs index 40b3327..6a56d66 100644 --- a/peg-macros/translate.rs +++ b/peg-macros/translate.rs @@ -319,7 +319,7 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream { let trace = if cfg!(feature = "trace") { let str_rule_name = rule.name.to_string(); - quote!( #[tracing::instrument(fields(rule = #str_rule_name))] ) + quote!( #[tracing::instrument(skip_all, fields(rule = #str_rule_name, pos = __pos))] ) } else { quote!() }; @@ -371,7 +371,7 @@ fn compile_rule_export(context: &Context, rule: &Rule) -> TokenStream { }; let trace = if cfg!(feature = "trace") { - quote!( #[tracing::instrument] ) + quote!( #[tracing::instrument(skip_all, err)] ) } else { quote!() }; @@ -943,7 +943,7 @@ fn compile_expr(context: &Context, e: &SpannedExpr, result_used: bool) -> TokenS }; let trace = if cfg!(feature = "trace") { - quote!( #[tracing::instrument] ) + quote!( #[tracing::instrument(skip_all, fields(min_prec, lpos))] ) } else { quote!() }; From 7a9204885d1f91e6ca646a9c234767d6b50f32af Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:39:00 +0800 Subject: [PATCH 5/8] add tracing to dev dependencies --- Cargo.lock | 1 + Cargo.toml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1636306..dca1f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,7 @@ version = "0.8.1" dependencies = [ "peg-macros", "peg-runtime", + "tracing", "trybuild", ] diff --git a/Cargo.toml b/Cargo.toml index 25302c4..e0070d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ peg-runtime = { path = "./peg-runtime", version = "= 0.8.1" } [dev-dependencies] trybuild = "1.0.80" +tracing = "0.1.37" [[test]] name = "trybuild" @@ -29,4 +30,4 @@ harness = false default = ["std"] trace = ["peg-macros/trace"] std = ["peg-runtime/std"] -unstable = ["peg-runtime/unstable"] \ No newline at end of file +unstable = ["peg-runtime/unstable"] From 6bc2559229e319923c8d06898905e5acd4753877 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:48:18 +0800 Subject: [PATCH 6/8] add loc back to the span --- peg-macros/translate.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/peg-macros/translate.rs b/peg-macros/translate.rs index 6a56d66..7f8560c 100644 --- a/peg-macros/translate.rs +++ b/peg-macros/translate.rs @@ -239,10 +239,10 @@ fn compile_rule(context: &Context, rule: &Rule) -> TokenStream { match __peg_result { ::peg::RuleResult::Matched(epos, _) => { let eloc = ::peg::Parse::position_repr(__input, epos); - tracing::trace!(eloc = %eloc, "Matched"); + tracing::trace!(loc = %loc, eloc = %eloc, "Matched"); } ::peg::RuleResult::Failed => { - tracing::trace!("Failed"); + tracing::trace!(loc = %loc, "Failed"); } } From 3db395abaea312e1d571faf99d33f35f6c619fcc Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Mon, 10 Apr 2023 22:49:09 +0800 Subject: [PATCH 7/8] remove Debug from ErrorState --- peg-runtime/error.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/peg-runtime/error.rs b/peg-runtime/error.rs index 18ac137..b092643 100644 --- a/peg-runtime/error.rs +++ b/peg-runtime/error.rs @@ -71,7 +71,6 @@ impl ::std::error::Error for ParseError { } #[doc(hidden)] -#[derive(Debug)] pub struct ErrorState { /// Furthest failure we've hit so far. pub max_err_pos: usize, From 9cfc49e7264885f59aed05635957e61bc166a879 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Tue, 11 Apr 2023 00:14:01 +0800 Subject: [PATCH 8/8] add a tracing test case --- Cargo.lock | 111 ++++++++++++++++++++++++++++++++++++-- Cargo.toml | 2 + tests/run-pass/tracing.rs | 72 +++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 tests/run-pass/tracing.rs diff --git a/Cargo.lock b/Cargo.lock index dca1f6e..d34087b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,34 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" [[package]] name = "itoa" -version = "0.4.6" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] [[package]] name = "once_cell" @@ -35,13 +60,21 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "peg" version = "0.8.1" dependencies = [ "peg-macros", "peg-runtime", + "serde_json", "tracing", + "tracing-subscriber", "trybuild", ] @@ -108,15 +141,30 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.57" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164eacbdb13512ec2745fb09d51fd5b22b0d65ed294a1dcf7285a360c80a675c" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "itoa", "ryu", "serde", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + [[package]] name = "syn" version = "1.0.109" @@ -148,6 +196,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "tracing" version = "0.1.37" @@ -178,6 +236,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -201,6 +298,12 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index e0070d5..9a883e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ peg-runtime = { path = "./peg-runtime", version = "= 0.8.1" } [dev-dependencies] trybuild = "1.0.80" tracing = "0.1.37" +tracing-subscriber = { version = "0.3.16", features = ["json"] } +serde_json = "1.0.95" [[test]] name = "trybuild" diff --git a/tests/run-pass/tracing.rs b/tests/run-pass/tracing.rs new file mode 100644 index 0000000..47e4547 --- /dev/null +++ b/tests/run-pass/tracing.rs @@ -0,0 +1,72 @@ +extern crate peg; + +peg::parser!(grammar parser() for str { + // JSON grammar (RFC 4627). Note that this only checks for valid JSON and does not build a syntax + // tree. + + pub rule json() = _ (object() / array()) _ + + rule _() = [' ' | '\t' | '\r' | '\n']* + rule value_separator() = _ "," _ + + rule value() + = "false" / "true" / "null" / object() / array() / number() / string() + + rule object() + = "{" _ member() ** value_separator() _ "}" + + rule member() + = string() _ ":" _ value() + + rule array() + = "[" _ (value() ** value_separator()) _ "]" + + rule number() + = "-"? int() frac()? exp()? {} + + rule int() + = ['0'] / ['1'..='9']['0'..='9']* + + rule exp() + = ("e" / "E") ("-" / "+")? ['0'..='9']*<1,> + + rule frac() + = "." ['0'..='9']*<1,> + + // note: escaped chars not handled + rule string() + = "\"" (!"\"" [_])* "\"" +}); + +fn main() { + #[cfg(feature = "trace")] + tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .json() + .init(); + + let input = r#" +{ + "X": 0.6e2, + "Y": 5, + "Z": -5.312344, + "Bool": false, + "Bool": true, + "Null": null, + "Attr": { + "Name": "bla", + "Siblings": [6, 1, 2, {}, {}, {}] + }, + "Nested Array": [[[[[[[[[]]]]]]]]], + "Obj": { + "Child": { + "A": [], + "Child": { + "Child": {} + } + } + } +} +"#; + parser::json(input).unwrap(); +}