From 5db05e86f09f075862f8a2873f31ceb4919d4dba Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 21 Nov 2024 15:23:21 -0300 Subject: [PATCH 01/11] First attempt: re-parse leading trivia and advance index accordingly --- .../parser/parser_support/parser_function.rs | 18 ++++++++++++++---- .../parser/parser_support/parser_function.rs | 18 ++++++++++++++---- .../generated/0.4.11-failure.yml | 10 ++++------ .../generated/0.4.25-failure.yml | 9 ++++----- .../generated/0.4.25-failure.yml | 9 ++++----- .../try_catch/generated/0.4.11-failure.yml | 4 ++-- .../with_var/generated/0.5.0-failure.yml | 9 ++++----- .../generated/0.6.0-failure.yml | 9 ++++----- .../parser/parser_support/parser_function.rs | 18 ++++++++++++++---- 9 files changed, 64 insertions(+), 40 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs index 2cb239078e..964268c940 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs @@ -57,16 +57,26 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { - let kind = if input.is_empty() { - TerminalKind::MISSING + let (kind, start) = if input.is_empty() { + (TerminalKind::MISSING, TextIndex::ZERO) } else { - TerminalKind::UNRECOGNIZED + let mut stream = ParserContext::new(input); + if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) + { + let mut ti = TextIndex::ZERO; + trivia.nodes.iter().for_each(|node| { + node.unparse().chars().for_each(|c| ti.advance(c, None)); + }); + (TerminalKind::UNRECOGNIZED, ti) + } else { + (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) + } }; ParseOutput { parse_tree: Node::terminal(kind, input.to_string()), errors: vec![ParseError::new( - TextIndex::ZERO..input.into(), + start..input.into(), no_match.expected_terminals, )], } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index 57f5d3ed4b..f271e6dfe2 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -59,16 +59,26 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { - let kind = if input.is_empty() { - TerminalKind::MISSING + let (kind, start) = if input.is_empty() { + (TerminalKind::MISSING, TextIndex::ZERO) } else { - TerminalKind::UNRECOGNIZED + let mut stream = ParserContext::new(input); + if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) + { + let mut ti = TextIndex::ZERO; + trivia.nodes.iter().for_each(|node| { + node.unparse().chars().for_each(|c| ti.advance(c, None)); + }); + (TerminalKind::UNRECOGNIZED, ti) + } else { + (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) + } }; ParseOutput { parse_tree: Node::terminal(kind, input.to_string()), errors: vec![ParseError::new( - TextIndex::ZERO..input.into(), + start..input.into(), no_match.expected_terminals, )], } diff --git a/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml b/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml index 53b662c853..b92ae184b3 100644 --- a/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml @@ -9,13 +9,11 @@ Source: > Errors: # 1 total - > Error: Expected ContractKeyword. - ╭─[crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/input.sol:4:1] │ - 1 │ ╭─▶ // ╒════════════════════════════════════════════════════════════════╕ - ┆ ┆ - 4 │ ├─▶ unexpected - │ │ - │ ╰──────────────── Error occurred here. + 4 │ unexpected + │ ─────┬───── + │ ╰─────── Error occurred here. ───╯ Tree: diff --git a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml index 9d82a9cbc4..fa426b798c 100644 --- a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml @@ -7,12 +7,11 @@ Source: > Errors: # 1 total - > Error: Expected DoubleQuotedStringLiteral or SingleQuotedStringLiteral. - ╭─[crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/input.sol:2:1] │ - 1 │ ╭─▶ // Disallowed post 0.4.25 - 2 │ ├─▶ "\a" - │ │ - │ ╰────────── Error occurred here. + 2 │ "\a" + │ ──┬── + │ ╰──── Error occurred here. ───╯ Tree: diff --git a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml index 687e39fa6e..48bab2bab6 100644 --- a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml @@ -7,12 +7,11 @@ Source: > Errors: # 1 total - > Error: Expected DoubleQuotedStringLiteral or SingleQuotedStringLiteral. - ╭─[crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/input.sol:2:1] │ - 1 │ ╭─▶ // Disallowed post 0.4.25 - 2 │ ├─▶ "\✅" - │ │ - │ ╰─────────── Error occurred here. + 2 │ "\✅" + │ ──┬─── + │ ╰───── Error occurred here. ───╯ Tree: diff --git a/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml b/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml index 112b08de0b..893a1bbd44 100644 --- a/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml @@ -12,9 +12,9 @@ Source: > Errors: # 1 total - > Error: Expected end of file. - ╭─[crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/input.sol:4:1] │ - 1 │ ╭─▶ // Make sure that error recovery won't lead to misparsing + 4 │ ╭─▶ try foo() { ┆ ┆ 7 │ ├─▶ } │ │ diff --git a/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml index 35c47c54f3..124aed234c 100644 --- a/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml @@ -7,12 +7,11 @@ Source: > Errors: # 1 total - > Error: Expected OpenParen. - ╭─[crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/input.sol:2:1] │ - 1 │ ╭─▶ // "var" should be disabled in "0.5.0": - 2 │ ├─▶ var (foo, bar) = rhs; - │ │ - │ ╰─────────────────────────── Error occurred here. + 2 │ var (foo, bar) = rhs; + │ ───────────┬────────── + │ ╰──────────── Error occurred here. ───╯ Tree: diff --git a/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml index c588db9952..df7c141a1b 100644 --- a/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml @@ -7,12 +7,11 @@ Source: > Errors: # 1 total - > Error: Expected end of file. - ╭─[crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/input.sol:1:1] + ╭─[crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/input.sol:2:1] │ - 1 │ ╭─▶ // Split into `view` and `pure` in 0.4.16 - 2 │ ├─▶ function () constant {} - │ │ - │ ╰───────────────────────────── Error occurred here. + 2 │ function () constant {} + │ ────────────┬─────────── + │ ╰───────────── Error occurred here. ───╯ Tree: diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index 57f5d3ed4b..f271e6dfe2 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -59,16 +59,26 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { - let kind = if input.is_empty() { - TerminalKind::MISSING + let (kind, start) = if input.is_empty() { + (TerminalKind::MISSING, TextIndex::ZERO) } else { - TerminalKind::UNRECOGNIZED + let mut stream = ParserContext::new(input); + if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) + { + let mut ti = TextIndex::ZERO; + trivia.nodes.iter().for_each(|node| { + node.unparse().chars().for_each(|c| ti.advance(c, None)); + }); + (TerminalKind::UNRECOGNIZED, ti) + } else { + (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) + } }; ParseOutput { parse_tree: Node::terminal(kind, input.to_string()), errors: vec![ParseError::new( - TextIndex::ZERO..input.into(), + start..input.into(), no_match.expected_terminals, )], } From 86540f43295133de507d5b8a75008e2adc180b84 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Mon, 25 Nov 2024 15:07:10 -0300 Subject: [PATCH 02/11] Adding nodes in NoMatch --- .../crate/src/runtime/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 2 +- .../parser/parser_support/parser_result.rs | 16 ++-- .../runtime/parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 11 ++- .../parser/parser_support/separated_helper.rs | 2 +- .../codegen/precedence_parser_definition.rs | 4 +- .../src/generated/parser/generated/parser.rs | 84 +++++++++---------- .../crate/src/generated/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 2 +- .../parser/parser_support/parser_result.rs | 16 ++-- .../parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 11 ++- .../parser/parser_support/separated_helper.rs | 2 +- .../src/generated/parser/generated/parser.rs | 12 +-- .../crate/src/generated/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 2 +- .../parser/parser_support/parser_result.rs | 16 ++-- .../parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 11 ++- .../parser/parser_support/separated_helper.rs | 2 +- 21 files changed, 128 insertions(+), 89 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs index 71d28c520b..1a1882e2e0 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs @@ -94,7 +94,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(vec![], vec![kind]); } let end = input.position(); @@ -129,7 +129,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(children, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs index a52654b63c..3253a7988d 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs @@ -21,7 +21,7 @@ pub struct ChoiceHelper { impl ChoiceHelper { pub fn new(input: &mut ParserContext<'_>) -> Self { Self { - result: ParserResult::no_match(vec![]), + result: ParserResult::default(), start_position: input.mark(), recovered_errors: vec![], last_progress: input.position(), diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs index e6afd564cb..e86d8c3550 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs @@ -16,6 +16,7 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { + nodes: vec![], expected_terminals: vec![], }) } @@ -36,11 +37,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![]) + Self::no_match(vec![], vec![]) } - pub fn no_match(expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(expected_terminals)) + pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) } #[must_use] @@ -230,13 +231,18 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { + /// The terminals that were parsed; trivia + pub nodes: Vec, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(expected_terminals: Vec) -> Self { - Self { expected_terminals } + pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + Self { + nodes, + expected_terminals, + } } } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs index cec55a393c..8cd4fd372c 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs @@ -105,7 +105,9 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => ParserResult::no_match(expected_terminals), + ParseResultKind::NoMatch => { + ParserResult::no_match(leading_trivia, expected_terminals) + } } } } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs index e4bcf1433d..b4f2c14abb 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs @@ -26,8 +26,11 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals - ParserResult::NoMatch(NoMatch { expected_terminals }) if MIN_COUNT == 0 => { - return ParserResult::r#match(vec![], expected_terminals); + ParserResult::NoMatch(NoMatch { + nodes, + expected_terminals, + }) if MIN_COUNT == 0 => { + return ParserResult::r#match(nodes, expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, @@ -60,7 +63,9 @@ impl RepetitionHelper { ParserResult::IncompleteMatch(IncompleteMatch { expected_terminals, .. }) - | ParserResult::NoMatch(NoMatch { expected_terminals }), + | ParserResult::NoMatch(NoMatch { + expected_terminals, .. + }), ) => { input.rewind(save); running.expected_terminals = expected_terminals; diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs index e1455a8155..44615a7d4f 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs @@ -89,7 +89,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; diff --git a/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs b/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs index 30c11f90a3..b917749aaa 100644 --- a/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs +++ b/crates/codegen/runtime/generator/src/parser/codegen/precedence_parser_definition.rs @@ -49,9 +49,9 @@ impl PrecedenceParserDefinitionCodegen for PrecedenceParserDefinitionRef { [inner @ cst::Edge { node: cst::Node::Nonterminal(node), .. }] if node.kind == NonterminalKind::#op_nonterminal_name => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } }; diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/generated/parser.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/generated/parser.rs index 4daf7c6f0c..91c358a3b4 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/generated/parser.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/generated/parser.rs @@ -254,9 +254,9 @@ impl Parser { }] if node.kind == NonterminalKind::AdditiveExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -301,9 +301,9 @@ impl Parser { }] if node.kind == NonterminalKind::AndExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -371,9 +371,9 @@ impl Parser { }] if node.kind == NonterminalKind::ArrayTypeName => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -476,9 +476,9 @@ impl Parser { }] if node.kind == NonterminalKind::AssignmentExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -499,9 +499,9 @@ impl Parser { }] if node.kind == NonterminalKind::BitwiseAndExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -522,9 +522,9 @@ impl Parser { }] if node.kind == NonterminalKind::BitwiseOrExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -545,9 +545,9 @@ impl Parser { }] if node.kind == NonterminalKind::BitwiseXorExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -646,9 +646,9 @@ impl Parser { }] if node.kind == NonterminalKind::CallOptionsExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -727,9 +727,9 @@ impl Parser { }] if node.kind == NonterminalKind::ComparisonExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -750,9 +750,9 @@ impl Parser { }] if node.kind == NonterminalKind::ConditionalExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -1320,9 +1320,9 @@ impl Parser { }] if node.kind == NonterminalKind::EqualityExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -1625,9 +1625,9 @@ impl Parser { }] if node.kind == NonterminalKind::ExponentiationExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -2639,9 +2639,9 @@ impl Parser { }] if node.kind == NonterminalKind::FunctionCallExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -3082,9 +3082,9 @@ impl Parser { }] if node.kind == NonterminalKind::IndexAccessExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -3375,9 +3375,9 @@ impl Parser { }] if node.kind == NonterminalKind::MemberAccessExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -3467,9 +3467,9 @@ impl Parser { }] if node.kind == NonterminalKind::MultiplicativeExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -3705,9 +3705,9 @@ impl Parser { }] if node.kind == NonterminalKind::OrExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -3932,9 +3932,9 @@ impl Parser { }] if node.kind == NonterminalKind::PostfixExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -4004,9 +4004,9 @@ impl Parser { }] if node.kind == NonterminalKind::PrefixExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -4192,9 +4192,9 @@ impl Parser { }] if node.kind == NonterminalKind::ShiftExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -6468,9 +6468,9 @@ impl Parser { }] if node.kind == NonterminalKind::YulFunctionCallExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 02bbe2b817..4212f7b4f4 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(vec![], vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(children, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 925582d5f7..84e9295266 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -23,7 +23,7 @@ pub struct ChoiceHelper { impl ChoiceHelper { pub fn new(input: &mut ParserContext<'_>) -> Self { Self { - result: ParserResult::no_match(vec![]), + result: ParserResult::default(), start_position: input.mark(), recovered_errors: vec![], last_progress: input.position(), diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index ed86a774bf..260dc6fc61 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -18,6 +18,7 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { + nodes: vec![], expected_terminals: vec![], }) } @@ -38,11 +39,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![]) + Self::no_match(vec![], vec![]) } - pub fn no_match(expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(expected_terminals)) + pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) } #[must_use] @@ -232,13 +233,18 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { + /// The terminals that were parsed; trivia + pub nodes: Vec, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(expected_terminals: Vec) -> Self { - Self { expected_terminals } + pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + Self { + nodes, + expected_terminals, + } } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index f54ae45934..848eb12e81 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -107,7 +107,9 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => ParserResult::no_match(expected_terminals), + ParseResultKind::NoMatch => { + ParserResult::no_match(leading_trivia, expected_terminals) + } } } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index a808e73796..95026d38ae 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -28,8 +28,11 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals - ParserResult::NoMatch(NoMatch { expected_terminals }) if MIN_COUNT == 0 => { - return ParserResult::r#match(vec![], expected_terminals); + ParserResult::NoMatch(NoMatch { + nodes, + expected_terminals, + }) if MIN_COUNT == 0 => { + return ParserResult::r#match(nodes, expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, @@ -62,7 +65,9 @@ impl RepetitionHelper { ParserResult::IncompleteMatch(IncompleteMatch { expected_terminals, .. }) - | ParserResult::NoMatch(NoMatch { expected_terminals }), + | ParserResult::NoMatch(NoMatch { + expected_terminals, .. + }), ) => { input.rewind(save); running.expected_terminals = expected_terminals; diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index bcc5dc64f0..527944afba 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/generated/parser.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/generated/parser.rs index 9526734434..ef1e4ba83a 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/generated/parser.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/generated/parser.rs @@ -87,9 +87,9 @@ impl Parser { }] if node.kind == NonterminalKind::AdditionExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -237,9 +237,9 @@ impl Parser { }] if node.kind == NonterminalKind::MemberAccessExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } @@ -260,9 +260,9 @@ impl Parser { }] if node.kind == NonterminalKind::NegationExpression => { ParserResult::r#match(vec![inner.clone()], r#match.expected_terminals.clone()) } - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), }, - _ => ParserResult::no_match(vec![]), + _ => ParserResult::default(), } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 02bbe2b817..4212f7b4f4 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(vec![], vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(vec![kind]); + return ParserResult::no_match(children, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 925582d5f7..84e9295266 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -23,7 +23,7 @@ pub struct ChoiceHelper { impl ChoiceHelper { pub fn new(input: &mut ParserContext<'_>) -> Self { Self { - result: ParserResult::no_match(vec![]), + result: ParserResult::default(), start_position: input.mark(), recovered_errors: vec![], last_progress: input.position(), diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index ed86a774bf..260dc6fc61 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -18,6 +18,7 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { + nodes: vec![], expected_terminals: vec![], }) } @@ -38,11 +39,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![]) + Self::no_match(vec![], vec![]) } - pub fn no_match(expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(expected_terminals)) + pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) } #[must_use] @@ -232,13 +233,18 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { + /// The terminals that were parsed; trivia + pub nodes: Vec, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(expected_terminals: Vec) -> Self { - Self { expected_terminals } + pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + Self { + nodes, + expected_terminals, + } } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index f54ae45934..848eb12e81 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -107,7 +107,9 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => ParserResult::no_match(expected_terminals), + ParseResultKind::NoMatch => { + ParserResult::no_match(leading_trivia, expected_terminals) + } } } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index a808e73796..95026d38ae 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -28,8 +28,11 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals - ParserResult::NoMatch(NoMatch { expected_terminals }) if MIN_COUNT == 0 => { - return ParserResult::r#match(vec![], expected_terminals); + ParserResult::NoMatch(NoMatch { + nodes, + expected_terminals, + }) if MIN_COUNT == 0 => { + return ParserResult::r#match(nodes, expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, @@ -62,7 +65,9 @@ impl RepetitionHelper { ParserResult::IncompleteMatch(IncompleteMatch { expected_terminals, .. }) - | ParserResult::NoMatch(NoMatch { expected_terminals }), + | ParserResult::NoMatch(NoMatch { + expected_terminals, .. + }), ) => { input.rewind(save); running.expected_terminals = expected_terminals; diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index bcc5dc64f0..527944afba 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; From a74ea7d3fc2117aee2d5fe4465889b3d9cc4b7ad Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Wed, 27 Nov 2024 13:35:09 -0300 Subject: [PATCH 03/11] NoMatch with tracking information about trivia and originator non-terminal --- .../crate/src/runtime/parser/lexer/mod.rs | 4 +-- .../parser/parser_support/choice_helper.rs | 6 +++- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 31 +++++++++++-------- .../parser/parser_support/parser_result.rs | 26 ++++++++++++---- .../runtime/parser/parser_support/recovery.rs | 8 +++-- .../parser_support/repetition_helper.rs | 1 + .../parser/parser_support/separated_helper.rs | 2 +- crates/metaslang/cst/src/text_index.rs | 8 +++++ .../crate/src/generated/parser/lexer/mod.rs | 4 +-- .../parser/parser_support/choice_helper.rs | 6 +++- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 31 +++++++++++-------- .../parser/parser_support/parser_result.rs | 26 ++++++++++++---- .../parser/parser_support/recovery.rs | 8 +++-- .../parser_support/repetition_helper.rs | 1 + .../parser/parser_support/separated_helper.rs | 2 +- 17 files changed, 116 insertions(+), 52 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs index 1a1882e2e0..5925617b28 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs @@ -94,7 +94,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], vec![kind]); + return ParserResult::no_match(vec![], None, vec![kind]); } let end = input.position(); @@ -129,7 +129,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, vec![kind]); + return ParserResult::no_match(children, None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs index 3253a7988d..353e090cf8 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs @@ -52,6 +52,10 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { + if running.nodes.is_empty() && !next.nodes.is_empty() { + // transfer nodes + running.nodes.extend(next.nodes.clone()); + } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -127,7 +131,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &[][..], + ParserResult::NoMatch(no_match) => &no_match.nodes, ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs index 9ef5430056..a052cc4f66 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs @@ -8,7 +8,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(vec![], no_match.expected_terminals) + ParserResult::r#match(no_match.nodes, no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs index 964268c940..189b9c7734 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs @@ -56,25 +56,30 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(no_match) => { + ParserResult::NoMatch(mut no_match) => { let (kind, start) = if input.is_empty() { (TerminalKind::MISSING, TextIndex::ZERO) } else { - let mut stream = ParserContext::new(input); - if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) - { - let mut ti = TextIndex::ZERO; - trivia.nodes.iter().for_each(|node| { - node.unparse().chars().for_each(|c| ti.advance(c, None)); - }); - (TerminalKind::UNRECOGNIZED, ti) - } else { - (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) - } + let mut ti = TextIndex::ZERO; + no_match.nodes.iter().for_each(|edge| { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + ti.advance_str(terminal.text.as_str()); + } + } + }); + (TerminalKind::UNRECOGNIZED, ti) }; + let node = Node::terminal(kind, input[start.utf8..].to_string()); + let tree = if no_match.kind.is_none() || start.utf8 == 0 { + node + } else { + no_match.nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + }; ParseOutput { - parse_tree: Node::terminal(kind, input.to_string()), + parse_tree: tree, errors: vec![ParseError::new( start..input.into(), no_match.expected_terminals, diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs index e86d8c3550..57db35864e 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs @@ -17,6 +17,7 @@ impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { nodes: vec![], + kind: None, expected_terminals: vec![], }) } @@ -35,13 +36,17 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], vec![]) + Self::no_match(vec![], None, vec![]) } - pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) + pub fn no_match( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) } #[must_use] @@ -62,7 +67,9 @@ impl ParserResult { nodes: vec![Edge::anonymous(Node::nonterminal(new_kind, skipped.nodes))], ..skipped }), - ParserResult::NoMatch(_) => self, + ParserResult::NoMatch(no_match) => { + ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") } @@ -233,14 +240,21 @@ impl IncompleteMatch { pub struct NoMatch { /// The terminals that were parsed; trivia pub nodes: Vec, + /// The nonterminal kind this match is coming from + pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + pub fn new( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { Self { nodes, + kind, expected_terminals, } } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs index 8cd4fd372c..1a610349d0 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs @@ -70,7 +70,11 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - (vec![], result.expected_terminals, ParseResultKind::NoMatch) + ( + vec![], /* trivia will be read below */ + result.expected_terminals, + ParseResultKind::NoMatch, + ) } // No need to recover, so just return as-is. _ => return self, @@ -106,7 +110,7 @@ impl ParserResult { ParserResult::incomplete_match(nodes, expected_terminals) } ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, expected_terminals) + ParserResult::no_match(leading_trivia, None, expected_terminals) } } } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs index b4f2c14abb..7a13af5bd1 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs @@ -28,6 +28,7 @@ impl RepetitionHelper { // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { nodes, + kind: _, expected_terminals, }) if MIN_COUNT == 0 => { return ParserResult::r#match(nodes, expected_terminals); diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs index 44615a7d4f..8efcac8f74 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs @@ -89,7 +89,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; diff --git a/crates/metaslang/cst/src/text_index.rs b/crates/metaslang/cst/src/text_index.rs index c107c4f191..c0144c4a3f 100644 --- a/crates/metaslang/cst/src/text_index.rs +++ b/crates/metaslang/cst/src/text_index.rs @@ -40,6 +40,14 @@ impl TextIndex { } } } + + pub fn advance_str(&mut self, text: &str) { + text.chars().for_each(|c| { + if c != '\r' { + self.advance(c, None); + } + }); + } } impl PartialOrd for TextIndex { diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 4212f7b4f4..480c652881 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], vec![kind]); + return ParserResult::no_match(vec![], None, vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, vec![kind]); + return ParserResult::no_match(children, None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 84e9295266..f7c44e0137 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -54,6 +54,10 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { + if running.nodes.is_empty() && !next.nodes.is_empty() { + // transfer nodes + running.nodes.extend(next.nodes.clone()); + } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -129,7 +133,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &[][..], + ParserResult::NoMatch(no_match) => &no_match.nodes, ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs index 989e259cda..7bcdead08e 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs @@ -10,7 +10,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(vec![], no_match.expected_terminals) + ParserResult::r#match(no_match.nodes, no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index f271e6dfe2..d1bd24d10e 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -58,25 +58,30 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(no_match) => { + ParserResult::NoMatch(mut no_match) => { let (kind, start) = if input.is_empty() { (TerminalKind::MISSING, TextIndex::ZERO) } else { - let mut stream = ParserContext::new(input); - if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) - { - let mut ti = TextIndex::ZERO; - trivia.nodes.iter().for_each(|node| { - node.unparse().chars().for_each(|c| ti.advance(c, None)); - }); - (TerminalKind::UNRECOGNIZED, ti) - } else { - (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) - } + let mut ti = TextIndex::ZERO; + no_match.nodes.iter().for_each(|edge| { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + ti.advance_str(terminal.text.as_str()); + } + } + }); + (TerminalKind::UNRECOGNIZED, ti) }; + let node = Node::terminal(kind, input[start.utf8..].to_string()); + let tree = if no_match.kind.is_none() || start.utf8 == 0 { + node + } else { + no_match.nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + }; ParseOutput { - parse_tree: Node::terminal(kind, input.to_string()), + parse_tree: tree, errors: vec![ParseError::new( start..input.into(), no_match.expected_terminals, diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 260dc6fc61..016f002ca4 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -19,6 +19,7 @@ impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { nodes: vec![], + kind: None, expected_terminals: vec![], }) } @@ -37,13 +38,17 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], vec![]) + Self::no_match(vec![], None, vec![]) } - pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) + pub fn no_match( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) } #[must_use] @@ -64,7 +69,9 @@ impl ParserResult { nodes: vec![Edge::anonymous(Node::nonterminal(new_kind, skipped.nodes))], ..skipped }), - ParserResult::NoMatch(_) => self, + ParserResult::NoMatch(no_match) => { + ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") } @@ -235,14 +242,21 @@ impl IncompleteMatch { pub struct NoMatch { /// The terminals that were parsed; trivia pub nodes: Vec, + /// The nonterminal kind this match is coming from + pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + pub fn new( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { Self { nodes, + kind, expected_terminals, } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 848eb12e81..843c908e16 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -72,7 +72,11 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - (vec![], result.expected_terminals, ParseResultKind::NoMatch) + ( + vec![], /* trivia will be read below */ + result.expected_terminals, + ParseResultKind::NoMatch, + ) } // No need to recover, so just return as-is. _ => return self, @@ -108,7 +112,7 @@ impl ParserResult { ParserResult::incomplete_match(nodes, expected_terminals) } ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, expected_terminals) + ParserResult::no_match(leading_trivia, None, expected_terminals) } } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index 95026d38ae..abfd3cf3c0 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -30,6 +30,7 @@ impl RepetitionHelper { // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { nodes, + kind: _, expected_terminals, }) if MIN_COUNT == 0 => { return ParserResult::r#match(nodes, expected_terminals); diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index 527944afba..e2d68a177e 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; From cf6b9646aadb2ccee1e28a2bf7c9608b95c399f6 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Wed, 27 Nov 2024 13:36:47 -0300 Subject: [PATCH 04/11] Missing files --- .../crate/src/generated/parser/lexer/mod.rs | 4 +-- .../parser/parser_support/choice_helper.rs | 6 +++- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 31 +++++++++++-------- .../parser/parser_support/parser_result.rs | 26 ++++++++++++---- .../parser/parser_support/recovery.rs | 8 +++-- .../parser_support/repetition_helper.rs | 1 + .../parser/parser_support/separated_helper.rs | 2 +- 8 files changed, 54 insertions(+), 26 deletions(-) diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 4212f7b4f4..480c652881 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], vec![kind]); + return ParserResult::no_match(vec![], None, vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, vec![kind]); + return ParserResult::no_match(children, None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 84e9295266..f7c44e0137 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -54,6 +54,10 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { + if running.nodes.is_empty() && !next.nodes.is_empty() { + // transfer nodes + running.nodes.extend(next.nodes.clone()); + } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -129,7 +133,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &[][..], + ParserResult::NoMatch(no_match) => &no_match.nodes, ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs index 989e259cda..7bcdead08e 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs @@ -10,7 +10,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(vec![], no_match.expected_terminals) + ParserResult::r#match(no_match.nodes, no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index f271e6dfe2..d1bd24d10e 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -58,25 +58,30 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(no_match) => { + ParserResult::NoMatch(mut no_match) => { let (kind, start) = if input.is_empty() { (TerminalKind::MISSING, TextIndex::ZERO) } else { - let mut stream = ParserContext::new(input); - if let ParserResult::Match(trivia) = Lexer::leading_trivia(parser, &mut stream) - { - let mut ti = TextIndex::ZERO; - trivia.nodes.iter().for_each(|node| { - node.unparse().chars().for_each(|c| ti.advance(c, None)); - }); - (TerminalKind::UNRECOGNIZED, ti) - } else { - (TerminalKind::UNRECOGNIZED, TextIndex::ZERO) - } + let mut ti = TextIndex::ZERO; + no_match.nodes.iter().for_each(|edge| { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + ti.advance_str(terminal.text.as_str()); + } + } + }); + (TerminalKind::UNRECOGNIZED, ti) }; + let node = Node::terminal(kind, input[start.utf8..].to_string()); + let tree = if no_match.kind.is_none() || start.utf8 == 0 { + node + } else { + no_match.nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + }; ParseOutput { - parse_tree: Node::terminal(kind, input.to_string()), + parse_tree: tree, errors: vec![ParseError::new( start..input.into(), no_match.expected_terminals, diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 260dc6fc61..016f002ca4 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -19,6 +19,7 @@ impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { nodes: vec![], + kind: None, expected_terminals: vec![], }) } @@ -37,13 +38,17 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], vec![]) + Self::no_match(vec![], None, vec![]) } - pub fn no_match(nodes: Vec, expected_terminals: Vec) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, expected_terminals)) + pub fn no_match( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { + ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) } #[must_use] @@ -64,7 +69,9 @@ impl ParserResult { nodes: vec![Edge::anonymous(Node::nonterminal(new_kind, skipped.nodes))], ..skipped }), - ParserResult::NoMatch(_) => self, + ParserResult::NoMatch(no_match) => { + ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") } @@ -235,14 +242,21 @@ impl IncompleteMatch { pub struct NoMatch { /// The terminals that were parsed; trivia pub nodes: Vec, + /// The nonterminal kind this match is coming from + pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. pub expected_terminals: Vec, } impl NoMatch { - pub fn new(nodes: Vec, expected_terminals: Vec) -> Self { + pub fn new( + nodes: Vec, + kind: Option, + expected_terminals: Vec, + ) -> Self { Self { nodes, + kind, expected_terminals, } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 848eb12e81..843c908e16 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -72,7 +72,11 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - (vec![], result.expected_terminals, ParseResultKind::NoMatch) + ( + vec![], /* trivia will be read below */ + result.expected_terminals, + ParseResultKind::NoMatch, + ) } // No need to recover, so just return as-is. _ => return self, @@ -108,7 +112,7 @@ impl ParserResult { ParserResult::incomplete_match(nodes, expected_terminals) } ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, expected_terminals) + ParserResult::no_match(leading_trivia, None, expected_terminals) } } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index 95026d38ae..abfd3cf3c0 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -30,6 +30,7 @@ impl RepetitionHelper { // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { nodes, + kind: _, expected_terminals, }) if MIN_COUNT == 0 => { return ParserResult::r#match(nodes, expected_terminals); diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index 527944afba..e2d68a177e 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, no_match.expected_terminals) + ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; From 3fbefa4d223b5e96ca18d3cd5b3317564b2ae42e Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 28 Nov 2024 09:25:04 -0300 Subject: [PATCH 05/11] Revert to not carry the trivia in the NoMatch result --- .../crate/src/runtime/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 6 +-- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 40 +++++++++++-------- .../parser/parser_support/parser_result.rs | 22 +++------- .../runtime/parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 3 +- .../parser/parser_support/separated_helper.rs | 2 +- crates/metaslang/cst/generated/public_api.txt | 1 + crates/metaslang/cst/src/text_index.rs | 16 +++----- .../crate/src/generated/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 6 +-- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 40 +++++++++++-------- .../parser/parser_support/parser_result.rs | 22 +++------- .../parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 3 +- .../parser/parser_support/separated_helper.rs | 2 +- .../generated/0.4.11-failure.yml | 9 ++++- .../generated/0.4.25-failure.yml | 5 ++- .../generated/0.4.25-failure.yml | 5 ++- .../try_catch/generated/0.4.11-failure.yml | 8 +++- .../with_var/generated/0.5.0-failure.yml | 5 ++- .../generated/0.6.0-failure.yml | 5 ++- .../crate/src/generated/parser/lexer/mod.rs | 4 +- .../parser/parser_support/choice_helper.rs | 6 +-- .../parser/parser_support/optional_helper.rs | 2 +- .../parser/parser_support/parser_function.rs | 40 +++++++++++-------- .../parser/parser_support/parser_result.rs | 22 +++------- .../parser/parser_support/recovery.rs | 4 +- .../parser_support/repetition_helper.rs | 3 +- .../parser/parser_support/separated_helper.rs | 2 +- 32 files changed, 146 insertions(+), 157 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs index 5925617b28..156ce5583d 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/lexer/mod.rs @@ -94,7 +94,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); @@ -129,7 +129,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs index 353e090cf8..b96c0c8557 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs @@ -52,10 +52,6 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { - if running.nodes.is_empty() && !next.nodes.is_empty() { - // transfer nodes - running.nodes.extend(next.nodes.clone()); - } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -131,7 +127,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(no_match) => &no_match.nodes, + ParserResult::NoMatch(_) => &vec![], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs index a052cc4f66..9ef5430056 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/optional_helper.rs @@ -8,7 +8,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(no_match.nodes, no_match.expected_terminals) + ParserResult::r#match(vec![], no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs index 189b9c7734..531f5a0c6e 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs @@ -56,32 +56,40 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(mut no_match) => { - let (kind, start) = if input.is_empty() { - (TerminalKind::MISSING, TextIndex::ZERO) + ParserResult::NoMatch(no_match) => { + let mut trivia_nodes = if let ParserResult::Match(matched) = + Lexer::leading_trivia(parser, &mut stream) + { + matched.nodes } else { - let mut ti = TextIndex::ZERO; - no_match.nodes.iter().for_each(|edge| { - if let Node::Terminal(terminal) = &edge.node { - if terminal.kind.is_valid() { - ti.advance_str(terminal.text.as_str()); - } - } - }); - (TerminalKind::UNRECOGNIZED, ti) + vec![] }; - let node = Node::terminal(kind, input[start.utf8..].to_string()); + let mut start = TextIndex::ZERO; + for edge in &trivia_nodes { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + start.advance_str(terminal.text.as_str()); + } + } + } + let input = &input[start.utf8..]; + let kind = if input.is_empty() { + TerminalKind::MISSING + } else { + TerminalKind::UNRECOGNIZED + }; + let node = Node::terminal(kind, input.to_string()); let tree = if no_match.kind.is_none() || start.utf8 == 0 { node } else { - no_match.nodes.push(Edge::anonymous(node)); - Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + trivia_nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), trivia_nodes) }; ParseOutput { parse_tree: tree, errors: vec![ParseError::new( - start..input.into(), + start..start + input.into(), no_match.expected_terminals, )], } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs index 57db35864e..eaf550849e 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs @@ -16,7 +16,6 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { - nodes: vec![], kind: None, expected_terminals: vec![], }) @@ -38,15 +37,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], None, vec![]) + Self::no_match(None, vec![]) } - pub fn no_match( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) + pub fn no_match(kind: Option, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(kind, expected_terminals)) } #[must_use] @@ -68,7 +63,7 @@ impl ParserResult { ..skipped }), ParserResult::NoMatch(no_match) => { - ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + ParserResult::no_match(Some(new_kind), no_match.expected_terminals) } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") @@ -238,8 +233,6 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { - /// The terminals that were parsed; trivia - pub nodes: Vec, /// The nonterminal kind this match is coming from pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. @@ -247,13 +240,8 @@ pub struct NoMatch { } impl NoMatch { - pub fn new( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { + pub fn new(kind: Option, expected_terminals: Vec) -> Self { Self { - nodes, kind, expected_terminals, } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs index 1a610349d0..ce57ca3b68 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs @@ -109,9 +109,7 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, None, expected_terminals) - } + ParseResultKind::NoMatch => ParserResult::no_match(None, expected_terminals), } } } diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs index 7a13af5bd1..3d612e4fe3 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/repetition_helper.rs @@ -27,11 +27,10 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { - nodes, kind: _, expected_terminals, }) if MIN_COUNT == 0 => { - return ParserResult::r#match(nodes, expected_terminals); + return ParserResult::r#match(vec![], expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs index 8efcac8f74..280d89c8cf 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/separated_helper.rs @@ -89,7 +89,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) + ParserResult::no_match(None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; diff --git a/crates/metaslang/cst/generated/public_api.txt b/crates/metaslang/cst/generated/public_api.txt index 3f00927431..9a2eb5a033 100644 --- a/crates/metaslang/cst/generated/public_api.txt +++ b/crates/metaslang/cst/generated/public_api.txt @@ -247,6 +247,7 @@ pub metaslang_cst::text_index::TextIndex::utf8: usize impl metaslang_cst::text_index::TextIndex pub const metaslang_cst::text_index::TextIndex::ZERO: metaslang_cst::text_index::TextIndex pub fn metaslang_cst::text_index::TextIndex::advance(&mut self, c: char, next: core::option::Option<&char>) +pub fn metaslang_cst::text_index::TextIndex::advance_str(&mut self, text: &str) impl core::clone::Clone for metaslang_cst::text_index::TextIndex pub fn metaslang_cst::text_index::TextIndex::clone(&self) -> metaslang_cst::text_index::TextIndex impl core::cmp::Eq for metaslang_cst::text_index::TextIndex diff --git a/crates/metaslang/cst/src/text_index.rs b/crates/metaslang/cst/src/text_index.rs index c0144c4a3f..18ed1b4dd8 100644 --- a/crates/metaslang/cst/src/text_index.rs +++ b/crates/metaslang/cst/src/text_index.rs @@ -42,11 +42,11 @@ impl TextIndex { } pub fn advance_str(&mut self, text: &str) { - text.chars().for_each(|c| { - if c != '\r' { - self.advance(c, None); - } - }); + let mut iter = text.chars().peekable(); + while let Some(c) = iter.next() { + let n = iter.peek(); + self.advance(c, n); + } } } @@ -71,11 +71,7 @@ impl Display for TextIndex { impl> From for TextIndex { fn from(s: T) -> Self { let mut result = Self::ZERO; - let mut iter = s.as_ref().chars().peekable(); - while let Some(c) = iter.next() { - let n = iter.peek(); - result.advance(c, n); - } + result.advance_str(s.as_ref()); result } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 480c652881..c648756daf 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index f7c44e0137..9042b7756c 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -54,10 +54,6 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { - if running.nodes.is_empty() && !next.nodes.is_empty() { - // transfer nodes - running.nodes.extend(next.nodes.clone()); - } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -133,7 +129,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(no_match) => &no_match.nodes, + ParserResult::NoMatch(_) => &vec![], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs index 7bcdead08e..989e259cda 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs @@ -10,7 +10,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(no_match.nodes, no_match.expected_terminals) + ParserResult::r#match(vec![], no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index d1bd24d10e..7a3afb9156 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -58,32 +58,40 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(mut no_match) => { - let (kind, start) = if input.is_empty() { - (TerminalKind::MISSING, TextIndex::ZERO) + ParserResult::NoMatch(no_match) => { + let mut trivia_nodes = if let ParserResult::Match(matched) = + Lexer::leading_trivia(parser, &mut stream) + { + matched.nodes } else { - let mut ti = TextIndex::ZERO; - no_match.nodes.iter().for_each(|edge| { - if let Node::Terminal(terminal) = &edge.node { - if terminal.kind.is_valid() { - ti.advance_str(terminal.text.as_str()); - } - } - }); - (TerminalKind::UNRECOGNIZED, ti) + vec![] }; - let node = Node::terminal(kind, input[start.utf8..].to_string()); + let mut start = TextIndex::ZERO; + for edge in &trivia_nodes { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + start.advance_str(terminal.text.as_str()); + } + } + } + let input = &input[start.utf8..]; + let kind = if input.is_empty() { + TerminalKind::MISSING + } else { + TerminalKind::UNRECOGNIZED + }; + let node = Node::terminal(kind, input.to_string()); let tree = if no_match.kind.is_none() || start.utf8 == 0 { node } else { - no_match.nodes.push(Edge::anonymous(node)); - Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + trivia_nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), trivia_nodes) }; ParseOutput { parse_tree: tree, errors: vec![ParseError::new( - start..input.into(), + start..start + input.into(), no_match.expected_terminals, )], } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 016f002ca4..64f7f41fe1 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -18,7 +18,6 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { - nodes: vec![], kind: None, expected_terminals: vec![], }) @@ -40,15 +39,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], None, vec![]) + Self::no_match(None, vec![]) } - pub fn no_match( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) + pub fn no_match(kind: Option, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(kind, expected_terminals)) } #[must_use] @@ -70,7 +65,7 @@ impl ParserResult { ..skipped }), ParserResult::NoMatch(no_match) => { - ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + ParserResult::no_match(Some(new_kind), no_match.expected_terminals) } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") @@ -240,8 +235,6 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { - /// The terminals that were parsed; trivia - pub nodes: Vec, /// The nonterminal kind this match is coming from pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. @@ -249,13 +242,8 @@ pub struct NoMatch { } impl NoMatch { - pub fn new( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { + pub fn new(kind: Option, expected_terminals: Vec) -> Self { Self { - nodes, kind, expected_terminals, } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 843c908e16..1833209d75 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -111,9 +111,7 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, None, expected_terminals) - } + ParseResultKind::NoMatch => ParserResult::no_match(None, expected_terminals), } } } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index abfd3cf3c0..a4fb3a2be5 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -29,11 +29,10 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { - nodes, kind: _, expected_terminals, }) if MIN_COUNT == 0 => { - return ParserResult::r#match(nodes, expected_terminals); + return ParserResult::r#match(vec![], expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index e2d68a177e..eca96115d0 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) + ParserResult::no_match(None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; diff --git a/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml b/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml index b92ae184b3..02e2ecb360 100644 --- a/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/ContractDefinition/unicode_in_doc_comments/generated/0.4.11-failure.yml @@ -17,4 +17,11 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): "// ╒══════════════════════════════════════════════..." # (0..489) + - (ContractDefinition): # "// ╒══════════════════════════════════════════════..." (0..489) + - (leading_trivia꞉ SingleLineComment): "// ╒══════════════════════════════════════════════..." # (0..201) + - (leading_trivia꞉ EndOfLine): "\n" # (201..202) + - (leading_trivia꞉ SingleLineComment): "// │ More Info: https://github.com/NomicFoundation..." # (202..275) + - (leading_trivia꞉ EndOfLine): "\n" # (275..276) + - (leading_trivia꞉ SingleLineComment): "// ╘══════════════════════════════════════════════..." # (276..477) + - (leading_trivia꞉ EndOfLine): "\n" # (477..478) + - (UNRECOGNIZED): "unexpected\n" # (478..489) diff --git a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml index fa426b798c..6d35cb954c 100644 --- a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_ascii/generated/0.4.25-failure.yml @@ -15,4 +15,7 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): '// Disallowed post 0.4.25\n"\a"\n' # (0..31) + - (StringLiteral): # '// Disallowed post 0.4.25\n"\a"\n' (0..31) + - (leading_trivia꞉ SingleLineComment): "// Disallowed post 0.4.25" # (0..25) + - (leading_trivia꞉ EndOfLine): "\n" # (25..26) + - (UNRECOGNIZED): '"\a"\n' # (26..31) diff --git a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml index 48bab2bab6..e98c1d5e98 100644 --- a/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/StringLiteral/escape_arbitrary_unicode/generated/0.4.25-failure.yml @@ -15,4 +15,7 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): '// Disallowed post 0.4.25\n"\✅"\n' # (0..33) + - (StringLiteral): # '// Disallowed post 0.4.25\n"\✅"\n' (0..33) + - (leading_trivia꞉ SingleLineComment): "// Disallowed post 0.4.25" # (0..25) + - (leading_trivia꞉ EndOfLine): "\n" # (25..26) + - (UNRECOGNIZED): '"\✅"\n' # (26..33) diff --git a/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml b/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml index 893a1bbd44..4f4bbd6c52 100644 --- a/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/TryStatement/try_catch/generated/0.4.11-failure.yml @@ -22,4 +22,10 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): "// Make sure that error recovery won't lead to mis..." # (0..171) + - (TryStatement): # "// Make sure that error recovery won't lead to mis..." (0..171) + - (leading_trivia꞉ SingleLineComment): "// Make sure that error recovery won't lead to mis..." # (0..57) + - (leading_trivia꞉ EndOfLine): "\n" # (57..58) + - (leading_trivia꞉ SingleLineComment): "// ambiguous function call options with the block ..." # (58..136) + - (leading_trivia꞉ EndOfLine): "\n" # (136..137) + - (leading_trivia꞉ EndOfLine): "\n" # (137..138) + - (UNRECOGNIZED): "try foo() {\n bar();\n} catch {\n}\n" # (138..171) diff --git a/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml index 124aed234c..12cc49290f 100644 --- a/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/TupleDeconstructionStatement/with_var/generated/0.5.0-failure.yml @@ -15,4 +15,7 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): '// "var" should be disabled in "0.5.0":\nvar (foo, ...' # (0..62) + - (TupleDeconstructionStatement): # '// "var" should be disabled in "0.5.0":\nvar (foo, ...' (0..62) + - (leading_trivia꞉ SingleLineComment): '// "var" should be disabled in "0.5.0":' # (0..39) + - (leading_trivia꞉ EndOfLine): "\n" # (39..40) + - (UNRECOGNIZED): "var (foo, bar) = rhs;\n" # (40..62) diff --git a/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml b/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml index df7c141a1b..7f629684df 100644 --- a/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml +++ b/crates/solidity/testing/snapshots/cst_output/UnnamedFunctionDefinition/constant_attribute/generated/0.6.0-failure.yml @@ -15,4 +15,7 @@ Errors: # 1 total ───╯ Tree: - - (UNRECOGNIZED): "// Split into `view` and `pure` in 0.4.16\nfunction..." # (0..66) + - (UnnamedFunctionDefinition): # "// Split into `view` and `pure` in 0.4.16\nfunction..." (0..66) + - (leading_trivia꞉ SingleLineComment): "// Split into `view` and `pure` in 0.4.16" # (0..41) + - (leading_trivia꞉ EndOfLine): "\n" # (41..42) + - (UNRECOGNIZED): "function () constant {}\n" # (42..66) diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs index 480c652881..c648756daf 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/lexer/mod.rs @@ -96,7 +96,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(start); - return ParserResult::no_match(vec![], None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); @@ -131,7 +131,7 @@ pub(crate) trait Lexer { .is_some_and(|t| t.accepted_as(kind)) { input.set_position(restore); - return ParserResult::no_match(children, None, vec![kind]); + return ParserResult::no_match(None, vec![kind]); } let end = input.position(); children.push(Edge::anonymous(Node::terminal( diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index f7c44e0137..9042b7756c 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -54,10 +54,6 @@ impl ChoiceHelper { // Still no match, extend the possible expected terminals. (ParserResult::NoMatch(running), ParserResult::NoMatch(next)) => { - if running.nodes.is_empty() && !next.nodes.is_empty() { - // transfer nodes - running.nodes.extend(next.nodes.clone()); - } running .expected_terminals .extend(next.expected_terminals.clone()); @@ -133,7 +129,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(no_match) => &no_match.nodes, + ParserResult::NoMatch(_) => &vec![], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs index 7bcdead08e..989e259cda 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/optional_helper.rs @@ -10,7 +10,7 @@ impl OptionalHelper { // If there's absolutely no match, we treat it as a match (for the purposes of the Result algebra) // but we bubble up the information which terminals would have allowed more progress. ParserResult::NoMatch(no_match) => { - ParserResult::r#match(no_match.nodes, no_match.expected_terminals) + ParserResult::r#match(vec![], no_match.expected_terminals) } // ... otherwise we return the result as-is ParserResult::Match(_) diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index d1bd24d10e..7a3afb9156 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -58,32 +58,40 @@ where match result { ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), - ParserResult::NoMatch(mut no_match) => { - let (kind, start) = if input.is_empty() { - (TerminalKind::MISSING, TextIndex::ZERO) + ParserResult::NoMatch(no_match) => { + let mut trivia_nodes = if let ParserResult::Match(matched) = + Lexer::leading_trivia(parser, &mut stream) + { + matched.nodes } else { - let mut ti = TextIndex::ZERO; - no_match.nodes.iter().for_each(|edge| { - if let Node::Terminal(terminal) = &edge.node { - if terminal.kind.is_valid() { - ti.advance_str(terminal.text.as_str()); - } - } - }); - (TerminalKind::UNRECOGNIZED, ti) + vec![] }; - let node = Node::terminal(kind, input[start.utf8..].to_string()); + let mut start = TextIndex::ZERO; + for edge in &trivia_nodes { + if let Node::Terminal(terminal) = &edge.node { + if terminal.kind.is_valid() { + start.advance_str(terminal.text.as_str()); + } + } + } + let input = &input[start.utf8..]; + let kind = if input.is_empty() { + TerminalKind::MISSING + } else { + TerminalKind::UNRECOGNIZED + }; + let node = Node::terminal(kind, input.to_string()); let tree = if no_match.kind.is_none() || start.utf8 == 0 { node } else { - no_match.nodes.push(Edge::anonymous(node)); - Node::nonterminal(no_match.kind.unwrap(), no_match.nodes) + trivia_nodes.push(Edge::anonymous(node)); + Node::nonterminal(no_match.kind.unwrap(), trivia_nodes) }; ParseOutput { parse_tree: tree, errors: vec![ParseError::new( - start..input.into(), + start..start + input.into(), no_match.expected_terminals, )], } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 016f002ca4..64f7f41fe1 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -18,7 +18,6 @@ pub enum ParserResult { impl Default for ParserResult { fn default() -> Self { Self::NoMatch(NoMatch { - nodes: vec![], kind: None, expected_terminals: vec![], }) @@ -40,15 +39,11 @@ impl ParserResult { /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. pub fn disabled() -> Self { - Self::no_match(vec![], None, vec![]) + Self::no_match(None, vec![]) } - pub fn no_match( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { - ParserResult::NoMatch(NoMatch::new(nodes, kind, expected_terminals)) + pub fn no_match(kind: Option, expected_terminals: Vec) -> Self { + ParserResult::NoMatch(NoMatch::new(kind, expected_terminals)) } #[must_use] @@ -70,7 +65,7 @@ impl ParserResult { ..skipped }), ParserResult::NoMatch(no_match) => { - ParserResult::no_match(no_match.nodes, Some(new_kind), no_match.expected_terminals) + ParserResult::no_match(Some(new_kind), no_match.expected_terminals) } ParserResult::PrattOperatorMatch(_) => { unreachable!("PrattOperatorMatch cannot be converted to a nonterminal") @@ -240,8 +235,6 @@ impl IncompleteMatch { #[derive(PartialEq, Eq, Clone, Debug)] pub struct NoMatch { - /// The terminals that were parsed; trivia - pub nodes: Vec, /// The nonterminal kind this match is coming from pub kind: Option, /// Terminals that would have allowed for more progress. Collected for the purposes of error reporting. @@ -249,13 +242,8 @@ pub struct NoMatch { } impl NoMatch { - pub fn new( - nodes: Vec, - kind: Option, - expected_terminals: Vec, - ) -> Self { + pub fn new(kind: Option, expected_terminals: Vec) -> Self { Self { - nodes, kind, expected_terminals, } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 843c908e16..1833209d75 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -111,9 +111,7 @@ impl ParserResult { ParseResultKind::Incomplete => { ParserResult::incomplete_match(nodes, expected_terminals) } - ParseResultKind::NoMatch => { - ParserResult::no_match(leading_trivia, None, expected_terminals) - } + ParseResultKind::NoMatch => ParserResult::no_match(None, expected_terminals), } } } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs index abfd3cf3c0..a4fb3a2be5 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/repetition_helper.rs @@ -29,11 +29,10 @@ impl RepetitionHelper { // Couldn't get a full match but we allow 0 items - return an empty match // so the parse is considered valid but note the expected terminals ParserResult::NoMatch(NoMatch { - nodes, kind: _, expected_terminals, }) if MIN_COUNT == 0 => { - return ParserResult::r#match(nodes, expected_terminals); + return ParserResult::r#match(vec![], expected_terminals); } // Don't try repeating if we don't have a full match and we require at least one incomplete_or_no_match => return incomplete_or_no_match, diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs index e2d68a177e..eca96115d0 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/separated_helper.rs @@ -91,7 +91,7 @@ impl SeparatedHelper { } ParserResult::NoMatch(no_match) => { return if accum.is_empty() { - ParserResult::no_match(no_match.nodes, None, no_match.expected_terminals) + ParserResult::no_match(None, no_match.expected_terminals) } else { ParserResult::incomplete_match(accum, no_match.expected_terminals) }; From 24aff8c1841c4a6f157b117a926f8f5159a7c944 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 28 Nov 2024 09:30:02 -0300 Subject: [PATCH 06/11] Revert little change with no semantic difference --- .../crate/src/runtime/parser/parser_support/choice_helper.rs | 2 +- .../crate/src/generated/parser/parser_support/choice_helper.rs | 2 +- .../crate/src/generated/parser/parser_support/choice_helper.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs index b96c0c8557..3253a7988d 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/choice_helper.rs @@ -127,7 +127,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &vec![], + ParserResult::NoMatch(_) => &[][..], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 9042b7756c..84e9295266 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -129,7 +129,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &vec![], + ParserResult::NoMatch(_) => &[][..], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs index 9042b7756c..84e9295266 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/choice_helper.rs @@ -129,7 +129,7 @@ pub fn total_not_skipped_span(result: &ParserResult) -> usize { ParserResult::Match(match_) => &match_.nodes, ParserResult::IncompleteMatch(incomplete_match) => &incomplete_match.nodes, ParserResult::SkippedUntil(skipped) => &skipped.nodes, - ParserResult::NoMatch(_) => &vec![], + ParserResult::NoMatch(_) => &[][..], ParserResult::PrattOperatorMatch(_) => unreachable!( "PrattOperatorMatch is always considered a better pick, so it should never be considered here" ), From ecd50620e0ee7b0fbba7f285563979bafef14e59 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 28 Nov 2024 09:31:25 -0300 Subject: [PATCH 07/11] Fixing comment --- .../crate/src/runtime/parser/parser_support/parser_result.rs | 2 +- .../crate/src/generated/parser/parser_support/parser_result.rs | 2 +- .../crate/src/generated/parser/parser_support/parser_result.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs index eaf550849e..1f9b6e8e14 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_result.rs @@ -35,7 +35,7 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(None, vec![])`. pub fn disabled() -> Self { Self::no_match(None, vec![]) } diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 64f7f41fe1..1ec7594426 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -37,7 +37,7 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(None, vec![])`. pub fn disabled() -> Self { Self::no_match(None, vec![]) } diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs index 64f7f41fe1..1ec7594426 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_result.rs @@ -37,7 +37,7 @@ impl ParserResult { ParserResult::IncompleteMatch(IncompleteMatch::new(nodes, expected_terminals)) } - /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(vec![], None, vec![])`. + /// Whenever a parser didn't run because it's disabled due to versioning. Shorthand for `no_match(None, vec![])`. pub fn disabled() -> Self { Self::no_match(None, vec![]) } From 41136096be4744aa8ded421819d236cb320a1df2 Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Thu, 28 Nov 2024 09:41:15 -0300 Subject: [PATCH 08/11] revert of unnecessary change --- .../crate/src/runtime/parser/parser_support/recovery.rs | 6 +----- .../crate/src/generated/parser/parser_support/recovery.rs | 6 +----- .../crate/src/generated/parser/parser_support/recovery.rs | 6 +----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs index ce57ca3b68..9885e723b3 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/recovery.rs @@ -70,11 +70,7 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - ( - vec![], /* trivia will be read below */ - result.expected_terminals, - ParseResultKind::NoMatch, - ) + (vec![], result.expected_terminals, ParseResultKind::NoMatch) } // No need to recover, so just return as-is. _ => return self, diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 1833209d75..c49819e30f 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -72,11 +72,7 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - ( - vec![], /* trivia will be read below */ - result.expected_terminals, - ParseResultKind::NoMatch, - ) + (vec![], result.expected_terminals, ParseResultKind::NoMatch) } // No need to recover, so just return as-is. _ => return self, diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs index 1833209d75..c49819e30f 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/recovery.rs @@ -72,11 +72,7 @@ impl ParserResult { ) } ParserResult::NoMatch(result) if acceptance_threshold.0 == 0 => { - ( - vec![], /* trivia will be read below */ - result.expected_terminals, - ParseResultKind::NoMatch, - ) + (vec![], result.expected_terminals, ParseResultKind::NoMatch) } // No need to recover, so just return as-is. _ => return self, From 7495413cb894ee8cfd68dd858466823b9150787a Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Tue, 10 Dec 2024 10:55:39 -0300 Subject: [PATCH 09/11] Adding comment --- .../src/runtime/parser/parser_support/parser_function.rs | 5 +++++ .../src/generated/parser/parser_support/parser_function.rs | 5 +++++ .../src/generated/parser/parser_support/parser_function.rs | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs index 531f5a0c6e..b921590b2f 100644 --- a/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs +++ b/crates/codegen/runtime/cargo/crate/src/runtime/parser/parser_support/parser_function.rs @@ -57,6 +57,11 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { + // Parse leading trivia (see #1172 for details). One could argue that trivia should be parsed + // just once, and returned in the `NoMatch` structure. However, in rules like (This | That), + // trivia is already parsed twice, one for each branch. And there's a good reason: each branch might + // accept different trivia, so it's not clear what the combination of the two rules should return in a + // NoMatch. Therefore, we just parse it again. Note that trivia is anyway cached by the parser (#1119). let mut trivia_nodes = if let ParserResult::Match(matched) = Lexer::leading_trivia(parser, &mut stream) { diff --git a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index 7a3afb9156..ec3766b706 100644 --- a/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/solidity/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -59,6 +59,11 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { + // Parse leading trivia (see #1172 for details). One could argue that trivia should be parsed + // just once, and returned in the `NoMatch` structure. However, in rules like (This | That), + // trivia is already parsed twice, one for each branch. And there's a good reason: each branch might + // accept different trivia, so it's not clear what the combination of the two rules should return in a + // NoMatch. Therefore, we just parse it again. Note that trivia is anyway cached by the parser (#1119). let mut trivia_nodes = if let ParserResult::Match(matched) = Lexer::leading_trivia(parser, &mut stream) { diff --git a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs index 7a3afb9156..ec3766b706 100644 --- a/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs +++ b/crates/testlang/outputs/cargo/crate/src/generated/parser/parser_support/parser_function.rs @@ -59,6 +59,11 @@ where ParserResult::PrattOperatorMatch(..) => unreachable!("PrattOperatorMatch is internal"), ParserResult::NoMatch(no_match) => { + // Parse leading trivia (see #1172 for details). One could argue that trivia should be parsed + // just once, and returned in the `NoMatch` structure. However, in rules like (This | That), + // trivia is already parsed twice, one for each branch. And there's a good reason: each branch might + // accept different trivia, so it's not clear what the combination of the two rules should return in a + // NoMatch. Therefore, we just parse it again. Note that trivia is anyway cached by the parser (#1119). let mut trivia_nodes = if let ParserResult::Match(matched) = Lexer::leading_trivia(parser, &mut stream) { From 4dc2d0bf4d94135c908debb040d8ac1988eadc4a Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Tue, 10 Dec 2024 10:58:04 -0300 Subject: [PATCH 10/11] Adding changeset --- .changeset/smart-cooks-suffer.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/smart-cooks-suffer.md diff --git a/.changeset/smart-cooks-suffer.md b/.changeset/smart-cooks-suffer.md new file mode 100644 index 0000000000..c1f5530082 --- /dev/null +++ b/.changeset/smart-cooks-suffer.md @@ -0,0 +1,5 @@ +--- +"@nomicfoundation/slang": minor +--- + +Improved error recovery, where leading trivia are always parsed and included before an errornous terminal. From 72d485800d7b00495ac7266b40dc4d14a91105ad Mon Sep 17 00:00:00 2001 From: Beta Ziliani Date: Wed, 11 Dec 2024 10:29:08 -0300 Subject: [PATCH 11/11] fixup --- .changeset/smart-cooks-suffer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/smart-cooks-suffer.md b/.changeset/smart-cooks-suffer.md index c1f5530082..2c46a5040c 100644 --- a/.changeset/smart-cooks-suffer.md +++ b/.changeset/smart-cooks-suffer.md @@ -2,4 +2,4 @@ "@nomicfoundation/slang": minor --- -Improved error recovery, where leading trivia are always parsed and included before an errornous terminal. +Improved error recovery, where leading trivia are always parsed and included before an erroneous terminal.