diff --git a/.changeset/famous-ants-heal.md b/.changeset/famous-ants-heal.md new file mode 100644 index 0000000000..19408d9633 --- /dev/null +++ b/.changeset/famous-ants-heal.md @@ -0,0 +1,5 @@ +--- +"@nomicfoundation/slang": patch +--- + +parse line breaks without newlines diff --git a/crates/solidity/inputs/language/src/definition.rs b/crates/solidity/inputs/language/src/definition.rs index e0320df567..6f4a24bcb9 100644 --- a/crates/solidity/inputs/language/src/definition.rs +++ b/crates/solidity/inputs/language/src/definition.rs @@ -392,7 +392,8 @@ codegen_language_macros::compile!(Language( ), Trivia( name = EndOfLine, - scanner = Sequence([Optional(Atom("\r")), Atom("\n")]) + scanner = + Choice([Atom("\n"), Sequence([Atom("\r"), Optional(Atom("\n"))])]) ), Trivia( name = SingleLineComment, diff --git a/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs b/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs index a115a3726d..929757bf94 100644 --- a/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs +++ b/crates/solidity/outputs/cargo/slang_solidity/src/generated/language.rs @@ -6985,9 +6985,13 @@ impl Language { #[allow(unused_assignments, unused_parens)] fn end_of_line(&self, input: &mut ParserContext<'_>) -> bool { - scan_sequence!( - scan_optional!(input, scan_chars!(input, '\r')), - scan_chars!(input, '\n') + scan_choice!( + input, + scan_chars!(input, '\n'), + scan_sequence!( + scan_chars!(input, '\r'), + scan_optional!(input, scan_chars!(input, '\n')) + ) ) } diff --git a/crates/solidity/outputs/cargo/tests/src/lib.rs b/crates/solidity/outputs/cargo/tests/src/lib.rs index 6c62e406c6..1e7cc66faa 100644 --- a/crates/solidity/outputs/cargo/tests/src/lib.rs +++ b/crates/solidity/outputs/cargo/tests/src/lib.rs @@ -2,3 +2,4 @@ mod cst_output; mod doc_examples; +mod trivia; diff --git a/crates/solidity/outputs/cargo/tests/src/trivia.rs b/crates/solidity/outputs/cargo/tests/src/trivia.rs new file mode 100644 index 0000000000..c2e86f4d86 --- /dev/null +++ b/crates/solidity/outputs/cargo/tests/src/trivia.rs @@ -0,0 +1,47 @@ +use anyhow::Result; +use semver::Version; +use slang_solidity::cst::Node; +use slang_solidity::kinds::{RuleKind, TokenKind}; +use slang_solidity::language::Language; + +#[test] +fn end_of_line() -> Result<()> { + // Only one is valid as a single token: + compare_end_of_lines("\r", &["\r"])?; + compare_end_of_lines("\n", &["\n"])?; + + // Two of the same are two tokens: + compare_end_of_lines("\n\n", &["\n", "\n"])?; + compare_end_of_lines("\r\r", &["\r", "\r"])?; + + // A carriage return followed by a newline is one token, but the opposite is not: + compare_end_of_lines("\r\n", &["\r\n"])?; + compare_end_of_lines("\n\r", &["\n", "\r"])?; + + Ok(()) +} + +fn compare_end_of_lines(input: &str, expected: &[&str]) -> Result<()> { + let version = Version::parse("0.8.0")?; + let language = &Language::new(version)?; + + let output = language.parse(RuleKind::SourceUnit, input); + assert!(output.is_valid()); + + let actual = output + .create_tree_cursor() + .filter_map(|node| match node { + Node::Rule(_) => None, + + Node::Token(token) => { + assert_eq!(token.kind, TokenKind::EndOfLine); + Some(token.text.to_owned()) + } + }) + .collect::>(); + + let expected = expected.to_vec(); + assert_eq!(actual, expected); + + Ok(()) +} diff --git a/crates/solidity/outputs/spec/generated/grammar.ebnf b/crates/solidity/outputs/spec/generated/grammar.ebnf index 0ae6a187d5..fd025ce4d0 100644 --- a/crates/solidity/outputs/spec/generated/grammar.ebnf +++ b/crates/solidity/outputs/spec/generated/grammar.ebnf @@ -154,7 +154,7 @@ UsingTarget = TypeName WHITESPACE = (" " | "\t")+; -END_OF_LINE = "\r"? "\n"; +END_OF_LINE = "\n" | ("\r" "\n"?); SINGLE_LINE_COMMENT = "//" (!("\r" "\n"))*; diff --git a/crates/solidity/outputs/spec/generated/public/01-file-structure/06-trivia.md b/crates/solidity/outputs/spec/generated/public/01-file-structure/06-trivia.md index 3c1d2df6b8..97560ccfd8 100644 --- a/crates/solidity/outputs/spec/generated/public/01-file-structure/06-trivia.md +++ b/crates/solidity/outputs/spec/generated/public/01-file-structure/06-trivia.md @@ -14,7 +14,7 @@ ``` -
END_OF_LINE = "\r"? "\n";
+
END_OF_LINE = "\n" | ("\r" "\n"?);
```{ .ebnf #SingleLineComment } diff --git a/crates/solidity/testing/utils/src/cst_snapshots/mod.rs b/crates/solidity/testing/utils/src/cst_snapshots/mod.rs index 2feb2ff10d..8cd92301c4 100644 --- a/crates/solidity/testing/utils/src/cst_snapshots/mod.rs +++ b/crates/solidity/testing/utils/src/cst_snapshots/mod.rs @@ -37,6 +37,9 @@ fn write_source(w: &mut String, source: &str) -> Result<()> { return Ok(()); } + // "lines()" only handles "\n", so let's normalize all line endings: + let source = source.replace("\r\n", "\n").replace('\r', "\n"); + let line_data = source .lines() .enumerate()