From 61c439b6cfc39b03ce8f6c9061c035671b8b5536 Mon Sep 17 00:00:00 2001 From: Pasha Stetsenko Date: Sat, 5 Nov 2022 20:30:55 -0700 Subject: [PATCH 1/2] trim whitespace before hashtags --- .../jenny/lib/src/parse/tokenize.dart | 8 +++- .../jenny/test/parse/tokenize_test.dart | 38 ++++++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart index 480155061f3..c2dc80616ca 100644 --- a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart +++ b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart @@ -651,6 +651,7 @@ class _Lexer { /// Emits the text token corresponding to the text processed. bool eatPlainText() { final position0 = position; + var positionBeforeWhitespace = position; while (!eof) { final cu = currentCodeUnit; if (cu == $lessThan || cu == $slash) { @@ -661,12 +662,17 @@ class _Lexer { } } else if (cu == $carriageReturn || cu == $lineFeed || - cu == $hash || cu == $backslash || cu == $leftBrace) { break; + } else if (cu == $hash) { + position = positionBeforeWhitespace; + break; } position += 1; + if (!(cu == $space || cu == $tab)) { + positionBeforeWhitespace = position; + } } if (position > position0) { pushToken(Token.text(text.substring(position0, position)), position0); diff --git a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart index a4cc15ce99a..7f6fbfa51ed 100644 --- a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart +++ b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart @@ -470,6 +470,23 @@ void main() { ], ); }); + + test('line with hash tags', () { + expect( + tokenize('---\n---\n' + 'Some text #with-tag\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('Some text'), + Token.hashtag('#with-tag'), + Token.newline, + Token.endBody, + ], + ); + }); }); group('modeText', () { @@ -823,7 +840,7 @@ void main() { Token.startHeader, Token.endHeader, Token.startBody, - Token.text('line1 '), + Token.text('line1'), Token.hashtag('#tag'), Token.hashtag('#some:other@tag!'), Token.newline, @@ -831,7 +848,6 @@ void main() { Token.startExpression, Token.number('33'), Token.endExpression, - Token.text(' '), Token.hashtag('#here-be-dragons'), Token.newline, Token.endBody, @@ -892,6 +908,24 @@ void main() { ], ); }); + + test('text with escaped content', () { + expect( + tokenize('---\n---\n' + 'One \\{ two\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('One '), + Token.text('{'), + Token.text(' two'), + Token.newline, + Token.endBody, + ], + ); + }); }); // This group is for testing the error mechanism itself, not any particular From 7c2266618e5af0e67f77516af64a962bfba77db7 Mon Sep 17 00:00:00 2001 From: Pasha Stetsenko Date: Sat, 5 Nov 2022 20:43:39 -0700 Subject: [PATCH 2/2] also eat whitespace at the end of line and before comments --- .../jenny/lib/src/parse/tokenize.dart | 24 +++++++++---------- .../jenny/test/parse/tokenize_test.dart | 21 +++++++++++++--- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart index c2dc80616ca..43f29f7e881 100644 --- a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart +++ b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart @@ -77,7 +77,11 @@ class _Lexer { /// Returns the integer code unit at the current parse position, or -1 if we /// reached the end of input. - int get currentCodeUnit => eof ? -1 : text.codeUnitAt(position); + int get currentCodeUnit => + position < text.length ? text.codeUnitAt(position) : -1; + + int get nextCodeUnit => + position < text.length - 1 ? text.codeUnitAt(position + 1) : -1; /// Pushes a new mode into the mode stack and returns `true`. bool pushMode(_ModeFn mode) { @@ -654,20 +658,16 @@ class _Lexer { var positionBeforeWhitespace = position; while (!eof) { final cu = currentCodeUnit; - if (cu == $lessThan || cu == $slash) { - position += 1; - if (currentCodeUnit == cu) { - position -= 1; - break; - } - } else if (cu == $carriageReturn || - cu == $lineFeed || + if ((cu == $slash && nextCodeUnit == $slash) || + cu == $hash || + cu == $carriageReturn || + cu == $lineFeed) { + position = positionBeforeWhitespace; + break; + } else if ((cu == $lessThan && nextCodeUnit == $lessThan) || cu == $backslash || cu == $leftBrace) { break; - } else if (cu == $hash) { - position = positionBeforeWhitespace; - break; } position += 1; if (!(cu == $space || cu == $tab)) { diff --git a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart index 7f6fbfa51ed..2209aa61d46 100644 --- a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart +++ b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart @@ -494,13 +494,13 @@ void main() { expect( tokenize('---\n---\n' 'some text // here be dragons\n' - 'other text\n' + 'other text \t\n' '===\n'), const [ Token.startHeader, Token.endHeader, Token.startBody, - Token.text('some text '), + Token.text('some text'), Token.newline, Token.text('other text'), Token.newline, @@ -563,7 +563,6 @@ void main() { Token.startBody, Token.startExpression, Token.endExpression, - Token.text(' '), Token.newline, Token.endBody, ], @@ -855,6 +854,22 @@ void main() { ); }); + test('comments in lines', () { + expect( + tokenize('---\n---\n' + 'line1 // whatever\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('line1'), + Token.newline, + Token.endBody, + ], + ); + }); + test('commands in lines', () { expect( tokenize('---\n---\n'