diff --git a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart index 480155061f3..43f29f7e881 100644 --- a/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart +++ b/packages/flame_jenny/jenny/lib/src/parse/tokenize.dart @@ -77,7 +77,11 @@ class _Lexer { /// Returns the integer code unit at the current parse position, or -1 if we /// reached the end of input. - int get currentCodeUnit => eof ? -1 : text.codeUnitAt(position); + int get currentCodeUnit => + position < text.length ? text.codeUnitAt(position) : -1; + + int get nextCodeUnit => + position < text.length - 1 ? text.codeUnitAt(position + 1) : -1; /// Pushes a new mode into the mode stack and returns `true`. bool pushMode(_ModeFn mode) { @@ -651,22 +655,24 @@ class _Lexer { /// Emits the text token corresponding to the text processed. bool eatPlainText() { final position0 = position; + var positionBeforeWhitespace = position; while (!eof) { final cu = currentCodeUnit; - if (cu == $lessThan || cu == $slash) { - position += 1; - if (currentCodeUnit == cu) { - position -= 1; - break; - } - } else if (cu == $carriageReturn || - cu == $lineFeed || + if ((cu == $slash && nextCodeUnit == $slash) || cu == $hash || + cu == $carriageReturn || + cu == $lineFeed) { + position = positionBeforeWhitespace; + break; + } else if ((cu == $lessThan && nextCodeUnit == $lessThan) || cu == $backslash || cu == $leftBrace) { break; } position += 1; + if (!(cu == $space || cu == $tab)) { + positionBeforeWhitespace = position; + } } if (position > position0) { pushToken(Token.text(text.substring(position0, position)), position0); diff --git a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart index a4cc15ce99a..2209aa61d46 100644 --- a/packages/flame_jenny/jenny/test/parse/tokenize_test.dart +++ b/packages/flame_jenny/jenny/test/parse/tokenize_test.dart @@ -470,6 +470,23 @@ void main() { ], ); }); + + test('line with hash tags', () { + expect( + tokenize('---\n---\n' + 'Some text #with-tag\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('Some text'), + Token.hashtag('#with-tag'), + Token.newline, + Token.endBody, + ], + ); + }); }); group('modeText', () { @@ -477,13 +494,13 @@ void main() { expect( tokenize('---\n---\n' 'some text // here be dragons\n' - 'other text\n' + 'other text \t\n' '===\n'), const [ Token.startHeader, Token.endHeader, Token.startBody, - Token.text('some text '), + Token.text('some text'), Token.newline, Token.text('other text'), Token.newline, @@ -546,7 +563,6 @@ void main() { Token.startBody, Token.startExpression, Token.endExpression, - Token.text(' '), Token.newline, Token.endBody, ], @@ -823,7 +839,7 @@ void main() { Token.startHeader, Token.endHeader, Token.startBody, - Token.text('line1 '), + Token.text('line1'), Token.hashtag('#tag'), Token.hashtag('#some:other@tag!'), Token.newline, @@ -831,7 +847,6 @@ void main() { Token.startExpression, Token.number('33'), Token.endExpression, - Token.text(' '), Token.hashtag('#here-be-dragons'), Token.newline, Token.endBody, @@ -839,6 +854,22 @@ void main() { ); }); + test('comments in lines', () { + expect( + tokenize('---\n---\n' + 'line1 // whatever\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('line1'), + Token.newline, + Token.endBody, + ], + ); + }); + test('commands in lines', () { expect( tokenize('---\n---\n' @@ -892,6 +923,24 @@ void main() { ], ); }); + + test('text with escaped content', () { + expect( + tokenize('---\n---\n' + 'One \\{ two\n' + '===\n'), + const [ + Token.startHeader, + Token.endHeader, + Token.startBody, + Token.text('One '), + Token.text('{'), + Token.text(' two'), + Token.newline, + Token.endBody, + ], + ); + }); }); // This group is for testing the error mechanism itself, not any particular