feat: Trim whitespace at the end of dialogue lines

packages/flame_jenny/jenny/lib/src/parse/tokenize.dart

@@ -77,7 +77,11 @@ class _Lexer {
 
  /// Returns the integer code unit at the current parse position, or -1 if we
  /// reached the end of input.
- int get currentCodeUnit => eof ? -1 : text.codeUnitAt(position);
+ int get currentCodeUnit =>
+ position < text.length ? text.codeUnitAt(position) : -1;
+
+ int get nextCodeUnit =>
+ position < text.length - 1 ? text.codeUnitAt(position + 1) : -1;
 
  /// Pushes a new mode into the mode stack and returns `true`.
  bool pushMode(_ModeFn mode) {
@@ -651,22 +655,24 @@ class _Lexer {
  /// Emits the text token corresponding to the text processed.
  bool eatPlainText() {
  final position0 = position;
+ var positionBeforeWhitespace = position;
  while (!eof) {
  final cu = currentCodeUnit;
- if (cu == $lessThan || cu == $slash) {
- position += 1;
- if (currentCodeUnit == cu) {
- position -= 1;
- break;
- }
- } else if (cu == $carriageReturn ||
- cu == $lineFeed ||
+ if ((cu == $slash && nextCodeUnit == $slash) ||
  cu == $hash ||
+ cu == $carriageReturn ||
+ cu == $lineFeed) {
+ position = positionBeforeWhitespace;
+ break;
+ } else if ((cu == $lessThan && nextCodeUnit == $lessThan) ||
  cu == $backslash ||
  cu == $leftBrace) {
  break;
  }
  position += 1;
+ if (!(cu == $space || cu == $tab)) {
+ positionBeforeWhitespace = position;
+ }
  }
  if (position > position0) {
  pushToken(Token.text(text.substring(position0, position)), position0);

packages/flame_jenny/jenny/test/parse/tokenize_test.dart

@@ -470,20 +470,37 @@ void main() {
  ],
  );
  });
+
+ test('line with hash tags', () {
+ expect(
+ tokenize('---\n---\n'
+ 'Some text #with-tag\n'
+ '===\n'),
+ const [
+ Token.startHeader,
+ Token.endHeader,
+ Token.startBody,
+ Token.text('Some text'),
+ Token.hashtag('#with-tag'),
+ Token.newline,
+ Token.endBody,
+ ],
+ );
+ });
  });
 
  group('modeText', () {
  test('text with comment', () {
  expect(
  tokenize('---\n---\n'
  'some text // here be dragons\n'
- 'other text\n'
+ 'other text \t\n'
  '===\n'),
  const [
  Token.startHeader,
  Token.endHeader,
  Token.startBody,
- Token.text('some text '),
+ Token.text('some text'),
  Token.newline,
  Token.text('other text'),
  Token.newline,
@@ -546,7 +563,6 @@ void main() {
  Token.startBody,
  Token.startExpression,
  Token.endExpression,
- Token.text(' '),
  Token.newline,
  Token.endBody,
  ],
@@ -823,22 +839,37 @@ void main() {
  Token.startHeader,
  Token.endHeader,
  Token.startBody,
- Token.text('line1 '),
+ Token.text('line1'),
  Token.hashtag('#tag'),
  Token.hashtag('#some:other@tag!'),
  Token.newline,
  Token.text('line2 '),
  Token.startExpression,
  Token.number('33'),
  Token.endExpression,
- Token.text(' '),
  Token.hashtag('#here-be-dragons'),
  Token.newline,
  Token.endBody,
  ],
  );
  });
 
+ test('comments in lines', () {
+ expect(
+ tokenize('---\n---\n'
+ 'line1 // whatever\n'
+ '===\n'),
+ const [
+ Token.startHeader,
+ Token.endHeader,
+ Token.startBody,
+ Token.text('line1'),
+ Token.newline,
+ Token.endBody,
+ ],
+ );
+ });
+
  test('commands in lines', () {
  expect(
  tokenize('---\n---\n'
@@ -892,6 +923,24 @@ void main() {
  ],
  );
  });
+
+ test('text with escaped content', () {
+ expect(
+ tokenize('---\n---\n'
+ 'One \\{ two\n'
+ '===\n'),
+ const [
+ Token.startHeader,
+ Token.endHeader,
+ Token.startBody,
+ Token.text('One '),
+ Token.text('{'),
+ Token.text(' two'),
+ Token.newline,
+ Token.endBody,
+ ],
+ );
+ });
  });
 
  // This group is for testing the error mechanism itself, not any particular