Remove explicit EOF token from token builder (#1276)

packages/langium/src/parser/token-builder.ts

@@ -7,10 +7,10 @@
 import type { CustomPatternMatcherFunc, TokenPattern, TokenType, TokenVocabulary } from 'chevrotain';
 import type { AbstractRule, Grammar, Keyword, TerminalRule } from '../grammar/generated/ast.js';
 import type { Stream } from '../utils/stream.js';
-import { Lexer, EOF } from 'chevrotain';
-import { isKeyword, isParserRule, isTerminalRule, isEndOfFile } from '../grammar/generated/ast.js';
+import { Lexer } from 'chevrotain';
+import { isKeyword, isParserRule, isTerminalRule } from '../grammar/generated/ast.js';
 import { terminalRegex } from '../grammar/internal-grammar-util.js';
-import { streamAllContents, streamAst } from '../utils/ast-util.js';
+import { streamAllContents } from '../utils/ast-util.js';
 import { getAllReachableRules } from '../utils/grammar-util.js';
 import { getCaseInsensitivePattern, isWhitespaceRegExp, partialMatches } from '../utils/regex-util.js';
 import { stream } from '../utils/stream.js';
@@ -38,11 +38,8 @@ export class DefaultTokenBuilder implements TokenBuilder {
                 tokens.push(terminalToken);
             }
         });
-
-        //reminder: EOF should always be the last token, because it is very unlikely that it will be matched within the input
-        if (reachableRules.some(r => streamAst(r.definition).some(isEndOfFile))) {
-            tokens.push(EOF);
-        }
+        // We don't need to add the EOF token explicitly.
+        // It is automatically available at the end of the token stream.
         return tokens;
     }