diff --git a/src/dparse/lexer.d b/src/dparse/lexer.d index dd0db6ad..471b9880 100644 --- a/src/dparse/lexer.d +++ b/src/dparse/lexer.d @@ -171,7 +171,7 @@ public struct LexerConfig { string fileName; StringBehavior stringBehavior; - WhitespaceBehavior whitespaceBehavior; + WhitespaceBehavior whitespaceBehavior = WhitespaceBehavior.skip; CommentBehavior commentBehavior = CommentBehavior.intern; } @@ -384,9 +384,10 @@ public bool isLiteral(IdType type) pure nothrow @safe @nogc } /** - * Returns: an array of tokens lexed from the given source code to the output range. All - * whitespace tokens are skipped and comments are attached to the token nearest - * to them. + * Returns: an array of tokens lexed from the given source code to the output range. + * Whitespace token are skipped by default, except `WhitespaceBehavior.include` + * has been set explicitly. + * comments are attached to the token nearest to them. */ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, StringCache* cache) @@ -409,7 +410,6 @@ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, return CommentType.notDoc; } - config.whitespaceBehavior = WhitespaceBehavior.skip; config.commentBehavior = CommentBehavior.noIntern; auto leadingCommentAppender = appender!(char[])(); @@ -425,6 +425,9 @@ const(Token)[] getTokensForParser(ubyte[] sourceCode, LexerConfig config, { case tok!"specialTokenSequence": case tok!"whitespace": + if (config.whitespaceBehavior == WhitespaceBehavior.include) + output.put(lexer.front); + lexer.popFront(); break; case tok!"comment":