Skip to content

[opt](nereids) Improve sql parse performance by avoid parse twice when collect hint map #52629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Aug 12, 2025
Merged

[opt](nereids) Improve sql parse performance by avoid parse twice when collect hint map #52629

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 44 additions & 29 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@

import java.lang.reflect.Method;
import java.util.BitSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -228,7 +229,7 @@ private static Token readUntilNonComment(TokenSource tokenSource) {
}

private List<StatementBase> parseSQLWithDialect(String sql,
SessionVariable sessionVariable) {
SessionVariable sessionVariable) {
@Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect());
if (sqlDialect == null) {
return parseSQL(sql);
Expand All @@ -244,7 +245,7 @@ private List<StatementBase> parseSQLWithDialect(String sql,
}
} catch (Throwable throwable) {
LOG.warn("Parse sql with dialect {} failed, plugin: {}, sql: {}.",
sqlDialect, plugin.getClass().getSimpleName(), sql, throwable);
sqlDialect, plugin.getClass().getSimpleName(), sql, throwable);
}
}

Expand Down Expand Up @@ -280,7 +281,7 @@ public List<Pair<LogicalPlan, StatementContext>> parseMultiple(String sql) {
}

public List<Pair<LogicalPlan, StatementContext>> parseMultiple(String sql,
@Nullable LogicalPlanBuilder logicalPlanBuilder) {
@Nullable LogicalPlanBuilder logicalPlanBuilder) {
return parse(sql, logicalPlanBuilder, DorisParser::multiStatements);
}

Expand Down Expand Up @@ -325,38 +326,31 @@ private <T> T parse(String sql, Function<DorisParser, ParserRuleContext> parseFu
}

private <T> T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder,
Function<DorisParser, ParserRuleContext> parseFunction) {
ParserRuleContext tree = toAst(sql, parseFunction);
Function<DorisParser, ParserRuleContext> parseFunction) {
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(tokenStream, parseFunction);
LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null
? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder;
? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint))
: logicalPlanBuilder;
return (T) realLogicalPlanBuilder.visit(tree);
}

public LogicalPlan parseForCreateView(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView(
getHintMap(sql, DorisParser::selectHint));
getHintMap(sql, tokenStream, DorisParser::selectHint));
return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
}

public Optional<String> parseForSyncMv(String sql) {
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
getHintMap(sql, DorisParser::selectHint));
logicalPlanBuilderForSyncMv.visit(tree);
return logicalPlanBuilderForSyncMv.getQuerySql();
}

/** get hint map */
public static Map<Integer, ParserRuleContext> getHintMap(String sql,
Function<DorisParser, ParserRuleContext> parseFunction) {
public static Map<Integer, ParserRuleContext> getHintMap(String sql, CommonTokenStream hintTokenStream,
Function<DorisParser, ParserRuleContext> parseFunction) {
// parse hint first round
DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer);

Map<Integer, ParserRuleContext> selectHintMap = Maps.newHashMap();

Token hintToken = hintTokenStream.getTokenSource().nextToken();
Iterator<Token> tokenIterator = hintTokenStream.getTokens().iterator();
Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null;
while (hintToken != null && hintToken.getType() != DorisLexer.EOF) {
if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') {
String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1);
Expand All @@ -366,15 +360,19 @@ public static Map<Integer, ParserRuleContext> getHintMap(String sql,
ParserRuleContext hintContext = parseFunction.apply(hintParser);
selectHintMap.put(hintToken.getStartIndex(), hintContext);
}
hintToken = hintTokenStream.getTokenSource().nextToken();
hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null;
}
return selectHintMap;
}

public static ParserRuleContext toAst(
String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
return toAst(parseAllTokens(sql), parseFunction);
}

/** toAst */
public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
public static ParserRuleContext toAst(
CommonTokenStream tokenStream, Function<DorisParser, ParserRuleContext> parseFunction) {
DorisParser parser = new DorisParser(tokenStream);

parser.addParseListener(POST_PROCESSOR);
Expand Down Expand Up @@ -405,9 +403,7 @@ public static ParserRuleContext toAst(String sql, Function<DorisParser, ParserRu
* will be normalized to: select \/*+SET_VAR(key=value)*\/ * , a, b from table
*/
public static String removeCommentAndTrimBlank(String sql) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
CommonTokenStream tokenStream = parseAllTokens(sql);

// maybe add more space char
StringBuilder newSql = new StringBuilder((int) (sql.length() * 1.2));
Expand All @@ -434,4 +430,23 @@ public static String removeCommentAndTrimBlank(String sql) {
}
return newSql.toString().trim();
}

private static CommonTokenStream parseAllTokens(String sql) {
DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql)));
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
return tokenStream;
}

/**
* sync mv sql parser
*/
public Optional<String> parseForSyncMv(String sql) {
CommonTokenStream tokenStream = parseAllTokens(sql);
ParserRuleContext tree = toAst(sql, DorisParser::singleStatement);
LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv(
getHintMap(sql, tokenStream, DorisParser::selectHint));
logicalPlanBuilderForSyncMv.visit(tree);
return logicalPlanBuilderForSyncMv.getQuerySql();
}
}