[Refactor](dialect) Add sql dialect converter plugins.

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -2279,5 +2279,4 @@ public class Config extends ConfigBase {
 
  @ConfField(mutable = true, masterOnly = true)
  public static int publish_topic_info_interval_ms = 30000; // 30s
-
 }

fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java

@@ -207,7 +207,6 @@
 import org.apache.doris.persist.meta.MetaReader;
 import org.apache.doris.persist.meta.MetaWriter;
 import org.apache.doris.planner.SingleTabletLoadRecorderMgr;
-import org.apache.doris.planner.TabletLoadIndexRecorderMgr;
 import org.apache.doris.plugin.DialectConverterPluginMgr;
 import org.apache.doris.plugin.PluginInfo;
 import org.apache.doris.plugin.PluginMgr;

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java

@@ -83,13 +83,6 @@ public List<StatementBase> parseSQL(String originStr, @Nullable LogicalPlanBuild
 
  private List<StatementBase> parseSQLWithDialect(String sql,
  SessionVariable sessionVariable) {
-<<<<<<< HEAD
- switch (sqlDialect) {
- case TRINO:
- final List<StatementBase> logicalPlans = TrinoParser.parse(sql, sessionVariable);
- if (CollectionUtils.isEmpty(logicalPlans)) {
- return parseSQL(sql);
-=======
  @Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect());
  if (sqlDialect == null) {
  return parseSQL(sql);
@@ -102,7 +95,6 @@ private List<StatementBase> parseSQLWithDialect(String sql,
  List<StatementBase> statementBases = plugin.parseSqlWithDialect(sql, sessionVariable);
  if (CollectionUtils.isNotEmpty(statementBases)) {
  return statementBases;
->>>>>>> 72f272e785 ([Refactor](dialect) Add sql dialect converter plugins.)
  }
  } catch (Throwable throwable) {
  LOG.warn("Parse sql with dialect {} failed, sql: {}.", sqlDialect, sql, throwable);

fe/fe-core/src/main/java/org/apache/doris/nereids/types/VarcharType.java

@@ -85,4 +85,8 @@ public boolean equals(Object o) {
  public int hashCode() {
  return Objects.hash(super.hashCode(), len);
  }
+
+ public boolean isWildcardVarchar() {
+ return len == -1 || len == MAX_VARCHAR_LENGTH;
+ }
 }

fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java

@@ -2020,10 +2020,6 @@ public String getSqlDialect() {
  return sqlDialect;
  }
 
- public int getWaitFullBlockScheduleTimes() {
- return waitFullBlockScheduleTimes;
- }
-
  public Dialect getSqlParseDialect() {
  return Dialect.getByName(sqlDialect);
  }

fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java

@@ -21,7 +21,6 @@
 import org.apache.doris.common.Config;
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.StatementContext;
-import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.exceptions.ParseException;
 import org.apache.doris.nereids.glue.LogicalPlanAdapter;
 import org.apache.doris.nereids.trees.expressions.Cast;

fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3FnCallTransformers.java

@@ -35,30 +35,54 @@ private SparkSql3FnCallTransformers() {
 
  @Override
  protected void registerTransformers() {
- doRegister("get_json_object", 2, "json_extract",
- Lists.newArrayList(
- PlaceholderExpression.of(Expression.class, 1),
- PlaceholderExpression.of(Expression.class, 2)), true);
+ // register json functions
+ registerJsonFunctionTransformers();
+ // register string functions
+ registerStringFunctionTransformers();
+ // register date functions
+ registerDateFunctionTransformers();
+ // register numeric functions
+ registerNumericFunctionTransformers();
+ // TODO: add other function transformer
+ }
+
+ @Override
+ protected void registerComplexTransformers() {
+ DateTruncFnCallTransformer dateTruncFnCallTransformer = new DateTruncFnCallTransformer();
+ doRegister(dateTruncFnCallTransformer.getSourceFnName(), dateTruncFnCallTransformer);
+ // TODO: add other complex function transformer
+ }
 
- doRegister("get_json_object", 2, "json_extract",
+ private void registerJsonFunctionTransformers() {
+ doRegister("get_json_object", "json_extract_string",
  Lists.newArrayList(
  PlaceholderExpression.of(Expression.class, 1),
- PlaceholderExpression.of(Expression.class, 2)), false);
+ PlaceholderExpression.of(Expression.class, 2)));
+ }
 
- doRegister("split", 2, "split_by_string",
+ private void registerStringFunctionTransformers() {
+ doRegister("split", "split_by_string",
  Lists.newArrayList(
  PlaceholderExpression.of(Expression.class, 1),
- PlaceholderExpression.of(Expression.class, 2)), true);
- doRegister("split", 2, "split_by_string",
+ PlaceholderExpression.of(Expression.class, 2)));
+ }
+
+ private void registerDateFunctionTransformers() {
+ // spark-sql support to_date(date_str, fmt) function but doris only support to_date(date_str)
+ // here try to compat with this situation by using str_to_date(str, fmt),
+ // this function support the following three formats which can handle the mainly situations:
+ // 1. yyyyMMdd
+ // 2. yyyy-MM-dd
+ // 3. yyyy-MM-dd HH:mm:ss
+ doRegister("to_date", "str_to_date",
  Lists.newArrayList(
  PlaceholderExpression.of(Expression.class, 1),
- PlaceholderExpression.of(Expression.class, 2)), false);
- // TODO: add other function transformer
+ PlaceholderExpression.of(Expression.class, 2)));
  }
 
- @Override
- protected void registerComplexTransformers() {
- // TODO: add other complex function transformer
+ private void registerNumericFunctionTransformers() {
+  doRegister("mean", "avg",
+  Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
  }
 
  static class SingletonHolder {

fe_plugins/sparksql-converter/src/main/java/org/apache/doris/plugin/dialect/spark/SparkSql3LogicalPlanBuilder.java

@@ -55,8 +55,8 @@ public LogicalPlan visitAliasedQuery(DorisParser.AliasedQueryContext ctx) {
  }
 
  @Override
- public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) {
- Expression expression = super.visitFunctionCallExpression(ctx);
+ public Expression visitFunctionCall(DorisParser.FunctionCallContext ctx) {
+ Expression expression = super.visitFunctionCall(ctx);
  if (!(expression instanceof UnboundFunction)) {
  return expression;
  }

fe_plugins/trino-converter/src/main/java/org/apache/doris/plugin/dialect/trino/TrinoFnCallTransformers.java

@@ -47,8 +47,8 @@ protected void registerComplexTransformers() {
  }
 
  protected void registerStringFunctionTransformer() {
- doRegister("codepoint", 1, "ascii",
- Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)), false);
+ doRegister("codepoint", "ascii",
+ Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
  // TODO: add other string function transformer
  }