diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8c90f6a5162af3..08ba77465fbf29 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2356,6 +2356,8 @@ public class Config extends ConfigBase { "Whether to enable the function of getting log files through http interface"}) public static boolean enable_get_log_file_api = false; + + @Deprecated @ConfField(description = {"用于SQL方言转换的服务地址。", "The service address for SQL dialect conversion."}) public static String sql_convertor_service = ""; diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 0375db287e6e21..86031b5e2422f6 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -680,12 +680,6 @@ under the License. kryo-shaded - - - io.trino - trino-parser - - org.apache.arrow diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java index 5cc6a363cff7d1..fb5b8072edbd19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java @@ -27,8 +27,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder; +import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.thrift.TNullSide; @@ -50,6 +49,8 @@ public class InlineViewRef extends TableRef { private static final Logger LOG = LogManager.getLogger(InlineViewRef.class); + private static final String DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL = "__auto_generated_subquery_name"; + // Catalog or local view that is referenced. // Null for inline views parsed directly from a query string. private final View view; @@ -198,12 +199,12 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (view == null && !hasExplicitAlias()) { String dialect = ConnectContext.get().getSessionVariable().getSqlDialect(); - ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(dialect); - if (ParseDialect.Dialect.SPARK_SQL != sqlDialect) { + Dialect sqlDialect = Dialect.getByName(dialect); + if (Dialect.SPARK_SQL != sqlDialect) { ErrorReport.reportAnalysisException(ErrorCode.ERR_DERIVED_MUST_HAVE_ALIAS); } hasExplicitAlias = true; - aliases = new String[] { SparkSql3LogicalPlanBuilder.DEFAULT_TABLE_ALIAS }; + aliases = new String[] { DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL }; } // Analyze the inline view query statement with its own analyzer diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 9ae7d4f1617660..ddb04dc480b755 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -217,6 +217,7 @@ import org.apache.doris.persist.meta.MetaReader; import org.apache.doris.persist.meta.MetaWriter; import org.apache.doris.planner.TabletLoadIndexRecorderMgr; +import org.apache.doris.plugin.DialectConverterPluginMgr; import org.apache.doris.plugin.PluginInfo; import org.apache.doris.plugin.PluginMgr; import org.apache.doris.policy.PolicyMgr; @@ -472,6 +473,8 @@ public class Env { private PluginMgr pluginMgr; + private DialectConverterPluginMgr dialectConverterPluginMgr; + private AuditEventProcessor auditEventProcessor; private RefreshManager refreshManager; @@ -721,6 +724,7 @@ private Env(boolean isCheckpointCatalog) { this.pluginMgr = new PluginMgr(); this.auditEventProcessor = new AuditEventProcessor(this.pluginMgr); + this.dialectConverterPluginMgr = new DialectConverterPluginMgr(this.pluginMgr); this.refreshManager = new RefreshManager(); this.policyMgr = new PolicyMgr(); this.extMetaCacheMgr = new ExternalMetaCacheMgr(); @@ -789,6 +793,10 @@ public PluginMgr getPluginMgr() { return pluginMgr; } + public DialectConverterPluginMgr getSqlDialectPluginMgr() { + return dialectConverterPluginMgr; + } + public Auth getAuth() { return auth; } @@ -954,6 +962,7 @@ public void initialize(String[] args) throws Exception { // init plugin manager pluginMgr.init(); + dialectConverterPluginMgr.init(); auditEventProcessor.start(); // 2. get cluster id and role (Observer or Follower) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java index 38a028c71dd262..3d96e6dd039898 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/DialectTransformException.java @@ -18,8 +18,7 @@ package org.apache.doris.nereids.exceptions; /** - * DialectTransformException when have not supported transforming for the - * {@link io.trino.sql.tree.Node}. + * DialectTransformException when have not supported transforming for dialect converters. */ public class DialectTransformException extends UnsupportedOperationException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java index 9e977fedbca309..cdf7944c61c158 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/exceptions/UnsupportedDialectException.java @@ -17,17 +17,16 @@ package org.apache.doris.nereids.exceptions; -import org.apache.doris.nereids.parser.ParseDialect; +import org.apache.doris.nereids.parser.Dialect; /** * UnsupportedDialectException when not match any in - * {@link org.apache.doris.nereids.parser.ParseDialect}. + * {@link Dialect}. */ public class UnsupportedDialectException extends UnsupportedOperationException { - public UnsupportedDialectException(ParseDialect dialect) { - super(String.format("Unsupported dialect name is %s, version is %s", - dialect.getDialect().getDialectName(), dialect.getVersion().getVersionName())); + public UnsupportedDialectException(Dialect dialect) { + super(String.format("Unsupported dialect name is %s", dialect.getDialectName())); } public UnsupportedDialectException(String type, String msg) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/Dialect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/Dialect.java new file mode 100644 index 00000000000000..684eea02ae6764 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/Dialect.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import javax.annotation.Nullable; + +/** + * ParseDialect enum, maybe support other dialect. + */ +public enum Dialect { + /** + * Doris parser dialect + */ + DORIS("doris"), + /** + * Trino parser dialect + */ + TRINO("trino"), + /** + * Presto parser dialect + */ + PRESTO("presto"), + /** + * Spark sql parser dialect + */ + SPARK_SQL("spark_sql"), + /** + * Hive parser dialect + */ + HIVE("hive"), + /** + * Iceberg parser dialect + */ + ICEBERG("iceberg"), + /** + * Hudi parser dialect + */ + HUDI("hudi"), + /** + * Paimon parser dialect + */ + PAIMON("paimon"), + /** + * Alibaba dlf parser dialect + */ + DLF("dlf"), + /** + * Alibaba max compute parser dialect + */ + MAX_COMPUTE("max_compute"), + /** + * Mysql parser dialect + */ + MYSQL("mysql"), + /** + * Postgresql parser dialect + */ + POSTGRESQL("postgresql"), + /** + * Sqlserver parser dialect + */ + SQLSERVER("sqlserver"), + /** + * Clickhouse parser dialect + */ + CLICKHOUSE("clickhouse"), + /** + * Sap hana parser dialect + */ + SAP_HANA("sap_hana"), + /** + * OceanBase parser dialect + */ + OCEANBASE("oceanbase"); + + + private final String dialectName; + + Dialect(String dialectName) { + this.dialectName = dialectName; + } + + public String getDialectName() { + return dialectName; + } + + /** + * Get dialect by name + */ + public static @Nullable Dialect getByName(String dialectName) { + if (dialectName == null) { + return null; + } + for (Dialect dialect : Dialect.values()) { + if (dialect.getDialectName().equals(dialectName.toLowerCase())) { + return dialect; + } + } + return null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 27172d312f7799..99c420ea88ad13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -18,20 +18,19 @@ package org.apache.doris.nereids.parser; import org.apache.doris.analysis.StatementBase; -import org.apache.doris.common.Config; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Pair; import org.apache.doris.nereids.DorisLexer; import org.apache.doris.nereids.DorisParser; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.parser.spark.SparkSql3LogicalPlanBuilder; -import org.apache.doris.nereids.parser.trino.TrinoParser; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.plugin.DialectConverterPlugin; +import org.apache.doris.plugin.DialectConverterPluginMgr; import org.apache.doris.qe.SessionVariable; -import com.google.common.base.Strings; import com.google.common.collect.Lists; import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; @@ -67,11 +66,13 @@ public List parseSQL(String originStr) { * ParseSQL with dialect. */ public List parseSQL(String sql, SessionVariable sessionVariable) { - @Nullable ParseDialect.Dialect sqlDialect = ParseDialect.Dialect.getByName(sessionVariable.getSqlDialect()); - return parseSQLWithDialect(sql, sqlDialect, sessionVariable); + return parseSQLWithDialect(sql, sessionVariable); } - private List parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) { + /** + * ParseSQL with logicalPlanBuilder. + */ + public List parseSQL(String originStr, @Nullable LogicalPlanBuilder logicalPlanBuilder) { List> logicalPlans = parseMultiple(originStr, logicalPlanBuilder); List statementBases = Lists.newArrayList(); for (Pair parsedPlanToContext : logicalPlans) { @@ -81,26 +82,27 @@ private List parseSQL(String originStr, @Nullable LogicalPlanBuil } private List parseSQLWithDialect(String sql, - @Nullable ParseDialect.Dialect sqlDialect, SessionVariable sessionVariable) { - if (!Strings.isNullOrEmpty(Config.sql_convertor_service)) { - // if sql convertor service is enabled, no need to parse sql again by specific dialect. + @Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect()); + if (sqlDialect == null) { return parseSQL(sql); } - switch (sqlDialect) { - case TRINO: - final List logicalPlans = TrinoParser.parse(sql, sessionVariable); - if (CollectionUtils.isEmpty(logicalPlans)) { - return parseSQL(sql); - } - return logicalPlans; - case SPARK_SQL: - return parseSQL(sql, new SparkSql3LogicalPlanBuilder()); - - default: - return parseSQL(sql); + DialectConverterPluginMgr pluginMgr = Env.getCurrentEnv().getSqlDialectPluginMgr(); + List plugins = pluginMgr.getDialectConverterPlugins(sqlDialect); + for (DialectConverterPlugin plugin : plugins) { + try { + List statementBases = plugin.parseSqlWithDialect(sql, sessionVariable); + if (CollectionUtils.isNotEmpty(statementBases)) { + return statementBases; + } + } catch (Throwable throwable) { + LOG.warn("Parse sql with dialect {} failed, sql: {}.", sqlDialect, sql, throwable); + } } + + // fallback if any exception occurs before + return parseSQL(sql); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java deleted file mode 100644 index b94de76e279b08..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParseDialect.java +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser; - -import javax.annotation.Nullable; - -/** - * ParseDialect enum, maybe support other dialect. - */ -public enum ParseDialect { - - /** - * Trino parser and it's version is 395. - */ - TRINO_395(Dialect.TRINO, Version.TRINO_395), - /** - * Doris parser and it's version is 2.0.0. - */ - DORIS_2_ALL(Dialect.DORIS, Version.DORIS_2_ALL), - /** - * Spark parser and it's version is 3.x. - */ - SPARK_SQL_3_ALL(Dialect.SPARK_SQL, Version.SPARK_SQL_3_ALL); - - private final Dialect dialect; - private final Version version; - - ParseDialect(Dialect dialect, Version version) { - this.dialect = dialect; - this.version = version; - } - - public Version getVersion() { - return version; - } - - public Dialect getDialect() { - return dialect; - } - - /** - * The version of parse dialect. - */ - public enum Version { - /** - * Trino parser and it's version is 395. - */ - TRINO_395("395"), - /** - * Doris parser and it's version is 2.0.0. - */ - DORIS_2_ALL("2.*"), - /** - * Spark sql parser and it's version is 3.x. - */ - SPARK_SQL_3_ALL("3.*"); - private final String version; - - Version(String version) { - this.version = version; - } - - public String getVersionName() { - return version; - } - } - - /** - * The dialect name of parse dialect. - */ - public enum Dialect { - /** - * Trino parser dialect - */ - TRINO("trino"), - /** - * Presto parser dialect - */ - PRESTO("presto"), - /** - * Doris parser dialect - */ - DORIS("doris"), - /** - * Spark sql parser dialect - */ - SPARK_SQL("spark_sql"); - - private String dialectName; - - Dialect(String dialectName) { - this.dialectName = dialectName; - } - - public String getDialectName() { - return dialectName; - } - - /** - * Get dialect by name - */ - public static @Nullable Dialect getByName(String dialectName) { - if (dialectName == null) { - return null; - } - for (Dialect dialect : Dialect.values()) { - if (dialect.getDialectName().equals(dialectName.toLowerCase())) { - return dialect; - } - } - return null; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java index c36767f1beab6e..91ef4afd06e4b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ParserContext.java @@ -22,13 +22,13 @@ */ public class ParserContext { - private final ParseDialect parseDialect; + private final Dialect parseDialect; - public ParserContext(ParseDialect parseDialect) { + public ParserContext(Dialect parseDialect) { this.parseDialect = parseDialect; } - public ParseDialect getParserDialect() { + public Dialect getParserDialect() { return parseDialect; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java deleted file mode 100644 index 5a6ec21fc9a1c5..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.spark; - -import org.apache.doris.nereids.analyzer.PlaceholderExpression; -import org.apache.doris.nereids.parser.AbstractFnCallTransformer; -import org.apache.doris.nereids.parser.AbstractFnCallTransformers; -import org.apache.doris.nereids.trees.expressions.Expression; - -import com.google.common.collect.Lists; - -/** - * The builder and factory for spark-sql 3.x {@link AbstractFnCallTransformer}, - * and supply transform facade ability. - */ -public class SparkSql3FnCallTransformers extends AbstractFnCallTransformers { - - private SparkSql3FnCallTransformers() { - } - - @Override - protected void registerTransformers() { - doRegister("get_json_object", 2, "json_extract", - Lists.newArrayList( - PlaceholderExpression.of(Expression.class, 1), - PlaceholderExpression.of(Expression.class, 2)), true); - - doRegister("get_json_object", 2, "json_extract", - Lists.newArrayList( - PlaceholderExpression.of(Expression.class, 1), - PlaceholderExpression.of(Expression.class, 2)), false); - - doRegister("split", 2, "split_by_string", - Lists.newArrayList( - PlaceholderExpression.of(Expression.class, 1), - PlaceholderExpression.of(Expression.class, 2)), true); - doRegister("split", 2, "split_by_string", - Lists.newArrayList( - PlaceholderExpression.of(Expression.class, 1), - PlaceholderExpression.of(Expression.class, 2)), false); - // TODO: add other function transformer - } - - @Override - protected void registerComplexTransformers() { - // TODO: add other complex function transformer - } - - static class SingletonHolder { - private static final SparkSql3FnCallTransformers INSTANCE = new SparkSql3FnCallTransformers(); - } - - public static SparkSql3FnCallTransformers getSingleton() { - return SingletonHolder.INSTANCE; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3LogicalPlanBuilder.java deleted file mode 100644 index 49eb2b74cc1f63..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3LogicalPlanBuilder.java +++ /dev/null @@ -1,88 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.spark; - -import org.apache.doris.nereids.DorisParser; -import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.exceptions.ParseException; -import org.apache.doris.nereids.parser.LogicalPlanBuilder; -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.nereids.parser.ParserContext; -import org.apache.doris.nereids.parser.ParserUtils; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; - -import org.apache.commons.lang3.StringUtils; - -/** - * Extends from {@link org.apache.doris.nereids.parser.LogicalPlanBuilder}, - * just focus on the difference between these query syntax. - */ -public class SparkSql3LogicalPlanBuilder extends LogicalPlanBuilder { - // use a default alias name if not exists, keep the same name with spark-sql - public static final String DEFAULT_TABLE_ALIAS = "__auto_generated_subquery_name"; - - private final ParserContext parserContext; - - public SparkSql3LogicalPlanBuilder() { - this.parserContext = new ParserContext(ParseDialect.SPARK_SQL_3_ALL); - } - - @Override - public LogicalPlan visitAliasedQuery(DorisParser.AliasedQueryContext ctx) { - LogicalPlan plan = withTableAlias(visitQuery(ctx.query()), ctx.tableAlias()); - for (DorisParser.LateralViewContext lateralViewContext : ctx.lateralView()) { - plan = withGenerate(plan, lateralViewContext); - } - return plan; - } - - @Override - public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpressionContext ctx) { - Expression expression = super.visitFunctionCallExpression(ctx); - if (!(expression instanceof UnboundFunction)) { - return expression; - } - UnboundFunction sourceFunction = (UnboundFunction) expression; - Function transformedFunction = SparkSql3FnCallTransformers.getSingleton().transform( - sourceFunction.getName(), - sourceFunction.getArguments(), - this.parserContext - ); - if (transformedFunction == null) { - return expression; - } - return transformedFunction; - } - - private LogicalPlan withTableAlias(LogicalPlan plan, DorisParser.TableAliasContext ctx) { - if (ctx.strictIdentifier() == null) { - return plan; - } - return ParserUtils.withOrigin(ctx.strictIdentifier(), () -> { - String alias = StringUtils.isEmpty(ctx.strictIdentifier().getText()) - ? DEFAULT_TABLE_ALIAS : ctx.strictIdentifier().getText(); - if (null != ctx.identifierList()) { - throw new ParseException("Do not implemented", ctx); - } - return new LogicalSubQueryAlias<>(alias, plan); - }); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java deleted file mode 100644 index d3a687a289feed..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.trino; - -import org.apache.doris.nereids.parser.AbstractFnCallTransformer; - -/** - * Trino complex function transformer - */ -public abstract class ComplexTrinoFnCallTransformer extends AbstractFnCallTransformer { - - protected abstract String getSourceFnName(); -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java deleted file mode 100644 index b59a9327bdedc0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.trino; - -import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.parser.ParserContext; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; - -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * DateDiff complex function transformer - */ -public class DateDiffFnCallTransformer extends ComplexTrinoFnCallTransformer { - - private static final String SECOND = "second"; - private static final String HOUR = "hour"; - private static final String DAY = "day"; - private static final String MILLI_SECOND = "millisecond"; - - @Override - public String getSourceFnName() { - return "date_diff"; - } - - @Override - protected boolean check(String sourceFnName, List sourceFnTransformedArguments, - ParserContext context) { - return getSourceFnName().equalsIgnoreCase(sourceFnName); - } - - @Override - protected Function transform(String sourceFnName, List sourceFnTransformedArguments, - ParserContext context) { - if (sourceFnTransformedArguments.size() != 3) { - return null; - } - VarcharLiteral diffGranularity = (VarcharLiteral) sourceFnTransformedArguments.get(0); - if (SECOND.equals(diffGranularity.getValue())) { - return new UnboundFunction( - "seconds_diff", - ImmutableList.of(sourceFnTransformedArguments.get(1), sourceFnTransformedArguments.get(2))); - } - // TODO: support other date diff granularity - return null; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java deleted file mode 100644 index 883cb1cd132e9c..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.trino; - -import org.apache.doris.nereids.analyzer.PlaceholderExpression; -import org.apache.doris.nereids.parser.AbstractFnCallTransformer; -import org.apache.doris.nereids.parser.AbstractFnCallTransformers; -import org.apache.doris.nereids.trees.expressions.Expression; - -import com.google.common.collect.Lists; - -/** - * The builder and factory for trino {@link AbstractFnCallTransformer}, - * and supply transform facade ability. - */ -public class TrinoFnCallTransformers extends AbstractFnCallTransformers { - - private TrinoFnCallTransformers() { - } - - @Override - protected void registerTransformers() { - registerStringFunctionTransformer(); - // TODO: add other function transformer - } - - @Override - protected void registerComplexTransformers() { - DateDiffFnCallTransformer dateDiffFnCallTransformer = new DateDiffFnCallTransformer(); - doRegister(dateDiffFnCallTransformer.getSourceFnName(), dateDiffFnCallTransformer); - // TODO: add other complex function transformer - } - - protected void registerStringFunctionTransformer() { - doRegister("codepoint", 1, "ascii", - Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)), false); - // TODO: add other string function transformer - } - - static class SingletonHolder { - private static final TrinoFnCallTransformers INSTANCE = new TrinoFnCallTransformers(); - } - - public static TrinoFnCallTransformers getSingleton() { - return SingletonHolder.INSTANCE; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoLogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoLogicalPlanBuilder.java deleted file mode 100644 index c1f9b9fbba8e77..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoLogicalPlanBuilder.java +++ /dev/null @@ -1,329 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.trino; - -import org.apache.doris.nereids.analyzer.UnboundAlias; -import org.apache.doris.nereids.analyzer.UnboundFunction; -import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; -import org.apache.doris.nereids.analyzer.UnboundRelation; -import org.apache.doris.nereids.analyzer.UnboundResultSink; -import org.apache.doris.nereids.analyzer.UnboundSlot; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.exceptions.DialectTransformException; -import org.apache.doris.nereids.parser.LogicalPlanBuilderAssistant; -import org.apache.doris.nereids.parser.ParserContext; -import org.apache.doris.nereids.trees.expressions.Cast; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; -import org.apache.doris.nereids.trees.expressions.functions.Function; -import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; -import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalProject; -import org.apache.doris.nereids.types.DataType; -import org.apache.doris.nereids.types.VarcharType; -import org.apache.doris.nereids.types.coercion.CharacterType; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; - -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -/** - * The actually planBuilder for Trino SQL to Doris logical plan. - * It depends on {@link io.trino.sql.tree.AstVisitor} - */ -public class TrinoLogicalPlanBuilder extends io.trino.sql.tree.AstVisitor { - - public Object visit(io.trino.sql.tree.Node node, ParserContext context) { - return this.process(node, context); - } - - public T visit(io.trino.sql.tree.Node node, ParserContext context, Class clazz) { - return clazz.cast(this.process(node, context)); - } - - public List visit(List nodes, ParserContext context, Class clazz) { - return nodes.stream() - .map(node -> clazz.cast(this.process(node, context))) - .collect(Collectors.toList()); - } - - public Object processOptional(Optional node, ParserContext context) { - return node.map(value -> this.process(value, context)).orElse(null); - } - - public T processOptional(Optional node, - ParserContext context, Class clazz) { - return node.map(value -> clazz.cast(this.process(value, context))).orElse(null); - } - - @Override - protected LogicalPlan visitQuery(io.trino.sql.tree.Query node, ParserContext context) { - io.trino.sql.tree.QueryBody queryBody = node.getQueryBody(); - LogicalPlan logicalPlan = (LogicalPlan) visit(queryBody, context); - if (!(queryBody instanceof io.trino.sql.tree.QuerySpecification)) { - // TODO: need to handle orderBy and limit - throw new DialectTransformException("transform querySpecification"); - } - return logicalPlan; - } - - @Override - protected LogicalPlan visitQuerySpecification(io.trino.sql.tree.QuerySpecification node, - ParserContext context) { - // from -> where -> group by -> having -> select - Optional from = node.getFrom(); - LogicalPlan fromPlan = processOptional(from, context, LogicalPlan.class); - List selectItems = node.getSelect().getSelectItems(); - if (from == null || !from.isPresent()) { - // TODO: support query values - List expressions = selectItems.stream() - .map(item -> visit(item, context, NamedExpression.class)) - .collect(ImmutableList.toImmutableList()); - return new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(), expressions); - } - // TODO: support predicate, aggregate, having, order by, limit - // TODO: support distinct - boolean isDistinct = node.getSelect().isDistinct(); - return new UnboundResultSink<>(withProjection(selectItems, fromPlan, isDistinct, context)); - } - - private LogicalProject withProjection(List selectItems, - LogicalPlan input, - boolean isDistinct, - ParserContext context) { - List expressions = selectItems.stream() - .map(item -> visit(item, context, NamedExpression.class)) - .collect(Collectors.toList()); - return new LogicalProject(expressions, ImmutableList.of(), isDistinct, input); - } - - @Override - protected Expression visitSingleColumn(io.trino.sql.tree.SingleColumn node, ParserContext context) { - String alias = node.getAlias().map(io.trino.sql.tree.Identifier::getValue).orElse(null); - Expression expr = visit(node.getExpression(), context, Expression.class); - if (expr instanceof NamedExpression) { - return (NamedExpression) expr; - } else { - return alias == null ? new UnboundAlias(expr) : new UnboundAlias(expr, alias); - } - } - - @Override - protected Object visitIdentifier(io.trino.sql.tree.Identifier node, ParserContext context) { - return new UnboundSlot(ImmutableList.of(node.getValue())); - } - - /* ******************************************************************************************** - * visitFunction - * ******************************************************************************************** */ - - @Override - protected Function visitFunctionCall(io.trino.sql.tree.FunctionCall node, ParserContext context) { - List exprs = visit(node.getArguments(), context, Expression.class); - Function transformedFn = - TrinoFnCallTransformers.getSingleton().transform(node.getName().toString(), exprs, context); - if (transformedFn == null) { - transformedFn = new UnboundFunction(node.getName().toString(), exprs); - - } - return transformedFn; - } - - /* ******************************************************************************************** - * visitTable - * ******************************************************************************************** */ - - @Override - protected LogicalPlan visitTable(io.trino.sql.tree.Table node, ParserContext context) { - io.trino.sql.tree.QualifiedName name = node.getName(); - List tableId = name.getParts(); - // build table - return LogicalPlanBuilderAssistant.withCheckPolicy( - new UnboundRelation(StatementScopeIdGenerator.newRelationId(), tableId, - ImmutableList.of(), false)); - } - - /* ******************************************************************************************** - * visit buildIn function - * ******************************************************************************************** */ - - @Override - protected Expression visitCast(io.trino.sql.tree.Cast node, ParserContext context) { - Expression expr = visit(node.getExpression(), context, Expression.class); - DataType dataType = mappingType(node.getType()); - Expression cast = new Cast(expr, dataType); - if (dataType.isStringLikeType() && ((CharacterType) dataType).getLen() >= 0) { - if (dataType.isVarcharType() && ((VarcharType) dataType).isWildcardVarchar()) { - return cast; - } - List args = ImmutableList.of( - cast, - new TinyIntLiteral((byte) 1), - Literal.of(((CharacterType) dataType).getLen()) - ); - return new UnboundFunction("substr", args); - } else { - return cast; - } - } - - /* ******************************************************************************************** - * visitLiteral - * ******************************************************************************************** */ - - @Override - protected Object visitLiteral(io.trino.sql.tree.Literal node, ParserContext context) { - // TODO: support literal transform - throw new DialectTransformException("transform literal"); - } - - @Override - protected Literal visitLongLiteral(io.trino.sql.tree.LongLiteral node, ParserContext context) { - return LogicalPlanBuilderAssistant.handleIntegerLiteral(String.valueOf(node.getValue())); - } - - @Override - protected Object visitDoubleLiteral(io.trino.sql.tree.DoubleLiteral node, ParserContext context) { - // TODO: support double literal transform - throw new DialectTransformException("transform double literal"); - } - - @Override - protected Object visitDecimalLiteral(io.trino.sql.tree.DecimalLiteral node, ParserContext context) { - // TODO: support decimal literal transform - throw new DialectTransformException("transform decimal literal"); - } - - @Override - protected Object visitTimestampLiteral(io.trino.sql.tree.TimestampLiteral node, ParserContext context) { - try { - String value = node.getValue(); - if (value.length() <= 10) { - value += " 00:00:00"; - } - return new DateTimeLiteral(value); - } catch (AnalysisException e) { - throw new DialectTransformException("transform timestamp literal"); - } - } - - @Override - protected Object visitGenericLiteral(io.trino.sql.tree.GenericLiteral node, ParserContext context) { - // TODO: support generic literal transform - throw new DialectTransformException("transform generic literal"); - } - - @Override - protected Object visitTimeLiteral(io.trino.sql.tree.TimeLiteral node, ParserContext context) { - // TODO: support time literal transform - throw new DialectTransformException("transform time literal"); - } - - @Override - protected Object visitCharLiteral(io.trino.sql.tree.CharLiteral node, ParserContext context) { - // TODO: support char literal transform - throw new DialectTransformException("transform char literal"); - } - - @Override - protected Expression visitStringLiteral(io.trino.sql.tree.StringLiteral node, ParserContext context) { - // TODO: add unescapeSQLString. - String txt = node.getValue(); - if (txt.length() <= 1) { - return new VarcharLiteral(txt); - } - return new VarcharLiteral(LogicalPlanBuilderAssistant.escapeBackSlash(txt.substring(0, txt.length()))); - } - - @Override - protected Object visitIntervalLiteral(io.trino.sql.tree.IntervalLiteral node, ParserContext context) { - // TODO: support interval literal transform - throw new DialectTransformException("transform char literal"); - } - - @Override - protected Object visitBinaryLiteral(io.trino.sql.tree.BinaryLiteral node, ParserContext context) { - // TODO: support binary literal transform - throw new DialectTransformException("transform binary literal"); - } - - @Override - protected Object visitNullLiteral(io.trino.sql.tree.NullLiteral node, ParserContext context) { - return NullLiteral.INSTANCE; - } - - @Override - protected Object visitBooleanLiteral(io.trino.sql.tree.BooleanLiteral node, ParserContext context) { - return BooleanLiteral.of(node.getValue()); - } - - private DataType mappingType(io.trino.sql.tree.DataType dataType) { - - if (dataType instanceof io.trino.sql.tree.GenericDataType) { - io.trino.sql.tree.GenericDataType genericDataType = (io.trino.sql.tree.GenericDataType) dataType; - String typeName = genericDataType.getName().getValue().toLowerCase(); - List types = Lists.newArrayList(typeName); - - String length = null; - String precision = null; - String scale = null; - List arguments = genericDataType.getArguments(); - if (!arguments.isEmpty()) { - if (arguments.get(0) instanceof io.trino.sql.tree.NumericParameter) { - precision = length = ((io.trino.sql.tree.NumericParameter) arguments.get(0)).getValue(); - } - if (arguments.size() > 1 && arguments.get(1) instanceof io.trino.sql.tree.NumericParameter) { - scale = ((io.trino.sql.tree.NumericParameter) arguments.get(1)).getValue(); - } - } - if ("decimal".equals(typeName)) { - if (precision != null) { - types.add(precision); - } - if (scale != null) { - types.add(scale); - } - } - if ("varchar".equals(typeName) || "char".equals(typeName)) { - if (length != null) { - types.add(length); - } - } - - // unsigned decimal in Trino is longDecimal, not handle now, support it later - if (!"decimal".equals(typeName) && typeName.contains("decimal")) { - throw new DialectTransformException("transform not standard decimal data type "); - } - // Trino only support signed, safe unsigned is false here - return DataType.convertPrimitiveFromStrings(types, false); - } else if (dataType instanceof io.trino.sql.tree.DateTimeDataType) { - // TODO: support date data type mapping - throw new DialectTransformException("transform date data type"); - } - throw new AnalysisException("Nereids do not support type: " + dataType); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java deleted file mode 100644 index 671af5e7f05dfb..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.parser.trino; - -import org.apache.doris.analysis.StatementBase; -import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.exceptions.UnsupportedDialectException; -import org.apache.doris.nereids.glue.LogicalPlanAdapter; -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.nereids.parser.ParserContext; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.qe.SessionVariable; - -import com.google.common.base.Preconditions; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import javax.annotation.Nullable; - -/** - * Trino Parser, depends on 395 trino-parser, and 4.9.3 antlr-runtime - */ -public class TrinoParser { - - public static final Logger LOG = LogManager.getLogger(TrinoParser.class); - - private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS = - new io.trino.sql.parser.ParsingOptions( - io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL); - - /** - * Parse with trino syntax, return null if parse failed - */ - public static @Nullable List parse(String sql, SessionVariable sessionVariable) { - final List logicalPlans = new ArrayList<>(); - try { - io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(sql); - ParserContext parserContext = new ParserContext(ParseDialect.TRINO_395); - StatementContext statementContext = new StatementContext(); - for (io.trino.sql.parser.StatementSplitter.Statement statement : splitter.getCompleteStatements()) { - Object parsedPlan = parseSingle(statement.statement(), parserContext); - logicalPlans.add(parsedPlan == null - ? null : new LogicalPlanAdapter((LogicalPlan) parsedPlan, statementContext)); - } - if (logicalPlans.isEmpty() || logicalPlans.stream().anyMatch(Objects::isNull)) { - return null; - } - return logicalPlans; - } catch (io.trino.sql.parser.ParsingException | UnsupportedDialectException e) { - LOG.debug("Failed to parse logical plan from trino, sql is :{}", sql, e); - return null; - } - } - - private static io.trino.sql.tree.Statement parse(String sql) { - io.trino.sql.parser.SqlParser sqlParser = new io.trino.sql.parser.SqlParser(); - return sqlParser.createStatement(sql, PARSING_OPTIONS); - } - - /** - * Parse trino dialect sql. - * - * @param sql sql string - * @param parserContext parse context - * @return logical plan - */ - public static T parseSingle(String sql, ParserContext parserContext) { - Preconditions.checkArgument(parserContext.getParserDialect() == ParseDialect.TRINO_395); - io.trino.sql.tree.Statement statement = TrinoParser.parse(sql); - return (T) new TrinoLogicalPlanBuilder().visit(statement, parserContext); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginInfo.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginInfo.java index a20030522f2b61..737e3c140d492c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginInfo.java @@ -52,7 +52,8 @@ public class PluginInfo implements Writable { public enum PluginType { AUDIT, IMPORT, - STORAGE; + STORAGE, + DIALECT; public static int MAX_PLUGIN_TYPE_SIZE = PluginType.values().length; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java index 455d6913e97db4..c116e7c16ae19e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditEventProcessor.java @@ -84,7 +84,7 @@ public void run() { AuditEvent auditEvent; while (!isStopped) { // update audit plugin list every UPDATE_PLUGIN_INTERVAL_MS. - // because some of plugins may be installed or uninstalled at runtime. + // because some plugins may be installed or uninstalled at runtime. if (auditPlugins == null || System.currentTimeMillis() - lastUpdateTime > UPDATE_PLUGIN_INTERVAL_MS) { auditPlugins = pluginMgr.getActivePluginList(PluginType.AUDIT); lastUpdateTime = System.currentTimeMillis(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 3fe4457ba5a81e..dc079c325ba56a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -35,7 +35,6 @@ import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugUtil; -import org.apache.doris.common.util.SQLDialectUtils; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.SqlUtils; import org.apache.doris.common.util.Util; @@ -49,8 +48,11 @@ import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; +import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.stats.StatsErrorEstimator; +import org.apache.doris.plugin.DialectConverterPlugin; +import org.apache.doris.plugin.DialectConverterPluginMgr; import org.apache.doris.proto.Data; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.thrift.TMasterOpRequest; @@ -60,6 +62,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -69,6 +72,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import javax.annotation.Nullable; /** * Process one connection, the life cycle is the same as connection @@ -170,8 +174,7 @@ protected void handleQuery(MysqlCommand mysqlCommand, String originStmt) { MetricRepo.COUNTER_REQUEST_ALL.increase(1L); } - String convertedStmt = SQLDialectUtils.convertStmtWithDialect(originStmt, ctx, mysqlCommand); - + String convertedStmt = convertOriginStmt(originStmt); String sqlHash = DigestUtils.md5Hex(convertedStmt); ctx.setSqlHash(sqlHash); ctx.getAuditEventBuilder().reset(); @@ -280,6 +283,28 @@ protected void handleQuery(MysqlCommand mysqlCommand, String originStmt) { } + private String convertOriginStmt(String originStmt) { + String convertedStmt = originStmt; + @Nullable Dialect sqlDialect = Dialect.getByName(ctx.getSessionVariable().getSqlDialect()); + if (sqlDialect != null) { + DialectConverterPluginMgr pluginMgr = Env.getCurrentEnv().getSqlDialectPluginMgr(); + List plugins = pluginMgr.getDialectConverterPlugins(sqlDialect); + for (DialectConverterPlugin plugin : plugins) { + try { + String convertedSql = plugin.convertSql(originStmt, ctx.getSessionVariable()); + if (StringUtils.isNotEmpty(convertedSql)) { + convertedStmt = convertedSql; + break; + } + } catch (Throwable throwable) { + LOG.warn("Convert sql with dialect {} failed, sql: {}, use origin sql.", + sqlDialect, originStmt, throwable); + } + } + } + return convertedStmt; + } + // Use a handler for exception to avoid big try catch block which is a little hard to understand protected void handleQueryException(Throwable throwable, String origStmt, StatementBase parsedStmt, Data.PQueryStatistics statistics) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 182cfc4424aff3..b3e46e2d6361bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -28,8 +28,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.metrics.Event; import org.apache.doris.nereids.metrics.EventSwitchParser; -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.nereids.parser.ParseDialect.Dialect; +import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.planner.GroupCommitBlockSink; import org.apache.doris.qe.VariableMgr.VarAttr; @@ -2267,8 +2266,8 @@ public int getWaitFullBlockScheduleTimes() { return waitFullBlockScheduleTimes; } - public ParseDialect.Dialect getSqlParseDialect() { - return ParseDialect.Dialect.getByName(sqlDialect); + public Dialect getSqlParseDialect() { + return Dialect.getByName(sqlDialect); } public void setSqlDialect(String sqlDialect) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index bae4d7459ce9f0..c093adce53e9dc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -21,7 +21,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -42,7 +41,6 @@ import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; -import org.apache.doris.qe.SessionVariable; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -189,41 +187,6 @@ public void testParseSQL() { Assertions.assertTrue(logicalPlan1 instanceof ExplainCommand); } - @Test - public void testParseSQLWithTrinoDialect() { - String sql = "select `AD``D` from t1 where a = 1;explain graph select `AD``D` from t1 where a = 1;"; - NereidsParser nereidsParser = new NereidsParser(); - SessionVariable sessionVariable = new SessionVariable(); - sessionVariable.setSqlDialect("trino"); - // test fall back to doris parser - List statementBases = nereidsParser.parseSQL(sql, sessionVariable); - Assertions.assertEquals(2, statementBases.size()); - Assertions.assertTrue(statementBases.get(0) instanceof LogicalPlanAdapter); - Assertions.assertTrue(statementBases.get(1) instanceof LogicalPlanAdapter); - LogicalPlan logicalPlan0 = ((LogicalPlanAdapter) statementBases.get(0)).getLogicalPlan(); - LogicalPlan logicalPlan1 = ((LogicalPlanAdapter) statementBases.get(1)).getLogicalPlan(); - Assertions.assertTrue(logicalPlan0 instanceof UnboundResultSink); - Assertions.assertTrue(logicalPlan1 instanceof ExplainCommand); - } - - @Test - public void testParseSQLWithSparkSqlDialect() { - // doris parser will throw a ParseException when derived table does not have alias - String sql1 = "select * from (select * from t1);"; - NereidsParser nereidsParser = new NereidsParser(); - Assertions.assertThrows(ParseException.class, () -> nereidsParser.parseSQL(sql1), - "Every derived table must have its own alias"); - - // test parse with spark-sql dialect - SessionVariable sessionVariable = new SessionVariable(); - sessionVariable.setSqlDialect("spark_sql"); - List statementBases = nereidsParser.parseSQL(sql1, sessionVariable); - Assertions.assertEquals(1, statementBases.size()); - Assertions.assertTrue(statementBases.get(0) instanceof LogicalPlanAdapter); - LogicalPlan logicalPlan = ((LogicalPlanAdapter) statementBases.get(0)).getLogicalPlan(); - Assertions.assertTrue(logicalPlan instanceof UnboundResultSink); - } - @Test public void testParseJoin() { NereidsParser nereidsParser = new NereidsParser(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java index 5e68a3809f9403..d8e92b69b8a569 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/ParserTestBase.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.util.ExpressionParseChecker; import org.apache.doris.nereids.util.MemoPatternMatchSupported; import org.apache.doris.nereids.util.PlanParseChecker; -import org.apache.doris.nereids.util.TrinoDialectPlanParseChecker; import org.apache.doris.qe.ConnectContext; import mockit.Mock; @@ -51,7 +50,4 @@ public ExpressionParseChecker parseExpression(String sql) { return new ExpressionParseChecker(sql); } - public TrinoDialectPlanParseChecker trinoDialectParsePlan(String sql) { - return new TrinoDialectPlanParseChecker(sql); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java deleted file mode 100644 index 9e22895cb79024..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/TrinoDialectPlanParseChecker.java +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.util; - -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.nereids.parser.ParserContext; -import org.apache.doris.nereids.parser.trino.TrinoParser; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; - -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; -import org.apache.commons.lang3.StringUtils; -import org.junit.jupiter.api.Assertions; - -/** - * Plan parse checker for trino. - * It supports equals or contain pattern match assert and so on. - */ -public class TrinoDialectPlanParseChecker extends ParseChecker { - - private final Supplier parsedPlanSupplier; - - public TrinoDialectPlanParseChecker(String sql) { - super(sql); - this.parsedPlanSupplier = - Suppliers.memoize(() -> TrinoParser.parseSingle(sql, new ParserContext(ParseDialect.TRINO_395))); - } - - public TrinoDialectPlanParseChecker assertEquals(LogicalPlan plan) { - LogicalPlan target = parsedPlanSupplier.get(); - Assertions.assertEquals(plan, target); - return this; - } - - public TrinoDialectPlanParseChecker assertContains(String... expects) { - LogicalPlan logicalPlan = parsedPlanSupplier.get(); - Assertions.assertNotNull(logicalPlan); - String targetPlanString = logicalPlan.toString(); - for (String expected : expects) { - Assertions.assertTrue(StringUtils.containsIgnoreCase(targetPlanString.toLowerCase(), expected), - "expected contain is: " + expected + " but plan is \n" + targetPlanString); - } - return this; - } -} diff --git a/fe/pom.xml b/fe/pom.xml index c8f862dd540811..fb0c314d321c00 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -344,7 +344,6 @@ under the License. 0.5.0-incubating 3.4.4 - 395 shade-format-flatbuffers 1.12.0 @@ -1455,13 +1454,6 @@ under the License. client ${vesoft.client.version} - - - io.trino - trino-parser - ${trino.parser.version} - io.grpc grpc-netty diff --git a/fe_plugins/pom.xml b/fe_plugins/pom.xml index 1315315c8976e1..b5831b1d97ceab 100644 --- a/fe_plugins/pom.xml +++ b/fe_plugins/pom.xml @@ -65,11 +65,19 @@ under the License. auditdemo auditloader + presto-converter + trino-converter + sparksql-converter - 2.18.0 - 2.0-SNAPSHOT github + 1 + 1.2-SNAPSHOT + 2.18.0 + 1.2.17 + 1.18.24 + 5.8.2 + 1.49 @@ -137,7 +145,35 @@ under the License. log4j log4j - 1.2.17 + ${log4j.verion} + + + + org.jmockit + jmockit + ${jmockit.version} + test + + + + org.junit.jupiter + junit-jupiter-engine + ${junit.version} + test + + + + org.junit.vintage + junit-vintage-engine + ${junit.version} + test + + + + org.junit.jupiter + junit-jupiter-params + ${junit.version} + test diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/SQLDialectUtils.java b/fe_plugins/presto-converter/src/main/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtils.java similarity index 84% rename from fe/fe-core/src/main/java/org/apache/doris/common/util/SQLDialectUtils.java rename to fe_plugins/presto-converter/src/main/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtils.java index 777f4ee2794e73..c2df5806abd78a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/SQLDialectUtils.java +++ b/fe_plugins/presto-converter/src/main/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtils.java @@ -15,12 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.common.util; - -import org.apache.doris.common.Config; -import org.apache.doris.mysql.MysqlCommand; -import org.apache.doris.nereids.parser.ParseDialect; -import org.apache.doris.qe.ConnectContext; +package org.apache.doris.plugin.dialect.presto; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -60,28 +55,7 @@ public class SQLDialectUtils { private static final Logger LOG = LogManager.getLogger(SQLDialectUtils.class); - public static String convertStmtWithDialect(String originStmt, ConnectContext ctx, MysqlCommand mysqlCommand) { - if (mysqlCommand != MysqlCommand.COM_QUERY) { - return originStmt; - } - if (Config.sql_convertor_service.isEmpty()) { - return originStmt; - } - ParseDialect.Dialect dialect = ctx.getSessionVariable().getSqlParseDialect(); - if (dialect == null) { - return originStmt; - } - switch (dialect) { - case PRESTO: - return convertStmtWithPresto(originStmt); - default: - LOG.debug("only support presto dialect now."); - return originStmt; - } - } - - private static String convertStmtWithPresto(String originStmt) { - String targetURL = Config.sql_convertor_service; + public static String convertSql(String targetURL, String originStmt) { ConvertRequest convertRequest = new ConvertRequest(originStmt, "presto"); HttpURLConnection connection = null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/SQLDialectUtilsTest.java b/fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtilsTest.java similarity index 53% rename from fe/fe-core/src/test/java/org/apache/doris/common/util/SQLDialectUtilsTest.java rename to fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtilsTest.java index 3a0812c89f2c95..23e9c89db33662 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/SQLDialectUtilsTest.java +++ b/fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SQLDialectUtilsTest.java @@ -15,13 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.common.util; - -import org.apache.doris.common.Config; -import org.apache.doris.mysql.MysqlCommand; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.utframe.SimpleHttpServer; -import org.apache.doris.utframe.TestWithFeService; +package org.apache.doris.plugin.dialect.presto; import org.junit.After; import org.junit.Assert; @@ -29,15 +23,18 @@ import org.junit.Test; import java.io.IOException; +import java.net.DatagramSocket; +import java.net.ServerSocket; +import java.net.SocketException; public class SQLDialectUtilsTest { - int port; - SimpleHttpServer server; + private int port; + private SimpleHttpServer server; @Before public void setUp() throws Exception { - port = TestWithFeService.findValidPort(); + port = findValidPort(); server = new SimpleHttpServer(port); server.start("/api/v1/convert"); } @@ -50,37 +47,44 @@ public void tearDown() { } @Test - public void testSqlConvert() throws IOException { + public void testSqlConvert() { String originSql = "select * from t1 where \"k1\" = 1"; String expectedSql = "select * from t1 where `k1` = 1"; - ConnectContext ctx = TestWithFeService.createDefaultCtx(); - // 1. not COM_QUERY - String res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_STMT_RESET); - Assert.assertEquals(originSql, res); - // 2. config sql_convertor_service not set - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); - Assert.assertEquals(originSql, res); - // 3. session var sql_dialect not set - Config.sql_convertor_service = "http://127.0.0.1:" + port + "/api/v1/convert"; - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); - Assert.assertEquals(originSql, res); - // 4. not support dialect - ctx.getSessionVariable().setSqlDialect("sqlserver"); - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); + + String targetURL = "http://127.0.0.1:" + port + "/api/v1/convert"; + String res = SQLDialectUtils.convertSql(targetURL, originSql); Assert.assertEquals(originSql, res); - // 5. test presto - ctx.getSessionVariable().setSqlDialect("presto"); + // test presto server.setResponse("{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); + res = SQLDialectUtils.convertSql(targetURL, originSql); Assert.assertEquals(expectedSql, res); - // 6. test response version error + // test response version error server.setResponse("{\"version\": \"v2\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); + res = SQLDialectUtils.convertSql(targetURL, originSql); Assert.assertEquals(originSql, res); // 7. test response code error server.setResponse( "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 400, \"message\": \"\"}"); - res = SQLDialectUtils.convertStmtWithDialect(originSql, ctx, MysqlCommand.COM_QUERY); + res = SQLDialectUtils.convertSql(targetURL, originSql); Assert.assertEquals(originSql, res); } + + private static int findValidPort() { + int port; + while (true) { + try (ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + port = socket.getLocalPort(); + try (DatagramSocket datagramSocket = new DatagramSocket(port)) { + datagramSocket.setReuseAddress(true); + break; + } catch (SocketException e) { + System.out.println("The port " + port + " is invalid and try another port."); + } + } catch (IOException e) { + throw new IllegalStateException("Could not find a free TCP/IP port to start HTTP Server on"); + } + } + return port; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/SimpleHttpServer.java b/fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SimpleHttpServer.java similarity index 98% rename from fe/fe-core/src/test/java/org/apache/doris/utframe/SimpleHttpServer.java rename to fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SimpleHttpServer.java index 57ffcf90824c40..4c5d9bdf723100 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/SimpleHttpServer.java +++ b/fe_plugins/presto-converter/src/test/java/org/apache/doris/plugin/dialect/presto/SimpleHttpServer.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.utframe; +package org.apache.doris.plugin.dialect.presto; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java b/fe_plugins/sparksql-converter/src/test/java/org/apache/doris/plugin/dialect/spark/FnTransformTest.java similarity index 78% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java rename to fe_plugins/sparksql-converter/src/test/java/org/apache/doris/plugin/dialect/spark/FnTransformTest.java index f652e171280617..5092b94107ab75 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java +++ b/fe_plugins/sparksql-converter/src/test/java/org/apache/doris/plugin/dialect/spark/FnTransformTest.java @@ -15,24 +15,41 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.parser.spark; +package org.apache.doris.plugin.dialect.spark; + import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.parser.ParserTestBase; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import mockit.Mock; +import mockit.MockUp; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; + /** * Spark SQL to Doris function mapping test. */ -public class FnTransformTest extends ParserTestBase { +public class FnTransformTest { + @BeforeAll + public static void init() { + ConnectContext ctx = new ConnectContext(); + SessionVariable sessionVariable = new SessionVariable(); + ctx.setSessionVariable(sessionVariable); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + } @Test public void testCommonFnTransform() { NereidsParser nereidsParser = new NereidsParser(); - String sql1 = "SELECT json_extract('{\"a\": 1}', '$.a') as b FROM t"; String dialectSql1 = "SELECT get_json_object('{\"a\": 1}', '$.a') as b FROM t"; LogicalPlan logicalPlan1 = nereidsParser.parseSingle(sql1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java b/fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/FnTransformTest.java similarity index 53% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java rename to fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/FnTransformTest.java index eefcf8599d6d4a..896dbf068136f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/FnTransformTest.java +++ b/fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/FnTransformTest.java @@ -15,32 +15,59 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.parser.trino; +package org.apache.doris.plugin.dialect.trino; + +import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.parser.ParserTestBase; +import org.apache.doris.nereids.parser.ParserContext; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; + /** * Trino to Doris function mapping test. */ -public class FnTransformTest extends ParserTestBase { +public class FnTransformTest { + + @BeforeAll + public static void init() { + ConnectContext ctx = new ConnectContext(); + SessionVariable sessionVariable = new SessionVariable(); + ctx.setSessionVariable(sessionVariable); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + } @Test public void testStringFnTransform() { - String sql = "SELECT ascii('a') as b FROM t"; + ParserContext parserContext = new ParserContext(Dialect.TRINO); NereidsParser nereidsParser = new NereidsParser(); + String sql = "SELECT ascii('a') as b FROM t"; LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); - String dialectSql = "SELECT codepoint('a') as b FROM t"; - trinoDialectParsePlan(dialectSql).assertEquals(logicalPlan); + LogicalPlan dialectLogicalPlan = TrinoParser.parseSingle(dialectSql, parserContext); + Assertions.assertEquals(dialectLogicalPlan, logicalPlan); } @Test public void testDateDiffFnTransform() { + ParserContext parserContext = new ParserContext(Dialect.TRINO); String dialectSql = "SELECT date_diff('second', TIMESTAMP '2020-12-25 22:00:00', TIMESTAMP '2020-12-25 21:00:00')"; - trinoDialectParsePlan(dialectSql).assertContains("seconds_diff(2020-12-25 22:00:00, 2020-12-25 21:00:00)"); + LogicalPlan logicalPlan = TrinoParser.parseSingle(dialectSql, parserContext); + Assertions.assertTrue(logicalPlan.child(0).toString().toLowerCase() + .contains("seconds_diff(2020-12-25 22:00:00, 2020-12-25 21:00:00)")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java b/fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/ParserTest.java similarity index 54% rename from fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java rename to fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/ParserTest.java index f438f4c252c182..8b741befc59340 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/trino/QueryTest.java +++ b/fe_plugins/trino-converter/src/test/java/org/apache/doris/plugin/dialect/trino/ParserTest.java @@ -15,28 +15,52 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.parser.trino; +package org.apache.doris.plugin.dialect.trino; +import mockit.Mock; +import mockit.MockUp; +import org.apache.doris.nereids.parser.Dialect; import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.parser.ParserTestBase; +import org.apache.doris.nereids.parser.ParserContext; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; + /** * Trino query tests. */ -public class QueryTest extends ParserTestBase { +public class ParserTest { + + @BeforeAll + public static void init() { + ConnectContext ctx = new ConnectContext(); + SessionVariable sessionVariable = new SessionVariable(); + ctx.setSessionVariable(sessionVariable); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + } @Test public void testParseCast1() { + ParserContext parserContext = new ParserContext(Dialect.TRINO); String sql = "SELECT CAST(1 AS DECIMAL(20, 6)) FROM t"; NereidsParser nereidsParser = new NereidsParser(); LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); - trinoDialectParsePlan(sql).assertEquals(logicalPlan); + LogicalPlan dialectLogicalPlan = TrinoParser.parseSingle(sql, parserContext); + Assertions.assertEquals(dialectLogicalPlan, logicalPlan); sql = "SELECT CAST(a AS DECIMAL(20, 6)) FROM t"; logicalPlan = nereidsParser.parseSingle(sql); - trinoDialectParsePlan(sql).assertEquals(logicalPlan); + dialectLogicalPlan = TrinoParser.parseSingle(sql, parserContext); + Assertions.assertEquals(dialectLogicalPlan, logicalPlan); } }