From 09b5c11c66c22757ba3ae439ee053d84ad3a0a8f Mon Sep 17 00:00:00 2001 From: wangxiangyu Date: Wed, 27 Dec 2023 18:20:49 +0800 Subject: [PATCH] [Refactor](dialect) Add sql dialect converter plugins. --- .../org/apache/doris/plugin/PluginMgr.java | 7 +- .../org/apache/doris/plugin/PluginTest.java | 71 +++++++++++-------- .../doris/plugin/TestHiveDialectPlugin.java | 46 ++++++++++++ .../doris/plugin/TestSparkDialectPlugin.java | 42 +++++++++++ 4 files changed, 135 insertions(+), 31 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/plugin/TestHiveDialectPlugin.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/plugin/TestSparkDialectPlugin.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java index 48406c743a0b69..9a38bce01737fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/PluginMgr.java @@ -71,7 +71,7 @@ public PluginMgr() { dialectPlugins = new Map[Dialect.MAX_DIALECT_SIZE]; for (int i = 0; i < Dialect.MAX_DIALECT_SIZE; i++) { // use synchronized wrapper for thread-safe - plugins[i] = Collections.synchronizedSortedMap(Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER)); + dialectPlugins[i] = Collections.synchronizedSortedMap(Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER)); } } @@ -218,7 +218,10 @@ public boolean registerBuiltinPlugin(PluginInfo pluginInfo, Plugin plugin) { PluginLoader loader = new BuiltinPluginLoader(Config.plugin_dir, pluginInfo, plugin); PluginLoader checkLoader = plugins[pluginInfo.getTypeId()].putIfAbsent(pluginInfo.getName(), loader); - + // add dialect plugin + if (plugin instanceof DialectConverterPlugin) { + addDialectPlugin((DialectConverterPlugin) plugin, pluginInfo); + } return checkLoader == null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/plugin/PluginTest.java b/fe/fe-core/src/test/java/org/apache/doris/plugin/PluginTest.java index 5d69e2cc6a396a..031ae52e3cc03b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/plugin/PluginTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/PluginTest.java @@ -17,43 +17,56 @@ package org.apache.doris.plugin; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class PluginTest extends Plugin { +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; +import org.apache.doris.nereids.parser.Dialect; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.utframe.TestWithFeService; - private Map map = new HashMap<>(); +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; - @Override - public void init(PluginInfo info, PluginContext ctx) { - System.out.println("this is init"); - } - - - @Override - public void close() throws IOException { - super.close(); - System.out.println("this is close"); - } +import java.io.IOException; +import java.util.List; +public class PluginTest extends TestWithFeService { @Override - public int flags() { - return 2; - } + public void runBeforeAll() throws IOException, InterruptedException { + connectContext.getState().setNereids(true); + connectContext.getSessionVariable().enableFallbackToOriginalPlanner = false; + connectContext.getSessionVariable().enableNereidsTimeout = false; + connectContext.getSessionVariable().enableNereidsDML = true; + FeConstants.runningUnitTest = true; - @Override - public void setVariable(String key, String value) { - map.put(key, value); + TestHiveDialectPlugin hivePlugin = new TestHiveDialectPlugin(); + PluginInfo hivePluginInfo = new PluginInfo("hiveDialectPlugin", PluginInfo.PluginType.DIALECT, "test"); + TestSparkDialectPlugin sparkPlugin = new TestSparkDialectPlugin(); + PluginInfo sparkPluginInfo = new PluginInfo("sparkDialectPlugin", PluginInfo.PluginType.DIALECT, "test"); + Env.getCurrentEnv().getPluginMgr().registerBuiltinPlugin(hivePluginInfo, hivePlugin); + Env.getCurrentEnv().getPluginMgr().registerBuiltinPlugin(sparkPluginInfo, sparkPlugin); } - @Override - public Map variable() { - return map; + @Test + public void testHivePlugin() { + connectContext.getSessionVariable().setSqlDialect(Dialect.HIVE.getDialectName()); + NereidsParser parser = new NereidsParser(); + List stmts = parser.parseSQL("select * from test_hive_table", + connectContext.getSessionVariable()); + Assertions.assertEquals(1, stmts.size()); + Assertions.assertTrue(stmts.get(0) instanceof LogicalPlan); + Assertions.assertTrue(stmts.get(0).toString().contains("select 1")); } - @Override - public Map status() { - return new HashMap<>(); + @Test + public void testSparkPlugin() { + connectContext.getSessionVariable().setSqlDialect(Dialect.SPARK_SQL.getDialectName()); + NereidsParser parser = new NereidsParser(); + List stmts = parser.parseSQL("select * from test_hive_table", + connectContext.getSessionVariable()); + Assertions.assertEquals(1, stmts.size()); + Assertions.assertTrue(stmts.get(0) instanceof LogicalPlan); + Assertions.assertTrue(stmts.get(0).toString().contains("select 2")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/plugin/TestHiveDialectPlugin.java b/fe/fe-core/src/test/java/org/apache/doris/plugin/TestHiveDialectPlugin.java new file mode 100644 index 00000000000000..28763a1c5619e2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/TestHiveDialectPlugin.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.plugin; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.nereids.parser.Dialect; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; + +public class TestHiveDialectPlugin extends Plugin implements DialectConverterPlugin { + + private static final String TEST_CONVERTED_SQL = "select 1"; + + @Override + public ImmutableSet acceptDialects() { + return ImmutableSet.of(Dialect.HIVE); + } + + @Override + public String convertSql(String originSql, SessionVariable sessionVariable) { + return TEST_CONVERTED_SQL; + } + + @Override + public List parseSqlWithDialect(String sql, SessionVariable sessionVariable) { + return null; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/plugin/TestSparkDialectPlugin.java b/fe/fe-core/src/test/java/org/apache/doris/plugin/TestSparkDialectPlugin.java new file mode 100644 index 00000000000000..26e30ad66deef1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/TestSparkDialectPlugin.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.plugin; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.nereids.parser.Dialect; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; + +public class TestSparkDialectPlugin extends Plugin implements DialectConverterPlugin { + + private static final String TEST_CONVERTED_SQL = "select 2"; + + @Override + public ImmutableSet acceptDialects() { + return ImmutableSet.of(Dialect.SPARK_SQL); + } + + @Override + public List parseSqlWithDialect(String sql, SessionVariable sessionVariable) { + return new NereidsParser().parseSQL(TEST_CONVERTED_SQL, sessionVariable); + } +}