diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index d57e518824d3aa..e6fd3eedf0df88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -41,7 +42,7 @@ * unbound result sink */ public class UnboundResultSink extends LogicalSink - implements Unbound, Sink, BlockFuncDepsPropagation { + implements NeedAuditEncryption, Unbound, Sink, BlockFuncDepsPropagation { public UnboundResultSink(CHILD_TYPE child) { super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child); @@ -94,4 +95,9 @@ public String toString() { public StmtType stmtType() { return StmtType.SELECT; } + + @Override + public boolean needAuditEncryption() { + return anyMatch(node -> node instanceof UnboundTVFRelation); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java new file mode 100644 index 00000000000000..d495c164b56bd4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.DorisParser.InsertTableContext; +import org.apache.doris.nereids.DorisParser.SupportedDmlStatementContext; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.apache.commons.collections4.MapUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/**LogicalPlanBuilderForEncryption*/ +public class LogicalPlanBuilderForEncryption extends LogicalPlanBuilder { + private final Map, String> indexInSqlToString; + + public LogicalPlanBuilderForEncryption(Map selectHintMap, + Map, String> indexInSqlToString) { + super(selectHintMap); + this.indexInSqlToString = Objects.requireNonNull(indexInSqlToString, "indexInSqlToString is null"); + } + + // select into outfile clause + @Override + public LogicalPlan visitStatementDefault(DorisParser.StatementDefaultContext ctx) { + if (ctx.outFileClause() != null && ctx.outFileClause().propertyClause() != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.outFileClause().propertyClause(); + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitStatementDefault(ctx); + } + + // export into outfile clause + @Override + public BrokerDesc visitWithRemoteStorageSystem(DorisParser.WithRemoteStorageSystemContext ctx) { + Map properties = visitPropertyItemList(ctx.brokerProperties); + encryptProperty(properties, ctx.brokerProperties.start.getStartIndex(), + ctx.brokerProperties.stop.getStopIndex()); + return super.visitWithRemoteStorageSystem(ctx); + } + + // load into outfile clause + @Override + public LogicalPlan visitLoad(DorisParser.LoadContext ctx) { + if (ctx.withRemoteStorageSystem() != null) { + Map properties = + new HashMap<>(visitPropertyItemList(ctx.withRemoteStorageSystem().brokerProperties)); + encryptProperty(properties, ctx.withRemoteStorageSystem().brokerProperties.start.getStartIndex(), + ctx.withRemoteStorageSystem().brokerProperties.stop.getStopIndex()); + } + return super.visitLoad(ctx); + } + + // create table clause + @Override + public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { + // property or ext property + if (ctx.propertyClause() != null) { + List propertyClauseContexts = ctx.propertyClause(); + for (DorisParser.PropertyClauseContext propertyClauseContext : propertyClauseContexts) { + if (propertyClauseContext != null) { + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + } + } + return super.visitCreateTable(ctx); + } + + // alter storage vault clause + @Override + public LogicalPlan visitAlterStorageVault(DorisParser.AlterStorageVaultContext ctx) { + if (ctx.properties != null && ctx.properties.fileProperties != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.properties; + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitAlterStorageVault(ctx); + } + + // select from tvf + @Override + public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionContext ctx) { + DorisParser.PropertyItemListContext properties = ctx.properties; + if (properties != null) { + encryptProperty(visitPropertyItemList(properties), properties.start.getStartIndex(), + properties.stop.getStopIndex()); + } + return super.visitTableValuedFunction(ctx); + } + + // create job select tvf + @Override + public LogicalPlan visitCreateScheduledJob(DorisParser.CreateScheduledJobContext ctx) { + SupportedDmlStatementContext supportedDmlStatementContext = ctx.supportedDmlStatement(); + visitInsertTable((InsertTableContext) supportedDmlStatementContext); + return super.visitCreateScheduledJob(ctx); + } + + private void encryptProperty(Map properties, int start, int stop) { + if (MapUtils.isNotEmpty(properties)) { + PrintableMap printableMap = new PrintableMap<>(properties, "=", + true, false, true); + indexInSqlToString.put(Pair.of(start, stop), printableMap.toString()); + } + } + + private void encryptPassword(int start, int stop) { + indexInSqlToString.put(Pair.of(start, stop), "'*XXX'"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index f9ac5b724ef922..50c1dddbe6c3b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -344,6 +344,14 @@ public LogicalPlan parseForCreateView(String sql) { return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + public LogicalPlan parseForEncryption(String sql, Map, String> indexInSqlToString) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); + LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForEncryption( + getHintMap(sql, tokenStream, DorisParser::selectHint), indexInSqlToString); + return (LogicalPlan) realLogicalPlanBuilder.visit(tree); + } + /** parseForSyncMv */ public Optional parseForSyncMv(String sql) { CommonTokenStream tokenStream = parseAllTokens(sql); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 3ace2194fc3068..5b4462b42970be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -70,7 +70,7 @@ * create table command */ @Developing -public class CreateTableCommand extends Command implements ForwardWithSync { +public class CreateTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(CreateTableCommand.class); private final Optional ctasQuery; @@ -221,5 +221,10 @@ public Optional getCtasQuery() { public StmtType stmtType() { return StmtType.CREATE; } + + @Override + public boolean needAuditEncryption() { + return !createTableInfo.getEngineName().equalsIgnoreCase(CreateTableInfo.ENGINE_OLAP); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index 35ddf3d86a28f0..d5e436f8c559a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -71,7 +71,7 @@ * [PROPERTIES("key"="value")] * WITH BROKER 'broker_name' [( $broker_attrs)] */ -public class ExportCommand extends Command implements ForwardWithSync { +public class ExportCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final String PARALLELISM = "parallelism"; public static final String LABEL = "label"; public static final String DATA_CONSISTENCY = "data_consistency"; @@ -405,5 +405,10 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.EXPORT; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java index 0701d5f1b80ed1..4d8f328666e143 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java @@ -85,7 +85,7 @@ /** * load OLAP table data from external bulk file */ -public class LoadCommand extends Command implements ForwardWithSync { +public class LoadCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(LoadCommand.class); @@ -509,4 +509,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.LOAD; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java new file mode 100644 index 00000000000000..823d9e6d4c3829 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo; + +import java.util.TreeMap; + +/** + * NeedAuditEncryption + */ +public interface NeedAuditEncryption { + boolean needAuditEncryption(); + + /** + * gene encryption SQL + */ + default String geneEncryptionSQL(String sql) { + if (!needAuditEncryption()) { + return sql; + } + TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + NereidsParser parser = new NereidsParser(); + parser.parseForEncryption(sql, indexInSqlToString); + return BaseViewInfo.rewriteSql(indexInSqlToString, sql); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index a9fa6d42df02c5..76ad9569eeaacf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -237,6 +237,14 @@ public String getTableName() { return tableName; } + public String getEngineName() { + return engineName; + } + + public Map getProperties() { + return properties; + } + /** * full qualifier table name. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index efd57eb7a97fa6..3ef1059d3a518e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -45,8 +45,10 @@ import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; @@ -86,7 +88,7 @@ * InsertIntoTableCommand(Query()) * ExplainCommand(Query()) */ -public class InsertIntoTableCommand extends Command implements ForwardWithSync, Explainable { +public class InsertIntoTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable { public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class); @@ -497,6 +499,11 @@ public RedirectStatus toRedirectStatus() { } } + @Override + public boolean needAuditEncryption() { + return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); + } + /** * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated * by NereidsPlanner diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index 43e490c00c7a17..f5e49cdba9e690 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import com.google.common.base.Preconditions; @@ -38,7 +39,7 @@ * logicalFileSink for select into outfile */ public class LogicalFileSink extends LogicalSink - implements Sink, PropagateFuncDeps { + implements NeedAuditEncryption, Sink, PropagateFuncDeps { private final String filePath; private final String format; @@ -120,4 +121,9 @@ public String getFormat() { public Map getProperties() { return properties; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index af9ef8ce51135d..97a6eaec2f12bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -42,6 +42,7 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; @@ -87,7 +88,6 @@ public class AuditLogHelper { public static void logAuditLog(ConnectContext ctx, String origStmt, StatementBase parsedStmt, org.apache.doris.proto.Data.PQueryStatistics statistics, boolean printFuzzyVariables) { try { - origStmt = handleStmt(origStmt, parsedStmt); logAuditLogImpl(ctx, origStmt, parsedStmt, statistics, printFuzzyVariables); } catch (Throwable t) { LOG.warn("Failed to write audit log.", t); @@ -373,12 +373,21 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme statistics == null ? 0 : statistics.getScanBytesFromRemoteStorage()); } - // We put origin query stmt at the end of audit log, for parsing the log more convenient. - if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - auditEventBuilder.setStmt(parsedStmt.toSql()); + boolean isAnalysisErr = ctx.getState().getStateType() == MysqlStateType.ERR + && ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR; + String encryptSql = isAnalysisErr ? ctx.getState().getErrorMessage() : origStmt; + + if (parsedStmt instanceof LogicalPlanAdapter) { + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + if ((logicalPlan instanceof NeedAuditEncryption)) { + encryptSql = ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(origStmt); + } } else { - auditEventBuilder.setStmt(origStmt); + if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { + encryptSql = parsedStmt.toSql(); + } } + auditEventBuilder.setStmt(handleStmt(encryptSql, parsedStmt)); if (!Env.getCurrentEnv().isMaster()) { if (ctx.executor != null && ctx.executor.isForwardToMaster()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java new file mode 100644 index 00000000000000..3bcc2bfa6fd2fa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.qe.AuditLogHelper; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class EncryptSQLTest extends ParserTestBase { + + NereidsParser parser = new NereidsParser(); + ConnectContext ctx = ConnectContext.get(); + WorkloadRuntimeStatusMgr mgr = Env.getCurrentEnv().getWorkloadRuntimeStatusMgr(); + List auditEvents = Deencapsulation.getField(mgr, "queryAuditEventList"); + + @Test + public void testEncryption() { + ctx.setDatabase("test"); + + String sql = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + String res = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + res = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"," + + " \"AWS_ENDPOINT\" = \"xxx\"," + + " \"AWS_ACCESS_KEY\" = \"abc\"," + + " \"AWS_SECRET_KEY\" = \"abc\"," + + " \"AWS_REGION\" = \"ap-beijing\"" + + ")"; + + res = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"123\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + + res = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"*XXX\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"123\"" + + ")"; + + res = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"*XXX\"" + + ")"; + parseAndCheck(sql, res); + + sql = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + } + + private void parseAndCheck(String sql, String expected) { + StatementBase parsedStmt = parser.parseSQL(sql).get(0); + AuditLogHelper.logAuditLog(ctx, sql, parsedStmt, null, false); + AuditEvent event = auditEvents.get(auditEvents.size() - 1); + Assertions.assertEquals(expected, event.stmt); + } +}