diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 8c21461f1e68b7..ce1f0fb7c44384 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -256,7 +256,7 @@ supportedShowStatement | SHOW DELETE ((FROM | IN) database=multipartIdentifier)? #showDelete | SHOW ALL? GRANTS #showGrants | SHOW GRANTS FOR userIdentify #showGrantsForUser - | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)? #showSyncJob + | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)? #showSyncJob | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL? limitClause? #showLoadProfile | SHOW CREATE REPOSITORY FOR identifier #showCreateRepository | SHOW VIEW @@ -849,10 +849,10 @@ optionWithoutType | (CHAR SET | CHARSET) (charsetName=identifierOrText | DEFAULT) #setCharset | NAMES (charsetName=identifierOrText | DEFAULT) (COLLATE collateName=identifierOrText | DEFAULT)? #setCollate - | PASSWORD (FOR userIdentify)? EQ (STRING_LITERAL - | (isPlain=PASSWORD LEFT_PAREN STRING_LITERAL RIGHT_PAREN)) #setPassword - | LDAP_ADMIN_PASSWORD EQ (STRING_LITERAL - | (PASSWORD LEFT_PAREN STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword + | PASSWORD (FOR userIdentify)? EQ (pwd=STRING_LITERAL + | (isPlain=PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setPassword + | LDAP_ADMIN_PASSWORD EQ (pwd=STRING_LITERAL + | (PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword | variable #setVariableWithoutType ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index d57e518824d3aa..e6fd3eedf0df88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -41,7 +42,7 @@ * unbound result sink */ public class UnboundResultSink extends LogicalSink - implements Unbound, Sink, BlockFuncDepsPropagation { + implements NeedAuditEncryption, Unbound, Sink, BlockFuncDepsPropagation { public UnboundResultSink(CHILD_TYPE child) { super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child); @@ -94,4 +95,9 @@ public String toString() { public StmtType stmtType() { return StmtType.SELECT; } + + @Override + public boolean needAuditEncryption() { + return anyMatch(node -> node instanceof UnboundTVFRelation); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java new file mode 100644 index 00000000000000..809ab277a18b11 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.trees.plans.commands.info.SetVarOp; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.apache.commons.collections.MapUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/**LogicalPlanBuilderForEncryption*/ +public class LogicalPlanBuilderForEncryption extends LogicalPlanBuilder { + private final Map, String> indexInSqlToString; + + public LogicalPlanBuilderForEncryption(Map selectHintMap, + Map, String> indexInSqlToString) { + super(selectHintMap); + this.indexInSqlToString = Objects.requireNonNull(indexInSqlToString, "indexInSqlToString is null"); + } + + // select into outfile clause + @Override + public LogicalPlan visitStatementDefault(DorisParser.StatementDefaultContext ctx) { + if (ctx.outFileClause() != null && ctx.outFileClause().propertyClause() != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.outFileClause().propertyClause(); + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitStatementDefault(ctx); + } + + // export into outfile clause + @Override + public BrokerDesc visitWithRemoteStorageSystem(DorisParser.WithRemoteStorageSystemContext ctx) { + Map properties = visitPropertyItemList(ctx.brokerProperties); + encryptProperty(properties, ctx.brokerProperties.start.getStartIndex(), + ctx.brokerProperties.stop.getStopIndex()); + return super.visitWithRemoteStorageSystem(ctx); + } + + // load into outfile clause + @Override + public LogicalPlan visitLoad(DorisParser.LoadContext ctx) { + if (ctx.withRemoteStorageSystem() != null) { + Map properties = + new HashMap<>(visitPropertyItemList(ctx.withRemoteStorageSystem().brokerProperties)); + encryptProperty(properties, ctx.withRemoteStorageSystem().brokerProperties.start.getStartIndex(), + ctx.withRemoteStorageSystem().brokerProperties.stop.getStopIndex()); + } + return super.visitLoad(ctx); + } + + // set password clause + @Override + public SetVarOp visitSetPassword(DorisParser.SetPasswordContext ctx) { + encryptPassword(ctx.pwd.getStartIndex(), ctx.pwd.getStopIndex()); + return super.visitSetPassword(ctx); + } + + // set ldap password clause + @Override + public SetVarOp visitSetLdapAdminPassword(DorisParser.SetLdapAdminPasswordContext ctx) { + encryptPassword(ctx.pwd.getStartIndex(), ctx.pwd.getStopIndex()); + return super.visitSetLdapAdminPassword(ctx); + } + + // create catalog clause + @Override + public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) { + if (ctx.propertyClause() != null) { + DorisParser.PropertyClauseContext context = ctx.propertyClause(); + encryptProperty(visitPropertyClause(context), context.fileProperties.start.getStartIndex(), + context.fileProperties.stop.getStopIndex()); + } + return super.visitCreateCatalog(ctx); + } + + // create table clause + @Override + public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { + // property or ext property + if (ctx.propertyClause() != null) { + List propertyClauseContexts = ctx.propertyClause(); + for (DorisParser.PropertyClauseContext propertyClauseContext : propertyClauseContexts) { + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + } + return super.visitCreateTable(ctx); + } + + // select from tvf + @Override + public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionContext ctx) { + DorisParser.PropertyItemListContext properties = ctx.properties; + encryptProperty(visitPropertyItemList(properties), properties.start.getStartIndex(), + properties.stop.getStopIndex()); + return super.visitTableValuedFunction(ctx); + } + + private void encryptProperty(Map properties, int start, int stop) { + if (MapUtils.isNotEmpty(properties)) { + PrintableMap printableMap = new PrintableMap<>(properties, "=", + true, false, true); + indexInSqlToString.put(Pair.of(start, stop), printableMap.toString()); + } + } + + private void encryptPassword(int start, int stop) { + indexInSqlToString.put(Pair.of(start, stop), "'*XXX'"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index c273f50b04ac44..df7b0b012af614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -343,6 +343,14 @@ public LogicalPlan parseForCreateView(String sql) { return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + public LogicalPlan parseForEncryption(String sql, Map, String> indexInSqlToString) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); + LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForEncryption( + getHintMap(sql, tokenStream, DorisParser::selectHint), indexInSqlToString); + return (LogicalPlan) realLogicalPlanBuilder.visit(tree); + } + /** parseForSyncMv */ public Optional parseForSyncMv(String sql) { CommonTokenStream tokenStream = parseAllTokens(sql); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java index 22c1285a0556fd..138773108e5296 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java @@ -22,7 +22,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalCatalog; @@ -122,23 +121,5 @@ public Map getProperties() { public boolean needAuditEncryption() { return true; } - - @Override - public String toSql() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("CREATE CATALOG ").append("`").append(catalogName).append("`"); - if (!Strings.isNullOrEmpty(resourceName)) { - stringBuilder.append(" WITH RESOURCE `").append(resourceName).append("`"); - } - if (!Strings.isNullOrEmpty(comment)) { - stringBuilder.append("\nCOMMENT \"").append(comment).append("\""); - } - if (properties.size() > 0) { - stringBuilder.append("\nPROPERTIES (\n"); - stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true)); - stringBuilder.append("\n)"); - } - return stringBuilder.toString(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 281eb73513e21c..afe56721e54eec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -70,7 +70,7 @@ * create table command */ @Developing -public class CreateTableCommand extends Command implements ForwardWithSync { +public class CreateTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(CreateTableCommand.class); private final Optional ctasQuery; @@ -208,7 +208,6 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitCreateTableCommand(this, context); } - // for test public CreateTableInfo getCreateTableInfo() { return createTableInfo; } @@ -217,5 +216,10 @@ public CreateTableInfo getCreateTableInfo() { public StmtType stmtType() { return StmtType.CREATE; } + + @Override + public boolean needAuditEncryption() { + return !createTableInfo.getEngineName().equalsIgnoreCase(CreateTableInfo.ENGINE_OLAP); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index b08c6a6b54afbf..b27ff77032ddea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -70,7 +70,7 @@ * [PROPERTIES("key"="value")] * WITH BROKER 'broker_name' [( $broker_attrs)] */ -public class ExportCommand extends Command implements ForwardWithSync { +public class ExportCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final String PARALLELISM = "parallelism"; public static final String LABEL = "label"; public static final String DATA_CONSISTENCY = "data_consistency"; @@ -380,4 +380,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.EXPORT; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java index 69cbf762c2afe6..3b810d72172057 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java @@ -84,7 +84,7 @@ /** * load OLAP table data from external bulk file */ -public class LoadCommand extends Command implements ForwardWithSync { +public class LoadCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(LoadCommand.class); @@ -509,4 +509,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.LOAD; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java index df1c22ffe531fd..823d9e6d4c3829 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java @@ -17,13 +17,28 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo; + +import java.util.TreeMap; + /** * NeedAuditEncryption */ public interface NeedAuditEncryption { - boolean needAuditEncryption(); - String toSql(); - + /** + * gene encryption SQL + */ + default String geneEncryptionSQL(String sql) { + if (!needAuditEncryption()) { + return sql; + } + TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + NereidsParser parser = new NereidsParser(); + parser.parseForEncryption(sql, indexInSqlToString); + return BaseViewInfo.rewriteSql(indexInSqlToString, sql); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java index d3e280d541d210..f62ff61ccf8f0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java @@ -80,21 +80,6 @@ public boolean needAuditEncryption() { return false; } - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("SET "); - int idx = 0; - for (SetVarOp variableInfo : setVarOpList) { - if (idx != 0) { - sb.append(", "); - } - sb.append(variableInfo.toSql()); - idx++; - } - return sb.toString(); - } - @Override public void afterForwardToMaster(ConnectContext ctx) throws Exception { for (SetVarOp varOp : setVarOpList) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index c43fe04fb2c1f1..9b3a35e1760ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -217,6 +217,14 @@ public String getTableName() { return tableName; } + public String getEngineName() { + return engineName; + } + + public Map getProperties() { + return properties; + } + /** * full qualifier table name. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index c65ce4a282ddf8..55941dcf8bdca5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -44,8 +44,10 @@ import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; @@ -87,7 +89,7 @@ * InsertIntoTableCommand(Query()) * ExplainCommand(Query()) */ -public class InsertIntoTableCommand extends Command implements ForwardWithSync, Explainable { +public class InsertIntoTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable { public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class); @@ -502,6 +504,11 @@ public RedirectStatus toRedirectStatus() { } } + @Override + public boolean needAuditEncryption() { + return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); + } + /** * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated * by NereidsPlanner diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 68c71de2d9e8b8..90656fff44915a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -46,8 +46,10 @@ import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; @@ -81,7 +83,7 @@ * InsertIntoTableCommand(Query()) * ExplainCommand(Query()) */ -public class InsertOverwriteTableCommand extends Command implements ForwardWithSync, Explainable { +public class InsertOverwriteTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable { private static final Logger LOG = LogManager.getLogger(InsertOverwriteTableCommand.class); @@ -409,4 +411,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.INSERT; } + + @Override + public boolean needAuditEncryption() { + return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index 43e490c00c7a17..f5e49cdba9e690 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import com.google.common.base.Preconditions; @@ -38,7 +39,7 @@ * logicalFileSink for select into outfile */ public class LogicalFileSink extends LogicalSink - implements Sink, PropagateFuncDeps { + implements NeedAuditEncryption, Sink, PropagateFuncDeps { private final String filePath; private final String format; @@ -120,4 +121,9 @@ public String getFormat() { public Map getProperties() { return properties; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 446960f9d56415..1e87546673cb34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -71,7 +71,6 @@ public class AuditLogHelper { public static void logAuditLog(ConnectContext ctx, String origStmt, StatementBase parsedStmt, org.apache.doris.proto.Data.PQueryStatistics statistics, boolean printFuzzyVariables) { try { - origStmt = handleStmt(origStmt, parsedStmt); logAuditLogImpl(ctx, origStmt, parsedStmt, statistics, printFuzzyVariables); } catch (Throwable t) { LOG.warn("Failed to write audit log.", t); @@ -266,25 +265,21 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme auditEventBuilder.setFeIp(FrontendOptions.getLocalHostAddress()); + boolean isAnalysisErr = ctx.getState().getStateType() == MysqlStateType.ERR + && ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR; + String encryptSql = isAnalysisErr ? ctx.getState().getErrorMessage() : origStmt; // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (parsedStmt instanceof LogicalPlanAdapter) { - if (!ctx.getState().isQuery() && (parsedStmt != null - && (((LogicalPlanAdapter) parsedStmt).getLogicalPlan() instanceof NeedAuditEncryption) - && ((NeedAuditEncryption) ((LogicalPlanAdapter) parsedStmt).getLogicalPlan()) - .needAuditEncryption())) { - auditEventBuilder - .setStmt(((NeedAuditEncryption) ((LogicalPlanAdapter) parsedStmt).getLogicalPlan()).toSql()); - } else { - auditEventBuilder.setStmt(origStmt); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + if ((logicalPlan instanceof NeedAuditEncryption)) { + encryptSql = ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(origStmt); } } else { if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - auditEventBuilder.setStmt(parsedStmt.toSql()); - } else { - auditEventBuilder.setStmt(origStmt); + encryptSql = parsedStmt.toSql(); } } - + auditEventBuilder.setStmt(handleStmt(encryptSql, parsedStmt)); auditEventBuilder.setStmtType(getStmtType(parsedStmt)); if (!Env.getCurrentEnv().isMaster()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java new file mode 100644 index 00000000000000..59f86b38fbb3e3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.qe.AuditLogHelper; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class EncryptSQLTest extends ParserTestBase { + + NereidsParser parser = new NereidsParser(); + ConnectContext ctx = ConnectContext.get(); + WorkloadRuntimeStatusMgr mgr = Env.getCurrentEnv().getWorkloadRuntimeStatusMgr(); + List auditEvents = Deencapsulation.getField(mgr, "queryAuditEventList"); + + @Test + public void testEncryption() { + ctx.setDatabase("test"); + + String sql = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + String res = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + res = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"," + + " \"AWS_ENDPOINT\" = \"xxx\"," + + " \"AWS_ACCESS_KEY\" = \"abc\"," + + " \"AWS_SECRET_KEY\" = \"abc\"," + + " \"AWS_REGION\" = \"ap-beijing\"" + + ")"; + + res = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE CATALOG ctl " + + "PROPERTIES(" + + " \"type\" = \"iceberg\"," + + " \"iceberg.catalog.type\" = \"hadoop\"," + + " \"warehouse\" = \"s3://bucket/dir/key\"," + + " \"s3.endpoint\" = \"s3.us-east-1.amazonaws.com\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"" + + ")"; + + res = "CREATE CATALOG ctl " + + "PROPERTIES(" + + " \"type\" = \"iceberg\"," + + " \"iceberg.catalog.type\" = \"hadoop\"," + + " \"warehouse\" = \"s3://bucket/dir/key\"," + + " \"s3.endpoint\" = \"s3.us-east-1.amazonaws.com\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"123\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + + res = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"*XXX\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"123\"" + + ")"; + + res = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"*XXX\"" + + ")"; + parseAndCheck(sql, res); + + sql = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')"; + res = "SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX')"; + parseAndCheck(sql, res); + + sql = "SET PASSWORD FOR 'admin' = PASSWORD('123456')"; + res = "SET PASSWORD FOR 'admin' = PASSWORD('*XXX')"; + parseAndCheck(sql, res); + } + + private void parseAndCheck(String sql, String expected) { + StatementBase parsedStmt = parser.parseSQL(sql).get(0); + AuditLogHelper.logAuditLog(ctx, sql, parsedStmt, null, false); + AuditEvent event = auditEvents.get(auditEvents.size() - 1); + Assertions.assertEquals(expected, event.stmt); + } +}