From fcec55bbc430e4473e4180d0b6a4826162d1cbd0 Mon Sep 17 00:00:00 2001 From: garenshi Date: Thu, 6 Feb 2025 19:40:30 +0800 Subject: [PATCH 01/12] [Feature](nereids) support encrypt sql for audit log --- .../org/apache/doris/nereids/DorisParser.g4 | 10 +- .../LogicalPlanBuilderForEncryption.java | 113 ++++++++++++++++ .../doris/nereids/parser/NereidsParser.java | 8 ++ .../plans/commands/CreateCatalogCommand.java | 19 --- .../trees/plans/commands/ExportCommand.java | 7 +- .../trees/plans/commands/LoadCommand.java | 7 +- .../plans/commands/NeedAuditEncryption.java | 21 ++- .../plans/commands/SetOptionsCommand.java | 15 --- .../trees/plans/logical/LogicalFileSink.java | 8 +- .../org/apache/doris/qe/AuditLogHelper.java | 19 +-- .../nereids_p0/encrypt/test_encrypt_sql.out | 9 ++ .../encrypt/test_encrypt_sql.groovy | 122 ++++++++++++++++++ 12 files changed, 300 insertions(+), 58 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java create mode 100644 regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out create mode 100644 regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 8c21461f1e68b7..ff15eb41a03e5d 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -256,7 +256,7 @@ supportedShowStatement | SHOW DELETE ((FROM | IN) database=multipartIdentifier)? #showDelete | SHOW ALL? GRANTS #showGrants | SHOW GRANTS FOR userIdentify #showGrantsForUser - | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)? #showSyncJob + | SHOW SYNC JOB ((FROM | IN) database=multipartIdentifier)? #showSyncJob | SHOW LOAD PROFILE loadIdPath=STRING_LITERAL? limitClause? #showLoadProfile | SHOW CREATE REPOSITORY FOR identifier #showCreateRepository | SHOW VIEW @@ -849,10 +849,10 @@ optionWithoutType | (CHAR SET | CHARSET) (charsetName=identifierOrText | DEFAULT) #setCharset | NAMES (charsetName=identifierOrText | DEFAULT) (COLLATE collateName=identifierOrText | DEFAULT)? #setCollate - | PASSWORD (FOR userIdentify)? EQ (STRING_LITERAL - | (isPlain=PASSWORD LEFT_PAREN STRING_LITERAL RIGHT_PAREN)) #setPassword - | LDAP_ADMIN_PASSWORD EQ (STRING_LITERAL - | (PASSWORD LEFT_PAREN STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword + | PASSWORD (FOR userIdentify)? EQ (pwd=STRING_LITERAL + | (isPlain=PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setPassword + | LDAP_ADMIN_PASSWORD EQ (pwd=STRING_LITERAL + | (PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword | variable #setVariableWithoutType ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java new file mode 100644 index 00000000000000..b602b3cf01ede5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.common.Pair; +import org.apache.doris.common.util.PrintableMap; +import org.apache.doris.nereids.DorisParser; +import org.apache.doris.nereids.trees.plans.commands.info.SetVarOp; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.apache.commons.collections.MapUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/**LogicalPlanBuilderForEncryption*/ +public class LogicalPlanBuilderForEncryption extends LogicalPlanBuilder { + private final Map, String> indexInSqlToString; + + public LogicalPlanBuilderForEncryption(Map selectHintMap, + Map, String> indexInSqlToString) { + super(selectHintMap); + this.indexInSqlToString = Objects.requireNonNull(indexInSqlToString, "indexInSqlToString is null"); + } + + // select into outfile clause + @Override + public LogicalPlan visitStatementDefault(DorisParser.StatementDefaultContext ctx) { + if (ctx.outFileClause() != null && ctx.outFileClause().propertyClause() != null) { + DorisParser.PropertyClauseContext propertyClauseContext = ctx.outFileClause().propertyClause(); + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } + return super.visitStatementDefault(ctx); + } + + // export into outfile clause + @Override + public BrokerDesc visitWithRemoteStorageSystem(DorisParser.WithRemoteStorageSystemContext ctx) { + Map properties = visitPropertyItemList(ctx.brokerProperties); + encryptProperty(properties, ctx.brokerProperties.start.getStartIndex(), + ctx.brokerProperties.stop.getStopIndex()); + return super.visitWithRemoteStorageSystem(ctx); + } + + // load into outfile clause + @Override + public LogicalPlan visitLoad(DorisParser.LoadContext ctx) { + if (ctx.withRemoteStorageSystem() != null) { + Map properties = + new HashMap<>(visitPropertyItemList(ctx.withRemoteStorageSystem().brokerProperties)); + encryptProperty(properties, ctx.withRemoteStorageSystem().brokerProperties.start.getStartIndex(), + ctx.withRemoteStorageSystem().brokerProperties.stop.getStopIndex()); + } + return super.visitLoad(ctx); + } + + // set password clause + @Override + public SetVarOp visitSetPassword(DorisParser.SetPasswordContext ctx) { + encryptPassword(ctx.pwd.getStartIndex(), ctx.pwd.getStopIndex()); + return super.visitSetPassword(ctx); + } + + // set ldap password clause + @Override + public SetVarOp visitSetLdapAdminPassword(DorisParser.SetLdapAdminPasswordContext ctx) { + encryptPassword(ctx.pwd.getStartIndex(), ctx.pwd.getStopIndex()); + return super.visitSetLdapAdminPassword(ctx); + } + + // create catalog clause + @Override + public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) { + if (ctx.propertyClause() != null) { + DorisParser.PropertyClauseContext context = ctx.propertyClause(); + encryptProperty(visitPropertyClause(context), context.fileProperties.start.getStartIndex(), + context.fileProperties.stop.getStopIndex()); + } + return super.visitCreateCatalog(ctx); + } + + private void encryptProperty(Map properties, int start, int stop) { + if (MapUtils.isNotEmpty(properties)) { + PrintableMap printableMap = new PrintableMap<>(properties, " = ", + true, false, true); + indexInSqlToString.put(Pair.of(start, stop), printableMap.toString()); + } + } + + private void encryptPassword(int start, int stop) { + indexInSqlToString.put(Pair.of(start, stop), "'*XXX'"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index c273f50b04ac44..df7b0b012af614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -343,6 +343,14 @@ public LogicalPlan parseForCreateView(String sql) { return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } + public LogicalPlan parseForEncryption(String sql, Map, String> indexInSqlToString) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); + LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForEncryption( + getHintMap(sql, tokenStream, DorisParser::selectHint), indexInSqlToString); + return (LogicalPlan) realLogicalPlanBuilder.visit(tree); + } + /** parseForSyncMv */ public Optional parseForSyncMv(String sql) { CommonTokenStream tokenStream = parseAllTokens(sql); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java index 22c1285a0556fd..138773108e5296 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateCatalogCommand.java @@ -22,7 +22,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalCatalog; @@ -122,23 +121,5 @@ public Map getProperties() { public boolean needAuditEncryption() { return true; } - - @Override - public String toSql() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("CREATE CATALOG ").append("`").append(catalogName).append("`"); - if (!Strings.isNullOrEmpty(resourceName)) { - stringBuilder.append(" WITH RESOURCE `").append(resourceName).append("`"); - } - if (!Strings.isNullOrEmpty(comment)) { - stringBuilder.append("\nCOMMENT \"").append(comment).append("\""); - } - if (properties.size() > 0) { - stringBuilder.append("\nPROPERTIES (\n"); - stringBuilder.append(new PrintableMap<>(properties, "=", true, true, true)); - stringBuilder.append("\n)"); - } - return stringBuilder.toString(); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index b08c6a6b54afbf..b27ff77032ddea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -70,7 +70,7 @@ * [PROPERTIES("key"="value")] * WITH BROKER 'broker_name' [( $broker_attrs)] */ -public class ExportCommand extends Command implements ForwardWithSync { +public class ExportCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final String PARALLELISM = "parallelism"; public static final String LABEL = "label"; public static final String DATA_CONSISTENCY = "data_consistency"; @@ -380,4 +380,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.EXPORT; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java index 69cbf762c2afe6..3b810d72172057 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java @@ -84,7 +84,7 @@ /** * load OLAP table data from external bulk file */ -public class LoadCommand extends Command implements ForwardWithSync { +public class LoadCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(LoadCommand.class); @@ -509,4 +509,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.LOAD; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java index df1c22ffe531fd..823d9e6d4c3829 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java @@ -17,13 +17,28 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo; + +import java.util.TreeMap; + /** * NeedAuditEncryption */ public interface NeedAuditEncryption { - boolean needAuditEncryption(); - String toSql(); - + /** + * gene encryption SQL + */ + default String geneEncryptionSQL(String sql) { + if (!needAuditEncryption()) { + return sql; + } + TreeMap, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>()); + NereidsParser parser = new NereidsParser(); + parser.parseForEncryption(sql, indexInSqlToString); + return BaseViewInfo.rewriteSql(indexInSqlToString, sql); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java index d3e280d541d210..f62ff61ccf8f0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/SetOptionsCommand.java @@ -80,21 +80,6 @@ public boolean needAuditEncryption() { return false; } - @Override - public String toSql() { - StringBuilder sb = new StringBuilder(); - sb.append("SET "); - int idx = 0; - for (SetVarOp variableInfo : setVarOpList) { - if (idx != 0) { - sb.append(", "); - } - sb.append(variableInfo.toSql()); - idx++; - } - return sb.toString(); - } - @Override public void afterForwardToMaster(ConnectContext ctx) throws Exception { for (SetVarOp varOp : setVarOpList) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java index 43e490c00c7a17..f5e49cdba9e690 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import com.google.common.base.Preconditions; @@ -38,7 +39,7 @@ * logicalFileSink for select into outfile */ public class LogicalFileSink extends LogicalSink - implements Sink, PropagateFuncDeps { + implements NeedAuditEncryption, Sink, PropagateFuncDeps { private final String filePath; private final String format; @@ -120,4 +121,9 @@ public String getFormat() { public Map getProperties() { return properties; } + + @Override + public boolean needAuditEncryption() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 446960f9d56415..17b27a409ce8a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -71,7 +71,6 @@ public class AuditLogHelper { public static void logAuditLog(ConnectContext ctx, String origStmt, StatementBase parsedStmt, org.apache.doris.proto.Data.PQueryStatistics statistics, boolean printFuzzyVariables) { try { - origStmt = handleStmt(origStmt, parsedStmt); logAuditLogImpl(ctx, origStmt, parsedStmt, statistics, printFuzzyVariables); } catch (Throwable t) { LOG.warn("Failed to write audit log.", t); @@ -266,25 +265,19 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme auditEventBuilder.setFeIp(FrontendOptions.getLocalHostAddress()); + String encryptSql = origStmt; // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (parsedStmt instanceof LogicalPlanAdapter) { - if (!ctx.getState().isQuery() && (parsedStmt != null - && (((LogicalPlanAdapter) parsedStmt).getLogicalPlan() instanceof NeedAuditEncryption) - && ((NeedAuditEncryption) ((LogicalPlanAdapter) parsedStmt).getLogicalPlan()) - .needAuditEncryption())) { - auditEventBuilder - .setStmt(((NeedAuditEncryption) ((LogicalPlanAdapter) parsedStmt).getLogicalPlan()).toSql()); - } else { - auditEventBuilder.setStmt(origStmt); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + if ((logicalPlan instanceof NeedAuditEncryption)) { + encryptSql = ((NeedAuditEncryption) logicalPlan).geneEncryptionSQL(origStmt); } } else { if (!ctx.getState().isQuery() && (parsedStmt != null && parsedStmt.needAuditEncryption())) { - auditEventBuilder.setStmt(parsedStmt.toSql()); - } else { - auditEventBuilder.setStmt(origStmt); + encryptSql = parsedStmt.toSql(); } } - + auditEventBuilder.setStmt(handleStmt(encryptSql, parsedStmt)); auditEventBuilder.setStmtType(getStmtType(parsedStmt)); if (!Env.getCurrentEnv().isMaster()) { diff --git a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out new file mode 100644 index 00000000000000..19aacadeda5d28 --- /dev/null +++ b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +CREATE CATALOG test_encrypt_sql_table \\n PROPERTIES( \\n "type" = "iceberg", "iceberg.catalog.type" = "hadoop", "warehouse" = "s3://bucket/dir/key", "s3.endpoint" = "s3.us-east-1.amazonaws.com", "s3.access_key" = "ak", "s3.secret_key" = "*XXX"\\n );\\n +EXPORT TABLE test_encrypt_sql_db.test_encrypt_sql_table TO "s3://abc/aaa"\\n PROPERTIES(\\n "format" = "csv",\\n "max_file_size" = "2048MB"\\n )\\n WITH s3 (\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n +LOAD LABEL test_load_s3_orc_encrypt\\n (\\n DATA INFILE("s3://abc/aaa")\\n INTO TABLE test_encrypt_sql_table\\n FORMAT AS "ORC"\\n )\\n WITH S3\\n (\\n "provider" = "S3"\\n )\\n +SELECT * FROM test_encrypt_sql_db.test_encrypt_sql_table\\n INTO OUTFILE "s3://abc/aaa"\\n FORMAT AS ORC\\n PROPERTIES(\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n +SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX') +SET PASSWORD FOR 'test_encrypt_sql_user' = PASSWORD('*XXX') + diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy new file mode 100644 index 00000000000000..f1c61d9168b278 --- /dev/null +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_encrypt_sql") { + + def dbName = "test_encrypt_sql_db" + def tableName = "test_encrypt_sql_table" + + sql """drop database if exists ${dbName}""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """CREATE TABLE `${tableName}` ( + `year` int NULL, + `country` text NULL, + `product` text NULL, + `profit` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`year`) + DISTRIBUTED BY HASH(`year`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + def user = "test_encrypt_sql_user" + def pwd = "123456" + + sql "drop user if exists ${user}" + sql "create user ${user} IDENTIFIED BY '${pwd}'" + sql "grant ADMIN_PRIV on *.*.* to ${user}" + + connect(user, "${pwd}", context.config.jdbcUrl) { + try { + sql """EXPORT TABLE ${dbName}.${tableName} TO "s3://abc/aaa" + PROPERTIES( + "format" = "csv", + "max_file_size" = "2048MB" + ) + WITH s3 ( + "s3.endpoint" = "xxx", + "s3.region" = "ap-beijing", + "s3.secret_key"="abc", + "s3.access_key" = "abc" + ); + """ + } catch (Exception e) {} + + try { + sql """SELECT * FROM ${dbName}.${tableName} + INTO OUTFILE "s3://abc/aaa" + FORMAT AS ORC + PROPERTIES( + "s3.endpoint" = "xxx", + "s3.region" = "ap-beijing", + "s3.secret_key"="abc", + "s3.access_key" = "abc" + ); + """ + } catch (Exception e) {} + + try { + sql """LOAD LABEL test_load_s3_orc_encrypt + ( + DATA INFILE("s3://abc/aaa") + INTO TABLE ${tableName} + FORMAT AS "ORC" + ) + WITH S3 + ( + "provider" = "S3", + "AWS_ENDPOINT" = "xxx", + "AWS_ACCESS_KEY" = "abc", + "AWS_SECRET_KEY" = "abc", + "AWS_REGION" = "ap-beijing" + ) + """ + } catch (Exception e) {} + + try { + sql"""CREATE CATALOG ${tableName} + PROPERTIES( + 'type'='iceberg', + 'iceberg.catalog.type' = 'hadoop', + 'warehouse' = 's3://bucket/dir/key', + 's3.endpoint' = 's3.us-east-1.amazonaws.com', + 's3.access_key' = 'ak', + 's3.secret_key' = 'sk' + ); + """ + } catch (Exception e) {} + + sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')" + + sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" + } + + Thread.sleep(3000) + sql "call flush_audit_log()" + + def dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd") + def date = dateFormat.format(new Date()) + qt_sql "select stmt from __internal_schema.audit_log where user = '${user}' and time > '${date}' and (stmt like '%${tableName}%' or stmt like '%PASSWORD%' order by stmt" + + sql "drop table ${tableName}" + sql "drop database ${dbName}" + sql "drop user ${user}" +} From 6632964c3ccfd7965aa49a3720be89bf3414487f Mon Sep 17 00:00:00 2001 From: garenshi Date: Fri, 7 Feb 2025 16:02:29 +0800 Subject: [PATCH 02/12] 1 --- .../suites/nereids_p0/encrypt/test_encrypt_sql.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index f1c61d9168b278..5e9b26e9214b8b 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -109,7 +109,7 @@ suite("test_encrypt_sql") { sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" } - Thread.sleep(3000) + Thread.sleep(15000) sql "call flush_audit_log()" def dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd") From 1a345302808087b14a686c75a50ce34e6d09d851 Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 11:14:01 +0800 Subject: [PATCH 03/12] 1 --- .../org/apache/doris/nereids/DorisParser.g4 | 4 ++-- .../LogicalPlanBuilderForEncryption.java | 11 ++++++++++ .../plans/commands/CreateTableCommand.java | 8 ++++++-- .../plans/commands/info/CreateTableInfo.java | 8 ++++++++ .../encrypt/test_encrypt_sql.groovy | 20 +++++++++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index ff15eb41a03e5d..ce1f0fb7c44384 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -850,9 +850,9 @@ optionWithoutType | NAMES (charsetName=identifierOrText | DEFAULT) (COLLATE collateName=identifierOrText | DEFAULT)? #setCollate | PASSWORD (FOR userIdentify)? EQ (pwd=STRING_LITERAL - | (isPlain=PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setPassword + | (isPlain=PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setPassword | LDAP_ADMIN_PASSWORD EQ (pwd=STRING_LITERAL - | (PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword + | (PASSWORD LEFT_PAREN pwd=STRING_LITERAL RIGHT_PAREN)) #setLdapAdminPassword | variable #setVariableWithoutType ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index b602b3cf01ede5..d60f40141fe2a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -99,6 +99,17 @@ public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) { return super.visitCreateCatalog(ctx); } + // create table clause + @Override + public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { + DorisParser.PropertyClauseContext context = ctx.properties; + if (context != null) { + encryptProperty(visitPropertyClause(context), context.fileProperties.start.getStartIndex(), + context.fileProperties.stop.getStopIndex()); + } + return super.visitCreateTable(ctx); + } + private void encryptProperty(Map properties, int start, int stop) { if (MapUtils.isNotEmpty(properties)) { PrintableMap printableMap = new PrintableMap<>(properties, " = ", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 281eb73513e21c..afe56721e54eec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -70,7 +70,7 @@ * create table command */ @Developing -public class CreateTableCommand extends Command implements ForwardWithSync { +public class CreateTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync { public static final Logger LOG = LogManager.getLogger(CreateTableCommand.class); private final Optional ctasQuery; @@ -208,7 +208,6 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitCreateTableCommand(this, context); } - // for test public CreateTableInfo getCreateTableInfo() { return createTableInfo; } @@ -217,5 +216,10 @@ public CreateTableInfo getCreateTableInfo() { public StmtType stmtType() { return StmtType.CREATE; } + + @Override + public boolean needAuditEncryption() { + return !createTableInfo.getEngineName().equalsIgnoreCase(CreateTableInfo.ENGINE_OLAP); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index c43fe04fb2c1f1..9b3a35e1760ff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -217,6 +217,14 @@ public String getTableName() { return tableName; } + public String getEngineName() { + return engineName; + } + + public Map getProperties() { + return properties; + } + /** * full qualifier table name. */ diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index 5e9b26e9214b8b..cd1b95bc345e3c 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -104,6 +104,26 @@ suite("test_encrypt_sql") { """ } catch (Exception e) {} + try { + sql"""CREATE TABLE mysql_tbl ( + k1 DATE, + k2 INT, + k3 SMALLINT, + k4 VARCHAR(2048), + k5 DATETIME + ) + ENGINE=mysql + PROPERTIES( + 'host' = '127.0.0.1', + 'port' = '8234', + 'user' = 'abc', + 'password' = '123', + 'database' = 'mysql_db', + 'table' = 'mysql_table' + ); + """ + } catch (Exception e) {} + sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')" sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" From 8422705b977dc2371042bd60398b05c7a57228fe Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 11:55:42 +0800 Subject: [PATCH 04/12] 1 --- .../LogicalPlanBuilderForEncryption.java | 13 +++++++--- .../encrypt/test_encrypt_sql.groovy | 25 ++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index d60f40141fe2a6..a8bf644ad7f5fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -28,6 +28,7 @@ import org.apache.commons.collections.MapUtils; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -102,10 +103,14 @@ public LogicalPlan visitCreateCatalog(DorisParser.CreateCatalogContext ctx) { // create table clause @Override public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { - DorisParser.PropertyClauseContext context = ctx.properties; - if (context != null) { - encryptProperty(visitPropertyClause(context), context.fileProperties.start.getStartIndex(), - context.fileProperties.stop.getStopIndex()); + // property or ext property + if (ctx.propertyClause() != null) { + List propertyClauseContexts = ctx.propertyClause(); + for (DorisParser.PropertyClauseContext propertyClauseContext : propertyClauseContexts) { + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } } return super.visitCreateTable(ctx); } diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index cd1b95bc345e3c..97472c19524e54 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -103,9 +103,9 @@ suite("test_encrypt_sql") { ); """ } catch (Exception e) {} - + // for jdbc table or es table try { - sql"""CREATE TABLE mysql_tbl ( + sql"""CREATE TABLE mysql_${tableName} ( k1 DATE, k2 INT, k3 SMALLINT, @@ -124,6 +124,25 @@ suite("test_encrypt_sql") { """ } catch (Exception e) {} + try { + sql"""CREATE EXTERNAL TABLE broker_${tableName}( + k1 tinyint, + k2 smallint, + k3 int, + k4 bigint) + ENGINE=broker + PROPERTIES( + "broker_name" = "hdfs", + "path" = "hdfs://xxxxxxx/qe/a.txt", + "column_separator" = "\\t" + ) + BROKER PROPERTIES( + "username"="root", + "password"="123" + ); + """ + } catch (Exception e) {} + sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')" sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" @@ -134,7 +153,7 @@ suite("test_encrypt_sql") { def dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd") def date = dateFormat.format(new Date()) - qt_sql "select stmt from __internal_schema.audit_log where user = '${user}' and time > '${date}' and (stmt like '%${tableName}%' or stmt like '%PASSWORD%' order by stmt" + qt_sql "select stmt from __internal_schema.audit_log where user = '${user}' and time > '${date}' and (stmt like '%${tableName}%' or stmt like '%PASSWORD%') order by stmt" sql "drop table ${tableName}" sql "drop database ${dbName}" From 6345216f4aceef258a1c51047ef42088c2e7c40a Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 14:16:47 +0800 Subject: [PATCH 05/12] 1 --- .../data/nereids_p0/encrypt/test_encrypt_sql.out | 10 ++++++---- .../suites/nereids_p0/encrypt/test_encrypt_sql.groovy | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out index 19aacadeda5d28..a09691826df3be 100644 --- a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out +++ b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out @@ -1,9 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -CREATE CATALOG test_encrypt_sql_table \\n PROPERTIES( \\n "type" = "iceberg", "iceberg.catalog.type" = "hadoop", "warehouse" = "s3://bucket/dir/key", "s3.endpoint" = "s3.us-east-1.amazonaws.com", "s3.access_key" = "ak", "s3.secret_key" = "*XXX"\\n );\\n -EXPORT TABLE test_encrypt_sql_db.test_encrypt_sql_table TO "s3://abc/aaa"\\n PROPERTIES(\\n "format" = "csv",\\n "max_file_size" = "2048MB"\\n )\\n WITH s3 (\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n -LOAD LABEL test_load_s3_orc_encrypt\\n (\\n DATA INFILE("s3://abc/aaa")\\n INTO TABLE test_encrypt_sql_table\\n FORMAT AS "ORC"\\n )\\n WITH S3\\n (\\n "provider" = "S3"\\n )\\n -SELECT * FROM test_encrypt_sql_db.test_encrypt_sql_table\\n INTO OUTFILE "s3://abc/aaa"\\n FORMAT AS ORC\\n PROPERTIES(\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n +CREATE CATALOG test_encrypt_sql_table \\n PROPERTIES( \\n "type" = "iceberg", "iceberg.catalog.type" = "hadoop", "warehouse" = "s3://bucket/dir/key", "s3.endpoint" = "s3.us-east-1.amazonaws.com", "s3.access_key" = "ak", "s3.secret_key" = "*XXX"\\n );\\n +CREATE EXTERNAL TABLE broker_test_encrypt_sql_table(\\n k1 tinyint, \\n k2 smallint, \\n k3 int, \\n k4 bigint) \\n ENGINE=broker \\n PROPERTIES(\\n "broker_name" = "hdfs", "path" = "hdfs://xxxxxxx/qe/a.txt", "column_separator" = "\\t"\\n ) \\n BROKER PROPERTIES(\\n "username" = "root", "password" = "*XXX"\\n );\\n +CREATE TABLE mysql_test_encrypt_sql_table (\\n k1 DATE, \\n k2 INT, \\n k3 SMALLINT, \\n k4 VARCHAR(2048), \\n k5 DATETIME\\n )\\n ENGINE=mysql\\n PROPERTIES(\\n "host" = "127.0.0.1", "port" = "8234", "user" = "abc", "password" = "*XXX", "database" = "mysql_db", "table" = "mysql_table"\\n );\\n +EXPORT TABLE test_encrypt_sql_db.test_encrypt_sql_table TO "s3://abc/aaa"\\n PROPERTIES(\\n "format" = "csv",\\n "max_file_size" = "2048MB"\\n )\\n WITH s3 (\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n +LOAD LABEL test_load_s3_orc_encrypt\\n (\\n DATA INFILE("s3://abc/aaa")\\n INTO TABLE test_encrypt_sql_table\\n FORMAT AS "ORC"\\n )\\n WITH S3\\n (\\n "provider" = "S3"\\n )\\n +SELECT * FROM test_encrypt_sql_db.test_encrypt_sql_table\\n INTO OUTFILE "s3://abc/aaa"\\n FORMAT AS ORC\\n PROPERTIES(\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX') SET PASSWORD FOR 'test_encrypt_sql_user' = PASSWORD('*XXX') diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index 97472c19524e54..bcd19901807c63 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -43,6 +43,7 @@ suite("test_encrypt_sql") { sql "drop user if exists ${user}" sql "create user ${user} IDENTIFIED BY '${pwd}'" sql "grant ADMIN_PRIV on *.*.* to ${user}" + sql "admin set frontend config('enable_nereids_load'='true')" connect(user, "${pwd}", context.config.jdbcUrl) { try { From 238734be2fc5fc23a453c9afd58eb00b5c2c4b9d Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 16:44:31 +0800 Subject: [PATCH 06/12] 1 --- .../nereids/analyzer/UnboundResultSink.java | 8 ++++- .../LogicalPlanBuilderForEncryption.java | 9 ++++++ .../insert/InsertIntoTableCommand.java | 9 +++++- .../insert/InsertOverwriteTableCommand.java | 9 +++++- .../encrypt/test_encrypt_sql.groovy | 31 +++++++++++++++++++ 5 files changed, 63 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java index d57e518824d3aa..e6fd3eedf0df88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; @@ -41,7 +42,7 @@ * unbound result sink */ public class UnboundResultSink extends LogicalSink - implements Unbound, Sink, BlockFuncDepsPropagation { + implements NeedAuditEncryption, Unbound, Sink, BlockFuncDepsPropagation { public UnboundResultSink(CHILD_TYPE child) { super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child); @@ -94,4 +95,9 @@ public String toString() { public StmtType stmtType() { return StmtType.SELECT; } + + @Override + public boolean needAuditEncryption() { + return anyMatch(node -> node instanceof UnboundTVFRelation); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index a8bf644ad7f5fe..e88b6625c150c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -115,6 +115,15 @@ public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { return super.visitCreateTable(ctx); } + // select from tvf + @Override + public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionContext ctx) { + DorisParser.PropertyItemListContext properties = ctx.properties; + encryptProperty(visitPropertyItemList(properties), properties.start.getStartIndex(), + properties.stop.getStopIndex()); + return super.visitTableValuedFunction(ctx); + } + private void encryptProperty(Map properties, int start, int stop) { if (MapUtils.isNotEmpty(properties)) { PrintableMap printableMap = new PrintableMap<>(properties, " = ", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index c65ce4a282ddf8..e767417f496190 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -46,6 +47,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; @@ -87,7 +89,7 @@ * InsertIntoTableCommand(Query()) * ExplainCommand(Query()) */ -public class InsertIntoTableCommand extends Command implements ForwardWithSync, Explainable { +public class InsertIntoTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable { public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class); @@ -502,6 +504,11 @@ public RedirectStatus toRedirectStatus() { } } + @Override + public boolean needAuditEncryption() { + return originLogicalQuery.anyMatch(node -> node instanceof UnboundTVFRelation); + } + /** * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated * by NereidsPlanner diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 68c71de2d9e8b8..4e95a48c88120b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; +import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -48,6 +49,7 @@ import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; @@ -81,7 +83,7 @@ * InsertIntoTableCommand(Query()) * ExplainCommand(Query()) */ -public class InsertOverwriteTableCommand extends Command implements ForwardWithSync, Explainable { +public class InsertOverwriteTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable { private static final Logger LOG = LogManager.getLogger(InsertOverwriteTableCommand.class); @@ -409,4 +411,9 @@ public R accept(PlanVisitor visitor, C context) { public StmtType stmtType() { return StmtType.INSERT; } + + @Override + public boolean needAuditEncryption() { + return originLogicalQuery.anyMatch(node -> node instanceof UnboundTVFRelation); + } } diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index bcd19901807c63..e661e948ea5d87 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -144,6 +144,37 @@ suite("test_encrypt_sql") { """ } catch (Exception e) {} + try { + sql"""INSERT INTO test_s3load + SELECT * FROM S3_${tableName} ( + "uri" = "s3://your_bucket_name/s3load_example.csv" + "format" = "csv" + "provider" = "OSS" + "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com" + "s3.region" = "oss-cn-hangzhou" + "s3.access_key" = "abc" + "s3.secret_key" = "abc" + "column_separator" = "," + "csv_schema" = "user_id:int;name:string;age:int" + ); + """ + } catch (Exception e) {} + + try { + sql"""SELECT * FROM S3_${tableName} ( + "uri" = "s3://your_bucket_name/s3load_example.csv" + "format" = "csv" + "provider" = "OSS" + "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com" + "s3.region" = "oss-cn-hangzhou" + "s3.access_key" = "abc" + "s3.secret_key" = "abc" + "column_separator" = "," + "csv_schema" = "user_id:int;name:string;age:int" + ); + """ + } catch (Exception e) {} + sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')" sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" From 071d4d8c5c01678a6b7b7d97137b44b492d94251 Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 17:36:32 +0800 Subject: [PATCH 07/12] 1 --- .../encrypt/test_encrypt_sql.groovy | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy index e661e948ea5d87..7327ab87d228e2 100644 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy @@ -147,14 +147,14 @@ suite("test_encrypt_sql") { try { sql"""INSERT INTO test_s3load SELECT * FROM S3_${tableName} ( - "uri" = "s3://your_bucket_name/s3load_example.csv" - "format" = "csv" - "provider" = "OSS" - "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com" - "s3.region" = "oss-cn-hangzhou" - "s3.access_key" = "abc" - "s3.secret_key" = "abc" - "column_separator" = "," + "uri" = "s3://your_bucket_name/s3load_example.csv", + "format" = "csv", + "provider" = "OSS", + "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", + "s3.region" = "oss-cn-hangzhou", + "s3.access_key" = "abc", + "s3.secret_key" = "abc", + "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int" ); """ @@ -162,14 +162,14 @@ suite("test_encrypt_sql") { try { sql"""SELECT * FROM S3_${tableName} ( - "uri" = "s3://your_bucket_name/s3load_example.csv" - "format" = "csv" - "provider" = "OSS" - "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com" - "s3.region" = "oss-cn-hangzhou" - "s3.access_key" = "abc" - "s3.secret_key" = "abc" - "column_separator" = "," + "uri" = "s3://your_bucket_name/s3load_example.csv", + "format" = "csv", + "provider" = "OSS", + "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", + "s3.region" = "oss-cn-hangzhou", + "s3.access_key" = "abc", + "s3.secret_key" = "abc", + "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int" ); """ From eb73f2003b0b3f104202c0091636ac726a1c3713 Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 12 Feb 2025 17:43:17 +0800 Subject: [PATCH 08/12] 1 --- regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out | 2 ++ 1 file changed, 2 insertions(+) diff --git a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out index a09691826df3be..c477d9e855dfb8 100644 --- a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out +++ b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out @@ -4,7 +4,9 @@ CREATE CATALOG test_encrypt_sql_table \\n PROPERTIES( \\n CREATE EXTERNAL TABLE broker_test_encrypt_sql_table(\\n k1 tinyint, \\n k2 smallint, \\n k3 int, \\n k4 bigint) \\n ENGINE=broker \\n PROPERTIES(\\n "broker_name" = "hdfs", "path" = "hdfs://xxxxxxx/qe/a.txt", "column_separator" = "\\t"\\n ) \\n BROKER PROPERTIES(\\n "username" = "root", "password" = "*XXX"\\n );\\n CREATE TABLE mysql_test_encrypt_sql_table (\\n k1 DATE, \\n k2 INT, \\n k3 SMALLINT, \\n k4 VARCHAR(2048), \\n k5 DATETIME\\n )\\n ENGINE=mysql\\n PROPERTIES(\\n "host" = "127.0.0.1", "port" = "8234", "user" = "abc", "password" = "*XXX", "database" = "mysql_db", "table" = "mysql_table"\\n );\\n EXPORT TABLE test_encrypt_sql_db.test_encrypt_sql_table TO "s3://abc/aaa"\\n PROPERTIES(\\n "format" = "csv",\\n "max_file_size" = "2048MB"\\n )\\n WITH s3 (\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n +INSERT INTO test_s3load\\n SELECT * FROM S3_test_encrypt_sql_table (\\n "uri" = "s3://your_bucket_name/s3load_example.csv", "format" = "csv", "provider" = "OSS", "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", "s3.region" = "oss-cn-hangzhou", "s3.access_key" = "abc", "s3.secret_key" = "*XXX", "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int"\\n );\\n LOAD LABEL test_load_s3_orc_encrypt\\n (\\n DATA INFILE("s3://abc/aaa")\\n INTO TABLE test_encrypt_sql_table\\n FORMAT AS "ORC"\\n )\\n WITH S3\\n (\\n "provider" = "S3"\\n )\\n +SELECT * FROM S3_test_encrypt_sql_table (\\n "uri" = "s3://your_bucket_name/s3load_example.csv", "format" = "csv", "provider" = "OSS", "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", "s3.region" = "oss-cn-hangzhou", "s3.access_key" = "abc", "s3.secret_key" = "*XXX", "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int"\\n );\\n SELECT * FROM test_encrypt_sql_db.test_encrypt_sql_table\\n INTO OUTFILE "s3://abc/aaa"\\n FORMAT AS ORC\\n PROPERTIES(\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX') SET PASSWORD FOR 'test_encrypt_sql_user' = PASSWORD('*XXX') From 85ca401380e9ddf98e8160be4275c63f43a7e23f Mon Sep 17 00:00:00 2001 From: garenshi Date: Fri, 14 Feb 2025 18:58:19 +0800 Subject: [PATCH 09/12] 1 --- .../LogicalPlanBuilderForEncryption.java | 2 +- .../doris/nereids/parser/EncryptSQLTest.java | 267 ++++++++++++++++++ .../nereids_p0/encrypt/test_encrypt_sql.out | 13 - .../encrypt/test_encrypt_sql.groovy | 193 ------------- 4 files changed, 268 insertions(+), 207 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java delete mode 100644 regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out delete mode 100644 regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index e88b6625c150c8..809ab277a18b11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -126,7 +126,7 @@ public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionConte private void encryptProperty(Map properties, int start, int stop) { if (MapUtils.isNotEmpty(properties)) { - PrintableMap printableMap = new PrintableMap<>(properties, " = ", + PrintableMap printableMap = new PrintableMap<>(properties, "=", true, false, true); indexInSqlToString.put(Pair.of(start, stop), printableMap.toString()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java new file mode 100644 index 00000000000000..59f86b38fbb3e3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java @@ -0,0 +1,267 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.qe.AuditLogHelper; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class EncryptSQLTest extends ParserTestBase { + + NereidsParser parser = new NereidsParser(); + ConnectContext ctx = ConnectContext.get(); + WorkloadRuntimeStatusMgr mgr = Env.getCurrentEnv().getWorkloadRuntimeStatusMgr(); + List auditEvents = Deencapsulation.getField(mgr, "queryAuditEventList"); + + @Test + public void testEncryption() { + ctx.setDatabase("test"); + + String sql = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + String res = "EXPORT TABLE export_table TO \"s3://abc/aaa\" " + + "PROPERTIES(" + + " \"format\" = \"csv\"," + + " \"max_file_size\" = \"2048MB\"" + + ")" + + "WITH s3 (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + + res = "SELECT * FROM tbl " + + " INTO OUTFILE \"s3://abc/aaa\"" + + " FORMAT AS ORC" + + " PROPERTIES (" + + " \"s3.endpoint\" = \"abc\"," + + " \"s3.region\" = \"ap-beijing\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"s3.access_key\" = \"abc\"" + + ")"; + parseAndCheck(sql, res); + + sql = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"," + + " \"AWS_ENDPOINT\" = \"xxx\"," + + " \"AWS_ACCESS_KEY\" = \"abc\"," + + " \"AWS_SECRET_KEY\" = \"abc\"," + + " \"AWS_REGION\" = \"ap-beijing\"" + + ")"; + + res = "LOAD LABEL test_load_s3_orc_encrypt(" + + " DATA INFILE(\"s3://abc/aaa\")" + + " INTO TABLE tbl" + + " FORMAT AS \"ORC\"" + + ")" + + "WITH S3(" + + " \"provider\" = \"S3\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE CATALOG ctl " + + "PROPERTIES(" + + " \"type\" = \"iceberg\"," + + " \"iceberg.catalog.type\" = \"hadoop\"," + + " \"warehouse\" = \"s3://bucket/dir/key\"," + + " \"s3.endpoint\" = \"s3.us-east-1.amazonaws.com\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"" + + ")"; + + res = "CREATE CATALOG ctl " + + "PROPERTIES(" + + " \"type\" = \"iceberg\"," + + " \"iceberg.catalog.type\" = \"hadoop\"," + + " \"warehouse\" = \"s3://bucket/dir/key\"," + + " \"s3.endpoint\" = \"s3.us-east-1.amazonaws.com\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"123\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + + res = "CREATE TABLE mysql_tbl(" + + " k1 DATE," + + " k2 INT," + + " k3 SMALLINT," + + " k4 VARCHAR(2048)," + + " k5 DATETIME" + + ") " + + "ENGINE=mysql " + + "PROPERTIES(" + + " \"host\" = \"127.0.0.1\"," + + " \"port\" = \"8234\"," + + " \"user\" = \"abc\"," + + " \"password\" = \"*XXX\"," + + " \"database\" = \"mysql_db\"," + + " \"table\" = \"mysql_table\"" + + ")"; + parseAndCheck(sql, res); + + sql = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"123\"" + + ")"; + + res = "CREATE EXTERNAL TABLE broker_tbl(" + + " k1 tinyint," + + " k2 smallint," + + " k3 int," + + " k4 bigint) " + + "ENGINE=broker " + + "PROPERTIES(" + + " \"broker_name\" = \"hdfs\"," + + " \"path\" = \"hdfs://abc/qe/a.txt\"" + + ") " + + "BROKER PROPERTIES(" + + " \"username\" = \"root\"," + + " \"password\" = \"*XXX\"" + + ")"; + parseAndCheck(sql, res); + + sql = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "INSERT INTO test_s3load " + + "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"abc\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + + res = "SELECT * FROM s3_tbl(" + + " \"uri\" = \"s3://your_bucket_name/s3load_example.csv\"," + + " \"format\" = \"csv\"," + + " \"provider\" = \"OSS\"," + + " \"s3.endpoint\" = \"oss-cn-hangzhou.aliyuncs.com\"," + + " \"s3.region\" = \"oss-cn-hangzhou\"," + + " \"s3.access_key\" = \"abc\"," + + " \"s3.secret_key\" = \"*XXX\"," + + " \"column_separator\" = \",\"," + + " \"csv_schema\" = \"user_id:int;name:string;age:int\"" + + ")"; + parseAndCheck(sql, res); + + sql = "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')"; + res = "SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX')"; + parseAndCheck(sql, res); + + sql = "SET PASSWORD FOR 'admin' = PASSWORD('123456')"; + res = "SET PASSWORD FOR 'admin' = PASSWORD('*XXX')"; + parseAndCheck(sql, res); + } + + private void parseAndCheck(String sql, String expected) { + StatementBase parsedStmt = parser.parseSQL(sql).get(0); + AuditLogHelper.logAuditLog(ctx, sql, parsedStmt, null, false); + AuditEvent event = auditEvents.get(auditEvents.size() - 1); + Assertions.assertEquals(expected, event.stmt); + } +} diff --git a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out b/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out deleted file mode 100644 index c477d9e855dfb8..00000000000000 --- a/regression-test/data/nereids_p0/encrypt/test_encrypt_sql.out +++ /dev/null @@ -1,13 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -CREATE CATALOG test_encrypt_sql_table \\n PROPERTIES( \\n "type" = "iceberg", "iceberg.catalog.type" = "hadoop", "warehouse" = "s3://bucket/dir/key", "s3.endpoint" = "s3.us-east-1.amazonaws.com", "s3.access_key" = "ak", "s3.secret_key" = "*XXX"\\n );\\n -CREATE EXTERNAL TABLE broker_test_encrypt_sql_table(\\n k1 tinyint, \\n k2 smallint, \\n k3 int, \\n k4 bigint) \\n ENGINE=broker \\n PROPERTIES(\\n "broker_name" = "hdfs", "path" = "hdfs://xxxxxxx/qe/a.txt", "column_separator" = "\\t"\\n ) \\n BROKER PROPERTIES(\\n "username" = "root", "password" = "*XXX"\\n );\\n -CREATE TABLE mysql_test_encrypt_sql_table (\\n k1 DATE, \\n k2 INT, \\n k3 SMALLINT, \\n k4 VARCHAR(2048), \\n k5 DATETIME\\n )\\n ENGINE=mysql\\n PROPERTIES(\\n "host" = "127.0.0.1", "port" = "8234", "user" = "abc", "password" = "*XXX", "database" = "mysql_db", "table" = "mysql_table"\\n );\\n -EXPORT TABLE test_encrypt_sql_db.test_encrypt_sql_table TO "s3://abc/aaa"\\n PROPERTIES(\\n "format" = "csv",\\n "max_file_size" = "2048MB"\\n )\\n WITH s3 (\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n -INSERT INTO test_s3load\\n SELECT * FROM S3_test_encrypt_sql_table (\\n "uri" = "s3://your_bucket_name/s3load_example.csv", "format" = "csv", "provider" = "OSS", "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", "s3.region" = "oss-cn-hangzhou", "s3.access_key" = "abc", "s3.secret_key" = "*XXX", "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int"\\n );\\n -LOAD LABEL test_load_s3_orc_encrypt\\n (\\n DATA INFILE("s3://abc/aaa")\\n INTO TABLE test_encrypt_sql_table\\n FORMAT AS "ORC"\\n )\\n WITH S3\\n (\\n "provider" = "S3"\\n )\\n -SELECT * FROM S3_test_encrypt_sql_table (\\n "uri" = "s3://your_bucket_name/s3load_example.csv", "format" = "csv", "provider" = "OSS", "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", "s3.region" = "oss-cn-hangzhou", "s3.access_key" = "abc", "s3.secret_key" = "*XXX", "column_separator" = ",", "csv_schema" = "user_id:int;name:string;age:int"\\n );\\n -SELECT * FROM test_encrypt_sql_db.test_encrypt_sql_table\\n INTO OUTFILE "s3://abc/aaa"\\n FORMAT AS ORC\\n PROPERTIES(\\n "s3.endpoint" = "xxx", "s3.region" = "ap-beijing", "s3.secret_key" = "*XXX", "s3.access_key" = "abc"\\n );\\n -SET LDAP_ADMIN_PASSWORD = PASSWORD('*XXX') -SET PASSWORD FOR 'test_encrypt_sql_user' = PASSWORD('*XXX') - diff --git a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy b/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy deleted file mode 100644 index 7327ab87d228e2..00000000000000 --- a/regression-test/suites/nereids_p0/encrypt/test_encrypt_sql.groovy +++ /dev/null @@ -1,193 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_encrypt_sql") { - - def dbName = "test_encrypt_sql_db" - def tableName = "test_encrypt_sql_table" - - sql """drop database if exists ${dbName}""" - sql """create database ${dbName}""" - sql """use ${dbName}""" - - sql """CREATE TABLE `${tableName}` ( - `year` int NULL, - `country` text NULL, - `product` text NULL, - `profit` int NULL - ) ENGINE=OLAP - DUPLICATE KEY(`year`) - DISTRIBUTED BY HASH(`year`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - """ - - def user = "test_encrypt_sql_user" - def pwd = "123456" - - sql "drop user if exists ${user}" - sql "create user ${user} IDENTIFIED BY '${pwd}'" - sql "grant ADMIN_PRIV on *.*.* to ${user}" - sql "admin set frontend config('enable_nereids_load'='true')" - - connect(user, "${pwd}", context.config.jdbcUrl) { - try { - sql """EXPORT TABLE ${dbName}.${tableName} TO "s3://abc/aaa" - PROPERTIES( - "format" = "csv", - "max_file_size" = "2048MB" - ) - WITH s3 ( - "s3.endpoint" = "xxx", - "s3.region" = "ap-beijing", - "s3.secret_key"="abc", - "s3.access_key" = "abc" - ); - """ - } catch (Exception e) {} - - try { - sql """SELECT * FROM ${dbName}.${tableName} - INTO OUTFILE "s3://abc/aaa" - FORMAT AS ORC - PROPERTIES( - "s3.endpoint" = "xxx", - "s3.region" = "ap-beijing", - "s3.secret_key"="abc", - "s3.access_key" = "abc" - ); - """ - } catch (Exception e) {} - - try { - sql """LOAD LABEL test_load_s3_orc_encrypt - ( - DATA INFILE("s3://abc/aaa") - INTO TABLE ${tableName} - FORMAT AS "ORC" - ) - WITH S3 - ( - "provider" = "S3", - "AWS_ENDPOINT" = "xxx", - "AWS_ACCESS_KEY" = "abc", - "AWS_SECRET_KEY" = "abc", - "AWS_REGION" = "ap-beijing" - ) - """ - } catch (Exception e) {} - - try { - sql"""CREATE CATALOG ${tableName} - PROPERTIES( - 'type'='iceberg', - 'iceberg.catalog.type' = 'hadoop', - 'warehouse' = 's3://bucket/dir/key', - 's3.endpoint' = 's3.us-east-1.amazonaws.com', - 's3.access_key' = 'ak', - 's3.secret_key' = 'sk' - ); - """ - } catch (Exception e) {} - // for jdbc table or es table - try { - sql"""CREATE TABLE mysql_${tableName} ( - k1 DATE, - k2 INT, - k3 SMALLINT, - k4 VARCHAR(2048), - k5 DATETIME - ) - ENGINE=mysql - PROPERTIES( - 'host' = '127.0.0.1', - 'port' = '8234', - 'user' = 'abc', - 'password' = '123', - 'database' = 'mysql_db', - 'table' = 'mysql_table' - ); - """ - } catch (Exception e) {} - - try { - sql"""CREATE EXTERNAL TABLE broker_${tableName}( - k1 tinyint, - k2 smallint, - k3 int, - k4 bigint) - ENGINE=broker - PROPERTIES( - "broker_name" = "hdfs", - "path" = "hdfs://xxxxxxx/qe/a.txt", - "column_separator" = "\\t" - ) - BROKER PROPERTIES( - "username"="root", - "password"="123" - ); - """ - } catch (Exception e) {} - - try { - sql"""INSERT INTO test_s3load - SELECT * FROM S3_${tableName} ( - "uri" = "s3://your_bucket_name/s3load_example.csv", - "format" = "csv", - "provider" = "OSS", - "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", - "s3.region" = "oss-cn-hangzhou", - "s3.access_key" = "abc", - "s3.secret_key" = "abc", - "column_separator" = ",", - "csv_schema" = "user_id:int;name:string;age:int" - ); - """ - } catch (Exception e) {} - - try { - sql"""SELECT * FROM S3_${tableName} ( - "uri" = "s3://your_bucket_name/s3load_example.csv", - "format" = "csv", - "provider" = "OSS", - "s3.endpoint" = "oss-cn-hangzhou.aliyuncs.com", - "s3.region" = "oss-cn-hangzhou", - "s3.access_key" = "abc", - "s3.secret_key" = "abc", - "column_separator" = ",", - "csv_schema" = "user_id:int;name:string;age:int" - ); - """ - } catch (Exception e) {} - - sql "SET LDAP_ADMIN_PASSWORD = PASSWORD('123456')" - - sql "SET PASSWORD FOR '${user}' = PASSWORD('123456')" - } - - Thread.sleep(15000) - sql "call flush_audit_log()" - - def dateFormat = new java.text.SimpleDateFormat("yyyy-MM-dd") - def date = dateFormat.format(new Date()) - qt_sql "select stmt from __internal_schema.audit_log where user = '${user}' and time > '${date}' and (stmt like '%${tableName}%' or stmt like '%PASSWORD%') order by stmt" - - sql "drop table ${tableName}" - sql "drop database ${dbName}" - sql "drop user ${user}" -} From 3343cff4184b5e3766a9b47ccf5e46c97d040496 Mon Sep 17 00:00:00 2001 From: garenshi Date: Tue, 18 Feb 2025 12:09:31 +0800 Subject: [PATCH 10/12] 1 --- .../trees/plans/commands/insert/InsertIntoTableCommand.java | 4 ++-- .../plans/commands/insert/InsertOverwriteTableCommand.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index e767417f496190..55941dcf8bdca5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -36,7 +36,6 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -45,6 +44,7 @@ import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; @@ -506,7 +506,7 @@ public RedirectStatus toRedirectStatus() { @Override public boolean needAuditEncryption() { - return originLogicalQuery.anyMatch(node -> node instanceof UnboundTVFRelation); + return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 4e95a48c88120b..90656fff44915a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; -import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -47,6 +46,7 @@ import org.apache.doris.nereids.trees.plans.Explainable; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.TVFRelation; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption; @@ -414,6 +414,6 @@ public StmtType stmtType() { @Override public boolean needAuditEncryption() { - return originLogicalQuery.anyMatch(node -> node instanceof UnboundTVFRelation); + return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation); } } From ff5e9b55adf672b394f7c51005cc809338e401ff Mon Sep 17 00:00:00 2001 From: garenshi Date: Tue, 18 Feb 2025 17:35:42 +0800 Subject: [PATCH 11/12] 1 --- .../src/main/java/org/apache/doris/qe/AuditLogHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 17b27a409ce8a5..642379326fe637 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -265,7 +265,9 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme auditEventBuilder.setFeIp(FrontendOptions.getLocalHostAddress()); - String encryptSql = origStmt; + boolean isAnalysisErr = ctx.getState().getStateType() == MysqlStateType.ERR + && ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR; + String encryptSql = isAnalysisErr ? ctx.getState().getErrorMessage() : origStmt; // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (parsedStmt instanceof LogicalPlanAdapter) { LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); From 6c12b594c5f797c3066cca0c05c3b9351a6d56e9 Mon Sep 17 00:00:00 2001 From: garenshi Date: Wed, 19 Feb 2025 09:54:33 +0800 Subject: [PATCH 12/12] 1 --- .../src/main/java/org/apache/doris/qe/AuditLogHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index 642379326fe637..1e87546673cb34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -266,7 +266,7 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme auditEventBuilder.setFeIp(FrontendOptions.getLocalHostAddress()); boolean isAnalysisErr = ctx.getState().getStateType() == MysqlStateType.ERR - && ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR; + && ctx.getState().getErrType() == QueryState.ErrType.ANALYSIS_ERR; String encryptSql = isAnalysisErr ? ctx.getState().getErrorMessage() : origStmt; // We put origin query stmt at the end of audit log, for parsing the log more convenient. if (parsedStmt instanceof LogicalPlanAdapter) {