branch-3.1:[Feature](nereids) support encrypt sql for audit log #47595

fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundResultSink.java

@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.algebra.Sink;
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
 import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.nereids.util.Utils;
@@ -41,7 +42,7 @@
  * unbound result sink
  */
 public class UnboundResultSink<CHILD_TYPE extends Plan> extends LogicalSink<CHILD_TYPE>
-        implements Unbound, Sink, BlockFuncDepsPropagation {
+        implements NeedAuditEncryption, Unbound, Sink, BlockFuncDepsPropagation {
 
     public UnboundResultSink(CHILD_TYPE child) {
         super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child);
@@ -94,4 +95,9 @@ public String toString() {
     public StmtType stmtType() {
         return StmtType.SELECT;
     }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return anyMatch(node -> node instanceof UnboundTVFRelation);
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java

@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser;
+
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.nereids.DorisParser;
+import org.apache.doris.nereids.DorisParser.InsertTableContext;
+import org.apache.doris.nereids.DorisParser.SupportedDmlStatementContext;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+
+import org.antlr.v4.runtime.ParserRuleContext;
+import org.apache.commons.collections4.MapUtils;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**LogicalPlanBuilderForEncryption*/
+public class LogicalPlanBuilderForEncryption extends LogicalPlanBuilder {
+    private final Map<Pair<Integer, Integer>, String> indexInSqlToString;
+
+    public LogicalPlanBuilderForEncryption(Map<Integer, ParserRuleContext> selectHintMap,
+                                           Map<Pair<Integer, Integer>, String> indexInSqlToString) {
+        super(selectHintMap);
+        this.indexInSqlToString = Objects.requireNonNull(indexInSqlToString, "indexInSqlToString is null");
+    }
+
+    // select into outfile clause
+    @Override
+    public LogicalPlan visitStatementDefault(DorisParser.StatementDefaultContext ctx) {
+        if (ctx.outFileClause() != null && ctx.outFileClause().propertyClause() != null) {
+            DorisParser.PropertyClauseContext propertyClauseContext = ctx.outFileClause().propertyClause();
+            encryptProperty(visitPropertyClause(propertyClauseContext),
+                    propertyClauseContext.fileProperties.start.getStartIndex(),
+                    propertyClauseContext.fileProperties.stop.getStopIndex());
+        }
+        return super.visitStatementDefault(ctx);
+    }
+
+    // export into outfile clause
+    @Override
+    public BrokerDesc visitWithRemoteStorageSystem(DorisParser.WithRemoteStorageSystemContext ctx) {
+        Map<String, String> properties = visitPropertyItemList(ctx.brokerProperties);
+        encryptProperty(properties, ctx.brokerProperties.start.getStartIndex(),
+                ctx.brokerProperties.stop.getStopIndex());
+        return super.visitWithRemoteStorageSystem(ctx);
+    }
+
+    // load into outfile clause
+    @Override
+    public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
+        if (ctx.withRemoteStorageSystem() != null) {
+            Map<String, String> properties =
+                    new HashMap<>(visitPropertyItemList(ctx.withRemoteStorageSystem().brokerProperties));
+            encryptProperty(properties, ctx.withRemoteStorageSystem().brokerProperties.start.getStartIndex(),
+                    ctx.withRemoteStorageSystem().brokerProperties.stop.getStopIndex());
+        }
+        return super.visitLoad(ctx);
+    }
+
+    // create table clause
+    @Override
+    public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) {
+        // property or ext property
+        if (ctx.propertyClause() != null) {
+            List<DorisParser.PropertyClauseContext> propertyClauseContexts = ctx.propertyClause();
+            for (DorisParser.PropertyClauseContext propertyClauseContext : propertyClauseContexts) {
+                if (propertyClauseContext != null) {
+                    encryptProperty(visitPropertyClause(propertyClauseContext),
+                            propertyClauseContext.fileProperties.start.getStartIndex(),
+                            propertyClauseContext.fileProperties.stop.getStopIndex());
+                }
+            }
+        }
+        return super.visitCreateTable(ctx);
+    }
+
+    // alter storage vault clause
+    @Override
+    public LogicalPlan visitAlterStorageVault(DorisParser.AlterStorageVaultContext ctx) {
+        if (ctx.properties != null && ctx.properties.fileProperties != null) {
+            DorisParser.PropertyClauseContext propertyClauseContext = ctx.properties;
+            encryptProperty(visitPropertyClause(propertyClauseContext),
+                    propertyClauseContext.fileProperties.start.getStartIndex(),
+                    propertyClauseContext.fileProperties.stop.getStopIndex());
+        }
+        return super.visitAlterStorageVault(ctx);
+    }
+
+    // select from tvf
+    @Override
+    public LogicalPlan visitTableValuedFunction(DorisParser.TableValuedFunctionContext ctx) {
+        DorisParser.PropertyItemListContext properties = ctx.properties;
+        if (properties != null) {
+            encryptProperty(visitPropertyItemList(properties), properties.start.getStartIndex(),
+                    properties.stop.getStopIndex());
+        }
+        return super.visitTableValuedFunction(ctx);
+    }
+
+    // create job select tvf
+    @Override
+    public LogicalPlan visitCreateScheduledJob(DorisParser.CreateScheduledJobContext ctx) {
+        SupportedDmlStatementContext supportedDmlStatementContext = ctx.supportedDmlStatement();
+        visitInsertTable((InsertTableContext) supportedDmlStatementContext);
+        return super.visitCreateScheduledJob(ctx);
+    }
+
+    private void encryptProperty(Map<String, String> properties, int start, int stop) {
+        if (MapUtils.isNotEmpty(properties)) {
+            PrintableMap<String, String> printableMap = new PrintableMap<>(properties, "=",
+                    true, false, true);
+            indexInSqlToString.put(Pair.of(start, stop), printableMap.toString());
+        }
+    }
+
+    private void encryptPassword(int start, int stop) {
+        indexInSqlToString.put(Pair.of(start, stop), "'*XXX'");
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java

@@ -344,6 +344,14 @@ public LogicalPlan parseForCreateView(String sql) {
         return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
     }
 
+    public LogicalPlan parseForEncryption(String sql, Map<Pair<Integer, Integer>, String> indexInSqlToString) {
+        CommonTokenStream tokenStream = parseAllTokens(sql);
+        ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement);
+        LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForEncryption(
+                getHintMap(sql, tokenStream, DorisParser::selectHint), indexInSqlToString);
+        return (LogicalPlan) realLogicalPlanBuilder.visit(tree);
+    }
+
     /** parseForSyncMv */
     public Optional<String> parseForSyncMv(String sql) {
         CommonTokenStream tokenStream = parseAllTokens(sql);

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java

@@ -70,7 +70,7 @@
  * create table command
  */
 @Developing
-public class CreateTableCommand extends Command implements ForwardWithSync {
+public class CreateTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync {
     public static final Logger LOG = LogManager.getLogger(CreateTableCommand.class);
 
     private final Optional<LogicalPlan> ctasQuery;
@@ -221,5 +221,10 @@ public Optional<LogicalPlan> getCtasQuery() {
     public StmtType stmtType() {
         return StmtType.CREATE;
     }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return !createTableInfo.getEngineName().equalsIgnoreCase(CreateTableInfo.ENGINE_OLAP);
+    }
 }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java

@@ -71,7 +71,7 @@
  *          [PROPERTIES("key"="value")]
  *          WITH BROKER 'broker_name' [( $broker_attrs)]
  */
-public class ExportCommand extends Command implements ForwardWithSync {
+public class ExportCommand extends Command implements NeedAuditEncryption, ForwardWithSync {
     public static final String PARALLELISM = "parallelism";
     public static final String LABEL = "label";
     public static final String DATA_CONSISTENCY = "data_consistency";
@@ -405,5 +405,10 @@ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
     public StmtType stmtType() {
         return StmtType.EXPORT;
     }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return true;
+    }
 }
 

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java

@@ -85,7 +85,7 @@
 /**
  * load OLAP table data from external bulk file
  */
-public class LoadCommand extends Command implements ForwardWithSync {
+public class LoadCommand extends Command implements NeedAuditEncryption, ForwardWithSync {
 
     public static final Logger LOG = LogManager.getLogger(LoadCommand.class);
 
@@ -509,4 +509,9 @@ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
     public StmtType stmtType() {
         return StmtType.LOAD;
     }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return true;
+    }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/NeedAuditEncryption.java

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo;
+
+import java.util.TreeMap;
+
+/**
+ * NeedAuditEncryption
+ */
+public interface NeedAuditEncryption {
+    boolean needAuditEncryption();
+
+    /**
+     * gene encryption SQL
+     */
+    default String geneEncryptionSQL(String sql) {
+        if (!needAuditEncryption()) {
+            return sql;
+        }
+        TreeMap<Pair<Integer, Integer>, String> indexInSqlToString = new TreeMap<>(new Pair.PairComparator<>());
+        NereidsParser parser = new NereidsParser();
+        parser.parseForEncryption(sql, indexInSqlToString);
+        return BaseViewInfo.rewriteSql(indexInSqlToString, sql);
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java

@@ -237,6 +237,14 @@ public String getTableName() {
         return tableName;
     }
 
+    public String getEngineName() {
+        return engineName;
+    }
+
+    public Map<String, String> getProperties() {
+        return properties;
+    }
+
     /**
      * full qualifier table name.
      */

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java

@@ -45,8 +45,10 @@
 import org.apache.doris.nereids.trees.plans.Explainable;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.algebra.TVFRelation;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation;
@@ -86,7 +88,7 @@
  * InsertIntoTableCommand(Query())
  * ExplainCommand(Query())
  */
-public class InsertIntoTableCommand extends Command implements ForwardWithSync, Explainable {
+public class InsertIntoTableCommand extends Command implements NeedAuditEncryption, ForwardWithSync, Explainable {
 
     public static final Logger LOG = LogManager.getLogger(InsertIntoTableCommand.class);
 
@@ -497,6 +499,11 @@ public RedirectStatus toRedirectStatus() {
         }
     }
 
+    @Override
+    public boolean needAuditEncryption() {
+        return originLogicalQuery.anyMatch(node -> node instanceof TVFRelation);
+    }
+
     /**
      * this factory is used to delay create the AbstractInsertExecutor until the DistributePlan is generated
      * by NereidsPlanner

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileSink.java

@@ -24,6 +24,7 @@
 import org.apache.doris.nereids.trees.plans.PlanType;
 import org.apache.doris.nereids.trees.plans.PropagateFuncDeps;
 import org.apache.doris.nereids.trees.plans.algebra.Sink;
+import org.apache.doris.nereids.trees.plans.commands.NeedAuditEncryption;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 
 import com.google.common.base.Preconditions;
@@ -38,7 +39,7 @@
  * logicalFileSink for select into outfile
  */
 public class LogicalFileSink<CHILD_TYPE extends Plan> extends LogicalSink<CHILD_TYPE>
-        implements Sink, PropagateFuncDeps {
+        implements NeedAuditEncryption, Sink, PropagateFuncDeps {
 
     private final String filePath;
     private final String format;
@@ -120,4 +121,9 @@ public String getFormat() {
     public Map<String, String> getProperties() {
         return properties;
     }
+
+    @Override
+    public boolean needAuditEncryption() {
+        return true;
+    }
 }