[Feature](multi-catalog) Support sql cache for hms catalog (#23391) (#…

…24324) **Support sql cache for hms catalog. Legacy planner and Nereids planner are all supported. Not support partition cache now, not support federated query now.**
apache · Sep 13, 2023 · d6b6626 · d6b6626
1 parent 4741648
commit d6b6626
Show file tree

Hide file tree

Showing 12 changed files with 559 additions and 90 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -71,6 +71,7 @@ public final class MetricRepo {
     public static LongCounterMetric COUNTER_QUERY_ERR;
     public static LongCounterMetric COUNTER_QUERY_TABLE;
     public static LongCounterMetric COUNTER_QUERY_OLAP_TABLE;
+    public static LongCounterMetric COUNTER_QUERY_HIVE_TABLE;
     public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ALL;
     public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ERR;
     public static Histogram HISTO_QUERY_LATENCY;
@@ -287,6 +288,9 @@ public Long getValue() {
         COUNTER_QUERY_OLAP_TABLE = new LongCounterMetric("query_olap_table", MetricUnit.REQUESTS,
                 "total query from olap table");
         DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_OLAP_TABLE);
+        COUNTER_QUERY_HIVE_TABLE = new LongCounterMetric("query_hive_table", MetricUnit.REQUESTS,
+                "total query from hive table");
+        DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_HIVE_TABLE);
         USER_COUNTER_QUERY_ALL = new AutoMappedMetric<>(name -> {
             LongCounterMetric userCountQueryAll  = new LongCounterMetric("query_total", MetricUnit.REQUESTS,
                     "total query for single user");

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -151,7 +151,7 @@ public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions
         ArrayList<String> columnLabelList = physicalPlan.getOutput().stream().map(NamedExpression::getName)
                 .collect(Collectors.toCollection(ArrayList::new));
         logicalPlanAdapter.setColLabels(columnLabelList);
-        logicalPlanAdapter.setViews(statementContext.getViews());
+        logicalPlanAdapter.setViewDdlSqls(statementContext.getViewDdlSqls());
     }
 
     @VisibleForTesting

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids;
 
 import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.View;
 import org.apache.doris.common.IdGenerator;
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.memo.Group;
@@ -83,7 +82,7 @@ public class StatementContext {
     // Used to update consumer's stats
     private final Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>> cteIdToConsumerGroup = new HashMap<>();
     private final Map<CTEId, LogicalPlan> rewrittenCtePlan = new HashMap<>();
-    private final Set<View> views = Sets.newHashSet();
+    private final Set<String> viewDdlSqlSet = Sets.newHashSet();
 
     public StatementContext() {
         this.connectContext = ConnectContext.get();
@@ -218,11 +217,11 @@ public Map<CTEId, LogicalPlan> getRewrittenCtePlan() {
         return rewrittenCtePlan;
     }
 
-    public void addView(View view) {
-        this.views.add(view);
+    public void addViewDdlSql(String ddlSql) {
+        this.viewDdlSqlSet.add(ddlSql);
     }
 
-    public List<View> getViews() {
-        return ImmutableList.copyOf(views);
+    public List<String> getViewDdlSqls() {
+        return ImmutableList.copyOf(viewDdlSqlSet);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
@@ -23,7 +23,6 @@
 import org.apache.doris.analysis.Queriable;
 import org.apache.doris.analysis.RedirectStatus;
 import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.View;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
@@ -44,7 +43,7 @@ public class LogicalPlanAdapter extends StatementBase implements Queriable {
     private final LogicalPlan logicalPlan;
     private List<Expr> resultExprs;
     private ArrayList<String> colLabels;
-    private List<View> views;
+    private List<String> viewDdlSqls;
 
     public LogicalPlanAdapter(LogicalPlan logicalPlan, StatementContext statementContext) {
         this.logicalPlan = logicalPlan;
@@ -81,8 +80,8 @@ public ArrayList<String> getColLabels() {
         return colLabels;
     }
 
-    public List<View> getViews() {
-        return views;
+    public List<String> getViewDdlSqls() {
+        return viewDdlSqls;
     }
 
     @Override
@@ -98,8 +97,8 @@ public void setColLabels(ArrayList<String> colLabels) {
         this.colLabels = colLabels;
     }
 
-    public void setViews(List<View> views) {
-        this.views = views;
+    public void setViewDdlSqls(List<String> viewDdlSqls) {
+        this.viewDdlSqls = viewDdlSqls;
     }
 
     public StatementContext getStatementContext() {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -62,7 +62,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 import java.util.Optional;
@@ -204,16 +203,14 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
             case OLAP:
                 return makeOlapScan(table, unboundRelation, tableQualifier);
             case VIEW:
-                cascadesContext.getStatementContext().addView((View) table);
                 Plan viewPlan = parseAndAnalyzeView(((View) table).getDdlSql(), cascadesContext);
                 return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);
             case HMS_EXTERNAL_TABLE:
-                if (Config.enable_query_hive_views) {
-                    if (((HMSExternalTable) table).isView()
-                                && StringUtils.isNotEmpty(((HMSExternalTable) table).getViewText())) {
-                        Plan hiveViewPlan = parseAndAnalyzeHiveView(table, cascadesContext);
-                        return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan);
-                    }
+                if (Config.enable_query_hive_views && ((HMSExternalTable) table).isView()) {
+                    String hiveCatalog = ((HMSExternalTable) table).getCatalog().getName();
+                    String ddlSql = ((HMSExternalTable) table).getViewText();
+                    Plan hiveViewPlan = parseAndAnalyzeHiveView(hiveCatalog, ddlSql, cascadesContext);
+                    return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan);
                 }
                 return new LogicalFileScan(StatementScopeIdGenerator.newRelationId(),
                         (HMSExternalTable) table, tableQualifier);
@@ -234,20 +231,20 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
         }
     }
 
-    private Plan parseAndAnalyzeHiveView(TableIf table, CascadesContext cascadesContext) {
-        HMSExternalTable hiveTable = (HMSExternalTable) table;
+    private Plan parseAndAnalyzeHiveView(String hiveCatalog, String ddlSql, CascadesContext cascadesContext) {
         ConnectContext ctx = cascadesContext.getConnectContext();
         String previousCatalog = ctx.getCurrentCatalog().getName();
         String previousDb = ctx.getDatabase();
-        ctx.changeDefaultCatalog(hiveTable.getCatalog().getName());
-        Plan hiveViewPlan = parseAndAnalyzeView(hiveTable.getViewText(), cascadesContext);
+        ctx.changeDefaultCatalog(hiveCatalog);
+        Plan hiveViewPlan = parseAndAnalyzeView(ddlSql, cascadesContext);
         ctx.changeDefaultCatalog(previousCatalog);
         ctx.setDatabase(previousDb);
         return hiveViewPlan;
     }
 
-    private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) {
-        LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(viewSql);
+    private Plan parseAndAnalyzeView(String ddlSql, CascadesContext parentContext) {
+        parentContext.getStatementContext().addViewDdlSql(ddlSql);
+        LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(ddlSql);
         // TODO: use a good to do this, such as eliminate UnboundResultSink
         if (parsedViewPlan instanceof UnboundResultSink) {
             parsedViewPlan = (LogicalPlan) ((UnboundResultSink<?>) parsedViewPlan).child();

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileScanNode.java
@@ -270,4 +270,8 @@ protected int getBlockIndex(BlockLocation[] blkLocations, long offset) {
         long fileLength = last.getOffset() + last.getLength() - 1L;
         throw new IllegalArgumentException(String.format("Offset %d is outside of file (0..%d)", offset, fileLength));
     }
+
+    public long getReadPartitionNum() {
+        return this.readPartitionNum;
+    }
 }