[Feature](multi-catalog) support sql-cache for hms catalog.

dutyu · Aug 23, 2023 · 3c9902b · 3c9902b
1 parent a538204
commit 3c9902b
Show file tree

Hide file tree

Showing 10 changed files with 152 additions and 1,467 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -71,6 +71,7 @@ public final class MetricRepo {
  public static LongCounterMetric COUNTER_QUERY_ERR;
  public static LongCounterMetric COUNTER_QUERY_TABLE;
  public static LongCounterMetric COUNTER_QUERY_OLAP_TABLE;
+ public static LongCounterMetric COUNTER_QUERY_HIVE_TABLE;
  public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ALL;
  public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ERR;
  public static Histogram HISTO_QUERY_LATENCY;
@@ -287,6 +288,9 @@ public Long getValue() {
  COUNTER_QUERY_OLAP_TABLE = new LongCounterMetric("query_olap_table", MetricUnit.REQUESTS,
  "total query from olap table");
  DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_OLAP_TABLE);
+ COUNTER_QUERY_HIVE_TABLE = new LongCounterMetric("query_hive_table", MetricUnit.REQUESTS,
+ "total query from hive table");
+ DORIS_METRIC_REGISTER.addMetrics(COUNTER_QUERY_HIVE_TABLE);
  USER_COUNTER_QUERY_ALL = new AutoMappedMetric<>(name -> {
  LongCounterMetric userCountQueryAll = new LongCounterMetric("query_total", MetricUnit.REQUESTS,
  "total query for single user");

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -143,7 +143,7 @@ public void plan(StatementBase queryStmt, org.apache.doris.thrift.TQueryOptions
  ArrayList<String> columnLabelList = physicalPlan.getOutput().stream().map(NamedExpression::getName)
  .collect(Collectors.toCollection(ArrayList::new));
  logicalPlanAdapter.setColLabels(columnLabelList);
- logicalPlanAdapter.setViews(statementContext.getViews());
+ logicalPlanAdapter.setViewDdlSqls(statementContext.getViewDdlSqls());
  }
 
  @VisibleForTesting

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids;
 
 import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.View;
 import org.apache.doris.common.IdGenerator;
 import org.apache.doris.common.Pair;
 import org.apache.doris.nereids.memo.Group;
@@ -81,7 +80,7 @@ public class StatementContext {
  // Used to update consumer's stats
  private final Map<CTEId, List<Pair<Map<Slot, Slot>, Group>>> cteIdToConsumerGroup = new HashMap<>();
  private final Map<CTEId, LogicalPlan> rewrittenCtePlan = new HashMap<>();
- private final Set<View> views = Sets.newHashSet();
+ private final Set<String> viewDdlSqlSet = Sets.newHashSet();
 
  public StatementContext() {
  this.connectContext = ConnectContext.get();
@@ -212,11 +211,11 @@ public Map<CTEId, LogicalPlan> getRewrittenCtePlan() {
  return rewrittenCtePlan;
  }
 
- public void addView(View view) {
- this.views.add(view);
+ public void addViewDdlSql(String ddlSql) {
+ this.viewDdlSqlSet.add(ddlSql);
  }
 
- public List<View> getViews() {
- return ImmutableList.copyOf(views);
+ public List<String> getViewDdlSqls() {
+ return ImmutableList.copyOf(viewDdlSqlSet);
  }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
@@ -23,7 +23,6 @@
 import org.apache.doris.analysis.Queriable;
 import org.apache.doris.analysis.RedirectStatus;
 import org.apache.doris.analysis.StatementBase;
-import org.apache.doris.catalog.View;
 import org.apache.doris.nereids.StatementContext;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
@@ -44,7 +43,7 @@ public class LogicalPlanAdapter extends StatementBase implements Queriable {
  private final LogicalPlan logicalPlan;
  private List<Expr> resultExprs;
  private ArrayList<String> colLabels;
- private List<View> views;
+ private List<String> viewDdlSqls;
 
  public LogicalPlanAdapter(LogicalPlan logicalPlan, StatementContext statementContext) {
  this.logicalPlan = logicalPlan;
@@ -81,8 +80,8 @@ public ArrayList<String> getColLabels() {
  return colLabels;
  }
 
- public List<View> getViews() {
- return views;
+ public List<String> getViewDdlSqls() {
+ return viewDdlSqls;
  }
 
  @Override
@@ -98,8 +97,8 @@ public void setColLabels(ArrayList<String> colLabels) {
  this.colLabels = colLabels;
  }
 
- public void setViews(List<View> views) {
- this.views = views;
+ public void setViewDdlSqls(List<String> viewDdlSqls) {
+ this.viewDdlSqls = viewDdlSqls;
  }
 
  public StatementContext getStatementContext() {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -61,7 +61,6 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 import java.util.Optional;
@@ -205,16 +204,14 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
  case OLAP:
  return makeOlapScan(table, unboundRelation, tableQualifier);
  case VIEW:
- cascadesContext.getStatementContext().addView((View) table);
  Plan viewPlan = parseAndAnalyzeView(((View) table).getDdlSql(), cascadesContext);
  return new LogicalSubQueryAlias<>(tableQualifier, viewPlan);
  case HMS_EXTERNAL_TABLE:
- if (Config.enable_query_hive_views) {
- if (((HMSExternalTable) table).isView()
- && StringUtils.isNotEmpty(((HMSExternalTable) table).getViewText())) {
- Plan hiveViewPlan = parseAndAnalyzeHiveView(table, cascadesContext);
- return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan);
- }
+ if (Config.enable_query_hive_views && ((HMSExternalTable) table).isView()) {
+ String hiveCatalog = ((HMSExternalTable) table).getCatalog().getName();
+ String ddlSql = ((HMSExternalTable) table).getViewText();
+ Plan hiveViewPlan = parseAndAnalyzeHiveView(hiveCatalog, ddlSql, cascadesContext);
+ return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan);
  }
  return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, tableQualifier);
  case ICEBERG_EXTERNAL_TABLE:
@@ -233,20 +230,20 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
  }
  }
 
- private Plan parseAndAnalyzeHiveView(TableIf table, CascadesContext cascadesContext) {
- HMSExternalTable hiveTable = (HMSExternalTable) table;
+ private Plan parseAndAnalyzeHiveView(String hiveCatalog, String ddlSql, CascadesContext cascadesContext) {
  ConnectContext ctx = cascadesContext.getConnectContext();
  String previousCatalog = ctx.getCurrentCatalog().getName();
  String previousDb = ctx.getDatabase();
- ctx.changeDefaultCatalog(hiveTable.getCatalog().getName());
- Plan hiveViewPlan = parseAndAnalyzeView(hiveTable.getViewText(), cascadesContext);
+ ctx.changeDefaultCatalog(hiveCatalog);
+ Plan hiveViewPlan = parseAndAnalyzeView(ddlSql, cascadesContext);
  ctx.changeDefaultCatalog(previousCatalog);
  ctx.setDatabase(previousDb);
  return hiveViewPlan;
  }
 
- private Plan parseAndAnalyzeView(String viewSql, CascadesContext parentContext) {
- LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(viewSql);
+ private Plan parseAndAnalyzeView(String ddlSql, CascadesContext parentContext) {
+ parentContext.getStatementContext().addViewDdlSql(ddlSql);
+ LogicalPlan parsedViewPlan = new NereidsParser().parseSingle(ddlSql);
  // TODO: use a good to do this, such as eliminate UnboundResultSink
  if (parsedViewPlan instanceof UnboundResultSink) {
  parsedViewPlan = (LogicalPlan) ((UnboundResultSink<?>) parsedViewPlan).child();