From e003cacce2cff5b42d41073f1d7c7803ba94646b Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Fri, 12 Jun 2020 11:43:34 +0800 Subject: [PATCH 01/19] Support bitmap_union, hll_union and count when creating and matching a materialized view. This commit mainly supports creating bitmap_union, hll_union, and count materialized views. * The main changes are as follows: 1. When creating a materialized view, doris judge the semantic analysis of the newly supported aggregate function. Only bitmap_union(to_bitmap(column)), hll_union(hll_hash(column)) and count(column) are supported. 2. Match the correct materialized view when querying. After the user sends the query, if there is a possibility of matching the materialized view, the query will be rewritten firstly. Such as: Table: k1 int, k2 int MV: k1 int, mv_bitmap_union_k2 bitmap mv_bitmap_union mv_bitmap_union = to_bitmap(k2) Query: select k1, count(distinct k2) from Table Found that there is a match between the materialized view column and the query column, the query is rewritten as: Rewritten query: select k1, bitmap_union_count(mv_bitmap_union_k2) from table Then when the materialized view is matched, it can be matched to the query materialized view table. Sometimes the rewritten query may not match any materialized view, which means that the rewriting failed. The query needs to be re-parsed and executed again. Change-Id: I96a7522c4e6a58150b0df1886cf22c895c1c2bad --- .../doris/alter/MaterializedViewHandler.java | 71 +++----- .../apache/doris/analysis/AnalyticExpr.java | 4 +- .../org/apache/doris/analysis/Analyzer.java | 19 ++ .../doris/analysis/BetweenPredicate.java | 2 +- .../org/apache/doris/analysis/ColumnDef.java | 2 +- .../analysis/CreateMaterializedViewStmt.java | 120 ++++++++---- .../java/org/apache/doris/analysis/Expr.java | 6 +- .../doris/analysis/FunctionCallExpr.java | 6 +- .../apache/doris/analysis/JoinOperator.java | 2 +- .../analysis/MVColumnBitmapUnionPattern.java | 57 ++++++ .../analysis/MVColumnHLLUnionPattern.java | 55 ++++++ .../apache/doris/analysis/MVColumnItem.java | 32 +++- .../analysis/MVColumnOneChildPattern.java | 56 ++++++ .../doris/analysis/MVColumnPattern.java | 23 +++ .../org/apache/doris/analysis/Predicate.java | 4 +- .../org/apache/doris/analysis/QueryStmt.java | 94 +++++++--- .../org/apache/doris/analysis/SelectList.java | 2 +- .../org/apache/doris/analysis/SelectStmt.java | 14 +- .../doris/analysis/SetOperationStmt.java | 4 +- .../org/apache/doris/analysis/SlotRef.java | 38 +++- .../apache/doris/analysis/StmtRewriter.java | 30 +-- .../doris/analysis/TupleDescriptor.java | 10 +- .../doris/catalog/AggregateFunction.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 11 ++ .../doris/load/BrokerFileGroupAggInfo.java | 2 +- .../apache/doris/planner/LoadScanNode.java | 2 +- .../planner/MaterializedViewSelector.java | 171 ++++++++---------- .../apache/doris/planner/OlapScanNode.java | 12 ++ .../org/apache/doris/planner/Planner.java | 6 +- .../apache/doris/planner/RollupSelector.java | 1 - .../doris/planner/SingleNodePlanner.java | 26 ++- .../org/apache/doris/qe/StmtExecutor.java | 126 +++++++------ .../apache/doris/rewrite/ExprRewriteRule.java | 6 +- .../mvrewrite/CountDistinctToBitmap.java | 101 +++++++++++ .../CountDistinctToBitmapOrHLLRule.java | 71 ++++++++ .../rewrite/mvrewrite/CountFieldToSum.java | 94 ++++++++++ .../mvrewrite/FunctionCallEqualRule.java | 78 ++++++++ .../mvrewrite/HLLHashToSlotRefRule.java | 114 ++++++++++++ .../rewrite/mvrewrite/MVExprEqualRule.java | 25 +++ .../rewrite/mvrewrite/MVExprEquivalent.java | 45 +++++ .../mvrewrite/MVSelectFailedException.java | 27 +++ .../doris/rewrite/mvrewrite/NDVToHll.java | 97 ++++++++++ .../rewrite/mvrewrite/SlotRefEqualRule.java | 39 ++++ .../mvrewrite/ToBitmapToSlotRefRule.java | 113 ++++++++++++ .../doris/task/PullLoadTaskPlanner.java | 2 +- .../doris/transaction/TransactionState.java | 2 +- .../alter/MaterializedViewHandlerTest.java | 6 +- .../apache/doris/alter/RollupJobV2Test.java | 4 +- .../org/apache/doris/analysis/ExprTest.java | 16 +- .../catalog/MaterializedIndexMetaTest.java | 4 +- .../planner/MaterializedViewFunctionTest.java | 155 ++++++++++++++++ .../planner/MaterializedViewSelectorTest.java | 64 ++++--- 52 files changed, 1713 insertions(+), 360 deletions(-) create mode 100644 fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java create mode 100644 fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java create mode 100644 fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java create mode 100644 fe/src/main/java/org/apache/doris/analysis/MVColumnPattern.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEqualRule.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVSelectFailedException.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/SlotRefEqualRule.java create mode 100644 fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java diff --git a/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 8626374bf9346e..27ef6d5eff0676 100644 --- a/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -42,7 +42,6 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletInvertedIndex; import org.apache.doris.catalog.TabletMeta; -import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -422,60 +421,46 @@ private List checkAndPrepareMaterializedView(CreateMaterializedViewStmt if (olapTable.hasMaterializedIndex(addMVClause.getMVName())) { throw new DdlException("Materialized view[" + addMVClause.getMVName() + "] already exists"); } - // check if rollup columns are valid - // a. all columns should exist in base rollup schema - // b. For aggregate table, mv columns with aggregate function should be same as base schema - // c. For aggregate table, the column which is the key of base table should be the key of mv as well. + // check if mv columns are valid + // a. Aggregate or Unique table: + // 1. For aggregate table, mv columns with aggregate function should be same as base schema + // 2. For aggregate table, the column which is the key of base table should be the key of mv as well. + // b. Duplicate table: + // 1. Columns resolved by semantics are legal // update mv columns List mvColumnItemList = addMVClause.getMVColumnItemList(); List newMVColumns = Lists.newArrayList(); int numOfKeys = 0; - for (MVColumnItem mvColumnItem : mvColumnItemList) { - String mvColumnName = mvColumnItem.getName(); - Column baseColumn = olapTable.getColumn(mvColumnName); - if (baseColumn == null) { - throw new DdlException("Column[" + mvColumnName + "] does not exist"); - } - if (mvColumnItem.isKey()) { - ++numOfKeys; - } - AggregateType baseAggregationType = baseColumn.getAggregationType(); - AggregateType mvAggregationType = mvColumnItem.getAggregationType(); - if (olapTable.getKeysType().isAggregationFamily()) { + if (olapTable.getKeysType().isAggregationFamily()) { + for (MVColumnItem mvColumnItem : mvColumnItemList) { + String mvColumnName = mvColumnItem.getName(); + Column baseColumn = olapTable.getColumn(mvColumnName); + if (mvColumnItem.isKey()) { + ++numOfKeys; + } + Preconditions.checkNotNull(baseColumn, "Column[" + mvColumnName + "] does not exist"); + AggregateType baseAggregationType = baseColumn.getAggregationType(); + AggregateType mvAggregationType = mvColumnItem.getAggregationType(); if (baseColumn.isKey() && !mvColumnItem.isKey()) { throw new DdlException("The column[" + mvColumnName + "] must be the key of materialized view"); } if (baseAggregationType != mvAggregationType) { - throw new DdlException("The aggregation type of column[" + mvColumnName + "] must be same as " - + "the aggregate type of base column in aggregate table"); + throw new DdlException( + "The aggregation type of column[" + mvColumnName + "] must be same as the aggregate " + + "type of base column in aggregate table"); } - if (baseAggregationType != null && baseAggregationType.isReplaceFamily() - && olapTable.getKeysNum() != numOfKeys) { - throw new DdlException("The materialized view should contain all keys of base table if there is a" - + " REPLACE value"); + if (baseAggregationType != null && baseAggregationType.isReplaceFamily() && olapTable + .getKeysNum() != numOfKeys) { + throw new DdlException( + "The materialized view should contain all keys of base table if there is a" + " REPLACE " + + "value"); } + newMVColumns.add(mvColumnItem.toMVColumn(olapTable)); } - if (olapTable.getKeysType() == KeysType.DUP_KEYS && mvAggregationType != null - && mvAggregationType.isReplaceFamily()) { - throw new DdlException("The aggregation type of REPLACE AND REPLACE IF NOT NULL is forbidden in " - + "duplicate table"); - } - Column newMVColumn = new Column(baseColumn); - newMVColumn.setIsKey(mvColumnItem.isKey()); - newMVColumn.setAggregationType(mvAggregationType, mvColumnItem.isAggregationTypeImplicit()); - if (mvColumnItem.getDefineExpr() != null) { - if (mvAggregationType.equals(AggregateType.BITMAP_UNION)) { - newMVColumn.setType(Type.BITMAP); - } else if (mvAggregationType.equals(AggregateType.HLL_UNION)){ - newMVColumn.setType(Type.HLL); - } else { - throw new DdlException("The define expr of column is only support bitmap_union or hll_union"); - } - newMVColumn.setIsKey(false); - newMVColumn.setIsAllowNull(false); - newMVColumn.setDefineExpr(mvColumnItem.getDefineExpr()); + } else { + for (MVColumnItem mvColumnItem : mvColumnItemList) { + newMVColumns.add(mvColumnItem.toMVColumn(olapTable)); } - newMVColumns.add(newMVColumn); } return newMVColumns; } diff --git a/fe/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/src/main/java/org/apache/doris/analysis/AnalyticExpr.java index 2ca2ab4b83935b..975b4607f6de47 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AnalyticExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/AnalyticExpr.java @@ -87,7 +87,7 @@ public class AnalyticExpr extends Expr { private static String SUM = "SUM"; private static String COUNT = "COUNT"; - // Internal function used to implement FIRST_VALUE with a window rewrite and + // Internal function used to implement FIRST_VALUE with a window equal and // additional null handling in the backend. public static String FIRST_VALUE_REWRITE = "FIRST_VALUE_REWRITE"; @@ -241,7 +241,7 @@ static private boolean isHllAggFn(Function fn) { * percent_rank(), cume_dist() and ntile() * * Returns a new Expr if the analytic expr is rewritten, returns null if it's not one - * that we want to rewrite. + * that we want to equal. */ public static Expr rewrite(AnalyticExpr analyticExpr) { Function fn = analyticExpr.getFnCall().getFn(); diff --git a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java index dee9d933c610ff..757b6e97d3316d 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -40,6 +40,12 @@ import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.rewrite.FoldConstantsRule; import org.apache.doris.rewrite.NormalizeBinaryPredicatesRule; +import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmap; +import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmapOrHLLRule; +import org.apache.doris.rewrite.mvrewrite.CountFieldToSum; +import org.apache.doris.rewrite.mvrewrite.HLLHashToSlotRefRule; +import org.apache.doris.rewrite.mvrewrite.NDVToHll; +import org.apache.doris.rewrite.mvrewrite.ToBitmapToSlotRefRule; import org.apache.doris.thrift.TQueryGlobals; import com.google.common.base.Joiner; @@ -235,6 +241,8 @@ private static class GlobalState { // Expr rewriter for normalizing and rewriting expressions. private final ExprRewriter exprRewriter_; + private final ExprRewriter mvExprRewriter; + public GlobalState(Catalog catalog, ConnectContext context) { this.catalog = catalog; this.context = context; @@ -248,6 +256,15 @@ public GlobalState(Catalog catalog, ConnectContext context) { rules.add(NormalizeBinaryPredicatesRule.INSTANCE); rules.add(FoldConstantsRule.INSTANCE); exprRewriter_ = new ExprRewriter(rules); + // init mv rewriter + List mvRewriteRules = Lists.newArrayList(); + mvRewriteRules.add(ToBitmapToSlotRefRule.INSTANCE); + mvRewriteRules.add(CountDistinctToBitmapOrHLLRule.INSTANCE); + mvRewriteRules.add(CountDistinctToBitmap.INSTANCE); + mvRewriteRules.add(NDVToHll.INSTANCE); + mvRewriteRules.add(HLLHashToSlotRefRule.INSTANCE); + mvRewriteRules.add(CountFieldToSum.INSTANCE); + mvExprRewriter = new ExprRewriter(mvRewriteRules); } }; private final GlobalState globalState; @@ -511,6 +528,8 @@ public Table getTable(TableName tblName) { public ExprRewriter getExprRewriter() { return globalState.exprRewriter_; } + public ExprRewriter getMVExprRewriter() { return globalState.mvExprRewriter; } + /** * Return descriptor of registered table/alias. * diff --git a/fe/src/main/java/org/apache/doris/analysis/BetweenPredicate.java b/fe/src/main/java/org/apache/doris/analysis/BetweenPredicate.java index 7a49eeb0ecf869..9729b89c97cfb4 100644 --- a/fe/src/main/java/org/apache/doris/analysis/BetweenPredicate.java +++ b/fe/src/main/java/org/apache/doris/analysis/BetweenPredicate.java @@ -23,7 +23,7 @@ import org.apache.logging.log4j.Logger; /** - * Class describing between predicates. After successful analysis, we rewrite + * Class describing between predicates. After successful analysis, we equal * the between predicate to a conjunctive/disjunctive compound predicate * to be handed to the backend. */ diff --git a/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java index d51e6edfeaf0bd..cc9c5794de3b3e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -68,7 +68,7 @@ public DefaultValue(boolean isSet, String value) { public static DefaultValue NOT_SET = new DefaultValue(false, null); // default null public static DefaultValue NULL_DEFAULT_VALUE = new DefaultValue(true, null); - private static String ZERO = new String(new byte[] {0}); + public static String ZERO = new String(new byte[] {0}); // default "value", "0" means empty hll public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(true, ZERO); // default "value", "0" means empty bitmap diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 6f0ff6f75b06c9..819369c162fbee 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; @@ -30,8 +31,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -50,7 +53,21 @@ * [PROPERTIES ("key" = "value")] */ public class CreateMaterializedViewStmt extends DdlStmt { - public static final String MATERIALIZED_VIEW_NAME_PRFIX = "__doris_materialized_view_"; + public static final String MATERIALIZED_VIEW_NAME_PREFIX = "mv_"; + public static final Map fnNameToPattern; + + static { + fnNameToPattern = Maps.newHashMap(); + fnNameToPattern.put(AggregateType.SUM.name().toLowerCase(), + new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase())); + fnNameToPattern.put(AggregateType.MIN.name().toLowerCase(), + new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase())); + fnNameToPattern.put(AggregateType.MAX.name().toLowerCase(), + new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase())); + fnNameToPattern.put("count", new MVColumnOneChildPattern("count")); + fnNameToPattern.put("bitmap_union", new MVColumnBitmapUnionPattern()); + fnNameToPattern.put("hll_union", new MVColumnHLLUnionPattern()); + } private String mvName; private SelectStmt selectStmt; @@ -108,6 +125,7 @@ public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); FeNameFormat.checkTableName(mvName); // TODO(ml): The mv name in from clause should pass the analyze without error. + selectStmt.forbiddenMVRewrite(); selectStmt.analyze(analyzer); if (selectStmt.getAggInfo() != null) { mvKeysType = KeysType.AGG_KEYS; @@ -163,35 +181,26 @@ public void analyzeSelectClause() throws AnalysisException { if (!mvColumnNameSet.add(columnName)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnName); } - MVColumnItem mvColumnItem = new MVColumnItem(columnName); - mvColumnItem.setType(slotRef.getType()); + MVColumnItem mvColumnItem = new MVColumnItem(columnName, slotRef.getType()); mvColumnItemList.add(mvColumnItem); } else if (selectListItem.getExpr() instanceof FunctionCallExpr) { + // Function must match pattern. FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItem.getExpr(); String functionName = functionCallExpr.getFnName().getFunction(); - Expr defineExpr = null; - // TODO(ml): support REPLACE, REPLACE_IF_NOT_NULL only for aggregate table, HLL_UNION, BITMAP_UNION - if (!functionName.equalsIgnoreCase("sum") - && !functionName.equalsIgnoreCase("min") - && !functionName.equalsIgnoreCase("max")) { - throw new AnalysisException("The materialized view only support the sum, min and max aggregate " - + "function. Error function: " + functionCallExpr.toSqlImpl()); + MVColumnPattern mvColumnPattern = fnNameToPattern.get(functionName.toLowerCase()); + if (mvColumnPattern == null) { + throw new AnalysisException( + "Materialized view does not support this function:" + functionCallExpr.toSqlImpl()); } - - Preconditions.checkState(functionCallExpr.getChildren().size() == 1); - Expr functionChild0 = functionCallExpr.getChild(0); - SlotRef slotRef; - if (functionChild0 instanceof SlotRef) { - slotRef = (SlotRef) functionChild0; - } else if (functionChild0 instanceof CastExpr && (functionChild0.getChild(0) instanceof SlotRef)) { - slotRef = (SlotRef) functionChild0.getChild(0); - } else { - throw new AnalysisException("The children of aggregate function only support one original column. " - + "Error function: " + functionCallExpr.toSqlImpl()); + if (!mvColumnPattern.match(functionCallExpr)) { + throw new AnalysisException( + "The function " + functionName + " must match pattern:" + mvColumnPattern.toString()); } - meetAggregate = true; // check duplicate column - String columnName = slotRef.getColumnName().toLowerCase(); + List slots = new ArrayList<>(); + functionCallExpr.collect(SlotRef.class, slots); + Preconditions.checkArgument(slots.size() == 1); + String columnName = slots.get(0).getColumnName().toLowerCase(); if (!mvColumnNameSet.add(columnName)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnName); } @@ -199,15 +208,14 @@ public void analyzeSelectClause() throws AnalysisException { if (beginIndexOfAggregation == -1) { beginIndexOfAggregation = i; } + meetAggregate = true; + // build mv column item + mvColumnItemList.add(buildMVColumnItem(functionCallExpr)); + // TODO(ml): support REPLACE, REPLACE_IF_NOT_NULL, bitmap_union, hll_union only for aggregate table // TODO(ml): support different type of column, int -> bigint(sum) - // TODO: change the column name of bitmap and hll - MVColumnItem mvColumnItem = new MVColumnItem(columnName); - mvColumnItem.setAggregationType(AggregateType.valueOf(functionName.toUpperCase()), false); - mvColumnItem.setDefineExpr(defineExpr); - mvColumnItemList.add(mvColumnItem); } } - // TODO(ML): only value columns of materialized view, such as select sum(v1) from table + // TODO(ml): only value columns of materialized view, such as select sum(v1) from table if (beginIndexOfAggregation == 0) { throw new AnalysisException("The materialized view must contain at least one key column"); } @@ -298,8 +306,7 @@ private void supplyOrderColumn() throws AnalysisException { for (; theBeginIndexOfValue < mvColumnItemList.size(); theBeginIndexOfValue++) { MVColumnItem column = mvColumnItemList.get(theBeginIndexOfValue); keySizeByte += column.getType().getIndexSize(); - if (theBeginIndexOfValue + 1 > FeConstants.shortkey_max_column_count - || keySizeByte > FeConstants.shortkey_maxsize_bytes) { + if (theBeginIndexOfValue + 1 > FeConstants.shortkey_max_column_count || keySizeByte > FeConstants.shortkey_maxsize_bytes) { if (theBeginIndexOfValue == 0 && column.getType().getPrimitiveType().isCharFamily()) { column.setIsKey(true); theBeginIndexOfValue++; @@ -325,7 +332,58 @@ private void supplyOrderColumn() throws AnalysisException { mvColumnItem.setAggregationType(AggregateType.NONE, true); } } + } + + public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws AnalysisException { + String functionName = functionCallExpr.getFnName().getFunction(); + List slots = new ArrayList<>(); + functionCallExpr.collect(SlotRef.class, slots); + Preconditions.checkArgument(slots.size() == 1); + SlotRef baseColumnRef = slots.get(0); + String baseColumnName = baseColumnRef.getColumnName().toLowerCase(); + Expr functionChild0 = functionCallExpr.getChild(0); + String mvColumnName; + AggregateType mvAggregateType; + Expr defineExpr = null; + Type type; + switch (functionName.toLowerCase()) { + case "sum": + case "min": + case "max": + mvColumnName = baseColumnName; + mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); + type = Type.BIGINT; + break; + case "bitmap_union": + mvColumnName = mvColumnBuilder(functionName, baseColumnName); + mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); + defineExpr = functionChild0; + type = Type.BITMAP; + break; + case "hll_union": + mvColumnName = mvColumnBuilder(functionName, baseColumnName); + mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); + defineExpr = functionChild0; + type = Type.HLL; + break; + case "count": + mvColumnName = mvColumnBuilder(functionName, baseColumnName); + mvAggregateType = AggregateType.SUM; + defineExpr = new CaseExpr(null, Lists.newArrayList(new CaseWhenClause( + new IsNullPredicate(baseColumnRef, false), + new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT)); + type = Type.BIGINT; + break; + default: + throw new AnalysisException("Unsupported function:" + functionName); + } + MVColumnItem mvColumnItem = new MVColumnItem(mvColumnName, type, mvAggregateType, false, defineExpr); + return mvColumnItem; + } + public static String mvColumnBuilder(String functionName, String sourceColumnName) { + return new StringBuilder().append(MATERIALIZED_VIEW_NAME_PREFIX).append(functionName).append("_") + .append(sourceColumnName).toString(); } @Override diff --git a/fe/src/main/java/org/apache/doris/analysis/Expr.java b/fe/src/main/java/org/apache/doris/analysis/Expr.java index c605e49cd6fba0..15d70e522fb6f8 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/src/main/java/org/apache/doris/analysis/Expr.java @@ -1110,10 +1110,10 @@ public void getIds(List tupleIds, List slotIds) { } } - public void getTableNameToColumnNames(Map> tableNameToColumnNames) { - Preconditions.checkState(tableNameToColumnNames != null); + public void getTableIdToColumnNames(Map> tableIdToColumnNames) { + Preconditions.checkState(tableIdToColumnNames != null); for (Expr child : children) { - child.getTableNameToColumnNames(tableNameToColumnNames); + child.getTableIdToColumnNames(tableIdToColumnNames); } } diff --git a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 174b41025c97f5..686b9079d4b69a 100644 --- a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -505,10 +505,10 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { } if (fnName.getFunction().equals("count") && fnParams.isDistinct()) { - // Treat COUNT(DISTINCT ...) special because of how we do the rewrite. + // Treat COUNT(DISTINCT ...) special because of how we do the equal. // There is no version of COUNT() that takes more than 1 argument but after - // the rewrite, we only need count(*). - // TODO: fix how we rewrite count distinct. + // the equal, we only need count(*). + // TODO: fix how we equal count distinct. fn = getBuiltinFunction(analyzer, fnName.getFunction(), new Type[0], Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); type = fn.getReturnType(); diff --git a/fe/src/main/java/org/apache/doris/analysis/JoinOperator.java b/fe/src/main/java/org/apache/doris/analysis/JoinOperator.java index f13e6ffbda8119..49d9e9f7b4471e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/JoinOperator.java +++ b/fe/src/main/java/org/apache/doris/analysis/JoinOperator.java @@ -31,7 +31,7 @@ public enum JoinOperator { FULL_OUTER_JOIN("FULL OUTER JOIN", TJoinOp.FULL_OUTER_JOIN), MERGE_JOIN("MERGE JOIN", TJoinOp.MERGE_JOIN), CROSS_JOIN("CROSS JOIN", TJoinOp.CROSS_JOIN), - // Variant of the LEFT ANTI JOIN that is used for the rewrite of + // Variant of the LEFT ANTI JOIN that is used for the equal of // NOT IN subqueries. It can have a single equality join conjunct // that returns TRUE when the rhs is NULL. NULL_AWARE_LEFT_ANTI_JOIN("NULL AWARE LEFT ANTI JOIN", diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java new file mode 100644 index 00000000000000..0a9e9b1d0ae496 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.FunctionSet; + +public class MVColumnBitmapUnionPattern implements MVColumnPattern { + @Override + public boolean match(Expr expr) { + if (!(expr instanceof FunctionCallExpr)) { + return false; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + String fnNameString = fnExpr.getFnName().getFunction(); + if (!fnNameString.equalsIgnoreCase(FunctionSet.BITMAP_UNION)) { + return false; + } + if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { + return false; + } + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")) { + return false; + } + if (child0FnExpr.getChild(0) instanceof SlotRef) { + return true; + } else if (child0FnExpr.getChild(0) instanceof CastExpr) { + CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); + if (!(castExpr.getChild(0) instanceof SlotRef)) { + return false; + } + return true; + } else { + return false; + } + } + + @Override + public String toString() { + return "bitmap_union(to_bitmap(column)) or bitmap_union(bitmap_column) in agg table"; + } +} diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java new file mode 100644 index 00000000000000..b15e092706b9b3 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +public class MVColumnHLLUnionPattern implements MVColumnPattern { + @Override + public boolean match(Expr expr) { + if (!(expr instanceof FunctionCallExpr)) { + return false; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + String fnNameString = fnExpr.getFnName().getFunction(); + if (!fnNameString.equalsIgnoreCase("hll_union")){ + return false; + } + if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { + return false; + } + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { + return false; + } + if (child0FnExpr.getChild(0) instanceof SlotRef) { + return true; + } else if (child0FnExpr.getChild(0) instanceof CastExpr) { + CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); + if (!(castExpr.getChild(0) instanceof SlotRef)) { + return false; + } + return true; + } else { + return false; + } + } + + @Override + public String toString() { + return "hll_union(hll_hash(column))"; + } +} diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java index e92bd4e9c1e653..d797e6b0c7abba 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java @@ -19,6 +19,11 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.DdlException; + +import com.google.common.base.Preconditions; /** * This is a result of semantic analysis for AddMaterializedViewClause. @@ -35,8 +40,18 @@ public class MVColumnItem { private boolean isAggregationTypeImplicit; private Expr defineExpr; - public MVColumnItem(String name) { + public MVColumnItem(String name, Type type, AggregateType aggregateType, boolean isAggregationTypeImplicit, + Expr defineExpr) { + this.name = name; + this.type = type; + this.aggregationType = aggregateType; + this.isAggregationTypeImplicit = isAggregationTypeImplicit; + this.defineExpr = defineExpr; + } + + public MVColumnItem(String name, Type type) { this.name = name; + this.type = type; } public String getName() { @@ -79,4 +94,19 @@ public Expr getDefineExpr() { public void setDefineExpr(Expr defineExpr) { this.defineExpr = defineExpr; } + + public Column toMVColumn(OlapTable olapTable) throws DdlException { + Column baseColumn = olapTable.getColumn(name); + if (baseColumn == null) { + Preconditions.checkNotNull(defineExpr != null); + Column result = new Column(name, type, isKey, aggregationType, ColumnDef.DefaultValue.ZERO, ""); + result.setDefineExpr(defineExpr); + return result; + } else { + Column result = new Column(baseColumn); + result.setIsKey(isKey); + result.setAggregationType(aggregationType, isAggregationTypeImplicit); + return result; + } + } } diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java new file mode 100644 index 00000000000000..7b045a79033d75 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +public class MVColumnOneChildPattern implements MVColumnPattern { + + private String functionName; + + public MVColumnOneChildPattern(String functionName) { + this.functionName = functionName; + } + + @Override + public boolean match(Expr expr) { + if (!(expr instanceof FunctionCallExpr)) { + return false; + } + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + String exprFnName = functionCallExpr.getFnName().getFunction(); + if (!exprFnName.equalsIgnoreCase(functionName)) { + return false; + } + if (functionCallExpr.getChildren().size() != 1) { + return false; + } + Expr functionChild0 = functionCallExpr.getChild(0); + if (functionChild0 instanceof SlotRef) { + return true; + } else if (functionChild0 instanceof CastExpr && (functionChild0.getChild(0) instanceof SlotRef)) { + return true; + } else { + return false; + } + + } + + @Override + public String toString() { + return functionName + "(column)"; + } +} diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnPattern.java new file mode 100644 index 00000000000000..ca3c09c847a8c5 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnPattern.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +public interface MVColumnPattern { + + boolean match(Expr expr); +} diff --git a/fe/src/main/java/org/apache/doris/analysis/Predicate.java b/fe/src/main/java/org/apache/doris/analysis/Predicate.java index 3e2c22b0fc435d..0f033b4e1ef0ce 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Predicate.java +++ b/fe/src/main/java/org/apache/doris/analysis/Predicate.java @@ -115,8 +115,8 @@ public static boolean canPushDownPredicate(Expr expr) { // and all SingleColumnPredicate will be rewritten as "column on the left and the constant on the right". // So usually the right child is constant. // - // But if there is a subquery in where clause, the planner will rewrite the subquery to join. - // During the rewrite, some auxiliary BinaryPredicate will be automatically generated, + // But if there is a subquery in where clause, the planner will equal the subquery to join. + // During the equal, some auxiliary BinaryPredicate will be automatically generated, // and these BinaryPredicates will not go through ExprRewriteRule. // As a result, these BinaryPredicates may be as "column on the right and the constant on the left". // Example can be found in QueryPlanTest.java -> testJoinPredicateTransitivityWithSubqueryInWhereClause(). diff --git a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java index 32cce4524784b2..ed70bb05c6d88d 100644 --- a/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -18,12 +18,11 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.Database; -import org.apache.doris.catalog.FunctionSet; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rewrite.ExprRewriter; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -37,6 +36,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Abstract base class for any statement that returns results @@ -113,6 +113,40 @@ public abstract class QueryStmt extends StatementBase { // represent the "INTO OUTFILE" clause protected OutFileClause outFileClause; + /** + * If the query stmt belongs to CreateMaterializedViewStmt, + * such as + * `CREATE MATERIALIZED VIEW mv AS SELECT bitmap_union(to_bitmap(k1)) from table` + * query stmt will not be rewrite by MVRewriter. + * The `bitmap_union(to_bitmap(k1))` is the definition of the mv column rather then a expr. + * So `forbiddenMVRewrite` will be set to true to protect the definition of the mv column from being overwritten. + *

+ * In other query case, `forbiddenMVRewrite` is always false. + */ + private boolean forbiddenMVRewrite = false; + + /** + * If the tuple id in `disableMVRewriteTupleIds`, the expr which belongs to this tuple will not be MVRewritten. + * Initially this set is an empty set. + * When the scan node is unable to match any index in selecting the materialized view, + * the tuple is added to this set. + * The query will be re-executed, and this tuple will not be mv rewritten. + * For example: + * TableA: (k1 int, k2 int, k3 int) + * MV: (k1 int, mv_bitmap_union_k2 bitmap bitmap_union) + * Query: select k3, bitmap_union(to_bitmap(k2)) from TableA + * First analyze: MV rewriter enable and this set is empty + * select k3, bitmap_union(mv_bitmap_union_k2) from TableA + * SingleNodePlanner: could not select any index for TableA + * Add table to disableMVRewriteTupleIds. + * `disableMVRewriteTupleIds` = {TableA} + * Re-executed: + * Second analyze: MV rewrite disable in table and use origin stmt. + * select k3, bitmap_union(to_bitmap(k2)) from TableA + * SingleNodePlanner: base index selected + */ + private Set disableTuplesMVRewriter = Sets.newHashSet(); + QueryStmt(ArrayList orderByElements, LimitElement limitElement) { this.orderByElements = orderByElements; this.limitElement = limitElement; @@ -208,36 +242,16 @@ public void setNeedToSql(boolean needToSql) { this.needToSql = needToSql; } - - // for bitmap type, we rewrite count distinct to bitmap_union_count - // for hll type, we rewrite count distinct to hll_union_agg - protected Expr rewriteCountDistinctForBitmapOrHLL(Expr expr, Analyzer analyzer) { - if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().isRewriteCountDistinct()) { + protected Expr rewriteQueryExprByMvColumnExpr(Expr expr, Analyzer analyzer) throws AnalysisException { + if (forbiddenMVRewrite) { return expr; } - - for (int i = 0; i < expr.getChildren().size(); ++i) { - expr.setChild(i, rewriteCountDistinctForBitmapOrHLL(expr.getChild(i), analyzer)); - } - - if (!(expr instanceof FunctionCallExpr)) { + if (expr.isBoundByTupleIds(disableTuplesMVRewriter.stream().collect(Collectors.toList()))) { return expr; } - - FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; - if (functionCallExpr.isCountDistinctBitmapOrHLL()) { - FunctionParams newParams = new FunctionParams(false, functionCallExpr.getParams().exprs()); - if (expr.getChild(0).type.isBitmapType()) { - FunctionCallExpr bitmapExpr = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, newParams); - bitmapExpr.analyzeNoThrow(analyzer); - return bitmapExpr; - } else { - FunctionCallExpr hllExpr = new FunctionCallExpr("hll_union_agg", newParams); - hllExpr.analyzeNoThrow(analyzer); - return hllExpr; - } - } - return expr; + ExprRewriter rewriter = analyzer.getMVExprRewriter(); + rewriter.reset(); + return rewriter.rewrite(expr, analyzer); } /** @@ -272,8 +286,8 @@ protected void createSortInfo(Analyzer analyzer) throws AnalysisException { // save the order by element after analyzed orderByElementsAfterAnalyzed = Lists.newArrayList(); for (int i = 0; i < orderByElements.size(); i++) { - // rewrite count distinct - orderingExprs.set(i, rewriteCountDistinctForBitmapOrHLL(orderingExprs.get(i), analyzer)); + // equal count distinct + orderingExprs.set(i, rewriteQueryExprByMvColumnExpr(orderingExprs.get(i), analyzer)); OrderByElement orderByElement = new OrderByElement(orderingExprs.get(i), isAscOrder.get(i), nullsFirstParams.get(i)); orderByElementsAfterAnalyzed.add(orderByElement); @@ -517,6 +531,26 @@ public void substituteResultExprs(ExprSubstitutionMap smap, Analyzer analyzer) { resultExprs = Expr.substituteList(resultExprs, smap, analyzer, true); } + public boolean isForbiddenMVRewrite() { + return forbiddenMVRewrite; + } + + public void forbiddenMVRewrite() { + this.forbiddenMVRewrite = true; + } + + public void updateDisableTuplesMVRewriter(TupleId tupleId) { + disableTuplesMVRewriter.add(tupleId); + } + + public void updateDisableTuplesMVRewriter(Set tupleIds) { + disableTuplesMVRewriter.addAll(tupleIds); + } + + public Set getDisableTuplesMVRewriter() { + return disableTuplesMVRewriter; + } + /** * Mark all slots that need to be materialized for the execution of this stmt. * This excludes slots referenced in resultExprs (it depends on the consumer of diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectList.java b/fe/src/main/java/org/apache/doris/analysis/SelectList.java index d56ed7d171ef88..3f3996d98454d9 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectList.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectList.java @@ -87,7 +87,7 @@ public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer) if (item.isStar()) { continue; } - // rewrite subquery in select list + // equal subquery in select list if (item.getExpr().contains(Predicates.instanceOf(Subquery.class))) { List subqueryExprs = Lists.newArrayList(); item.getExpr().collect(Subquery.class, subqueryExprs); diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java index 2ad443b23b68b4..d3b8419c31ef0b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -366,7 +366,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { item.getExpr().contains(Predicates.instanceOf(Subquery.class))) { throw new AnalysisException("Subquery is not supported in the select list."); } - Expr expr = rewriteCountDistinctForBitmapOrHLL(item.getExpr(), analyzer); + Expr expr = rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer); resultExprs.add(expr); SlotRef aliasRef = new SlotRef(null, item.toColumnLabel()); Expr existingAliasExpr = aliasSMap.get(aliasRef); @@ -661,7 +661,7 @@ private Expr processDuplicateOrs(List> exprs) { : makeCompound(temp, CompoundPredicate.Operator.OR); } if (LOG.isDebugEnabled()) { - LOG.debug("rewrite ors: " + result.toSql()); + LOG.debug("equal ors: " + result.toSql()); } return result; } @@ -995,7 +995,7 @@ private void analyzeAggregation(Analyzer analyzer) throws AnalysisException { * TODO: the a.key should be replaced by a.k1 instead of unknown column 'key' in 'a' */ havingClauseAfterAnaylzed = havingClause.substitute(aliasSMap, analyzer, false); - havingClauseAfterAnaylzed = rewriteCountDistinctForBitmapOrHLL(havingClauseAfterAnaylzed, analyzer); + havingClauseAfterAnaylzed = rewriteQueryExprByMvColumnExpr(havingClauseAfterAnaylzed, analyzer); havingClauseAfterAnaylzed.checkReturnsBool("HAVING clause", true); // can't contain analytic exprs Expr analyticExpr = havingClauseAfterAnaylzed.findFirstOf(AnalyticExpr.class); @@ -1353,7 +1353,7 @@ private void createAnalyticInfo(Analyzer analyzer) throws AnalysisException { Expr.substituteList(analyticExprs, rewriteSmap, analyzer, false); // This is to get rid the original exprs which have been rewritten. analyticExprs.clear(); - // Collect the new exprs introduced through the rewrite and the non-rewrite exprs. + // Collect the new exprs introduced through the equal and the non-equal exprs. TreeNode.collect(updatedAnalyticExprs, AnalyticExpr.class, analyticExprs); } @@ -1387,7 +1387,7 @@ public void rewriteExprs(ExprRewriter rewriter) throws AnalysisException { for (TableRef ref : fromClause_) { ref.rewriteExprs(rewriter, analyzer); } - // Also rewrite exprs in the statements of subqueries. + // Also equal exprs in the statements of subqueries. List subqueryExprs = Lists.newArrayList(); if (whereClause != null) { whereClause = rewriter.rewrite(whereClause, analyzer); @@ -1423,7 +1423,7 @@ private void rewriteSelectList(ExprRewriter rewriter) throws AnalysisException { selectList.rewriteExprs(rewriter, analyzer); } - /** rewrite subquery in case when to an inline view + /** equal subquery in case when to an inline view * subquery in case when statement like * * SELECT CASE @@ -1440,7 +1440,7 @@ private void rewriteSelectList(ExprRewriter rewriter) throws AnalysisException { * ) * END AS kk4 * FROM t; - * this statement will be rewrite to + * this statement will be equal to * * SELECT CASE * WHEN t1.a > k4 THEN t2.a diff --git a/fe/src/main/java/org/apache/doris/analysis/SetOperationStmt.java b/fe/src/main/java/org/apache/doris/analysis/SetOperationStmt.java index f03ceccbd621ef..3a7ac703e5193e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SetOperationStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/SetOperationStmt.java @@ -661,10 +661,10 @@ public void analyze(Analyzer parent) throws AnalysisException, UserException { if (isAnalyzed()) { return; } - // union statement support const expr, so not need to rewrite + // union statement support const expr, so not need to equal if (operation != Operation.UNION && queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).fromClause_.isEmpty()) { - // rewrite select 1 to select * from (select 1) __DORIS_DUAL__ , because when using select 1 it will be + // equal select 1 to select * from (select 1) __DORIS_DUAL__ , because when using select 1 it will be // transformed to a union node, select 1 is a literal, it doesn't have a tuple but will produce a slot, // this will cause be core dump QueryStmt inlineQuery = queryStmt.clone(); diff --git a/fe/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/src/main/java/org/apache/doris/analysis/SlotRef.java index b7ea3ff2328cc5..13cdd2dc5faa86 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -103,12 +104,24 @@ public SlotId getSlotId() { return desc.getId(); } + public Column getColumn() { + if (desc == null) { + return null; + } else { + return desc.getColumn(); + } + } + // NOTE: this is used to set tblName to null, // so we can to get the only column name when calling toSql public void setTblName(TableName name) { this.tblName = name; } + public void setDesc(SlotDescriptor desc) { + this.desc = desc; + } + @Override public void vectorizedAnalyze(Analyzer analyzer) { computeOutputColumn(analyzer); @@ -186,6 +199,15 @@ public String toMySql() { } public TableName getTableName() { + Preconditions.checkState(isAnalyzed); + Preconditions.checkNotNull(desc); + if (tblName == null) { + Preconditions.checkNotNull(desc.getParent()); + if (desc.getParent().getRef() == null) { + return null; + } + return desc.getParent().getRef().getName(); + } return tblName; } @@ -273,14 +295,14 @@ public void getIds(List tupleIds, List slotIds) { } @Override - public void getTableNameToColumnNames(Map> tupleDescToColumnNames) { + public void getTableIdToColumnNames(Map> tableIdToColumnNames) { Preconditions.checkState(desc != null); if (!desc.isMaterialized()) { return; } if (col == null) { for (Expr expr : desc.getSourceExprs()) { - expr.getTableNameToColumnNames(tupleDescToColumnNames); + expr.getTableIdToColumnNames(tableIdToColumnNames); } } else { Table table = desc.getParent().getTable(); @@ -288,16 +310,22 @@ public void getTableNameToColumnNames(Map> tupleDescToColumn // Maybe this column comes from inline view. return; } - String tableName = table.getName(); - Set columnNames = tupleDescToColumnNames.get(tableName); + Long tableId = table.getId(); + Set columnNames = tableIdToColumnNames.get(tableId); if (columnNames == null) { columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - tupleDescToColumnNames.put(tableName, columnNames); + tableIdToColumnNames.put(tableId, columnNames); } columnNames.add(desc.getColumn().getName()); } } + public Table getTable() { + Preconditions.checkState(desc != null); + Table table = desc.getParent().getTable(); + return table; + } + public String getColumnName() { return col; } diff --git a/fe/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/src/main/java/org/apache/doris/analysis/StmtRewriter.java index 05674f5a4d3b4d..f6b10b7735cea5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/StmtRewriter.java +++ b/fe/src/main/java/org/apache/doris/analysis/StmtRewriter.java @@ -64,7 +64,7 @@ public static StatementBase rewrite(Analyzer analyzer, StatementBase parsedStmt) } /** - * Calls the appropriate rewrite method based on the specific type of query stmt. See + * Calls the appropriate equal method based on the specific type of query stmt. See * rewriteSelectStatement() and rewriteUnionStatement() documentation. */ public static QueryStmt rewriteQueryStatement(QueryStmt stmt, Analyzer analyzer) @@ -96,7 +96,7 @@ private static SelectStmt rewriteSelectStatement(SelectStmt stmt, Analyzer analy if (result.hasWhereClause()) { // Push negation to leaf operands. result.whereClause = Expr.pushNegationToOperands(result.whereClause); - // Check if we can rewrite the subqueries in the WHERE clause. OR predicates with + // Check if we can equal the subqueries in the WHERE clause. OR predicates with // subqueries are not supported. if (hasSubqueryInDisjunction(result.whereClause)) { throw new AnalysisException("Subqueries in OR predicates are not supported: " @@ -116,15 +116,15 @@ private static SelectStmt rewriteSelectStatement(SelectStmt stmt, Analyzer analy /** * Rewrite having subquery. - * Step1: rewrite having subquery to where subquery - * Step2: rewrite where subquery + * Step1: equal having subquery to where subquery + * Step2: equal where subquery *

* For example: * select cs_item_sk, sum(cs_sales_price) from catalog_sales a group by cs_item_sk * having sum(cs_sales_price) > * (select min(cs_sales_price) from catalog_sales b where a.cs_item_sk = b.cs_item_sk); *

- * Step1: rewrite having subquery to where subquery + * Step1: equal having subquery to where subquery * Outer query is changed to inline view in rewritten query * Inline view of outer query: * from (select cs_item_sk, sum(cs_sales_price) sum_cs_sales_price from catalog_sales group by cs_item_sk) a @@ -137,7 +137,7 @@ private static SelectStmt rewriteSelectStatement(SelectStmt stmt, Analyzer analy * where a.sum_cs_sales_price > * (select min(cs_sales_price) from catalog_sales b where a.cs_item_sk = b.cs_item_sk) *

- * Step2: rewrite where subquery + * Step2: equal where subquery * Inline view of subquery: * from (select b.cs_item_sk, min(cs_sales_price) from catalog_sales b group by cs_item_sk) c * Rewritten correlated predicate: @@ -269,7 +269,7 @@ private static SelectStmt rewriteHavingClauseSubqueries(SelectStmt stmt, Analyze } LOG.info("New stmt {} is constructed after rewritten subquery of having clause.", result.toSql()); - // rewrite where subquery + // equal where subquery result = rewriteSelectStatement(result, analyzer); LOG.debug("The final stmt is " + result.toSql()); return result; @@ -436,7 +436,7 @@ private static void rewriteWhereClauseSubqueries( stmt.whereClause = stmt.whereClause.substitute(smap, analyzer, false); boolean hasNewVisibleTuple = false; - // Recursively rewrite all the exprs that contain subqueries and merge them + // Recursively equal all the exprs that contain subqueries and merge them // with 'stmt'. for (Expr expr : exprsWithSubqueries) { if (mergeExpr(stmt, rewriteExpr(expr, analyzer), analyzer)) { @@ -490,7 +490,7 @@ private static boolean canEliminate(Expr expr) { */ private static Expr rewriteExpr(Expr expr, Analyzer analyzer) throws AnalysisException { - // Extract the subquery and rewrite it. + // Extract the subquery and equal it. Subquery subquery = expr.getSubquery(); Preconditions.checkNotNull(subquery); QueryStmt rewrittenStmt = rewriteSelectStatement((SelectStmt) subquery.getStatement(), subquery.getAnalyzer()); @@ -541,7 +541,7 @@ private static boolean mergeExpr(SelectStmt stmt, Expr expr, // Create a new inline view from the subquery stmt. The inline view will be added // to the stmt's table refs later. Explicitly set the inline view's column labels // to eliminate any chance that column aliases from the parent query could reference - // select items from the inline view after the rewrite. + // select items from the inline view after the equal. List colLabels = Lists.newArrayList(); // add a new alias for all of columns in subquery for (int i = 0; i < subqueryStmt.getColLabels().size(); ++i) { @@ -555,7 +555,7 @@ private static boolean mergeExpr(SelectStmt stmt, Expr expr, List onClauseConjuncts = extractCorrelatedPredicates(subqueryStmt); if (!onClauseConjuncts.isEmpty()) { canRewriteCorrelatedSubquery(expr, onClauseConjuncts); - // For correlated subqueries that are eligible for rewrite by transforming + // For correlated subqueries that are eligible for equal by transforming // into a join, a LIMIT clause has no effect on the results, so we can // safely remove it. // subqueryStmt.limitElement = null; @@ -715,7 +715,7 @@ private static boolean mergeExpr(SelectStmt stmt, Expr expr, expr.toSql()); } - // We can rewrite the aggregate subquery using a cross join. All conjuncts + // We can equal the aggregate subquery using a cross join. All conjuncts // that were extracted from the subquery are added to stmt's WHERE clause. stmt.whereClause = CompoundPredicate.createConjunction(onClausePredicate, stmt.whereClause); @@ -911,11 +911,11 @@ private static boolean canExtractCorrelatedPredicates(Expr expr, } /** - * Checks if an expr containing a correlated subquery is eligible for rewrite by + * Checks if an expr containing a correlated subquery is eligible for equal by * tranforming into a join. 'correlatedPredicates' contains the correlated * predicates identified in the subquery. Throws an AnalysisException if 'expr' - * is not eligible for rewrite. - * TODO: Merge all the rewrite eligibility tests into a single function. + * is not eligible for equal. + * TODO: Merge all the equal eligibility tests into a single function. */ private static void canRewriteCorrelatedSubquery( Expr expr, List correlatedPredicates) diff --git a/fe/src/main/java/org/apache/doris/analysis/TupleDescriptor.java b/fe/src/main/java/org/apache/doris/analysis/TupleDescriptor.java index c15b2c1b2f1a07..1cfdf47ec58b9a 100644 --- a/fe/src/main/java/org/apache/doris/analysis/TupleDescriptor.java +++ b/fe/src/main/java/org/apache/doris/analysis/TupleDescriptor.java @@ -259,7 +259,7 @@ public void materializeSlots() { for (SlotDescriptor slot: slots) slot.setIsMaterialized(true); } - public void getTableNameToColumnNames(Map> tupleDescToColumnNames) { + public void getTableIdToColumnNames(Map> tableIdToColumnNames) { for (SlotDescriptor slotDescriptor : slots) { if (!slotDescriptor.isMaterialized()) { continue; @@ -269,16 +269,16 @@ public void getTableNameToColumnNames(Map> tupleDescToColumn Preconditions.checkState(parent != null); Table table = parent.getTable(); Preconditions.checkState(table != null); - String tableName = table.getName(); - Set columnNames = tupleDescToColumnNames.get(tableName); + Long tableId = table.getId(); + Set columnNames = tableIdToColumnNames.get(tableId); if (columnNames == null) { columnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - tupleDescToColumnNames.put(tableName, columnNames); + tableIdToColumnNames.put(tableId, columnNames); } columnNames.add(slotDescriptor.getColumn().getName()); } else { for (Expr expr : slotDescriptor.getSourceExprs()) { - expr.getTableNameToColumnNames(tupleDescToColumnNames); + expr.getTableIdToColumnNames(tableIdToColumnNames); } } } diff --git a/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java b/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java index f318cc25cdbae1..34e1416780858d 100644 --- a/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java +++ b/fe/src/main/java/org/apache/doris/catalog/AggregateFunction.java @@ -80,7 +80,7 @@ public class AggregateFunction extends Function { private boolean isAggregateFn; // True if this function returns a non-null value on an empty input. It is used - // primarily during the rewrite of scalar subqueries. + // primarily during the equal of scalar subqueries. // TODO: Instead of manually setting this flag, we should identify this // property from the function itself (e.g. evaluating the function on an // empty input in BE). diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index c0ac047eb2bf10..2611a15cbddcca 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -346,6 +346,17 @@ public List getVisibleIndex() { return partition.getMaterializedIndices(IndexExtState.VISIBLE); } + public Column getVisibleColumn(String columnName) { + for (MaterializedIndexMeta meta : getVisibleIndexIdToMeta().values()) { + for (Column column : meta.getSchema()) { + if (column.getName().equalsIgnoreCase(columnName)) { + return column; + } + } + } + return null; + } + // this is only for schema change. public void renameIndexForSchemaChange(String name, String newName) { long idxId = indexNameToId.remove(name); diff --git a/fe/src/main/java/org/apache/doris/load/BrokerFileGroupAggInfo.java b/fe/src/main/java/org/apache/doris/load/BrokerFileGroupAggInfo.java index 55b6c134b00200..5214cce4b2d448 100644 --- a/fe/src/main/java/org/apache/doris/load/BrokerFileGroupAggInfo.java +++ b/fe/src/main/java/org/apache/doris/load/BrokerFileGroupAggInfo.java @@ -85,7 +85,7 @@ * PARTITION (p2, p3) * * will throw an Exception, because there is an overlap partition(p2) between 2 data descriptions. And we - * currently not allow this. You can rewrite the data descriptions like this: + * currently not allow this. You can equal the data descriptions like this: * * DATA INFILE("hdfs://hdfs_host:hdfs_port/input/file1") * INTO TABLE `tbl1` diff --git a/fe/src/main/java/org/apache/doris/planner/LoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/LoadScanNode.java index e816b87ceb8f40..a8478b24d7f333 100644 --- a/fe/src/main/java/org/apache/doris/planner/LoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/LoadScanNode.java @@ -51,7 +51,7 @@ protected void initWhereExpr(Expr whereExpr, Analyzer analyzer) throws UserExcep } // substitute SlotRef in filter expression - // where expr must be rewrite first to transfer some predicates(eg: BetweenPredicate to BinaryPredicate) + // where expr must be equal first to transfer some predicates(eg: BetweenPredicate to BinaryPredicate) whereExpr = analyzer.getExprRewriter().rewrite(whereExpr, analyzer); List slots = Lists.newArrayList(); whereExpr.collect(SlotRef.class, slots); diff --git a/fe/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java b/fe/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java index 35a99905cb4def..13a63d4d52810f 100644 --- a/fe/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java +++ b/fe/src/main/java/org/apache/doris/planner/MaterializedViewSelector.java @@ -18,10 +18,10 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; -import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SelectStmt; +import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.TupleDescriptor; @@ -30,9 +30,10 @@ import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rewrite.mvrewrite.MVExprEquivalent; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -47,7 +48,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.TreeSet; @@ -70,16 +70,17 @@ public class MaterializedViewSelector { private final Analyzer analyzer; /** - * The key of following maps is table name. + * The key of following maps is table id. * The value of following maps is column names. * `columnNamesInPredicates` means that the column names in where clause. * And so on. */ - private Map> columnNamesInPredicates = Maps.newHashMap(); + private Map> columnNamesInPredicates = Maps.newHashMap(); private boolean isSPJQuery; - private Map> columnNamesInGrouping = Maps.newHashMap(); - private Map> aggregateColumnsInQuery = Maps.newHashMap(); - private Map> columnNamesInQueryOutput = Maps.newHashMap(); + private Map> columnNamesInGrouping = Maps.newHashMap(); + private Map> aggColumnsInQuery = Maps.newHashMap(); + // private Map> aggregateColumnsInQuery = Maps.newHashMap(); + private Map> columnNamesInQueryOutput = Maps.newHashMap(); private boolean disableSPJGView; private String reasonOfDisable; @@ -107,28 +108,30 @@ public BestIndexInfo selectBestMV(ScanNode scanNode) throws UserException { long start = System.currentTimeMillis(); Preconditions.checkState(scanNode instanceof OlapScanNode); OlapScanNode olapScanNode = (OlapScanNode) scanNode; - Map> candidateIndexIdToSchema = predicates(olapScanNode); + if (candidateIndexIdToSchema.keySet().size() == 0) { + return null; + } long bestIndexId = priorities(olapScanNode, candidateIndexIdToSchema); LOG.debug("The best materialized view is {} for scan node {} in query {}, cost {}", bestIndexId, scanNode.getId(), selectStmt.toSql(), (System.currentTimeMillis() - start)); return new BestIndexInfo(bestIndexId, isPreAggregation, reasonOfDisable); } - private Map> predicates(OlapScanNode scanNode) { + private Map> predicates(OlapScanNode scanNode) throws AnalysisException { // Step1: all of predicates is compensating predicates Map candidateIndexIdToMeta = scanNode.getOlapTable().getVisibleIndexIdToMeta(); OlapTable table = scanNode.getOlapTable(); Preconditions.checkState(table != null); - String tableName = table.getName(); + long tableId = table.getId(); // Step2: check all columns in compensating predicates are available in the view output - checkCompensatingPredicates(columnNamesInPredicates.get(tableName), candidateIndexIdToMeta); + checkCompensatingPredicates(columnNamesInPredicates.get(tableId), candidateIndexIdToMeta); // Step3: group by list in query is the subset of group by list in view or view contains no aggregation - checkGrouping(columnNamesInGrouping.get(tableName), candidateIndexIdToMeta); + checkGrouping(columnNamesInGrouping.get(tableId), candidateIndexIdToMeta); // Step4: aggregation functions are available in the view output - checkAggregationFunction(aggregateColumnsInQuery.get(tableName), candidateIndexIdToMeta); + checkAggregationFunction(aggColumnsInQuery.get(tableId), candidateIndexIdToMeta); // Step5: columns required to compute output expr are available in the view output - checkOutputColumns(columnNamesInQueryOutput.get(tableName), candidateIndexIdToMeta); + checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta); // Step6: if table type is aggregate and the candidateIndexIdToSchema is empty, if ((table.getKeysType() == KeysType.AGG_KEYS || table.getKeysType() == KeysType.UNIQUE_KEYS) && candidateIndexIdToMeta.size() == 0) { @@ -149,7 +152,7 @@ private Map> predicates(OlapScanNode scanNode) { */ compensateCandidateIndex(candidateIndexIdToMeta, scanNode.getOlapTable().getVisibleIndexIdToMeta(), table); - checkOutputColumns(columnNamesInQueryOutput.get(tableName), candidateIndexIdToMeta); + checkOutputColumns(columnNamesInQueryOutput.get(tableId), candidateIndexIdToMeta); } Map> result = Maps.newHashMap(); for (Map.Entry entry : candidateIndexIdToMeta.entrySet()) { @@ -332,18 +335,15 @@ aggregate mv index (k1, k2) + Joiner.on(",").join(candidateIndexIdToMeta.keySet())); } - private void checkAggregationFunction(Set aggregatedColumnsInQueryOutput, - Map candidateIndexIdToMeta) { + private void checkAggregationFunction(Set aggregatedColumnsInQueryOutput, + Map candidateIndexIdToMeta) throws AnalysisException { Iterator> iterator = candidateIndexIdToMeta.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry entry = iterator.next(); - List indexAggregatedColumns = Lists.newArrayList(); - List candidateIndexSchema = entry.getValue().getSchema(); - candidateIndexSchema.stream().filter(column -> column.isAggregated()) - .forEach(column -> indexAggregatedColumns.add( - new AggregatedColumn(column.getName(), column.getAggregationType().name()))); + MaterializedIndexMeta candidateIndexMeta = entry.getValue(); + List indexAggColumnExpsList = mvAggColumnsToExprList(candidateIndexMeta); // When the candidate index is SPJ type, it passes the verification directly - if (indexAggregatedColumns.size() == 0) { + if (indexAggColumnExpsList.size() == 0 && candidateIndexMeta.getKeysType() == KeysType.DUP_KEYS) { continue; } // When the query is SPJ type but the candidate index is SPJG type, it will not pass directly. @@ -360,7 +360,7 @@ private void checkAggregationFunction(Set aggregatedColumnsInQ continue; } // The aggregated columns in query output must be subset of the aggregated columns in view - if (!indexAggregatedColumns.containsAll(aggregatedColumnsInQueryOutput)) { + if (!aggFunctionsMatchAggColumns(aggregatedColumnsInQueryOutput, indexAggColumnExpsList)) { iterator.remove(); } } @@ -368,8 +368,8 @@ private void checkAggregationFunction(Set aggregatedColumnsInQ + Joiner.on(",").join(candidateIndexIdToMeta.keySet())); } - private void checkOutputColumns(Set columnNamesInQueryOutput, Map - candidateIndexIdToMeta) { + private void checkOutputColumns(Set columnNamesInQueryOutput, + Map candidateIndexIdToMeta) { if (columnNamesInQueryOutput == null) { return; } @@ -379,7 +379,7 @@ private void checkOutputColumns(Set columnNamesInQueryOutput, Map indexColumnNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); List candidateIndexSchema = entry.getValue().getSchema(); candidateIndexSchema.stream().forEach(column -> indexColumnNames.add(column.getName())); - // The aggregated columns in query output must be subset of the aggregated columns in view + // The columns in query output must be subset of the columns in SPJ view if (!indexColumnNames.containsAll(columnNamesInQueryOutput)) { iterator.remove(); } @@ -407,13 +407,13 @@ private void init() { // Step1: compute the columns in compensating predicates Expr whereClause = selectStmt.getWhereClause(); if (whereClause != null) { - whereClause.getTableNameToColumnNames(columnNamesInPredicates); + whereClause.getTableIdToColumnNames(columnNamesInPredicates); } for (TableRef tableRef : selectStmt.getTableRefs()) { if (tableRef.getOnClause() == null) { continue; } - tableRef.getOnClause().getTableNameToColumnNames(columnNamesInPredicates); + tableRef.getOnClause().getTableIdToColumnNames(columnNamesInPredicates); } if (selectStmt.getAggInfo() == null) { @@ -423,96 +423,79 @@ private void init() { if (selectStmt.getAggInfo().getGroupingExprs() != null) { List groupingExprs = selectStmt.getAggInfo().getGroupingExprs(); for (Expr expr : groupingExprs) { - expr.getTableNameToColumnNames(columnNamesInGrouping); + expr.getTableIdToColumnNames(columnNamesInGrouping); } } // Step3: compute the aggregation function for (FunctionCallExpr aggExpr : selectStmt.getAggInfo().getAggregateExprs()) { - // Only sum, min, max function could appear in materialized views. - // The number of children in these functions is one. - if (aggExpr.getChildren().size() != 1) { - reasonOfDisable = "aggExpr has more than one child"; - disableSPJGView = true; - break; - } - Expr aggChild0 = aggExpr.getChild(0); - if (aggChild0 instanceof SlotRef) { - SlotRef slotRef = (SlotRef) aggChild0; - Table table = slotRef.getDesc().getParent().getTable(); - /* If this column come from subquery, the parent table will be null - * For example: select k1 from (select name as k1 from tableA) A - * The parent table of k1 column in outer query is null. - */ - if (table == null) { - continue; - } - Preconditions.checkState(slotRef.getColumnName() != null); - addAggregatedColumn(slotRef.getColumnName(), aggExpr.getFnName().getFunction(), - table.getName()); - } else if ((aggChild0 instanceof CastExpr) && (aggChild0.getChild(0) instanceof SlotRef)) { - SlotRef slotRef = (SlotRef) aggChild0.getChild(0); - Table table = slotRef.getDesc().getParent().getTable(); - /* - * Same as above - */ - if (table == null) { - continue; - } - Preconditions.checkState(slotRef.getColumnName() != null); - addAggregatedColumn(slotRef.getColumnName(), aggExpr.getFnName().getFunction(), - table.getName()); - } else { - reasonOfDisable = "aggExpr.getChild(0)[" + aggExpr.getChild(0).debugString() - + "] is not SlotRef or CastExpr|CaseExpr"; + Map> tableIdToAggColumnNames = Maps.newHashMap(); + aggExpr.getTableIdToColumnNames(tableIdToAggColumnNames); + // count(*): tableIdToAggColumnNames is empty which must forbidden the SPJG MV. + // TODO(ml): support count(*) + if (tableIdToAggColumnNames.size() != 1) { + reasonOfDisable = "aggExpr[" + aggExpr.debugString() + "] should involved only one column"; disableSPJGView = true; break; } + addAggColumnInQuery(tableIdToAggColumnNames.keySet().stream().findFirst().get(), aggExpr); // TODO(ml): select rollup by case expr } } // Step4: compute the output column // ISSUE-3174: all of columns which belong to top tuple should be considered in selector. - List topTupleIds = selectStmt.getTableRefIdsWithoutInlineView(); - for (TupleId tupleId : topTupleIds) { + List tupleIds = selectStmt.getTableRefIdsWithoutInlineView(); + for (TupleId tupleId : tupleIds) { TupleDescriptor tupleDescriptor = analyzer.getTupleDesc(tupleId); - tupleDescriptor.getTableNameToColumnNames(columnNamesInQueryOutput); + tupleDescriptor.getTableIdToColumnNames(columnNamesInQueryOutput); } } - private void addAggregatedColumn(String columnName, String functionName, String tableName) { - AggregatedColumn newAggregatedColumn = new AggregatedColumn(columnName, functionName); - Set aggregatedColumns = aggregateColumnsInQuery.computeIfAbsent(tableName, k -> Sets.newHashSet()); - aggregatedColumns.add(newAggregatedColumn); - } - - class AggregatedColumn { - private String columnName; - private String aggFunctionName; - - public AggregatedColumn(String columnName, String aggFunctionName) { - this.columnName = columnName; - this.aggFunctionName = aggFunctionName; + private void addAggColumnInQuery(Long tableId, FunctionCallExpr fnExpr) { + Set aggColumns = aggColumnsInQuery.get(tableId); + if (aggColumns == null) { + aggColumns = Sets.newHashSet(); + aggColumnsInQuery.put(tableId, aggColumns); } + aggColumns.add(fnExpr); + } - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; + private boolean aggFunctionsMatchAggColumns(Set queryExprList, + List mvColumnExprList) throws AnalysisException { + for (Expr queryExpr : queryExprList) { + boolean match = false; + for (Expr mvColumnExpr : mvColumnExprList) { + if (MVExprEquivalent.mvExprEqual(queryExpr, mvColumnExpr)) { + match = true; + break; + } } - if (!(obj instanceof AggregatedColumn)) { + + if (!match) { return false; } - AggregatedColumn input = (AggregatedColumn) obj; - return this.columnName.equalsIgnoreCase(input.columnName) - && this.aggFunctionName.equalsIgnoreCase(input.aggFunctionName); } + return true; + } - @Override - public int hashCode() { - return Objects.hash(this.columnName, this.aggFunctionName); + private List mvAggColumnsToExprList(MaterializedIndexMeta mvMeta) { + List result = Lists.newArrayList(); + List schema = mvMeta.getSchema(); + for (Column column : schema) { + if (!column.isAggregated()) { + continue; + } + SlotRef slotRef = new SlotRef(null, column.getName()); + // This slot desc is only used to temporarily store column that will be used in subsequent MVExprRewriter. + SlotDescriptor slotDescriptor = new SlotDescriptor(null, null); + slotDescriptor.setColumn(column); + slotRef.setDesc(slotDescriptor); + FunctionCallExpr fnExpr = new FunctionCallExpr(column.getAggregationType().name(), + Lists.newArrayList(slotRef)); + result.add(fnExpr); } + return result; } public class BestIndexInfo { diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index c7a993fb4fa85b..9c55afb17a3917 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; @@ -226,7 +227,9 @@ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isP this.selectedIndexId = selectedIndexId; this.isPreAggregation = isPreAggregation; this.reasonOfPreAggregation = reasonOfDisable; + long start = System.currentTimeMillis(); computeTabletInfo(); + LOG.debug("distribution prune cost: {} ms", (System.currentTimeMillis() - start)); LOG.info("Using the new scan range info instead of the old one. {}, {}", situation ,scanRangeInfo); } else { LOG.warn("Using the old scan range info instead of the new one. {}, {}", situation, scanRangeInfo); @@ -441,6 +444,10 @@ private void getScanRangeLocations(Analyzer analyzer) throws UserException { isPreAggregation = true; } // TODO: Remove the logic of old selector. + if (olapTable.getKeysType() == KeysType.DUP_KEYS) { + LOG.debug("The best index will be selected later in mv selector"); + return; + } final RollupSelector rollupSelector = new RollupSelector(analyzer, desc, olapTable); selectedIndexId = rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts, isPreAggregation); LOG.debug("select best roll up cost: {} ms, best index id: {}", @@ -650,6 +657,11 @@ public void collectColumns(Analyzer analyzer, Set equivalenceColumns, Se } } + public TupleId getTupleId(){ + Preconditions.checkNotNull(desc); + return desc.getId(); + } + private boolean isEquivalenceExpr(Expr expr) { if (expr instanceof InPredicate) { diff --git a/fe/src/main/java/org/apache/doris/planner/Planner.java b/fe/src/main/java/org/apache/doris/planner/Planner.java index 77665dff6039fe..a88de531cd681b 100644 --- a/fe/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/src/main/java/org/apache/doris/planner/Planner.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.UserException; +import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TQueryOptions; @@ -162,7 +163,10 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue analyzer.getDescTbl().computeMemLayout(); singleNodePlan.finalize(analyzer); // materialized view selector - singleNodePlanner.selectMaterializedView(queryStmt, analyzer); + boolean selectFailed = singleNodePlanner.selectMaterializedView(queryStmt, analyzer); + if (selectFailed) { + throw new MVSelectFailedException("Failed to select materialize view"); + } if (queryOptions.num_nodes == 1) { // single-node execution; we're almost done singleNodePlan = addUnassignedConjuncts(analyzer, singleNodePlan); diff --git a/fe/src/main/java/org/apache/doris/planner/RollupSelector.java b/fe/src/main/java/org/apache/doris/planner/RollupSelector.java index 88ed70316a9054..035e1ad29c6af3 100644 --- a/fe/src/main/java/org/apache/doris/planner/RollupSelector.java +++ b/fe/src/main/java/org/apache/doris/planner/RollupSelector.java @@ -75,7 +75,6 @@ public long selectBestRollup( return v2RollupIndexId; } } - // Get first partition to select best prefix index rollups, because MaterializedIndex ids in one rollup's partitions are all same. final List bestPrefixIndexRollups = selectBestPrefixIndexRollup(conjuncts, isPreAggregation); return selectBestRowCountRollup(bestPrefixIndexRollups, partitionIds); diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index bef038d7a35df0..52c7b4ddf3888b 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -752,17 +752,20 @@ private PlanNode createRepeatNodePlan(SelectStmt selectStmt, Analyzer analyzer, return root; } - public void selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) throws UserException { + public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) + throws UserException { + boolean selectFailed = false; if (queryStmt instanceof SelectStmt) { SelectStmt selectStmt = (SelectStmt) queryStmt; for (TableRef tableRef : selectStmt.getTableRefs()) { if (tableRef instanceof InlineViewRef) { - selectMaterializedView(((InlineViewRef) tableRef).getViewStmt(), analyzer); + selectFailed |= selectMaterializedView(((InlineViewRef) tableRef).getViewStmt(), + ((InlineViewRef) tableRef).getAnalyzer()); } } List scanNodeList = selectStmtToScanNodes.get(selectStmt.getId()); if (scanNodeList == null) { - return; + return selectFailed; } MaterializedViewSelector materializedViewSelector = new MaterializedViewSelector(selectStmt, analyzer); for (ScanNode scanNode : scanNodeList) { @@ -773,19 +776,26 @@ public void selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) throw if (olapScanNode.getSelectedPartitionIds().size() == 0 && !FeConstants.runningUnitTest) { continue; } - MaterializedViewSelector.BestIndexInfo bestIndexInfo = materializedViewSelector.selectBestMV( - olapScanNode); - olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), bestIndexInfo.isPreAggregation(), - bestIndexInfo.getReasonOfDisable()); + MaterializedViewSelector.BestIndexInfo bestIndexInfo = materializedViewSelector.selectBestMV(olapScanNode); + if (bestIndexInfo == null) { + selectFailed |= true; + TupleId tupleId = olapScanNode.getTupleId(); + selectStmt.updateDisableTuplesMVRewriter(tupleId); + LOG.info("MV rewriter of tuple [] will be disable", tupleId); + continue; + } + olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), + bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable()); } } else { Preconditions.checkState(queryStmt instanceof SetOperationStmt); SetOperationStmt unionStmt = (SetOperationStmt) queryStmt; for (SetOperationStmt.SetOperand unionOperand : unionStmt.getOperands()) { - selectMaterializedView(unionOperand.getQueryStmt(), analyzer); + selectFailed |= selectMaterializedView(unionOperand.getQueryStmt(), analyzer); } } + return selectFailed; } /** diff --git a/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java index 6c862932bc067c..8202e0f0ba54d1 100644 --- a/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -49,7 +49,6 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.util.DebugUtil; @@ -57,8 +56,8 @@ import org.apache.doris.common.util.RuntimeProfile; import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.metric.MetricRepo; import org.apache.doris.load.EtlJobType; +import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlEofPacket; import org.apache.doris.mysql.MysqlSerializer; @@ -69,6 +68,7 @@ import org.apache.doris.qe.QueryDetailQueue; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.rewrite.ExprRewriter; +import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.rpc.RpcException; import org.apache.doris.task.LoadEtlTask; import org.apache.doris.thrift.TExplainLevel; @@ -371,8 +371,7 @@ private void unLock(Map dbs) { } // Analyze one statement to structure in memory. - public void analyze(TQueryOptions tQueryOptions) throws AnalysisException, UserException, - NotImplementedException { + public void analyze(TQueryOptions tQueryOptions) throws UserException { LOG.info("begin to analyze stmt: {}, forwarded stmt id: {}", context.getStmtId(), context.getForwardedStmtId()); // parsedStmt may already by set when constructing this StmtExecutor(); @@ -440,59 +439,10 @@ public void analyze(TQueryOptions tQueryOptions) throws AnalysisException, UserE lock(dbs); try { - parsedStmt.analyze(analyzer); - if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { - boolean isExplain = parsedStmt.isExplain(); - boolean isVerbose = parsedStmt.isVerbose(); - // Apply expr and subquery rewrites. - boolean reAnalyze = false; - - ExprRewriter rewriter = analyzer.getExprRewriter(); - rewriter.reset(); - parsedStmt.rewriteExprs(rewriter); - reAnalyze = rewriter.changed(); - if (analyzer.containSubquery()) { - parsedStmt = StmtRewriter.rewrite(analyzer, parsedStmt); - reAnalyze = true; - } - if (reAnalyze) { - // The rewrites should have no user-visible effect. Remember the original result - // types and column labels to restore them after the rewritten stmt has been - // reset() and re-analyzed. - List origResultTypes = Lists.newArrayList(); - for (Expr e: parsedStmt.getResultExprs()) { - origResultTypes.add(e.getType()); - } - List origColLabels = - Lists.newArrayList(parsedStmt.getColLabels()); - - // Re-analyze the stmt with a new analyzer. - analyzer = new Analyzer(context.getCatalog(), context); - - // query re-analyze - parsedStmt.reset(); - parsedStmt.analyze(analyzer); - - // Restore the original result types and column labels. - parsedStmt.castResultExprs(origResultTypes); - parsedStmt.setColLabels(origColLabels); - if (LOG.isTraceEnabled()) { - LOG.trace("rewrittenStmt: " + parsedStmt.toSql()); - } - if (isExplain) parsedStmt.setIsExplain(isExplain, isVerbose); - } - } - - // create plan - planner = new Planner(); - if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { - planner.plan(parsedStmt, analyzer, tQueryOptions); - } else { - planner.plan(((CreateTableAsSelectStmt) parsedStmt).getInsertStmt(), - analyzer, new TQueryOptions()); - } - // TODO(zc): - // Preconditions.checkState(!analyzer.hasUnassignedConjuncts()); + analyzeAndGenerateQueryPlan(tQueryOptions); + } catch (MVSelectFailedException e) { + resetAnalyzerAndStmt(); + analyzeAndGenerateQueryPlan(tQueryOptions); } catch (UserException e) { throw e; } catch (Exception e) { @@ -513,6 +463,68 @@ public void analyze(TQueryOptions tQueryOptions) throws AnalysisException, UserE } } + private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws UserException { + parsedStmt.analyze(analyzer); + if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { + boolean isExplain = parsedStmt.isExplain(); + boolean isVerbose = parsedStmt.isVerbose(); + // Apply expr and subquery rewrites. + boolean reAnalyze = false; + + ExprRewriter rewriter = analyzer.getExprRewriter(); + rewriter.reset(); + parsedStmt.rewriteExprs(rewriter); + reAnalyze = rewriter.changed(); + if (analyzer.containSubquery()) { + parsedStmt = StmtRewriter.rewrite(analyzer, parsedStmt); + reAnalyze = true; + } + if (reAnalyze) { + // The rewrites should have no user-visible effect. Remember the original result + // types and column labels to restore them after the rewritten stmt has been + // reset() and re-analyzed. + List origResultTypes = Lists.newArrayList(); + for (Expr e: parsedStmt.getResultExprs()) { + origResultTypes.add(e.getType()); + } + List origColLabels = + Lists.newArrayList(parsedStmt.getColLabels()); + + // Re-analyze the stmt with a new analyzer. + analyzer = new Analyzer(context.getCatalog(), context); + + // query re-analyze + parsedStmt.reset(); + parsedStmt.analyze(analyzer); + + // Restore the original result types and column labels. + parsedStmt.castResultExprs(origResultTypes); + parsedStmt.setColLabels(origColLabels); + if (LOG.isTraceEnabled()) { + LOG.trace("rewrittenStmt: " + parsedStmt.toSql()); + } + if (isExplain) parsedStmt.setIsExplain(isExplain, isVerbose); + } + } + + // create plan + planner = new Planner(); + if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { + planner.plan(parsedStmt, analyzer, tQueryOptions); + } else { + planner.plan(((CreateTableAsSelectStmt) parsedStmt).getInsertStmt(), + analyzer, new TQueryOptions()); + } + // TODO(zc): + // Preconditions.checkState(!analyzer.hasUnassignedConjuncts()); + } + + private void resetAnalyzerAndStmt() { + analyzer = new Analyzer(context.getCatalog(), context); + + parsedStmt.reset(); + } + // Because this is called by other thread public void cancel() { Coordinator coordRef = coord; diff --git a/fe/src/main/java/org/apache/doris/rewrite/ExprRewriteRule.java b/fe/src/main/java/org/apache/doris/rewrite/ExprRewriteRule.java index 4bc195a27b3e89..bb8b2738a6c393 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/ExprRewriteRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/ExprRewriteRule.java @@ -22,7 +22,7 @@ import org.apache.doris.common.AnalysisException; /** - * Base class for all Expr rewrite rules. A rule is free to modify Exprs in place, + * Base class for all Expr equal rules. A rule is free to modify Exprs in place, * but must return a different Expr object if any modifications were made. * An ExprRewriteRule is intended to apply its transformation on a single Expr and not * recursively on all its children. The recursive and repeated application of @@ -35,10 +35,10 @@ */ public interface ExprRewriteRule { /** - * Applies this rewrite rule to the given analyzed Expr. Returns the transformed and + * Applies this equal rule to the given analyzed Expr. Returns the transformed and * analyzed Expr or the original unmodified Expr if no changes were made. If any * changes were made, the transformed Expr is guaranteed to be a different Expr object, * so callers can rely on object reference comparison for change detection. */ - public abstract Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException; + Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException; } diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java new file mode 100644 index 00000000000000..4479e2f3b2e6e4 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.rewrite.ExprRewriteRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * For duplicate table, the count distinct(k1) could be rewritten to bitmap_union_count(mv_to_bitmap_k1) when bitmap + * mv exists. + * For example: + * Table: (k1 int, k2 int) + * MV: (k1 int, mv_bitmap_union_k2 bitmap bitmap_union) + * mv_to_bitmap_k2 = to_bitmap(k2) + * Query: select k1, count(distinct k2) from table group by k1 + * Rewritten query: select k1, bitmap_union_count(mv_to_bitmap_k2) from table group by k1 + */ +public class CountDistinctToBitmap implements ExprRewriteRule { + + public static final ExprRewriteRule INSTANCE = new CountDistinctToBitmap(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + // meet condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + if (!fnExpr.getParams().isDistinct()) { + return expr; + } + if (!fnExpr.getFnName().getFunction().equalsIgnoreCase("count")) { + return expr; + } + if (fnExpr.getChildren().size() != 1 || !(fnExpr.getChild(0) instanceof SlotRef)) { + return expr; + } + SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); + Column column = fnChild0.getColumn(); + Table table = fnChild0.getTable(); + if (column == null || table == null || !(table instanceof OlapTable)) { + return expr; + } + OlapTable olapTable = (OlapTable) table; + + // check column + String queryColumnName = column.getName(); + String mvColumnName = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), queryColumnName); + Column mvColumn = olapTable.getVisibleColumn(mvColumnName); + if (mvColumn == null) { + return expr; + } + + // rewrite expr + return rewriteExpr(fnChild0, mvColumn, analyzer); + } + + public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + Preconditions.checkNotNull(mvColumn); + Preconditions.checkNotNull(queryColumnSlotRef); + TableName tableName = queryColumnSlotRef.getTableName(); + Preconditions.checkNotNull(tableName); + SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName()); + List newFnParams = Lists.newArrayList(); + newFnParams.add(mvSlotRef); + FunctionCallExpr result = new FunctionCallExpr("bitmap_union_count", newFnParams); + result.analyzeNoThrow(analyzer); + return result; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java new file mode 100644 index 00000000000000..cec3b5d172f4aa --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmapOrHLLRule.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rewrite.ExprRewriteRule; + +/** + * For agg keys type, the count distinct could be rewritten to bitmap or hll depends on the type of column. + * For example: + * Table: (k1 int, k2 bitmap bitmap_union) agg key(k1) + * Query: select k1, count(distinct k2) from table group by k1 + * Rewritten query: select k1, bitmap_union_count(k2) from table group by k1 + *

+ * Table: (k1 int, k2 hll hll_union) agg key(k1) + * Query: select k1, count(distinct k2) from table group by k1 + * Rewritten query: select k1, hll_union_agg(k2) from table group by k1 + *

+ * Attention: this rule only apply AGG keys type. + */ +public class CountDistinctToBitmapOrHLLRule implements ExprRewriteRule { + public static final ExprRewriteRule INSTANCE = new CountDistinctToBitmapOrHLLRule(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + if (ConnectContext.get() == null || !ConnectContext.get().getSessionVariable().isRewriteCountDistinct()) { + return expr; + } + + // meet condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + if (!fnExpr.isCountDistinctBitmapOrHLL()) { + return expr; + } + // rewrite expr + FunctionParams newParams = new FunctionParams(false, fnExpr.getParams().exprs()); + if (fnExpr.getChild(0).getType().isBitmapType()) { + FunctionCallExpr bitmapExpr = new FunctionCallExpr(FunctionSet.BITMAP_UNION_COUNT, newParams); + bitmapExpr.analyzeNoThrow(analyzer); + return bitmapExpr; + } else { + FunctionCallExpr hllExpr = new FunctionCallExpr("hll_union_agg", newParams); + hllExpr.analyzeNoThrow(analyzer); + return hllExpr; + } + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java new file mode 100644 index 00000000000000..9fa85fc7cf9970 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.rewrite.ExprRewriteRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * Rewrite count(k1) to sum(mv_count_k1) when MV Column exists. + * For example: + * Table: (k1 int ,k2 varchar) + * MV: (k1 int, mv_count_k2 bigint sum) + * mv_count_k1 = case when k2 is null then 0 else 1 + * Query: select k1, count(k2) from table group by k1 + * Rewritten query: select k1, sum(mv_count_k2) from table group by k1 + */ +public class CountFieldToSum implements ExprRewriteRule { + public static final ExprRewriteRule INSTANCE = new CountFieldToSum(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + // meet condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + if (!fnExpr.getFnName().getFunction().equalsIgnoreCase("count")) { + return expr; + } + if (fnExpr.getChildren().size() != 1 || !(fnExpr.getChild(0) instanceof SlotRef)) { + return expr; + } + SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); + Column column = fnChild0.getColumn(); + Table table = fnChild0.getTable(); + if (column == null || table == null || !(table instanceof OlapTable)) { + return expr; + } + OlapTable olapTable = (OlapTable) table; + + // check column + String queryColumnName = column.getName(); + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("count", queryColumnName); + Column mvColumn = olapTable.getVisibleColumn(mvColumnName); + if (mvColumn == null) { + return expr; + } + + // rewrite expr + return rewriteExpr(fnChild0, mvColumn, analyzer); + } + + public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + Preconditions.checkNotNull(mvColumn); + Preconditions.checkNotNull(queryColumnSlotRef); + TableName tableName = queryColumnSlotRef.getTableName(); + Preconditions.checkNotNull(tableName); + SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName()); + List newFnParams = Lists.newArrayList(); + newFnParams.add(mvSlotRef); + FunctionCallExpr result = new FunctionCallExpr("sum", newFnParams); + result.analyzeNoThrow(analyzer); + return result; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java new file mode 100644 index 00000000000000..bde6a8142e2c2f --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSetMultimap; + +public class FunctionCallEqualRule implements MVExprEqualRule { + + public static MVExprEqualRule INSTANCE = new FunctionCallEqualRule(); + private static final ImmutableSetMultimap columnAggTypeMatchFnName; + + static { + ImmutableSetMultimap.Builder builder = ImmutableSetMultimap.builder(); + builder.put("sum", "sum"); + builder.put("max", "max"); + builder.put("min", "min"); + builder.put("bitmap_union", "bitmap_union"); + builder.put("bitmap_union", "bitmap_union_count"); + builder.put("hll_union", "hll_union_agg"); + builder.put("hll_union", "hll_union"); + builder.put("hll_union", "hll_raw_agg"); + builder.put("to_bitmap", "to_bitmap"); + builder.put("hll_hash", "hll_hash"); + columnAggTypeMatchFnName = builder.build(); + } + + @Override + public boolean equal(Expr queryExpr, Expr mvColumnExpr) { + if ((!(queryExpr instanceof FunctionCallExpr)) || (!(mvColumnExpr instanceof FunctionCallExpr))) { + return false; + } + FunctionCallExpr queryFn = (FunctionCallExpr) queryExpr; + FunctionCallExpr mvColumnFn = (FunctionCallExpr) mvColumnExpr; + // match fn name + if (!columnAggTypeMatchFnName.get(mvColumnFn.getFnName().getFunction()) + .contains(queryFn.getFnName().getFunction().toLowerCase())) { + return false; + } + // match children + if (queryFn.getChildren().size() != mvColumnFn.getChildren().size()) { + return false; + } + Preconditions.checkState(queryFn.getChildren().size() == 1); + // remove cast to function + Expr queryFnChild0 = queryFn.getChild(0); + if (queryFnChild0 instanceof CastExpr) { + queryFnChild0 = queryFnChild0.getChild(0); + } + Expr mvColumnFnChild0 = mvColumnFn.getChild(0); + if (mvColumnFnChild0 instanceof CastExpr) { + mvColumnFnChild0 = mvColumnFnChild0.getChild(0); + } + if (MVExprEquivalent.mvExprEqual(queryFnChild0, mvColumnFnChild0)) { + return true; + } + return false; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java new file mode 100644 index 00000000000000..8eadcada4d3e41 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.rewrite.ExprRewriteRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.util.List; + +/* +Rewrite hll_union(hll_hash(c1)) to hll_union(mv_hll_union_c1) +Rewrite hll_raw_agg(hll_hash(c1)) to hll_raw_agg(mv_hll_union_c1) +Rewrite hll_union_agg(hll_hash(c1)) to hll_union_agg(mv_hll_union_c1) + */ +public class HLLHashToSlotRefRule implements ExprRewriteRule { + + public static final ExprRewriteRule INSTANCE = new HLLHashToSlotRefRule(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + SlotRef queryColumnSlotRef; + Column mvColumn; + + // meet the condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + String fnNameString = fnExpr.getFnName().getFunction(); + if (!fnNameString.equalsIgnoreCase("hll_union") + && !fnNameString.equalsIgnoreCase("hll_raw_agg") + && !fnNameString.equalsIgnoreCase("hll_union_agg")) { + return expr; + } + if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { + return expr; + } + if (child0FnExpr.getChild(0) instanceof SlotRef) { + queryColumnSlotRef = (SlotRef) child0FnExpr.getChild(0); + } else if (child0FnExpr.getChild(0) instanceof CastExpr) { + CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); + if (!(castExpr.getChild(0) instanceof SlotRef)) { + return expr; + } + queryColumnSlotRef = (SlotRef) castExpr.getChild(0); + } else { + return expr; + } + Column column = queryColumnSlotRef.getColumn(); + Table table = queryColumnSlotRef.getTable(); + if (column == null || table == null || !(table instanceof OlapTable)) { + return expr; + } + OlapTable olapTable = (OlapTable) table; + + // check column + String queryColumnName = column.getName(); + String mvColumnName = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.HLL_UNION.name().toLowerCase(), queryColumnName); + mvColumn = olapTable.getVisibleColumn(mvColumnName); + if (mvColumn == null) { + return expr; + } + + // equal expr + return rewriteExpr(fnNameString, queryColumnSlotRef, mvColumn, analyzer); + } + + public Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + Preconditions.checkNotNull(mvColumn); + Preconditions.checkNotNull(queryColumnSlotRef); + TableName tableName = queryColumnSlotRef.getTableName(); + Preconditions.checkNotNull(tableName); + SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName()); + List newFnParams = Lists.newArrayList(); + newFnParams.add(mvSlotRef); + FunctionCallExpr result = new FunctionCallExpr(fnName, newFnParams); + result.analyzeNoThrow(analyzer); + return result; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEqualRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEqualRule.java new file mode 100644 index 00000000000000..6595f61472f13b --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEqualRule.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Expr; + +public interface MVExprEqualRule { + + boolean equal(Expr queryExpr, Expr mvColumnExpr); +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java new file mode 100644 index 00000000000000..2376c631b2cac6 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Expr; + +import com.google.common.collect.ImmutableList; + +/* +Only support the once match from originExpr and newExpr +TODO：one query expr could be calculate by a group by mv columns +TODO: mvExprEqual(queryexpr, mvColumnExprList) + */ +public class MVExprEquivalent { + + private static final ImmutableList exprRewriteRuleList = ImmutableList + .builder() + .add(FunctionCallEqualRule.INSTANCE) + .add(SlotRefEqualRule.INSTANCE) + .build(); + + public static boolean mvExprEqual(Expr queryExpr, Expr mvColumnExpr) { + for (MVExprEqualRule rule : exprRewriteRuleList) { + if (rule.equal(queryExpr, mvColumnExpr)) { + return true; + } + } + return false; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVSelectFailedException.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVSelectFailedException.java new file mode 100644 index 00000000000000..286fc71067ce67 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVSelectFailedException.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.common.UserException; + +public class MVSelectFailedException extends UserException { + + public MVSelectFailedException(String msg) { + super(msg); + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java new file mode 100644 index 00000000000000..e0334b61c27281 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.rewrite.ExprRewriteRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.util.List; + +/** + * For duplicate table, the ndv(k1) could be rewritten to hll_union_agg(mv_hll_hash_k1) when bitmap + * mv exists. + * For example: + * Table: (k1 int, k2 int) + * MV: (k1 int, mv_hll_hash_k2 hll hll_union) + * mv_hll_hash_k2 = hll_hash(k2) + * Query: select k1, count(distinct k2) from table group by k1 + * Rewritten query: select k1, hll_union_agg(mv_hll_hash_k2) from table group by k1 + */ +public class NDVToHll implements ExprRewriteRule{ + public static final ExprRewriteRule INSTANCE = new NDVToHll(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + // meet condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + if (!fnExpr.getFnName().getFunction().equalsIgnoreCase("ndv")) { + return expr; + } + if (fnExpr.getChildren().size() != 1 || !(fnExpr.getChild(0) instanceof SlotRef)) { + return expr; + } + SlotRef fnChild0 = (SlotRef) fnExpr.getChild(0); + Column column = fnChild0.getColumn(); + Table table = fnChild0.getTable(); + if (column == null || table == null || !(table instanceof OlapTable)) { + return expr; + } + OlapTable olapTable = (OlapTable) table; + + // check column + String queryColumnName = column.getName(); + String mvColumnName = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.HLL_UNION.name().toLowerCase(), queryColumnName); + Column mvColumn = olapTable.getVisibleColumn(mvColumnName); + if (mvColumn == null) { + return expr; + } + + // rewrite expr + return rewriteExpr(fnChild0, mvColumn, analyzer); + } + + public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + Preconditions.checkNotNull(mvColumn); + Preconditions.checkNotNull(queryColumnSlotRef); + TableName tableName = queryColumnSlotRef.getTableName(); + Preconditions.checkNotNull(tableName); + SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName()); + List newFnParams = Lists.newArrayList(); + newFnParams.add(mvSlotRef); + FunctionCallExpr result = new FunctionCallExpr("hll_union_agg", newFnParams); + result.analyzeNoThrow(analyzer); + return result; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/SlotRefEqualRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/SlotRefEqualRule.java new file mode 100644 index 00000000000000..76b59046f99203 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/SlotRefEqualRule.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; + +public class SlotRefEqualRule implements MVExprEqualRule { + + public static MVExprEqualRule INSTANCE = new SlotRefEqualRule(); + + @Override + public boolean equal(Expr queryExpr, Expr mvColumnExpr) { + if ((!(queryExpr instanceof SlotRef)) || (!(mvColumnExpr instanceof SlotRef))) { + return false; + } + SlotRef querySlotRef = (SlotRef) queryExpr; + SlotRef mvColumnSlotRef = (SlotRef) mvColumnExpr; + if (querySlotRef.getColumnName().equalsIgnoreCase(mvColumnSlotRef.getColumnName())) { + return true; + } + return false; + } +} diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java new file mode 100644 index 00000000000000..20d98a351cfb01 --- /dev/null +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.rewrite.mvrewrite; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CastExpr; +import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TableName; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.rewrite.ExprRewriteRule; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.util.List; + +/* +Rewrite bitmap_union(to_bitmap(c1)) to bitmap_union(mv_bitmap_c1) +Rewrite bitmap_union_count(to_bitmap(c1)) to bitmap_union_count(mv_bitmap_c1) + */ +public class ToBitmapToSlotRefRule implements ExprRewriteRule { + + public static final ExprRewriteRule INSTANCE = new ToBitmapToSlotRefRule(); + + @Override + public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { + SlotRef queryColumnSlotRef; + Column mvColumn; + + // meet the condition + if (!(expr instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr fnExpr = (FunctionCallExpr) expr; + String fnNameString = fnExpr.getFnName().getFunction(); + if (!fnNameString.equalsIgnoreCase(FunctionSet.BITMAP_UNION) + && !fnNameString.equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)) { + return expr; + } + if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { + return expr; + } + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")) { + return expr; + } + if (child0FnExpr.getChild(0) instanceof SlotRef) { + queryColumnSlotRef = (SlotRef) child0FnExpr.getChild(0); + } else if (child0FnExpr.getChild(0) instanceof CastExpr) { + CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); + if (!(castExpr.getChild(0) instanceof SlotRef)) { + return expr; + } + queryColumnSlotRef = (SlotRef) castExpr.getChild(0); + } else { + return expr; + } + Column column = queryColumnSlotRef.getColumn(); + Table table = queryColumnSlotRef.getTable(); + if (column == null || table == null || !(table instanceof OlapTable)) { + return expr; + } + OlapTable olapTable = (OlapTable) table; + + // check column + String queryColumnName = column.getName(); + String mvColumnName = CreateMaterializedViewStmt + .mvColumnBuilder(AggregateType.BITMAP_UNION.name().toLowerCase(), queryColumnName); + mvColumn = olapTable.getVisibleColumn(mvColumnName); + if (mvColumn == null) { + return expr; + } + + // equal expr + return rewriteExpr(fnNameString, queryColumnSlotRef, mvColumn, analyzer); + } + + public Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + Preconditions.checkNotNull(mvColumn); + Preconditions.checkNotNull(queryColumnSlotRef); + TableName tableName = queryColumnSlotRef.getTableName(); + Preconditions.checkNotNull(tableName); + SlotRef mvSlotRef = new SlotRef(tableName, mvColumn.getName()); + List newFnParams = Lists.newArrayList(); + newFnParams.add(mvSlotRef); + FunctionCallExpr result = new FunctionCallExpr(fnName, newFnParams); + result.analyzeNoThrow(analyzer); + return result; + } +} diff --git a/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java b/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java index e4903ae9667176..251ee2de840b1b 100644 --- a/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java +++ b/fe/src/main/java/org/apache/doris/task/PullLoadTaskPlanner.java @@ -102,7 +102,7 @@ public void plan(List> fileStatusesList, int filesAdded) scanNode.init(analyzer); scanNodes.add(scanNode); - // rewrite node + // equal node OlapRewriteNode rewriteNode = new OlapRewriteNode( new PlanNodeId(nextNodeId++), scanNode, table, tupleDesc, slotRefs); rewriteNode.init(analyzer); diff --git a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java index dbccc14183f3d3..d422f5036b231a 100644 --- a/fe/src/main/java/org/apache/doris/transaction/TransactionState.java +++ b/fe/src/main/java/org/apache/doris/transaction/TransactionState.java @@ -501,7 +501,7 @@ public void addTableIndexes(OlapTable table) { indexIds = Sets.newHashSet(); loadedTblIndexes.put(table.getId(), indexIds); } - // always rewrite the index ids + // always equal the index ids indexIds.clear(); for (Long indexId : table.getIndexIdToMeta().keySet()) { indexIds.add(indexId); diff --git a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java index 585d78bfc38363..e9f593bd4e548f 100644 --- a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java +++ b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java @@ -177,7 +177,7 @@ public void testInvalidMVColumn(@Injectable CreateMaterializedViewStmt createMat @Injectable OlapTable olapTable) { final String mvName = "mv1"; final String mvColumnName = "mv_k1"; - MVColumnItem mvColumnItem = new MVColumnItem(mvColumnName); + MVColumnItem mvColumnItem = new MVColumnItem(mvColumnName, Type.BIGINT); new Expectations() { { olapTable.hasMaterializedIndex(mvName); @@ -206,7 +206,7 @@ public void testInvalidAggregateType(@Injectable CreateMaterializedViewStmt crea final String mvName = "mv1"; final String columnName = "mv_k1"; Column baseColumn = new Column(columnName, Type.INT, false, AggregateType.SUM, "", ""); - MVColumnItem mvColumnItem = new MVColumnItem(columnName); + MVColumnItem mvColumnItem = new MVColumnItem(columnName, Type.BIGINT); mvColumnItem.setIsKey(true); mvColumnItem.setAggregationType(null, false); new Expectations() { @@ -239,7 +239,7 @@ public void testDuplicateTable(@Injectable CreateMaterializedViewStmt createMate final String mvName = "mv1"; final String columnName1 = "k1"; Column baseColumn1 = new Column(columnName1, Type.VARCHAR, false, AggregateType.NONE, "", ""); - MVColumnItem mvColumnItem = new MVColumnItem(columnName1); + MVColumnItem mvColumnItem = new MVColumnItem(columnName1, Type.VARCHAR); mvColumnItem.setIsKey(true); mvColumnItem.setAggregationType(null, false); new Expectations() { diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index 70932fe2100c1e..1232cc7bb43f91 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -377,7 +377,7 @@ public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) th short keysCount = 1; List columns = Lists.newArrayList(); - String mvColumnName =CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PRFIX + "bitmap_" + "c1"; + String mvColumnName = CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX + "to_bitmap_" + "c1"; Column column = new Column(mvColumnName, Type.BITMAP, false, AggregateType.BITMAP_UNION, false, "1", ""); columns.add(column); RollupJobV2 rollupJobV2 = new RollupJobV2(1, 1, 1, "test", 1, 1, 1, "test", "rollup", columns, 1, 1, @@ -393,7 +393,7 @@ public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) th List itemList = Lists.newArrayList(); MVColumnItem item = new MVColumnItem( - mvColumnName); + mvColumnName, Type.BITMAP); List params = Lists.newArrayList(); SlotRef param1 = new SlotRef(new TableName(null, "test"), "c1"); params.add(param1); diff --git a/fe/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/src/test/java/org/apache/doris/analysis/ExprTest.java index cf924f0c50b00d..991cf79fcb009e 100644 --- a/fe/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -69,21 +69,21 @@ public void testGetTableNameToColumnNames(@Mocked Analyzer analyzer, result = tableA; tupleDescriptor2.getTable(); result = tableB; - tableA.getName(); - result = "tableA"; - tableB.getName(); - result = "tableB"; + tableA.getId(); + result = 1; + tableB.getId(); + result = 2; } }; - Map> tableNameToColumnNames = Maps.newHashMap(); - whereExpr.getTableNameToColumnNames(tableNameToColumnNames); + Map> tableNameToColumnNames = Maps.newHashMap(); + whereExpr.getTableIdToColumnNames(tableNameToColumnNames); Assert.assertEquals(tableNameToColumnNames.size(), 2); - Set tableAColumns = tableNameToColumnNames.get("tableA"); + Set tableAColumns = tableNameToColumnNames.get(new Long(1)); Assert.assertNotEquals(tableAColumns, null); Assert.assertTrue(tableAColumns.contains("c1")); - Set tableBColumns = tableNameToColumnNames.get("tableB"); + Set tableBColumns = tableNameToColumnNames.get(new Long(2)); Assert.assertNotEquals(tableBColumns, null); Assert.assertTrue(tableBColumns.contains("c1")); } diff --git a/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java b/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java index f661e6239dcb12..017993c638510a 100644 --- a/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java @@ -61,7 +61,7 @@ public void testSerializeMaterializedIndexMeta(@Mocked CreateMaterializedViewStm file.createNewFile(); DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); - String mvColumnName = CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PRFIX + "bitmap_" + "k1"; + String mvColumnName = CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX + "to_bitmap_" + "k1"; List schema = Lists.newArrayList(); schema.add(new Column("k1", Type.TINYINT, true, null, true, "1", "abc")); schema.add(new Column("k2", Type.SMALLINT, true, null, true, "1", "debug")); @@ -89,7 +89,7 @@ TStorageType.COLUMN, KeysType.DUP_KEYS, new OriginStatement( out.close(); List itemList = Lists.newArrayList(); - MVColumnItem item = new MVColumnItem(mvColumnName); + MVColumnItem item = new MVColumnItem(mvColumnName, Type.BITMAP); List params = Lists.newArrayList(); SlotRef param1 = new SlotRef(new TableName(null, "test"), "c1"); params.add(param1); diff --git a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java index bd9179223efa96..fda215cbae68b8 100644 --- a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java +++ b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java @@ -17,6 +17,7 @@ package org.apache.doris.planner; +import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.common.FeConstants; import org.apache.doris.utframe.DorisAssert; import org.apache.doris.utframe.UtFrameUtils; @@ -42,6 +43,10 @@ public class MaterializedViewFunctionTest { private static final String DEPTS_MV_NAME = "depts_mv"; private static final String QUERY_USE_DEPTS_MV = "rollup: " + DEPTS_MV_NAME; private static final String QUERY_USE_DEPTS = "rollup: " + DEPTS_TABLE_NAME; + private static final String USER_TAG_TABLE_NAME = "user_tags"; + private static final String USER_TAG_MV_NAME = "user_tags_mv"; + private static final String QUERY_USE_USER_TAG_MV = "rollup: " + USER_TAG_MV_NAME; + private static final String QUERY_USE_USER_TAG = "rollup: " + USER_TAG_TABLE_NAME; private static final String TEST_TABLE_NAME = "test_tb"; private static DorisAssert dorisAssert; @@ -66,12 +71,18 @@ public void beforeMethod() throws Exception { + "(partition p1 values less than MAXVALUE) " + "distributed by hash(deptno) buckets 3 properties('replication_num' = '1');"; dorisAssert.withTable(createTableSQL); + createTableSQL = "create table " + HR_DB_NAME + "." + USER_TAG_TABLE_NAME + + " (user_id int, user_name varchar(20), tag_id int) partition by range (user_id) " + + " (partition p1 values less than MAXVALUE) " + + "distributed by hash(user_id) buckets 3 properties('replication_num' = '1');"; + dorisAssert.withTable(createTableSQL); } @After public void afterMethod() throws Exception { dorisAssert.dropTable(EMPS_TABLE_NAME); dorisAssert.dropTable(DEPTS_TABLE_NAME); + dorisAssert.dropTable(USER_TAG_TABLE_NAME); } @AfterClass @@ -638,4 +649,148 @@ public void testUniqueTableInQuery() throws Exception { } + + @Test + public void testBitmapUnionInQuery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + + " as select user_id, bitmap_union(to_bitmap(tag_id)) from " + + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + + " group by user_id having a>1 order by a;"; + dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG_MV); + } + + @Test + public void testBitmapUnionInSubquery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select user_id from " + USER_TAG_TABLE_NAME + " where user_id in (select user_id from " + + USER_TAG_TABLE_NAME + " group by user_id having bitmap_union_count(to_bitmap(tag_id)) >1 ) ;"; + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, USER_TAG_TABLE_NAME); + } + + @Test + public void testIncorrectMVRewriteInQuery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String createEMPMVSQL = "create materialized view " + EMPS_MV_NAME + " as select name, deptno from " + + EMPS_TABLE_NAME + ";"; + dorisAssert.withMaterializedView(createEMPMVSQL); + String query = "select user_name, bitmap_union_count(to_bitmap(tag_id)) a from " + USER_TAG_TABLE_NAME + ", " + + "(select name, deptno from " + EMPS_TABLE_NAME + ") a" + " where user_name=a.name group by " + + "user_name having a>1 order by a;"; + dorisAssert.query(query).explainContains(QUERY_USE_EMPS_MV); + dorisAssert.query(query).explainWithout(QUERY_USE_USER_TAG_MV); + } + + @Test + public void testIncorrectMVRewriteInSubquery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select user_id, bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " where " + + "user_name in (select user_name from " + USER_TAG_TABLE_NAME + " group by user_name having " + + "bitmap_union_count(to_bitmap(tag_id)) >1 )" + " group by user_id;"; + dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG); + } + + @Test + public void testTwoTupleInQuery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select * from (select user_id, bitmap_union_count(to_bitmap(tag_id)) x from " + + USER_TAG_TABLE_NAME + " group by user_id) a, (select user_name, bitmap_union_count(to_bitmap(tag_id))" + + "" + " y from " + USER_TAG_TABLE_NAME + " group by user_name) b where a.x=b.y;"; + dorisAssert.query(query).explainContains(QUERY_USE_USER_TAG, QUERY_USE_USER_TAG_MV); + } + + @Test + public void testAggTableCountDistinctInBitmapType() throws Exception { + String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 bitmap bitmap_union) Aggregate KEY (k1) " + + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');"; + dorisAssert.withTable(aggTable); + String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;"; + dorisAssert.query(query).explainContains(TEST_TABLE_NAME, "bitmap_union_count"); + dorisAssert.dropTable(TEST_TABLE_NAME); + } + + @Test + public void testAggTableCountDistinctInHllType() throws Exception { + String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 hll hll_union) Aggregate KEY (k1) " + + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');"; + dorisAssert.withTable(aggTable); + String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;"; + dorisAssert.query(query).explainContains(TEST_TABLE_NAME, "hll_union_agg"); + dorisAssert.dropTable(TEST_TABLE_NAME); + } + + @Test + public void testCountDistinctToBitmap() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select count(distinct tag_id) from " + USER_TAG_TABLE_NAME + ";"; + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "bitmap_union_count"); + } + + @Test + public void testIncorrectRewriteCountDistinct() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; + dorisAssert.query(query).explainContains(USER_TAG_TABLE_NAME, "count"); + } + + @Test + public void testNDVToHll() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "`hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";"; + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg"); + } + + @Test + public void testHLLUnionFamilyRewrite() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "`hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select `hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("hll_union", "tag_id"); + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); + query = "select hll_union_agg(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); + query = "select hll_raw_agg(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); + } + + /* + ISSUE-3174 + */ + @Test + public void testAggInHaving() throws Exception { + String createMVSQL = "create materialized view " + EMPS_MV_NAME + " as select empid, deptno from " + + EMPS_TABLE_NAME + " group by empid, deptno;"; + dorisAssert.withMaterializedView(createMVSQL); + String query = "select empid from " + EMPS_TABLE_NAME + " group by empid having max(salary) > 1;"; + dorisAssert.query(query).explainWithout(QUERY_USE_EMPS_MV); + } + + @Test + public void testCountFieldInQuery() throws Exception { + String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + + "count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + dorisAssert.withMaterializedView(createUserTagMVSql); + String query = "select count(tag_id) from " + USER_TAG_TABLE_NAME + ";"; + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("count", "tag_id"); + dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); + query = "select user_name, count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; + dorisAssert.query(query).explainWithout(USER_TAG_MV_NAME); + } + } diff --git a/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java b/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java index 6dfb9b85618cdf..af4a8fc4c0b60d 100644 --- a/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java +++ b/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java @@ -19,6 +19,8 @@ import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.SlotDescriptor; @@ -38,7 +40,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.commons.lang3.builder.ToStringExclude; import org.junit.Assert; import org.junit.Test; @@ -90,8 +91,8 @@ public void initTest(@Injectable SelectStmt selectStmt, result = tableADesc; tableADesc.getTable(); result = tableA; - tableA.getName(); - result = "tableA"; + tableA.getId(); + result = 1; aggregateInfo.getAggregateExprs(); result = Lists.newArrayList(tableAColumn2Sum, tableBColumn1Max); @@ -114,36 +115,51 @@ public void initTest(@Injectable SelectStmt selectStmt, result = tableBDesc; tableBDesc.getTable(); result = tableB; - tableB.getName(); - result = "tableB"; + tableB.getId(); + result = 2; + + tableAColumn2Desc.isMaterialized(); + result = true; + tableBColumn1Desc.isMaterialized(); + result = true; + tableAColumn2Desc.getColumn().getName(); + result = "c2"; + tableBColumn1Desc.getColumn().getName(); + result = "c1"; } }; MaterializedViewSelector materializedViewSelector = new MaterializedViewSelector(selectStmt, analyzer); - Map> columnNamesInPredicates = + Map> columnNamesInPredicates = Deencapsulation.getField(materializedViewSelector, "columnNamesInPredicates"); Assert.assertEquals(0, columnNamesInPredicates.size()); Assert.assertFalse(Deencapsulation.getField(materializedViewSelector, "isSPJQuery")); - Map> columnNamesInGrouping = + Map> columnNamesInGrouping = Deencapsulation.getField(materializedViewSelector, "columnNamesInGrouping"); Assert.assertEquals(1, columnNamesInGrouping.size()); - Set tableAColumnNamesInGrouping = columnNamesInGrouping.get("tableA"); + Set tableAColumnNamesInGrouping = columnNamesInGrouping.get(new Long(1)); Assert.assertNotEquals(tableAColumnNamesInGrouping, null); Assert.assertEquals(1, tableAColumnNamesInGrouping.size()); Assert.assertTrue(tableAColumnNamesInGrouping.contains("c1")); - Map> aggregateColumnsInQuery = - Deencapsulation.getField(materializedViewSelector, "aggregateColumnsInQuery"); + Map> aggregateColumnsInQuery = + Deencapsulation.getField(materializedViewSelector, "aggColumnsInQuery"); Assert.assertEquals(2, aggregateColumnsInQuery.size()); - Set tableAAgggregatedColumns = aggregateColumnsInQuery.get("tableA"); + Set tableAAgggregatedColumns = aggregateColumnsInQuery.get(new Long(1)); Assert.assertEquals(1, tableAAgggregatedColumns.size()); - MaterializedViewSelector.AggregatedColumn aggregatedColumn1 = tableAAgggregatedColumns.iterator().next(); - Assert.assertEquals("c2", Deencapsulation.getField(aggregatedColumn1, "columnName")); - Assert.assertTrue("SUM".equalsIgnoreCase(Deencapsulation.getField(aggregatedColumn1, "aggFunctionName"))); - Set tableBAgggregatedColumns = aggregateColumnsInQuery.get("tableB"); + FunctionCallExpr aggregatedColumn1 = tableAAgggregatedColumns.iterator().next(); + List aggColumn1Params = aggregatedColumn1.getParams().exprs(); + Assert.assertEquals(1, aggColumn1Params.size()); + Assert.assertTrue(aggColumn1Params.get(0) instanceof SlotRef); + Assert.assertEquals("c2", ((SlotRef) aggColumn1Params.get(0)).getColumnName()); + Assert.assertTrue("SUM".equalsIgnoreCase(aggregatedColumn1.getFnName().getFunction())); + Set tableBAgggregatedColumns = aggregateColumnsInQuery.get(new Long(2)); Assert.assertEquals(1, tableBAgggregatedColumns.size()); - MaterializedViewSelector.AggregatedColumn aggregatedColumn2 = tableBAgggregatedColumns.iterator().next(); - Assert.assertEquals("c1", Deencapsulation.getField(aggregatedColumn2, "columnName")); - Assert.assertTrue("MAX".equalsIgnoreCase(Deencapsulation.getField(aggregatedColumn2, "aggFunctionName"))); + FunctionCallExpr aggregatedColumn2 = tableBAgggregatedColumns.iterator().next(); + List aggColumn2Params = aggregatedColumn2.getParams().exprs(); + Assert.assertEquals(1, aggColumn2Params.size()); + Assert.assertTrue(aggColumn2Params.get(0) instanceof SlotRef); + Assert.assertEquals("c1", ((SlotRef) aggColumn2Params.get(0)).getColumnName()); + Assert.assertTrue("MAX".equalsIgnoreCase(aggregatedColumn2.getFnName().getFunction())); } @Test @@ -276,13 +292,15 @@ public void testCheckAggregationFunction(@Injectable SelectStmt selectStmt, @Inj }; MaterializedViewSelector selector = new MaterializedViewSelector(selectStmt, analyzer); - MaterializedViewSelector.AggregatedColumn aggregatedColumn = Deencapsulation.newInnerInstance - (MaterializedViewSelector.AggregatedColumn.class, selector, "C1", "sum"); - Set aggregatedColumnsInQueryOutput = Sets.newHashSet(); - aggregatedColumnsInQueryOutput.add(aggregatedColumn); + TableName tableName = new TableName("db1", "table1"); + SlotRef slotRef = new SlotRef(tableName, "C1"); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", Lists.newArrayList(slotRef)); + Set aggregatedColumnsInQueryOutput = Sets.newHashSet(); + aggregatedColumnsInQueryOutput.add(functionCallExpr); Deencapsulation.setField(selector, "isSPJQuery", false); + Map candidateIndexIdToRewriteSMap = Maps.newHashMap(); Deencapsulation.invoke(selector, "checkAggregationFunction", aggregatedColumnsInQueryOutput, - candidateIndexIdToSchema); + candidateIndexIdToSchema, candidateIndexIdToRewriteSMap); Assert.assertEquals(2, candidateIndexIdToSchema.size()); Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(1))); Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(3))); From dbf57071cc1abd9631a317b49c8a28a8453bba62 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Mon, 6 Jul 2020 17:26:11 +0800 Subject: [PATCH 02/19] Add comment and change field name Change-Id: I4c8568d4b57384c342eaf5432856eaaa7393f49d --- .../analysis/CreateMaterializedViewStmt.java | 18 +++++++++--------- .../doris/planner/SingleNodePlanner.java | 2 +- .../java/org/apache/doris/qe/StmtExecutor.java | 7 +++++-- .../mvrewrite/CountDistinctToBitmap.java | 2 +- .../rewrite/mvrewrite/CountFieldToSum.java | 2 +- .../mvrewrite/HLLHashToSlotRefRule.java | 2 +- .../doris/rewrite/mvrewrite/NDVToHll.java | 2 +- .../mvrewrite/ToBitmapToSlotRefRule.java | 2 +- 8 files changed, 20 insertions(+), 17 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 819369c162fbee..3b2dac76214048 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -54,19 +54,19 @@ */ public class CreateMaterializedViewStmt extends DdlStmt { public static final String MATERIALIZED_VIEW_NAME_PREFIX = "mv_"; - public static final Map fnNameToPattern; + public static final Map FN_NAME_TO_PATTERN; static { - fnNameToPattern = Maps.newHashMap(); - fnNameToPattern.put(AggregateType.SUM.name().toLowerCase(), + FN_NAME_TO_PATTERN = Maps.newHashMap(); + FN_NAME_TO_PATTERN.put(AggregateType.SUM.name().toLowerCase(), new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase())); - fnNameToPattern.put(AggregateType.MIN.name().toLowerCase(), + FN_NAME_TO_PATTERN.put(AggregateType.MIN.name().toLowerCase(), new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase())); - fnNameToPattern.put(AggregateType.MAX.name().toLowerCase(), + FN_NAME_TO_PATTERN.put(AggregateType.MAX.name().toLowerCase(), new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase())); - fnNameToPattern.put("count", new MVColumnOneChildPattern("count")); - fnNameToPattern.put("bitmap_union", new MVColumnBitmapUnionPattern()); - fnNameToPattern.put("hll_union", new MVColumnHLLUnionPattern()); + FN_NAME_TO_PATTERN.put("count", new MVColumnOneChildPattern("count")); + FN_NAME_TO_PATTERN.put("bitmap_union", new MVColumnBitmapUnionPattern()); + FN_NAME_TO_PATTERN.put("hll_union", new MVColumnHLLUnionPattern()); } private String mvName; @@ -187,7 +187,7 @@ public void analyzeSelectClause() throws AnalysisException { // Function must match pattern. FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItem.getExpr(); String functionName = functionCallExpr.getFnName().getFunction(); - MVColumnPattern mvColumnPattern = fnNameToPattern.get(functionName.toLowerCase()); + MVColumnPattern mvColumnPattern = FN_NAME_TO_PATTERN.get(functionName.toLowerCase()); if (mvColumnPattern == null) { throw new AnalysisException( "Materialized view does not support this function:" + functionCallExpr.toSqlImpl()); diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 52c7b4ddf3888b..2aec056fd63ca1 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -781,7 +781,7 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) selectFailed |= true; TupleId tupleId = olapScanNode.getTupleId(); selectStmt.updateDisableTuplesMVRewriter(tupleId); - LOG.info("MV rewriter of tuple [] will be disable", tupleId); + LOG.debug("MV rewriter of tuple [] will be disable", tupleId); continue; } olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), diff --git a/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java index 8202e0f0ba54d1..09e2e3cee7c20c 100644 --- a/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -64,8 +64,6 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.planner.Planner; import org.apache.doris.proto.PQueryStatistics; -import org.apache.doris.qe.QueryDetail; -import org.apache.doris.qe.QueryDetailQueue; import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; @@ -441,6 +439,11 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { try { analyzeAndGenerateQueryPlan(tQueryOptions); } catch (MVSelectFailedException e) { + /** + * If there is MVSelectFailedException after the first planner, there will be error mv rewritten in query. + * So, the query should be reanalyzed without mv rewritten and planner again. + * Attention: Only error rewritten tuple is forbidden to mv rewrite in the second time. + */ resetAnalyzerAndStmt(); analyzeAndGenerateQueryPlan(tQueryOptions); } catch (UserException e) { diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java index 4479e2f3b2e6e4..b50eab71bb6add 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java @@ -86,7 +86,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return rewriteExpr(fnChild0, mvColumn, analyzer); } - public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + private Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { Preconditions.checkNotNull(mvColumn); Preconditions.checkNotNull(queryColumnSlotRef); TableName tableName = queryColumnSlotRef.getTableName(); diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java index 9fa85fc7cf9970..138f30643b8c34 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java @@ -79,7 +79,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return rewriteExpr(fnChild0, mvColumn, analyzer); } - public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + private Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { Preconditions.checkNotNull(mvColumn); Preconditions.checkNotNull(queryColumnSlotRef); TableName tableName = queryColumnSlotRef.getTableName(); diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java index 8eadcada4d3e41..571f1c3c51cd75 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/HLLHashToSlotRefRule.java @@ -99,7 +99,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return rewriteExpr(fnNameString, queryColumnSlotRef, mvColumn, analyzer); } - public Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + private Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { Preconditions.checkNotNull(mvColumn); Preconditions.checkNotNull(queryColumnSlotRef); TableName tableName = queryColumnSlotRef.getTableName(); diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java index e0334b61c27281..37eb7767c020e8 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java @@ -82,7 +82,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return rewriteExpr(fnChild0, mvColumn, analyzer); } - public Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + private Expr rewriteExpr(SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { Preconditions.checkNotNull(mvColumn); Preconditions.checkNotNull(queryColumnSlotRef); TableName tableName = queryColumnSlotRef.getTableName(); diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java index 20d98a351cfb01..ddc60b40ab7969 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java @@ -98,7 +98,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return rewriteExpr(fnNameString, queryColumnSlotRef, mvColumn, analyzer); } - public Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { + private Expr rewriteExpr(String fnName, SlotRef queryColumnSlotRef, Column mvColumn, Analyzer analyzer) { Preconditions.checkNotNull(mvColumn); Preconditions.checkNotNull(queryColumnSlotRef); TableName tableName = queryColumnSlotRef.getTableName(); From 688f631c46b30d3fc1fd12c6652acfe3a12bcda1 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Tue, 7 Jul 2020 11:25:19 +0800 Subject: [PATCH 03/19] Change code style Change-Id: Ibdc44e527279308d6cad77aa7f1d230d27abd02e --- .../apache/doris/rewrite/mvrewrite/MVExprEquivalent.java | 8 ++++---- .../doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java index 2376c631b2cac6..d36df4d29356ff 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/MVExprEquivalent.java @@ -21,10 +21,10 @@ import com.google.common.collect.ImmutableList; -/* -Only support the once match from originExpr and newExpr -TODO：one query expr could be calculate by a group by mv columns -TODO: mvExprEqual(queryexpr, mvColumnExprList) +/** + * Only support the once match from originExpr and newExpr + * TODO：one query expr could be calculate by a group by mv columns + * TODO: mvExprEqual(queryexpr, mvColumnExprList) */ public class MVExprEquivalent { diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java index ddc60b40ab7969..45d8b756270c68 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/ToBitmapToSlotRefRule.java @@ -37,9 +37,9 @@ import java.util.List; -/* -Rewrite bitmap_union(to_bitmap(c1)) to bitmap_union(mv_bitmap_c1) -Rewrite bitmap_union_count(to_bitmap(c1)) to bitmap_union_count(mv_bitmap_c1) +/** + * Rewrite bitmap_union(to_bitmap(c1)) to bitmap_union(mv_bitmap_c1) + * Rewrite bitmap_union_count(to_bitmap(c1)) to bitmap_union_count(mv_bitmap_c1) */ public class ToBitmapToSlotRefRule implements ExprRewriteRule { From 83dbcdd05d2a9d6b99b5c27b48cac46418185878 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 8 Jul 2020 18:20:33 +0800 Subject: [PATCH 04/19] Add unit test Change-Id: I313b896c20a210ce5a2d9ccbc5ca6d8e78ade3e7 --- .../apache/doris/analysis/AggregateInfo.java | 5 +- .../doris/analysis/AggregateInfoBase.java | 2 +- .../analysis/CreateMaterializedViewStmt.java | 13 ++- .../doris/analysis/DataDescription.java | 6 +- .../java/org/apache/doris/analysis/Expr.java | 3 +- .../doris/analysis/FunctionCallExpr.java | 14 +-- .../analysis/MVColumnBitmapUnionPattern.java | 5 +- .../analysis/MVColumnHLLUnionPattern.java | 8 +- .../org/apache/doris/analysis/SelectStmt.java | 5 +- .../apache/doris/catalog/AggregateType.java | 2 +- .../org/apache/doris/catalog/FunctionSet.java | 11 +- .../main/java/org/apache/doris/load/Load.java | 4 +- .../doris/load/loadv2/dpp/SparkDpp.java | 52 +++++---- .../apache/doris/planner/OlapScanNode.java | 9 +- .../doris/planner/SingleNodePlanner.java | 2 +- .../doris/planner/StreamLoadScanNode.java | 19 +-- .../mvrewrite/CountDistinctToBitmap.java | 3 +- .../rewrite/mvrewrite/CountFieldToSum.java | 5 +- .../mvrewrite/FunctionCallEqualRule.java | 15 +-- .../doris/rewrite/mvrewrite/NDVToHll.java | 8 +- .../alter/MaterializedViewHandlerTest.java | 28 ----- .../CreateMaterializedViewStmtTest.java | 102 +++++----------- .../MVColumnBitmapUnionPatternTest.java | 110 ++++++++++++++++++ .../analysis/MVColumnHLLUnionPatternTest.java | 104 +++++++++++++++++ .../analysis/MVColumnOneChildPatternTest.java | 102 ++++++++++++++++ .../planner/MaterializedViewFunctionTest.java | 32 +++-- .../planner/MaterializedViewSelectorTest.java | 6 +- .../doris/planner/StreamLoadScanNodeTest.java | 5 +- 28 files changed, 485 insertions(+), 195 deletions(-) create mode 100644 fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java create mode 100644 fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java create mode 100644 fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java diff --git a/fe/src/main/java/org/apache/doris/analysis/AggregateInfo.java b/fe/src/main/java/org/apache/doris/analysis/AggregateInfo.java index 170bd01b5ce638..f5e8baae3b9a1b 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AggregateInfo.java +++ b/fe/src/main/java/org/apache/doris/analysis/AggregateInfo.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.common.AnalysisException; import org.apache.doris.planner.DataPartition; import org.apache.doris.thrift.TPartitionType; @@ -532,7 +533,7 @@ private void createSecondPhaseAggInfo( Preconditions.checkState(inputExpr.isAggregateFunction()); FunctionCallExpr aggExpr = null; if (!isMultiDistinct_) { - if (inputExpr.getFnName().getFunction().equalsIgnoreCase("COUNT")) { + if (inputExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { // COUNT(DISTINCT ...) -> // COUNT(IF(IsNull(), NULL, IF(IsNull(), NULL, ...))) // We need the nested IF to make sure that we do not count @@ -543,7 +544,7 @@ private void createSecondPhaseAggInfo( inputDesc.getSlots()); Preconditions.checkNotNull(ifExpr); ifExpr.analyzeNoThrow(analyzer); - aggExpr = new FunctionCallExpr("count", Lists.newArrayList(ifExpr)); + aggExpr = new FunctionCallExpr(FunctionSet.COUNT, Lists.newArrayList(ifExpr)); } else if (inputExpr.getFnName().getFunction().equals("group_concat")) { // Syntax: GROUP_CONCAT([DISTINCT] expression [, separator]) ArrayList exprList = Lists.newArrayList(); diff --git a/fe/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java b/fe/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java index a78bb352b441f6..9102c57d995a30 100644 --- a/fe/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java +++ b/fe/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java @@ -150,7 +150,7 @@ private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple // COUNT(), NDV() and NDV_NO_FINALIZE() are non-nullable. The latter two are used // by compute stats and compute incremental stats, respectively. - if (aggExpr.getFnName().getFunction().equals("count") + if (aggExpr.getFnName().getFunction().equals(FunctionSet.COUNT) || aggExpr.getFnName().getFunction().equals("ndv") || aggExpr.getFnName().getFunction().equals(FunctionSet.BITMAP_UNION_INT) || aggExpr.getFnName().getFunction().equals("ndv_no_finalize")) { diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 3b2dac76214048..eccccac0f1a05c 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -18,6 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; @@ -64,9 +65,9 @@ public class CreateMaterializedViewStmt extends DdlStmt { new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase())); FN_NAME_TO_PATTERN.put(AggregateType.MAX.name().toLowerCase(), new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase())); - FN_NAME_TO_PATTERN.put("count", new MVColumnOneChildPattern("count")); - FN_NAME_TO_PATTERN.put("bitmap_union", new MVColumnBitmapUnionPattern()); - FN_NAME_TO_PATTERN.put("hll_union", new MVColumnHLLUnionPattern()); + FN_NAME_TO_PATTERN.put(FunctionSet.COUNT, new MVColumnOneChildPattern(FunctionSet.COUNT)); + FN_NAME_TO_PATTERN.put(FunctionSet.BITMAP_UNION, new MVColumnBitmapUnionPattern()); + FN_NAME_TO_PATTERN.put(FunctionSet.HLL_UNION, new MVColumnHLLUnionPattern()); } private String mvName; @@ -354,19 +355,19 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); type = Type.BIGINT; break; - case "bitmap_union": + case FunctionSet.BITMAP_UNION: mvColumnName = mvColumnBuilder(functionName, baseColumnName); mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); defineExpr = functionChild0; type = Type.BITMAP; break; - case "hll_union": + case FunctionSet.HLL_UNION: mvColumnName = mvColumnBuilder(functionName, baseColumnName); mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); defineExpr = functionChild0; type = Type.HLL; break; - case "count": + case FunctionSet.COUNT: mvColumnName = mvColumnBuilder(functionName, baseColumnName); mvAggregateType = AggregateType.SUM; defineExpr = new CaseExpr(null, Lists.newArrayList(new CaseWhenClause( diff --git a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java index 7a1f339aae16fd..0d621c05ce136e 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DataDescription.java +++ b/fe/src/main/java/org/apache/doris/analysis/DataDescription.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.BinaryPredicate.Operator; import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -74,7 +75,6 @@ */ public class DataDescription { private static final Logger LOG = LogManager.getLogger(DataDescription.class); - public static String FUNCTION_HASH_HLL = "hll_hash"; // function isn't built-in function, hll_hash is not built-in function in hadoop load. private static final List HADOOP_SUPPORT_FUNCTION_NAMES = Arrays.asList( "strftime", @@ -84,7 +84,7 @@ public class DataDescription { "md5sum", "replace_value", "now", - "hll_hash", + FunctionSet.HLL_HASH, "substitute"); private final String tableName; @@ -399,7 +399,7 @@ public static void validateMappingFunction(String functionName, List arg validateMd5sum(args, columnNameMap); } else if (functionName.equalsIgnoreCase("replace_value")) { validateReplaceValue(args, mappingColumn); - } else if (functionName.equalsIgnoreCase(FUNCTION_HASH_HLL)) { + } else if (functionName.equalsIgnoreCase(FunctionSet.HLL_HASH)) { validateHllHash(args, columnNameMap); } else if (functionName.equalsIgnoreCase("now")) { validateNowFunction(mappingColumn); diff --git a/fe/src/main/java/org/apache/doris/analysis/Expr.java b/fe/src/main/java/org/apache/doris/analysis/Expr.java index 15d70e522fb6f8..afe7e243a7e7e9 100644 --- a/fe/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/src/main/java/org/apache/doris/analysis/Expr.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; @@ -136,7 +137,7 @@ public boolean apply(Expr arg) { || fnName.equalsIgnoreCase("max") || fnName.equalsIgnoreCase("min") || fnName.equalsIgnoreCase("avg") - || fnName.equalsIgnoreCase("count")); + || fnName.equalsIgnoreCase(FunctionSet.COUNT)); } else { return false; } diff --git a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 686b9079d4b69a..d0568dbc0a29dc 100644 --- a/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -104,7 +104,7 @@ public FunctionCallExpr(FunctionName fnName, FunctionParams params) { private FunctionCallExpr( FunctionName fnName, FunctionParams params, boolean isMergeAggFn) { - super(); + super(); this.fnName = fnName; fnParams = params; this.isMergeAggFn = isMergeAggFn; @@ -234,7 +234,7 @@ public boolean isDistinct() { } public boolean isCountStar() { - if (fnName.getFunction().equalsIgnoreCase("count")) { + if (fnName.getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { if (fnParams.isStar()) { return true; } else if (fnParams.exprs() == null || fnParams.exprs().isEmpty()) { @@ -255,7 +255,7 @@ public boolean isCountDistinctBitmapOrHLL() { return false; } - if (!fnName.getFunction().equalsIgnoreCase("count")) { + if (!fnName.getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { return false; } @@ -282,12 +282,12 @@ protected void toThrift(TExprNode msg) { } private void analyzeBuiltinAggFunction(Analyzer analyzer) throws AnalysisException { - if (fnParams.isStar() && !fnName.getFunction().equalsIgnoreCase("count")) { + if (fnParams.isStar() && !fnName.getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { throw new AnalysisException( "'*' can only be used in conjunction with COUNT: " + this.toSql()); } - if (fnName.getFunction().equalsIgnoreCase("count")) { + if (fnName.getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { // for multiple exprs count must be qualified with distinct if (children.size() > 1 && !fnParams.isDistinct()) { throw new AnalysisException( @@ -464,7 +464,7 @@ protected String getFunctionNotFoundError(Type[] argTypes) { return "'*' can only be used in conjunction with COUNT"; } - if (fnName.getFunction().equalsIgnoreCase("count")) { + if (fnName.getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { if (!fnParams.isDistinct() && argTypes.length > 1) { return "COUNT must have DISTINCT for multiple arguments: " + toSql(); } @@ -504,7 +504,7 @@ public void analyzeImpl(Analyzer analyzer) throws AnalysisException { return; } - if (fnName.getFunction().equals("count") && fnParams.isDistinct()) { + if (fnName.getFunction().equals(FunctionSet.COUNT) && fnParams.isDistinct()) { // Treat COUNT(DISTINCT ...) special because of how we do the equal. // There is no version of COUNT() that takes more than 1 argument but after // the equal, we only need count(*). diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java index 0a9e9b1d0ae496..db7822d1e24406 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java @@ -34,7 +34,7 @@ public boolean match(Expr expr) { return false; } FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); - if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("to_bitmap")) { + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) { return false; } if (child0FnExpr.getChild(0) instanceof SlotRef) { @@ -52,6 +52,7 @@ public boolean match(Expr expr) { @Override public String toString() { - return "bitmap_union(to_bitmap(column)) or bitmap_union(bitmap_column) in agg table"; + return FunctionSet.BITMAP_UNION + "(" + FunctionSet.TO_BITMAP + "(column)) " + + "or " + FunctionSet.BITMAP_UNION + "(bitmap_column) in agg table"; } } diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java index b15e092706b9b3..51b1068e7171c4 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.FunctionSet; + public class MVColumnHLLUnionPattern implements MVColumnPattern { @Override public boolean match(Expr expr) { @@ -25,14 +27,14 @@ public boolean match(Expr expr) { } FunctionCallExpr fnExpr = (FunctionCallExpr) expr; String fnNameString = fnExpr.getFnName().getFunction(); - if (!fnNameString.equalsIgnoreCase("hll_union")){ + if (!fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION)){ return false; } if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { return false; } FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); - if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase("hll_hash")) { + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) { return false; } if (child0FnExpr.getChild(0) instanceof SlotRef) { @@ -50,6 +52,6 @@ public boolean match(Expr expr) { @Override public String toString() { - return "hll_union(hll_hash(column))"; + return "hll_union(" + FunctionSet.HLL_HASH + "(column))"; } } diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java index d3b8419c31ef0b..1e3fef3c53dc7f 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Type; @@ -1223,7 +1224,7 @@ private ExprSubstitutionMap createSumOrCountMultiDistinctSMap( for (FunctionCallExpr inputExpr : distinctExprs) { Expr replaceExpr = null; final String functionName = inputExpr.getFnName().getFunction(); - if (functionName.equalsIgnoreCase("COUNT")) { + if (functionName.equalsIgnoreCase(FunctionSet.COUNT)) { final List countInputExpr = Lists.newArrayList(inputExpr.getChild(0).clone(null)); replaceExpr = new FunctionCallExpr("MULTI_DISTINCT_COUNT", new FunctionParams(inputExpr.isDistinct(), countInputExpr)); @@ -1283,7 +1284,7 @@ public boolean apply(FunctionCallExpr expr) { com.google.common.base.Predicate isCountPred = new com.google.common.base.Predicate() { public boolean apply(FunctionCallExpr expr) { - return expr.getFnName().getFunction().equals("count"); + return expr.getFnName().getFunction().equals(FunctionSet.COUNT); } }; diff --git a/fe/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/src/main/java/org/apache/doris/catalog/AggregateType.java index bce3c14b9fa524..7741814a5dbe7f 100644 --- a/fe/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -38,7 +38,7 @@ public enum AggregateType { private static EnumMap> compatibilityMap; static { - compatibilityMap = new EnumMap>(AggregateType.class); + compatibilityMap = new EnumMap<>(AggregateType.class); List primitiveTypeList = Lists.newArrayList(); primitiveTypeList.add(PrimitiveType.TINYINT); diff --git a/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java index 4f9a0c75da7468..d581ba4733be27 100644 --- a/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -284,6 +284,9 @@ public boolean isNonNullResultWithNullParamFunctions(String funcName) { "16knuth_var_updateIN9doris_udf9DoubleValEEEvPNS2_15FunctionContextERKT_PNS2_9StringValE") .build(); + public static final String HLL_HASH = "hll_hash"; + public static final String HLL_UNION = "hll_union"; + private static final Map HLL_UPDATE_SYMBOL = ImmutableMap.builder() .put(Type.BOOLEAN, @@ -535,6 +538,7 @@ public boolean isNonNullResultWithNullParamFunctions(String funcName) { .build(); + public static final String TO_BITMAP = "to_bitmap"; public static final String BITMAP_UNION = "bitmap_union"; public static final String BITMAP_UNION_COUNT = "bitmap_union_count"; public static final String BITMAP_UNION_INT = "bitmap_union_int"; @@ -819,6 +823,7 @@ public void addBuiltin(Function fn) { addFunction(fn, true); } + public static final String COUNT = "count"; // Populate all the aggregate builtins in the catalog. // null symbols indicate the function does not need that step of the evaluation. // An empty symbol indicates a TODO for the BE to implement the function. @@ -834,7 +839,7 @@ private void initAggregateBuiltins() { // Type stringType[] = {Type.CHAR, Type.VARCHAR}; // count(*) - addBuiltin(AggregateFunction.createBuiltin("count", + addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, new ArrayList(), Type.BIGINT, Type.BIGINT, prefix + "9init_zeroIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "17count_star_updateEPN9doris_udf15FunctionContextEPNS1_9BigIntValE", @@ -851,7 +856,7 @@ private void initAggregateBuiltins() { continue; // promoted to STRING } // Count - addBuiltin(AggregateFunction.createBuiltin("count", + addBuiltin(AggregateFunction.createBuiltin(FunctionSet.COUNT, Lists.newArrayList(t), Type.BIGINT, Type.BIGINT, prefix + "9init_zeroIN9doris_udf9BigIntValEEEvPNS2_15FunctionContextEPT_", prefix + "12count_updateEPN9doris_udf15FunctionContextERKNS1_6AnyValEPNS1_9BigIntValE", @@ -1027,7 +1032,7 @@ private void initAggregateBuiltins() { true, true, true)); // HLL_UNION - addBuiltin(AggregateFunction.createBuiltin("hll_union", + addBuiltin(AggregateFunction.createBuiltin(HLL_UNION, Lists.newArrayList(t), Type.HLL, Type.HLL, "_ZN5doris12HllFunctions8hll_initEPN9doris_udf15FunctionContextEPNS1_9StringValE", "_ZN5doris12HllFunctions9hll_mergeEPN9doris_udf15FunctionContextERKNS1_9StringValEPS4_", diff --git a/fe/src/main/java/org/apache/doris/load/Load.java b/fe/src/main/java/org/apache/doris/load/Load.java index 683f5a9c9921d4..99218b090441ef 100644 --- a/fe/src/main/java/org/apache/doris/load/Load.java +++ b/fe/src/main/java/org/apache/doris/load/Load.java @@ -48,6 +48,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; @@ -338,8 +339,7 @@ public boolean addMiniLoadJob(TMiniLoadRequest request) throws DdlException { final String resultColumn = pairList.get(0); final String hashColumn = pairList.get(1); - final Pair> pair = new Pair>( - DataDescription.FUNCTION_HASH_HLL, + final Pair> pair = new Pair>(FunctionSet.HLL_HASH, Arrays.asList(hashColumn)); dataDescription.addColumnMapping(resultColumn, pair); } diff --git a/fe/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java b/fe/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java index 3bc8c9575de104..3a1002638b32af 100644 --- a/fe/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java +++ b/fe/src/main/java/org/apache/doris/load/loadv2/dpp/SparkDpp.java @@ -17,63 +17,65 @@ package org.apache.doris.load.loadv2.dpp; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.common.UserException; +import org.apache.doris.load.loadv2.etl.EtlJobConfig; + import com.google.common.base.Strings; import com.google.gson.Gson; + import org.apache.commons.lang3.StringUtils; -import org.apache.doris.common.UserException; -import org.apache.doris.load.loadv2.etl.EtlJobConfig; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.spark.Partitioner; +import org.apache.spark.TaskContext; +import org.apache.spark.api.java.JavaPairRDD; +import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.function.ForeachPartitionFunction; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.api.java.function.PairFunction; +import org.apache.spark.sql.Column; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; import org.apache.spark.sql.catalyst.encoders.RowEncoder; import org.apache.spark.sql.execution.datasources.parquet.ParquetWriteSupport; import org.apache.spark.sql.functions; -import org.apache.spark.sql.RowFactory; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.Dataset; -import org.apache.spark.sql.Column; -import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; import org.apache.spark.util.LongAccumulator; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.TaskContext; - -import scala.Tuple2; -import scala.collection.Seq; import java.io.IOException; import java.math.BigInteger; import java.net.URI; import java.net.URISyntaxException; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.List; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; -import java.util.HashSet; -import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; import java.util.stream.Collectors; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.FSDataOutputStream; +import scala.Tuple2; import scala.collection.JavaConverters; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import scala.collection.Seq; // This class is a Spark-based data preprocessing program, // which will make use of the distributed compute framework of spark to @@ -490,7 +492,7 @@ private Dataset convertSrcDataframeToDstDataframe(EtlJobConfig.EtlIndex bas // 2. process the mapping columns for (String mappingColumn : mappingColumns) { String mappingDescription = columnMappings.get(mappingColumn).toDescription(); - if (mappingDescription.toLowerCase().contains("hll_hash")) { + if (mappingDescription.toLowerCase().contains(FunctionSet.HLL_HASH)) { continue; } // here should cast data type to dst column type diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 9c55afb17a3917..26b5b286280822 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -185,7 +185,7 @@ public Collection getSelectedPartitionIds() { * @throws UserException */ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isPreAggregation, String - reasonOfDisable) + reasonOfDisable, Analyzer analyzer) throws UserException { if (selectedIndexId == this.selectedIndexId && isPreAggregation == this.isPreAggregation) { return; @@ -229,6 +229,7 @@ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isP this.reasonOfPreAggregation = reasonOfDisable; long start = System.currentTimeMillis(); computeTabletInfo(); + computeStats(analyzer); LOG.debug("distribution prune cost: {} ms", (System.currentTimeMillis() - start)); LOG.info("Using the new scan range info instead of the old one. {}, {}", situation ,scanRangeInfo); } else { @@ -325,7 +326,11 @@ private void addScanRangeLocations(Partition partition, MaterializedIndex index, List tablets, long localBeId) throws UserException { - + /** + * The addScanRangeLocations could be invoked only once. + * So the scanBackendIds should be empty in the beginning. + */ + Preconditions.checkState(scanBackendIds.size() == 0); int logNum = 0; int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); String schemaHashStr = String.valueOf(schemaHash); diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 2aec056fd63ca1..11b102f781b438 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -785,7 +785,7 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) continue; } olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), - bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable()); + bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable(), analyzer); } } else { diff --git a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 75bb1bf6d8e745..2fa49190b0418b 100644 --- a/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -17,8 +17,6 @@ package org.apache.doris.planner; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.Expr; @@ -31,6 +29,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; @@ -49,6 +48,10 @@ import org.apache.doris.thrift.TScanRange; import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TUniqueId; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -188,13 +191,15 @@ private void finalizeParams() throws UserException { // check hll_hash if (dstSlotDesc.getType().getPrimitiveType() == PrimitiveType.HLL) { if (!(expr instanceof FunctionCallExpr)) { - throw new AnalysisException("HLL column must use hll_hash function, like " - + dstSlotDesc.getColumn().getName() + "=hll_hash(xxx)"); + throw new AnalysisException("HLL column must use " + FunctionSet.HLL_HASH + " function, like " + + dstSlotDesc.getColumn().getName() + "=" + FunctionSet.HLL_HASH + "(xxx)"); } FunctionCallExpr fn = (FunctionCallExpr) expr; - if (!fn.getFnName().getFunction().equalsIgnoreCase("hll_hash") && !fn.getFnName().getFunction().equalsIgnoreCase("hll_empty")) { - throw new AnalysisException("HLL column must use hll_hash function, like " - + dstSlotDesc.getColumn().getName() + "=hll_hash(xxx) or " + dstSlotDesc.getColumn().getName() + "=hll_empty()"); + if (!fn.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH) + && !fn.getFnName().getFunction().equalsIgnoreCase("hll_empty")) { + throw new AnalysisException("HLL column must use " + FunctionSet.HLL_HASH + " function, like " + + dstSlotDesc.getColumn().getName() + "=" + FunctionSet.HLL_HASH + + "(xxx) or " + dstSlotDesc.getColumn().getName() + "=hll_empty()"); } expr.setType(Type.HLL); } diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java index b50eab71bb6add..79312b5f820e0d 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountDistinctToBitmap.java @@ -25,6 +25,7 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; @@ -59,7 +60,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { if (!fnExpr.getParams().isDistinct()) { return expr; } - if (!fnExpr.getFnName().getFunction().equalsIgnoreCase("count")) { + if (!fnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { return expr; } if (fnExpr.getChildren().size() != 1 || !(fnExpr.getChild(0) instanceof SlotRef)) { diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java index 138f30643b8c34..4f55a9b85d1ca3 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/CountFieldToSum.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; @@ -53,7 +54,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { return expr; } FunctionCallExpr fnExpr = (FunctionCallExpr) expr; - if (!fnExpr.getFnName().getFunction().equalsIgnoreCase("count")) { + if (!fnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.COUNT)) { return expr; } if (fnExpr.getChildren().size() != 1 || !(fnExpr.getChild(0) instanceof SlotRef)) { @@ -69,7 +70,7 @@ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException { // check column String queryColumnName = column.getName(); - String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("count", queryColumnName); + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder(FunctionSet.COUNT, queryColumnName); Column mvColumn = olapTable.getVisibleColumn(mvColumnName); if (mvColumn == null) { return expr; diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java index bde6a8142e2c2f..aee0871851c450 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/FunctionCallEqualRule.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.catalog.FunctionSet; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSetMultimap; @@ -34,13 +35,13 @@ public class FunctionCallEqualRule implements MVExprEqualRule { builder.put("sum", "sum"); builder.put("max", "max"); builder.put("min", "min"); - builder.put("bitmap_union", "bitmap_union"); - builder.put("bitmap_union", "bitmap_union_count"); - builder.put("hll_union", "hll_union_agg"); - builder.put("hll_union", "hll_union"); - builder.put("hll_union", "hll_raw_agg"); - builder.put("to_bitmap", "to_bitmap"); - builder.put("hll_hash", "hll_hash"); + builder.put(FunctionSet.BITMAP_UNION, FunctionSet.BITMAP_UNION); + builder.put(FunctionSet.BITMAP_UNION, FunctionSet.BITMAP_UNION_COUNT); + builder.put(FunctionSet.HLL_UNION, "hll_union_agg"); + builder.put(FunctionSet.HLL_UNION, "hll_union"); + builder.put(FunctionSet.HLL_UNION, "hll_raw_agg"); + builder.put(FunctionSet.TO_BITMAP, FunctionSet.TO_BITMAP); + builder.put(FunctionSet.HLL_HASH, FunctionSet.HLL_HASH); columnAggTypeMatchFnName = builder.build(); } diff --git a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java index 37eb7767c020e8..e96abb4b218bc4 100644 --- a/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java +++ b/fe/src/main/java/org/apache/doris/rewrite/mvrewrite/NDVToHll.java @@ -36,14 +36,14 @@ import java.util.List; /** - * For duplicate table, the ndv(k1) could be rewritten to hll_union_agg(mv_hll_hash_k1) when bitmap + * For duplicate table, the ndv(k1) could be rewritten to hll_union_agg(mv_hll_union_k1) when bitmap * mv exists. * For example: * Table: (k1 int, k2 int) - * MV: (k1 int, mv_hll_hash_k2 hll hll_union) - * mv_hll_hash_k2 = hll_hash(k2) + * MV: (k1 int, mv_hll_union_k2 hll hll_union) + * mv_hll_union_k2 = hll_hash(k2) * Query: select k1, count(distinct k2) from table group by k1 - * Rewritten query: select k1, hll_union_agg(mv_hll_hash_k2) from table group by k1 + * Rewritten query: select k1, hll_union_agg(mv_hll_union_k2) from table group by k1 */ public class NDVToHll implements ExprRewriteRule{ public static final ExprRewriteRule INSTANCE = new NDVToHll(); diff --git a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java index e9f593bd4e548f..de315e49ea34a8 100644 --- a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java +++ b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java @@ -172,34 +172,6 @@ public void testDuplicateMVName(@Injectable CreateMaterializedViewStmt createMat } } - @Test - public void testInvalidMVColumn(@Injectable CreateMaterializedViewStmt createMaterializedViewStmt, - @Injectable OlapTable olapTable) { - final String mvName = "mv1"; - final String mvColumnName = "mv_k1"; - MVColumnItem mvColumnItem = new MVColumnItem(mvColumnName, Type.BIGINT); - new Expectations() { - { - olapTable.hasMaterializedIndex(mvName); - result = false; - createMaterializedViewStmt.getMVName(); - result = mvName; - createMaterializedViewStmt.getMVColumnItemList(); - result = Lists.newArrayList(mvColumnItem); - olapTable.getColumn(mvColumnName); - result = null; - } - }; - MaterializedViewHandler materializedViewHandler = new MaterializedViewHandler(); - try { - Deencapsulation.invoke(materializedViewHandler, "checkAndPrepareMaterializedView", - createMaterializedViewStmt, olapTable); - Assert.fail(); - } catch (Exception e) { - System.out.print(e.getMessage()); - } - } - @Test public void testInvalidAggregateType(@Injectable CreateMaterializedViewStmt createMaterializedViewStmt, @Injectable OlapTable olapTable) { diff --git a/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java index 1a778fbdd19001..61ff906977a3db 100644 --- a/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java @@ -91,7 +91,7 @@ public void testAggregateWithFunctionColumnInSelectClause(@Injectable Arithmetic selectStmt.analyze(analyzer); selectStmt.getSelectList(); result = selectList; - arithmeticExpr.toSql(); + arithmeticExpr.toString(); result = "a+b"; } }; @@ -250,13 +250,15 @@ public void testOrderOfColumn(@Injectable SlotRef slotRef1, @Test public void testOrderByAggregateColumn(@Injectable SlotRef slotRef1, - @Injectable SlotRef slotRef2, - @Injectable FunctionCallExpr functionCallExpr, @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException { SelectList selectList = new SelectList(); SelectListItem selectListItem1 = new SelectListItem(slotRef1, null); selectList.addItem(selectListItem1); + TableName tableName = new TableName("db", "table"); + SlotRef slotRef2 = new SlotRef(tableName, "v1"); + List fnChildren = Lists.newArrayList(slotRef2); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", fnChildren); SelectListItem selectListItem2 = new SelectListItem(functionCallExpr, null); selectList.addItem(selectListItem2); OrderByElement orderByElement1 = new OrderByElement(functionCallExpr, false, false); @@ -280,14 +282,6 @@ public void testOrderByAggregateColumn(@Injectable SlotRef slotRef1, result = orderByElementList; slotRef1.getColumnName(); result = "k1"; - slotRef2.getColumnName(); - result = "v1"; - functionCallExpr.getFnName().getFunction(); - result = "sum"; - functionCallExpr.getChildren(); - result = Lists.newArrayList(slotRef2); - functionCallExpr.getChild(0); - result = slotRef2; } }; CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null); @@ -300,12 +294,14 @@ public void testOrderByAggregateColumn(@Injectable SlotRef slotRef1, } @Test - public void testDuplicateColumn(@Injectable SlotRef slotRef1, - @Injectable FunctionCallExpr functionCallExpr, - @Injectable SelectStmt selectStmt) throws UserException { + public void testDuplicateColumn(@Injectable SelectStmt selectStmt) throws UserException { SelectList selectList = new SelectList(); + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "k1"); SelectListItem selectListItem1 = new SelectListItem(slotRef1, null); selectList.addItem(selectListItem1); + List fnChildren = Lists.newArrayList(slotRef1); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", fnChildren); SelectListItem selectListItem2 = new SelectListItem(functionCallExpr, null); selectList.addItem(selectListItem2); @@ -316,14 +312,6 @@ public void testDuplicateColumn(@Injectable SlotRef slotRef1, selectStmt.analyze(analyzer); selectStmt.getSelectList(); result = selectList; - slotRef1.getColumnName(); - result = "k1"; - functionCallExpr.getFnName().getFunction(); - result = "sum"; - functionCallExpr.getChildren(); - result = Lists.newArrayList(slotRef1); - functionCallExpr.getChild(0); - result = slotRef1; } }; CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null); @@ -336,14 +324,18 @@ public void testDuplicateColumn(@Injectable SlotRef slotRef1, } @Test - public void testDuplicateColumn1(@Injectable SlotRef slotRef1, @Injectable SlotRef slotRef2, - @Injectable FunctionCallExpr functionCallExpr1, @Injectable FunctionCallExpr functionCallExpr2, + public void testDuplicateColumn1(@Injectable SlotRef slotRef1, @Injectable SelectStmt selectStmt) throws UserException { SelectList selectList = new SelectList(); SelectListItem selectListItem1 = new SelectListItem(slotRef1, null); selectList.addItem(selectListItem1); + TableName tableName = new TableName("db", "table"); + SlotRef slotRef2 = new SlotRef(tableName, "k2"); + List fn1Children = Lists.newArrayList(slotRef2); + FunctionCallExpr functionCallExpr1 = new FunctionCallExpr("sum", fn1Children); SelectListItem selectListItem2 = new SelectListItem(functionCallExpr1, null); selectList.addItem(selectListItem2); + FunctionCallExpr functionCallExpr2 = new FunctionCallExpr("max", fn1Children); SelectListItem selectListItem3 = new SelectListItem(functionCallExpr2, null); selectList.addItem(selectListItem3); @@ -356,20 +348,6 @@ public void testDuplicateColumn1(@Injectable SlotRef slotRef1, @Injectable SlotR result = selectList; slotRef1.getColumnName(); result = "k1"; - slotRef2.getColumnName(); - result = "k2"; - functionCallExpr1.getFnName().getFunction(); - result = "sum"; - functionCallExpr1.getChildren(); - result = Lists.newArrayList(slotRef2); - functionCallExpr1.getChild(0); - result = slotRef2; - functionCallExpr2.getFnName().getFunction(); - result = "max"; - functionCallExpr2.getChildren(); - result = Lists.newArrayList(slotRef2); - functionCallExpr2.getChild(0); - result = slotRef2; } }; CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null); @@ -384,8 +362,6 @@ public void testDuplicateColumn1(@Injectable SlotRef slotRef1, @Injectable SlotR @Test public void testOrderByColumnsLessThenGroupByColumns(@Injectable SlotRef slotRef1, @Injectable SlotRef slotRef2, - @Injectable FunctionCallExpr functionCallExpr, - @Injectable SlotRef functionChild0, @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException { SelectList selectList = new SelectList(); @@ -393,6 +369,10 @@ public void testOrderByColumnsLessThenGroupByColumns(@Injectable SlotRef slotRef selectList.addItem(selectListItem1); SelectListItem selectListItem2 = new SelectListItem(slotRef2, null); selectList.addItem(selectListItem2); + TableName tableName = new TableName("db", "table"); + SlotRef functionChild0 = new SlotRef(tableName, "v1"); + List fn1Children = Lists.newArrayList(functionChild0); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", fn1Children); SelectListItem selectListItem3 = new SelectListItem(functionCallExpr, null); selectList.addItem(selectListItem3); OrderByElement orderByElement1 = new OrderByElement(slotRef1, false, false); @@ -417,14 +397,6 @@ public void testOrderByColumnsLessThenGroupByColumns(@Injectable SlotRef slotRef result = "k1"; slotRef2.getColumnName(); result = "non-k2"; - functionChild0.getColumnName(); - result = "v1"; - functionCallExpr.getFnName().getFunction(); - result = "sum"; - functionCallExpr.getChildren(); - result = Lists.newArrayList(slotRef1); - functionCallExpr.getChild(0); - result = functionChild0; } }; CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null); @@ -441,8 +413,6 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1, @Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4, - @Injectable FunctionCallExpr functionCallExpr, - @Injectable SlotRef functionChild0, @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt, @Injectable AggregateInfo aggregateInfo) throws UserException { @@ -455,14 +425,17 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1, selectList.addItem(selectListItem3); SelectListItem selectListItem4 = new SelectListItem(slotRef4, null); selectList.addItem(selectListItem4); - + TableName tableName = new TableName("db", "table"); + final String columnName5 = "sum_v2"; + SlotRef functionChild0 = new SlotRef(tableName, columnName5); + List fn1Children = Lists.newArrayList(functionChild0); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", fn1Children); SelectListItem selectListItem5 = new SelectListItem(functionCallExpr, null); selectList.addItem(selectListItem5); final String columnName1 = "k1"; final String columnName2 = "k2"; final String columnName3 = "k3"; final String columnName4 = "v1"; - final String columnName5 = "sum_v2"; new Expectations() { { @@ -483,12 +456,6 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1, selectStmt.getLimit(); result = -1; selectStmt.analyze(analyzer); - functionCallExpr.getChild(0); - result = functionChild0; - functionCallExpr.getChildren(); - result = Lists.newArrayList(functionChild0); - functionCallExpr.getFnName().getFunction(); - result = "sum"; slotRef1.getColumnName(); result = columnName1; slotRef2.getColumnName(); @@ -497,8 +464,6 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1, result = columnName3; slotRef4.getColumnName(); result = columnName4; - functionChild0.getColumnName(); - result = columnName5; } }; @@ -939,8 +904,6 @@ public void testMVColumnsWithFirstVarchar(@Injectable SlotRef slotRef1, @Test public void testMVColumns(@Injectable SlotRef slotRef1, @Injectable SlotRef slotRef2, - @Injectable FunctionCallExpr functionCallExpr, - @Injectable SlotRef functionChild0, @Injectable TableRef tableRef, @Injectable SelectStmt selectStmt, @Injectable AggregateInfo aggregateInfo) throws UserException { @@ -949,6 +912,11 @@ public void testMVColumns(@Injectable SlotRef slotRef1, selectList.addItem(selectListItem1); SelectListItem selectListItem2 = new SelectListItem(slotRef2, null); selectList.addItem(selectListItem2); + TableName tableName = new TableName("db", "table"); + final String columnName3 = "sum_v2"; + SlotRef slotRef = new SlotRef(tableName, columnName3); + List children = Lists.newArrayList(slotRef); + FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", children); SelectListItem selectListItem3 = new SelectListItem(functionCallExpr, null); selectList.addItem(selectListItem3); OrderByElement orderByElement1 = new OrderByElement(slotRef1, false, false); @@ -956,7 +924,7 @@ public void testMVColumns(@Injectable SlotRef slotRef1, ArrayList orderByElementList = Lists.newArrayList(orderByElement1, orderByElement2); final String columnName1 = "k1"; final String columnName2 = "v1"; - final String columnName3 = "sum_v2"; + new Expectations() { { analyzer.getClusterName(); @@ -976,18 +944,10 @@ public void testMVColumns(@Injectable SlotRef slotRef1, selectStmt.getLimit(); result = -1; selectStmt.analyze(analyzer); - functionCallExpr.getChild(0); - result = functionChild0; - functionCallExpr.getChildren(); - result = Lists.newArrayList(functionChild0); - functionCallExpr.getFnName().getFunction(); - result = "sum"; slotRef1.getColumnName(); result = columnName1; slotRef2.getColumnName(); result = columnName2; - functionChild0.getColumnName(); - result = columnName3; } }; diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java new file mode 100644 index 00000000000000..dbe6b5763eed54 --- /dev/null +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.FunctionSet; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import mockit.Expectations; +import mockit.Injectable; + +public class MVColumnBitmapUnionPatternTest { + + @Test + public void testCorrectExpr1() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testCorrectExpr2(@Injectable CastExpr castExpr) { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + new Expectations() { + { + castExpr.getChild(0); + result = slotRef; + } + }; + List child0Params = Lists.newArrayList(); + child0Params.add(castExpr); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testUpperCaseOfFunction() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP.toUpperCase(), child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION.toUpperCase(), params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testIncorrectArithmeticExpr1() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + SlotRef slotRef2 = new SlotRef(tableName, "c2"); + ArithmeticExpr arithmeticExpr = new ArithmeticExpr(ArithmeticExpr.Operator.ADD, slotRef1, slotRef2); + List params = Lists.newArrayList(); + params.add(arithmeticExpr); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } + + @Test + public void testIncorrectArithmeticExpr2() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + SlotRef slotRef2 = new SlotRef(tableName, "c2"); + ArithmeticExpr arithmeticExpr = new ArithmeticExpr(ArithmeticExpr.Operator.ADD, slotRef1, slotRef2); + List child0Params = Lists.newArrayList(); + child0Params.add(arithmeticExpr); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.TO_BITMAP, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } +} diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java new file mode 100644 index 00000000000000..e98ab4b10ee7f9 --- /dev/null +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.FunctionSet; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import mockit.Expectations; +import mockit.Injectable; + +public class MVColumnHLLUnionPatternTest { + + @Test + public void testCorrectExpr1() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.HLL_HASH, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testCorrectExpr2(@Injectable CastExpr castExpr) { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + new Expectations() { + { + castExpr.getChild(0); + result = slotRef; + } + }; + List child0Params = Lists.newArrayList(); + child0Params.add(castExpr); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.HLL_HASH, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testUpperCaseOfFunction() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.HLL_HASH.toUpperCase(), child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION.toUpperCase(), params); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + + @Test + public void testIncorrectLiteralExpr1() { + IntLiteral intLiteral = new IntLiteral(1); + List params = Lists.newArrayList(); + params.add(intLiteral); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } + + @Test + public void testIncorrectLiteralExpr2() { + IntLiteral intLiteral = new IntLiteral(1); + List child0Params = Lists.newArrayList(); + child0Params.add(intLiteral); + FunctionCallExpr child0 = new FunctionCallExpr(FunctionSet.HLL_HASH, child0Params); + List params = Lists.newArrayList(); + params.add(child0); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertFalse(pattern.match(expr)); + } +} diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java new file mode 100644 index 00000000000000..976483ade999fb --- /dev/null +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.FunctionSet; + +import com.google.common.collect.Lists; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +import mockit.Expectations; +import mockit.Injectable; + +public class MVColumnOneChildPatternTest { + + @Test + public void testCorrectSum() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List params = Lists.newArrayList(); + params.add(slotRef); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(AggregateType.SUM.name(), params); + MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern( + AggregateType.SUM.name().toLowerCase()); + Assert.assertTrue(mvColumnOneChildPattern.match(functionCallExpr)); + } + + @Test + public void testCorrectMin(@Injectable CastExpr castExpr) { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List child0Params = Lists.newArrayList(); + child0Params.add(slotRef); + new Expectations() { + { + castExpr.getChild(0); + result = child0Params; + } + }; + List params = Lists.newArrayList(); + params.add(castExpr); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(AggregateType.MIN.name(), params); + MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern( + AggregateType.MIN.name().toLowerCase()); + Assert.assertTrue(mvColumnOneChildPattern.match(functionCallExpr)); + } + + @Test + public void testCorrectCountField() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + List params = Lists.newArrayList(); + params.add(slotRef); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(FunctionSet.COUNT, params); + MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern(FunctionSet.COUNT.toLowerCase()); + Assert.assertTrue(mvColumnOneChildPattern.match(functionCallExpr)); + } + + @Test + public void testIncorrectLiteral() { + IntLiteral intLiteral = new IntLiteral(1); + List params = Lists.newArrayList(); + params.add(intLiteral); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(AggregateType.SUM.name(), params); + MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern( + AggregateType.SUM.name().toLowerCase()); + Assert.assertFalse(mvColumnOneChildPattern.match(functionCallExpr)); + } + + @Test + public void testIncorrectArithmeticExpr() { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + SlotRef slotRef2 = new SlotRef(tableName, "c2"); + ArithmeticExpr arithmeticExpr = new ArithmeticExpr(ArithmeticExpr.Operator.ADD, slotRef1, slotRef2); + List params = Lists.newArrayList(); + params.add(arithmeticExpr); + FunctionCallExpr functionCallExpr = new FunctionCallExpr(AggregateType.SUM.name(), params); + MVColumnOneChildPattern mvColumnOneChildPattern = new MVColumnOneChildPattern( + AggregateType.SUM.name().toLowerCase()); + Assert.assertFalse(mvColumnOneChildPattern.match(functionCallExpr)); + } +} diff --git a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java index fda215cbae68b8..6d30b324d7df8a 100644 --- a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java +++ b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java @@ -18,12 +18,14 @@ package org.apache.doris.planner; import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.common.FeConstants; import org.apache.doris.utframe.DorisAssert; import org.apache.doris.utframe.UtFrameUtils; import org.junit.After; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -720,7 +722,7 @@ public void testAggTableCountDistinctInBitmapType() throws Exception { @Test public void testAggTableCountDistinctInHllType() throws Exception { - String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 hll hll_union) Aggregate KEY (k1) " + + String aggTable = "CREATE TABLE " + TEST_TABLE_NAME + " (k1 int, v1 hll " + FunctionSet.HLL_UNION + ") Aggregate KEY (k1) " + "DISTRIBUTED BY HASH(k1) BUCKETS 3 PROPERTIES ('replication_num' = '1');"; dorisAssert.withTable(aggTable); String query = "select k1, count(distinct v1) from " + TEST_TABLE_NAME + " group by k1;"; @@ -743,13 +745,13 @@ public void testIncorrectRewriteCountDistinct() throws Exception { "bitmap_union(to_bitmap(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; dorisAssert.withMaterializedView(createUserTagMVSql); String query = "select user_name, count(distinct tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; - dorisAssert.query(query).explainContains(USER_TAG_TABLE_NAME, "count"); + dorisAssert.query(query).explainContains(USER_TAG_TABLE_NAME, FunctionSet.COUNT); } @Test public void testNDVToHll() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + - "`hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; dorisAssert.withMaterializedView(createUserTagMVSql); String query = "select ndv(tag_id) from " + USER_TAG_TABLE_NAME + ";"; dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, "hll_union_agg"); @@ -758,14 +760,14 @@ public void testNDVToHll() throws Exception { @Test public void testHLLUnionFamilyRewrite() throws Exception { String createUserTagMVSql = "create materialized view " + USER_TAG_MV_NAME + " as select user_id, " + - "`hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; + "`" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + " group by user_id;"; dorisAssert.withMaterializedView(createUserTagMVSql); - String query = "select `hll_union`(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; - String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("hll_union", "tag_id"); + String query = "select `" + FunctionSet.HLL_UNION + "`(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("" + FunctionSet.HLL_UNION + "", "tag_id"); dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); - query = "select hll_union_agg(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + query = "select hll_union_agg(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); - query = "select hll_raw_agg(hll_hash(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; + query = "select hll_raw_agg(" + FunctionSet.HLL_HASH + "(tag_id)) from " + USER_TAG_TABLE_NAME + ";"; dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); } @@ -787,10 +789,22 @@ public void testCountFieldInQuery() throws Exception { "count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_id;"; dorisAssert.withMaterializedView(createUserTagMVSql); String query = "select count(tag_id) from " + USER_TAG_TABLE_NAME + ";"; - String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder("count", "tag_id"); + String mvColumnName = CreateMaterializedViewStmt.mvColumnBuilder(FunctionSet.COUNT, "tag_id"); dorisAssert.query(query).explainContains(USER_TAG_MV_NAME, mvColumnName); query = "select user_name, count(tag_id) from " + USER_TAG_TABLE_NAME + " group by user_name;"; dorisAssert.query(query).explainWithout(USER_TAG_MV_NAME); } + @Test + public void testInvalidColumnInCreateMVStmt() throws Exception { + String createMVSQL = "create materialized view " + USER_TAG_MV_NAME + " as select invalid_column, user_id from " + + USER_TAG_TABLE_NAME + ";"; + try { + dorisAssert.withMaterializedView(createMVSQL); + Assert.fail(); + } catch (Exception e) { + System.out.println(e.getMessage()); + } + } + } diff --git a/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java b/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java index af4a8fc4c0b60d..1c8ef7c78b0241 100644 --- a/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java +++ b/fe/src/test/java/org/apache/doris/planner/MaterializedViewSelectorTest.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.AggregateInfo; import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SelectStmt; import org.apache.doris.analysis.SlotDescriptor; @@ -284,6 +283,8 @@ public void testCheckAggregationFunction(@Injectable SelectStmt selectStmt, @Inj result = null; indexMeta1.getSchema(); result = index1Columns; + indexMeta1.getKeysType(); + result = KeysType.DUP_KEYS; indexMeta2.getSchema(); result = index2Columns; indexMeta3.getSchema(); @@ -298,9 +299,8 @@ public void testCheckAggregationFunction(@Injectable SelectStmt selectStmt, @Inj Set aggregatedColumnsInQueryOutput = Sets.newHashSet(); aggregatedColumnsInQueryOutput.add(functionCallExpr); Deencapsulation.setField(selector, "isSPJQuery", false); - Map candidateIndexIdToRewriteSMap = Maps.newHashMap(); Deencapsulation.invoke(selector, "checkAggregationFunction", aggregatedColumnsInQueryOutput, - candidateIndexIdToSchema, candidateIndexIdToRewriteSMap); + candidateIndexIdToSchema); Assert.assertEquals(2, candidateIndexIdToSchema.size()); Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(1))); Assert.assertTrue(candidateIndexIdToSchema.keySet().contains(new Long(3))); diff --git a/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java b/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java index 9e4ebdc77fda6e..5e2511badf96e1 100644 --- a/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java +++ b/fe/src/test/java/org/apache/doris/planner/StreamLoadScanNodeTest.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Function; +import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarFunction; @@ -300,7 +301,7 @@ public void testHllColumnsNormal() throws UserException { new Expectations() {{ catalog.getFunction((Function) any, (Function.CompareMode) any); - result = new ScalarFunction(new FunctionName("hll_hash"), Lists.newArrayList(), Type.BIGINT, false); + result = new ScalarFunction(new FunctionName(FunctionSet.HLL_HASH), Lists.newArrayList(), Type.BIGINT, false); }}; new Expectations() { @@ -318,7 +319,7 @@ public void testHllColumnsNormal() throws UserException { TStreamLoadPutRequest request = getBaseRequest(); request.setFileType(TFileType.FILE_STREAM); - request.setColumns("k1,k2, v1=hll_hash(k2)"); + request.setColumns("k1,k2, v1=" + FunctionSet.HLL_HASH + "(k2)"); StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(request, null); StreamLoadScanNode scanNode = getStreamLoadScanNode(dstDesc, request); From f1571f81670225cee27bd745cc1001ca8f3e7536 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 8 Jul 2020 20:48:48 +0800 Subject: [PATCH 05/19] Support bitmap_union and hll_union for agg model by Create MV Stmt Change-Id: I897301af916b96377664df2d321aad3c4a2b24e1 --- .../analysis/CreateMaterializedViewStmt.java | 18 ++++++++--- .../analysis/MVColumnBitmapUnionPattern.java | 30 +++++++++-------- .../analysis/MVColumnHLLUnionPattern.java | 32 +++++++++++-------- .../apache/doris/analysis/MVColumnItem.java | 2 +- .../analysis/MVColumnOneChildPattern.java | 10 ++---- .../org/apache/doris/catalog/OlapTable.java | 9 ++++++ .../MVColumnBitmapUnionPatternTest.java | 24 +++++++++++++- .../analysis/MVColumnHLLUnionPatternTest.java | 29 +++++++++++++++-- .../planner/MaterializedViewFunctionTest.java | 17 ++++++++++ 9 files changed, 128 insertions(+), 43 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index eccccac0f1a05c..3f4d7ada270562 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -356,15 +356,25 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws type = Type.BIGINT; break; case FunctionSet.BITMAP_UNION: - mvColumnName = mvColumnBuilder(functionName, baseColumnName); + // Compatible aggregation models + if (baseColumnRef.getType() == Type.BITMAP) { + mvColumnName = baseColumnName; + } else { + mvColumnName = mvColumnBuilder(functionName, baseColumnName); + defineExpr = functionChild0; + } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); - defineExpr = functionChild0; type = Type.BITMAP; break; case FunctionSet.HLL_UNION: - mvColumnName = mvColumnBuilder(functionName, baseColumnName); + // Compatible aggregation models + if (baseColumnRef.getType() == Type.HLL) { + mvColumnName = baseColumnName; + } else { + mvColumnName = mvColumnBuilder(functionName, baseColumnName); + defineExpr = functionChild0; + } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); - defineExpr = functionChild0; type = Type.HLL; break; case FunctionSet.COUNT: diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java index db7822d1e24406..a95d05a0fc2669 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java @@ -18,8 +18,10 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.Type; public class MVColumnBitmapUnionPattern implements MVColumnPattern { + @Override public boolean match(Expr expr) { if (!(expr instanceof FunctionCallExpr)) { @@ -30,21 +32,23 @@ public boolean match(Expr expr) { if (!fnNameString.equalsIgnoreCase(FunctionSet.BITMAP_UNION)) { return false; } - if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { - return false; - } - FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); - if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) { - return false; - } - if (child0FnExpr.getChild(0) instanceof SlotRef) { - return true; - } else if (child0FnExpr.getChild(0) instanceof CastExpr) { - CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); - if (!(castExpr.getChild(0) instanceof SlotRef)) { + if (fnExpr.getChild(0) instanceof SlotRef) { + SlotRef slotRef = (SlotRef) fnExpr.getChild(0); + if (slotRef.getType() == Type.BITMAP && slotRef.getColumn() != null) { + return true; + } else { + return false; + } + } else if(fnExpr.getChild(0) instanceof FunctionCallExpr) { + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) { + return false; + } + if (child0FnExpr.getChild(0).unwrapSlotRef() == null) { return false; + } else { + return true; } - return true; } else { return false; } diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java index 51b1068e7171c4..ac58787a24c6b8 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java @@ -18,6 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.PrimitiveType; public class MVColumnHLLUnionPattern implements MVColumnPattern { @Override @@ -30,21 +31,23 @@ public boolean match(Expr expr) { if (!fnNameString.equalsIgnoreCase(FunctionSet.HLL_UNION)){ return false; } - if (!(fnExpr.getChild(0) instanceof FunctionCallExpr)) { - return false; - } - FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); - if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) { - return false; - } - if (child0FnExpr.getChild(0) instanceof SlotRef) { - return true; - } else if (child0FnExpr.getChild(0) instanceof CastExpr) { - CastExpr castExpr = (CastExpr) child0FnExpr.getChild(0); - if (!(castExpr.getChild(0) instanceof SlotRef)) { + if (fnExpr.getChild(0) instanceof SlotRef) { + SlotRef slotRef = (SlotRef) fnExpr.getChild(0); + if (slotRef.getType().getPrimitiveType() == PrimitiveType.HLL && slotRef.getColumn() != null) { + return true; + } else { + return false; + } + } else if(fnExpr.getChild(0) instanceof FunctionCallExpr) { + FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); + if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) { + return false; + } + if (child0FnExpr.getChild(0).unwrapSlotRef() == null) { return false; + } else { + return true; } - return true; } else { return false; } @@ -52,6 +55,7 @@ public boolean match(Expr expr) { @Override public String toString() { - return "hll_union(" + FunctionSet.HLL_HASH + "(column))"; + return FunctionSet.HLL_UNION + "(" + FunctionSet.HLL_HASH + "(column))" + + "or " + FunctionSet.HLL_UNION + "(hll_column) in agg table"; } } diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java index d797e6b0c7abba..55762cbbd1ab00 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java @@ -96,7 +96,7 @@ public void setDefineExpr(Expr defineExpr) { } public Column toMVColumn(OlapTable olapTable) throws DdlException { - Column baseColumn = olapTable.getColumn(name); + Column baseColumn = olapTable.getBaseColumn(name); if (baseColumn == null) { Preconditions.checkNotNull(defineExpr != null); Column result = new Column(name, type, isKey, aggregationType, ColumnDef.DefaultValue.ZERO, ""); diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java index 7b045a79033d75..ed50c91fed6d15 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnOneChildPattern.java @@ -38,15 +38,11 @@ public boolean match(Expr expr) { if (functionCallExpr.getChildren().size() != 1) { return false; } - Expr functionChild0 = functionCallExpr.getChild(0); - if (functionChild0 instanceof SlotRef) { - return true; - } else if (functionChild0 instanceof CastExpr && (functionChild0.getChild(0) instanceof SlotRef)) { - return true; - } else { + if (functionCallExpr.getChild(0).unwrapSlotRef() == null) { return false; + } else { + return true; } - } @Override diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 2611a15cbddcca..0d3d1a3b5a70b9 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1298,6 +1298,15 @@ public List getBaseSchema() { return getSchemaByIndexId(baseIndexId); } + public Column getBaseColumn(String columnName) { + for (Column column : getBaseSchema()) { + if (column.getName().equalsIgnoreCase(columnName)){ + return column; + } + } + return null; + } + public int getKeysNum() { int keysNum = 0; for (Column column : getBaseSchema()) { diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java index dbe6b5763eed54..f5998acb4a651f 100644 --- a/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnBitmapUnionPatternTest.java @@ -17,7 +17,9 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.Type; import com.google.common.collect.Lists; @@ -51,7 +53,7 @@ public void testCorrectExpr2(@Injectable CastExpr castExpr) { SlotRef slotRef = new SlotRef(tableName, "c1"); new Expectations() { { - castExpr.getChild(0); + castExpr.unwrapSlotRef(); result = slotRef; } }; @@ -107,4 +109,24 @@ public void testIncorrectArithmeticExpr2() { MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); Assert.assertFalse(pattern.match(expr)); } + + @Test + public void testAggTableBitmapColumn(@Injectable SlotDescriptor desc, + @Injectable Column column) { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + List params = Lists.newArrayList(); + params.add(slotRef1); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.BITMAP_UNION, params); + slotRef1.setType(Type.BITMAP); + slotRef1.setDesc(desc); + new Expectations() { + { + desc.getColumn(); + result = column; + } + }; + MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } } diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java index e98ab4b10ee7f9..7625ce6c75b8f0 100644 --- a/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnHLLUnionPatternTest.java @@ -17,7 +17,9 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.Type; import com.google.common.collect.Lists; @@ -51,7 +53,7 @@ public void testCorrectExpr2(@Injectable CastExpr castExpr) { SlotRef slotRef = new SlotRef(tableName, "c1"); new Expectations() { { - castExpr.getChild(0); + castExpr.unwrapSlotRef(); result = slotRef; } }; @@ -85,7 +87,7 @@ public void testIncorrectLiteralExpr1() { List params = Lists.newArrayList(); params.add(intLiteral); FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); - MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); Assert.assertFalse(pattern.match(expr)); } @@ -98,7 +100,28 @@ public void testIncorrectLiteralExpr2() { List params = Lists.newArrayList(); params.add(child0); FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); - MVColumnBitmapUnionPattern pattern = new MVColumnBitmapUnionPattern(); + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); Assert.assertFalse(pattern.match(expr)); } + + @Test + public void testAggTableHLLColumn(@Injectable SlotDescriptor desc, + @Injectable Column column) { + TableName tableName = new TableName("db", "table"); + SlotRef slotRef1 = new SlotRef(tableName, "c1"); + List params = Lists.newArrayList(); + params.add(slotRef1); + FunctionCallExpr expr = new FunctionCallExpr(FunctionSet.HLL_UNION, params); + slotRef1.setType(Type.HLL); + slotRef1.setDesc(desc); + new Expectations() { + { + desc.getColumn(); + result = column; + } + }; + MVColumnHLLUnionPattern pattern = new MVColumnHLLUnionPattern(); + Assert.assertTrue(pattern.match(expr)); + } + } diff --git a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java index 6d30b324d7df8a..49ec316ae13989 100644 --- a/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java +++ b/fe/src/test/java/org/apache/doris/planner/MaterializedViewFunctionTest.java @@ -807,4 +807,21 @@ public void testInvalidColumnInCreateMVStmt() throws Exception { } } + @Test + public void testCreateMVBaseBitmapAggTable() throws Exception { + String createTableSQL = "create table " + HR_DB_NAME + ".agg_table " + + "(empid int, name varchar, salary bitmap " + FunctionSet.BITMAP_UNION + ") " + + "aggregate key (empid, name) " + + "partition by range (empid) " + + "(partition p1 values less than MAXVALUE) " + + "distributed by hash(empid) buckets 3 properties('replication_num' = '1');"; + dorisAssert.withTable(createTableSQL); + String createMVSQL = "create materialized view mv as select empid, " + FunctionSet.BITMAP_UNION + + "(salary) from agg_table " + + "group by empid;"; + dorisAssert.withMaterializedView(createMVSQL); + String query = "select count(distinct salary) from agg_table;"; + dorisAssert.query(query).explainContains("mv"); + dorisAssert.dropTable("agg_table"); + } } From 65196a3e808e32dd4a6234f868e7259b7c4c3876 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Thu, 9 Jul 2020 12:30:31 +0800 Subject: [PATCH 06/19] Fix replay error Change-Id: Ibf277124144e012773de310a17cb5cd57cdc76de --- .../org/apache/doris/alter/RollupJobV2.java | 13 ++-- .../apache/doris/analysis/CaseWhenClause.java | 2 +- .../analysis/CreateMaterializedViewStmt.java | 61 +++++++++++++++++-- .../analysis/MVColumnBitmapUnionPattern.java | 6 +- .../analysis/MVColumnHLLUnionPattern.java | 2 +- .../org/apache/doris/analysis/SelectList.java | 2 +- .../apache/doris/analysis/SelectListItem.java | 2 +- .../doris/catalog/MaterializedIndexMeta.java | 15 ++--- 8 files changed, 76 insertions(+), 27 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java index 18ff621bb4d13e..4ee587029529a3 100644 --- a/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.MVColumnItem; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.catalog.Catalog; @@ -732,11 +731,11 @@ public void setJobState(JobState jobState) { this.jobState = jobState; } - private void setColumnsDefineExpr(List items) { - for (MVColumnItem item : items) { + private void setColumnsDefineExpr(Map columnNameToDefineExpr) { + for (Entry entry : columnNameToDefineExpr.entrySet()) { for (Column column : rollupSchema) { - if (column.getName().equals(item.getName())) { - column.setDefineExpr(item.getDefineExpr()); + if (column.getName().equals(entry.getKey())) { + column.setDefineExpr(entry.getValue()); break; } } @@ -812,8 +811,8 @@ public void gsonPostProcess() throws IOException { CreateMaterializedViewStmt stmt; try { stmt = (CreateMaterializedViewStmt) SqlParserUtils.getStmt(parser, origStmt.idx); - stmt.analyzeSelectClause(); - setColumnsDefineExpr(stmt.getMVColumnItemList()); + Map columnNameToDefineExpr = stmt.parseDefineExprWithoutAnalyze(); + setColumnsDefineExpr(columnNameToDefineExpr); } catch (Exception e) { throw new IOException("error happens when parsing create materialized view stmt: " + origStmt, e); } diff --git a/fe/src/main/java/org/apache/doris/analysis/CaseWhenClause.java b/fe/src/main/java/org/apache/doris/analysis/CaseWhenClause.java index 3a68ce296f4485..c07bd8d79e3bb7 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CaseWhenClause.java +++ b/fe/src/main/java/org/apache/doris/analysis/CaseWhenClause.java @@ -21,7 +21,7 @@ /** * captures info of a single WHEN expr THEN expr clause. */ -class CaseWhenClause { +public class CaseWhenClause { private final Expr whenExpr; private final Expr thenExpr; diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 3f4d7ada270562..7843cacc8899aa 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -98,6 +98,10 @@ public String getMVName() { return mvName; } + public SelectStmt getSelectStmt() { + return selectStmt; + } + public List getMVColumnItemList() { return mvColumnItemList; } @@ -172,11 +176,11 @@ public void analyzeSelectClause() throws AnalysisException { throw new AnalysisException("The materialized view only support the single column or function expr. " + "Error column: " + selectListItemExpr.toSql()); } - if (selectListItem.getExpr() instanceof SlotRef) { + if (selectListItemExpr instanceof SlotRef) { if (meetAggregate) { throw new AnalysisException("The aggregate column should be after the single column"); } - SlotRef slotRef = (SlotRef) selectListItem.getExpr(); + SlotRef slotRef = (SlotRef) selectListItemExpr; // check duplicate column String columnName = slotRef.getColumnName().toLowerCase(); if (!mvColumnNameSet.add(columnName)) { @@ -184,9 +188,9 @@ public void analyzeSelectClause() throws AnalysisException { } MVColumnItem mvColumnItem = new MVColumnItem(columnName, slotRef.getType()); mvColumnItemList.add(mvColumnItem); - } else if (selectListItem.getExpr() instanceof FunctionCallExpr) { + } else if (selectListItemExpr instanceof FunctionCallExpr) { // Function must match pattern. - FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItem.getExpr(); + FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItemExpr; String functionName = functionCallExpr.getFnName().getFunction(); MVColumnPattern mvColumnPattern = FN_NAME_TO_PATTERN.get(functionName.toLowerCase()); if (mvColumnPattern == null) { @@ -357,7 +361,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws break; case FunctionSet.BITMAP_UNION: // Compatible aggregation models - if (baseColumnRef.getType() == Type.BITMAP) { + if (baseColumnRef.getType().getPrimitiveType() == PrimitiveType.BITMAP) { mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); @@ -368,7 +372,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws break; case FunctionSet.HLL_UNION: // Compatible aggregation models - if (baseColumnRef.getType() == Type.HLL) { + if (baseColumnRef.getType().getPrimitiveType() == PrimitiveType.HLL) { mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); @@ -392,6 +396,51 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws return mvColumnItem; } + public Map parseDefineExprWithoutAnalyze() throws AnalysisException { + Map result = Maps.newHashMap(); + SelectList selectList = selectStmt.getSelectList(); + for (SelectListItem selectListItem : selectList.getItems()) { + Expr selectListItemExpr = selectListItem.getExpr(); + if (selectListItemExpr instanceof SlotRef) { + SlotRef slotRef = (SlotRef) selectListItemExpr; + result.put(slotRef.getColumnName(), null); + } else if (selectListItemExpr instanceof FunctionCallExpr) { + FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItemExpr; + List slots = new ArrayList<>(); + functionCallExpr.collect(SlotRef.class, slots); + Preconditions.checkArgument(slots.size() == 1); + String baseColumnName = slots.get(0).getColumnName(); + String functionName = functionCallExpr.getFnName().getFunction(); + switch (functionName.toLowerCase()) { + case "sum": + case "min": + case "max": + result.put(baseColumnName, null); + break; + case FunctionSet.BITMAP_UNION: + case FunctionSet.HLL_UNION: + if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { + result.put(mvColumnBuilder(functionName, baseColumnName), functionCallExpr.getChild(0)); + } else { + result.put(baseColumnName, null); + } + break; + case FunctionSet.COUNT: + Expr defineExpr = new CaseExpr(null, Lists.newArrayList( + new CaseWhenClause(new IsNullPredicate(slots.get(0), false), + new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT)); + result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr); + break; + default: + throw new AnalysisException("Unsupported function:" + functionName); + } + } else { + throw new AnalysisException("Unsupported select item:" + selectListItem.toSql()); + } + } + return result; + } + public static String mvColumnBuilder(String functionName, String sourceColumnName) { return new StringBuilder().append(MATERIALIZED_VIEW_NAME_PREFIX).append(functionName).append("_") .append(sourceColumnName).toString(); diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java index a95d05a0fc2669..e2a26102a9c07c 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnBitmapUnionPattern.java @@ -18,7 +18,7 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.FunctionSet; -import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.PrimitiveType; public class MVColumnBitmapUnionPattern implements MVColumnPattern { @@ -34,12 +34,12 @@ public boolean match(Expr expr) { } if (fnExpr.getChild(0) instanceof SlotRef) { SlotRef slotRef = (SlotRef) fnExpr.getChild(0); - if (slotRef.getType() == Type.BITMAP && slotRef.getColumn() != null) { + if (slotRef.getType().getPrimitiveType() == PrimitiveType.BITMAP && slotRef.getColumn() != null) { return true; } else { return false; } - } else if(fnExpr.getChild(0) instanceof FunctionCallExpr) { + } else if (fnExpr.getChild(0) instanceof FunctionCallExpr) { FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.TO_BITMAP)) { return false; diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java index ac58787a24c6b8..11eae282cdf542 100644 --- a/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java +++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnHLLUnionPattern.java @@ -38,7 +38,7 @@ public boolean match(Expr expr) { } else { return false; } - } else if(fnExpr.getChild(0) instanceof FunctionCallExpr) { + } else if (fnExpr.getChild(0) instanceof FunctionCallExpr) { FunctionCallExpr child0FnExpr = (FunctionCallExpr) fnExpr.getChild(0); if (!child0FnExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.HLL_HASH)) { return false; diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectList.java b/fe/src/main/java/org/apache/doris/analysis/SelectList.java index 3f3996d98454d9..ab1c859a573f6a 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectList.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectList.java @@ -28,7 +28,7 @@ /** * Select list items plus distinct clause. */ -class SelectList { +public class SelectList { private boolean isDistinct; // /////////////////////////////////////// diff --git a/fe/src/main/java/org/apache/doris/analysis/SelectListItem.java b/fe/src/main/java/org/apache/doris/analysis/SelectListItem.java index f1e4c36f935cea..c55bd7c9772ec5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/SelectListItem.java +++ b/fe/src/main/java/org/apache/doris/analysis/SelectListItem.java @@ -19,7 +19,7 @@ import com.google.common.base.Preconditions; -class SelectListItem { +public class SelectListItem { private Expr expr; // for "[name.]*" private final TableName tblName; diff --git a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 2e224f9c2099c8..3499f18b7504e6 100644 --- a/fe/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -18,7 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.analysis.CreateMaterializedViewStmt; -import org.apache.doris.analysis.MVColumnItem; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.common.io.Text; @@ -39,6 +39,7 @@ import java.io.IOException; import java.io.StringReader; import java.util.List; +import java.util.Map; public class MaterializedIndexMeta implements Writable, GsonPostProcessable { @SerializedName(value = "indexId") @@ -106,11 +107,11 @@ public int getSchemaVersion() { return schemaVersion; } - private void setColumnsDefineExpr(List items) { - for (MVColumnItem item : items) { + private void setColumnsDefineExpr(Map columnNameToDefineExpr) { + for (Map.Entry entry : columnNameToDefineExpr.entrySet()) { for (Column column : schema) { - if (column.getName().equals(item.getName())) { - column.setDefineExpr(item.getDefineExpr()); + if (column.getName().equals(entry.getKey())) { + column.setDefineExpr(entry.getValue()); break; } } @@ -169,8 +170,8 @@ public void gsonPostProcess() throws IOException { CreateMaterializedViewStmt stmt; try { stmt = (CreateMaterializedViewStmt) SqlParserUtils.getStmt(parser, defineStmt.idx); - stmt.analyzeSelectClause(); - setColumnsDefineExpr(stmt.getMVColumnItemList()); + Map columnNameToDefineExpr = stmt.parseDefineExprWithoutAnalyze(); + setColumnsDefineExpr(columnNameToDefineExpr); } catch (Exception e) { throw new IOException("error happens when parsing create materialized view stmt: " + defineStmt, e); } From 3a448729e159dd61fc8bca16db8c41372a46be04 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Thu, 9 Jul 2020 15:13:32 +0800 Subject: [PATCH 07/19] Change display column name Change-Id: I8feab7ac56aa17412bbf45b15b41754f3f3b7410 --- .../main/java/org/apache/doris/analysis/DescribeStmt.java | 5 +---- fe/src/main/java/org/apache/doris/catalog/Column.java | 8 ++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java index bf7cfaccfb099c..c41c7fa86022b5 100644 --- a/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -164,14 +164,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (bfColumns != null && bfColumns.contains(column.getName())) { extras.add("BLOOM_FILTER"); } - if (column.getDefineExpr() != null) { - extras.add(column.getDefineExpr().toSql().toUpperCase()); - } String extraStr = StringUtils.join(extras, ","); List row = Arrays.asList("", "", - column.getName(), + column.getDisplayName(), column.getOriginType().toString(), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), diff --git a/fe/src/main/java/org/apache/doris/catalog/Column.java b/fe/src/main/java/org/apache/doris/catalog/Column.java index 634757939a695a..63aa2c51779f73 100644 --- a/fe/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/src/main/java/org/apache/doris/catalog/Column.java @@ -141,6 +141,14 @@ public String getName() { return this.name; } + public String getDisplayName() { + if (defineExpr == null) { + return name; + } else { + return defineExpr.toSql(); + } + } + public String getNameWithoutPrefix(String prefix) { if (isNameWithPrefix(prefix)) { return name.substring(prefix.length()); From 607aff3551a732729db97135a5befe3be097faab Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Thu, 9 Jul 2020 15:32:43 +0800 Subject: [PATCH 08/19] Remove mv column when drop index Change-Id: I73bd860091b14202acb731a0eab460d992237f03 --- fe/src/main/java/org/apache/doris/catalog/OlapTable.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 0d3d1a3b5a70b9..641fd3faed4bea 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -307,6 +307,8 @@ public boolean deleteIndexInfo(String indexName) { long indexId = this.indexNameToId.remove(indexName); this.indexIdToMeta.remove(indexId); + // Some column of deleted index should be removed during `deleteIndexInfo` such as `mv_bitmap_union_c1` + rebuildFullSchema(); return true; } From 5d424d2a02e07aaef2d54929ac1b5385945219dc Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Fri, 10 Jul 2020 16:08:43 +0800 Subject: [PATCH 09/19] Reset the define expr Change-Id: Ib8ab5fa56b1caec597937f6bf3defa4acfe8aab1 --- .../org/apache/doris/analysis/CreateMaterializedViewStmt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 7843cacc8899aa..39dcc075b13866 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -365,7 +365,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); - defineExpr = functionChild0; + defineExpr = functionChild0.reset(); } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); type = Type.BITMAP; @@ -376,7 +376,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); - defineExpr = functionChild0; + defineExpr = functionChild0.reset(); } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); type = Type.HLL; From d6c885d45df0e2ad1d25c0f1de1cfcbe3bd1e621 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Mon, 13 Jul 2020 10:45:52 +0800 Subject: [PATCH 10/19] Analyze define expr when replay log Change-Id: If3e553ebb29e59b0e35ed4423094ad8caaa710ac --- .../analysis/CreateMaterializedViewStmt.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 39dcc075b13866..29bf729990a6f3 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -365,7 +365,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); - defineExpr = functionChild0.reset(); + defineExpr = functionChild0; } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); type = Type.BITMAP; @@ -376,7 +376,7 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws mvColumnName = baseColumnName; } else { mvColumnName = mvColumnBuilder(functionName, baseColumnName); - defineExpr = functionChild0.reset(); + defineExpr = functionChild0; } mvAggregateType = AggregateType.valueOf(functionName.toUpperCase()); type = Type.HLL; @@ -411,6 +411,7 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio Preconditions.checkArgument(slots.size() == 1); String baseColumnName = slots.get(0).getColumnName(); String functionName = functionCallExpr.getFnName().getFunction(); + SlotRef baseSlotRef = slots.get(0); switch (functionName.toLowerCase()) { case "sum": case "min": @@ -418,9 +419,23 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio result.put(baseColumnName, null); break; case FunctionSet.BITMAP_UNION: + if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { + CastExpr castExpr = new CastExpr(Type.VARCHAR, baseSlotRef); + List params = Lists.newArrayList(); + params.add(castExpr); + FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP, params); + result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr); + } else { + result.put(baseColumnName, null); + } + break; case FunctionSet.HLL_UNION: if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { - result.put(mvColumnBuilder(functionName, baseColumnName), functionCallExpr.getChild(0)); + CastExpr castExpr = new CastExpr(Type.VARCHAR, baseSlotRef); + List params = Lists.newArrayList(); + params.add(castExpr); + FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.HLL_HASH, params); + result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr); } else { result.put(baseColumnName, null); } From 4bd80e3197b5af042b34b75d3c0df0e0acb23716 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Tue, 14 Jul 2020 11:30:44 +0800 Subject: [PATCH 11/19] Add cast when replay log Change-Id: Id867098ded33667a0f80f67daf88ee068a1973a0 --- .../main/java/org/apache/doris/analysis/CastExpr.java | 10 +++++++--- .../doris/analysis/CreateMaterializedViewStmt.java | 4 ++-- .../test/java/org/apache/doris/utframe/DemoTest.java | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java index bcf65c0ed4a255..de5dd9a50bdb0f 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -137,10 +137,14 @@ public String toSqlImpl() { if (isImplicit) { return getChild(0).toSql(); } - if (type.isStringType()) { - return "CAST(" + getChild(0).toSql() + " AS " + "CHARACTER" + ")"; + if (isAnalyzed) { + if (type.isStringType()) { + return "CAST(" + getChild(0).toSql() + " AS " + "CHARACTER" + ")"; + } else { + return "CAST(" + getChild(0).toSql() + " AS " + type.toString() + ")"; + } } else { - return "CAST(" + getChild(0).toSql() + " AS " + type.toString() + ")"; + return "CAST(" + getChild(0).toSql() + " AS " + targetTypeDef.toSql() + ")"; } } diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 29bf729990a6f3..26c7229d2e2230 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -420,7 +420,7 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio break; case FunctionSet.BITMAP_UNION: if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { - CastExpr castExpr = new CastExpr(Type.VARCHAR, baseSlotRef); + CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef); List params = Lists.newArrayList(); params.add(castExpr); FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP, params); @@ -431,7 +431,7 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio break; case FunctionSet.HLL_UNION: if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { - CastExpr castExpr = new CastExpr(Type.VARCHAR, baseSlotRef); + CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef); List params = Lists.newArrayList(); params.add(castExpr); FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.HLL_HASH, params); diff --git a/fe/src/test/java/org/apache/doris/utframe/DemoTest.java b/fe/src/test/java/org/apache/doris/utframe/DemoTest.java index 499f9594b3044b..d2b8fc05c1b7af 100644 --- a/fe/src/test/java/org/apache/doris/utframe/DemoTest.java +++ b/fe/src/test/java/org/apache/doris/utframe/DemoTest.java @@ -65,6 +65,7 @@ public class DemoTest { public static void beforeClass() throws EnvVarNotSetException, IOException, FeStartException, NotInitException, DdlException, InterruptedException { FeConstants.default_scheduler_interval_millisecond = 10; + FeConstants.runningUnitTest = true; UtFrameUtils.createMinDorisCluster(runningDir); } From 39d38542ae2001ddecacd3a64b5ea78cc83402b1 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Tue, 14 Jul 2020 14:24:12 +0800 Subject: [PATCH 12/19] Change display name Change-Id: I3d5481fab5a09bf5f43b606d1fe0bda41eb2e17b --- fe/src/main/java/org/apache/doris/analysis/CastExpr.java | 6 +++++- .../apache/doris/analysis/CreateMaterializedViewStmt.java | 2 ++ fe/src/main/java/org/apache/doris/catalog/OlapTable.java | 7 ++++++- fe/src/test/java/org/apache/doris/utframe/DemoTest.java | 1 - 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java index de5dd9a50bdb0f..89b5138619a790 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -43,7 +43,7 @@ public class CastExpr extends Expr { private final TypeDef targetTypeDef; // True if this is a "pre-analyzed" implicit cast. - private final boolean isImplicit; + private boolean isImplicit; // True if this cast does not change the type. private boolean noOp = false; @@ -171,6 +171,10 @@ public boolean isImplicit() { return isImplicit; } + public void setImplicit(boolean implicit) { + isImplicit = implicit; + } + public void analyze() throws AnalysisException { // cast was asked for in the query, check for validity of cast Type childType = getChild(0).getType(); diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 26c7229d2e2230..35a00e7cd2d139 100644 --- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -421,6 +421,7 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio case FunctionSet.BITMAP_UNION: if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef); + castExpr.setImplicit(true); List params = Lists.newArrayList(); params.add(castExpr); FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.TO_BITMAP, params); @@ -432,6 +433,7 @@ public Map parseDefineExprWithoutAnalyze() throws AnalysisExceptio case FunctionSet.HLL_UNION: if (functionCallExpr.getChild(0) instanceof FunctionCallExpr) { CastExpr castExpr = new CastExpr(new TypeDef(Type.VARCHAR), baseSlotRef); + castExpr.setImplicit(true); List params = Lists.newArrayList(); params.add(castExpr); FunctionCallExpr defineExpr = new FunctionCallExpr(FunctionSet.HLL_HASH, params); diff --git a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java index 641fd3faed4bea..f8f442337ee4cf 100644 --- a/fe/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -308,7 +308,12 @@ public boolean deleteIndexInfo(String indexName) { long indexId = this.indexNameToId.remove(indexName); this.indexIdToMeta.remove(indexId); // Some column of deleted index should be removed during `deleteIndexInfo` such as `mv_bitmap_union_c1` - rebuildFullSchema(); + // If deleted index id == base index id, the schema will not be rebuilt. + // The reason is that the base index has been removed from indexIdToMeta while the new base index hasn't changed. + // The schema could not be rebuild in here with error base index id. + if (indexId != baseIndexId) { + rebuildFullSchema(); + } return true; } diff --git a/fe/src/test/java/org/apache/doris/utframe/DemoTest.java b/fe/src/test/java/org/apache/doris/utframe/DemoTest.java index d2b8fc05c1b7af..499f9594b3044b 100644 --- a/fe/src/test/java/org/apache/doris/utframe/DemoTest.java +++ b/fe/src/test/java/org/apache/doris/utframe/DemoTest.java @@ -65,7 +65,6 @@ public class DemoTest { public static void beforeClass() throws EnvVarNotSetException, IOException, FeStartException, NotInitException, DdlException, InterruptedException { FeConstants.default_scheduler_interval_millisecond = 10; - FeConstants.runningUnitTest = true; UtFrameUtils.createMinDorisCluster(runningDir); } From 0337c0c38b430676fe794426384d185885ca94a1 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Tue, 14 Jul 2020 20:08:03 +0800 Subject: [PATCH 13/19] Change the finalize Change-Id: I38273603ae21245b38e0aa5dd2b266aceab06575 --- .../java/org/apache/doris/load/ExportJob.java | 2 + .../apache/doris/planner/OlapScanNode.java | 38 +++++++++++-------- .../org/apache/doris/planner/Planner.java | 2 +- .../doris/planner/SingleNodePlanner.java | 4 ++ .../alter/MaterializedViewHandlerTest.java | 2 +- .../apache/doris/alter/RollupJobV2Test.java | 16 ++++---- .../CreateMaterializedViewStmtTest.java | 2 +- .../analysis/MVColumnOneChildPatternTest.java | 4 +- .../catalog/MaterializedIndexMetaTest.java | 19 +++++----- 9 files changed, 50 insertions(+), 39 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/load/ExportJob.java b/fe/src/main/java/org/apache/doris/load/ExportJob.java index 71fd29a7701a82..6589f629ed050e 100644 --- a/fe/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/src/main/java/org/apache/doris/load/ExportJob.java @@ -281,6 +281,8 @@ private ScanNode genScanNode() throws UserException { ((OlapScanNode) scanNode).setColumnFilters(Maps.newHashMap()); ((OlapScanNode) scanNode).setIsPreAggregation(false, "This an export operation"); ((OlapScanNode) scanNode).setCanTurnOnPreAggr(false); + scanNode.init(analyzer); + ((OlapScanNode) scanNode).selectBestRollupByRollupSelector(analyzer); break; case MYSQL: scanNode = new MysqlScanNode(new PlanNodeId(0), exportTupleDesc, (MysqlTable) this.exportTable); diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 26b5b286280822..f4e87d35c4c6d1 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -229,7 +229,7 @@ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isP this.reasonOfPreAggregation = reasonOfDisable; long start = System.currentTimeMillis(); computeTabletInfo(); - computeStats(analyzer); +// computeStats(analyzer); LOG.debug("distribution prune cost: {} ms", (System.currentTimeMillis() - start)); LOG.info("Using the new scan range info instead of the old one. {}, {}", situation ,scanRangeInfo); } else { @@ -249,6 +249,12 @@ protected String debugString() { return helper.toString(); } + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + computePartitionInfo(); + } + @Override public void finalize(Analyzer analyzer) throws UserException { if (isFinalized) { @@ -257,7 +263,7 @@ public void finalize(Analyzer analyzer) throws UserException { LOG.debug("OlapScanNode finalize. Tuple: {}", desc); try { - getScanRangeLocations(analyzer); + getScanRangeLocations(); } catch (AnalysisException e) { throw new UserException(e.getMessage()); } @@ -412,7 +418,7 @@ private void addScanRangeLocations(Partition partition, } } - private void getScanRangeLocations(Analyzer analyzer) throws UserException { + private void computePartitionInfo() throws AnalysisException { long start = System.currentTimeMillis(); // Step1: compute partition ids PartitionNames partitionNames = ((BaseTableRef) desc.getRef()).getPartitionNames(); @@ -438,28 +444,28 @@ private void getScanRangeLocations(Analyzer analyzer) throws UserException { } selectedPartitionNum = selectedPartitionIds.size(); LOG.debug("partition prune cost: {} ms, partitions: {}", - (System.currentTimeMillis() - start), selectedPartitionIds); - if (selectedPartitionIds.size() == 0) { - return; - } + (System.currentTimeMillis() - start), selectedPartitionIds); + } + public void selectBestRollupByRollupSelector(Analyzer analyzer) throws UserException { // Step2: select best rollup - start = System.currentTimeMillis(); + long start = System.currentTimeMillis(); if (olapTable.getKeysType() == KeysType.DUP_KEYS) { isPreAggregation = true; } - // TODO: Remove the logic of old selector. - if (olapTable.getKeysType() == KeysType.DUP_KEYS) { - LOG.debug("The best index will be selected later in mv selector"); - return; - } final RollupSelector rollupSelector = new RollupSelector(analyzer, desc, olapTable); selectedIndexId = rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts, isPreAggregation); LOG.debug("select best roll up cost: {} ms, best index id: {}", - (System.currentTimeMillis() - start), selectedIndexId); + (System.currentTimeMillis() - start), selectedIndexId); + } - // Step3: compute tablet info by selected index id and selected partition ids - start = System.currentTimeMillis(); + private void getScanRangeLocations() throws UserException { + if (selectedPartitionIds.size() == 0) { + return; + } + Preconditions.checkState(selectedIndexId != -1); + // compute tablet info by selected index id and selected partition ids + long start = System.currentTimeMillis(); computeTabletInfo(); LOG.debug("distribution prune cost: {} ms", (System.currentTimeMillis() - start)); } diff --git a/fe/src/main/java/org/apache/doris/planner/Planner.java b/fe/src/main/java/org/apache/doris/planner/Planner.java index a88de531cd681b..cc70e5d46cb3e8 100644 --- a/fe/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/src/main/java/org/apache/doris/planner/Planner.java @@ -161,12 +161,12 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue // compute mem layout *before* finalize(); finalize() may reference // TupleDescriptor.avgSerializedSize analyzer.getDescTbl().computeMemLayout(); - singleNodePlan.finalize(analyzer); // materialized view selector boolean selectFailed = singleNodePlanner.selectMaterializedView(queryStmt, analyzer); if (selectFailed) { throw new MVSelectFailedException("Failed to select materialize view"); } + singleNodePlan.finalize(analyzer); if (queryOptions.num_nodes == 1) { // single-node execution; we're almost done singleNodePlan = addUnassignedConjuncts(analyzer, singleNodePlan); diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 11b102f781b438..a9e88bed99a24e 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -776,6 +776,9 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) if (olapScanNode.getSelectedPartitionIds().size() == 0 && !FeConstants.runningUnitTest) { continue; } + // select index by the old Rollup selector + olapScanNode.selectBestRollupByRollupSelector(analyzer); + // select index by the new Materialized selector MaterializedViewSelector.BestIndexInfo bestIndexInfo = materializedViewSelector.selectBestMV(olapScanNode); if (bestIndexInfo == null) { selectFailed |= true; @@ -784,6 +787,7 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) LOG.debug("MV rewriter of tuple [] will be disable", tupleId); continue; } + // if the new selected index id is different from the old one, scan node will be updated. olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable(), analyzer); } diff --git a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java index de315e49ea34a8..d1e55362626a86 100644 --- a/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java +++ b/fe/src/test/java/org/apache/doris/alter/MaterializedViewHandlerTest.java @@ -222,7 +222,7 @@ public void testDuplicateTable(@Injectable CreateMaterializedViewStmt createMate result = mvName; createMaterializedViewStmt.getMVColumnItemList(); result = Lists.newArrayList(mvColumnItem); - olapTable.getColumn(columnName1); + olapTable.getBaseColumn(columnName1); result = baseColumn1; olapTable.getKeysType(); result = KeysType.DUP_KEYS; diff --git a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java index 1232cc7bb43f91..6fb4d06ac2d62c 100644 --- a/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java +++ b/fe/src/test/java/org/apache/doris/alter/RollupJobV2Test.java @@ -28,7 +28,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; -import org.apache.doris.analysis.MVColumnItem; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; @@ -63,6 +62,7 @@ import org.apache.doris.transaction.GlobalTransactionMgr; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.junit.After; import org.junit.Assert; @@ -369,7 +369,8 @@ public void testSchemaChangeWhileTabletNotStable() throws Exception { @Test - public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) throws IOException { + public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) throws IOException, + AnalysisException { // prepare file File file = new File(fileName); file.createNewFile(); @@ -391,18 +392,15 @@ public void testSerializeOfRollupJob(@Mocked CreateMaterializedViewStmt stmt) th out.flush(); out.close(); - List itemList = Lists.newArrayList(); - MVColumnItem item = new MVColumnItem( - mvColumnName, Type.BITMAP); List params = Lists.newArrayList(); SlotRef param1 = new SlotRef(new TableName(null, "test"), "c1"); params.add(param1); - item.setDefineExpr(new FunctionCallExpr(new FunctionName("to_bitmap"), params)); - itemList.add(item); + Map columnNameToDefineExpr = Maps.newHashMap(); + columnNameToDefineExpr.put(mvColumnName, new FunctionCallExpr(new FunctionName("to_bitmap"), params)); new Expectations() { { - stmt.getMVColumnItemList(); - result = itemList; + stmt.parseDefineExprWithoutAnalyze(); + result = columnNameToDefineExpr; } }; diff --git a/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java index 61ff906977a3db..da42668fefa06e 100644 --- a/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java @@ -100,7 +100,7 @@ public void testAggregateWithFunctionColumnInSelectClause(@Injectable Arithmetic try { createMaterializedViewStmt.analyze(analyzer); Assert.fail(); - } catch (UserException e) { + } catch (IllegalArgumentException e) { System.out.print(e.getMessage()); } } diff --git a/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java b/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java index 976483ade999fb..dfd47e8861f5e0 100644 --- a/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/MVColumnOneChildPatternTest.java @@ -52,8 +52,8 @@ public void testCorrectMin(@Injectable CastExpr castExpr) { child0Params.add(slotRef); new Expectations() { { - castExpr.getChild(0); - result = child0Params; + castExpr.unwrapSlotRef(); + result = slotRef; } }; List params = Lists.newArrayList(); diff --git a/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java b/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java index 017993c638510a..c3841ec81f2a8f 100644 --- a/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java +++ b/fe/src/test/java/org/apache/doris/catalog/MaterializedIndexMetaTest.java @@ -21,13 +21,14 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.FunctionName; -import org.apache.doris.analysis.MVColumnItem; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; +import org.apache.doris.common.AnalysisException; import org.apache.doris.qe.OriginStatement; import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.junit.After; import org.junit.Assert; @@ -40,6 +41,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.List; +import java.util.Map; import mockit.Expectations; import mockit.Mocked; @@ -55,13 +57,14 @@ public void tearDown() { } @Test - public void testSerializeMaterializedIndexMeta(@Mocked CreateMaterializedViewStmt stmt) throws IOException { + public void testSerializeMaterializedIndexMeta(@Mocked CreateMaterializedViewStmt stmt) + throws IOException, AnalysisException { // 1. Write objects to file File file = new File(fileName); file.createNewFile(); DataOutputStream out = new DataOutputStream(new FileOutputStream(file)); - String mvColumnName = CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX + "to_bitmap_" + "k1"; + String mvColumnName = CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX + FunctionSet.BITMAP_UNION + "_" + "k1"; List schema = Lists.newArrayList(); schema.add(new Column("k1", Type.TINYINT, true, null, true, "1", "abc")); schema.add(new Column("k2", Type.SMALLINT, true, null, true, "1", "debug")); @@ -88,17 +91,15 @@ TStorageType.COLUMN, KeysType.DUP_KEYS, new OriginStatement( out.flush(); out.close(); - List itemList = Lists.newArrayList(); - MVColumnItem item = new MVColumnItem(mvColumnName, Type.BITMAP); List params = Lists.newArrayList(); SlotRef param1 = new SlotRef(new TableName(null, "test"), "c1"); params.add(param1); - item.setDefineExpr(new FunctionCallExpr(new FunctionName("to_bitmap"), params)); - itemList.add(item); + Map columnNameToDefineExpr = Maps.newHashMap(); + columnNameToDefineExpr.put(mvColumnName, new FunctionCallExpr(new FunctionName("to_bitmap"), params)); new Expectations() { { - stmt.getMVColumnItemList(); - result = itemList; + stmt.parseDefineExprWithoutAnalyze(); + result = columnNameToDefineExpr; } }; From 68746a23f51cc7f06f0d28849ac5537438600bbd Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 15 Jul 2020 12:03:04 +0800 Subject: [PATCH 14/19] Remove duplicate compute tablet Change-Id: Id42b9102585901e647d56601e61c0a5d53a7208c --- fe/src/main/java/org/apache/doris/planner/OlapScanNode.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index f4e87d35c4c6d1..93dc045c21e030 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -228,8 +228,6 @@ public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isP this.isPreAggregation = isPreAggregation; this.reasonOfPreAggregation = reasonOfDisable; long start = System.currentTimeMillis(); - computeTabletInfo(); -// computeStats(analyzer); LOG.debug("distribution prune cost: {} ms", (System.currentTimeMillis() - start)); LOG.info("Using the new scan range info instead of the old one. {}, {}", situation ,scanRangeInfo); } else { From fe359d97d7bb2f7abc3b8589b51a21f9663488fd Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 15 Jul 2020 15:16:20 +0800 Subject: [PATCH 15/19] Remove duplicate tablet Change-Id: I6e16b29f8cdcd8fa4772d235fbeddae6b7ec9511 --- .../apache/doris/planner/OlapScanNode.java | 26 +++---------------- .../doris/planner/SingleNodePlanner.java | 2 +- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 93dc045c21e030..b622a469759d93 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -330,11 +330,6 @@ private void addScanRangeLocations(Partition partition, MaterializedIndex index, List tablets, long localBeId) throws UserException { - /** - * The addScanRangeLocations could be invoked only once. - * So the scanBackendIds should be empty in the beginning. - */ - Preconditions.checkState(scanBackendIds.size() == 0); int logNum = 0; int schemaHash = olapTable.getSchemaHashByIndexId(index.getId()); String schemaHashStr = String.valueOf(schemaHash); @@ -473,14 +468,12 @@ private void computeTabletInfo() throws UserException { if (Config.enable_local_replica_selection) { localBeId = Catalog.getCurrentSystemInfo().getBackendIdByHost(FrontendOptions.getLocalHostAddress()); } - /** - * Reset the tablet and scan range info before compute it. - * The old rollup selector has computed tablet and scan range info. - * Then the new mv selector maybe compute tablet and scan range info again sometimes. - * So, we need to reset those info in here. + * The tablet info could be computed only once. + * So the scanBackendIds should be empty in the beginning. */ - resetTabletAndScanRangeInfo(); + Preconditions.checkState(scanBackendIds.size() == 0); + Preconditions.checkState(scanTabletIds.size() == 0); for (Long partitionId : selectedPartitionIds) { final Partition partition = olapTable.getPartition(partitionId); final MaterializedIndex selectedTable = partition.getIndex(selectedIndexId); @@ -509,17 +502,6 @@ private void computeTabletInfo() throws UserException { } } - private void resetTabletAndScanRangeInfo() { - scanTabletIds = Lists.newArrayList(); - tabletId2BucketSeq = Maps.newHashMap(); - bucketSeq2locations = ArrayListMultimap.create(); - totalTabletsNum = 0; - selectedTabletsNum = 0; - cardinality = 0; - totalBytes = 0; - result = Lists.newArrayList(); - } - /** * We query Palo Meta to get request's data location * extra result info will pass to backend ScanNode diff --git a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index a9e88bed99a24e..3f936cb46b88a9 100644 --- a/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -789,7 +789,7 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) } // if the new selected index id is different from the old one, scan node will be updated. olapScanNode.updateScanRangeInfoByNewMVSelector(bestIndexInfo.getBestIndexId(), - bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable(), analyzer); + bestIndexInfo.isPreAggregation(), bestIndexInfo.getReasonOfDisable()); } } else { From bc30cfc90518d339ac87e1fdafc7ceae8ece1b57 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 15 Jul 2020 15:34:31 +0800 Subject: [PATCH 16/19] Fix compile error Change-Id: I6656a91208fa58bd870977a6a873bcd9cf9893c9 --- fe/src/main/java/org/apache/doris/planner/OlapScanNode.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index b622a469759d93..aafd7082d39e82 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -184,8 +184,7 @@ public Collection getSelectedPartitionIds() { * @param reasonOfDisable * @throws UserException */ - public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isPreAggregation, String - reasonOfDisable, Analyzer analyzer) + public void updateScanRangeInfoByNewMVSelector(long selectedIndexId, boolean isPreAggregation, String reasonOfDisable) throws UserException { if (selectedIndexId == this.selectedIndexId && isPreAggregation == this.isPreAggregation) { return; From 3bd6812764258256cedbf0e139be08a062516a72 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Wed, 15 Jul 2020 20:09:35 +0800 Subject: [PATCH 17/19] Fix ut Change-Id: Ib034a61801fa2d9c0eb566fa5f67f20fc065aea2 --- fe/src/main/java/org/apache/doris/planner/OlapScanNode.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index aafd7082d39e82..6805c80f5d118c 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -443,7 +443,8 @@ public void selectBestRollupByRollupSelector(Analyzer analyzer) throws UserExcep // Step2: select best rollup long start = System.currentTimeMillis(); if (olapTable.getKeysType() == KeysType.DUP_KEYS) { - isPreAggregation = true; + LOG.debug("The best index will be selected later in mv selector"); + return; } final RollupSelector rollupSelector = new RollupSelector(analyzer, desc, olapTable); selectedIndexId = rollupSelector.selectBestRollup(selectedPartitionIds, conjuncts, isPreAggregation); From a28a07da6c4010c64629cc9fa9774a9ee8e4e8f0 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Mon, 20 Jul 2020 11:47:52 +0800 Subject: [PATCH 18/19] Fix compile error Change-Id: Id9106590c48052ace64eea26831f7e283dddb0de --- fe/src/main/java/org/apache/doris/analysis/InsertStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java index 296aeecbf50efd..f3dd61f343b6c0 100644 --- a/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -459,7 +459,7 @@ private void analyzeSubquery(Analyzer analyzer) throws UserException { } } } - if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PRFIX)) { + if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)) { SlotRef refColumn = column.getRefColumn(); if (refColumn == null) { ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR, column.getName(), targetTable.getName()); From 3f5c9cbf06b438aea3676726888f094dc39d0c11 Mon Sep 17 00:00:00 2001 From: emmymiao87 <522274284@qq.com> Date: Mon, 20 Jul 2020 14:42:01 +0800 Subject: [PATCH 19/19] Fix ut Change-Id: I45a957598f0a4f8bbc3c8404530f86a614fc9890 --- .../apache/doris/analysis/InsertStmtTest.java | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/fe/src/test/java/org/apache/doris/analysis/InsertStmtTest.java b/fe/src/test/java/org/apache/doris/analysis/InsertStmtTest.java index bcc9d87b63d901..2ad113d7cabef5 100644 --- a/fe/src/test/java/org/apache/doris/analysis/InsertStmtTest.java +++ b/fe/src/test/java/org/apache/doris/analysis/InsertStmtTest.java @@ -17,37 +17,23 @@ package org.apache.doris.analysis; -import mockit.Tested; import org.apache.doris.catalog.AggregateType; -import org.apache.doris.catalog.Catalog; import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.Function; -import org.apache.doris.catalog.KeysType; -import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Type; -import org.apache.doris.clone.TabletScheduler; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; -import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.SqlParserUtils; -import org.apache.doris.planner.StreamLoadScanNode; -import org.apache.doris.planner.StreamLoadScanNodeTest; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.DorisAssert; +import org.apache.doris.utframe.UtFrameUtils; import com.google.common.collect.Lists; -import org.apache.doris.thrift.TStreamLoadPutRequest; -import org.apache.doris.utframe.DorisAssert; -import org.apache.doris.utframe.UtFrameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -58,7 +44,6 @@ import mockit.Expectations; import mockit.Injectable; -import mockit.Mocked; public class InsertStmtTest { private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; @@ -136,7 +121,8 @@ List getFullSchema() throws Exception { v2.setIsAllowNull(false); columns.add(v2); - Column v3 = new Column("__doris_materialized_view_bitmap_k1", PrimitiveType.BITMAP); + Column v3 = new Column(CreateMaterializedViewStmt.mvColumnBuilder("bitmap_union", "k1"), + PrimitiveType.BITMAP); v3.setIsKey(false); v3.setAggregationType(AggregateType.BITMAP_UNION, false); v3.setIsAllowNull(false); @@ -150,7 +136,7 @@ List getFullSchema() throws Exception { v3.setDefineExpr(defineExpr); columns.add(v3); - Column v4 = new Column("__doris_materialized_view_hll_k2", PrimitiveType.HLL); + Column v4 = new Column(CreateMaterializedViewStmt.mvColumnBuilder("hll_union", "k2"), PrimitiveType.HLL); v4.setIsKey(false); v4.setAggregationType(AggregateType.HLL_UNION, false); v4.setIsAllowNull(false);