Skip to content

Add framework of mv selector #4014

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 19 commits into from
Jul 20, 2020
Merged

Add framework of mv selector #4014

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
e003cac
Support bitmap_union, hll_union and count when creating and matching …
EmmyMiao87 Jun 12, 2020
dbf5707
Add comment and change field name
EmmyMiao87 Jul 6, 2020
688f631
Change code style
EmmyMiao87 Jul 7, 2020
83dbcdd
Add unit test
EmmyMiao87 Jul 8, 2020
f1571f8
Support bitmap_union and hll_union for agg model by Create MV Stmt
EmmyMiao87 Jul 8, 2020
65196a3
Fix replay error
EmmyMiao87 Jul 9, 2020
3a44872
Change display column name
EmmyMiao87 Jul 9, 2020
607aff3
Remove mv column when drop index
EmmyMiao87 Jul 9, 2020
5d424d2
Reset the define expr
EmmyMiao87 Jul 10, 2020
d6c885d
Analyze define expr when replay log
EmmyMiao87 Jul 13, 2020
4bd80e3
Add cast when replay log
EmmyMiao87 Jul 14, 2020
39d3854
Change display name
EmmyMiao87 Jul 14, 2020
0337c0c
Change the finalize
EmmyMiao87 Jul 14, 2020
68746a2
Remove duplicate compute tablet
EmmyMiao87 Jul 15, 2020
fe359d9
Remove duplicate tablet
EmmyMiao87 Jul 15, 2020
bc30cfc
Fix compile error
EmmyMiao87 Jul 15, 2020
3bd6812
Fix ut
EmmyMiao87 Jul 15, 2020
a28a07d
Fix compile error
EmmyMiao87 Jul 20, 2020
3f5c9cb
Fix ut
EmmyMiao87 Jul 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 28 additions & 43 deletions fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
Expand Down Expand Up @@ -422,60 +421,46 @@ private List<Column> checkAndPrepareMaterializedView(CreateMaterializedViewStmt
if (olapTable.hasMaterializedIndex(addMVClause.getMVName())) {
throw new DdlException("Materialized view[" + addMVClause.getMVName() + "] already exists");
}
// check if rollup columns are valid
// a. all columns should exist in base rollup schema
// b. For aggregate table, mv columns with aggregate function should be same as base schema
// c. For aggregate table, the column which is the key of base table should be the key of mv as well.
// check if mv columns are valid
// a. Aggregate or Unique table:
// 1. For aggregate table, mv columns with aggregate function should be same as base schema
// 2. For aggregate table, the column which is the key of base table should be the key of mv as well.
// b. Duplicate table:
// 1. Columns resolved by semantics are legal
// update mv columns
List<MVColumnItem> mvColumnItemList = addMVClause.getMVColumnItemList();
List<Column> newMVColumns = Lists.newArrayList();
int numOfKeys = 0;
for (MVColumnItem mvColumnItem : mvColumnItemList) {
String mvColumnName = mvColumnItem.getName();
Column baseColumn = olapTable.getColumn(mvColumnName);
if (baseColumn == null) {
throw new DdlException("Column[" + mvColumnName + "] does not exist");
}
if (mvColumnItem.isKey()) {
++numOfKeys;
}
AggregateType baseAggregationType = baseColumn.getAggregationType();
AggregateType mvAggregationType = mvColumnItem.getAggregationType();
if (olapTable.getKeysType().isAggregationFamily()) {
if (olapTable.getKeysType().isAggregationFamily()) {
for (MVColumnItem mvColumnItem : mvColumnItemList) {
String mvColumnName = mvColumnItem.getName();
Column baseColumn = olapTable.getColumn(mvColumnName);
if (mvColumnItem.isKey()) {
++numOfKeys;
}
Preconditions.checkNotNull(baseColumn, "Column[" + mvColumnName + "] does not exist");
AggregateType baseAggregationType = baseColumn.getAggregationType();
AggregateType mvAggregationType = mvColumnItem.getAggregationType();
if (baseColumn.isKey() && !mvColumnItem.isKey()) {
throw new DdlException("The column[" + mvColumnName + "] must be the key of materialized view");
}
if (baseAggregationType != mvAggregationType) {
throw new DdlException("The aggregation type of column[" + mvColumnName + "] must be same as "
+ "the aggregate type of base column in aggregate table");
throw new DdlException(
"The aggregation type of column[" + mvColumnName + "] must be same as the aggregate " +
"type of base column in aggregate table");
}
if (baseAggregationType != null && baseAggregationType.isReplaceFamily()
&& olapTable.getKeysNum() != numOfKeys) {
throw new DdlException("The materialized view should contain all keys of base table if there is a"
+ " REPLACE value");
if (baseAggregationType != null && baseAggregationType.isReplaceFamily() && olapTable
.getKeysNum() != numOfKeys) {
throw new DdlException(
"The materialized view should contain all keys of base table if there is a" + " REPLACE "
+ "value");
}
newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
}
if (olapTable.getKeysType() == KeysType.DUP_KEYS && mvAggregationType != null
&& mvAggregationType.isReplaceFamily()) {
throw new DdlException("The aggregation type of REPLACE AND REPLACE IF NOT NULL is forbidden in "
+ "duplicate table");
}
Column newMVColumn = new Column(baseColumn);
newMVColumn.setIsKey(mvColumnItem.isKey());
newMVColumn.setAggregationType(mvAggregationType, mvColumnItem.isAggregationTypeImplicit());
if (mvColumnItem.getDefineExpr() != null) {
if (mvAggregationType.equals(AggregateType.BITMAP_UNION)) {
newMVColumn.setType(Type.BITMAP);
} else if (mvAggregationType.equals(AggregateType.HLL_UNION)){
newMVColumn.setType(Type.HLL);
} else {
throw new DdlException("The define expr of column is only support bitmap_union or hll_union");
}
newMVColumn.setIsKey(false);
newMVColumn.setIsAllowNull(false);
newMVColumn.setDefineExpr(mvColumnItem.getDefineExpr());
} else {
for (MVColumnItem mvColumnItem : mvColumnItemList) {
newMVColumns.add(mvColumnItem.toMVColumn(olapTable));
}
newMVColumns.add(newMVColumn);
}
return newMVColumns;
}
Expand Down
13 changes: 6 additions & 7 deletions fe/src/main/java/org/apache/doris/alter/RollupJobV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.MVColumnItem;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.catalog.Catalog;
Expand Down Expand Up @@ -732,11 +731,11 @@ public void setJobState(JobState jobState) {
this.jobState = jobState;
}

private void setColumnsDefineExpr(List<MVColumnItem> items) {
for (MVColumnItem item : items) {
private void setColumnsDefineExpr(Map<String, Expr> columnNameToDefineExpr) {
for (Entry<String, Expr> entry : columnNameToDefineExpr.entrySet()) {
for (Column column : rollupSchema) {
if (column.getName().equals(item.getName())) {
column.setDefineExpr(item.getDefineExpr());
if (column.getName().equals(entry.getKey())) {
column.setDefineExpr(entry.getValue());
break;
}
}
Expand Down Expand Up @@ -812,8 +811,8 @@ public void gsonPostProcess() throws IOException {
CreateMaterializedViewStmt stmt;
try {
stmt = (CreateMaterializedViewStmt) SqlParserUtils.getStmt(parser, origStmt.idx);
stmt.analyzeSelectClause();
setColumnsDefineExpr(stmt.getMVColumnItemList());
Map<String, Expr> columnNameToDefineExpr = stmt.parseDefineExprWithoutAnalyze();
setColumnsDefineExpr(columnNameToDefineExpr);
} catch (Exception e) {
throw new IOException("error happens when parsing create materialized view stmt: " + origStmt, e);
}
Expand Down
5 changes: 3 additions & 2 deletions fe/src/main/java/org/apache/doris/analysis/AggregateInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.analysis;

import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.planner.DataPartition;
import org.apache.doris.thrift.TPartitionType;
Expand Down Expand Up @@ -532,7 +533,7 @@ private void createSecondPhaseAggInfo(
Preconditions.checkState(inputExpr.isAggregateFunction());
FunctionCallExpr aggExpr = null;
if (!isMultiDistinct_) {
if (inputExpr.getFnName().getFunction().equalsIgnoreCase("COUNT")) {
if (inputExpr.getFnName().getFunction().equalsIgnoreCase(FunctionSet.COUNT)) {
// COUNT(DISTINCT ...) ->
// COUNT(IF(IsNull(<agg slot 1>), NULL, IF(IsNull(<agg slot 2>), NULL, ...)))
// We need the nested IF to make sure that we do not count
Expand All @@ -543,7 +544,7 @@ private void createSecondPhaseAggInfo(
inputDesc.getSlots());
Preconditions.checkNotNull(ifExpr);
ifExpr.analyzeNoThrow(analyzer);
aggExpr = new FunctionCallExpr("count", Lists.newArrayList(ifExpr));
aggExpr = new FunctionCallExpr(FunctionSet.COUNT, Lists.newArrayList(ifExpr));
} else if (inputExpr.getFnName().getFunction().equals("group_concat")) {
// Syntax: GROUP_CONCAT([DISTINCT] expression [, separator])
ArrayList<Expr> exprList = Lists.newArrayList();
Expand Down
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/analysis/AggregateInfoBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ private TupleDescriptor createTupleDesc(Analyzer analyzer, boolean isOutputTuple

// COUNT(), NDV() and NDV_NO_FINALIZE() are non-nullable. The latter two are used
// by compute stats and compute incremental stats, respectively.
if (aggExpr.getFnName().getFunction().equals("count")
if (aggExpr.getFnName().getFunction().equals(FunctionSet.COUNT)
|| aggExpr.getFnName().getFunction().equals("ndv")
|| aggExpr.getFnName().getFunction().equals(FunctionSet.BITMAP_UNION_INT)
|| aggExpr.getFnName().getFunction().equals("ndv_no_finalize")) {
Expand Down
4 changes: 2 additions & 2 deletions fe/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public class AnalyticExpr extends Expr {
private static String SUM = "SUM";
private static String COUNT = "COUNT";

// Internal function used to implement FIRST_VALUE with a window rewrite and
// Internal function used to implement FIRST_VALUE with a window equal and
// additional null handling in the backend.
public static String FIRST_VALUE_REWRITE = "FIRST_VALUE_REWRITE";

Expand Down Expand Up @@ -241,7 +241,7 @@ static private boolean isHllAggFn(Function fn) {
* percent_rank(), cume_dist() and ntile()
*
* Returns a new Expr if the analytic expr is rewritten, returns null if it's not one
* that we want to rewrite.
* that we want to equal.
*/
public static Expr rewrite(AnalyticExpr analyticExpr) {
Function fn = analyticExpr.getFnCall().getFn();
Expand Down
19 changes: 19 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.rewrite.FoldConstantsRule;
import org.apache.doris.rewrite.NormalizeBinaryPredicatesRule;
import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmap;
import org.apache.doris.rewrite.mvrewrite.CountDistinctToBitmapOrHLLRule;
import org.apache.doris.rewrite.mvrewrite.CountFieldToSum;
import org.apache.doris.rewrite.mvrewrite.HLLHashToSlotRefRule;
import org.apache.doris.rewrite.mvrewrite.NDVToHll;
import org.apache.doris.rewrite.mvrewrite.ToBitmapToSlotRefRule;
import org.apache.doris.thrift.TQueryGlobals;

import com.google.common.base.Joiner;
Expand Down Expand Up @@ -235,6 +241,8 @@ private static class GlobalState {
// Expr rewriter for normalizing and rewriting expressions.
private final ExprRewriter exprRewriter_;

private final ExprRewriter mvExprRewriter;

public GlobalState(Catalog catalog, ConnectContext context) {
this.catalog = catalog;
this.context = context;
Expand All @@ -248,6 +256,15 @@ public GlobalState(Catalog catalog, ConnectContext context) {
rules.add(NormalizeBinaryPredicatesRule.INSTANCE);
rules.add(FoldConstantsRule.INSTANCE);
exprRewriter_ = new ExprRewriter(rules);
// init mv rewriter
List<ExprRewriteRule> mvRewriteRules = Lists.newArrayList();
mvRewriteRules.add(ToBitmapToSlotRefRule.INSTANCE);
mvRewriteRules.add(CountDistinctToBitmapOrHLLRule.INSTANCE);
mvRewriteRules.add(CountDistinctToBitmap.INSTANCE);
mvRewriteRules.add(NDVToHll.INSTANCE);
mvRewriteRules.add(HLLHashToSlotRefRule.INSTANCE);
mvRewriteRules.add(CountFieldToSum.INSTANCE);
mvExprRewriter = new ExprRewriter(mvRewriteRules);
}
};
private final GlobalState globalState;
Expand Down Expand Up @@ -511,6 +528,8 @@ public Table getTable(TableName tblName) {

public ExprRewriter getExprRewriter() { return globalState.exprRewriter_; }

public ExprRewriter getMVExprRewriter() { return globalState.mvExprRewriter; }

/**
* Return descriptor of registered table/alias.
*
Expand Down
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.logging.log4j.Logger;

/**
* Class describing between predicates. After successful analysis, we rewrite
* Class describing between predicates. After successful analysis, we equal
* the between predicate to a conjunctive/disjunctive compound predicate
* to be handed to the backend.
*/
Expand Down
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/analysis/CaseWhenClause.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
/**
* captures info of a single WHEN expr THEN expr clause.
*/
class CaseWhenClause {
public class CaseWhenClause {
private final Expr whenExpr;
private final Expr thenExpr;

Expand Down
16 changes: 12 additions & 4 deletions fe/src/main/java/org/apache/doris/analysis/CastExpr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public class CastExpr extends Expr {
private final TypeDef targetTypeDef;

// True if this is a "pre-analyzed" implicit cast.
private final boolean isImplicit;
private boolean isImplicit;

// True if this cast does not change the type.
private boolean noOp = false;
Expand Down Expand Up @@ -137,10 +137,14 @@ public String toSqlImpl() {
if (isImplicit) {
return getChild(0).toSql();
}
if (type.isStringType()) {
return "CAST(" + getChild(0).toSql() + " AS " + "CHARACTER" + ")";
if (isAnalyzed) {
if (type.isStringType()) {
return "CAST(" + getChild(0).toSql() + " AS " + "CHARACTER" + ")";
} else {
return "CAST(" + getChild(0).toSql() + " AS " + type.toString() + ")";
}
} else {
return "CAST(" + getChild(0).toSql() + " AS " + type.toString() + ")";
return "CAST(" + getChild(0).toSql() + " AS " + targetTypeDef.toSql() + ")";
}
}

Expand All @@ -167,6 +171,10 @@ public boolean isImplicit() {
return isImplicit;
}

public void setImplicit(boolean implicit) {
isImplicit = implicit;
}

public void analyze() throws AnalysisException {
// cast was asked for in the query, check for validity of cast
Type childType = getChild(0).getType();
Expand Down
2 changes: 1 addition & 1 deletion fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public DefaultValue(boolean isSet, String value) {
public static DefaultValue NOT_SET = new DefaultValue(false, null);
// default null
public static DefaultValue NULL_DEFAULT_VALUE = new DefaultValue(true, null);
private static String ZERO = new String(new byte[] {0});
public static String ZERO = new String(new byte[] {0});
// default "value", "0" means empty hll
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new DefaultValue(true, ZERO);
// default "value", "0" means empty bitmap
Expand Down
Loading