Skip to content

Support create materialized view with bitmap or hll #3651

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EmmyMiao87 merged 2 commits into from
May 25, 2020
Merged

Support create materialized view with bitmap or hll #3651

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
14368cb
support create materialized view
HangyuanLiu May 21, 2020
9260f13
support create materialized view
HangyuanLiu May 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
Expand Down Expand Up @@ -88,6 +89,7 @@
public class MaterializedViewHandler extends AlterHandler {
private static final Logger LOG = LogManager.getLogger(MaterializedViewHandler.class);
public static final String NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX = "__v2_";
public static final String MATERIALIZED_VIEW_NAME_PRFIX = "__doris_materialized_view_";

public MaterializedViewHandler() {
super("materialized view");
Expand Down Expand Up @@ -459,6 +461,20 @@ private List<Column> checkAndPrepareMaterializedView(CreateMaterializedViewStmt
Column newMVColumn = new Column(baseColumn);
newMVColumn.setIsKey(mvColumnItem.isKey());
newMVColumn.setAggregationType(mvAggregationType, mvColumnItem.isAggregationTypeImplicit());
if (mvColumnItem.getDefineExpr() != null) {
if (mvAggregationType.equals(AggregateType.BITMAP_UNION)) {
newMVColumn.setType(Type.BITMAP);
newMVColumn.setName(MATERIALIZED_VIEW_NAME_PRFIX + "bitmap_" + baseColumn.getName());
} else if (mvAggregationType.equals(AggregateType.HLL_UNION)){
newMVColumn.setType(Type.HLL);
newMVColumn.setName(MATERIALIZED_VIEW_NAME_PRFIX + "hll_" + baseColumn.getName());
} else {
throw new DdlException("The define expr of column is only support bitmap_union or hll_union");
}
newMVColumn.setIsKey(false);
Copy link
Contributor

@EmmyMiao87 EmmyMiao87 May 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The isKey has been set.

Copy link
Contributor Author

@HangyuanLiu HangyuanLiu May 22, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the aggregate function is computed for the original column, it should be not key whether the original column is a key column or a value column

newMVColumn.setIsAllowNull(false);
newMVColumn.setDefineExpr(mvColumnItem.getDefineExpr());
}
newMVColumns.add(newMVColumn);
}
return newMVColumns;
Expand Down
3 changes: 3 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,15 @@ private void analyzeSelectClause() throws AnalysisException {
} else if (selectListItem.getExpr() instanceof FunctionCallExpr) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItem.getExpr();
String functionName = functionCallExpr.getFnName().getFunction();
Expr defineExpr = null;
// TODO(ml): support REPLACE, REPLACE_IF_NOT_NULL only for aggregate table, HLL_UNION, BITMAP_UNION
if (!functionName.equalsIgnoreCase("sum")
&& !functionName.equalsIgnoreCase("min")
&& !functionName.equalsIgnoreCase("max")) {
throw new AnalysisException("The materialized view only support the sum, min and max aggregate "
+ "function. Error function: " + functionCallExpr.toSqlImpl());
}

Preconditions.checkState(functionCallExpr.getChildren().size() == 1);
Expr functionChild0 = functionCallExpr.getChild(0);
SlotRef slotRef;
Expand All @@ -200,6 +202,7 @@ else if (functionChild0 instanceof CastExpr
// TODO(ml): support different type of column, int -> bigint(sum)
MVColumnItem mvColumnItem = new MVColumnItem(columnName);
mvColumnItem.setAggregationType(AggregateType.valueOf(functionName.toUpperCase()), false);
mvColumnItem.setDefineExpr(defineExpr);
mvColumnItemList.add(mvColumnItem);
}
}
Expand Down
3 changes: 3 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,9 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
if (bfColumns != null && bfColumns.contains(column.getName())) {
extras.add("BLOOM_FILTER");
}
if (column.getDefineExpr() != null) {
extras.add(column.getDefineExpr().toSql().toUpperCase());
}
String extraStr = StringUtils.join(extras, ",");

List<String> row = Arrays.asList("",
Expand Down
9 changes: 9 additions & 0 deletions fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ public class MVColumnItem {
private boolean isKey;
private AggregateType aggregationType;
private boolean isAggregationTypeImplicit;
private Expr defineExpr;

public MVColumnItem(String name) {
this.name = name;
Expand Down Expand Up @@ -59,4 +60,12 @@ public AggregateType getAggregationType() {
public boolean isAggregationTypeImplicit() {
return isAggregationTypeImplicit;
}

public Expr getDefineExpr() {
return defineExpr;
}

public void setDefineExpr(Expr defineExpr) {
this.defineExpr = defineExpr;
}
}
17 changes: 17 additions & 0 deletions fe/src/main/java/org/apache/doris/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.catalog;

import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.Expr;
import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
Expand Down Expand Up @@ -66,6 +67,7 @@ public class Column implements Writable {
private String comment;
@SerializedName(value = "stats")
private ColumnStats stats; // cardinality and selectivity etc.
private Expr defineExpr; //use to define materialize view

public Column() {
this.name = "";
Expand Down Expand Up @@ -157,6 +159,10 @@ public boolean isKey() {

public Type getType() { return ScalarType.createType(type.getPrimitiveType()); }

public void setType(Type type) {
this.type = type;
}

public Type getOriginType() { return type; }

public int getStrLen() { return ((ScalarType) type).getLength(); }
Expand Down Expand Up @@ -240,6 +246,9 @@ public TColumn toThrift() {
tColumn.setIs_key(this.isKey);
tColumn.setIs_allow_null(this.isAllowNull);
tColumn.setDefault_value(this.defaultValue);
if (this.defineExpr != null) {
tColumn.setDefine_expr(this.defineExpr.treeToThrift());
}
return tColumn;
}

Expand Down Expand Up @@ -310,6 +319,14 @@ public static String removeNamePrefix(String colName) {
return colName;
}

public Expr getDefineExpr() {
return defineExpr;
}

public void setDefineExpr(Expr expr) {
defineExpr = expr;
}

public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("`").append(name).append("` ");
Expand Down
2 changes: 2 additions & 0 deletions gensrc/thrift/AgentService.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ include "Status.thrift"
include "Types.thrift"
include "PaloInternalService.thrift"
include "Descriptors.thrift"
include "Exprs.thrift"

struct TColumn {
1: required string column_name
Expand All @@ -31,6 +32,7 @@ struct TColumn {
5: optional bool is_allow_null
6: optional string default_value
7: optional bool is_bloom_filter_column
8: optional Exprs.TExpr define_expr
}

struct TTabletSchema {
Expand Down