Skip to content

Change type of sum, min, max function column in mv #4199

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 7 commits into from
Aug 1, 2020
Merged

Change type of sum, min, max function column in mv #4199

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4d94f36
Change type of sum, min, max function column in mv
EmmyMiao87 Jul 27, 2020
caebfc7
Add unit test
EmmyMiao87 Jul 28, 2020
9a7ab50
Change import sequence
EmmyMiao87 Jul 28, 2020
f107770
If the type of base column is tinyint, smallint or int, the type of m…
EmmyMiao87 Jul 28, 2020
ea5b2d3
float -> double
EmmyMiao87 Jul 29, 2020
58849a3
Import
EmmyMiao87 Jul 30, 2020
7427628
Import
EmmyMiao87 Jul 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 32 additions & 20 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,10 @@
* <p>
* Syntax:
* CREATE MATERIALIZED VIEW [MV name] (
* SELECT select_expr[, select_expr ...]
* FROM [Base view name]
* GROUP BY column_name[, column_name ...]
* ORDER BY column_name[, column_name ...])
* SELECT select_expr[, select_expr ...]
* FROM [Base view name]
* GROUP BY column_name[, column_name ...]
* ORDER BY column_name[, column_name ...])
* [PROPERTIES ("key" = "value")]
*/
public class CreateMaterializedViewStmt extends DdlStmt {
Expand All @@ -60,11 +60,11 @@ public class CreateMaterializedViewStmt extends DdlStmt {
static {
FN_NAME_TO_PATTERN = Maps.newHashMap();
FN_NAME_TO_PATTERN.put(AggregateType.SUM.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.SUM.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MIN.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.MIN.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(AggregateType.MAX.name().toLowerCase(),
new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
new MVColumnOneChildPattern(AggregateType.MAX.name().toLowerCase()));
FN_NAME_TO_PATTERN.put(FunctionSet.COUNT, new MVColumnOneChildPattern(FunctionSet.COUNT));
FN_NAME_TO_PATTERN.put(FunctionSet.BITMAP_UNION, new MVColumnBitmapUnionPattern());
FN_NAME_TO_PATTERN.put(FunctionSet.HLL_UNION, new MVColumnHLLUnionPattern());
Expand All @@ -78,9 +78,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
/**
* origin stmt: select k1, k2, v1, sum(v2) from base_table group by k1, k2, v1
* mvColumnItemList: [k1: {name: k1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
* k2: {name: k2, isKey: true, aggType: null, isAggregationTypeImplicit: false},
* v1: {name: v1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
* v2: {name: v2, isKey: false, aggType: sum, isAggregationTypeImplicit: false}]
* k2: {name: k2, isKey: true, aggType: null, isAggregationTypeImplicit: false},
* v1: {name: v1, isKey: true, aggType: null, isAggregationTypeImplicit: false},
* v2: {name: v2, isKey: false, aggType: sum, isAggregationTypeImplicit: false}]
* This order of mvColumnItemList is meaningful.
*/
private List<MVColumnItem> mvColumnItemList = Lists.newArrayList();
Expand Down Expand Up @@ -139,16 +139,16 @@ public void analyze(Analyzer analyzer) throws UserException {
analyzeFromClause();
if (selectStmt.getWhereClause() != null) {
throw new AnalysisException("The where clause is not supported in add materialized view clause, expr:"
+ selectStmt.getWhereClause().toSql());
+ selectStmt.getWhereClause().toSql());
}
if (selectStmt.getHavingPred() != null) {
throw new AnalysisException("The having clause is not supported in add materialized view clause, expr:"
+ selectStmt.getHavingPred().toSql());
+ selectStmt.getHavingPred().toSql());
}
analyzeOrderByClause();
if (selectStmt.getLimit() != -1) {
throw new AnalysisException("The limit clause is not supported in add materialized view clause, expr:"
+ " limit " + selectStmt.getLimit());
+ " limit " + selectStmt.getLimit());
}
}

Expand All @@ -174,7 +174,7 @@ public void analyzeSelectClause() throws AnalysisException {
Expr selectListItemExpr = selectListItem.getExpr();
if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr)) {
throw new AnalysisException("The materialized view only support the single column or function expr. "
+ "Error column: " + selectListItemExpr.toSql());
+ "Error column: " + selectListItemExpr.toSql());
}
if (selectListItemExpr instanceof SlotRef) {
if (meetAggregate) {
Expand Down Expand Up @@ -245,7 +245,7 @@ private void analyzeOrderByClause() throws AnalysisException {
List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
if (orderByElements.size() > mvColumnItemList.size()) {
throw new AnalysisException("The number of columns in order clause must be less then " + "the number of "
+ "columns in select clause");
+ "columns in select clause");
}
if (beginIndexOfAggregation != -1 && (orderByElements.size() != (beginIndexOfAggregation))) {
throw new AnalysisException("The key of columns in mv must be all of group by columns");
Expand All @@ -254,13 +254,13 @@ private void analyzeOrderByClause() throws AnalysisException {
Expr orderByElement = orderByElements.get(i).getExpr();
if (!(orderByElement instanceof SlotRef)) {
throw new AnalysisException("The column in order clause must be original column without calculation. "
+ "Error column: " + orderByElement.toSql());
+ "Error column: " + orderByElement.toSql());
}
MVColumnItem mvColumnItem = mvColumnItemList.get(i);
SlotRef slotRef = (SlotRef) orderByElement;
if (!mvColumnItem.getName().equalsIgnoreCase(slotRef.getColumnName())) {
throw new AnalysisException("The order of columns in order by clause must be same as "
+ "the order of columns in select list");
+ "the order of columns in select list");
}
Preconditions.checkState(mvColumnItem.getAggregationType() == null);
mvColumnItem.setIsKey(true);
Expand Down Expand Up @@ -339,7 +339,7 @@ private void supplyOrderColumn() throws AnalysisException {
}
}

public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws AnalysisException {
private MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws AnalysisException {
String functionName = functionCallExpr.getFnName().getFunction();
List<SlotRef> slots = new ArrayList<>();
functionCallExpr.collect(SlotRef.class, slots);
Expand All @@ -353,11 +353,23 @@ public MVColumnItem buildMVColumnItem(FunctionCallExpr functionCallExpr) throws
Type type;
switch (functionName.toLowerCase()) {
case "sum":
mvColumnName = baseColumnName;
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
PrimitiveType baseColumnType = baseColumnRef.getType().getPrimitiveType();
if (baseColumnType == PrimitiveType.TINYINT || baseColumnType == PrimitiveType.SMALLINT
|| baseColumnType == PrimitiveType.INT) {
type = Type.BIGINT;
} else if (baseColumnType == PrimitiveType.FLOAT) {
type = Type.DOUBLE;
} else {
type = Type.fromPrimitiveType(baseColumnRef.getType().getPrimitiveType());
Copy link
Contributor

@kangkaisen kangkaisen Jul 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sum float result type should be double.

}
break;
case "min":
case "max":
mvColumnName = baseColumnName;
mvAggregateType = AggregateType.valueOf(functionName.toUpperCase());
type = Type.BIGINT;
type = Type.fromPrimitiveType(baseColumnRef.getType().getPrimitiveType());
break;
case FunctionSet.BITMAP_UNION:
// Compatible aggregation models
Expand Down Expand Up @@ -445,7 +457,7 @@ public Map<String, Expr> parseDefineExprWithoutAnalyze() throws AnalysisExceptio
case FunctionSet.COUNT:
Expr defineExpr = new CaseExpr(null, Lists.newArrayList(
new CaseWhenClause(new IsNullPredicate(slots.get(0), false),
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
new IntLiteral(0, Type.BIGINT))), new IntLiteral(1, Type.BIGINT));
result.put(mvColumnBuilder(functionName, baseColumnName), defineExpr);
break;
default:
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public Column toMVColumn(OlapTable olapTable) throws DdlException {
} else {
Column result = new Column(baseColumn);
result.setIsKey(isKey);
result.setType(type);
result.setAggregationType(aggregationType, isAggregationTypeImplicit);
return result;
}
Expand Down
73 changes: 56 additions & 17 deletions fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
Expand Down Expand Up @@ -325,7 +326,7 @@ public void testDuplicateColumn(@Injectable SelectStmt selectStmt) throws UserEx

@Test
public void testDuplicateColumn1(@Injectable SlotRef slotRef1,
@Injectable SelectStmt selectStmt) throws UserException {
@Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -415,7 +416,7 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef4,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable AggregateInfo aggregateInfo) throws UserException {
@Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -506,8 +507,8 @@ public void testMVColumnsWithoutOrderby(@Injectable SlotRef slotRef1,

@Test
public void testMVColumnsWithoutOrderbyWithoutAggregation(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -606,8 +607,8 @@ public void testMVColumnsWithoutOrderbyWithoutAggregation(@Injectable SlotRef sl
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithFloat(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -704,8 +705,8 @@ public void testMVColumnsWithoutOrderbyWithoutAggregationWithFloat(@Injectable S
*/
@Test
public void testMVColumnsWithoutOrderbyWithoutAggregationWithVarchar(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
@Injectable SlotRef slotRef2, @Injectable SlotRef slotRef3, @Injectable SlotRef slotRef4,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -802,7 +803,7 @@ public void testMVColumnsWithoutOrderbyWithoutAggregationWithVarchar(@Injectable
*/
@Test
public void testMVColumnsWithFirstFloat(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -850,7 +851,7 @@ public void testMVColumnsWithFirstFloat(@Injectable SlotRef slotRef1,
*/
@Test
public void testMVColumnsWithFirstVarchar(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
@Injectable TableRef tableRef, @Injectable SelectStmt selectStmt) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -903,10 +904,10 @@ public void testMVColumnsWithFirstVarchar(@Injectable SlotRef slotRef1,

@Test
public void testMVColumns(@Injectable SlotRef slotRef1,
@Injectable SlotRef slotRef2,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable AggregateInfo aggregateInfo) throws UserException {
@Injectable SlotRef slotRef2,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -981,9 +982,9 @@ public void testMVColumns(@Injectable SlotRef slotRef1,

@Test
public void testDeduplicateMV(@Injectable SlotRef slotRef1,
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable AggregateInfo aggregateInfo) throws UserException {
@Injectable TableRef tableRef,
@Injectable SelectStmt selectStmt,
@Injectable AggregateInfo aggregateInfo) throws UserException {
SelectList selectList = new SelectList();
SelectListItem selectListItem1 = new SelectListItem(slotRef1, null);
selectList.addItem(selectListItem1);
Expand Down Expand Up @@ -1022,5 +1023,43 @@ public void testDeduplicateMV(@Injectable SlotRef slotRef1,
}

}

@Test
public void testBuildMVColumnItem(@Injectable SelectStmt selectStmt) {
CreateMaterializedViewStmt createMaterializedViewStmt = new CreateMaterializedViewStmt("test", selectStmt, null);
SlotRef slotRef = new SlotRef(new TableName("db", "table"), "a");
List<Expr> params = Lists.newArrayList();
params.add(slotRef);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("sum", params);
slotRef.setType(Type.LARGEINT);
MVColumnItem mvColumnItem = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr);
Assert.assertEquals(Type.LARGEINT, mvColumnItem.getType());

SlotRef slotRef2 = new SlotRef(new TableName("db", "table"), "a");
List<Expr> params2 = Lists.newArrayList();
params2.add(slotRef2);
FunctionCallExpr functionCallExpr2 = new FunctionCallExpr("sum", params2);
slotRef2.setType(Type.SMALLINT);
MVColumnItem mvColumnItem2 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr2);
Assert.assertEquals(Type.BIGINT, mvColumnItem2.getType());

SlotRef slotRef3 = new SlotRef(new TableName("db", "table"), "a");
List<Expr> params3 = Lists.newArrayList();
params3.add(slotRef3);
FunctionCallExpr functionCallExpr3 = new FunctionCallExpr("min", params3);
slotRef3.setType(Type.VARCHAR);
MVColumnItem mvColumnItem3 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr3);
Assert.assertEquals(Type.VARCHAR, mvColumnItem3.getType());

SlotRef slotRef4 = new SlotRef(new TableName("db", "table"), "a");
List<Expr> params4 = Lists.newArrayList();
params4.add(slotRef4);
FunctionCallExpr functionCallExpr4 = new FunctionCallExpr("sum", params4);
slotRef4.setType(Type.DOUBLE);
MVColumnItem mvColumnItem4 = Deencapsulation.invoke(createMaterializedViewStmt, "buildMVColumnItem", functionCallExpr4);
Assert.assertEquals(Type.DOUBLE, mvColumnItem4.getType());


}
}