From 40ac111fa5d197c4828913c2d3f40ab63e004f7c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 23 May 2024 16:58:00 +0800 Subject: [PATCH 01/21] [Feat](nereids) support generated column --- .../org/apache/doris/nereids/DorisLexer.g4 | 2 + .../org/apache/doris/nereids/DorisParser.g4 | 5 +- fe/fe-core/src/main/cup/sql_parser.cup | 23 +- .../doris/alter/SchemaChangeHandler.java | 39 ++ .../org/apache/doris/analysis/ColumnDef.java | 58 ++- .../doris/analysis/CreateTableStmt.java | 158 +++++++ .../doris/analysis/FunctionCallExpr.java | 11 + .../java/org/apache/doris/catalog/Column.java | 60 ++- .../doris/catalog/GeneratedColumnInfo.java | 78 +++ .../org/apache/doris/common/TreeNode.java | 10 + .../common/proc/IndexSchemaProcNode.java | 3 + .../apache/doris/common/util/ExprUtil.java | 45 ++ .../common/util/GeneratedColumnUtil.java | 69 +++ .../main/java/org/apache/doris/load/Load.java | 27 +- .../nereids/analyzer/UnboundTableSink.java | 5 + .../nereids/parser/LogicalPlanBuilder.java | 6 +- .../nereids/rules/analysis/BindSink.java | 68 ++- .../plans/commands/info/ColumnDefinition.java | 46 +- .../plans/commands/info/CreateTableInfo.java | 232 ++++++++- .../commands/info/GeneratedColumnDesc.java | 64 +++ .../plans/commands/insert/InsertUtils.java | 42 ++ fe/fe-core/src/main/jflex/sql_scanner.flex | 2 + .../gen_col_data.csv | 2 + .../gen_col_data.json | 4 + .../test_create_table_generated_column.out | 337 +++++++++++++ ...t_create_table_generated_column_legacy.out | 93 ++++ .../test_routine_load_generated_column.out | 5 + .../three_column_gen_col_data.csv | 2 + .../test_create_table_generated_column.groovy | 445 ++++++++++++++++++ ...reate_table_generated_column_legacy.groovy | 229 +++++++++ .../test_routine_load_generated_column.groovy | 98 ++++ 31 files changed, 2195 insertions(+), 73 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.csv create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.json create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/three_column_gen_col_data.csv create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 8b256a7b07cda4..9971d277ebebcc 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -115,6 +115,7 @@ AT: 'AT'; AUTHORS: 'AUTHORS'; AUTO: 'AUTO'; AUTO_INCREMENT: 'AUTO_INCREMENT'; +ALWAYS: 'ALWAYS'; BACKEND: 'BACKEND'; BACKENDS: 'BACKENDS'; BACKUP: 'BACKUP'; @@ -276,6 +277,7 @@ FRONTENDS: 'FRONTENDS'; FULL: 'FULL'; FUNCTION: 'FUNCTION'; FUNCTIONS: 'FUNCTIONS'; +GENERATED: 'GENERATED'; GENERIC: 'GENERIC'; GLOBAL: 'GLOBAL'; GRANT: 'GRANT'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 8ac823200ee3a0..68da90f9da1ee3 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -585,6 +585,7 @@ columnDef : colName=identifier type=dataType KEY? (aggType=aggTypeDef)? + ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | stringValue=STRING_LITERAL @@ -592,7 +593,7 @@ columnDef (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? ; - + indexDefs : indexes+=indexDef (COMMA indexes+=indexDef)* ; @@ -1028,6 +1029,7 @@ nonReserved | AGG_STATE | AGGREGATE | ALIAS + | ALWAYS | ANALYZED | ARRAY | ARRAY_RANGE @@ -1142,6 +1144,7 @@ nonReserved | FREE | FRONTENDS | FUNCTION + | GENERATED | GENERIC | GLOBAL | GRAPH diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 151e1c1f5f97ee..fdacb6f9df7b40 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -43,6 +43,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; @@ -62,6 +63,7 @@ import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.Optional; import java.util.stream.Collectors; import java_cup.runtime.Symbol; @@ -270,6 +272,7 @@ terminal String KW_AS, KW_ASC, KW_AT, + KW_ALWAYS, KW_AUTHORS, KW_BACKEND, KW_BACKENDS, @@ -408,6 +411,7 @@ terminal String KW_FULL, KW_FUNCTION, KW_FUNCTIONS, + KW_GENERATED, KW_GENERIC, KW_GLOBAL, KW_GRANT, @@ -1004,6 +1008,9 @@ nonterminal String stage_name; nonterminal StageAndPattern stage_and_pattern; nonterminal List copy_select_expr_list; +//genearted column +nonterminal Boolean opt_generated_always; + precedence nonassoc COMMA; precedence nonassoc STRING_LITERAL; precedence nonassoc KW_COLUMNS; @@ -3968,6 +3975,11 @@ column_definition ::= ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, aggType, nullable_type, autoIncInitValue, defaultValue, comment); RESULT = columnDef; :} + | ident:columnName type_def:typeDef opt_is_key:isKey opt_generated_always KW_AS LPAREN expr:expr RPAREN opt_nullable_type:nullable_type opt_comment:comment + {: + ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, nullable_type, comment, Optional.of(new GeneratedColumnInfo(null, expr))); + RESULT = columnDef; + :} ; index_definition ::= @@ -7942,7 +7954,16 @@ opt_work ::= RESULT = null; :} ; - +opt_generated_always ::= + /* empty */ + {: + RESULT = null; + :} + | KW_GENERATED KW_ALWAYS + {: + RESULT = null; + :} + ; opt_chain ::= {: RESULT = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 24d373a0d5f34a..36d63adbca416d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DropColumnClause; import org.apache.doris.analysis.DropIndexClause; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.ModifyColumnClause; import org.apache.doris.analysis.ModifyTablePropertiesClause; @@ -42,6 +43,7 @@ import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; @@ -377,6 +379,21 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa } } + // generated column check + Map nameToColumn = new HashMap<>(); + for (Column c : indexSchemaMap.get(baseIndexId)) { + nameToColumn.put(c.getName(), c); + } + if (nameToColumn.containsKey(dropColName)) { + Column column = nameToColumn.get(dropColName); + Set generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis(); + if (!generatedColumnsThatReferToThis.isEmpty()) { + throw new DdlException( + "Column '" + dropColName + "' has a generated column dependency on :" + + generatedColumnsThatReferToThis); + } + } + Iterator it = indexes.iterator(); while (it.hasNext()) { Index index = it.next(); @@ -403,6 +420,8 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa if (column.getName().equalsIgnoreCase(dropColName)) { baseIter.remove(); found = true; + // find generated column referred column + removeColumnWhenDropGeneratedColumn(column, nameToColumn); break; } } @@ -3194,4 +3213,24 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List nameToColumn) { + GeneratedColumnInfo generatedColumnInfo = dropColumn.getGeneratedColumnInfo(); + if (generatedColumnInfo == null) { + return; + } + String dropColName = dropColumn.getName(); + Expr expr = generatedColumnInfo.getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRef : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRef).getColumnName(); + if (!nameToColumn.containsKey(name)) { + continue; + } + Column c = nameToColumn.get(name); + Set sets = c.getGeneratedColumnsThatReferToThis(); + sets.remove(dropColName); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 6dd00bfe58d01d..a36896851c5e71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; @@ -38,6 +39,10 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; // Column definition which is generated by SQL syntax parser // Syntax: @@ -115,7 +120,7 @@ public DefaultValue(boolean isSet, String value, String exprName, Long precision public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precision) { if (precision > ScalarType.MAX_DATETIMEV2_SCALE || precision < 0) { throw new IllegalArgumentException("column's default value current_timestamp" - + " precision must be between 0 and 6"); + + " precision must be between 0 and 6"); } if (precision == 0) { return new DefaultValue(true, CURRENT_TIMESTAMP, NOW); @@ -183,6 +188,9 @@ public String getValue() { private String comment; private boolean visible; private int clusterKeyId = -1; + private Optional generatedColumnInfo = Optional.empty(); + private Set generatedColumnsThatReferToThis = new HashSet<>(); + public ColumnDef(String name, TypeDef typeDef) { this(name, typeDef, false, null, ColumnNullableType.NOT_NULLABLE, DefaultValue.NOT_SET, ""); @@ -198,17 +206,19 @@ public ColumnDef(String name, TypeDef typeDef, ColumnNullableType nullableType) public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, long autoIncInitValue, DefaultValue defaultValue, String comment) { - this(name, typeDef, isKey, aggregateType, nullableType, autoIncInitValue, defaultValue, comment, true); + this(name, typeDef, isKey, aggregateType, nullableType, autoIncInitValue, defaultValue, comment, true, + Optional.empty()); } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, DefaultValue defaultValue, String comment) { - this(name, typeDef, isKey, aggregateType, nullableType, -1, defaultValue, comment, true); + this(name, typeDef, isKey, aggregateType, nullableType, -1, defaultValue, comment, true, + Optional.empty()); } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, long autoIncInitValue, DefaultValue defaultValue, String comment, - boolean visible) { + boolean visible, Optional generatedColumnInfo) { this.name = name; this.typeDef = typeDef; this.isKey = isKey; @@ -221,6 +231,7 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr this.defaultValue = defaultValue; this.comment = comment; this.visible = visible; + this.generatedColumnInfo = generatedColumnInfo; } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, boolean isAllowNull, @@ -242,44 +253,51 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr this.visible = visible; } + public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType nullableType, String comment, + Optional generatedColumnInfo) { + this(name, typeDef, isKey, null, nullableType, -1, DefaultValue.NOT_SET, + comment, true, generatedColumnInfo); + } + public static ColumnDef newDeleteSignColumnDef() { return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris delete flag hidden column", false); + "doris delete flag hidden column", false, Optional.empty()); } public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris delete flag hidden column", false); + "doris delete flag hidden column", false, Optional.empty()); } public static ColumnDef newSequenceColumnDef(Type type) { return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1, - DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false); + DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); } public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) { return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE, - -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false); + -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, + Optional.empty()); } public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""), - "doris row store hidden column", false); + "doris row store hidden column", false, Optional.empty()); } public static ColumnDef newVersionColumnDef() { return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris version hidden column", false); + "doris version hidden column", false, Optional.empty()); } public static ColumnDef newVersionColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris version hidden column", false); + "doris version hidden column", false, Optional.empty()); } public boolean isAllowNull() { @@ -411,9 +429,9 @@ public void analyze(boolean isOlap) throws AnalysisException { + " the duplicate table at present."); } if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE - && !defaultValue.value.equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value)) { + && !defaultValue.value.equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value)) { throw new AnalysisException("Array type column default value only support null or " - + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value); + + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value); } } if (isKey() && type.getPrimitiveType() == PrimitiveType.STRING && isOlap) { @@ -626,7 +644,7 @@ public Column toColumn() { Type type = typeDef.getType(); return new Column(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue.value, comment, visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue(), - clusterKeyId); + clusterKeyId, generatedColumnInfo.orElse(null), generatedColumnsThatReferToThis); } @Override @@ -637,4 +655,16 @@ public String toString() { public void setAllowNull(boolean allowNull) { isAllowNull = allowNull; } + + public Optional getGeneratedColumnInfo() { + return generatedColumnInfo; + } + + public long getAutoIncInitValue() { + return autoIncInitValue; + } + + public void addGeneratedColumnsThatReferToThis(List list) { + generatedColumnsThatReferToThis.addAll(list); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 6398d171d0eb62..ee94ccf36a5c00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; @@ -35,6 +36,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.AutoBucketUtils; +import org.apache.doris.common.util.GeneratedColumnUtil; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PrintableMap; @@ -42,6 +44,8 @@ import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rewrite.ExprRewriteRule; +import org.apache.doris.rewrite.ExprRewriter; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -53,11 +57,13 @@ import java.io.DataInput; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -585,6 +591,7 @@ public void analyze(Analyzer analyzer) throws UserException { throw new AnalysisException("same index columns have multiple same type index is not allowed."); } } + generatedColumnCheck(analyzer); } private void analyzeEngineName() throws AnalysisException { @@ -713,4 +720,155 @@ public String toString() { public boolean needAuditEncryption() { return !engineName.equals("olap"); } + + private void generatedColumnCheck(Analyzer analyzer) throws AnalysisException { + genreatedColumnCommonCheck(); + Map> nameToColumnDef = Maps.newHashMap(); + for (int i = 0; i < columnDefs.size(); i++) { + ColumnDef columnDef = columnDefs.get(i); + nameToColumnDef.put(columnDef.getName(), Pair.of(columnDef, i)); + } + SlotRefRewriteRule.initializeslotRefMap(nameToColumnDef); + List exprAndnames = Lists.newArrayList(); + for (int i = 0; i < columnDefs.size(); i++) { + ColumnDef columnDef = columnDefs.get(i); + if (!columnDef.getGeneratedColumnInfo().isPresent()) { + continue; + } + SlotRefRewriteRule slotRefRewriteRule = new SlotRefRewriteRule(i); + ExprRewriter rewriter = new ExprRewriter(slotRefRewriteRule); + GeneratedColumnInfo generatedColumnInfo = columnDef.getGeneratedColumnInfo().get(); + Expr expr = rewriter.rewrite(generatedColumnInfo.getExpr(), analyzer); + expr.foreach(e -> { + if (e instanceof LambdaFunctionCallExpr) { + throw new AnalysisException("Generated column does not support lambda."); + } else if (e instanceof Subquery) { + throw new AnalysisException("Generated column does not support subquery."); + } + }); + expr.analyze(analyzer); + expr.foreach(e -> { + if (e instanceof VariableExpr) { + throw new AnalysisException("Generated column expression cannot contain variable."); + } else if (e instanceof SlotRef && nameToColumnDef.containsKey(((SlotRef) e).getColumnName())) { + ColumnDef refColumnDef = nameToColumnDef.get(((SlotRef) e).getColumnName()).first; + if (refColumnDef.getAutoIncInitValue() != -1) { + throw new AnalysisException("Generated column '" + columnDef.getName() + + "' cannot refer to auto-increment column."); + } + } else if (e instanceof FunctionCallExpr && !checkFunctionInGeneratedColumn((FunctionCallExpr) e)) { + throw new AnalysisException( + "Expression of generated column '" + + columnDef.getName() + "' contains a disallowed function:'" + + ((FunctionCallExpr) e).getFnName() + "'"); + } else if (e instanceof AnalyticExpr) { + throw new AnalysisException( + "Expression of generated column '" + + columnDef.getName() + "' contains a disallowed expression:'" + + ((AnalyticExpr) e).toSqlWithoutTbl() + "'"); + } + }); + if (!Type.canCastTo(expr.getType(), columnDef.getType())) { + throw new AnalysisException("can not cast from origin type " + + expr.getType() + " to target type=" + columnDef.getType()); + } + exprAndnames.add(new GeneratedColumnUtil.ExprAndname(expr, columnDef.getName())); + } + + // for alter drop column + Map> columnToListOfGeneratedColumnsThatReferToThis = new HashMap<>(); + for (ColumnDef column : columnDefs) { + Optional info = column.getGeneratedColumnInfo(); + if (!info.isPresent()) { + continue; + } + Expr generatedExpr = info.get().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + generatedExpr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRefExpr : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRefExpr).getColumnName(); + columnToListOfGeneratedColumnsThatReferToThis + .computeIfAbsent(name, k -> new ArrayList<>()) + .add(column.getName()); + } + } + for (Map.Entry> entry : columnToListOfGeneratedColumnsThatReferToThis.entrySet()) { + ColumnDef col = nameToColumnDef.get(entry.getKey()).first; + col.addGeneratedColumnsThatReferToThis(entry.getValue()); + } + + GeneratedColumnUtil.rewriteColumns(exprAndnames); + for (GeneratedColumnUtil.ExprAndname exprAndname : exprAndnames) { + if (nameToColumnDef.containsKey(exprAndname.getName())) { + ColumnDef columnDef = nameToColumnDef.get(exprAndname.getName()).first; + Optional info = columnDef.getGeneratedColumnInfo(); + info.ifPresent(columnInfo -> columnInfo.setExpandExprForLoad(exprAndname.getExpr())); + } + } + } + + private boolean checkFunctionInGeneratedColumn(FunctionCallExpr funcExpr) { + if (!funcExpr.isScalarFunction()) { + return false; + } + if (funcExpr.fn.isUdf()) { + return false; + } + if (funcExpr instanceof GroupingFunctionCallExpr) { + return false; + } + return true; + } + + public static final class SlotRefRewriteRule implements ExprRewriteRule { + private static Map> nameToColumnDefMap = new HashMap<>(); + private final int index; + + public SlotRefRewriteRule(int index) { + this.index = index; + } + + public static void initializeslotRefMap(Map> map) { + nameToColumnDefMap = map; + } + + @Override + public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException { + if (!(expr instanceof SlotRef)) { + return expr; + } + SlotRef slotRef = (SlotRef) expr; + if (!nameToColumnDefMap.containsKey(slotRef.getColumnName())) { + throw new AnalysisException("Unknown column '" + slotRef.getColumnName() + + "' in 'generated column function'"); + } + Pair columnAndIdx = nameToColumnDefMap.get(slotRef.getColumnName()); + ColumnDef columnDef = columnAndIdx.first; + if (columnDef.getGeneratedColumnInfo().isPresent() && columnAndIdx.second >= index) { + throw new AnalysisException( + "Generated column can refer only to generated columns defined prior to it."); + } + slotRef.setType(columnDef.getType()); + slotRef.setAnalyzed(true); + return slotRef; + } + } + + private void genreatedColumnCommonCheck() throws AnalysisException { + boolean hasGeneratedCol = false; + for (ColumnDef column : columnDefs) { + if (column.getGeneratedColumnInfo().isPresent()) { + hasGeneratedCol = true; + break; + } + } + if (hasGeneratedCol && !engineName.equalsIgnoreCase("olap")) { + throw new AnalysisException( + "Tables can only have generated columns if the olap engine is used"); + } + if (hasGeneratedCol && keysDesc.getKeysType() == KeysType.AGG_KEYS) { + throw new AnalysisException( + "Generated Column cannot be used in the aggregate table"); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index c0c4f8d049b7d3..2e635beaee139b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -686,6 +686,17 @@ public String toSqlImpl() { sb.append(((FunctionCallExpr) expr).fnName); sb.append(" "); sb.append(children.get(1).toSql()); + } else if (fnName.getFunction().equalsIgnoreCase("encryptkeyref")) { + sb.append("key "); + for (int i = 0; i < children.size(); i++) { + String str = ((StringLiteral) children.get(i)).getValue(); + if (str.isEmpty()) { + continue; + } + sb.append(str); + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); } else { sb.append(((FunctionCallExpr) expr).fnName); sb.append(paramsToSql()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index ca710188f5f553..1ac4c81c4d1413 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -51,6 +51,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -145,6 +146,12 @@ public class Column implements Writable, GsonPostProcessable { @SerializedName(value = "onUpdateDefaultValueExprDef") private DefaultValueExprDef onUpdateDefaultValueExprDef; + @SerializedName(value = "generatedColumnInfo") + private GeneratedColumnInfo generatedColumnInfo; + + @SerializedName(value = "generatedColumnsThatReferToThis") + private Set generatedColumnsThatReferToThis = new HashSet<>(); + public Column() { this.name = ""; this.type = Type.NULL; @@ -185,39 +192,41 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, true, null, - COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, false, null); + COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, false, null, null, + Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, null, comment, visible, null, colUniqueId, null, - false, null); + false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null, - false, null); + false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, defaultValueExprDef, - colUniqueId, realDefaultValue, false, null); + colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, - defaultValueExprDef, colUniqueId, realDefaultValue, false, null); + defaultValueExprDef, colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, - boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef) { + boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this.name = name; if (this.name == null) { this.name = ""; @@ -245,6 +254,8 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType this.uniqueId = colUniqueId; this.hasOnUpdateDefaultValue = hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = onUpdateDefaultValueExprDef; + this.generatedColumnInfo = generatedColumnInfo; + this.generatedColumnsThatReferToThis.addAll(generatedColumnsThatReferToThis); if (type.isAggStateType()) { AggStateType aggState = (AggStateType) type; @@ -262,18 +273,22 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, boolean hasOnUpdateDefaultValue, - DefaultValueExprDef onUpdateDefaultValueExprDef, int clusterKeyId) { + DefaultValueExprDef onUpdateDefaultValueExprDef, int clusterKeyId, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, defaultValueExprDef, colUniqueId, realDefaultValue, - hasOnUpdateDefaultValue, onUpdateDefaultValueExprDef); + hasOnUpdateDefaultValue, onUpdateDefaultValueExprDef, generatedColumnInfo, + generatedColumnsThatReferToThis); this.clusterKeyId = clusterKeyId; } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, - DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, int clusterKeyId) { + DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, int clusterKeyId, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, - defaultValueExprDef, colUniqueId, realDefaultValue); + defaultValueExprDef, colUniqueId, realDefaultValue, false, null, generatedColumnInfo, + generatedColumnsThatReferToThis); this.clusterKeyId = clusterKeyId; } @@ -299,6 +314,7 @@ public Column(Column column) { this.hasOnUpdateDefaultValue = column.hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = column.onUpdateDefaultValueExprDef; this.clusterKeyId = column.getClusterKeyId(); + this.generatedColumnInfo = column.generatedColumnInfo; } public void createChildrenColumn(Type type, Column column) { @@ -933,6 +949,9 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { } else { sb.append(typeStr); } + if (generatedColumnInfo != null) { + return generatedColumnToSql(sb); + } if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable && !isAggregationTypeImplicit) { sb.append(" ").append(aggregationType.toSql()); @@ -961,6 +980,19 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { return sb.toString(); } + private String generatedColumnToSql(StringBuilder sb) { + sb.append(" AS (").append(generatedColumnInfo.getExpr().toSql()).append(")"); + if (isAllowNull) { + sb.append(" NULL"); + } else { + sb.append(" NOT NULL"); + } + if (StringUtils.isNotBlank(comment)) { + sb.append(" COMMENT '").append(getComment(true)).append("'"); + } + return sb.toString(); + } + @Override public String toString() { return toSql(); @@ -1166,4 +1198,12 @@ public boolean isMaterializedViewColumn() { return getName().startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX) || getName().startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX); } + + public GeneratedColumnInfo getGeneratedColumnInfo() { + return generatedColumnInfo; + } + + public Set getGeneratedColumnsThatReferToThis() { + return generatedColumnsThatReferToThis; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java new file mode 100644 index 00000000000000..37bc8001020a29 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.Expr; + +import com.google.gson.annotations.SerializedName; +import jline.internal.Nullable; + +/**GeneratedColumnInfo*/ +public class GeneratedColumnInfo { + /**GeneratedColumnType*/ + public enum GeneratedColumnType { + VIRTUAL, + STORED + } + + @SerializedName(value = "type") + private final GeneratedColumnType type; + @SerializedName(value = "exprSql") + private final String exprSql; + @SerializedName(value = "expr") + private final Expr expr; + + /* e.g. a,b,c=a+b,d=c+1 -> a,b,c=a+b,d=a+b+1 + e.g. this is column d generated column info + expr is c+1, expandExprForLoad is a+b+1 + expandExprForLoad is used in streamload, routineload, mysqlload, etc */ + @SerializedName(value = "exprForLoad") + private Expr expandExprForLoad; + + /** constructor */ + public GeneratedColumnInfo(@Nullable String exprSql, Expr expr) { + this(exprSql, expr, null); + } + + public GeneratedColumnInfo(@Nullable String exprSql, Expr expr, Expr expandExprForLoad) { + if (exprSql != null) { + this.exprSql = exprSql; + } else { + this.exprSql = expr.toSqlWithoutTbl(); + } + this.expr = expr; + this.expandExprForLoad = expandExprForLoad; + this.type = GeneratedColumnType.STORED; + } + + public String getExprSql() { + return exprSql; + } + + public Expr getExpr() { + return expr; + } + + public Expr getExpandExprForLoad() { + return expandExprForLoad; + } + + public void setExpandExprForLoad(Expr expandExprForLoad) { + this.expandExprForLoad = expandExprForLoad; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java index 7693acf3bb33d2..8070505798b98c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java @@ -239,4 +239,14 @@ public C findFirstOf(Class cl) { return null; } + public interface ThrowingConsumer { + void accept(T t) throws AnalysisException; + } + + public void foreach(ThrowingConsumer> func) throws AnalysisException { + func.accept(this); + for (NodeType child : getChildren()) { + child.foreach(func); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 0f2fb911d4658b..572151210b6df4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -65,6 +65,9 @@ public static ProcResult createResult(List schema, Set bfColumns if (column.isAutoInc()) { extras.add("AUTO_INCREMENT"); } + if (column.getGeneratedColumnInfo() != null) { + extras.add("GENERATED"); + } String extraStr = StringUtils.join(extras, ","); List rowList = Arrays.asList(column.getDisplayName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java new file mode 100644 index 00000000000000..331a8db49cbffe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; + +import org.apache.commons.collections.CollectionUtils; + +import java.util.Map; + +public class ExprUtil { + + public static void recursiveRewrite(Expr expr, Map derivativeColumns) { + if (CollectionUtils.isEmpty(expr.getChildren())) { + return; + } + for (int i = 0; i < expr.getChildren().size(); i++) { + Expr e = expr.getChild(i); + if (e instanceof SlotRef) { + String columnName = ((SlotRef) e).getColumnName(); + if (derivativeColumns.containsKey(columnName)) { + expr.setChild(i, derivativeColumns.get(columnName)); + } + } else { + recursiveRewrite(e, derivativeColumns); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java new file mode 100644 index 00000000000000..59d61b76dcf1e1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; + +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +public class GeneratedColumnUtil { + public static class ExprAndname { + private Expr expr; + private String name; + + public ExprAndname(Expr expr, String name) { + this.expr = expr; + this.name = name; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + public static void rewriteColumns(List exprAndnames) { + Map nameToExprMap = Maps.newHashMap(); + for (ExprAndname exprAndname : exprAndnames) { + if (exprAndname.getExpr() instanceof SlotRef) { + String columnName = ((SlotRef) exprAndname.getExpr()).getColumnName(); + if (nameToExprMap.containsKey(columnName)) { + exprAndname.setExpr(nameToExprMap.get(columnName)); + } + } else { + ExprUtil.recursiveRewrite(exprAndname.getExpr(), nameToExprMap); + } + nameToExprMap.put(exprAndname.getName(), exprAndname.getExpr()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index 398fba0d261662..0323f3ea62ee15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -37,6 +37,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -61,6 +62,7 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.ExprUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -77,7 +79,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.Gson; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -354,6 +355,11 @@ private static void initColumns(Table tbl, List columnExprs, for (Column column : tbl.getBaseSchema()) { String columnName = column.getName(); colToType.put(columnName, column.getType()); + if (column.getGeneratedColumnInfo() != null) { + GeneratedColumnInfo info = column.getGeneratedColumnInfo(); + exprsByName.put(column.getName(), info.getExpandExprForLoad()); + continue; + } if (columnExprMap.containsKey(columnName)) { continue; } @@ -617,7 +623,7 @@ public static void rewriteColumns(LoadTaskInfo.ImportColumnDescs columnDescs) { importColumnDesc.setExpr(derivativeColumns.get(columnName)); } } else { - recursiveRewrite(importColumnDesc.getExpr(), derivativeColumns); + ExprUtil.recursiveRewrite(importColumnDesc.getExpr(), derivativeColumns); } derivativeColumns.put(importColumnDesc.getColumnName(), importColumnDesc.getExpr()); } @@ -626,23 +632,6 @@ public static void rewriteColumns(LoadTaskInfo.ImportColumnDescs columnDescs) { columnDescs.isColumnDescsRewrited = true; } - private static void recursiveRewrite(Expr expr, Map derivativeColumns) { - if (CollectionUtils.isEmpty(expr.getChildren())) { - return; - } - for (int i = 0; i < expr.getChildren().size(); i++) { - Expr e = expr.getChild(i); - if (e instanceof SlotRef) { - String columnName = ((SlotRef) e).getColumnName(); - if (derivativeColumns.containsKey(columnName)) { - expr.setChild(i, derivativeColumns.get(columnName)); - } - } else { - recursiveRewrite(e, derivativeColumns); - } - } - } - /** * This method is used to transform hadoop function. * The hadoop function includes: replace_value, strftime, time_format, alignment_timestamp, default_value, now. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index 23c58ba42fb17e..54e5b5d5de8112 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -173,6 +173,11 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr isPartialUpdate, dmlCommandType, groupExpression, logicalProperties, children.get(0)); } + public Plan withColNames(List colNames) { + return new UnboundTableSink<>(nameParts, colNames, hints, temporaryPartition, partitions, autoDetectPartition, + isPartialUpdate, dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); + } + @Override public LogicalProperties computeLogicalProperties() { return UnboundLogicalProperties.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index abc3c75fb207f5..4f782373e97bbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -413,6 +413,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.DropMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.FixedRangePartition; import org.apache.doris.nereids.trees.plans.commands.info.FuncNameInfo; +import org.apache.doris.nereids.trees.plans.commands.info.GeneratedColumnDesc; import org.apache.doris.nereids.trees.plans.commands.info.InPartition; import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition; @@ -2747,8 +2748,11 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { autoIncInitValue = Long.valueOf(1); } } + Optional desc = ctx.generatedExpr != null + ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) + : Optional.empty(); return new ColumnDefinition(colName, colType, isKey, aggType, nullableType, autoIncInitValue, defaultValue, - onUpdateDefaultValue, comment, true); + onUpdateDefaultValue, comment, desc); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 0c0f9e2e25f47d..f28640041cc6b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -276,28 +277,18 @@ private static Map getColumnToOutput( Map columnToOutput = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); NereidsParser expressionParser = new NereidsParser(); + List generatedColumns = Lists.newArrayList(); + List materializedViewColumn = Lists.newArrayList(); // generate slots not mentioned in sql, mv slots and shaded slots. for (Column column : boundSink.getTargetTable().getFullSchema()) { - if (column.isMaterializedViewColumn()) { - List refs = column.getRefColumns(); - // now we have to replace the column to slots. - Preconditions.checkArgument(refs != null, - "mv column %s 's ref column cannot be null", column); - Expression parsedExpression = expressionParser.parseExpression( - column.getDefineExpr().toSqlWithoutTbl()); - Expression boundSlotExpression = SlotReplacer.INSTANCE - .replace(parsedExpression, columnToOutput); - // the boundSlotExpression is an expression whose slots are bound but function - // may not be bound, we have to bind it again. - // for example: to_bitmap. - Expression boundExpression = FunctionBinder.INSTANCE.rewrite( - boundSlotExpression, new ExpressionRewriteContext(ctx.cascadesContext)); - if (boundExpression instanceof Alias) { - boundExpression = ((Alias) boundExpression).child(); - } - NamedExpression slot = new Alias(boundExpression, column.getDefineExpr().toSqlWithoutTbl()); - columnToOutput.put(column.getName(), slot); - } else if (columnToChildOutput.containsKey(column) + if (column.getGeneratedColumnInfo() != null) { + generatedColumns.add(column); + continue; + } else if (column.isMaterializedViewColumn()) { + materializedViewColumn.add(column); + continue; + } + if (columnToChildOutput.containsKey(column) // do not process explicitly use DEFAULT value here: // insert into table t values(DEFAULT) && !(columnToChildOutput.get(column) instanceof DefaultValueSlot)) { @@ -381,6 +372,40 @@ private static Map getColumnToOutput( } } } + for (Column column : generatedColumns) { + GeneratedColumnInfo info = column.getGeneratedColumnInfo(); + Expression parsedExpression = new NereidsParser().parseExpression(info.getExpr().toSqlWithoutTbl()); + Expression boundSlotExpression = SlotReplacer.INSTANCE.replace(parsedExpression, columnToOutput); + Expression boundExpression = FunctionBinder.INSTANCE.rewrite(boundSlotExpression, + new ExpressionRewriteContext(ctx.cascadesContext)); + if (boundExpression instanceof Alias) { + boundExpression = ((Alias) boundExpression).child(); + } + NamedExpression slot = new Alias(boundExpression, info.getExprSql()); + columnToOutput.put(column.getName(), slot); + } + for (Column column : materializedViewColumn) { + if (column.isMaterializedViewColumn()) { + List refs = column.getRefColumns(); + // now we have to replace the column to slots. + Preconditions.checkArgument(refs != null, + "mv column %s 's ref column cannot be null", column); + Expression parsedExpression = expressionParser.parseExpression( + column.getDefineExpr().toSqlWithoutTbl()); + Expression boundSlotExpression = SlotReplacer.INSTANCE + .replace(parsedExpression, columnToOutput); + // the boundSlotExpression is an expression whose slots are bound but function + // may not be bound, we have to bind it again. + // for example: to_bitmap. + Expression boundExpression = FunctionBinder.INSTANCE.rewrite( + boundSlotExpression, new ExpressionRewriteContext(ctx.cascadesContext)); + if (boundExpression instanceof Alias) { + boundExpression = ((Alias) boundExpression).child(); + } + NamedExpression slot = new Alias(boundExpression, column.getDefineExpr().toSqlWithoutTbl()); + columnToOutput.put(column.getName(), slot); + } + } return columnToOutput; } @@ -543,6 +568,9 @@ private Pair, Integer> bindTargetColumns(OlapTable table, List generatedColumnDesc = Optional.empty(); + private Set generatedColumnsThatReferToThis = new HashSet<>(); public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable, Optional defaultValue, String comment) { @@ -72,10 +74,11 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType } public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, - boolean isNullable, long autoIncInitValue, Optional defaultValue, - Optional onUpdateDefaultValue, String comment) { - this(name, type, isKey, aggType, isNullable, autoIncInitValue, defaultValue, onUpdateDefaultValue, - comment, true); + ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, + Optional onUpdateDefaultValue, String comment, + Optional generatedColumnDesc) { + this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, + comment, true, generatedColumnDesc); } /** @@ -116,7 +119,8 @@ private ColumnDefinition(String name, DataType type, boolean isKey, AggregateTyp */ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, - Optional onUpdateDefaultValue, String comment, boolean isVisible) { + Optional onUpdateDefaultValue, String comment, boolean isVisible, + Optional generatedColumnDesc) { this.name = name; this.type = type; this.isKey = isKey; @@ -127,6 +131,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; this.isVisible = isVisible; + this.generatedColumnDesc = generatedColumnDesc; } public ColumnDefinition(String name, DataType type, boolean isNullable) { @@ -404,6 +409,7 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO if (type.isTimeLikeType()) { throw new AnalysisException("Time type is not supported for olap table"); } + validateGeneratedColumnInfo(); } // from TypeDef.java analyze() @@ -650,7 +656,9 @@ public Column translateToCatalogStyle() { autoIncInitValue, defaultValue.map(DefaultValue::getRawValue).orElse(null), comment, isVisible, defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.map(DefaultValue::getValue).orElse(null), onUpdateDefaultValue.isPresent(), - onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId); + onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, + generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), + generatedColumnsThatReferToThis); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } @@ -686,4 +694,30 @@ public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregat Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false); } + public Optional getGeneratedColumnDesc() { + return generatedColumnDesc; + } + + public long getAutoIncInitValue() { + return autoIncInitValue; + } + + public void addGeneratedColumnsThatReferToThis(List list) { + generatedColumnsThatReferToThis.addAll(list); + } + + private void validateGeneratedColumnInfo() { + // for generated column + if (generatedColumnDesc.isPresent()) { + if (autoIncInitValue != -1) { + throw new AnalysisException("Generated columns cannot be auto_increment."); + } + if (defaultValue.isPresent()) { + throw new AnalysisException("Generated columns cannot have default value."); + } + if (onUpdateDefaultValue.isPresent()) { + throw new AnalysisException("Generated columns cannot have on update default value."); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index fc93b068e4b528..e23219f087f2d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -20,9 +20,11 @@ import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DistributionDesc; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.KeysDesc; import org.apache.doris.analysis.PartitionDesc; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; @@ -37,6 +39,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.Pair; import org.apache.doris.common.util.AutoBucketUtils; +import org.apache.doris.common.util.GeneratedColumnUtil; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PropertyAnalyzer; @@ -46,9 +49,30 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.glue.translator.ExpressionTranslator; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.parser.PartitionTableInfo; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.Variable; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; @@ -59,10 +83,12 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -512,7 +538,6 @@ public void validate(ConnectContext ctx) { engineName, columns, columnMap, properties, ctx, false, true); } } - // validate column try { if (!engineName.equals(ENGINE_ELASTICSEARCH) && columns.isEmpty()) { @@ -565,6 +590,7 @@ public void validate(ConnectContext ctx) { "same index columns have multiple same type index is not allowed."); } } + generatedColumnCheck(ctx); } private void paddingEngineName(String ctlName, ConnectContext ctx) { @@ -812,5 +838,209 @@ public CreateTableStmt translateToLegacyStmt() { public void setIsExternal(boolean isExternal) { this.isExternal = isExternal; } + + private void genreatedColumnCommonCheck() { + boolean hasGeneratedCol = false; + for (ColumnDefinition column : columns) { + if (column.getGeneratedColumnDesc().isPresent()) { + hasGeneratedCol = true; + break; + } + } + if (hasGeneratedCol && !engineName.equalsIgnoreCase("olap")) { + throw new AnalysisException("Tables can only have generated columns if the olap engine is used"); + } + if (hasGeneratedCol && keysType == KeysType.AGG_KEYS) { + throw new AnalysisException("Generated Column cannot be used in the aggregate table"); + } + } + + private void generatedColumnCheck(ConnectContext ctx) { + genreatedColumnCommonCheck(); + LogicalEmptyRelation plan = new LogicalEmptyRelation( + ConnectContext.get().getStatementContext().getNextRelationId(), + new ArrayList<>()); + CascadesContext cascadesContext = CascadesContext.initContext(ctx.getStatementContext(), plan, + PhysicalProperties.ANY); + Map columnToSlotReference = Maps.newHashMap(); + Map nameToColumnDefinition = Maps.newHashMap(); + Map translateMap = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + ColumnDefinition column = columns.get(i); + Slot slot = new SlotReference(column.getName(), column.getType()); + columnToSlotReference.put(column.getName(), slot); + nameToColumnDefinition.put(column.getName(), column); + SlotRef slotRef = new SlotRef(null, column.getName()); + slotRef.setType(column.getType().toCatalogDataType()); + translateMap.put(slot, new SlotRefAndIdx(slotRef, i, column.getGeneratedColumnDesc().isPresent())); + } + ExpressionToExpr.initializeslotRefMap(translateMap); + PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext(cascadesContext); + List slots = Lists.newArrayList(columnToSlotReference.values()); + List exprAndnames = Lists.newArrayList(); + for (int i = 0; i < columns.size(); i++) { + ColumnDefinition column = columns.get(i); + Optional info = column.getGeneratedColumnDesc(); + if (!info.isPresent()) { + continue; + } + Expression parsedExpression = info.get().getExpression(); + checkparsedExpressionInGeneratedColumn(parsedExpression); + Expression boundSlotExpression = SlotReplacer.INSTANCE.replace(parsedExpression, columnToSlotReference); + Scope scope = new Scope(slots); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope, cascadesContext, false, false); + Expression expr = analyzer.analyze(boundSlotExpression, new ExpressionRewriteContext(cascadesContext)); + checkExpressionInGeneratedColumn(expr, column, nameToColumnDefinition); + TypeCoercionUtils.checkCanCastTo(expr.getDataType(), column.getType()); + ExpressionToExpr translator = new ExpressionToExpr(i); + Expr e = expr.accept(translator, planTranslatorContext); + info.get().setExpr(e); + exprAndnames.add(new GeneratedColumnUtil.ExprAndname(e.clone(), column.getName())); + } + + // for alter drop column + Map> columnToListOfGeneratedColumnsThatReferToThis = new HashMap<>(); + for (ColumnDefinition column : columns) { + Optional info = column.getGeneratedColumnDesc(); + if (!info.isPresent()) { + continue; + } + Expr generatedExpr = info.get().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + generatedExpr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRefExpr : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRefExpr).getColumnName(); + columnToListOfGeneratedColumnsThatReferToThis + .computeIfAbsent(name, k -> new ArrayList<>()) + .add(column.getName()); + } + } + for (Map.Entry> entry : columnToListOfGeneratedColumnsThatReferToThis.entrySet()) { + ColumnDefinition col = nameToColumnDefinition.get(entry.getKey()); + col.addGeneratedColumnsThatReferToThis(entry.getValue()); + } + + // expand expr + GeneratedColumnUtil.rewriteColumns(exprAndnames); + for (GeneratedColumnUtil.ExprAndname exprAndname : exprAndnames) { + if (nameToColumnDefinition.containsKey(exprAndname.getName())) { + ColumnDefinition columnDefinition = nameToColumnDefinition.get(exprAndname.getName()); + Optional info = columnDefinition.getGeneratedColumnDesc(); + info.ifPresent(genCol -> genCol.setExpandExprForLoad(exprAndname.getExpr())); + } + } + } + + private static class SlotReplacer extends DefaultExpressionRewriter> { + public static final SlotReplacer INSTANCE = new SlotReplacer(); + + public Expression replace(Expression e, Map replaceMap) { + return e.accept(this, replaceMap); + } + + @Override + public Expression visitUnboundSlot(UnboundSlot unboundSlot, Map replaceMap) { + if (!replaceMap.containsKey(unboundSlot.getName())) { + throw new AnalysisException("Unknown column '" + unboundSlot.getName() + + "' in 'generated column function'"); + } + return replaceMap.get(unboundSlot.getName()); + } + } + + void checkparsedExpressionInGeneratedColumn(Expression expr) { + expr.foreach(e -> { + if (e instanceof SubqueryExpr) { + throw new AnalysisException("Generated column does not support subquery."); + } else if (e instanceof Lambda) { + throw new AnalysisException("Generated column does not support lambda."); + } + }); + } + + void checkExpressionInGeneratedColumn(Expression expr, ColumnDefinition column, + Map nameToColumnDefinition) { + expr.foreach(e -> { + if (e instanceof Variable) { + throw new AnalysisException("Generated column expression cannot contain variable."); + } else if (e instanceof Slot && nameToColumnDefinition.containsKey(((Slot) e).getName())) { + ColumnDefinition columnDefinition = nameToColumnDefinition.get(((Slot) e).getName()); + if (columnDefinition.getAutoIncInitValue() != -1) { + throw new AnalysisException( + "Generated column '" + column.getName() + + "' cannot refer to auto-increment column."); + } + } else if (e instanceof BoundFunction && !checkFunctionInGeneratedColumn((BoundFunction) e)) { + throw new AnalysisException("Expression of generated column '" + + column.getName() + "' contains a disallowed function:'" + + ((BoundFunction) e).getName() + "'"); + } + }); + } + + boolean checkFunctionInGeneratedColumn(Expression expr) { + if (!(expr instanceof ScalarFunction)) { + return false; + } + if (expr instanceof Udf) { + return false; + } + if (expr instanceof GroupingScalarFunction) { + return false; + } + return true; + } + + private static class ExpressionToExpr extends ExpressionTranslator { + private static Map slotRefMap; + private final int index; + + public ExpressionToExpr(int index) { + this.index = index; + } + + public static void initializeslotRefMap(Map map) { + slotRefMap = map; + } + + @Override + public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContext context) { + if (!slotRefMap.containsKey(slotReference)) { + throw new AnalysisException("Unknown column '" + slotReference.getName() + "'"); + } + int idx = slotRefMap.get(slotReference).getIdx(); + boolean isGenCol = slotRefMap.get(slotReference).isGeneratedColumn(); + if (isGenCol && idx >= index) { + throw new AnalysisException( + "Generated column can refer only to generated columns defined prior to it."); + } + return slotRefMap.get(slotReference).getSlotRef(); + } + } + + /** SlotRefAndIdx */ + public static class SlotRefAndIdx { + private final SlotRef slotRef; + private final int idx; + private final boolean isGeneratedColumn; + + public SlotRefAndIdx(SlotRef slotRef, int idx, boolean isGeneratedColumn) { + this.slotRef = slotRef; + this.idx = idx; + this.isGeneratedColumn = isGeneratedColumn; + } + + public int getIdx() { + return idx; + } + + public SlotRef getSlotRef() { + return slotRef; + } + + public boolean isGeneratedColumn() { + return isGeneratedColumn; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java new file mode 100644 index 00000000000000..6175389dcdd877 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.GeneratedColumnInfo; +import org.apache.doris.nereids.trees.expressions.Expression; + +/**GeneratedColumnDesc for nereids*/ +public class GeneratedColumnDesc { + /**GeneratedColumnType*/ + public enum GeneratedColumnType { + VIRTUAL, + STORED + } + + private final GeneratedColumnType type; + private final String exprSql; + private Expr expr; + private Expr expandExprForLoad; + private final Expression expression; + + /** constructor */ + public GeneratedColumnDesc(String exprSql, Expression expression) { + this.exprSql = exprSql; + this.expression = expression; + this.type = GeneratedColumnType.STORED; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + public void setExpandExprForLoad(Expr expandExprForLoad) { + this.expandExprForLoad = expandExprForLoad; + } + + public Expression getExpression() { + return expression; + } + + public GeneratedColumnInfo translateToInfo() { + return new GeneratedColumnInfo(exprSql, expr, expandExprForLoad); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index c452a242dcc46b..20f78edcdb8d9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; @@ -324,6 +325,12 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Unknown column '" + unboundLogicalSink.getColNames().get(i) + "' in target table."); } + if (sameNameColumn.getGeneratedColumnInfo() != null + && !(values.get(i) instanceof DefaultValueSlot)) { + throw new AnalysisException("The value specified for generated column '" + + sameNameColumn.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } if (values.get(i) instanceof DefaultValueSlot) { constantExprs.add(generateDefaultExpression(sameNameColumn)); } else { @@ -336,6 +343,12 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Column count doesn't match value count"); } for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).getGeneratedColumnInfo() != null + && !(values.get(i) instanceof DefaultValueSlot)) { + throw new AnalysisException("The value specified for generated column '" + + columns.get(i).getName() + + "' in table '" + table.getName() + "' is not allowed."); + } if (values.get(i) instanceof DefaultValueSlot) { constantExprs.add(generateDefaultExpression(columns.get(i))); } else { @@ -349,6 +362,31 @@ public static Plan normalizePlan(Plan plan, TableIf table) { StatementScopeIdGenerator.newRelationId(), constantExprs.build())); } List oneRowRelations = oneRowRelationBuilder.build(); + + // if (plan instanceof UnboundTableSink) { + // List newColNames = new ArrayList<>(); + // if (unboundLogicalSink.getColNames().isEmpty()) { + // for (Column column : table.getBaseSchema(false)) { + // if (column.getGeneratedColumnInfo() != null) { + // continue; + // } + // newColNames.add(column.getName()); + // } + // } else { + // for (String name : unboundLogicalSink.getColNames()) { + // Column column = table.getColumn(name); + // if (column.getGeneratedColumnInfo() != null) { + // continue; + // } + // newColNames.add(name); + // } + // } + // if (!unboundLogicalSink.getColNames().isEmpty() && newColNames.isEmpty()) { + // + // } + // plan = ((UnboundTableSink) plan).withColNames(newColNames); + // } + if (oneRowRelations.size() == 1) { return plan.withChildren(oneRowRelations.get(0)); } else { @@ -388,6 +426,10 @@ public static TableIf getTargetTable(Plan plan, ConnectContext ctx) { private static NamedExpression generateDefaultExpression(Column column) { try { + GeneratedColumnInfo generatedColumnInfo = column.getGeneratedColumnInfo(); + if (generatedColumnInfo != null) { + return new Alias(new NullLiteral(DataType.fromCatalogType(column.getType())), column.getName()); + } if (column.getDefaultValue() == null) { throw new AnalysisException("Column has no default value, column=" + column.getName()); } diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index a53cef89fa10fa..be16c0a6a00440 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -101,6 +101,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("alias", new Integer(SqlParserSymbols.KW_ALIAS)); keywordMap.put("all", new Integer(SqlParserSymbols.KW_ALL)); keywordMap.put("alter", new Integer(SqlParserSymbols.KW_ALTER)); + keywordMap.put("always", new Integer(SqlParserSymbols.KW_ALWAYS)); keywordMap.put("and", new Integer(SqlParserSymbols.KW_AND)); keywordMap.put("analyze", new Integer(SqlParserSymbols.KW_ANALYZE)); keywordMap.put("anti", new Integer(SqlParserSymbols.KW_ANTI)); @@ -253,6 +254,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("function", new Integer(SqlParserSymbols.KW_FUNCTION)); keywordMap.put("functions", new Integer(SqlParserSymbols.KW_FUNCTIONS)); keywordMap.put("type_cast", new Integer(SqlParserSymbols.KW_TYPECAST)); + keywordMap.put("generated", new Integer(SqlParserSymbols.KW_GENERATED)); keywordMap.put("generic", new Integer(SqlParserSymbols.KW_GENERIC)); keywordMap.put("global", new Integer(SqlParserSymbols.KW_GLOBAL)); keywordMap.put("grant", new Integer(SqlParserSymbols.KW_GRANT)); diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.csv b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.csv new file mode 100644 index 00000000000000..33a3e989bb3833 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.csv @@ -0,0 +1,2 @@ +1,2 +3,5 diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.json b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.json new file mode 100644 index 00000000000000..95fc8a7f4903e6 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data.json @@ -0,0 +1,4 @@ +[ +{"a":1,"b":2}, +{"a":3,"b":5} +] diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.out new file mode 100644 index 00000000000000..a14ff58ec18211 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.out @@ -0,0 +1,337 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_default -- +0 + +-- !common_default_insert -- +1 + +-- !common_default_insert_with_specific_column -- +1 + +-- !common_default_test_insert_default -- +1 + +-- !commont_default_select -- +1 2 3.0 +3 5 8.0 +6 7 13.0 + +-- !common_without_generated_always -- +0 + +-- !common_without_generated_always_insert -- +1 + +-- !common_without_generated_always_insert_with_specific_column -- +1 + +-- !commont_without_generated_always_select -- +1 2 3.0 +6 7 13.0 + +-- !gencol_in_middle -- +0 + +-- !gencol_in_middle_insert -- +1 + +-- !gencol_in_middle_insert_with_specific_column -- +1 + +-- !gencol_in_middle_insert_with_specific_column_2 -- +1 + +-- !gencol_in_middle_insert_multi_values -- +2 + +-- !gencol_in_middle_select -- +1 3.0 2 +1 6.0 5 +1 7.0 6 +3 7.0 4 +4 9.0 5 + +-- !gencol_empty_values -- +0 + +-- !gencol_empty_values_insert -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_refer_gencol -- +0 + +-- !gencol_refer_gencol_insert -- +1 + +-- !gencol_refer_gencol_insert2 -- +1 + +-- !gencol_refer_gencol_insert3 -- +1 + +-- !gencol_refer_gencol_insert4 -- +1 + +-- !gencol_refer_gencol_insert_multi_values -- +2 + +-- !gencol_refer_gencol_select -- +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +5 11.0 6 12 +5 12.0 7 13 + +-- !test_insert_mv -- +2 + +-- !test_select -- +1 2 9.0 11 +2 2 11.0 12 +3 9 20.0 23 +5 10 23.0 25 + +-- !gencol_array_function_create -- +0 + +-- !gencol_array_function_insert -- +1 + +-- !gencol_array_function_select -- +1 [1, 2] [3, 2] [3, 2, 1] + +-- !gencol_array_function_element_at_create -- +0 + +-- !gencol_array_function_element_at_insert -- +1 + +-- !gencol_array_function_element_at_select -- +1 [1, 2] [3, 2] 1 + +-- !create_table_like -- +0 + +-- !create_table_like_insert -- +1 + +-- !create_table_like_select -- +1 2 3.0 + +-- !refer_gencol_create_table_like -- +0 + +-- !refer_gencol_create_table_like_insert -- +1 + +-- !refer_gencol_create_table_like_select -- +1 3.0 2 4 + +-- !test_row_store -- +0 + +-- !row_store_insert -- +1 + +-- !row_store_select -- +1 6.0 5 7 + +-- !test_aes_encrypt -- +0 + +-- !test_aes_encrypt_insert -- +1 + +-- !test_aes_encrypt_select -- +1 2 CBDF2AA1377910BA93D9A86F33F6B924 + +-- !test_aes_encrypt_with_db -- +0 + +-- !test_aes_encrypt_insert_with_db -- +1 + +-- !test_aes_encrypt_select_with_db -- +1 2 CBDF2AA1377910BA93D9A86F33F6B924 + +-- !describe -- +a INT Yes true \N +c DOUBLE No false \N NONE,GENERATED +b INT Yes false \N NONE +d INT Yes false \N NONE,GENERATED + +-- !test_stream_load_common -- +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !test_stream_load_refer_gencol -- +1 3.0 2 4 +1 3.0 2 4 +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +3 8.0 5 9 +3 8.0 5 9 +5 11.0 6 12 +5 12.0 7 13 + +-- !test_stream_load_common_json -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !test_stream_load_refer_gencol_json -- +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +5 11.0 6 12 +5 12.0 7 13 + +-- !http_stream_load_common_ -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !http_stream_load_refer_gencol -- +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +5 11.0 6 12 +5 12.0 7 13 + +-- !test_mysql_load_common -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !test_mysql_load_refer_gencol -- +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +5 11.0 6 12 +5 12.0 7 13 + +-- !specify_value_for_gencol -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +2 3 5.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !specify_value_for_gencol -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +2 3 5.0 +2 3 5.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +6 7 13.0 + +-- !test_update -- +1 + +-- !test_update_generated_column -- +1 20 21 + +-- !gen_col_unique_key -- +0 + +-- !gen_col_unique_key_insert -- +1 + +-- !gen_col_unique_key_insert -- +1 + +-- !gen_col_unique_key_select -- +6 7 13 + +-- !gen_col_unique_key_insert_is_key -- +1 + +-- !gen_col_unique_key_insert_is_key -- +1 + +-- !gen_col_unique_key_select_is_key -- +6 7 7 + +-- !gencol_refer_gencol -- +0 + +-- !test_drop_column -- +3 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.out new file mode 100644 index 00000000000000..afe56855ba3e99 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.out @@ -0,0 +1,93 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_default -- +0 + +-- !common_without_generated_always -- +0 + +-- !gencol_in_middle -- +0 + +-- !gencol_refer_gencol -- +0 + +-- !gencol_array_function_create -- +0 + +-- !gencol_array_function_element_at_create -- +0 + +-- !common_default_insert -- +1 + +-- !common_default_insert_with_specific_column -- +1 + +-- !common_default_test_insert_default -- +1 + +-- !commont_default_select -- +1 2 3.0 +3 5 8.0 +6 7 13.0 + +-- !common_without_generated_always_insert -- +1 + +-- !common_without_generated_always_insert_with_specific_column -- +1 + +-- !commont_without_generated_always_select -- +1 2 3.0 +6 7 13.0 + +-- !gencol_in_middle_insert -- +1 + +-- !gencol_in_middle_insert_with_specific_column -- +1 + +-- !gencol_in_middle_insert_with_specific_column_2 -- +1 + +-- !gencol_in_middle_select -- +1 6.0 5 +1 7.0 6 +4 9.0 5 + +-- !gencol_refer_gencol_insert -- +1 + +-- !gencol_refer_gencol_insert2 -- +1 + +-- !gencol_refer_gencol_insert3 -- +1 + +-- !gencol_refer_gencol_insert4 -- +1 + +-- !gencol_refer_gencol_select -- +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +5 11.0 6 12 + +-- !gencol_array_function_insert -- +1 + +-- !gencol_array_function_select -- +1 [1, 2] [3, 2] [3, 2, 1] + +-- !gencol_array_function_element_at_insert -- +1 + +-- !gencol_array_function_element_at_select -- +1 [1, 2] [3, 2] 1 + +-- !gencol_refer_gencol -- +0 + +-- !test_drop_column -- +3 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.out new file mode 100644 index 00000000000000..7a4be75a056713 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_default -- +1 3.0 2 4 +3 8.0 5 9 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/three_column_gen_col_data.csv b/regression-test/data/ddl_p0/test_create_table_generated_column/three_column_gen_col_data.csv new file mode 100644 index 00000000000000..6de6d643c8ae81 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/three_column_gen_col_data.csv @@ -0,0 +1,2 @@ +2,3,4 +1,2,2 diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy new file mode 100644 index 00000000000000..cc211d1c7ffacc --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy @@ -0,0 +1,445 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + String db = context.config.getDbNameByFile(context.file) + + sql "drop table if exists test_gen_col_common" + qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_common_default_insert "INSERT INTO test_gen_col_common values(6,7,default);" + qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common(a,b) values(1,2);" + qt_common_default_test_insert_default "INSERT INTO test_gen_col_common values(3,5,default);" + + qt_commont_default_select "select * from test_gen_col_common order by 1,2,3;" + // qt_common_default_test_insert_null + test { + sql "INSERT INTO test_gen_col_common(a,b) values(1,null);" + exception "Insert has filtered data in strict mode." + } + + // qt_common_default_test_insert_gencol + test { + sql "INSERT INTO test_gen_col_common values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common' is not allowed." + } + + sql "drop table if exists test_gen_col_without_generated_always" + qt_common_without_generated_always """create table test_gen_col_without_generated_always(a int,b int,c double as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always values(6,7,default);" + qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always(a,b) values(1,2);" + + qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always order by 1,2,3;" + + sql "drop table if exists test_gen_col_in_middle" + qt_gencol_in_middle """create table test_gen_col_in_middle(a int,c double generated always as (abs(a+b)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gencol_in_middle_insert "insert into test_gen_col_in_middle values(1,default,5);" + qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle(a,b) values(4,5);" + qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle(a,b,c) values(1,6,default);" + qt_gencol_in_middle_insert_multi_values "insert into test_gen_col_in_middle values(1,default,2),(3,default,4);" + qt_gencol_in_middle_select "select * from test_gen_col_in_middle order by 1,2,3;" + + sql "drop table if exists test_gen_col_empty_values" + qt_gencol_empty_values """create table test_gen_col_empty_values(a int default 10,c double generated always as (abs(a+b)) not null,b int default 100) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gencol_empty_values_insert "insert into test_gen_col_empty_values values(),()" + qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(c) values(default),(default)" + qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(a,c) values(1,default),(3,default)" + qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(c,a) values(default,5),(default,6)" + + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol values(1,default,5,default);" + qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol(a,b) values(5,6);" + qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol(a,b,c) values(2,9,default);" + qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default);" + qt_gencol_refer_gencol_insert_multi_values "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default),(5,7,default,default);" + qt_gencol_refer_gencol_select "select * from gencol_refer_gencol order by 1,2,3,4;" + + sql "drop materialized view if exists test_mv_gen_col on gencol_refer_gencol" + createMV ("""create materialized view test_mv_gen_col as select a,sum(a),sum(c) ,sum(d) from gencol_refer_gencol group by a;""") + qt_test_insert_mv "insert into gencol_refer_gencol(a,b) values(1,2),(3,5)" + qt_test_select "select a,sum(a),sum(c),sum(d) from gencol_refer_gencol group by a order by 1,2,3,4" + explain{ + sql "select a,sum(a),sum(c),sum(d) from gencol_refer_gencol group by a" + contains "test_mv_gen_col" + } + + sql "drop table if exists test_gen_col_array_func" + qt_gencol_array_function_create """ + create table test_gen_col_array_func(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + qt_gencol_array_function_insert "insert into test_gen_col_array_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_select "select * from test_gen_col_array_func" + + sql "drop table if exists test_gen_col_element_at_func" + qt_gencol_array_function_element_at_create """ + create table test_gen_col_element_at_func(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func" + + sql "drop table if exists gencol_like_t;" + qt_create_table_like "create table gencol_like_t like test_gen_col_common;" + qt_create_table_like_insert "insert into gencol_like_t values(1,2,default);" + qt_create_table_like_select "select * from gencol_like_t" + sql "drop table if exists gencol_like_t;" + qt_refer_gencol_create_table_like "create table gencol_like_t like gencol_refer_gencol;" + qt_refer_gencol_create_table_like_insert "insert into gencol_like_t values(1,default,2,default);" + qt_refer_gencol_create_table_like_select "select * from gencol_like_t" + + sql "drop table if exists gencol_refer_gencol_row_store" + qt_test_row_store """create table gencol_refer_gencol_row_store(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1","store_row_column" = "true");""" + qt_row_store_insert "insert into gencol_refer_gencol_row_store values(1,default,5,default);" + qt_row_store_select "select * from gencol_refer_gencol_row_store" + + sql "drop ENCRYPTKEY if exists my_key" + sql """CREATE ENCRYPTKEY my_key AS "ABCD123456789";""" + sql "drop table if exists test_encrypt_key_gen_col" + qt_test_aes_encrypt """create table test_encrypt_key_gen_col(a int default 10, b int default 100, + c varchar(100) as(hex(aes_encrypt("abc",key my_key))));""" + qt_test_aes_encrypt_insert "insert into test_encrypt_key_gen_col values(1,2,default)" + qt_test_aes_encrypt_select "select * from test_encrypt_key_gen_col" + + sql "drop table if exists test_encrypt_key_gen_col_with_db" + qt_test_aes_encrypt_with_db """create table test_encrypt_key_gen_col_with_db(a int default 10, b int default 100, + c varchar(100) as(hex(aes_encrypt("abc",key ${db}.my_key))));""" + qt_test_aes_encrypt_insert_with_db "insert into test_encrypt_key_gen_col_with_db values(1,2,default)" + qt_test_aes_encrypt_select_with_db "select * from test_encrypt_key_gen_col_with_db" + + qt_describe "describe gencol_refer_gencol" + test { + sql """ + create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Can not found function 'abs' which has 2 arity." + } + + // gencol_has_sum + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Expression of generated column 'c' contains a disallowed function" + } + + // gencol_has_column_not_define + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Unknown column 'd' in 'generated column function'" + } + + // gencol_refer_gencol_after + test { + sql """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column can refer only to generated columns defined prior to it." + } + + sql "set @myvar=2" + // gencol_has_var + test { + sql """ + create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column expression cannot contain variable." + } + + test { + sql """ + create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) + distributed by hash(a) properties("replication_num" = "1"); + """ + exception "Generated column 'c' cannot refer to auto-increment column." + } + + test{ + sql """ + create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support subquery." + } + + test { + sql """ + create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support lambda." + } + + test { + sql """ + create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Cannot cast from ARRAY to numeric type" + } + + test { + sql """ + create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated Column cannot be used in the aggregate table" + } + + test { + sql """ + create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot be auto_increment." + } + + test { + sql """ + create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + + test { + sql """ + create table test_gen_col_on_update(a int,b int,c datetimev2 generated always as (CURRENT_TIMESTAMP) not null default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + test { + sql """ + create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'rank'" + } + + test { + sql """ + create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function" + } + // test load + streamLoad { + table 'test_gen_col_common' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common "select * from test_gen_col_common order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + + streamLoad { + table 'test_gen_col_common' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common_json "select * from test_gen_col_common order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol order by 1,2,3,4" + + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.test_gen_col_common(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_common_ "select * from test_gen_col_common order by 1,2,3" + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.gencol_refer_gencol(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + // mysql load + def filepath = getLoalFilePath "gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b); """ + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE gencol_refer_gencol + COLUMNS TERMINATED BY ',' + (a,b); + """ + sql "sync" + qt_test_mysql_load_common "select * from test_gen_col_common order by 1,2,3" + qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + + // specify value for generated column + sql "truncate table gencol_refer_gencol" + filepath = getLoalFilePath "three_column_gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b,c); + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" + + sql "truncate table gencol_refer_gencol" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b,tmp) + set (c=tmp+1) ; + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" + + //test update + sql "drop table if exists test_gen_col_update" + sql """create table test_gen_col_update (a int, b int, c int as (a+b)) + unique key(a) + distributed by hash(a) properties("replication_num"="1")""" + sql "insert into test_gen_col_update values(1,3,default)" + qt_test_update "update test_gen_col_update set b=20" + qt_test_update_generated_column "select * from test_gen_col_update" + + // test unique table, generated column is not key + sql "drop table if exists test_gen_col_unique_key" + qt_gen_col_unique_key """create table test_gen_col_unique_key(a int,b int,c int generated always as (abs(a+b)) not null) + unique key(a,b) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_gen_col_unique_key_insert "INSERT INTO test_gen_col_unique_key values(6,7,default);" + qt_gen_col_unique_key_insert "INSERT INTO test_gen_col_unique_key values(6,7,default);" + qt_gen_col_unique_key_select "select * from test_gen_col_unique_key order by 1,2,3;" + + // test unique table,generated column is key + sql "drop table if exists test_gen_col_unique_key2" + sql """create table test_gen_col_unique_key2(a int,c int generated always as (abs(a+1)) not null,b int) + unique key(a,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gen_col_unique_key_insert_is_key "INSERT INTO test_gen_col_unique_key2 values(6,default,7);" + qt_gen_col_unique_key_insert_is_key "INSERT INTO test_gen_col_unique_key2 values(6,default,7);" + qt_gen_col_unique_key_select_is_key "select * from test_gen_col_unique_key2" + + // test drop dependency + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "insert into gencol_refer_gencol(a,b) values(3,4)" + test { + sql "alter table gencol_refer_gencol drop column a" + exception "Column 'a' has a generated column dependency on :[c]" + } + test { + sql "alter table gencol_refer_gencol drop column c" + exception "Column 'c' has a generated column dependency on :[d]" + } + sql "alter table gencol_refer_gencol drop column d" + sql "alter table gencol_refer_gencol drop column c" + sql "alter table gencol_refer_gencol drop column b" + qt_test_drop_column "select * from gencol_refer_gencol" + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy new file mode 100644 index 00000000000000..7e072804298510 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_table_generated_column_legacy") { + // test legacy planner create + sql "SET enable_nereids_planner=false;" + sql "drop table if exists test_gen_col_common" + qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + sql "drop table if exists test_gen_col_without_generated_always" + qt_common_without_generated_always """create table test_gen_col_without_generated_always(a int,b int,c double as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + sql "drop table if exists test_gen_col_in_middle" + qt_gencol_in_middle """create table test_gen_col_in_middle(a int,c double generated always as (abs(a+b)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "drop table if exists test_gen_col_array_func" + qt_gencol_array_function_create """ + create table test_gen_col_array_func(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + sql "drop table if exists test_gen_col_element_at_func" + qt_gencol_array_function_element_at_create """ + create table test_gen_col_element_at_func(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + test { + sql """ + create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "No matching function with signature" + } + + // gencol_has_sum + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Expression of generated column 'c' contains a disallowed function" + } + + // gencol_has_column_not_define + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Unknown column 'd' in 'generated column function'" + } + + // gencol_refer_gencol_after + test { + sql """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column can refer only to generated columns defined prior to it." + } + + sql "set @myvar=2" + // gencol_has_var + test { + sql """ + create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column expression cannot contain variable." + } + + test { + sql """ + create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) + distributed by hash(a) properties("replication_num" = "1"); + """ + exception "Generated column 'c' cannot refer to auto-increment column." + } + + test{ + sql """ + create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support subquery." + } + + test { + sql """ + create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support lambda." + } + + test { + sql """ + create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "can not cast from origin type" + } + + test { + sql """ + create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed expression:'rank() OVER ()'" + } + test { + sql """ + create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'grouping'" + } + + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + qt_common_default_insert "INSERT INTO test_gen_col_common values(6,7,default);" + qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common(a,b) values(1,2);" + qt_common_default_test_insert_default "INSERT INTO test_gen_col_common values(3,5,default);" + qt_commont_default_select "select * from test_gen_col_common order by 1,2,3;" + + // qt_common_default_test_insert_null + test { + sql "INSERT INTO test_gen_col_common(a,b) values(1,null);" + exception "Insert has filtered data in strict mode." + } + + // qt_common_default_test_insert_gencol + test { + sql "INSERT INTO test_gen_col_common values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common' is not allowed." + } + + + qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always values(6,7,default);" + qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always(a,b) values(1,2);" + qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always order by 1,2,3;" + + + qt_gencol_in_middle_insert "insert into test_gen_col_in_middle values(1,default,5);" + qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle(a,b) values(4,5);" + qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle(a,b,c) values(1,6,default);" + qt_gencol_in_middle_select "select * from test_gen_col_in_middle order by 1,2,3;" + + + qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol values(1,default,5,default);" + qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol(a,b) values(5,6);" + qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol(a,b,c) values(2,9,default);" + qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default);" + qt_gencol_refer_gencol_select "select * from gencol_refer_gencol order by 1,2,3,4;" + + + qt_gencol_array_function_insert "insert into test_gen_col_array_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_select "select * from test_gen_col_array_func" + + + qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func" + + test { + sql """ + create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated Column cannot be used in the aggregate table" + } + + // test drop dependency + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "insert into gencol_refer_gencol(a,b) values(3,4)" + test { + sql "alter table gencol_refer_gencol drop column a" + exception "Column 'a' has a generated column dependency on :[c]" + } + test { + sql "alter table gencol_refer_gencol drop column c" + exception "Column 'c' has a generated column dependency on :[d]" + } + sql "alter table gencol_refer_gencol drop column d" + sql "alter table gencol_refer_gencol drop column c" + sql "alter table gencol_refer_gencol drop column b" + qt_test_drop_column "select * from gencol_refer_gencol" +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy new file mode 100644 index 00000000000000..1c6c7dd4803e41 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_routine_load_generated_column") { + // 1.需要准备kafka的输入数据文件 + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // define kafka + String topic = "testGenCol"; + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + def producer = new KafkaProducer<>(props) + filepath = getLoalFilePath "gen_col_data.csv" + def txt = new File("${filepath}").text + def lines = txt.readLines(); + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(topic, null, line) + producer.send(record) + } + def tableName = "gencol_refer_gencol" + sql "drop table if exists ${tableName}" + sql """create table ${tableName}(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + + sql """ + CREATE ROUTINE LOAD gen_col_routine_load ON ${tableName} + COLUMNS(a,b), + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${topic}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + while (true) { + sleep(1000) + def res = sql "show routine load for gen_col_routine_load" + def state = res[0][8].toString() + if (state == "NEED_SCHEDULE") { + continue; + } + log.info("reason of state changed: ${res[0][17].toString()}".toString()) + assertEquals(res[0][8].toString(), "RUNNING") + break; + } + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for gen_col_routine_load" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + + qt_common_default "select * from ${tableName} order by 1,2,3" + + sql "stop routine load for gen_col_routine_load;" + } + +} \ No newline at end of file From 71b1c8b3d8174c2c1192c88a079c365255b318a3 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 31 May 2024 14:47:26 +0800 Subject: [PATCH 02/21] [Feat](nereids) support generated column --- .../stream_load_and_mysql_load.out | 89 +++++++ .../test_generated_column_nereids.out | 205 +++++++++++++++ .../fault_tolerance_nereids.groovy | 159 +++++++++++ .../stream_load_and_mysql_load.groovy | 144 ++++++++++ ...y => test_generated_column_nereids.groovy} | 246 ------------------ .../test_routine_load_generated_column.groovy | 1 - 6 files changed, 597 insertions(+), 247 deletions(-) create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy rename regression-test/suites/ddl_p0/test_create_table_generated_column/{test_create_table_generated_column.groovy => test_generated_column_nereids.groovy} (57%) diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out new file mode 100644 index 00000000000000..469cafafc41ba3 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out @@ -0,0 +1,89 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_default -- +0 + +-- !gencol_refer_gencol -- +0 + +-- !test_stream_load_common -- +1 2 3.0 +3 5 8.0 + +-- !test_stream_load_refer_gencol -- +1 3.0 2 4 +3 8.0 5 9 + +-- !test_stream_load_common_json -- +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 + +-- !test_stream_load_refer_gencol_json -- +1 3.0 2 4 +1 3.0 2 4 +3 8.0 5 9 +3 8.0 5 9 + +-- !http_stream_load_common_ -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 + +-- !http_stream_load_refer_gencol -- +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 + +-- !test_mysql_load_common -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 + +-- !test_mysql_load_refer_gencol -- +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +1 3.0 2 4 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 +3 8.0 5 9 + +-- !specify_value_for_gencol -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +2 3 5.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 + +-- !specify_value_for_gencol -- +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +1 2 3.0 +2 3 5.0 +2 3 5.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 +3 5 8.0 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out new file mode 100644 index 00000000000000..f18231ae9bd09b --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out @@ -0,0 +1,205 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !common_default -- +0 + +-- !common_default_insert -- +1 + +-- !common_default_insert_with_specific_column -- +1 + +-- !common_default_test_insert_default -- +1 + +-- !commont_default_select -- +1 2 3.0 +3 5 8.0 +6 7 13.0 + +-- !common_without_generated_always -- +0 + +-- !common_without_generated_always_insert -- +1 + +-- !common_without_generated_always_insert_with_specific_column -- +1 + +-- !commont_without_generated_always_select -- +1 2 3.0 +6 7 13.0 + +-- !gencol_in_middle -- +0 + +-- !gencol_in_middle_insert -- +1 + +-- !gencol_in_middle_insert_with_specific_column -- +1 + +-- !gencol_in_middle_insert_with_specific_column_2 -- +1 + +-- !gencol_in_middle_insert_multi_values -- +2 + +-- !gencol_in_middle_select -- +1 3.0 2 +1 6.0 5 +1 7.0 6 +3 7.0 4 +4 9.0 5 + +-- !gencol_empty_values -- +0 + +-- !gencol_empty_values_insert -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_empty_values_insert_2 -- +2 + +-- !gencol_refer_gencol -- +0 + +-- !gencol_refer_gencol_insert -- +1 + +-- !gencol_refer_gencol_insert2 -- +1 + +-- !gencol_refer_gencol_insert3 -- +1 + +-- !gencol_refer_gencol_insert4 -- +1 + +-- !gencol_refer_gencol_insert_multi_values -- +2 + +-- !gencol_refer_gencol_select -- +1 6.0 5 7 +2 11.0 9 12 +3 6.0 3 7 +3 6.0 3 7 +5 11.0 6 12 +5 12.0 7 13 + +-- !test_insert_mv -- +2 + +-- !test_select -- +1 2 9.0 11 +2 2 11.0 12 +3 9 20.0 23 +5 10 23.0 25 + +-- !gencol_array_function_create -- +0 + +-- !gencol_array_function_insert -- +1 + +-- !gencol_array_function_select -- +1 [1, 2] [3, 2] [3, 2, 1] + +-- !gencol_array_function_element_at_create -- +0 + +-- !gencol_array_function_element_at_insert -- +1 + +-- !gencol_array_function_element_at_select -- +1 [1, 2] [3, 2] 1 + +-- !create_table_like -- +0 + +-- !create_table_like_insert -- +1 + +-- !create_table_like_select -- +1 2 3.0 + +-- !refer_gencol_create_table_like -- +0 + +-- !refer_gencol_create_table_like_insert -- +1 + +-- !refer_gencol_create_table_like_select -- +1 3.0 2 4 + +-- !test_row_store -- +0 + +-- !row_store_insert -- +1 + +-- !row_store_select -- +1 6.0 5 7 + +-- !test_aes_encrypt -- +0 + +-- !test_aes_encrypt_insert -- +1 + +-- !test_aes_encrypt_select -- +1 2 CBDF2AA1377910BA93D9A86F33F6B924 + +-- !test_aes_encrypt_with_db -- +0 + +-- !test_aes_encrypt_insert_with_db -- +1 + +-- !test_aes_encrypt_select_with_db -- +1 2 CBDF2AA1377910BA93D9A86F33F6B924 + +-- !describe -- +a INT Yes true \N +c DOUBLE No false \N NONE,GENERATED +b INT Yes false \N NONE +d INT Yes false \N NONE,GENERATED + +-- !test_update -- +1 + +-- !test_update_generated_column -- +1 20 21 + +-- !gen_col_unique_key -- +0 + +-- !gen_col_unique_key_insert -- +1 + +-- !gen_col_unique_key_insert -- +1 + +-- !gen_col_unique_key_select -- +6 7 13 + +-- !gen_col_unique_key_insert_is_key -- +1 + +-- !gen_col_unique_key_insert_is_key -- +1 + +-- !gen_col_unique_key_select_is_key -- +6 7 7 + +-- !gencol_refer_gencol -- +0 + +-- !test_drop_column -- +3 + diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy new file mode 100644 index 00000000000000..259165efea60d5 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_fault_tolerance_nereids") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + test { + sql """ + create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Can not found function 'abs' which has 2 arity." + } + + // gencol_has_sum + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Expression of generated column 'c' contains a disallowed function" + } + + // gencol_has_column_not_define + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Unknown column 'd' in 'generated column function'" + } + + // gencol_refer_gencol_after + test { + sql """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column can refer only to generated columns defined prior to it." + } + + sql "set @myvar=2" + // gencol_has_var + test { + sql """ + create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column expression cannot contain variable." + } + + test { + sql """ + create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) + distributed by hash(a) properties("replication_num" = "1"); + """ + exception "Generated column 'c' cannot refer to auto-increment column." + } + + test{ + sql """ + create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support subquery." + } + + test { + sql """ + create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support lambda." + } + + test { + sql """ + create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Cannot cast from ARRAY to numeric type" + } + + test { + sql """ + create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated Column cannot be used in the aggregate table" + } + + test { + sql """ + create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot be auto_increment." + } + + test { + sql """ + create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + + test { + sql """ + create table test_gen_col_on_update(a int,b int,c datetimev2 generated always as (CURRENT_TIMESTAMP) not null default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + test { + sql """ + create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'rank'" + } + + test { + sql """ + create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function" + } +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy new file mode 100644 index 00000000000000..f384aeae2a8d37 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_stream_mysql_load") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + String db = context.config.getDbNameByFile(context.file) + + sql "drop table if exists test_gen_col_common" + qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + + // test load + streamLoad { + table 'test_gen_col_common' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common "select * from test_gen_col_common order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + + streamLoad { + table 'test_gen_col_common' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common_json "select * from test_gen_col_common order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol order by 1,2,3,4" + + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.test_gen_col_common(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_common_ "select * from test_gen_col_common order by 1,2,3" + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.gencol_refer_gencol(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + // mysql load + def filepath = getLoalFilePath "gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b); """ + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE gencol_refer_gencol + COLUMNS TERMINATED BY ',' + (a,b); + """ + sql "sync" + qt_test_mysql_load_common "select * from test_gen_col_common order by 1,2,3" + qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + + // specify value for generated column + sql "truncate table gencol_refer_gencol" + filepath = getLoalFilePath "three_column_gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b,c); + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" + + sql "truncate table gencol_refer_gencol" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common + COLUMNS TERMINATED BY ',' + (a,b,tmp) + set (c=tmp+1) ; + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy similarity index 57% rename from regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy rename to regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index cc211d1c7ffacc..d2fa8a7dded615 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -144,252 +144,6 @@ suite("test_generated_column") { qt_test_aes_encrypt_select_with_db "select * from test_encrypt_key_gen_col_with_db" qt_describe "describe gencol_refer_gencol" - test { - sql """ - create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Can not found function 'abs' which has 2 arity." - } - - // gencol_has_sum - test { - sql """ - create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Expression of generated column 'c' contains a disallowed function" - } - - // gencol_has_column_not_define - test { - sql """ - create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Unknown column 'd' in 'generated column function'" - } - - // gencol_refer_gencol_after - test { - sql """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated column can refer only to generated columns defined prior to it." - } - - sql "set @myvar=2" - // gencol_has_var - test { - sql """ - create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated column expression cannot contain variable." - } - - test { - sql """ - create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) - distributed by hash(a) properties("replication_num" = "1"); - """ - exception "Generated column 'c' cannot refer to auto-increment column." - } - - test{ - sql """ - create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated column does not support subquery." - } - - test { - sql """ - create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) - DISTRIBUTED BY HASH(pk) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated column does not support lambda." - } - - test { - sql """ - create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) - DISTRIBUTED BY HASH(pk) - PROPERTIES("replication_num" = "1"); - """ - exception "Cannot cast from ARRAY to numeric type" - } - - test { - sql """ - create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) - aggregate key(a,b,c) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated Column cannot be used in the aggregate table" - } - - test { - sql """ - create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) - aggregate key(a,b,c) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated columns cannot be auto_increment." - } - - test { - sql """ - create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) - aggregate key(a,b,c) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated columns cannot have default value." - } - - test { - sql """ - create table test_gen_col_on_update(a int,b int,c datetimev2 generated always as (CURRENT_TIMESTAMP) not null default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ) - aggregate key(a,b,c) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1"); - """ - exception "Generated columns cannot have default value." - } - test { - sql """ - create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1");""" - exception "Expression of generated column 'c' contains a disallowed function:'rank'" - } - - test { - sql """ - create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) - DISTRIBUTED BY HASH(a) - PROPERTIES("replication_num" = "1");""" - exception "Expression of generated column 'c' contains a disallowed function" - } - // test load - streamLoad { - table 'test_gen_col_common' - set 'column_separator', ',' - file 'gen_col_data.csv' - set 'columns', 'a,b' - time 10000 // limit inflight 10s - } - sql "sync" - qt_test_stream_load_common "select * from test_gen_col_common order by 1,2,3" - - streamLoad { - table 'gencol_refer_gencol' - set 'column_separator', ',' - file 'gen_col_data.csv' - set 'columns', 'a,b' - time 10000 // limit inflight 10s - } - sql "sync" - qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" - - streamLoad { - table 'test_gen_col_common' - set 'strip_outer_array', 'true' - set 'format', 'json' - set 'columns', 'a, b' - file 'gen_col_data.json' - set 'jsonpaths', '["$.a", "$.b"]' - time 10000 // limit inflight 10s - } - sql "sync" - qt_test_stream_load_common_json "select * from test_gen_col_common order by 1,2,3" - - streamLoad { - table 'gencol_refer_gencol' - set 'strip_outer_array', 'true' - set 'format', 'json' - set 'columns', 'a, b' - file 'gen_col_data.json' - set 'jsonpaths', '["$.a", "$.b"]' - time 10000 // limit inflight 10s - } - sql "sync" - qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol order by 1,2,3,4" - - streamLoad { - set 'version', '1' - file 'gen_col_data.csv' - time 10000 - set 'sql',""" - insert into ${db}.test_gen_col_common(a,b) select * from http_stream("format"="csv", "column_separator"=",") - """ - } - sql "sync" - qt_http_stream_load_common_ "select * from test_gen_col_common order by 1,2,3" - streamLoad { - set 'version', '1' - file 'gen_col_data.csv' - time 10000 - set 'sql',""" - insert into ${db}.gencol_refer_gencol(a,b) select * from http_stream("format"="csv", "column_separator"=",") - """ - } - sql "sync" - qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" - // mysql load - def filepath = getLoalFilePath "gen_col_data.csv" - sql """ - LOAD DATA LOCAL - INFILE '${filepath}' - INTO TABLE test_gen_col_common - COLUMNS TERMINATED BY ',' - (a,b); """ - sql """ - LOAD DATA LOCAL - INFILE '${filepath}' - INTO TABLE gencol_refer_gencol - COLUMNS TERMINATED BY ',' - (a,b); - """ - sql "sync" - qt_test_mysql_load_common "select * from test_gen_col_common order by 1,2,3" - qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" - - // specify value for generated column - sql "truncate table gencol_refer_gencol" - filepath = getLoalFilePath "three_column_gen_col_data.csv" - sql """ - LOAD DATA LOCAL - INFILE '${filepath}' - INTO TABLE test_gen_col_common - COLUMNS TERMINATED BY ',' - (a,b,c); - """ - sql "sync" - qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" - - sql "truncate table gencol_refer_gencol" - sql """ - LOAD DATA LOCAL - INFILE '${filepath}' - INTO TABLE test_gen_col_common - COLUMNS TERMINATED BY ',' - (a,b,tmp) - set (c=tmp+1) ; - """ - sql "sync" - qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" //test update sql "drop table if exists test_gen_col_update" diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy index 1c6c7dd4803e41..76fc0330dc3da6 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy @@ -20,7 +20,6 @@ import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig suite("test_routine_load_generated_column") { - // 1.需要准备kafka的输入数据文件 String enabled = context.config.otherConfigs.get("enableKafkaTest") String kafka_port = context.config.otherConfigs.get("kafka_port") String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") From 9bd11036685cb0f7638cce9f762245c68eb5bba7 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 31 May 2024 17:03:32 +0800 Subject: [PATCH 03/21] [Feat](nereids) support generated column --- .../test_generated_column_nereids.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out index f18231ae9bd09b..d1099b55760cc1 100644 --- a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out @@ -60,10 +60,10 @@ -- !gencol_empty_values_insert_2 -- 2 --- !gencol_empty_values_insert_2 -- +-- !gencol_empty_values_insert_3 -- 2 --- !gencol_empty_values_insert_2 -- +-- !gencol_empty_values_insert_4 -- 2 -- !gencol_refer_gencol -- From ef0ef4a1eea930c95e6adbc85d676545031da4c7 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 31 May 2024 17:07:43 +0800 Subject: [PATCH 04/21] [Feat](nereids) support generated column --- .../test_generated_column_nereids.groovy | 4 ++-- .../test_routine_load_generated_column.groovy | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index d2fa8a7dded615..d6deab18c71de5 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -68,8 +68,8 @@ suite("test_generated_column") { PROPERTIES("replication_num" = "1");""" qt_gencol_empty_values_insert "insert into test_gen_col_empty_values values(),()" qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(c) values(default),(default)" - qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(a,c) values(1,default),(3,default)" - qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(c,a) values(default,5),(default,6)" + qt_gencol_empty_values_insert_3 "insert into test_gen_col_empty_values(a,c) values(1,default),(3,default)" + qt_gencol_empty_values_insert_4 "insert into test_gen_col_empty_values(c,a) values(default,5),(default,6)" sql "drop table if exists gencol_refer_gencol" qt_gencol_refer_gencol """ diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy index 76fc0330dc3da6..f87df5bd72a6f3 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy @@ -58,7 +58,6 @@ suite("test_routine_load_generated_column") { "property.kafka_default_offsets" = "OFFSET_BEGINNING" ); """ - sql "sync" while (true) { sleep(1000) def res = sql "show routine load for gen_col_routine_load" From 0409c3f0208b061f6d7de2764b9b95887428249e Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 31 May 2024 20:50:55 +0800 Subject: [PATCH 05/21] [Feat](nereids) support generated column --- .../stream_load_and_mysql_load.groovy | 52 +++++------ ...reate_table_generated_column_legacy.groovy | 92 +++++++++---------- .../test_routine_load_generated_column.groovy | 2 +- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy index f384aeae2a8d37..17fda59bfaeea5 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy @@ -20,42 +20,42 @@ suite("test_generated_column_stream_mysql_load") { sql "SET enable_fallback_to_original_planner=false;" String db = context.config.getDbNameByFile(context.file) - sql "drop table if exists test_gen_col_common" - qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + sql "drop table if exists test_gen_col_common_steam_mysql_load" + qt_common_default """create table test_gen_col_common_steam_mysql_load(a int,b int,c double generated always as (abs(a+b)) not null) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); ;""" - sql "drop table if exists gencol_refer_gencol" + sql "drop table if exists gencol_refer_gencol_steam_mysql_load" qt_gencol_refer_gencol """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + create table gencol_refer_gencol_steam_mysql_load(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ // test load streamLoad { - table 'test_gen_col_common' + table 'test_gen_col_common_steam_mysql_load' set 'column_separator', ',' file 'gen_col_data.csv' set 'columns', 'a,b' time 10000 // limit inflight 10s } sql "sync" - qt_test_stream_load_common "select * from test_gen_col_common order by 1,2,3" + qt_test_stream_load_common "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" streamLoad { - table 'gencol_refer_gencol' + table 'gencol_refer_gencol_steam_mysql_load' set 'column_separator', ',' file 'gen_col_data.csv' set 'columns', 'a,b' time 10000 // limit inflight 10s } sql "sync" - qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" streamLoad { - table 'test_gen_col_common' + table 'test_gen_col_common_steam_mysql_load' set 'strip_outer_array', 'true' set 'format', 'json' set 'columns', 'a, b' @@ -64,10 +64,10 @@ suite("test_generated_column_stream_mysql_load") { time 10000 // limit inflight 10s } sql "sync" - qt_test_stream_load_common_json "select * from test_gen_col_common order by 1,2,3" + qt_test_stream_load_common_json "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" streamLoad { - table 'gencol_refer_gencol' + table 'gencol_refer_gencol_steam_mysql_load' set 'strip_outer_array', 'true' set 'format', 'json' set 'columns', 'a, b' @@ -76,69 +76,69 @@ suite("test_generated_column_stream_mysql_load") { time 10000 // limit inflight 10s } sql "sync" - qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol order by 1,2,3,4" + qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" streamLoad { set 'version', '1' file 'gen_col_data.csv' time 10000 set 'sql',""" - insert into ${db}.test_gen_col_common(a,b) select * from http_stream("format"="csv", "column_separator"=",") + insert into ${db}.test_gen_col_common_steam_mysql_load(a,b) select * from http_stream("format"="csv", "column_separator"=",") """ } sql "sync" - qt_http_stream_load_common_ "select * from test_gen_col_common order by 1,2,3" + qt_http_stream_load_common_ "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" streamLoad { set 'version', '1' file 'gen_col_data.csv' time 10000 set 'sql',""" - insert into ${db}.gencol_refer_gencol(a,b) select * from http_stream("format"="csv", "column_separator"=",") + insert into ${db}.gencol_refer_gencol_steam_mysql_load(a,b) select * from http_stream("format"="csv", "column_separator"=",") """ } sql "sync" - qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" // mysql load def filepath = getLoalFilePath "gen_col_data.csv" sql """ LOAD DATA LOCAL INFILE '${filepath}' - INTO TABLE test_gen_col_common + INTO TABLE test_gen_col_common_steam_mysql_load COLUMNS TERMINATED BY ',' (a,b); """ sql """ LOAD DATA LOCAL INFILE '${filepath}' - INTO TABLE gencol_refer_gencol + INTO TABLE gencol_refer_gencol_steam_mysql_load COLUMNS TERMINATED BY ',' (a,b); """ sql "sync" - qt_test_mysql_load_common "select * from test_gen_col_common order by 1,2,3" - qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol order by 1,2,3,4" + qt_test_mysql_load_common "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" // specify value for generated column - sql "truncate table gencol_refer_gencol" + sql "truncate table gencol_refer_gencol_steam_mysql_load" filepath = getLoalFilePath "three_column_gen_col_data.csv" sql """ LOAD DATA LOCAL INFILE '${filepath}' - INTO TABLE test_gen_col_common + INTO TABLE test_gen_col_common_steam_mysql_load COLUMNS TERMINATED BY ',' (a,b,c); """ sql "sync" - qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" + qt_specify_value_for_gencol "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" - sql "truncate table gencol_refer_gencol" + sql "truncate table gencol_refer_gencol_steam_mysql_load" sql """ LOAD DATA LOCAL INFILE '${filepath}' - INTO TABLE test_gen_col_common + INTO TABLE test_gen_col_common_steam_mysql_load COLUMNS TERMINATED BY ',' (a,b,tmp) set (c=tmp+1) ; """ sql "sync" - qt_specify_value_for_gencol "select * from test_gen_col_common order by 1,2,3" + qt_specify_value_for_gencol "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" } \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy index 7e072804298510..5da23ed6cc809e 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy @@ -18,36 +18,36 @@ suite("test_create_table_generated_column_legacy") { // test legacy planner create sql "SET enable_nereids_planner=false;" - sql "drop table if exists test_gen_col_common" - qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + sql "drop table if exists test_gen_col_common_legacy" + qt_common_default """create table test_gen_col_common_legacy(a int,b int,c double generated always as (abs(a+b)) not null) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); ;""" - sql "drop table if exists test_gen_col_without_generated_always" - qt_common_without_generated_always """create table test_gen_col_without_generated_always(a int,b int,c double as (abs(a+b)) not null) + sql "drop table if exists test_gen_col_without_generated_always_legacy" + qt_common_without_generated_always """create table test_gen_col_without_generated_always_legacy(a int,b int,c double as (abs(a+b)) not null) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); ;""" - sql "drop table if exists test_gen_col_in_middle" - qt_gencol_in_middle """create table test_gen_col_in_middle(a int,c double generated always as (abs(a+b)) not null,b int) + sql "drop table if exists test_gen_col_in_middle_legacy" + qt_gencol_in_middle """create table test_gen_col_in_middle_legacy(a int,c double generated always as (abs(a+b)) not null,b int) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1");""" - sql "drop table if exists gencol_refer_gencol" + sql "drop table if exists gencol_refer_gencol_legacy" qt_gencol_refer_gencol """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - sql "drop table if exists test_gen_col_array_func" + sql "drop table if exists test_gen_col_array_func_legacy" qt_gencol_array_function_create """ - create table test_gen_col_array_func(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) + create table test_gen_col_array_func_legacy(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) DISTRIBUTED BY HASH(pk) PROPERTIES("replication_num" = "1"); ; """ - sql "drop table if exists test_gen_col_element_at_func" + sql "drop table if exists test_gen_col_element_at_func_legacy" qt_gencol_array_function_element_at_create """ - create table test_gen_col_element_at_func(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) + create table test_gen_col_element_at_func_legacy(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) DISTRIBUTED BY HASH(pk) PROPERTIES("replication_num" = "1"); ; @@ -84,7 +84,7 @@ suite("test_create_table_generated_column_legacy") { // gencol_refer_gencol_after test { sql """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ @@ -130,7 +130,7 @@ suite("test_create_table_generated_column_legacy") { test { sql """ - create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) + create table test_gen_col_array_func_legacy(pk int,a array,b array, c double generated always as (a+b) not null) DISTRIBUTED BY HASH(pk) PROPERTIES("replication_num" = "1"); """ @@ -153,48 +153,48 @@ suite("test_create_table_generated_column_legacy") { sql "SET enable_nereids_planner=true;" sql "SET enable_fallback_to_original_planner=false;" - qt_common_default_insert "INSERT INTO test_gen_col_common values(6,7,default);" - qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common(a,b) values(1,2);" - qt_common_default_test_insert_default "INSERT INTO test_gen_col_common values(3,5,default);" - qt_commont_default_select "select * from test_gen_col_common order by 1,2,3;" + qt_common_default_insert "INSERT INTO test_gen_col_common_legacy values(6,7,default);" + qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common_legacy(a,b) values(1,2);" + qt_common_default_test_insert_default "INSERT INTO test_gen_col_common_legacy values(3,5,default);" + qt_commont_default_select "select * from test_gen_col_common_legacy order by 1,2,3;" // qt_common_default_test_insert_null test { - sql "INSERT INTO test_gen_col_common(a,b) values(1,null);" + sql "INSERT INTO test_gen_col_common_legacy(a,b) values(1,null);" exception "Insert has filtered data in strict mode." } // qt_common_default_test_insert_gencol test { - sql "INSERT INTO test_gen_col_common values(1,2,3);" - exception "The value specified for generated column 'c' in table 'test_gen_col_common' is not allowed." + sql "INSERT INTO test_gen_col_common_legacy values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_legacy' is not allowed." } - qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always values(6,7,default);" - qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always(a,b) values(1,2);" - qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always order by 1,2,3;" + qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always_legacy values(6,7,default);" + qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always_legacy(a,b) values(1,2);" + qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always_legacy order by 1,2,3;" - qt_gencol_in_middle_insert "insert into test_gen_col_in_middle values(1,default,5);" - qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle(a,b) values(4,5);" - qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle(a,b,c) values(1,6,default);" - qt_gencol_in_middle_select "select * from test_gen_col_in_middle order by 1,2,3;" + qt_gencol_in_middle_insert "insert into test_gen_col_in_middle_legacy values(1,default,5);" + qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle_legacy(a,b) values(4,5);" + qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle_legacy(a,b,c) values(1,6,default);" + qt_gencol_in_middle_select "select * from test_gen_col_in_middle_legacy order by 1,2,3;" - qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol values(1,default,5,default);" - qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol(a,b) values(5,6);" - qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol(a,b,c) values(2,9,default);" - qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default);" - qt_gencol_refer_gencol_select "select * from gencol_refer_gencol order by 1,2,3,4;" + qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol_legacy values(1,default,5,default);" + qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol_legacy(a,b) values(5,6);" + qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol_legacy(a,b,c) values(2,9,default);" + qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol_legacy(a,b,c,d) values(3,3,default,default);" + qt_gencol_refer_gencol_select "select * from gencol_refer_gencol_legacy order by 1,2,3,4;" - qt_gencol_array_function_insert "insert into test_gen_col_array_func values(1,[1,2],[3,2],default);" - qt_gencol_array_function_select "select * from test_gen_col_array_func" + qt_gencol_array_function_insert "insert into test_gen_col_array_func_legacy values(1,[1,2],[3,2],default);" + qt_gencol_array_function_select "select * from test_gen_col_array_func_legacy" - qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func values(1,[1,2],[3,2],default);" - qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func" + qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func_legacy values(1,[1,2],[3,2],default);" + qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func_legacy" test { sql """ @@ -207,23 +207,23 @@ suite("test_create_table_generated_column_legacy") { } // test drop dependency - sql "drop table if exists gencol_refer_gencol" + sql "drop table if exists gencol_refer_gencol_legacy" qt_gencol_refer_gencol """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - sql "insert into gencol_refer_gencol(a,b) values(3,4)" + sql "insert into gencol_refer_gencol_legacy(a,b) values(3,4)" test { - sql "alter table gencol_refer_gencol drop column a" + sql "alter table gencol_refer_gencol_legacy drop column a" exception "Column 'a' has a generated column dependency on :[c]" } test { - sql "alter table gencol_refer_gencol drop column c" + sql "alter table gencol_refer_gencol_legacy drop column c" exception "Column 'c' has a generated column dependency on :[d]" } - sql "alter table gencol_refer_gencol drop column d" - sql "alter table gencol_refer_gencol drop column c" - sql "alter table gencol_refer_gencol drop column b" - qt_test_drop_column "select * from gencol_refer_gencol" + sql "alter table gencol_refer_gencol_legacy drop column d" + sql "alter table gencol_refer_gencol_legacy drop column c" + sql "alter table gencol_refer_gencol_legacy drop column b" + qt_test_drop_column "select * from gencol_refer_gencol_legacy" } \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy index f87df5bd72a6f3..edf8ab2532345a 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy @@ -40,7 +40,7 @@ suite("test_routine_load_generated_column") { def record = new ProducerRecord<>(topic, null, line) producer.send(record) } - def tableName = "gencol_refer_gencol" + def tableName = "gencol_refer_gencol_rload" sql "drop table if exists ${tableName}" sql """create table ${tableName}(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) From f2b99f5e32ee8c0d7ff353bb924992b610da981c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 2 Jun 2024 12:21:00 +0800 Subject: [PATCH 06/21] [Feat](nereids) support generated column --- .../test_generated_column_nereids.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index d6deab18c71de5..9a60e71d3b852c 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -139,7 +139,7 @@ suite("test_generated_column") { sql "drop table if exists test_encrypt_key_gen_col_with_db" qt_test_aes_encrypt_with_db """create table test_encrypt_key_gen_col_with_db(a int default 10, b int default 100, - c varchar(100) as(hex(aes_encrypt("abc",key ${db}.my_key))));""" + c varchar(100) as(hex(aes_encrypt("abc",key ${db}.my_key)))) distributed by hash(a) properties("replication_num"="1");""" qt_test_aes_encrypt_insert_with_db "insert into test_encrypt_key_gen_col_with_db values(1,2,default)" qt_test_aes_encrypt_select_with_db "select * from test_encrypt_key_gen_col_with_db" From fe4ba69ec30f4ca9e5d40066fac34cc831657ab5 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 2 Jun 2024 22:26:46 +0800 Subject: [PATCH 07/21] [Feat](nereids) support generated column --- .../test_generated_column_nereids.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index 9a60e71d3b852c..9548e15beea935 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -133,7 +133,7 @@ suite("test_generated_column") { sql """CREATE ENCRYPTKEY my_key AS "ABCD123456789";""" sql "drop table if exists test_encrypt_key_gen_col" qt_test_aes_encrypt """create table test_encrypt_key_gen_col(a int default 10, b int default 100, - c varchar(100) as(hex(aes_encrypt("abc",key my_key))));""" + c varchar(100) as(hex(aes_encrypt("abc",key my_key)))) distributed by hash(a) properties("replication_num"="1");;""" qt_test_aes_encrypt_insert "insert into test_encrypt_key_gen_col values(1,2,default)" qt_test_aes_encrypt_select "select * from test_encrypt_key_gen_col" From cff3c30d6aa39599f2c3e61ceafddbdf5e7a0734 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 3 Jun 2024 16:18:15 +0800 Subject: [PATCH 08/21] [Feat](nereids) support generated column --- fe/fe-core/src/main/cup/sql_parser.cup | 6 +++++- .../doris/catalog/GeneratedColumnInfo.java | 8 ++++---- .../stream_load_and_mysql_load.out | 18 ------------------ .../stream_load_and_mysql_load.groovy | 4 ++-- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index fdacb6f9df7b40..d423475267e6f4 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -263,6 +263,7 @@ terminal String KW_ALIAS, KW_ALL, KW_ALTER, + KW_ALWAYS, KW_ANALYZE, KW_AND, KW_ANTI, @@ -272,7 +273,6 @@ terminal String KW_AS, KW_ASC, KW_AT, - KW_ALWAYS, KW_AUTHORS, KW_BACKEND, KW_BACKENDS, @@ -8003,6 +8003,8 @@ type_func_name_keyword ::= keyword ::= KW_AFTER:id {: RESULT = id; :} + | KW_ALWAYS:id + {: RESULT = id; :} | KW_AGGREGATE:id {: RESULT = id; :} | KW_ALIAS:id @@ -8163,6 +8165,8 @@ keyword ::= {: RESULT = id; :} | KW_GLOBAL:id {: RESULT = id; :} + | KW_GENERATED:id + {: RESULT = id; :} | KW_GENERIC:id {: RESULT = id; :} | KW_GRAPH:id diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java index 37bc8001020a29..b14343a5c9f5f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java @@ -30,18 +30,18 @@ public enum GeneratedColumnType { STORED } - @SerializedName(value = "type") + @SerializedName(value = "t") private final GeneratedColumnType type; - @SerializedName(value = "exprSql") + @SerializedName(value = "es") private final String exprSql; - @SerializedName(value = "expr") + @SerializedName(value = "e") private final Expr expr; /* e.g. a,b,c=a+b,d=c+1 -> a,b,c=a+b,d=a+b+1 e.g. this is column d generated column info expr is c+1, expandExprForLoad is a+b+1 expandExprForLoad is used in streamload, routineload, mysqlload, etc */ - @SerializedName(value = "exprForLoad") + @SerializedName(value = "efl") private Expr expandExprForLoad; /** constructor */ diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out index 469cafafc41ba3..59c5006a2d2b27 100644 --- a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out @@ -63,27 +63,9 @@ -- !specify_value_for_gencol -- 1 2 3.0 -1 2 3.0 -1 2 3.0 -1 2 3.0 -1 2 3.0 2 3 5.0 -3 5 8.0 -3 5 8.0 -3 5 8.0 -3 5 8.0 -- !specify_value_for_gencol -- 1 2 3.0 -1 2 3.0 -1 2 3.0 -1 2 3.0 -1 2 3.0 -1 2 3.0 -2 3 5.0 2 3 5.0 -3 5 8.0 -3 5 8.0 -3 5 8.0 -3 5 8.0 diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy index 17fda59bfaeea5..a8d4025b9b82fc 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy @@ -118,7 +118,7 @@ suite("test_generated_column_stream_mysql_load") { qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" // specify value for generated column - sql "truncate table gencol_refer_gencol_steam_mysql_load" + sql "truncate table test_gen_col_common_steam_mysql_load" filepath = getLoalFilePath "three_column_gen_col_data.csv" sql """ LOAD DATA LOCAL @@ -130,7 +130,7 @@ suite("test_generated_column_stream_mysql_load") { sql "sync" qt_specify_value_for_gencol "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" - sql "truncate table gencol_refer_gencol_steam_mysql_load" + sql "truncate table test_gen_col_common_steam_mysql_load" sql """ LOAD DATA LOCAL INFILE '${filepath}' From ec1d9fc6d622d0841bf02e53f7862a1aa35fb719 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 6 Jun 2024 15:06:21 +0800 Subject: [PATCH 09/21] [Feat](nereids) support generated column --- .../org/apache/doris/alter/SchemaChangeHandler.java | 5 +++++ .../main/java/org/apache/doris/catalog/Column.java | 8 ++++++-- .../fault_tolerance_nereids.groovy | 12 ++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 36d63adbca416d..278ec6aa41a0f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -547,6 +547,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol Map> indexSchemaMap) throws DdlException { Column modColumn = alterClause.getColumn(); boolean lightSchemaChange = false; + if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (modColumn.isKey() && null != modColumn.getAggregationType()) { throw new DdlException("Can not assign aggregation method on key column: " + modColumn.getName()); @@ -1023,6 +1024,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP } } + if (newColumn.getGeneratedColumnInfo() != null) { + throw new DdlException("Temporarily not supporting alter table add generated columns."); + } + /* * add new column to indexes. * UNIQUE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 1ac4c81c4d1413..d5f790f4745e67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -146,10 +146,10 @@ public class Column implements Writable, GsonPostProcessable { @SerializedName(value = "onUpdateDefaultValueExprDef") private DefaultValueExprDef onUpdateDefaultValueExprDef; - @SerializedName(value = "generatedColumnInfo") + @SerializedName(value = "gci") private GeneratedColumnInfo generatedColumnInfo; - @SerializedName(value = "generatedColumnsThatReferToThis") + @SerializedName(value = "gctt") private Set generatedColumnsThatReferToThis = new HashSet<>(); public Column() { @@ -864,6 +864,10 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { return; } // TODO check cluster key + + if (generatedColumnInfo != null || other.getGeneratedColumnInfo() != null) { + throw new DdlException("Temporarily not supporting alter table modify generated columns."); + } } public boolean nameEquals(String otherColName, boolean ignorePrefix) { diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy index 259165efea60d5..938b5af36b8b64 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy @@ -156,4 +156,16 @@ suite("test_generated_column_fault_tolerance_nereids") { PROPERTIES("replication_num" = "1");""" exception "Expression of generated column 'c' contains a disallowed function" } + + sql "drop table if exists gen_col_test_modify" + sql """ create table gen_col_test_modify(pk int, a int, b int, c int as (a+b)) distributed by hash(pk) buckets 10 + properties('replication_num' = '1'); """ + test { + sql """ALTER TABLE gen_col_test_modify modify COLUMN c int AS (a+b+1)""" + exception "Temporarily not supporting alter table modify generated columns." + } + test { + sql """ALTER TABLE gen_col_test_modify ADD COLUMN d int AS (a+b);""" + exception "Temporarily not supporting alter table add generated columns." + } } \ No newline at end of file From c80e0ad80e5ca484ed456b7b3395c10dc45b4411 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 6 Jun 2024 21:27:52 +0800 Subject: [PATCH 10/21] [Feat](nereids) support generated column --- .../doris/analysis/CreateTableStmt.java | 32 ++++----- .../java/org/apache/doris/catalog/Column.java | 19 +----- .../apache/doris/common/UserException.java | 4 ++ .../common/proc/IndexSchemaProcNode.java | 2 +- .../nereids/analyzer/UnboundTableSink.java | 5 -- .../nereids/rules/analysis/BindSink.java | 3 + .../plans/commands/info/CreateTableInfo.java | 31 ++++----- .../plans/commands/insert/InsertUtils.java | 59 ++++++++++------- .../org/apache/doris/nereids/util/Utils.java | 65 ++++++++++++++----- .../test_generated_column_nereids.out | 13 +++- .../fault_tolerance_nereids.groovy | 45 +++++++++++-- ...reate_table_generated_column_legacy.groovy | 10 +-- .../test_generated_column_nereids.groovy | 23 +++---- 13 files changed, 194 insertions(+), 117 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index ee94ccf36a5c00..47aa61bcb6c6f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -43,6 +43,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.ExprRewriteRule; import org.apache.doris.rewrite.ExprRewriter; @@ -722,7 +723,7 @@ public boolean needAuditEncryption() { } private void generatedColumnCheck(Analyzer analyzer) throws AnalysisException { - genreatedColumnCommonCheck(); + generatedColumnCommonCheck(); Map> nameToColumnDef = Maps.newHashMap(); for (int i = 0; i < columnDefs.size(); i++) { ColumnDef columnDef = columnDefs.get(i); @@ -746,7 +747,12 @@ private void generatedColumnCheck(Analyzer analyzer) throws AnalysisException { throw new AnalysisException("Generated column does not support subquery."); } }); - expr.analyze(analyzer); + try { + expr.analyze(analyzer); + } catch (AnalysisException e) { + throw new AnalysisException("In generated column '" + columnDef.getName() + "', " + + Utils.convertFirstChar(e.getDetailMessage())); + } expr.foreach(e -> { if (e instanceof VariableExpr) { throw new AnalysisException("Generated column expression cannot contain variable."); @@ -854,21 +860,17 @@ public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseTy } } - private void genreatedColumnCommonCheck() throws AnalysisException { - boolean hasGeneratedCol = false; + private void generatedColumnCommonCheck() { for (ColumnDef column : columnDefs) { - if (column.getGeneratedColumnInfo().isPresent()) { - hasGeneratedCol = true; - break; + if (keysDesc.getKeysType() == KeysType.AGG_KEYS && column.getGeneratedColumnInfo().isPresent() + && !column.isKey()) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Generated Columns in aggregate table must be keys."); + } + if (column.getGeneratedColumnInfo().isPresent() && !engineName.equalsIgnoreCase("olap")) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "Tables can only have generated columns if the olap engine is used"); } - } - if (hasGeneratedCol && !engineName.equalsIgnoreCase("olap")) { - throw new AnalysisException( - "Tables can only have generated columns if the olap engine is used"); - } - if (hasGeneratedCol && keysDesc.getKeysType() == KeysType.AGG_KEYS) { - throw new AnalysisException( - "Generated Column cannot be used in the aggregate table"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index d5f790f4745e67..e98a849ad4d436 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -953,13 +953,13 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { } else { sb.append(typeStr); } - if (generatedColumnInfo != null) { - return generatedColumnToSql(sb); - } if (aggregationType != null && aggregationType != AggregateType.NONE && !isUniqueTable && !isAggregationTypeImplicit) { sb.append(" ").append(aggregationType.toSql()); } + if (generatedColumnInfo != null) { + sb.append(" AS (").append(generatedColumnInfo.getExpr().toSql()).append(")"); + } if (isAllowNull) { sb.append(" NULL"); } else { @@ -984,19 +984,6 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { return sb.toString(); } - private String generatedColumnToSql(StringBuilder sb) { - sb.append(" AS (").append(generatedColumnInfo.getExpr().toSql()).append(")"); - if (isAllowNull) { - sb.append(" NULL"); - } else { - sb.append(" NOT NULL"); - } - if (StringUtils.isNotBlank(comment)) { - sb.append(" COMMENT '").append(getComment(true)).append("'"); - } - return sb.toString(); - } - @Override public String toString() { return toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java b/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java index 0ccb2b1c17e4b6..09421e108534ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java @@ -73,4 +73,8 @@ public void setMysqlErrorCode(ErrorCode mysqlErrorCode) { public String getMessage() { return errorCode + ", detailMessage = " + super.getMessage(); } + + public String getDetailMessage() { + return super.getMessage(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 572151210b6df4..7ffff0c449ac76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -66,7 +66,7 @@ public static ProcResult createResult(List schema, Set bfColumns extras.add("AUTO_INCREMENT"); } if (column.getGeneratedColumnInfo() != null) { - extras.add("GENERATED"); + extras.add("STORED GENERATED"); } String extraStr = StringUtils.join(extras, ","); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index 54e5b5d5de8112..23c58ba42fb17e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -173,11 +173,6 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr isPartialUpdate, dmlCommandType, groupExpression, logicalProperties, children.get(0)); } - public Plan withColNames(List colNames) { - return new UnboundTableSink<>(nameParts, colNames, hints, temporaryPartition, partitions, autoDetectPartition, - isPartialUpdate, dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); - } - @Override public LogicalProperties computeLogicalProperties() { return UnboundLogicalProperties.INSTANCE; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index f28640041cc6b2..4384197b629c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -372,6 +372,9 @@ private static Map getColumnToOutput( } } } + // the generated columns can use all ordinary columns, + // if processed in upper for loop, will lead to not found slot error + //It's the same reason for moving the processing of materialized columns down. for (Column column : generatedColumns) { GeneratedColumnInfo info = column.getGeneratedColumnInfo(); Expression parsedExpression = new NereidsParser().parseExpression(info.getExpr().toSqlWithoutTbl()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index e23219f087f2d3..1c00061b44d841 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -839,24 +839,19 @@ public void setIsExternal(boolean isExternal) { this.isExternal = isExternal; } - private void genreatedColumnCommonCheck() { - boolean hasGeneratedCol = false; + private void generatedColumnCommonCheck() { for (ColumnDefinition column : columns) { - if (column.getGeneratedColumnDesc().isPresent()) { - hasGeneratedCol = true; - break; + if (keysType == KeysType.AGG_KEYS && column.getGeneratedColumnDesc().isPresent() && !column.isKey()) { + throw new AnalysisException("Generated Columns in aggregate table must be keys."); + } + if (column.getGeneratedColumnDesc().isPresent() && !engineName.equalsIgnoreCase("olap")) { + throw new AnalysisException("Tables can only have generated columns if the olap engine is used"); } - } - if (hasGeneratedCol && !engineName.equalsIgnoreCase("olap")) { - throw new AnalysisException("Tables can only have generated columns if the olap engine is used"); - } - if (hasGeneratedCol && keysType == KeysType.AGG_KEYS) { - throw new AnalysisException("Generated Column cannot be used in the aggregate table"); } } private void generatedColumnCheck(ConnectContext ctx) { - genreatedColumnCommonCheck(); + generatedColumnCommonCheck(); LogicalEmptyRelation plan = new LogicalEmptyRelation( ConnectContext.get().getStatementContext().getNextRelationId(), new ArrayList<>()); @@ -885,11 +880,17 @@ private void generatedColumnCheck(ConnectContext ctx) { continue; } Expression parsedExpression = info.get().getExpression(); - checkparsedExpressionInGeneratedColumn(parsedExpression); + checkParsedExpressionInGeneratedColumn(parsedExpression); Expression boundSlotExpression = SlotReplacer.INSTANCE.replace(parsedExpression, columnToSlotReference); Scope scope = new Scope(slots); ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope, cascadesContext, false, false); - Expression expr = analyzer.analyze(boundSlotExpression, new ExpressionRewriteContext(cascadesContext)); + Expression expr; + try { + expr = analyzer.analyze(boundSlotExpression, new ExpressionRewriteContext(cascadesContext)); + } catch (AnalysisException e) { + throw new AnalysisException("In generated column '" + column.getName() + "', " + + Utils.convertFirstChar(e.getMessage())); + } checkExpressionInGeneratedColumn(expr, column, nameToColumnDefinition); TypeCoercionUtils.checkCanCastTo(expr.getDataType(), column.getType()); ExpressionToExpr translator = new ExpressionToExpr(i); @@ -948,7 +949,7 @@ public Expression visitUnboundSlot(UnboundSlot unboundSlot, Map re } } - void checkparsedExpressionInGeneratedColumn(Expression expr) { + void checkParsedExpressionInGeneratedColumn(Expression expr) { expr.foreach(e -> { if (e instanceof SubqueryExpr) { throw new AnalysisException("Generated column does not support subquery."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 20f78edcdb8d9c..92ec3676c1462d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -75,9 +75,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; +import org.glassfish.jersey.internal.guava.Sets; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -292,6 +294,7 @@ public static Plan normalizePlan(Plan plan, TableIf table) { } } Plan query = unboundLogicalSink.child(); + checkGeneratedColumnForInsertIntoSelect(table, query, unboundLogicalSink); if (!(query instanceof LogicalInlineTable)) { return plan; } @@ -362,31 +365,6 @@ public static Plan normalizePlan(Plan plan, TableIf table) { StatementScopeIdGenerator.newRelationId(), constantExprs.build())); } List oneRowRelations = oneRowRelationBuilder.build(); - - // if (plan instanceof UnboundTableSink) { - // List newColNames = new ArrayList<>(); - // if (unboundLogicalSink.getColNames().isEmpty()) { - // for (Column column : table.getBaseSchema(false)) { - // if (column.getGeneratedColumnInfo() != null) { - // continue; - // } - // newColNames.add(column.getName()); - // } - // } else { - // for (String name : unboundLogicalSink.getColNames()) { - // Column column = table.getColumn(name); - // if (column.getGeneratedColumnInfo() != null) { - // continue; - // } - // newColNames.add(name); - // } - // } - // if (!unboundLogicalSink.getColNames().isEmpty() && newColNames.isEmpty()) { - // - // } - // plan = ((UnboundTableSink) plan).withColNames(newColNames); - // } - if (oneRowRelations.size() == 1) { return plan.withChildren(oneRowRelations.get(0)); } else { @@ -427,6 +405,9 @@ public static TableIf getTargetTable(Plan plan, ConnectContext ctx) { private static NamedExpression generateDefaultExpression(Column column) { try { GeneratedColumnInfo generatedColumnInfo = column.getGeneratedColumnInfo(); + // Using NullLiteral as a placeholder. + // If return the expr in generatedColumnInfo, will lead to slot not found error in analyze. + // Instead, getting the generated column expr and analyze the expr in BindSink can avoid the error. if (generatedColumnInfo != null) { return new Alias(new NullLiteral(DataType.fromCatalogType(column.getType())), column.getName()); } @@ -464,4 +445,32 @@ public static Plan getPlanForExplain(ConnectContext ctx, LogicalPlan logicalQuer } return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx)); } + + private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, Plan query, + UnboundLogicalSink unboundLogicalSink) { + if (table instanceof OlapTable && !(query instanceof LogicalInlineTable)) { + OlapTable olapTable = (OlapTable) table; + Set insertNames = Sets.newHashSet(); + if (unboundLogicalSink.getColNames() != null) { + insertNames.addAll(unboundLogicalSink.getColNames()); + } + if (insertNames.isEmpty()) { + for (Column col : olapTable.getFullSchema()) { + if (col.getGeneratedColumnInfo() != null) { + throw new AnalysisException("The value specified for generated column '" + + col.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } + } + } else { + for (Column col : olapTable.getFullSchema()) { + if (col.getGeneratedColumnInfo() != null && insertNames.contains(col.getName())) { + throw new AnalysisException("The value specified for generated column '" + + col.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index 852e148ef1d9cb..8525d2629e34ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -63,8 +63,8 @@ public static String quoteIfNeeded(String part) { * Helper function to eliminate unnecessary checked exception caught requirement from the main logic of translator. * * @param f function which would invoke the logic of - * stale code from old optimizer that could throw - * a checked exception. + * stale code from old optimizer that could throw + * a checked exception. */ public static void execWithUncheckedException(FuncWrapper f) { try { @@ -128,14 +128,18 @@ public static String qualifiedName(List qualifier, String name) { return StringUtils.join(qualifiedNameParts(qualifier, name), "."); } - /** get qualified name with Backtick */ + /** + * get qualified name with Backtick + */ public static String qualifiedNameWithBackquote(List qualifiers, String name) { List fullName = new ArrayList<>(qualifiers); fullName.add(name); return qualifiedNameWithBackquote(fullName); } - /** get qualified name with Backtick */ + /** + * get qualified name with Backtick + */ public static String qualifiedNameWithBackquote(List qualifiers) { List qualifierWithBackquote = Lists.newArrayListWithCapacity(qualifiers.size()); for (String qualifier : qualifiers) { @@ -162,7 +166,7 @@ public static String toSqlString(String planName, Object... variables) { } for (int i = 0; i < variables.length - 1; i += 2) { - if (! "".equals(toStringOrNull(variables[i + 1]))) { + if (!"".equals(toStringOrNull(variables[i + 1]))) { if (i != 0) { stringBuilder.append(", "); } @@ -245,7 +249,9 @@ public static void identityRemove(List list, T item) { Preconditions.checkState(false, "item not found in list"); } - /** allCombinations */ + /** + * allCombinations + */ public static List> allCombinations(List> lists) { if (lists.size() == 1) { List first = lists.get(0); @@ -280,7 +286,7 @@ private static List> doAllCombinations(List> lists) { .flatMap(firstValue -> combinationWithoutFirst.stream() .map(restList -> Stream.concat(Stream.of(firstValue), restList.stream()) - .collect(ImmutableList.toImmutableList()) + .collect(ImmutableList.toImmutableList()) ) ).collect(ImmutableList.toImmutableList()); } @@ -326,7 +332,9 @@ public static List fastMapList(List list, int additionSize, Functio return newList; } - /** fastToImmutableList */ + /** + * fastToImmutableList + */ public static ImmutableList fastToImmutableList(E[] array) { switch (array.length) { case 0: @@ -344,14 +352,17 @@ public static ImmutableList fastToImmutableList(E[] array) { } } - /** fastToImmutableList */ + /** + * fastToImmutableList + */ public static ImmutableList fastToImmutableList(Collection collection) { if (collection instanceof ImmutableList) { return (ImmutableList) collection; } switch (collection.size()) { - case 0: return ImmutableList.of(); + case 0: + return ImmutableList.of(); case 1: return collection instanceof List ? ImmutableList.of(((List) collection).get(0)) @@ -366,7 +377,9 @@ public static ImmutableList fastToImmutableList(Collection c } } - /** fastToImmutableSet */ + /** + * fastToImmutableSet + */ public static ImmutableSet fastToImmutableSet(Collection collection) { if (collection instanceof ImmutableSet) { return (ImmutableSet) collection; @@ -389,7 +402,9 @@ public static ImmutableSet fastToImmutableSet(Collection col } } - /** reverseImmutableList */ + /** + * reverseImmutableList + */ public static ImmutableList reverseImmutableList(List list) { Builder reverseList = ImmutableList.builderWithExpectedSize(list.size()); for (int i = list.size() - 1; i >= 0; i--) { @@ -398,7 +413,9 @@ public static ImmutableList reverseImmutableList(List list) return reverseList.build(); } - /** filterImmutableList */ + /** + * filterImmutableList + */ public static ImmutableList filterImmutableList(List list, Predicate filter) { Builder newList = ImmutableList.builderWithExpectedSize(list.size()); for (int i = 0; i < list.size(); i++) { @@ -410,7 +427,9 @@ public static ImmutableList filterImmutableList(List list, P return newList.build(); } - /** concatToSet */ + /** + * concatToSet + */ public static Set concatToSet(Collection left, Collection right) { ImmutableSet.Builder required = ImmutableSet.builderWithExpectedSize( left.size() + right.size() @@ -420,7 +439,9 @@ public static Set concatToSet(Collection left, Collection Optional fastReduce(List list, BiFunction reduceOp) { if (list.isEmpty()) { return Optional.empty(); @@ -431,4 +452,18 @@ public static Optional fastReduce(List list, BiFunction to numeric type" + exception "In generated column 'c', cannot cast from ARRAY to numeric type" } test { sql """ - create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) - aggregate key(a,b,c) + create table test_gen_col_aggregate_value(a int,b int,c int sum generated always as (abs(a+1)) not null) + aggregate key(a,b) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - exception "Generated Column cannot be used in the aggregate table" + exception "Generated Columns in aggregate table must be keys." } test { @@ -154,7 +154,7 @@ suite("test_generated_column_fault_tolerance_nereids") { create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1");""" - exception "Expression of generated column 'c' contains a disallowed function" + exception "Expression of generated column 'c' contains a disallowed function:'Grouping'" } sql "drop table if exists gen_col_test_modify" @@ -168,4 +168,35 @@ suite("test_generated_column_fault_tolerance_nereids") { sql """ALTER TABLE gen_col_test_modify ADD COLUMN d int AS (a+b);""" exception "Temporarily not supporting alter table add generated columns." } + + sql "drop table if exists test_gen_col_common_ft" + sql """create table test_gen_col_common_ft(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + // qt_common_default_test_insert_null + test { + sql "INSERT INTO test_gen_col_common_ft(a,b) values(1,null);" + exception "Insert has filtered data in strict mode." + } + + // qt_common_default_test_insert_gencol + test { + sql "INSERT INTO test_gen_col_common_ft values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft(a,b,c) values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft select 1,2,5" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft(a,b,c) select * from test_gen_col_common_ft" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + + } \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy index 5da23ed6cc809e..e56e2405203c3e 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy @@ -58,7 +58,7 @@ suite("test_create_table_generated_column_legacy") { DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - exception "No matching function with signature" + exception "In generated column 'c', no matching function with signature" } // gencol_has_sum @@ -134,7 +134,7 @@ suite("test_create_table_generated_column_legacy") { DISTRIBUTED BY HASH(pk) PROPERTIES("replication_num" = "1"); """ - exception "can not cast from origin type" + exception "In generated column 'c', can not cast from origin type" } test { @@ -198,12 +198,12 @@ suite("test_create_table_generated_column_legacy") { test { sql """ - create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) - aggregate key(a,b,c) + create table test_gen_col_aggregate(a int,b int,c int sum generated always as (abs(a+1)) not null) + aggregate key(a,b) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - exception "Generated Column cannot be used in the aggregate table" + exception "Generated Columns in aggregate table must be keys." } // test drop dependency diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index 9548e15beea935..659f13b304e6f7 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -30,17 +30,6 @@ suite("test_generated_column") { qt_common_default_test_insert_default "INSERT INTO test_gen_col_common values(3,5,default);" qt_commont_default_select "select * from test_gen_col_common order by 1,2,3;" - // qt_common_default_test_insert_null - test { - sql "INSERT INTO test_gen_col_common(a,b) values(1,null);" - exception "Insert has filtered data in strict mode." - } - - // qt_common_default_test_insert_gencol - test { - sql "INSERT INTO test_gen_col_common values(1,2,3);" - exception "The value specified for generated column 'c' in table 'test_gen_col_common' is not allowed." - } sql "drop table if exists test_gen_col_without_generated_always" qt_common_without_generated_always """create table test_gen_col_without_generated_always(a int,b int,c double as (abs(a+b)) not null) @@ -175,6 +164,18 @@ suite("test_generated_column") { qt_gen_col_unique_key_insert_is_key "INSERT INTO test_gen_col_unique_key2 values(6,default,7);" qt_gen_col_unique_key_select_is_key "select * from test_gen_col_unique_key2" + // test aggregate table,generated column is key + sql "drop table if exists test_gen_col_aggregate" + sql """ + create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + qt_gen_col_aggregate_insert "INSERT INTO test_gen_col_aggregate values(6,7,default);" + qt_gen_col_aggregate_insert "INSERT INTO test_gen_col_aggregate values(6,7,default);" + qt_gen_col_aggregate_select "select * from test_gen_col_aggregate" + // test drop dependency sql "drop table if exists gencol_refer_gencol" qt_gencol_refer_gencol """ From bfc1f5bbc388927684714b2934c7a206751d24ee Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 7 Jun 2024 10:13:49 +0800 Subject: [PATCH 11/21] [Feat](nereids) support generated column --- .../doris/nereids/trees/plans/commands/insert/InsertUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 92ec3676c1462d..224b06cb71e128 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -74,8 +74,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; -import org.glassfish.jersey.internal.guava.Sets; import java.util.List; import java.util.Optional; From 90fc7cf2c063cf253e1dcb03d641f792cc7332d4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 7 Jun 2024 14:45:46 +0800 Subject: [PATCH 12/21] [Feat](nereids) support generated column --- .../main/java/org/apache/doris/analysis/CreateTableStmt.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 47aa61bcb6c6f1..93a5ba57550b10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -862,8 +862,8 @@ public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseTy private void generatedColumnCommonCheck() { for (ColumnDef column : columnDefs) { - if (keysDesc.getKeysType() == KeysType.AGG_KEYS && column.getGeneratedColumnInfo().isPresent() - && !column.isKey()) { + if (keysDesc != null && keysDesc.getKeysType() == KeysType.AGG_KEYS + && column.getGeneratedColumnInfo().isPresent() && !column.isKey()) { throw new org.apache.doris.nereids.exceptions.AnalysisException( "Generated Columns in aggregate table must be keys."); } From 1df666bae4aa1707e83dc6e5451474bd13245fe2 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 7 Jun 2024 19:56:51 +0800 Subject: [PATCH 13/21] [Feat](nereids) support generated column --- .../main/java/org/apache/doris/alter/SchemaChangeHandler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 278ec6aa41a0f8..2210a78e830aad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -823,6 +823,9 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol if (oneCol == null) { throw new DdlException("Column[" + colName + "] not exists"); } + if (oneCol.getGeneratedColumnInfo() != null) { + throw new DdlException("Temporarily does not support changing the position of generated columns."); + } newSchema.add(oneCol); if (colNameSet.contains(colName)) { throw new DdlException("Reduplicative column[" + colName + "]"); From c8a1a1874d26b021ad61c938db6fb0ead75d65d1 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 12 Jun 2024 17:10:50 +0800 Subject: [PATCH 14/21] [Feat](nereids) support generated column --- .../java/org/apache/doris/alter/SchemaChangeHandler.java | 2 +- .../java/org/apache/doris/analysis/CreateTableStmt.java | 6 +++--- .../src/main/java/org/apache/doris/catalog/Column.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 2210a78e830aad..3371fa55d33553 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1028,7 +1028,7 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP } if (newColumn.getGeneratedColumnInfo() != null) { - throw new DdlException("Temporarily not supporting alter table add generated columns."); + throw new DdlException("Not supporting alter table add generated columns."); } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 93a5ba57550b10..0834a400ad4f32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -860,15 +860,15 @@ public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseTy } } - private void generatedColumnCommonCheck() { + private void generatedColumnCommonCheck() throws AnalysisException { for (ColumnDef column : columnDefs) { if (keysDesc != null && keysDesc.getKeysType() == KeysType.AGG_KEYS && column.getGeneratedColumnInfo().isPresent() && !column.isKey()) { - throw new org.apache.doris.nereids.exceptions.AnalysisException( + throw new AnalysisException( "Generated Columns in aggregate table must be keys."); } if (column.getGeneratedColumnInfo().isPresent() && !engineName.equalsIgnoreCase("olap")) { - throw new org.apache.doris.nereids.exceptions.AnalysisException( + throw new AnalysisException( "Tables can only have generated columns if the olap engine is used"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index e98a849ad4d436..a926d9a9962766 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -866,7 +866,7 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { // TODO check cluster key if (generatedColumnInfo != null || other.getGeneratedColumnInfo() != null) { - throw new DdlException("Temporarily not supporting alter table modify generated columns."); + throw new DdlException("Not supporting alter table modify generated columns."); } } From 74faa45af3958f8fde54892659895ad0101f2af7 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 19 Jun 2024 21:30:32 +0800 Subject: [PATCH 15/21] [Feat](nereids) support generated column --- .../doris/alter/SchemaChangeHandler.java | 63 ++++++-- .../org/apache/doris/analysis/ColumnDef.java | 17 +++ .../doris/analysis/CreateTableStmt.java | 10 +- .../java/org/apache/doris/catalog/Env.java | 7 + .../nereids/analyzer/UnboundTableSink.java | 1 + .../plans/commands/info/ColumnDefinition.java | 2 +- .../plans/commands/info/CreateTableInfo.java | 9 +- .../insert/BatchInsertIntoTableCommand.java | 2 +- .../insert/InsertIntoTableCommand.java | 2 +- .../insert/InsertOverwriteTableCommand.java | 6 +- .../plans/commands/insert/InsertUtils.java | 34 ++++- .../insert/OlapInsertCommandContext.java | 27 ++-- .../doris/planner/StreamLoadPlanner.java | 6 + .../alter_column_test_generated_column.out | 28 ++++ .../gen_col_data_delete.csv | 1 + .../test_delete_generated_column.out | 53 +++++++ .../test_generated_column_nereids.out | 9 ++ ...test_insert_overwrite_generated_column.out | 64 ++++++++ .../test_load_delete_generated_column.out | 9 ++ .../test_partial_update_generated_column.out | 11 ++ .../test_update_generated_column.out | 22 +++ .../plugins/plugin_must_contains.groovy | 2 +- .../alter_column_test_generated_column.groovy | 140 ++++++++++++++++++ .../fault_tolerance_nereids.groovy | 8 +- ...reate_table_generated_column_legacy.groovy | 21 ++- .../test_delete_generated_column.groovy | 74 +++++++++ .../test_generated_column_nereids.groovy | 36 +++++ ...t_insert_overwrite_generated_column.groovy | 117 +++++++++++++++ .../test_load_delete_generated_column.groovy | 52 +++++++ ...est_partial_update_generated_column.groovy | 78 ++++++++++ .../test_update_generated_column.groovy | 75 ++++++++++ 31 files changed, 942 insertions(+), 44 deletions(-) create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data_delete.csv create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.out create mode 100644 regression-test/data/ddl_p0/test_create_table_generated_column/test_update_generated_column.out create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy create mode 100644 regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3371fa55d33553..fc3e5d97834244 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -384,13 +384,15 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa for (Column c : indexSchemaMap.get(baseIndexId)) { nameToColumn.put(c.getName(), c); } - if (nameToColumn.containsKey(dropColName)) { - Column column = nameToColumn.get(dropColName); - Set generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis(); - if (!generatedColumnsThatReferToThis.isEmpty()) { - throw new DdlException( - "Column '" + dropColName + "' has a generated column dependency on :" - + generatedColumnsThatReferToThis); + if (null == targetIndexName) { + if (nameToColumn.containsKey(dropColName)) { + Column column = nameToColumn.get(dropColName); + Set generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis(); + if (!generatedColumnsThatReferToThis.isEmpty()) { + throw new DdlException( + "Column '" + dropColName + "' has a generated column dependency on :" + + generatedColumnsThatReferToThis); + } } } @@ -804,12 +806,15 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol if (targetIndexName == null) { targetIndexName = baseIndexName; } - long targetIndexId = olapTable.getIndexIdByName(targetIndexName); LinkedList newSchema = new LinkedList(); List targetIndexSchema = indexSchemaMap.get(targetIndexId); - + // When rollup is specified, there is no need to check the order of generated columns. + // When rollup is not specified and the order of baseIndex needs to be modified, the order needs to be checked. + if (alterClause.getRollupName() == null) { + checkOrder(targetIndexSchema, orderedColNames); + } // check and create new ordered column list Set colNameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (String colName : orderedColNames) { @@ -823,9 +828,6 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol if (oneCol == null) { throw new DdlException("Column[" + colName + "] not exists"); } - if (oneCol.getGeneratedColumnInfo() != null) { - throw new DdlException("Temporarily does not support changing the position of generated columns."); - } newSchema.add(oneCol); if (colNameSet.contains(colName)) { throw new DdlException("Reduplicative column[" + colName + "]"); @@ -3241,4 +3243,41 @@ private void removeColumnWhenDropGeneratedColumn(Column dropColumn, Map targetIndexSchema, List orderedColNames) throws DdlException { + Set nameSet = new HashSet<>(); + for (Column column : targetIndexSchema) { + if (column.isVisible() && null == column.getGeneratedColumnInfo()) { + nameSet.add(column.getName()); + } + } + for (String colName : orderedColNames) { + Column oneCol = null; + for (Column column : targetIndexSchema) { + if (column.getName().equalsIgnoreCase(colName) && column.isVisible()) { + oneCol = column; + break; + } + } + if (oneCol == null) { + throw new DdlException("Column[" + colName + "] not exists"); + } + if (null == oneCol.getGeneratedColumnInfo()) { + continue; + } + Expr expr = oneCol.getGeneratedColumnInfo().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRef : slotRefsInGeneratedExpr) { + String slotName = ((SlotRef) slotRef).getColumnName(); + if (!nameSet.contains(slotName)) { + throw new DdlException("The specified column order is incorrect, `" + colName + + "` should come after `" + slotName + + "`, because both of them are generated columns, and `" + + colName + "` refers to `" + slotName + "`."); + } + } + nameSet.add(colName); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index a36896851c5e71..f306bb782fe3a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -146,6 +146,10 @@ public long getCurrentTimeStampPrecision() { return 0; } + public boolean isNullDefaultValue() { + return !isSet && value == null && defaultValueExprDef == null; + } + public String getValue() { if (isCurrentTimeStamp()) { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); @@ -488,6 +492,7 @@ public void analyze(boolean isOlap) throws AnalysisException { if (type.isScalarType() && defaultValue.isSet && defaultValue.value != null) { validateDefaultValue(type, defaultValue.value, defaultValue.defaultValueExprDef); } + validateGeneratedColumnInfo(); } @SuppressWarnings("checkstyle:Indentation") @@ -667,4 +672,16 @@ public long getAutoIncInitValue() { public void addGeneratedColumnsThatReferToThis(List list) { generatedColumnsThatReferToThis.addAll(list); } + + private void validateGeneratedColumnInfo() throws AnalysisException { + // for generated column + if (generatedColumnInfo.isPresent()) { + if (autoIncInitValue != -1) { + throw new AnalysisException("Generated columns cannot be auto_increment."); + } + if (defaultValue != null && !defaultValue.isNullDefaultValue()) { + throw new AnalysisException("Generated columns cannot have default value."); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 0834a400ad4f32..b4972931fb79ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -49,6 +49,7 @@ import org.apache.doris.rewrite.ExprRewriter; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -73,6 +74,8 @@ public class CreateTableStmt extends DdlStmt { private static final Logger LOG = LogManager.getLogger(CreateTableStmt.class); protected static final String DEFAULT_ENGINE_NAME = "olap"; + private static final ImmutableSet GENERATED_COLUMN_ALLOW_AGG_TYPE = + ImmutableSet.of(AggregateType.REPLACE, AggregateType.REPLACE_IF_NOT_NULL); protected boolean ifNotExists; private boolean isExternal; @@ -863,9 +866,10 @@ public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseTy private void generatedColumnCommonCheck() throws AnalysisException { for (ColumnDef column : columnDefs) { if (keysDesc != null && keysDesc.getKeysType() == KeysType.AGG_KEYS - && column.getGeneratedColumnInfo().isPresent() && !column.isKey()) { - throw new AnalysisException( - "Generated Columns in aggregate table must be keys."); + && column.getGeneratedColumnInfo().isPresent() + && (!column.isKey() && !GENERATED_COLUMN_ALLOW_AGG_TYPE.contains(column.getAggregateType()))) { + throw new AnalysisException("The generated columns can be key columns, " + + "or value columns of replace and replace_if_not_null aggregation type."); } if (column.getGeneratedColumnInfo().isPresent() && !engineName.equalsIgnoreCase("olap")) { throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 4b33b2079b4fdf..4ab4c11d6c89f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4911,7 +4911,14 @@ private void renameColumn(Database db, OlapTable table, String colName, } // check if have materialized view on rename column + // and check whether colName is referenced by generated columns for (Column column : entry.getValue().getSchema()) { + if (column.getName().equals(colName) && !column.getGeneratedColumnsThatReferToThis().isEmpty()) { + throw new DdlException( + "Cannot rename column, because column '" + colName + + "' has a generated column dependency on :" + + column.getGeneratedColumnsThatReferToThis()); + } Expr expr = column.getDefineExpr(); if (expr == null) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index 23c58ba42fb17e..ee515e21b07f27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -51,6 +51,7 @@ public class UnboundTableSink extends UnboundLogicalSin private boolean isPartialUpdate; private final DMLCommandType dmlCommandType; private final boolean autoDetectPartition; + private final boolean normalized = false; public UnboundTableSink(List nameParts, List colNames, List hints, List partitions, CHILD_TYPE child) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 01483e189348ff..569151430cb24d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -712,7 +712,7 @@ private void validateGeneratedColumnInfo() { if (autoIncInitValue != -1) { throw new AnalysisException("Generated columns cannot be auto_increment."); } - if (defaultValue.isPresent()) { + if (defaultValue.isPresent() && !defaultValue.get().equals(DefaultValue.NULL_DEFAULT_VALUE)) { throw new AnalysisException("Generated columns cannot have default value."); } if (onUpdateDefaultValue.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 1c00061b44d841..1ec8bd47a7de61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -79,6 +79,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -107,6 +108,8 @@ public class CreateTableInfo { public static final String ENGINE_BROKER = "broker"; public static final String ENGINE_HIVE = "hive"; public static final String ENGINE_ICEBERG = "iceberg"; + private static final ImmutableSet GENERATED_COLUMN_ALLOW_AGG_TYPE = + ImmutableSet.of(AggregateType.REPLACE, AggregateType.REPLACE_IF_NOT_NULL); private final boolean ifNotExists; private String ctlName; @@ -841,8 +844,10 @@ public void setIsExternal(boolean isExternal) { private void generatedColumnCommonCheck() { for (ColumnDefinition column : columns) { - if (keysType == KeysType.AGG_KEYS && column.getGeneratedColumnDesc().isPresent() && !column.isKey()) { - throw new AnalysisException("Generated Columns in aggregate table must be keys."); + if (keysType == KeysType.AGG_KEYS && column.getGeneratedColumnDesc().isPresent() + && (!column.isKey() && !GENERATED_COLUMN_ALLOW_AGG_TYPE.contains(column.getAggType()))) { + throw new AnalysisException("The generated columns can be key columns, " + + "or value columns of replace and replace_if_not_null aggregation type."); } if (column.getGeneratedColumnDesc().isPresent() && !engineName.equalsIgnoreCase("olap")) { throw new AnalysisException("Tables can only have generated columns if the olap engine is used"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 4b7afb1f6a855b..cdb5a4c7eebc85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -100,7 +100,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); targetTableIf.readLock(); try { - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index a0268f38a2b3ca..39507d726858b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -142,7 +142,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor targetTableIf.readLock(); try { // 1. process inline table (default values, empty values) - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, insertCtx); if (cte.isPresent()) { this.logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 05d51521ff196e..3acd3d8123d8f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -118,7 +118,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (targetTableIf instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ctx)) { throw new AnalysisException("Not allowed to perform current operation on async materialized view"); } - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); if (cte.isPresent()) { this.logicalQuery = (LogicalPlan) logicalQuery.withChildren(cte.get().withChildren( this.logicalQuery.child(0))); @@ -227,7 +227,7 @@ private void insertInto(ConnectContext ctx, StmtExecutor executor, List (LogicalPlan) (sink.child(0))); // 1. for overwrite situation, we disable auto create partition. // 2. we save and pass overwrite auto detect by insertCtx - insertCtx = new OlapInsertCommandContext(false); + insertCtx = new OlapInsertCommandContext(false, true); } else if (logicalQuery instanceof UnboundHiveTableSink) { UnboundHiveTableSink sink = (UnboundHiveTableSink) logicalQuery; copySink = (UnboundLogicalSink) UnboundTableSinkCreator.createUnboundTableSink( @@ -259,7 +259,7 @@ private void insertInto(ConnectContext ctx, StmtExecutor executor, long groupId) InsertCommandContext insertCtx; if (logicalQuery instanceof UnboundTableSink) { insertCtx = new OlapInsertCommandContext(false, - ((UnboundTableSink) logicalQuery).isAutoDetectPartition(), groupId); + ((UnboundTableSink) logicalQuery).isAutoDetectPartition(), groupId, true); } else if (logicalQuery instanceof UnboundHiveTableSink) { insertCtx = new HiveInsertCommandContext(); ((HiveInsertCommandContext) insertCtx).setOverwrite(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 224b06cb71e128..a1611718ec875a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -256,7 +256,7 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, /** * normalize plan to let it could be process correctly by nereids */ - public static Plan normalizePlan(Plan plan, TableIf table) { + public static Plan normalizePlan(Plan plan, TableIf table, Optional insertCtx) { UnboundLogicalSink unboundLogicalSink = (UnboundLogicalSink) plan; if (table instanceof HMSExternalTable) { HMSExternalTable hiveTable = (HMSExternalTable) table; @@ -288,13 +288,19 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Partial update should include all key columns, missing: " + col.getName()); } + if (!col.getGeneratedColumnsThatReferToThis().isEmpty() + && col.getGeneratedColumnInfo() == null && !insertCol.isPresent()) { + throw new AnalysisException("Partial update should include" + + " all ordinary columns referenced" + + " by generated columns, missing: " + col.getName()); + } } } } } } Plan query = unboundLogicalSink.child(); - checkGeneratedColumnForInsertIntoSelect(table, query, unboundLogicalSink); + checkGeneratedColumnForInsertIntoSelect(table, unboundLogicalSink, insertCtx); if (!(query instanceof LogicalInlineTable)) { return plan; } @@ -443,11 +449,29 @@ public static Plan getPlanForExplain(ConnectContext ctx, LogicalPlan logicalQuer } throw new AnalysisException("Nereids DML is disabled, will try to fall back to the original planner"); } - return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx)); + return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), Optional.empty()); } - private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, Plan query, - UnboundLogicalSink unboundLogicalSink) { + // check for insert into t1(a,b,gen_col) select 1,2,3; + private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, + UnboundLogicalSink unboundLogicalSink, Optional insertCtx) { + // should not check delete stmt, because deletestmt can transform to insert delete sign + if (unboundLogicalSink.getDMLCommandType() == DMLCommandType.DELETE) { + return; + } + // This is for the insert overwrite values(),() + // Insert overwrite stmt can enter normalizePlan() twice. + // Insert overwrite values(),() is checked in the first time when deal with ConstantExprsList, + // and then the insert into values(),() will be transformed to insert into union all in normalizePlan, + // and this function is for insert into select, which will check the insert into union all again, + // and that is no need, also will lead to problems. + // So for insert overwrite values(),(), this check is only performed + // when it first enters the normalizePlan checking constantExprsList + if (insertCtx.isPresent() && insertCtx.get() instanceof OlapInsertCommandContext + && ((OlapInsertCommandContext) insertCtx.get()).isOverwrite()) { + return; + } + Plan query = unboundLogicalSink.child(); if (table instanceof OlapTable && !(query instanceof LogicalInlineTable)) { OlapTable olapTable = (OlapTable) table; Set insertNames = Sets.newHashSet(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java index bebade142d97e1..35adc3031b1441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java @@ -21,18 +21,25 @@ * For Olap Table */ public class OlapInsertCommandContext extends InsertCommandContext { - private boolean allowAutoPartition; - private boolean autoDetectOverwrite = false; - private long overwriteGroupId = 0; + private final boolean allowAutoPartition; + private final boolean autoDetectOverwrite; + private final long overwriteGroupId; + private final boolean isOverwrite; - public OlapInsertCommandContext(boolean allowAutoPartition) { - this.allowAutoPartition = allowAutoPartition; - } - - public OlapInsertCommandContext(boolean allowAutoPartition, boolean autoDetectOverwrite, long overwriteGroupId) { + public OlapInsertCommandContext(boolean allowAutoPartition, boolean autoDetectOverwrite, long overwriteGroupId, + boolean isOverwrite) { this.allowAutoPartition = allowAutoPartition; this.autoDetectOverwrite = autoDetectOverwrite; this.overwriteGroupId = overwriteGroupId; + this.isOverwrite = isOverwrite; + } + + public OlapInsertCommandContext(boolean allowAutoPartition) { + this(allowAutoPartition, false, 0, false); + } + + public OlapInsertCommandContext(boolean allowAutoPartition, boolean isOverwrite) { + this(allowAutoPartition, false, 0, isOverwrite); } public boolean isAllowAutoPartition() { @@ -47,7 +54,7 @@ public long getOverwriteGroupId() { return overwriteGroupId; } - public void setAllowAutoPartition(boolean allowAutoPartition) { - this.allowAutoPartition = allowAutoPartition; + public boolean isOverwrite() { + return isOverwrite; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 973134563b638f..14d792e15076ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -174,6 +174,12 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde if (col.isKey() && !existInExpr) { throw new UserException("Partial update should include all key columns, missing: " + col.getName()); } + if (!col.getGeneratedColumnsThatReferToThis().isEmpty() + && col.getGeneratedColumnInfo() == null && !existInExpr) { + throw new UserException("Partial update should include" + + " all ordinary columns referenced" + + " by generated columns, missing: " + col.getName()); + } } if (taskInfo.getMergeType() == LoadTask.MergeType.DELETE) { partialUpdateInputColumns.add(Column.DELETE_SIGN); diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.out new file mode 100644 index 00000000000000..a57510f0291b5a --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !drop_gen_col_rollup -- +0 + +-- !reorder -- +0 + +-- !after_reorder_insert -- +1 + +-- !reorder_rollup -- +0 + +-- !rename_gen_col -- +0 + +-- !after_rename_insert -- +1 + +-- !after_rename_insert_select -- +1 3 2 4 5 +6 8 2 9 5 +6 8 2 9 5 +9 11 2 12 3 +9 11 2 12 3 +12 15 3 16 4 +16 18 2 19 4 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data_delete.csv b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data_delete.csv new file mode 100644 index 00000000000000..0e88c499a701c2 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/gen_col_data_delete.csv @@ -0,0 +1 @@ +1,2 \ No newline at end of file diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out new file mode 100644 index 00000000000000..6a00e1cb90e6f0 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_delete_generated_column.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !delete_where_gen_col -- +0 + +-- !delete_where_gen_col_select -- +2 22 24 +10 2 12 + +-- !delete_where_gen_col_partition_has_no_satisfied_row -- +0 + +-- !delete_where_gen_col_partition_has_no_satisfied_row_select -- +2 22 24 +10 2 12 + +-- !delete_where_gen_col_and_other_col -- +0 + +-- !delete_where_gen_col_and_other_col_select -- +2 22 24 + +-- !delete_where_gen_col_unique -- +0 + +-- !delete_where_gen_col_select_unique -- +2 22 24 +10 2 12 + +-- !delete_where_gen_col_partition_has_no_satisfied_row_unique -- +0 + +-- !delete_where_gen_col_partition_has_no_satisfied_row_select_unique -- +2 22 24 +10 2 12 + +-- !delete_where_gen_col_and_other_col_unique -- +0 + +-- !delete_where_gen_col_and_other_col_select_unique -- +2 22 24 + +-- !delete_query -- +1 + +-- !delete_query_select -- + +-- !delete_query_cte -- +1 + +-- !delete_query_cte_select -- +1 2 3 +10 2 12 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out index f7a1e3dec8a70d..8d34dd8f2b094f 100644 --- a/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.out @@ -212,3 +212,12 @@ d INT Yes false \N NONE,STORED GENERATED -- !test_drop_column -- 3 +-- !test_agg_key -- +1 2 3 6 + +-- !agg_gen_col_replace -- +0 + +-- !agg_replace_null -- +1 2 3 4 13 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.out new file mode 100644 index 00000000000000..1860cae9d0953c --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.out @@ -0,0 +1,64 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !overwrite_value_partial_column -- +1 + +-- !overwrite_value -- +2 + +-- !overwrite_value_select -- +1 2 3 4 +3 4 7 8 + +-- !overwrite_multi_value -- +2 + +-- !overwrite_multi_value_select -- +1 2 3 4 +3 4 7 8 + +-- !overwrite_multi_value_partial_column -- +2 + +-- !overwrite_multi_value_partial_column_select -- +1 2 3 4 +3 4 7 8 + +-- !overwrite_select_value_partial_column -- +1 + +-- !overwrite_select_value_partial_column_select -- +1 2 3 4 + +-- !overwrite_select_table_partial_column -- +2 + +-- !overwrite_select_table_partial_column_select -- +1 2 3 4 +3 23 26 27 + +-- !overwrite_partition_single_value -- +1 2 3 + +-- !overwrite_partition_partial_single_value -- +1 2 3 +4 3 7 + +-- !overwrite_partition_multi_value -- +1 2 3 +4 4 8 +5 4 9 + +-- !overwrite_partition_partial_multi_value -- +1 4 5 +2 4 6 +3 4 7 +4 3 7 +5 4 9 + +-- !auto_partition_overwrite -- +1 4 5 +2 4 6 +3 4 7 +4 3 7 +5 4 9 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.out new file mode 100644 index 00000000000000..bd405225537d6d --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_stream_load_delete -- +2 9 11 12 +3 5 8 9 + +-- !test_stream_load_merge -- +1 2 3 4 +2 9 11 12 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.out new file mode 100644 index 00000000000000..b01c0ef5326bf1 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !partial_update_select -- +1 5 6 7 7 +2 4 6 7 \N +3 6 9 10 \N + +-- !test_stream_load_refer_gencol -- +1 2 5 6 0 +2 9 11 12 \N +3 5 5 6 8 + diff --git a/regression-test/data/ddl_p0/test_create_table_generated_column/test_update_generated_column.out b/regression-test/data/ddl_p0/test_create_table_generated_column/test_update_generated_column.out new file mode 100644 index 00000000000000..e73ead6a8023cf --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_generated_column/test_update_generated_column.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !update_gen_col -- +2 + +-- !update_select -- +3 13.0 10 14 +4 14.0 10 15 + +-- !update_use_expr -- +1 + +-- !update_use_expr_select -- +3 106.0 103 107 +4 14.0 10 15 + +-- !update_use_other_table_col -- +1 + +-- !update_use_expr_select -- +3 6.0 3 7 +4 14.0 10 15 + diff --git a/regression-test/plugins/plugin_must_contains.groovy b/regression-test/plugins/plugin_must_contains.groovy index 876c87f21ac352..fc138372f2aafc 100644 --- a/regression-test/plugins/plugin_must_contains.groovy +++ b/regression-test/plugins/plugin_must_contains.groovy @@ -27,4 +27,4 @@ Suite.metaClass.mustContain = {String str1, String str2 -> } return true } -logger.info("Added 'mustContains' function to Suite") +logger.info("Added 'mustContain' function to Suite") diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy new file mode 100644 index 00000000000000..45b0c3a47c3c12 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("alter_column_test_generated_column") { + def waitSchemaChangeJob = { String tableName /* param */ -> + int tryTimes = 30 + while (tryTimes-- > 0) { + def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + def jobState = jobResult[0][9].toString() + if ('cancelled'.equalsIgnoreCase(jobState)) { + logger.info("jobResult:{}", jobResult) + throw new IllegalStateException("${tableName}'s job has been cancelled") + } + logger.info(jobState) + if ('finished'.equalsIgnoreCase(jobState)) { + logger.info("jobResult:{}", jobResult) + sleep(3000) + return + } + sleep(1000) + } + assertTrue(false) + } + def getMVJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1 """ + return jobStateResult[0][8] + } + def waitForMVJob = (tbName, timeout) -> { + while (timeout--){ + String result = getMVJobState(tbName) + if (result == "FINISHED") { + sleep(3000) + break + } else { + sleep(100) + if (timeout < 1){ + assertEquals(1,2) + } + } + } + } + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists alter_column_gen_col; + create table alter_column_gen_col (a int, b int, c int as(a+b), d int as(c+1), e int) + duplicate key(a) distributed by hash(a) ; + alter table alter_column_gen_col add rollup r1 (c,b,a) + """ + waitForMVJob("alter_column_gen_col", 3000) + // add column + test { + sql "alter table alter_column_gen_col add column f int as (a+1);" + exception "Not supporting alter table add generated columns." + } + test { + sql "alter table alter_column_gen_col add column f int as (a+1) to r1" + exception "Not supporting alter table add generated columns." + } + test { + sql "alter table alter_column_gen_col add column f int as (a+1) after a to r1;" + exception "Not supporting alter table add generated columns." + } + test { + sql """alter table alter_column_gen_col add column (f int as (a+1), g int) to r1;""" + exception "Not supporting alter table add generated columns." + } + + // drop column + // rollup + multi_sql """ + insert into alter_column_gen_col values(1,2,default,default,5); + insert into alter_column_gen_col values(9,2,default,default,3); + insert into alter_column_gen_col values(6,2,default,default,5); + """ + explain { + sql "select c,b from alter_column_gen_col where c=10;" + contains "r1" + } + qt_drop_gen_col_rollup "alter table alter_column_gen_col drop column c from r1;" + waitSchemaChangeJob("alter_column_gen_col") + explain { + sql "select b from alter_column_gen_col where b=10;" + contains "r1" + } + multi_sql """ + insert into alter_column_gen_col values(9,2,default,default,3); + insert into alter_column_gen_col values(6,2,default,default,5); + """ + // modify column + test { + sql "alter table alter_column_gen_col modify column c double as (a+b);" + exception "Not supporting alter table modify generated columns." + } + test { + sql "alter table alter_column_gen_col modify column c double as (a+b) after e;" + exception "Not supporting alter table modify generated columns." + } + test { + sql "alter table alter_column_gen_col modify column c int as (a+b) after e;" + exception "Not supporting alter table modify generated columns." + } + + // reorder column + qt_reorder "alter table alter_column_gen_col order by(a,c,b,d,e);" + waitSchemaChangeJob("alter_column_gen_col") + + test { + sql "alter table alter_column_gen_col order by(a,d,b,c,e);" + exception "The specified column order is incorrect, `d` should come after `c`, because both of them are generated columns, and `d` refers to `c`." + } + qt_after_reorder_insert "insert into alter_column_gen_col(a,b,e) values(12,3,4);" + qt_reorder_rollup "alter table alter_column_gen_col order by (a,b) from r1" + waitSchemaChangeJob("alter_column_gen_col") + + // rename column + test { + sql "alter table alter_column_gen_col rename column c c1" + exception "" + } + qt_rename_gen_col "alter table alter_column_gen_col rename column d d1" + waitSchemaChangeJob("alter_column_gen_col") + qt_after_rename_insert "insert into alter_column_gen_col(a,b,e) values(16,2,4);" + qt_after_rename_insert_select "select * from alter_column_gen_col order by 1,2,3,4,5" +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy index 2b87307317641a..291916b9a448b2 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy @@ -50,7 +50,7 @@ suite("test_generated_column_fault_tolerance_nereids") { // gencol_refer_gencol_after test { sql """ - create table gencol_refer_gencol(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + create table gencol_refer_gencol_ft(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ @@ -110,7 +110,7 @@ suite("test_generated_column_fault_tolerance_nereids") { DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - exception "Generated Columns in aggregate table must be keys." + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." } test { @@ -162,11 +162,11 @@ suite("test_generated_column_fault_tolerance_nereids") { properties('replication_num' = '1'); """ test { sql """ALTER TABLE gen_col_test_modify modify COLUMN c int AS (a+b+1)""" - exception "Temporarily not supporting alter table modify generated columns." + exception "Not supporting alter table modify generated columns." } test { sql """ALTER TABLE gen_col_test_modify ADD COLUMN d int AS (a+b);""" - exception "Temporarily not supporting alter table add generated columns." + exception "Not supporting alter table add generated columns." } sql "drop table if exists test_gen_col_common_ft" diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy index e56e2405203c3e..7bf58260d1f046 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy @@ -203,7 +203,7 @@ suite("test_create_table_generated_column_legacy") { DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1"); """ - exception "Generated Columns in aggregate table must be keys." + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." } // test drop dependency @@ -226,4 +226,23 @@ suite("test_create_table_generated_column_legacy") { sql "alter table gencol_refer_gencol_legacy drop column c" sql "alter table gencol_refer_gencol_legacy drop column b" qt_test_drop_column "select * from gencol_refer_gencol_legacy" + test { + sql """ + create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + test { + sql """ + create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot be auto_increment." + } + } \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy new file mode 100644 index 00000000000000..09b4b3dfaa2b81 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_delete") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists test_par_gen_col; + create table test_par_gen_col (a int, b int, c int as (a+b)) + partition by range(c) + ( + partition p1 values [('1'),('10')), + partition p2 values [('10'),('20')), + partition p3 values [('20'),('30')) + ) + DISTRIBUTED BY RANDOM BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + insert into test_par_gen_col values(1,2,default),(10,2,default),(2,22,default); + + drop table if exists test_par_gen_col_unique; + create table test_par_gen_col_unique (a int, b int, c int as (a+b)) unique key(a,b,c) + partition by range(c) + ( + partition p1 values [('1'),('10')), + partition p2 values [('10'),('20')), + partition p3 values [('20'),('30')) + ) + DISTRIBUTED BY hash(c) + PROPERTIES ("replication_num" = "1"); + insert into test_par_gen_col_unique values(1,2,default),(10,2,default),(2,22,default),(10,2,default); + """ + + qt_delete_where_gen_col "delete from test_par_gen_col partition p1 where c=3;" + qt_delete_where_gen_col_select "select * from test_par_gen_col order by a,b,c;" + qt_delete_where_gen_col_partition_has_no_satisfied_row "delete from test_par_gen_col partition p1 where c=12;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_select "select * from test_par_gen_col order by a,b,c;;" + qt_delete_where_gen_col_and_other_col "delete from test_par_gen_col partition p2 where c=12 and a=10;" + qt_delete_where_gen_col_and_other_col_select "select * from test_par_gen_col order by a,b,c;;" + + qt_delete_where_gen_col_unique "delete from test_par_gen_col_unique partition p1 where c=3;" + qt_delete_where_gen_col_select_unique "select * from test_par_gen_col_unique order by a,b,c;;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_unique "delete from test_par_gen_col_unique partition p1 where c=12;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_select_unique "select * from test_par_gen_col_unique order by a,b,c;;" + qt_delete_where_gen_col_and_other_col_unique "delete from test_par_gen_col_unique partition p2 where c=12 and a=10;" + qt_delete_where_gen_col_and_other_col_select_unique "select * from test_par_gen_col_unique order by a,b,c;" + + qt_delete_query """delete from test_par_gen_col_unique t1 using test_par_gen_col t2 inner join test_par_gen_col t3 + on t2.b=t3.b where t1.c=t2.c and t1.b=t2.b""" + qt_delete_query_select "select * from test_par_gen_col_unique order by a,b,c;" + sql "insert into test_par_gen_col_unique values(1,2,default),(10,2,default),(2,22,default),(10,2,default);" + qt_delete_query_cte """ + with cte as( + select t2.* from + test_par_gen_col t2 inner join test_par_gen_col t3 on t2.b=t3.b + ) delete from test_par_gen_col_unique t1 using cte where t1.c=cte.c and t1.b=cte.b""" + qt_delete_query_cte_select "select * from test_par_gen_col_unique order by a,b,c" + + +} diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy index 659f13b304e6f7..c2c451b59ff7cc 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -197,4 +197,40 @@ suite("test_generated_column") { sql "alter table gencol_refer_gencol drop column b" qt_test_drop_column "select * from gencol_refer_gencol" + // test agg table, gen col is the key or replace/replace_if_not_null + multi_sql """ + drop table if exists agg_gen_col_key; + create table agg_gen_col_key(a int,b int, c int as (a+b), d int sum) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + insert into agg_gen_col_key(a,b) values(1,2); + insert into agg_gen_col_key(a,b,d) values(1,2,6); + """ + qt_test_agg_key "select * from agg_gen_col_key order by 1,2,3,4" + + sql "drop table if exists agg_gen_col_replace" + qt_agg_gen_col_replace """create table agg_gen_col_replace(a int,b int, c int as (a+b), d int replace as (c+1)) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + + multi_sql """ + drop table if exists agg_gen_col_replace_if_not_null; + create table agg_gen_col_replace_if_not_null(a int,b int, c int as (a+b), d int REPLACE_IF_NOT_NULL as (c+1), f int sum default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + insert into agg_gen_col_replace_if_not_null(a,b,f) values(1,2,3); + insert into agg_gen_col_replace_if_not_null(a,b) values(1,2); + """ + qt_agg_replace_null "select * from agg_gen_col_replace_if_not_null order by 1,2,3,4" + + test { + sql """create table agg_gen_col_replace(a int,b int, c int as (a+b), d int sum as (c+1)) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." + } } \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy new file mode 100644 index 00000000000000..5a5f696a0b50a7 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_insert_overwrite_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists gen_col_insert_overwrite_src; + CREATE TABLE gen_col_insert_overwrite_src(a int, b int, c int,d int) ENGINE=OLAP + UNIQUE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`a`) BUCKETS 10 properties("replication_num"="1"); + insert into gen_col_insert_overwrite_src values(1,2,3,5),(3,23,5,1); + + drop table if exists gen_col_insert_overwrite; + CREATE TABLE gen_col_insert_overwrite(a int, b int, c int AS(a+b), d int AS (c+1)) + ENGINE=OLAP + UNIQUE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`a`) BUCKETS 10 properties("replication_num"="1"); + INSERT into gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default); + """ + + qt_overwrite_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) values(3,4)" + qt_overwrite_value "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default);" + qt_overwrite_value_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_multi_value "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default);" + qt_overwrite_multi_value_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_multi_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) values(1,2),(3,4);" + qt_overwrite_multi_value_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_select_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) select 1,2;" + qt_overwrite_select_value_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_select_table_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) select a,b from gen_col_insert_overwrite_src;" + qt_overwrite_select_table_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + + multi_sql """ + drop table if exists gen_col_insert_overwrite_par; + CREATE TABLE IF NOT EXISTS gen_col_insert_overwrite_par ( + `c1` int NOT NULL DEFAULT "1", + `c2` int NOT NULL DEFAULT "4", + `c3` int as(c1+c2) + ) ENGINE=OLAP + UNIQUE KEY(`c1`) + PARTITION BY LIST (`c1`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`c1`) BUCKETS 3 + properties("replication_num"="1"); + """ + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1,p2) (c1,c2) VALUES (1, 2);" + qt_overwrite_partition_single_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1) (c1,c2) VALUES (1, 2);" + qt_overwrite_partition_partial_single_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1,p2) (c1,c2) VALUES (1, 2), (4, 2 * 2), (5, DEFAULT);" + qt_overwrite_partition_multi_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1) (c1) VALUES (1),(2),(3);" + qt_overwrite_partition_partial_multi_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(*) (c1) VALUES (1), (2),(3);" + qt_auto_partition_overwrite "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + test { + sql "insert into gen_col_insert_overwrite_par values(1,5,2),(4,3,3);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par values(1,5,2);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) values(1,5,2),(4,3,3);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) values(1,5,2);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test{ + sql "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,5,default),(3,4,DEFAULT,3);" + exception "The value specified for generated column 'c' in table 'gen_col_insert_overwrite' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par select 1,2,3;" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par select * from gen_col_insert_overwrite_src" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) select * from gen_col_insert_overwrite_src;" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy new file mode 100644 index 00000000000000..6b89934f1a2ca4 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_load_delete_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql""" + drop table if exists test_stream_delete; + create table test_stream_delete(a int, b int, c int as (a + b) , d int as (c + 1) ) + unique key(a, b, c) distributed by hash(a) properties("replication_num"="1");; + insert into test_stream_delete(a, b) values(1, 2) , (3, 5 ), (2, 9 ); + """ + + streamLoad { + table 'test_stream_delete' + set 'column_separator', ',' + file 'gen_col_data_delete.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'merge_type', 'DELETE' + } + sql "sync" + qt_test_stream_load_delete "select * from test_stream_delete order by 1,2,3,4" + + streamLoad { + table 'test_stream_delete' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'merge_type', 'MERGE' + set 'delete','a=3' + } + sql "sync" + qt_test_stream_load_merge "select * from test_stream_delete order by 1,2,3,4" + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy new file mode 100644 index 00000000000000..f8cd5cf567adff --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partial_update_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + sql "set enable_unique_key_partial_update=true" + sql "set enable_insert_strict=false" + + sql "drop table if exists test_partial_column_unique_gen_col" + sql """create table test_partial_column_unique_gen_col (a int, b int, c int as(a+b), d int as(c+1), e int) + unique key(a) distributed by hash(a) properties( + "enable_unique_key_merge_on_write" = "true", + "replication_num"="1" + );""" + + multi_sql """ + insert into test_partial_column_unique_gen_col(a,b,e) values(1,2,7); + insert into test_partial_column_unique_gen_col(a,b) select 2,3; + insert into test_partial_column_unique_gen_col(a,b) select 2,4; + insert into test_partial_column_unique_gen_col(a,b) values(1,6),(3,6); + insert into test_partial_column_unique_gen_col(a,b) values(1,5);""" + qt_partial_update_select "select * from test_partial_column_unique_gen_col order by 1,2,3,4,5" + + test { + sql "insert into test_partial_column_unique_gen_col(a) values(3);" + exception "Partial update should include all ordinary columns referenced by generated columns, missing: b" + } + + // test stream load partial update + multi_sql """ + truncate table test_partial_column_unique_gen_col; + insert into test_partial_column_unique_gen_col(a,b,e) values(1,4,0),(3,2,8),(2,9,null); + """ + + streamLoad { + table 'test_partial_column_unique_gen_col' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'partial_columns', 'true' + } + sql "sync" + qt_test_stream_load_refer_gencol "select * from test_partial_column_unique_gen_col order by 1,2,3,4" + + streamLoad { + table 'test_partial_column_unique_gen_col' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,e' + time 10000 // limit inflight 10s + set 'partial_columns', 'true' + check {result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + mustContain(json.Message, "Partial update should include all ordinary columns referenced by generated columns, missing: b") + } + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy new file mode 100644 index 00000000000000..3ff05817d965ed --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_update") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + // duplicate table + sql "drop table if exists test_gen_col_update2" + sql """create table test_gen_col_update2(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "insert into test_gen_col_update2 values(1,default,5,default);" + sql "insert into test_gen_col_update2(a,b) values(4,5);" + + sql "drop table if exists test_gen_col_update2" + sql """CREATE TABLE `test_gen_col_update2` ( + `id` INT NULL, + `c1` DECIMAL(10, 4) NULL, + `c2` TEXT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `c1`) + DISTRIBUTED BY RANDOM BUCKETS 10 + PROPERTIES ("replication_num" = "1");""" + sql "insert into test_gen_col_update2 values(1,2,'d'),(2,3,'ds'),(3,4,'aa')" + + test { + sql "update test_gen_col_update2 set a=9,b=10" + exception "Only unique table could be updated." + } + + // unique + sql "drop table if exists test_gen_col_update_unique" + sql """create table test_gen_col_update_unique(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + unique key(a) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "insert into test_gen_col_update_unique values(3,default,5,default);" + sql "insert into test_gen_col_update_unique(a,b) values(4,5);" + qt_update_gen_col "update test_gen_col_update_unique set b=10" + qt_update_select "select * from test_gen_col_update_unique order by a,b,c,d" + test { + sql "update test_gen_col_update_unique set c=100" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } + test { + sql "update test_gen_col_update_unique set c=(select 10)" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } + + qt_update_use_expr "update test_gen_col_update_unique set b=a+100 WHERE a=3;" + qt_update_use_expr_select "select * from test_gen_col_update_unique order by a,b,c,d" + + qt_update_use_other_table_col "update test_gen_col_update_unique t1 set t1.b=test_gen_col_update2.id from test_gen_col_update2 where t1.a=test_gen_col_update2.id;" + qt_update_use_expr_select "select * from test_gen_col_update_unique order by a,b,c,d" + + test{ + sql "update test_gen_col_update_unique t1 set t1.c=test_gen_col_update2.id from test_gen_col_update2 where t1.a=test_gen_col_update2.id;" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } +} \ No newline at end of file From b373e55d3fa90dae60dc5857bbdc24181862b525 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 20 Jun 2024 10:20:06 +0800 Subject: [PATCH 16/21] [Feat](nereids) support generated column --- .../alter_column_test_generated_column.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy index 45b0c3a47c3c12..d6b9ec5bf122ce 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -59,7 +59,7 @@ suite("alter_column_test_generated_column") { multi_sql """ drop table if exists alter_column_gen_col; create table alter_column_gen_col (a int, b int, c int as(a+b), d int as(c+1), e int) - duplicate key(a) distributed by hash(a) ; + duplicate key(a) distributed by hash(a) PROPERTIES("replication_num" = "1"); alter table alter_column_gen_col add rollup r1 (c,b,a) """ waitForMVJob("alter_column_gen_col", 3000) From fc0efd83a3a2f4a2a1208bdd808c86c71de96b30 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 20 Jun 2024 15:27:19 +0800 Subject: [PATCH 17/21] [Feat](nereids) support generated column --- .../alter_column_test_generated_column.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy index d6b9ec5bf122ce..6ac7bc0fcfa10d 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -21,11 +21,13 @@ suite("alter_column_test_generated_column") { while (tryTimes-- > 0) { def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ def jobState = jobResult[0][9].toString() + if (jobState == null) { + return; + } if ('cancelled'.equalsIgnoreCase(jobState)) { logger.info("jobResult:{}", jobResult) throw new IllegalStateException("${tableName}'s job has been cancelled") } - logger.info(jobState) if ('finished'.equalsIgnoreCase(jobState)) { logger.info("jobResult:{}", jobResult) sleep(3000) From b02f4bc9044fd96df730031ab0d908e9568bb3a4 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Thu, 20 Jun 2024 20:09:55 +0800 Subject: [PATCH 18/21] [Feat](nereids) support generated column --- .../alter_column_test_generated_column.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy index 6ac7bc0fcfa10d..73a4193771e2b5 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -20,10 +20,10 @@ suite("alter_column_test_generated_column") { int tryTimes = 30 while (tryTimes-- > 0) { def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ - def jobState = jobResult[0][9].toString() - if (jobState == null) { + if (jobResult == null) { return; } + def jobState = jobResult[0][9].toString() if ('cancelled'.equalsIgnoreCase(jobState)) { logger.info("jobResult:{}", jobResult) throw new IllegalStateException("${tableName}'s job has been cancelled") From df1b9f48d282e8383169d89ebec6556a051ad0a1 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 21 Jun 2024 11:06:43 +0800 Subject: [PATCH 19/21] [Feat](nereids) support generated column --- .../alter_column_test_generated_column.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy index 73a4193771e2b5..6d645d56d6d104 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -20,7 +20,7 @@ suite("alter_column_test_generated_column") { int tryTimes = 30 while (tryTimes-- > 0) { def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ - if (jobResult == null) { + if (jobResult == null || jobResult.isEmpty()) { return; } def jobState = jobResult[0][9].toString() From 49b90e8576688f71266d846b7c15aa8e4cef62a8 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 21 Jun 2024 14:11:32 +0800 Subject: [PATCH 20/21] [Feat](nereids) support generated column --- .../alter_column_test_generated_column.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy index 6d645d56d6d104..d6a6695bce6c1b 100644 --- a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -21,6 +21,7 @@ suite("alter_column_test_generated_column") { while (tryTimes-- > 0) { def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ if (jobResult == null || jobResult.isEmpty()) { + sleep(3000) return; } def jobState = jobResult[0][9].toString() From f6362644d11c65ad089ae799a19aecd01195d9d9 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 24 Jun 2024 20:44:03 +0800 Subject: [PATCH 21/21] [Feat](nereids) support generated column --- .../java/org/apache/doris/nereids/analyzer/UnboundTableSink.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java index ee515e21b07f27..23c58ba42fb17e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSink.java @@ -51,7 +51,6 @@ public class UnboundTableSink extends UnboundLogicalSin private boolean isPartialUpdate; private final DMLCommandType dmlCommandType; private final boolean autoDetectPartition; - private final boolean normalized = false; public UnboundTableSink(List nameParts, List colNames, List hints, List partitions, CHILD_TYPE child) {