diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 8b256a7b07cda4..9971d277ebebcc 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -115,6 +115,7 @@ AT: 'AT'; AUTHORS: 'AUTHORS'; AUTO: 'AUTO'; AUTO_INCREMENT: 'AUTO_INCREMENT'; +ALWAYS: 'ALWAYS'; BACKEND: 'BACKEND'; BACKENDS: 'BACKENDS'; BACKUP: 'BACKUP'; @@ -276,6 +277,7 @@ FRONTENDS: 'FRONTENDS'; FULL: 'FULL'; FUNCTION: 'FUNCTION'; FUNCTIONS: 'FUNCTIONS'; +GENERATED: 'GENERATED'; GENERIC: 'GENERIC'; GLOBAL: 'GLOBAL'; GRANT: 'GRANT'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 8ac823200ee3a0..68da90f9da1ee3 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -585,6 +585,7 @@ columnDef : colName=identifier type=dataType KEY? (aggType=aggTypeDef)? + ((GENERATED ALWAYS)? AS LEFT_PAREN generatedExpr=expression RIGHT_PAREN)? ((NOT)? nullable=NULL)? (AUTO_INCREMENT (LEFT_PAREN autoIncInitValue=number RIGHT_PAREN)?)? (DEFAULT (nullValue=NULL | INTEGER_VALUE | DECIMAL_VALUE | stringValue=STRING_LITERAL @@ -592,7 +593,7 @@ columnDef (ON UPDATE CURRENT_TIMESTAMP (LEFT_PAREN onUpdateValuePrecision=number RIGHT_PAREN)?)? (COMMENT comment=STRING_LITERAL)? ; - + indexDefs : indexes+=indexDef (COMMA indexes+=indexDef)* ; @@ -1028,6 +1029,7 @@ nonReserved | AGG_STATE | AGGREGATE | ALIAS + | ALWAYS | ANALYZED | ARRAY | ARRAY_RANGE @@ -1142,6 +1144,7 @@ nonReserved | FREE | FRONTENDS | FUNCTION + | GENERATED | GENERIC | GLOBAL | GRAPH diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 151e1c1f5f97ee..d423475267e6f4 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -43,6 +43,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; @@ -62,6 +63,7 @@ import org.apache.doris.resource.workloadschedpolicy.WorkloadActionMeta; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import java.util.Optional; import java.util.stream.Collectors; import java_cup.runtime.Symbol; @@ -261,6 +263,7 @@ terminal String KW_ALIAS, KW_ALL, KW_ALTER, + KW_ALWAYS, KW_ANALYZE, KW_AND, KW_ANTI, @@ -408,6 +411,7 @@ terminal String KW_FULL, KW_FUNCTION, KW_FUNCTIONS, + KW_GENERATED, KW_GENERIC, KW_GLOBAL, KW_GRANT, @@ -1004,6 +1008,9 @@ nonterminal String stage_name; nonterminal StageAndPattern stage_and_pattern; nonterminal List copy_select_expr_list; +//genearted column +nonterminal Boolean opt_generated_always; + precedence nonassoc COMMA; precedence nonassoc STRING_LITERAL; precedence nonassoc KW_COLUMNS; @@ -3968,6 +3975,11 @@ column_definition ::= ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, aggType, nullable_type, autoIncInitValue, defaultValue, comment); RESULT = columnDef; :} + | ident:columnName type_def:typeDef opt_is_key:isKey opt_generated_always KW_AS LPAREN expr:expr RPAREN opt_nullable_type:nullable_type opt_comment:comment + {: + ColumnDef columnDef = new ColumnDef(columnName, typeDef, isKey, nullable_type, comment, Optional.of(new GeneratedColumnInfo(null, expr))); + RESULT = columnDef; + :} ; index_definition ::= @@ -7942,7 +7954,16 @@ opt_work ::= RESULT = null; :} ; - +opt_generated_always ::= + /* empty */ + {: + RESULT = null; + :} + | KW_GENERATED KW_ALWAYS + {: + RESULT = null; + :} + ; opt_chain ::= {: RESULT = null; @@ -7982,6 +8003,8 @@ type_func_name_keyword ::= keyword ::= KW_AFTER:id {: RESULT = id; :} + | KW_ALWAYS:id + {: RESULT = id; :} | KW_AGGREGATE:id {: RESULT = id; :} | KW_ALIAS:id @@ -8142,6 +8165,8 @@ keyword ::= {: RESULT = id; :} | KW_GLOBAL:id {: RESULT = id; :} + | KW_GENERATED:id + {: RESULT = id; :} | KW_GENERIC:id {: RESULT = id; :} | KW_GRAPH:id diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 24d373a0d5f34a..fc3e5d97834244 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DropColumnClause; import org.apache.doris.analysis.DropIndexClause; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.ModifyColumnClause; import org.apache.doris.analysis.ModifyTablePropertiesClause; @@ -42,6 +43,7 @@ import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; @@ -377,6 +379,23 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa } } + // generated column check + Map nameToColumn = new HashMap<>(); + for (Column c : indexSchemaMap.get(baseIndexId)) { + nameToColumn.put(c.getName(), c); + } + if (null == targetIndexName) { + if (nameToColumn.containsKey(dropColName)) { + Column column = nameToColumn.get(dropColName); + Set generatedColumnsThatReferToThis = column.getGeneratedColumnsThatReferToThis(); + if (!generatedColumnsThatReferToThis.isEmpty()) { + throw new DdlException( + "Column '" + dropColName + "' has a generated column dependency on :" + + generatedColumnsThatReferToThis); + } + } + } + Iterator it = indexes.iterator(); while (it.hasNext()) { Index index = it.next(); @@ -403,6 +422,8 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa if (column.getName().equalsIgnoreCase(dropColName)) { baseIter.remove(); found = true; + // find generated column referred column + removeColumnWhenDropGeneratedColumn(column, nameToColumn); break; } } @@ -528,6 +549,7 @@ private boolean processModifyColumn(ModifyColumnClause alterClause, OlapTable ol Map> indexSchemaMap) throws DdlException { Column modColumn = alterClause.getColumn(); boolean lightSchemaChange = false; + if (KeysType.AGG_KEYS == olapTable.getKeysType()) { if (modColumn.isKey() && null != modColumn.getAggregationType()) { throw new DdlException("Can not assign aggregation method on key column: " + modColumn.getName()); @@ -784,12 +806,15 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol if (targetIndexName == null) { targetIndexName = baseIndexName; } - long targetIndexId = olapTable.getIndexIdByName(targetIndexName); LinkedList newSchema = new LinkedList(); List targetIndexSchema = indexSchemaMap.get(targetIndexId); - + // When rollup is specified, there is no need to check the order of generated columns. + // When rollup is not specified and the order of baseIndex needs to be modified, the order needs to be checked. + if (alterClause.getRollupName() == null) { + checkOrder(targetIndexSchema, orderedColNames); + } // check and create new ordered column list Set colNameSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (String colName : orderedColNames) { @@ -1004,6 +1029,10 @@ private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnP } } + if (newColumn.getGeneratedColumnInfo() != null) { + throw new DdlException("Not supporting alter table add generated columns."); + } + /* * add new column to indexes. * UNIQUE: @@ -3194,4 +3223,61 @@ public boolean updateBinlogConfig(Database db, OlapTable olapTable, List nameToColumn) { + GeneratedColumnInfo generatedColumnInfo = dropColumn.getGeneratedColumnInfo(); + if (generatedColumnInfo == null) { + return; + } + String dropColName = dropColumn.getName(); + Expr expr = generatedColumnInfo.getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRef : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRef).getColumnName(); + if (!nameToColumn.containsKey(name)) { + continue; + } + Column c = nameToColumn.get(name); + Set sets = c.getGeneratedColumnsThatReferToThis(); + sets.remove(dropColName); + } + } + + private void checkOrder(List targetIndexSchema, List orderedColNames) throws DdlException { + Set nameSet = new HashSet<>(); + for (Column column : targetIndexSchema) { + if (column.isVisible() && null == column.getGeneratedColumnInfo()) { + nameSet.add(column.getName()); + } + } + for (String colName : orderedColNames) { + Column oneCol = null; + for (Column column : targetIndexSchema) { + if (column.getName().equalsIgnoreCase(colName) && column.isVisible()) { + oneCol = column; + break; + } + } + if (oneCol == null) { + throw new DdlException("Column[" + colName + "] not exists"); + } + if (null == oneCol.getGeneratedColumnInfo()) { + continue; + } + Expr expr = oneCol.getGeneratedColumnInfo().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + expr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRef : slotRefsInGeneratedExpr) { + String slotName = ((SlotRef) slotRef).getColumnName(); + if (!nameSet.contains(slotName)) { + throw new DdlException("The specified column order is incorrect, `" + colName + + "` should come after `" + slotName + + "`, because both of them are generated columns, and `" + + colName + "` refers to `" + slotName + "`."); + } + } + nameSet.add(colName); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 6dd00bfe58d01d..f306bb782fe3a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; @@ -38,6 +39,10 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; // Column definition which is generated by SQL syntax parser // Syntax: @@ -115,7 +120,7 @@ public DefaultValue(boolean isSet, String value, String exprName, Long precision public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precision) { if (precision > ScalarType.MAX_DATETIMEV2_SCALE || precision < 0) { throw new IllegalArgumentException("column's default value current_timestamp" - + " precision must be between 0 and 6"); + + " precision must be between 0 and 6"); } if (precision == 0) { return new DefaultValue(true, CURRENT_TIMESTAMP, NOW); @@ -141,6 +146,10 @@ public long getCurrentTimeStampPrecision() { return 0; } + public boolean isNullDefaultValue() { + return !isSet && value == null && defaultValueExprDef == null; + } + public String getValue() { if (isCurrentTimeStamp()) { return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' '); @@ -183,6 +192,9 @@ public String getValue() { private String comment; private boolean visible; private int clusterKeyId = -1; + private Optional generatedColumnInfo = Optional.empty(); + private Set generatedColumnsThatReferToThis = new HashSet<>(); + public ColumnDef(String name, TypeDef typeDef) { this(name, typeDef, false, null, ColumnNullableType.NOT_NULLABLE, DefaultValue.NOT_SET, ""); @@ -198,17 +210,19 @@ public ColumnDef(String name, TypeDef typeDef, ColumnNullableType nullableType) public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, long autoIncInitValue, DefaultValue defaultValue, String comment) { - this(name, typeDef, isKey, aggregateType, nullableType, autoIncInitValue, defaultValue, comment, true); + this(name, typeDef, isKey, aggregateType, nullableType, autoIncInitValue, defaultValue, comment, true, + Optional.empty()); } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, DefaultValue defaultValue, String comment) { - this(name, typeDef, isKey, aggregateType, nullableType, -1, defaultValue, comment, true); + this(name, typeDef, isKey, aggregateType, nullableType, -1, defaultValue, comment, true, + Optional.empty()); } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, ColumnNullableType nullableType, long autoIncInitValue, DefaultValue defaultValue, String comment, - boolean visible) { + boolean visible, Optional generatedColumnInfo) { this.name = name; this.typeDef = typeDef; this.isKey = isKey; @@ -221,6 +235,7 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr this.defaultValue = defaultValue; this.comment = comment; this.visible = visible; + this.generatedColumnInfo = generatedColumnInfo; } public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggregateType, boolean isAllowNull, @@ -242,44 +257,51 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr this.visible = visible; } + public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType nullableType, String comment, + Optional generatedColumnInfo) { + this(name, typeDef, isKey, null, nullableType, -1, DefaultValue.NOT_SET, + comment, true, generatedColumnInfo); + } + public static ColumnDef newDeleteSignColumnDef() { return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris delete flag hidden column", false); + "doris delete flag hidden column", false, Optional.empty()); } public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris delete flag hidden column", false); + "doris delete flag hidden column", false, Optional.empty()); } public static ColumnDef newSequenceColumnDef(Type type) { return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1, - DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false); + DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); } public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) { return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE, - -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false); + -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, + Optional.empty()); } public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""), - "doris row store hidden column", false); + "doris row store hidden column", false, Optional.empty()); } public static ColumnDef newVersionColumnDef() { return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris version hidden column", false); + "doris version hidden column", false, Optional.empty()); } public static ColumnDef newVersionColumnDef(AggregateType aggregateType) { return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), - "doris version hidden column", false); + "doris version hidden column", false, Optional.empty()); } public boolean isAllowNull() { @@ -411,9 +433,9 @@ public void analyze(boolean isOlap) throws AnalysisException { + " the duplicate table at present."); } if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE - && !defaultValue.value.equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value)) { + && !defaultValue.value.equals(DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value)) { throw new AnalysisException("Array type column default value only support null or " - + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value); + + DefaultValue.ARRAY_EMPTY_DEFAULT_VALUE.value); } } if (isKey() && type.getPrimitiveType() == PrimitiveType.STRING && isOlap) { @@ -470,6 +492,7 @@ public void analyze(boolean isOlap) throws AnalysisException { if (type.isScalarType() && defaultValue.isSet && defaultValue.value != null) { validateDefaultValue(type, defaultValue.value, defaultValue.defaultValueExprDef); } + validateGeneratedColumnInfo(); } @SuppressWarnings("checkstyle:Indentation") @@ -626,7 +649,7 @@ public Column toColumn() { Type type = typeDef.getType(); return new Column(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue.value, comment, visible, defaultValue.defaultValueExprDef, Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.getValue(), - clusterKeyId); + clusterKeyId, generatedColumnInfo.orElse(null), generatedColumnsThatReferToThis); } @Override @@ -637,4 +660,28 @@ public String toString() { public void setAllowNull(boolean allowNull) { isAllowNull = allowNull; } + + public Optional getGeneratedColumnInfo() { + return generatedColumnInfo; + } + + public long getAutoIncInitValue() { + return autoIncInitValue; + } + + public void addGeneratedColumnsThatReferToThis(List list) { + generatedColumnsThatReferToThis.addAll(list); + } + + private void validateGeneratedColumnInfo() throws AnalysisException { + // for generated column + if (generatedColumnInfo.isPresent()) { + if (autoIncInitValue != -1) { + throw new AnalysisException("Generated columns cannot be auto_increment."); + } + if (defaultValue != null && !defaultValue.isNullDefaultValue()) { + throw new AnalysisException("Generated columns cannot have default value."); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 6398d171d0eb62..b4972931fb79ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.Index; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PrimitiveType; @@ -35,15 +36,20 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.AutoBucketUtils; +import org.apache.doris.common.util.GeneratedColumnUtil; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.rewrite.ExprRewriteRule; +import org.apache.doris.rewrite.ExprRewriter; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -53,11 +59,13 @@ import java.io.DataInput; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -66,6 +74,8 @@ public class CreateTableStmt extends DdlStmt { private static final Logger LOG = LogManager.getLogger(CreateTableStmt.class); protected static final String DEFAULT_ENGINE_NAME = "olap"; + private static final ImmutableSet GENERATED_COLUMN_ALLOW_AGG_TYPE = + ImmutableSet.of(AggregateType.REPLACE, AggregateType.REPLACE_IF_NOT_NULL); protected boolean ifNotExists; private boolean isExternal; @@ -585,6 +595,7 @@ public void analyze(Analyzer analyzer) throws UserException { throw new AnalysisException("same index columns have multiple same type index is not allowed."); } } + generatedColumnCheck(analyzer); } private void analyzeEngineName() throws AnalysisException { @@ -713,4 +724,157 @@ public String toString() { public boolean needAuditEncryption() { return !engineName.equals("olap"); } + + private void generatedColumnCheck(Analyzer analyzer) throws AnalysisException { + generatedColumnCommonCheck(); + Map> nameToColumnDef = Maps.newHashMap(); + for (int i = 0; i < columnDefs.size(); i++) { + ColumnDef columnDef = columnDefs.get(i); + nameToColumnDef.put(columnDef.getName(), Pair.of(columnDef, i)); + } + SlotRefRewriteRule.initializeslotRefMap(nameToColumnDef); + List exprAndnames = Lists.newArrayList(); + for (int i = 0; i < columnDefs.size(); i++) { + ColumnDef columnDef = columnDefs.get(i); + if (!columnDef.getGeneratedColumnInfo().isPresent()) { + continue; + } + SlotRefRewriteRule slotRefRewriteRule = new SlotRefRewriteRule(i); + ExprRewriter rewriter = new ExprRewriter(slotRefRewriteRule); + GeneratedColumnInfo generatedColumnInfo = columnDef.getGeneratedColumnInfo().get(); + Expr expr = rewriter.rewrite(generatedColumnInfo.getExpr(), analyzer); + expr.foreach(e -> { + if (e instanceof LambdaFunctionCallExpr) { + throw new AnalysisException("Generated column does not support lambda."); + } else if (e instanceof Subquery) { + throw new AnalysisException("Generated column does not support subquery."); + } + }); + try { + expr.analyze(analyzer); + } catch (AnalysisException e) { + throw new AnalysisException("In generated column '" + columnDef.getName() + "', " + + Utils.convertFirstChar(e.getDetailMessage())); + } + expr.foreach(e -> { + if (e instanceof VariableExpr) { + throw new AnalysisException("Generated column expression cannot contain variable."); + } else if (e instanceof SlotRef && nameToColumnDef.containsKey(((SlotRef) e).getColumnName())) { + ColumnDef refColumnDef = nameToColumnDef.get(((SlotRef) e).getColumnName()).first; + if (refColumnDef.getAutoIncInitValue() != -1) { + throw new AnalysisException("Generated column '" + columnDef.getName() + + "' cannot refer to auto-increment column."); + } + } else if (e instanceof FunctionCallExpr && !checkFunctionInGeneratedColumn((FunctionCallExpr) e)) { + throw new AnalysisException( + "Expression of generated column '" + + columnDef.getName() + "' contains a disallowed function:'" + + ((FunctionCallExpr) e).getFnName() + "'"); + } else if (e instanceof AnalyticExpr) { + throw new AnalysisException( + "Expression of generated column '" + + columnDef.getName() + "' contains a disallowed expression:'" + + ((AnalyticExpr) e).toSqlWithoutTbl() + "'"); + } + }); + if (!Type.canCastTo(expr.getType(), columnDef.getType())) { + throw new AnalysisException("can not cast from origin type " + + expr.getType() + " to target type=" + columnDef.getType()); + } + exprAndnames.add(new GeneratedColumnUtil.ExprAndname(expr, columnDef.getName())); + } + + // for alter drop column + Map> columnToListOfGeneratedColumnsThatReferToThis = new HashMap<>(); + for (ColumnDef column : columnDefs) { + Optional info = column.getGeneratedColumnInfo(); + if (!info.isPresent()) { + continue; + } + Expr generatedExpr = info.get().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + generatedExpr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRefExpr : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRefExpr).getColumnName(); + columnToListOfGeneratedColumnsThatReferToThis + .computeIfAbsent(name, k -> new ArrayList<>()) + .add(column.getName()); + } + } + for (Map.Entry> entry : columnToListOfGeneratedColumnsThatReferToThis.entrySet()) { + ColumnDef col = nameToColumnDef.get(entry.getKey()).first; + col.addGeneratedColumnsThatReferToThis(entry.getValue()); + } + + GeneratedColumnUtil.rewriteColumns(exprAndnames); + for (GeneratedColumnUtil.ExprAndname exprAndname : exprAndnames) { + if (nameToColumnDef.containsKey(exprAndname.getName())) { + ColumnDef columnDef = nameToColumnDef.get(exprAndname.getName()).first; + Optional info = columnDef.getGeneratedColumnInfo(); + info.ifPresent(columnInfo -> columnInfo.setExpandExprForLoad(exprAndname.getExpr())); + } + } + } + + private boolean checkFunctionInGeneratedColumn(FunctionCallExpr funcExpr) { + if (!funcExpr.isScalarFunction()) { + return false; + } + if (funcExpr.fn.isUdf()) { + return false; + } + if (funcExpr instanceof GroupingFunctionCallExpr) { + return false; + } + return true; + } + + public static final class SlotRefRewriteRule implements ExprRewriteRule { + private static Map> nameToColumnDefMap = new HashMap<>(); + private final int index; + + public SlotRefRewriteRule(int index) { + this.index = index; + } + + public static void initializeslotRefMap(Map> map) { + nameToColumnDefMap = map; + } + + @Override + public Expr apply(Expr expr, Analyzer analyzer, ExprRewriter.ClauseType clauseType) throws AnalysisException { + if (!(expr instanceof SlotRef)) { + return expr; + } + SlotRef slotRef = (SlotRef) expr; + if (!nameToColumnDefMap.containsKey(slotRef.getColumnName())) { + throw new AnalysisException("Unknown column '" + slotRef.getColumnName() + + "' in 'generated column function'"); + } + Pair columnAndIdx = nameToColumnDefMap.get(slotRef.getColumnName()); + ColumnDef columnDef = columnAndIdx.first; + if (columnDef.getGeneratedColumnInfo().isPresent() && columnAndIdx.second >= index) { + throw new AnalysisException( + "Generated column can refer only to generated columns defined prior to it."); + } + slotRef.setType(columnDef.getType()); + slotRef.setAnalyzed(true); + return slotRef; + } + } + + private void generatedColumnCommonCheck() throws AnalysisException { + for (ColumnDef column : columnDefs) { + if (keysDesc != null && keysDesc.getKeysType() == KeysType.AGG_KEYS + && column.getGeneratedColumnInfo().isPresent() + && (!column.isKey() && !GENERATED_COLUMN_ALLOW_AGG_TYPE.contains(column.getAggregateType()))) { + throw new AnalysisException("The generated columns can be key columns, " + + "or value columns of replace and replace_if_not_null aggregation type."); + } + if (column.getGeneratedColumnInfo().isPresent() && !engineName.equalsIgnoreCase("olap")) { + throw new AnalysisException( + "Tables can only have generated columns if the olap engine is used"); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index c0c4f8d049b7d3..2e635beaee139b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -686,6 +686,17 @@ public String toSqlImpl() { sb.append(((FunctionCallExpr) expr).fnName); sb.append(" "); sb.append(children.get(1).toSql()); + } else if (fnName.getFunction().equalsIgnoreCase("encryptkeyref")) { + sb.append("key "); + for (int i = 0; i < children.size(); i++) { + String str = ((StringLiteral) children.get(i)).getValue(); + if (str.isEmpty()) { + continue; + } + sb.append(str); + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); } else { sb.append(((FunctionCallExpr) expr).fnName); sb.append(paramsToSql()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index ca710188f5f553..a926d9a9962766 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -51,6 +51,7 @@ import java.io.DataOutput; import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -145,6 +146,12 @@ public class Column implements Writable, GsonPostProcessable { @SerializedName(value = "onUpdateDefaultValueExprDef") private DefaultValueExprDef onUpdateDefaultValueExprDef; + @SerializedName(value = "gci") + private GeneratedColumnInfo generatedColumnInfo; + + @SerializedName(value = "gctt") + private Set generatedColumnsThatReferToThis = new HashSet<>(); + public Column() { this.name = ""; this.type = Type.NULL; @@ -185,39 +192,41 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, true, null, - COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, false, null); + COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue, false, null, null, + Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, null, comment, visible, null, colUniqueId, null, - false, null); + false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null, - false, null); + false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, defaultValueExprDef, - colUniqueId, realDefaultValue, false, null); + colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, - defaultValueExprDef, colUniqueId, realDefaultValue, false, null); + defaultValueExprDef, colUniqueId, realDefaultValue, false, null, null, Sets.newHashSet()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, - boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef) { + boolean hasOnUpdateDefaultValue, DefaultValueExprDef onUpdateDefaultValueExprDef, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this.name = name; if (this.name == null) { this.name = ""; @@ -245,6 +254,8 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType this.uniqueId = colUniqueId; this.hasOnUpdateDefaultValue = hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = onUpdateDefaultValueExprDef; + this.generatedColumnInfo = generatedColumnInfo; + this.generatedColumnsThatReferToThis.addAll(generatedColumnsThatReferToThis); if (type.isAggStateType()) { AggStateType aggState = (AggStateType) type; @@ -262,18 +273,22 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, boolean hasOnUpdateDefaultValue, - DefaultValueExprDef onUpdateDefaultValueExprDef, int clusterKeyId) { + DefaultValueExprDef onUpdateDefaultValueExprDef, int clusterKeyId, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, defaultValueExprDef, colUniqueId, realDefaultValue, - hasOnUpdateDefaultValue, onUpdateDefaultValueExprDef); + hasOnUpdateDefaultValue, onUpdateDefaultValueExprDef, generatedColumnInfo, + generatedColumnsThatReferToThis); this.clusterKeyId = clusterKeyId; } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, long autoIncInitValue, String defaultValue, String comment, boolean visible, - DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, int clusterKeyId) { + DefaultValueExprDef defaultValueExprDef, int colUniqueId, String realDefaultValue, int clusterKeyId, + GeneratedColumnInfo generatedColumnInfo, Set generatedColumnsThatReferToThis) { this(name, type, isKey, aggregateType, isAllowNull, autoIncInitValue, defaultValue, comment, visible, - defaultValueExprDef, colUniqueId, realDefaultValue); + defaultValueExprDef, colUniqueId, realDefaultValue, false, null, generatedColumnInfo, + generatedColumnsThatReferToThis); this.clusterKeyId = clusterKeyId; } @@ -299,6 +314,7 @@ public Column(Column column) { this.hasOnUpdateDefaultValue = column.hasOnUpdateDefaultValue; this.onUpdateDefaultValueExprDef = column.onUpdateDefaultValueExprDef; this.clusterKeyId = column.getClusterKeyId(); + this.generatedColumnInfo = column.generatedColumnInfo; } public void createChildrenColumn(Type type, Column column) { @@ -848,6 +864,10 @@ public void checkSchemaChangeAllowed(Column other) throws DdlException { return; } // TODO check cluster key + + if (generatedColumnInfo != null || other.getGeneratedColumnInfo() != null) { + throw new DdlException("Not supporting alter table modify generated columns."); + } } public boolean nameEquals(String otherColName, boolean ignorePrefix) { @@ -937,6 +957,9 @@ public String toSql(boolean isUniqueTable, boolean isCompatible) { && !isAggregationTypeImplicit) { sb.append(" ").append(aggregationType.toSql()); } + if (generatedColumnInfo != null) { + sb.append(" AS (").append(generatedColumnInfo.getExpr().toSql()).append(")"); + } if (isAllowNull) { sb.append(" NULL"); } else { @@ -1166,4 +1189,12 @@ public boolean isMaterializedViewColumn() { return getName().startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX) || getName().startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX); } + + public GeneratedColumnInfo getGeneratedColumnInfo() { + return generatedColumnInfo; + } + + public Set getGeneratedColumnsThatReferToThis() { + return generatedColumnsThatReferToThis; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 4b33b2079b4fdf..4ab4c11d6c89f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -4911,7 +4911,14 @@ private void renameColumn(Database db, OlapTable table, String colName, } // check if have materialized view on rename column + // and check whether colName is referenced by generated columns for (Column column : entry.getValue().getSchema()) { + if (column.getName().equals(colName) && !column.getGeneratedColumnsThatReferToThis().isEmpty()) { + throw new DdlException( + "Cannot rename column, because column '" + colName + + "' has a generated column dependency on :" + + column.getGeneratedColumnsThatReferToThis()); + } Expr expr = column.getDefineExpr(); if (expr == null) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java new file mode 100644 index 00000000000000..b14343a5c9f5f8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GeneratedColumnInfo.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.analysis.Expr; + +import com.google.gson.annotations.SerializedName; +import jline.internal.Nullable; + +/**GeneratedColumnInfo*/ +public class GeneratedColumnInfo { + /**GeneratedColumnType*/ + public enum GeneratedColumnType { + VIRTUAL, + STORED + } + + @SerializedName(value = "t") + private final GeneratedColumnType type; + @SerializedName(value = "es") + private final String exprSql; + @SerializedName(value = "e") + private final Expr expr; + + /* e.g. a,b,c=a+b,d=c+1 -> a,b,c=a+b,d=a+b+1 + e.g. this is column d generated column info + expr is c+1, expandExprForLoad is a+b+1 + expandExprForLoad is used in streamload, routineload, mysqlload, etc */ + @SerializedName(value = "efl") + private Expr expandExprForLoad; + + /** constructor */ + public GeneratedColumnInfo(@Nullable String exprSql, Expr expr) { + this(exprSql, expr, null); + } + + public GeneratedColumnInfo(@Nullable String exprSql, Expr expr, Expr expandExprForLoad) { + if (exprSql != null) { + this.exprSql = exprSql; + } else { + this.exprSql = expr.toSqlWithoutTbl(); + } + this.expr = expr; + this.expandExprForLoad = expandExprForLoad; + this.type = GeneratedColumnType.STORED; + } + + public String getExprSql() { + return exprSql; + } + + public Expr getExpr() { + return expr; + } + + public Expr getExpandExprForLoad() { + return expandExprForLoad; + } + + public void setExpandExprForLoad(Expr expandExprForLoad) { + this.expandExprForLoad = expandExprForLoad; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java index 7693acf3bb33d2..8070505798b98c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java @@ -239,4 +239,14 @@ public C findFirstOf(Class cl) { return null; } + public interface ThrowingConsumer { + void accept(T t) throws AnalysisException; + } + + public void foreach(ThrowingConsumer> func) throws AnalysisException { + func.accept(this); + for (NodeType child : getChildren()) { + child.foreach(func); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java b/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java index 0ccb2b1c17e4b6..09421e108534ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/UserException.java @@ -73,4 +73,8 @@ public void setMysqlErrorCode(ErrorCode mysqlErrorCode) { public String getMessage() { return errorCode + ", detailMessage = " + super.getMessage(); } + + public String getDetailMessage() { + return super.getMessage(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 0f2fb911d4658b..7ffff0c449ac76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -65,6 +65,9 @@ public static ProcResult createResult(List schema, Set bfColumns if (column.isAutoInc()) { extras.add("AUTO_INCREMENT"); } + if (column.getGeneratedColumnInfo() != null) { + extras.add("STORED GENERATED"); + } String extraStr = StringUtils.join(extras, ","); List rowList = Arrays.asList(column.getDisplayName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java new file mode 100644 index 00000000000000..331a8db49cbffe --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ExprUtil.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; + +import org.apache.commons.collections.CollectionUtils; + +import java.util.Map; + +public class ExprUtil { + + public static void recursiveRewrite(Expr expr, Map derivativeColumns) { + if (CollectionUtils.isEmpty(expr.getChildren())) { + return; + } + for (int i = 0; i < expr.getChildren().size(); i++) { + Expr e = expr.getChild(i); + if (e instanceof SlotRef) { + String columnName = ((SlotRef) e).getColumnName(); + if (derivativeColumns.containsKey(columnName)) { + expr.setChild(i, derivativeColumns.get(columnName)); + } + } else { + recursiveRewrite(e, derivativeColumns); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java new file mode 100644 index 00000000000000..59d61b76dcf1e1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/GeneratedColumnUtil.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; + +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; + +public class GeneratedColumnUtil { + public static class ExprAndname { + private Expr expr; + private String name; + + public ExprAndname(Expr expr, String name) { + this.expr = expr; + this.name = name; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + public static void rewriteColumns(List exprAndnames) { + Map nameToExprMap = Maps.newHashMap(); + for (ExprAndname exprAndname : exprAndnames) { + if (exprAndname.getExpr() instanceof SlotRef) { + String columnName = ((SlotRef) exprAndname.getExpr()).getColumnName(); + if (nameToExprMap.containsKey(columnName)) { + exprAndname.setExpr(nameToExprMap.get(columnName)); + } + } else { + ExprUtil.recursiveRewrite(exprAndname.getExpr(), nameToExprMap); + } + nameToExprMap.put(exprAndname.getName(), exprAndname.getExpr()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index 398fba0d261662..0323f3ea62ee15 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -37,6 +37,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -61,6 +62,7 @@ import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.PatternMatcherWrapper; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.ExprUtil; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; @@ -77,7 +79,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.Gson; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -354,6 +355,11 @@ private static void initColumns(Table tbl, List columnExprs, for (Column column : tbl.getBaseSchema()) { String columnName = column.getName(); colToType.put(columnName, column.getType()); + if (column.getGeneratedColumnInfo() != null) { + GeneratedColumnInfo info = column.getGeneratedColumnInfo(); + exprsByName.put(column.getName(), info.getExpandExprForLoad()); + continue; + } if (columnExprMap.containsKey(columnName)) { continue; } @@ -617,7 +623,7 @@ public static void rewriteColumns(LoadTaskInfo.ImportColumnDescs columnDescs) { importColumnDesc.setExpr(derivativeColumns.get(columnName)); } } else { - recursiveRewrite(importColumnDesc.getExpr(), derivativeColumns); + ExprUtil.recursiveRewrite(importColumnDesc.getExpr(), derivativeColumns); } derivativeColumns.put(importColumnDesc.getColumnName(), importColumnDesc.getExpr()); } @@ -626,23 +632,6 @@ public static void rewriteColumns(LoadTaskInfo.ImportColumnDescs columnDescs) { columnDescs.isColumnDescsRewrited = true; } - private static void recursiveRewrite(Expr expr, Map derivativeColumns) { - if (CollectionUtils.isEmpty(expr.getChildren())) { - return; - } - for (int i = 0; i < expr.getChildren().size(); i++) { - Expr e = expr.getChild(i); - if (e instanceof SlotRef) { - String columnName = ((SlotRef) e).getColumnName(); - if (derivativeColumns.containsKey(columnName)) { - expr.setChild(i, derivativeColumns.get(columnName)); - } - } else { - recursiveRewrite(e, derivativeColumns); - } - } - } - /** * This method is used to transform hadoop function. * The hadoop function includes: replace_value, strftime, time_format, alignment_timestamp, default_value, now. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index abc3c75fb207f5..4f782373e97bbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -413,6 +413,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.DropMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.FixedRangePartition; import org.apache.doris.nereids.trees.plans.commands.info.FuncNameInfo; +import org.apache.doris.nereids.trees.plans.commands.info.GeneratedColumnDesc; import org.apache.doris.nereids.trees.plans.commands.info.InPartition; import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition; @@ -2747,8 +2748,11 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) { autoIncInitValue = Long.valueOf(1); } } + Optional desc = ctx.generatedExpr != null + ? Optional.of(new GeneratedColumnDesc(ctx.generatedExpr.getText(), getExpression(ctx.generatedExpr))) + : Optional.empty(); return new ColumnDefinition(colName, colType, isKey, aggType, nullableType, autoIncInitValue, defaultValue, - onUpdateDefaultValue, comment, true); + onUpdateDefaultValue, comment, desc); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 0c0f9e2e25f47d..4384197b629c6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -276,28 +277,18 @@ private static Map getColumnToOutput( Map columnToOutput = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); NereidsParser expressionParser = new NereidsParser(); + List generatedColumns = Lists.newArrayList(); + List materializedViewColumn = Lists.newArrayList(); // generate slots not mentioned in sql, mv slots and shaded slots. for (Column column : boundSink.getTargetTable().getFullSchema()) { - if (column.isMaterializedViewColumn()) { - List refs = column.getRefColumns(); - // now we have to replace the column to slots. - Preconditions.checkArgument(refs != null, - "mv column %s 's ref column cannot be null", column); - Expression parsedExpression = expressionParser.parseExpression( - column.getDefineExpr().toSqlWithoutTbl()); - Expression boundSlotExpression = SlotReplacer.INSTANCE - .replace(parsedExpression, columnToOutput); - // the boundSlotExpression is an expression whose slots are bound but function - // may not be bound, we have to bind it again. - // for example: to_bitmap. - Expression boundExpression = FunctionBinder.INSTANCE.rewrite( - boundSlotExpression, new ExpressionRewriteContext(ctx.cascadesContext)); - if (boundExpression instanceof Alias) { - boundExpression = ((Alias) boundExpression).child(); - } - NamedExpression slot = new Alias(boundExpression, column.getDefineExpr().toSqlWithoutTbl()); - columnToOutput.put(column.getName(), slot); - } else if (columnToChildOutput.containsKey(column) + if (column.getGeneratedColumnInfo() != null) { + generatedColumns.add(column); + continue; + } else if (column.isMaterializedViewColumn()) { + materializedViewColumn.add(column); + continue; + } + if (columnToChildOutput.containsKey(column) // do not process explicitly use DEFAULT value here: // insert into table t values(DEFAULT) && !(columnToChildOutput.get(column) instanceof DefaultValueSlot)) { @@ -381,6 +372,43 @@ private static Map getColumnToOutput( } } } + // the generated columns can use all ordinary columns, + // if processed in upper for loop, will lead to not found slot error + //It's the same reason for moving the processing of materialized columns down. + for (Column column : generatedColumns) { + GeneratedColumnInfo info = column.getGeneratedColumnInfo(); + Expression parsedExpression = new NereidsParser().parseExpression(info.getExpr().toSqlWithoutTbl()); + Expression boundSlotExpression = SlotReplacer.INSTANCE.replace(parsedExpression, columnToOutput); + Expression boundExpression = FunctionBinder.INSTANCE.rewrite(boundSlotExpression, + new ExpressionRewriteContext(ctx.cascadesContext)); + if (boundExpression instanceof Alias) { + boundExpression = ((Alias) boundExpression).child(); + } + NamedExpression slot = new Alias(boundExpression, info.getExprSql()); + columnToOutput.put(column.getName(), slot); + } + for (Column column : materializedViewColumn) { + if (column.isMaterializedViewColumn()) { + List refs = column.getRefColumns(); + // now we have to replace the column to slots. + Preconditions.checkArgument(refs != null, + "mv column %s 's ref column cannot be null", column); + Expression parsedExpression = expressionParser.parseExpression( + column.getDefineExpr().toSqlWithoutTbl()); + Expression boundSlotExpression = SlotReplacer.INSTANCE + .replace(parsedExpression, columnToOutput); + // the boundSlotExpression is an expression whose slots are bound but function + // may not be bound, we have to bind it again. + // for example: to_bitmap. + Expression boundExpression = FunctionBinder.INSTANCE.rewrite( + boundSlotExpression, new ExpressionRewriteContext(ctx.cascadesContext)); + if (boundExpression instanceof Alias) { + boundExpression = ((Alias) boundExpression).child(); + } + NamedExpression slot = new Alias(boundExpression, column.getDefineExpr().toSqlWithoutTbl()); + columnToOutput.put(column.getName(), slot); + } + } return columnToOutput; } @@ -543,6 +571,9 @@ private Pair, Integer> bindTargetColumns(OlapTable table, List generatedColumnDesc = Optional.empty(); + private Set generatedColumnsThatReferToThis = new HashSet<>(); public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable, Optional defaultValue, String comment) { @@ -72,10 +74,11 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType } public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, - boolean isNullable, long autoIncInitValue, Optional defaultValue, - Optional onUpdateDefaultValue, String comment) { - this(name, type, isKey, aggType, isNullable, autoIncInitValue, defaultValue, onUpdateDefaultValue, - comment, true); + ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, + Optional onUpdateDefaultValue, String comment, + Optional generatedColumnDesc) { + this(name, type, isKey, aggType, nullableType, autoIncInitValue, defaultValue, onUpdateDefaultValue, + comment, true, generatedColumnDesc); } /** @@ -116,7 +119,8 @@ private ColumnDefinition(String name, DataType type, boolean isKey, AggregateTyp */ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, ColumnNullableType nullableType, long autoIncInitValue, Optional defaultValue, - Optional onUpdateDefaultValue, String comment, boolean isVisible) { + Optional onUpdateDefaultValue, String comment, boolean isVisible, + Optional generatedColumnDesc) { this.name = name; this.type = type; this.isKey = isKey; @@ -127,6 +131,7 @@ public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType this.onUpdateDefaultValue = onUpdateDefaultValue; this.comment = comment; this.isVisible = isVisible; + this.generatedColumnDesc = generatedColumnDesc; } public ColumnDefinition(String name, DataType type, boolean isNullable) { @@ -404,6 +409,7 @@ public void validate(boolean isOlap, Set keysSet, boolean isEnableMergeO if (type.isTimeLikeType()) { throw new AnalysisException("Time type is not supported for olap table"); } + validateGeneratedColumnInfo(); } // from TypeDef.java analyze() @@ -650,7 +656,9 @@ public Column translateToCatalogStyle() { autoIncInitValue, defaultValue.map(DefaultValue::getRawValue).orElse(null), comment, isVisible, defaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), Column.COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue.map(DefaultValue::getValue).orElse(null), onUpdateDefaultValue.isPresent(), - onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId); + onUpdateDefaultValue.map(DefaultValue::getDefaultValueExprDef).orElse(null), clusterKeyId, + generatedColumnDesc.map(GeneratedColumnDesc::translateToInfo).orElse(null), + generatedColumnsThatReferToThis); column.setAggregationTypeImplicit(aggTypeImplicit); return column; } @@ -686,4 +694,30 @@ public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregat Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false); } + public Optional getGeneratedColumnDesc() { + return generatedColumnDesc; + } + + public long getAutoIncInitValue() { + return autoIncInitValue; + } + + public void addGeneratedColumnsThatReferToThis(List list) { + generatedColumnsThatReferToThis.addAll(list); + } + + private void validateGeneratedColumnInfo() { + // for generated column + if (generatedColumnDesc.isPresent()) { + if (autoIncInitValue != -1) { + throw new AnalysisException("Generated columns cannot be auto_increment."); + } + if (defaultValue.isPresent() && !defaultValue.get().equals(DefaultValue.NULL_DEFAULT_VALUE)) { + throw new AnalysisException("Generated columns cannot have default value."); + } + if (onUpdateDefaultValue.isPresent()) { + throw new AnalysisException("Generated columns cannot have on update default value."); + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index fc93b068e4b528..1ec8bd47a7de61 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -20,9 +20,11 @@ import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DistributionDesc; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.KeysDesc; import org.apache.doris.analysis.PartitionDesc; +import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; @@ -37,6 +39,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.Pair; import org.apache.doris.common.util.AutoBucketUtils; +import org.apache.doris.common.util.GeneratedColumnUtil; import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PropertyAnalyzer; @@ -46,23 +49,47 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.analyzer.Scope; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.glue.translator.ExpressionTranslator; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.nereids.parser.PartitionTableInfo; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; +import org.apache.doris.nereids.trees.expressions.Variable; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.functions.Udf; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; +import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; +import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -81,6 +108,8 @@ public class CreateTableInfo { public static final String ENGINE_BROKER = "broker"; public static final String ENGINE_HIVE = "hive"; public static final String ENGINE_ICEBERG = "iceberg"; + private static final ImmutableSet GENERATED_COLUMN_ALLOW_AGG_TYPE = + ImmutableSet.of(AggregateType.REPLACE, AggregateType.REPLACE_IF_NOT_NULL); private final boolean ifNotExists; private String ctlName; @@ -512,7 +541,6 @@ public void validate(ConnectContext ctx) { engineName, columns, columnMap, properties, ctx, false, true); } } - // validate column try { if (!engineName.equals(ENGINE_ELASTICSEARCH) && columns.isEmpty()) { @@ -565,6 +593,7 @@ public void validate(ConnectContext ctx) { "same index columns have multiple same type index is not allowed."); } } + generatedColumnCheck(ctx); } private void paddingEngineName(String ctlName, ConnectContext ctx) { @@ -812,5 +841,212 @@ public CreateTableStmt translateToLegacyStmt() { public void setIsExternal(boolean isExternal) { this.isExternal = isExternal; } + + private void generatedColumnCommonCheck() { + for (ColumnDefinition column : columns) { + if (keysType == KeysType.AGG_KEYS && column.getGeneratedColumnDesc().isPresent() + && (!column.isKey() && !GENERATED_COLUMN_ALLOW_AGG_TYPE.contains(column.getAggType()))) { + throw new AnalysisException("The generated columns can be key columns, " + + "or value columns of replace and replace_if_not_null aggregation type."); + } + if (column.getGeneratedColumnDesc().isPresent() && !engineName.equalsIgnoreCase("olap")) { + throw new AnalysisException("Tables can only have generated columns if the olap engine is used"); + } + } + } + + private void generatedColumnCheck(ConnectContext ctx) { + generatedColumnCommonCheck(); + LogicalEmptyRelation plan = new LogicalEmptyRelation( + ConnectContext.get().getStatementContext().getNextRelationId(), + new ArrayList<>()); + CascadesContext cascadesContext = CascadesContext.initContext(ctx.getStatementContext(), plan, + PhysicalProperties.ANY); + Map columnToSlotReference = Maps.newHashMap(); + Map nameToColumnDefinition = Maps.newHashMap(); + Map translateMap = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + ColumnDefinition column = columns.get(i); + Slot slot = new SlotReference(column.getName(), column.getType()); + columnToSlotReference.put(column.getName(), slot); + nameToColumnDefinition.put(column.getName(), column); + SlotRef slotRef = new SlotRef(null, column.getName()); + slotRef.setType(column.getType().toCatalogDataType()); + translateMap.put(slot, new SlotRefAndIdx(slotRef, i, column.getGeneratedColumnDesc().isPresent())); + } + ExpressionToExpr.initializeslotRefMap(translateMap); + PlanTranslatorContext planTranslatorContext = new PlanTranslatorContext(cascadesContext); + List slots = Lists.newArrayList(columnToSlotReference.values()); + List exprAndnames = Lists.newArrayList(); + for (int i = 0; i < columns.size(); i++) { + ColumnDefinition column = columns.get(i); + Optional info = column.getGeneratedColumnDesc(); + if (!info.isPresent()) { + continue; + } + Expression parsedExpression = info.get().getExpression(); + checkParsedExpressionInGeneratedColumn(parsedExpression); + Expression boundSlotExpression = SlotReplacer.INSTANCE.replace(parsedExpression, columnToSlotReference); + Scope scope = new Scope(slots); + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope, cascadesContext, false, false); + Expression expr; + try { + expr = analyzer.analyze(boundSlotExpression, new ExpressionRewriteContext(cascadesContext)); + } catch (AnalysisException e) { + throw new AnalysisException("In generated column '" + column.getName() + "', " + + Utils.convertFirstChar(e.getMessage())); + } + checkExpressionInGeneratedColumn(expr, column, nameToColumnDefinition); + TypeCoercionUtils.checkCanCastTo(expr.getDataType(), column.getType()); + ExpressionToExpr translator = new ExpressionToExpr(i); + Expr e = expr.accept(translator, planTranslatorContext); + info.get().setExpr(e); + exprAndnames.add(new GeneratedColumnUtil.ExprAndname(e.clone(), column.getName())); + } + + // for alter drop column + Map> columnToListOfGeneratedColumnsThatReferToThis = new HashMap<>(); + for (ColumnDefinition column : columns) { + Optional info = column.getGeneratedColumnDesc(); + if (!info.isPresent()) { + continue; + } + Expr generatedExpr = info.get().getExpr(); + Set slotRefsInGeneratedExpr = new HashSet<>(); + generatedExpr.collect(e -> e instanceof SlotRef, slotRefsInGeneratedExpr); + for (Expr slotRefExpr : slotRefsInGeneratedExpr) { + String name = ((SlotRef) slotRefExpr).getColumnName(); + columnToListOfGeneratedColumnsThatReferToThis + .computeIfAbsent(name, k -> new ArrayList<>()) + .add(column.getName()); + } + } + for (Map.Entry> entry : columnToListOfGeneratedColumnsThatReferToThis.entrySet()) { + ColumnDefinition col = nameToColumnDefinition.get(entry.getKey()); + col.addGeneratedColumnsThatReferToThis(entry.getValue()); + } + + // expand expr + GeneratedColumnUtil.rewriteColumns(exprAndnames); + for (GeneratedColumnUtil.ExprAndname exprAndname : exprAndnames) { + if (nameToColumnDefinition.containsKey(exprAndname.getName())) { + ColumnDefinition columnDefinition = nameToColumnDefinition.get(exprAndname.getName()); + Optional info = columnDefinition.getGeneratedColumnDesc(); + info.ifPresent(genCol -> genCol.setExpandExprForLoad(exprAndname.getExpr())); + } + } + } + + private static class SlotReplacer extends DefaultExpressionRewriter> { + public static final SlotReplacer INSTANCE = new SlotReplacer(); + + public Expression replace(Expression e, Map replaceMap) { + return e.accept(this, replaceMap); + } + + @Override + public Expression visitUnboundSlot(UnboundSlot unboundSlot, Map replaceMap) { + if (!replaceMap.containsKey(unboundSlot.getName())) { + throw new AnalysisException("Unknown column '" + unboundSlot.getName() + + "' in 'generated column function'"); + } + return replaceMap.get(unboundSlot.getName()); + } + } + + void checkParsedExpressionInGeneratedColumn(Expression expr) { + expr.foreach(e -> { + if (e instanceof SubqueryExpr) { + throw new AnalysisException("Generated column does not support subquery."); + } else if (e instanceof Lambda) { + throw new AnalysisException("Generated column does not support lambda."); + } + }); + } + + void checkExpressionInGeneratedColumn(Expression expr, ColumnDefinition column, + Map nameToColumnDefinition) { + expr.foreach(e -> { + if (e instanceof Variable) { + throw new AnalysisException("Generated column expression cannot contain variable."); + } else if (e instanceof Slot && nameToColumnDefinition.containsKey(((Slot) e).getName())) { + ColumnDefinition columnDefinition = nameToColumnDefinition.get(((Slot) e).getName()); + if (columnDefinition.getAutoIncInitValue() != -1) { + throw new AnalysisException( + "Generated column '" + column.getName() + + "' cannot refer to auto-increment column."); + } + } else if (e instanceof BoundFunction && !checkFunctionInGeneratedColumn((BoundFunction) e)) { + throw new AnalysisException("Expression of generated column '" + + column.getName() + "' contains a disallowed function:'" + + ((BoundFunction) e).getName() + "'"); + } + }); + } + + boolean checkFunctionInGeneratedColumn(Expression expr) { + if (!(expr instanceof ScalarFunction)) { + return false; + } + if (expr instanceof Udf) { + return false; + } + if (expr instanceof GroupingScalarFunction) { + return false; + } + return true; + } + + private static class ExpressionToExpr extends ExpressionTranslator { + private static Map slotRefMap; + private final int index; + + public ExpressionToExpr(int index) { + this.index = index; + } + + public static void initializeslotRefMap(Map map) { + slotRefMap = map; + } + + @Override + public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContext context) { + if (!slotRefMap.containsKey(slotReference)) { + throw new AnalysisException("Unknown column '" + slotReference.getName() + "'"); + } + int idx = slotRefMap.get(slotReference).getIdx(); + boolean isGenCol = slotRefMap.get(slotReference).isGeneratedColumn(); + if (isGenCol && idx >= index) { + throw new AnalysisException( + "Generated column can refer only to generated columns defined prior to it."); + } + return slotRefMap.get(slotReference).getSlotRef(); + } + } + + /** SlotRefAndIdx */ + public static class SlotRefAndIdx { + private final SlotRef slotRef; + private final int idx; + private final boolean isGeneratedColumn; + + public SlotRefAndIdx(SlotRef slotRef, int idx, boolean isGeneratedColumn) { + this.slotRef = slotRef; + this.idx = idx; + this.isGeneratedColumn = isGeneratedColumn; + } + + public int getIdx() { + return idx; + } + + public SlotRef getSlotRef() { + return slotRef; + } + + public boolean isGeneratedColumn() { + return isGeneratedColumn; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java new file mode 100644 index 00000000000000..6175389dcdd877 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/GeneratedColumnDesc.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.GeneratedColumnInfo; +import org.apache.doris.nereids.trees.expressions.Expression; + +/**GeneratedColumnDesc for nereids*/ +public class GeneratedColumnDesc { + /**GeneratedColumnType*/ + public enum GeneratedColumnType { + VIRTUAL, + STORED + } + + private final GeneratedColumnType type; + private final String exprSql; + private Expr expr; + private Expr expandExprForLoad; + private final Expression expression; + + /** constructor */ + public GeneratedColumnDesc(String exprSql, Expression expression) { + this.exprSql = exprSql; + this.expression = expression; + this.type = GeneratedColumnType.STORED; + } + + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + + public void setExpandExprForLoad(Expr expandExprForLoad) { + this.expandExprForLoad = expandExprForLoad; + } + + public Expression getExpression() { + return expression; + } + + public GeneratedColumnInfo translateToInfo() { + return new GeneratedColumnInfo(exprSql, expr, expandExprForLoad); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 4b7afb1f6a855b..cdb5a4c7eebc85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -100,7 +100,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { TableIf targetTableIf = InsertUtils.getTargetTable(logicalQuery, ctx); targetTableIf.readLock(); try { - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(logicalQuery, ctx.getStatementContext()); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.plan(logicalPlanAdapter, ctx.getSessionVariable().toThrift()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index a0268f38a2b3ca..39507d726858b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -142,7 +142,7 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor targetTableIf.readLock(); try { // 1. process inline table (default values, empty values) - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, insertCtx); if (cte.isPresent()) { this.logicalQuery = ((LogicalPlan) cte.get().withChildren(logicalQuery)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java index 05d51521ff196e..3acd3d8123d8f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertOverwriteTableCommand.java @@ -118,7 +118,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (targetTableIf instanceof MTMV && !MTMVUtil.allowModifyMTMVData(ctx)) { throw new AnalysisException("Not allowed to perform current operation on async materialized view"); } - this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf); + this.logicalQuery = (LogicalPlan) InsertUtils.normalizePlan(logicalQuery, targetTableIf, Optional.empty()); if (cte.isPresent()) { this.logicalQuery = (LogicalPlan) logicalQuery.withChildren(cte.get().withChildren( this.logicalQuery.child(0))); @@ -227,7 +227,7 @@ private void insertInto(ConnectContext ctx, StmtExecutor executor, List (LogicalPlan) (sink.child(0))); // 1. for overwrite situation, we disable auto create partition. // 2. we save and pass overwrite auto detect by insertCtx - insertCtx = new OlapInsertCommandContext(false); + insertCtx = new OlapInsertCommandContext(false, true); } else if (logicalQuery instanceof UnboundHiveTableSink) { UnboundHiveTableSink sink = (UnboundHiveTableSink) logicalQuery; copySink = (UnboundLogicalSink) UnboundTableSinkCreator.createUnboundTableSink( @@ -259,7 +259,7 @@ private void insertInto(ConnectContext ctx, StmtExecutor executor, long groupId) InsertCommandContext insertCtx; if (logicalQuery instanceof UnboundTableSink) { insertCtx = new OlapInsertCommandContext(false, - ((UnboundTableSink) logicalQuery).isAutoDetectPartition(), groupId); + ((UnboundTableSink) logicalQuery).isAutoDetectPartition(), groupId, true); } else if (logicalQuery instanceof UnboundHiveTableSink) { insertCtx = new HiveInsertCommandContext(); ((HiveInsertCommandContext) insertCtx).setOverwrite(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index c452a242dcc46b..a1611718ec875a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; @@ -73,10 +74,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.commons.collections.CollectionUtils; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -253,7 +256,7 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, /** * normalize plan to let it could be process correctly by nereids */ - public static Plan normalizePlan(Plan plan, TableIf table) { + public static Plan normalizePlan(Plan plan, TableIf table, Optional insertCtx) { UnboundLogicalSink unboundLogicalSink = (UnboundLogicalSink) plan; if (table instanceof HMSExternalTable) { HMSExternalTable hiveTable = (HMSExternalTable) table; @@ -285,12 +288,19 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Partial update should include all key columns, missing: " + col.getName()); } + if (!col.getGeneratedColumnsThatReferToThis().isEmpty() + && col.getGeneratedColumnInfo() == null && !insertCol.isPresent()) { + throw new AnalysisException("Partial update should include" + + " all ordinary columns referenced" + + " by generated columns, missing: " + col.getName()); + } } } } } } Plan query = unboundLogicalSink.child(); + checkGeneratedColumnForInsertIntoSelect(table, unboundLogicalSink, insertCtx); if (!(query instanceof LogicalInlineTable)) { return plan; } @@ -324,6 +334,12 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Unknown column '" + unboundLogicalSink.getColNames().get(i) + "' in target table."); } + if (sameNameColumn.getGeneratedColumnInfo() != null + && !(values.get(i) instanceof DefaultValueSlot)) { + throw new AnalysisException("The value specified for generated column '" + + sameNameColumn.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } if (values.get(i) instanceof DefaultValueSlot) { constantExprs.add(generateDefaultExpression(sameNameColumn)); } else { @@ -336,6 +352,12 @@ public static Plan normalizePlan(Plan plan, TableIf table) { throw new AnalysisException("Column count doesn't match value count"); } for (int i = 0; i < columns.size(); i++) { + if (columns.get(i).getGeneratedColumnInfo() != null + && !(values.get(i) instanceof DefaultValueSlot)) { + throw new AnalysisException("The value specified for generated column '" + + columns.get(i).getName() + + "' in table '" + table.getName() + "' is not allowed."); + } if (values.get(i) instanceof DefaultValueSlot) { constantExprs.add(generateDefaultExpression(columns.get(i))); } else { @@ -388,6 +410,13 @@ public static TableIf getTargetTable(Plan plan, ConnectContext ctx) { private static NamedExpression generateDefaultExpression(Column column) { try { + GeneratedColumnInfo generatedColumnInfo = column.getGeneratedColumnInfo(); + // Using NullLiteral as a placeholder. + // If return the expr in generatedColumnInfo, will lead to slot not found error in analyze. + // Instead, getting the generated column expr and analyze the expr in BindSink can avoid the error. + if (generatedColumnInfo != null) { + return new Alias(new NullLiteral(DataType.fromCatalogType(column.getType())), column.getName()); + } if (column.getDefaultValue() == null) { throw new AnalysisException("Column has no default value, column=" + column.getName()); } @@ -420,6 +449,52 @@ public static Plan getPlanForExplain(ConnectContext ctx, LogicalPlan logicalQuer } throw new AnalysisException("Nereids DML is disabled, will try to fall back to the original planner"); } - return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx)); + return InsertUtils.normalizePlan(logicalQuery, InsertUtils.getTargetTable(logicalQuery, ctx), Optional.empty()); + } + + // check for insert into t1(a,b,gen_col) select 1,2,3; + private static void checkGeneratedColumnForInsertIntoSelect(TableIf table, + UnboundLogicalSink unboundLogicalSink, Optional insertCtx) { + // should not check delete stmt, because deletestmt can transform to insert delete sign + if (unboundLogicalSink.getDMLCommandType() == DMLCommandType.DELETE) { + return; + } + // This is for the insert overwrite values(),() + // Insert overwrite stmt can enter normalizePlan() twice. + // Insert overwrite values(),() is checked in the first time when deal with ConstantExprsList, + // and then the insert into values(),() will be transformed to insert into union all in normalizePlan, + // and this function is for insert into select, which will check the insert into union all again, + // and that is no need, also will lead to problems. + // So for insert overwrite values(),(), this check is only performed + // when it first enters the normalizePlan checking constantExprsList + if (insertCtx.isPresent() && insertCtx.get() instanceof OlapInsertCommandContext + && ((OlapInsertCommandContext) insertCtx.get()).isOverwrite()) { + return; + } + Plan query = unboundLogicalSink.child(); + if (table instanceof OlapTable && !(query instanceof LogicalInlineTable)) { + OlapTable olapTable = (OlapTable) table; + Set insertNames = Sets.newHashSet(); + if (unboundLogicalSink.getColNames() != null) { + insertNames.addAll(unboundLogicalSink.getColNames()); + } + if (insertNames.isEmpty()) { + for (Column col : olapTable.getFullSchema()) { + if (col.getGeneratedColumnInfo() != null) { + throw new AnalysisException("The value specified for generated column '" + + col.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } + } + } else { + for (Column col : olapTable.getFullSchema()) { + if (col.getGeneratedColumnInfo() != null && insertNames.contains(col.getName())) { + throw new AnalysisException("The value specified for generated column '" + + col.getName() + + "' in table '" + table.getName() + "' is not allowed."); + } + } + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java index bebade142d97e1..35adc3031b1441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/OlapInsertCommandContext.java @@ -21,18 +21,25 @@ * For Olap Table */ public class OlapInsertCommandContext extends InsertCommandContext { - private boolean allowAutoPartition; - private boolean autoDetectOverwrite = false; - private long overwriteGroupId = 0; + private final boolean allowAutoPartition; + private final boolean autoDetectOverwrite; + private final long overwriteGroupId; + private final boolean isOverwrite; - public OlapInsertCommandContext(boolean allowAutoPartition) { - this.allowAutoPartition = allowAutoPartition; - } - - public OlapInsertCommandContext(boolean allowAutoPartition, boolean autoDetectOverwrite, long overwriteGroupId) { + public OlapInsertCommandContext(boolean allowAutoPartition, boolean autoDetectOverwrite, long overwriteGroupId, + boolean isOverwrite) { this.allowAutoPartition = allowAutoPartition; this.autoDetectOverwrite = autoDetectOverwrite; this.overwriteGroupId = overwriteGroupId; + this.isOverwrite = isOverwrite; + } + + public OlapInsertCommandContext(boolean allowAutoPartition) { + this(allowAutoPartition, false, 0, false); + } + + public OlapInsertCommandContext(boolean allowAutoPartition, boolean isOverwrite) { + this(allowAutoPartition, false, 0, isOverwrite); } public boolean isAllowAutoPartition() { @@ -47,7 +54,7 @@ public long getOverwriteGroupId() { return overwriteGroupId; } - public void setAllowAutoPartition(boolean allowAutoPartition) { - this.allowAutoPartition = allowAutoPartition; + public boolean isOverwrite() { + return isOverwrite; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index 852e148ef1d9cb..8525d2629e34ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -63,8 +63,8 @@ public static String quoteIfNeeded(String part) { * Helper function to eliminate unnecessary checked exception caught requirement from the main logic of translator. * * @param f function which would invoke the logic of - * stale code from old optimizer that could throw - * a checked exception. + * stale code from old optimizer that could throw + * a checked exception. */ public static void execWithUncheckedException(FuncWrapper f) { try { @@ -128,14 +128,18 @@ public static String qualifiedName(List qualifier, String name) { return StringUtils.join(qualifiedNameParts(qualifier, name), "."); } - /** get qualified name with Backtick */ + /** + * get qualified name with Backtick + */ public static String qualifiedNameWithBackquote(List qualifiers, String name) { List fullName = new ArrayList<>(qualifiers); fullName.add(name); return qualifiedNameWithBackquote(fullName); } - /** get qualified name with Backtick */ + /** + * get qualified name with Backtick + */ public static String qualifiedNameWithBackquote(List qualifiers) { List qualifierWithBackquote = Lists.newArrayListWithCapacity(qualifiers.size()); for (String qualifier : qualifiers) { @@ -162,7 +166,7 @@ public static String toSqlString(String planName, Object... variables) { } for (int i = 0; i < variables.length - 1; i += 2) { - if (! "".equals(toStringOrNull(variables[i + 1]))) { + if (!"".equals(toStringOrNull(variables[i + 1]))) { if (i != 0) { stringBuilder.append(", "); } @@ -245,7 +249,9 @@ public static void identityRemove(List list, T item) { Preconditions.checkState(false, "item not found in list"); } - /** allCombinations */ + /** + * allCombinations + */ public static List> allCombinations(List> lists) { if (lists.size() == 1) { List first = lists.get(0); @@ -280,7 +286,7 @@ private static List> doAllCombinations(List> lists) { .flatMap(firstValue -> combinationWithoutFirst.stream() .map(restList -> Stream.concat(Stream.of(firstValue), restList.stream()) - .collect(ImmutableList.toImmutableList()) + .collect(ImmutableList.toImmutableList()) ) ).collect(ImmutableList.toImmutableList()); } @@ -326,7 +332,9 @@ public static List fastMapList(List list, int additionSize, Functio return newList; } - /** fastToImmutableList */ + /** + * fastToImmutableList + */ public static ImmutableList fastToImmutableList(E[] array) { switch (array.length) { case 0: @@ -344,14 +352,17 @@ public static ImmutableList fastToImmutableList(E[] array) { } } - /** fastToImmutableList */ + /** + * fastToImmutableList + */ public static ImmutableList fastToImmutableList(Collection collection) { if (collection instanceof ImmutableList) { return (ImmutableList) collection; } switch (collection.size()) { - case 0: return ImmutableList.of(); + case 0: + return ImmutableList.of(); case 1: return collection instanceof List ? ImmutableList.of(((List) collection).get(0)) @@ -366,7 +377,9 @@ public static ImmutableList fastToImmutableList(Collection c } } - /** fastToImmutableSet */ + /** + * fastToImmutableSet + */ public static ImmutableSet fastToImmutableSet(Collection collection) { if (collection instanceof ImmutableSet) { return (ImmutableSet) collection; @@ -389,7 +402,9 @@ public static ImmutableSet fastToImmutableSet(Collection col } } - /** reverseImmutableList */ + /** + * reverseImmutableList + */ public static ImmutableList reverseImmutableList(List list) { Builder reverseList = ImmutableList.builderWithExpectedSize(list.size()); for (int i = list.size() - 1; i >= 0; i--) { @@ -398,7 +413,9 @@ public static ImmutableList reverseImmutableList(List list) return reverseList.build(); } - /** filterImmutableList */ + /** + * filterImmutableList + */ public static ImmutableList filterImmutableList(List list, Predicate filter) { Builder newList = ImmutableList.builderWithExpectedSize(list.size()); for (int i = 0; i < list.size(); i++) { @@ -410,7 +427,9 @@ public static ImmutableList filterImmutableList(List list, P return newList.build(); } - /** concatToSet */ + /** + * concatToSet + */ public static Set concatToSet(Collection left, Collection right) { ImmutableSet.Builder required = ImmutableSet.builderWithExpectedSize( left.size() + right.size() @@ -420,7 +439,9 @@ public static Set concatToSet(Collection left, Collection Optional fastReduce(List list, BiFunction reduceOp) { if (list.isEmpty()) { return Optional.empty(); @@ -431,4 +452,18 @@ public static Optional fastReduce(List list, BiFunction } return true } -logger.info("Added 'mustContains' function to Suite") +logger.info("Added 'mustContain' function to Suite") diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy new file mode 100644 index 00000000000000..d6a6695bce6c1b --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/alter_column_test_generated_column.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("alter_column_test_generated_column") { + def waitSchemaChangeJob = { String tableName /* param */ -> + int tryTimes = 30 + while (tryTimes-- > 0) { + def jobResult = sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + if (jobResult == null || jobResult.isEmpty()) { + sleep(3000) + return; + } + def jobState = jobResult[0][9].toString() + if ('cancelled'.equalsIgnoreCase(jobState)) { + logger.info("jobResult:{}", jobResult) + throw new IllegalStateException("${tableName}'s job has been cancelled") + } + if ('finished'.equalsIgnoreCase(jobState)) { + logger.info("jobResult:{}", jobResult) + sleep(3000) + return + } + sleep(1000) + } + assertTrue(false) + } + def getMVJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE ROLLUP WHERE TableName='${tableName}' ORDER BY CreateTime DESC LIMIT 1 """ + return jobStateResult[0][8] + } + def waitForMVJob = (tbName, timeout) -> { + while (timeout--){ + String result = getMVJobState(tbName) + if (result == "FINISHED") { + sleep(3000) + break + } else { + sleep(100) + if (timeout < 1){ + assertEquals(1,2) + } + } + } + } + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists alter_column_gen_col; + create table alter_column_gen_col (a int, b int, c int as(a+b), d int as(c+1), e int) + duplicate key(a) distributed by hash(a) PROPERTIES("replication_num" = "1"); + alter table alter_column_gen_col add rollup r1 (c,b,a) + """ + waitForMVJob("alter_column_gen_col", 3000) + // add column + test { + sql "alter table alter_column_gen_col add column f int as (a+1);" + exception "Not supporting alter table add generated columns." + } + test { + sql "alter table alter_column_gen_col add column f int as (a+1) to r1" + exception "Not supporting alter table add generated columns." + } + test { + sql "alter table alter_column_gen_col add column f int as (a+1) after a to r1;" + exception "Not supporting alter table add generated columns." + } + test { + sql """alter table alter_column_gen_col add column (f int as (a+1), g int) to r1;""" + exception "Not supporting alter table add generated columns." + } + + // drop column + // rollup + multi_sql """ + insert into alter_column_gen_col values(1,2,default,default,5); + insert into alter_column_gen_col values(9,2,default,default,3); + insert into alter_column_gen_col values(6,2,default,default,5); + """ + explain { + sql "select c,b from alter_column_gen_col where c=10;" + contains "r1" + } + qt_drop_gen_col_rollup "alter table alter_column_gen_col drop column c from r1;" + waitSchemaChangeJob("alter_column_gen_col") + explain { + sql "select b from alter_column_gen_col where b=10;" + contains "r1" + } + multi_sql """ + insert into alter_column_gen_col values(9,2,default,default,3); + insert into alter_column_gen_col values(6,2,default,default,5); + """ + // modify column + test { + sql "alter table alter_column_gen_col modify column c double as (a+b);" + exception "Not supporting alter table modify generated columns." + } + test { + sql "alter table alter_column_gen_col modify column c double as (a+b) after e;" + exception "Not supporting alter table modify generated columns." + } + test { + sql "alter table alter_column_gen_col modify column c int as (a+b) after e;" + exception "Not supporting alter table modify generated columns." + } + + // reorder column + qt_reorder "alter table alter_column_gen_col order by(a,c,b,d,e);" + waitSchemaChangeJob("alter_column_gen_col") + + test { + sql "alter table alter_column_gen_col order by(a,d,b,c,e);" + exception "The specified column order is incorrect, `d` should come after `c`, because both of them are generated columns, and `d` refers to `c`." + } + qt_after_reorder_insert "insert into alter_column_gen_col(a,b,e) values(12,3,4);" + qt_reorder_rollup "alter table alter_column_gen_col order by (a,b) from r1" + waitSchemaChangeJob("alter_column_gen_col") + + // rename column + test { + sql "alter table alter_column_gen_col rename column c c1" + exception "" + } + qt_rename_gen_col "alter table alter_column_gen_col rename column d d1" + waitSchemaChangeJob("alter_column_gen_col") + qt_after_rename_insert "insert into alter_column_gen_col(a,b,e) values(16,2,4);" + qt_after_rename_insert_select "select * from alter_column_gen_col order by 1,2,3,4,5" +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy new file mode 100644 index 00000000000000..291916b9a448b2 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/fault_tolerance_nereids.groovy @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_fault_tolerance_nereids") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + test { + sql """ + create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "In generated column 'c', can not found function 'abs' which has 2 arity." + } + + // gencol_has_sum + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Expression of generated column 'c' contains a disallowed function:'sum'" + } + + // gencol_has_column_not_define + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Unknown column 'd' in 'generated column function'" + } + + // gencol_refer_gencol_after + test { + sql """ + create table gencol_refer_gencol_ft(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column can refer only to generated columns defined prior to it." + } + + sql "set @myvar=2" + // gencol_has_var + test { + sql """ + create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column expression cannot contain variable." + } + + test { + sql """ + create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) + distributed by hash(a) properties("replication_num" = "1"); + """ + exception "Generated column 'c' cannot refer to auto-increment column." + } + + test{ + sql """ + create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support subquery." + } + + test { + sql """ + create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support lambda." + } + + test { + sql """ + create table test_gen_col_array_func(pk int,a array,b array, c double generated always as (a+b) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "In generated column 'c', cannot cast from ARRAY to numeric type" + } + + test { + sql """ + create table test_gen_col_aggregate_value(a int,b int,c int sum generated always as (abs(a+1)) not null) + aggregate key(a,b) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." + } + + test { + sql """ + create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot be auto_increment." + } + + test { + sql """ + create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + + test { + sql """ + create table test_gen_col_on_update(a int,b int,c datetimev2 generated always as (CURRENT_TIMESTAMP) not null default CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP ) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + test { + sql """ + create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'rank'" + } + + test { + sql """ + create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'Grouping'" + } + + sql "drop table if exists gen_col_test_modify" + sql """ create table gen_col_test_modify(pk int, a int, b int, c int as (a+b)) distributed by hash(pk) buckets 10 + properties('replication_num' = '1'); """ + test { + sql """ALTER TABLE gen_col_test_modify modify COLUMN c int AS (a+b+1)""" + exception "Not supporting alter table modify generated columns." + } + test { + sql """ALTER TABLE gen_col_test_modify ADD COLUMN d int AS (a+b);""" + exception "Not supporting alter table add generated columns." + } + + sql "drop table if exists test_gen_col_common_ft" + sql """create table test_gen_col_common_ft(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + // qt_common_default_test_insert_null + test { + sql "INSERT INTO test_gen_col_common_ft(a,b) values(1,null);" + exception "Insert has filtered data in strict mode." + } + + // qt_common_default_test_insert_gencol + test { + sql "INSERT INTO test_gen_col_common_ft values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft(a,b,c) values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft select 1,2,5" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + test { + sql "INSERT INTO test_gen_col_common_ft(a,b,c) select * from test_gen_col_common_ft" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_ft' is not allowed." + } + + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy new file mode 100644 index 00000000000000..a8d4025b9b82fc --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_stream_mysql_load") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + String db = context.config.getDbNameByFile(context.file) + + sql "drop table if exists test_gen_col_common_steam_mysql_load" + qt_common_default """create table test_gen_col_common_steam_mysql_load(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + + sql "drop table if exists gencol_refer_gencol_steam_mysql_load" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol_steam_mysql_load(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + + // test load + streamLoad { + table 'test_gen_col_common_steam_mysql_load' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol_steam_mysql_load' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" + + streamLoad { + table 'test_gen_col_common_steam_mysql_load' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_common_json "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + + streamLoad { + table 'gencol_refer_gencol_steam_mysql_load' + set 'strip_outer_array', 'true' + set 'format', 'json' + set 'columns', 'a, b' + file 'gen_col_data.json' + set 'jsonpaths', '["$.a", "$.b"]' + time 10000 // limit inflight 10s + } + sql "sync" + qt_test_stream_load_refer_gencol_json "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" + + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.test_gen_col_common_steam_mysql_load(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_common_ "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + streamLoad { + set 'version', '1' + file 'gen_col_data.csv' + time 10000 + set 'sql',""" + insert into ${db}.gencol_refer_gencol_steam_mysql_load(a,b) select * from http_stream("format"="csv", "column_separator"=",") + """ + } + sql "sync" + qt_http_stream_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" + // mysql load + def filepath = getLoalFilePath "gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common_steam_mysql_load + COLUMNS TERMINATED BY ',' + (a,b); """ + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE gencol_refer_gencol_steam_mysql_load + COLUMNS TERMINATED BY ',' + (a,b); + """ + sql "sync" + qt_test_mysql_load_common "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + qt_test_mysql_load_refer_gencol "select * from gencol_refer_gencol_steam_mysql_load order by 1,2,3,4" + + // specify value for generated column + sql "truncate table test_gen_col_common_steam_mysql_load" + filepath = getLoalFilePath "three_column_gen_col_data.csv" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common_steam_mysql_load + COLUMNS TERMINATED BY ',' + (a,b,c); + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" + + sql "truncate table test_gen_col_common_steam_mysql_load" + sql """ + LOAD DATA LOCAL + INFILE '${filepath}' + INTO TABLE test_gen_col_common_steam_mysql_load + COLUMNS TERMINATED BY ',' + (a,b,tmp) + set (c=tmp+1) ; + """ + sql "sync" + qt_specify_value_for_gencol "select * from test_gen_col_common_steam_mysql_load order by 1,2,3" +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy new file mode 100644 index 00000000000000..7bf58260d1f046 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_create_table_generated_column_legacy.groovy @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_table_generated_column_legacy") { + // test legacy planner create + sql "SET enable_nereids_planner=false;" + sql "drop table if exists test_gen_col_common_legacy" + qt_common_default """create table test_gen_col_common_legacy(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + sql "drop table if exists test_gen_col_without_generated_always_legacy" + qt_common_without_generated_always """create table test_gen_col_without_generated_always_legacy(a int,b int,c double as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + sql "drop table if exists test_gen_col_in_middle_legacy" + qt_gencol_in_middle """create table test_gen_col_in_middle_legacy(a int,c double generated always as (abs(a+b)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "drop table if exists gencol_refer_gencol_legacy" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "drop table if exists test_gen_col_array_func_legacy" + qt_gencol_array_function_create """ + create table test_gen_col_array_func_legacy(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + sql "drop table if exists test_gen_col_element_at_func_legacy" + qt_gencol_array_function_element_at_create """ + create table test_gen_col_element_at_func_legacy(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + test { + sql """ + create table gencol_type_check(a int,b int, c array generated always as (abs(a+b,3)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "In generated column 'c', no matching function with signature" + } + + // gencol_has_sum + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (sum(a)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Expression of generated column 'c' contains a disallowed function" + } + + // gencol_has_column_not_define + test { + sql """ + create table gencol_has_sum(a int,b int, c int generated always as (abs(d)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Unknown column 'd' in 'generated column function'" + } + + // gencol_refer_gencol_after + test { + sql """ + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+d)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column can refer only to generated columns defined prior to it." + } + + sql "set @myvar=2" + // gencol_has_var + test { + sql """ + create table test_gen_col_not_null100(a varchar(10),c double generated always as (abs(a+b+@myvar)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column expression cannot contain variable." + } + + test { + sql """ + create table test_gen_col_auto_increment(a bigint not null auto_increment, b int, c int as (a*b)) + distributed by hash(a) properties("replication_num" = "1"); + """ + exception "Generated column 'c' cannot refer to auto-increment column." + } + + test{ + sql """ + create table test_gen_col_subquery(a int,b int, c int generated always as (a+(select 1)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support subquery." + } + + test { + sql """ + create table test_gen_col_array_func_lambda(pk int,a array,b array, c array generated always as (array_count(x->(x%2=0),b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated column does not support lambda." + } + + test { + sql """ + create table test_gen_col_array_func_legacy(pk int,a array,b array, c double generated always as (a+b) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + """ + exception "In generated column 'c', can not cast from origin type" + } + + test { + sql """ + create table test_window_func(a int default 10, b int default 100, c boolean as(rank() over())) DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed expression:'rank() OVER ()'" + } + test { + sql """ + create table test_grouping(a int default 10, b int default 100, c boolean as(grouping(a))) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "Expression of generated column 'c' contains a disallowed function:'grouping'" + } + + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + qt_common_default_insert "INSERT INTO test_gen_col_common_legacy values(6,7,default);" + qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common_legacy(a,b) values(1,2);" + qt_common_default_test_insert_default "INSERT INTO test_gen_col_common_legacy values(3,5,default);" + qt_commont_default_select "select * from test_gen_col_common_legacy order by 1,2,3;" + + // qt_common_default_test_insert_null + test { + sql "INSERT INTO test_gen_col_common_legacy(a,b) values(1,null);" + exception "Insert has filtered data in strict mode." + } + + // qt_common_default_test_insert_gencol + test { + sql "INSERT INTO test_gen_col_common_legacy values(1,2,3);" + exception "The value specified for generated column 'c' in table 'test_gen_col_common_legacy' is not allowed." + } + + + qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always_legacy values(6,7,default);" + qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always_legacy(a,b) values(1,2);" + qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always_legacy order by 1,2,3;" + + + qt_gencol_in_middle_insert "insert into test_gen_col_in_middle_legacy values(1,default,5);" + qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle_legacy(a,b) values(4,5);" + qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle_legacy(a,b,c) values(1,6,default);" + qt_gencol_in_middle_select "select * from test_gen_col_in_middle_legacy order by 1,2,3;" + + + qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol_legacy values(1,default,5,default);" + qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol_legacy(a,b) values(5,6);" + qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol_legacy(a,b,c) values(2,9,default);" + qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol_legacy(a,b,c,d) values(3,3,default,default);" + qt_gencol_refer_gencol_select "select * from gencol_refer_gencol_legacy order by 1,2,3,4;" + + + qt_gencol_array_function_insert "insert into test_gen_col_array_func_legacy values(1,[1,2],[3,2],default);" + qt_gencol_array_function_select "select * from test_gen_col_array_func_legacy" + + + qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func_legacy values(1,[1,2],[3,2],default);" + qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func_legacy" + + test { + sql """ + create table test_gen_col_aggregate(a int,b int,c int sum generated always as (abs(a+1)) not null) + aggregate key(a,b) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." + } + + // test drop dependency + sql "drop table if exists gencol_refer_gencol_legacy" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol_legacy(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "insert into gencol_refer_gencol_legacy(a,b) values(3,4)" + test { + sql "alter table gencol_refer_gencol_legacy drop column a" + exception "Column 'a' has a generated column dependency on :[c]" + } + test { + sql "alter table gencol_refer_gencol_legacy drop column c" + exception "Column 'c' has a generated column dependency on :[d]" + } + sql "alter table gencol_refer_gencol_legacy drop column d" + sql "alter table gencol_refer_gencol_legacy drop column c" + sql "alter table gencol_refer_gencol_legacy drop column b" + qt_test_drop_column "select * from gencol_refer_gencol_legacy" + test { + sql """ + create table test_gen_col_default(a int,b int,c int generated always as (abs(a+1)) not null default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot have default value." + } + test { + sql """ + create table test_gen_col_increment(a int,b int,c int generated always as (abs(a+1)) not null auto_increment) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + exception "Generated columns cannot be auto_increment." + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy new file mode 100644 index 00000000000000..09b4b3dfaa2b81 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_delete_generated_column.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_delete") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists test_par_gen_col; + create table test_par_gen_col (a int, b int, c int as (a+b)) + partition by range(c) + ( + partition p1 values [('1'),('10')), + partition p2 values [('10'),('20')), + partition p3 values [('20'),('30')) + ) + DISTRIBUTED BY RANDOM BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + insert into test_par_gen_col values(1,2,default),(10,2,default),(2,22,default); + + drop table if exists test_par_gen_col_unique; + create table test_par_gen_col_unique (a int, b int, c int as (a+b)) unique key(a,b,c) + partition by range(c) + ( + partition p1 values [('1'),('10')), + partition p2 values [('10'),('20')), + partition p3 values [('20'),('30')) + ) + DISTRIBUTED BY hash(c) + PROPERTIES ("replication_num" = "1"); + insert into test_par_gen_col_unique values(1,2,default),(10,2,default),(2,22,default),(10,2,default); + """ + + qt_delete_where_gen_col "delete from test_par_gen_col partition p1 where c=3;" + qt_delete_where_gen_col_select "select * from test_par_gen_col order by a,b,c;" + qt_delete_where_gen_col_partition_has_no_satisfied_row "delete from test_par_gen_col partition p1 where c=12;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_select "select * from test_par_gen_col order by a,b,c;;" + qt_delete_where_gen_col_and_other_col "delete from test_par_gen_col partition p2 where c=12 and a=10;" + qt_delete_where_gen_col_and_other_col_select "select * from test_par_gen_col order by a,b,c;;" + + qt_delete_where_gen_col_unique "delete from test_par_gen_col_unique partition p1 where c=3;" + qt_delete_where_gen_col_select_unique "select * from test_par_gen_col_unique order by a,b,c;;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_unique "delete from test_par_gen_col_unique partition p1 where c=12;" + qt_delete_where_gen_col_partition_has_no_satisfied_row_select_unique "select * from test_par_gen_col_unique order by a,b,c;;" + qt_delete_where_gen_col_and_other_col_unique "delete from test_par_gen_col_unique partition p2 where c=12 and a=10;" + qt_delete_where_gen_col_and_other_col_select_unique "select * from test_par_gen_col_unique order by a,b,c;" + + qt_delete_query """delete from test_par_gen_col_unique t1 using test_par_gen_col t2 inner join test_par_gen_col t3 + on t2.b=t3.b where t1.c=t2.c and t1.b=t2.b""" + qt_delete_query_select "select * from test_par_gen_col_unique order by a,b,c;" + sql "insert into test_par_gen_col_unique values(1,2,default),(10,2,default),(2,22,default),(10,2,default);" + qt_delete_query_cte """ + with cte as( + select t2.* from + test_par_gen_col t2 inner join test_par_gen_col t3 on t2.b=t3.b + ) delete from test_par_gen_col_unique t1 using cte where t1.c=cte.c and t1.b=cte.b""" + qt_delete_query_cte_select "select * from test_par_gen_col_unique order by a,b,c" + + +} diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy new file mode 100644 index 00000000000000..c2c451b59ff7cc --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_generated_column_nereids.groovy @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + String db = context.config.getDbNameByFile(context.file) + + sql "drop table if exists test_gen_col_common" + qt_common_default """create table test_gen_col_common(a int,b int,c double generated always as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_common_default_insert "INSERT INTO test_gen_col_common values(6,7,default);" + qt_common_default_insert_with_specific_column "INSERT INTO test_gen_col_common(a,b) values(1,2);" + qt_common_default_test_insert_default "INSERT INTO test_gen_col_common values(3,5,default);" + + qt_commont_default_select "select * from test_gen_col_common order by 1,2,3;" + + sql "drop table if exists test_gen_col_without_generated_always" + qt_common_without_generated_always """create table test_gen_col_without_generated_always(a int,b int,c double as (abs(a+b)) not null) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_common_without_generated_always_insert "INSERT INTO test_gen_col_without_generated_always values(6,7,default);" + qt_common_without_generated_always_insert_with_specific_column "INSERT INTO test_gen_col_without_generated_always(a,b) values(1,2);" + + qt_commont_without_generated_always_select "select * from test_gen_col_without_generated_always order by 1,2,3;" + + sql "drop table if exists test_gen_col_in_middle" + qt_gencol_in_middle """create table test_gen_col_in_middle(a int,c double generated always as (abs(a+b)) not null,b int) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gencol_in_middle_insert "insert into test_gen_col_in_middle values(1,default,5);" + qt_gencol_in_middle_insert_with_specific_column "insert into test_gen_col_in_middle(a,b) values(4,5);" + qt_gencol_in_middle_insert_with_specific_column_2 "insert into test_gen_col_in_middle(a,b,c) values(1,6,default);" + qt_gencol_in_middle_insert_multi_values "insert into test_gen_col_in_middle values(1,default,2),(3,default,4);" + qt_gencol_in_middle_select "select * from test_gen_col_in_middle order by 1,2,3;" + + sql "drop table if exists test_gen_col_empty_values" + qt_gencol_empty_values """create table test_gen_col_empty_values(a int default 10,c double generated always as (abs(a+b)) not null,b int default 100) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gencol_empty_values_insert "insert into test_gen_col_empty_values values(),()" + qt_gencol_empty_values_insert_2 "insert into test_gen_col_empty_values(c) values(default),(default)" + qt_gencol_empty_values_insert_3 "insert into test_gen_col_empty_values(a,c) values(1,default),(3,default)" + qt_gencol_empty_values_insert_4 "insert into test_gen_col_empty_values(c,a) values(default,5),(default,6)" + + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + qt_gencol_refer_gencol_insert "insert into gencol_refer_gencol values(1,default,5,default);" + qt_gencol_refer_gencol_insert2 "insert into gencol_refer_gencol(a,b) values(5,6);" + qt_gencol_refer_gencol_insert3 "insert into gencol_refer_gencol(a,b,c) values(2,9,default);" + qt_gencol_refer_gencol_insert4 "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default);" + qt_gencol_refer_gencol_insert_multi_values "insert into gencol_refer_gencol(a,b,c,d) values(3,3,default,default),(5,7,default,default);" + qt_gencol_refer_gencol_select "select * from gencol_refer_gencol order by 1,2,3,4;" + + sql "drop materialized view if exists test_mv_gen_col on gencol_refer_gencol" + createMV ("""create materialized view test_mv_gen_col as select a,sum(a),sum(c) ,sum(d) from gencol_refer_gencol group by a;""") + qt_test_insert_mv "insert into gencol_refer_gencol(a,b) values(1,2),(3,5)" + qt_test_select "select a,sum(a),sum(c),sum(d) from gencol_refer_gencol group by a order by 1,2,3,4" + explain{ + sql "select a,sum(a),sum(c),sum(d) from gencol_refer_gencol group by a" + contains "test_mv_gen_col" + } + + sql "drop table if exists test_gen_col_array_func" + qt_gencol_array_function_create """ + create table test_gen_col_array_func(pk int,a array,b array, c array generated always as (array_union(a,b)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + qt_gencol_array_function_insert "insert into test_gen_col_array_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_select "select * from test_gen_col_array_func" + + sql "drop table if exists test_gen_col_element_at_func" + qt_gencol_array_function_element_at_create """ + create table test_gen_col_element_at_func(pk int,a array,b array, c int generated always as (element_at(a, 1)) not null) + DISTRIBUTED BY HASH(pk) + PROPERTIES("replication_num" = "1"); + ; + """ + qt_gencol_array_function_element_at_insert "insert into test_gen_col_element_at_func values(1,[1,2],[3,2],default);" + qt_gencol_array_function_element_at_select "select * from test_gen_col_element_at_func" + + sql "drop table if exists gencol_like_t;" + qt_create_table_like "create table gencol_like_t like test_gen_col_common;" + qt_create_table_like_insert "insert into gencol_like_t values(1,2,default);" + qt_create_table_like_select "select * from gencol_like_t" + sql "drop table if exists gencol_like_t;" + qt_refer_gencol_create_table_like "create table gencol_like_t like gencol_refer_gencol;" + qt_refer_gencol_create_table_like_insert "insert into gencol_like_t values(1,default,2,default);" + qt_refer_gencol_create_table_like_select "select * from gencol_like_t" + + sql "drop table if exists gencol_refer_gencol_row_store" + qt_test_row_store """create table gencol_refer_gencol_row_store(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1","store_row_column" = "true");""" + qt_row_store_insert "insert into gencol_refer_gencol_row_store values(1,default,5,default);" + qt_row_store_select "select * from gencol_refer_gencol_row_store" + + sql "drop ENCRYPTKEY if exists my_key" + sql """CREATE ENCRYPTKEY my_key AS "ABCD123456789";""" + sql "drop table if exists test_encrypt_key_gen_col" + qt_test_aes_encrypt """create table test_encrypt_key_gen_col(a int default 10, b int default 100, + c varchar(100) as(hex(aes_encrypt("abc",key my_key)))) distributed by hash(a) properties("replication_num"="1");;""" + qt_test_aes_encrypt_insert "insert into test_encrypt_key_gen_col values(1,2,default)" + qt_test_aes_encrypt_select "select * from test_encrypt_key_gen_col" + + sql "drop table if exists test_encrypt_key_gen_col_with_db" + qt_test_aes_encrypt_with_db """create table test_encrypt_key_gen_col_with_db(a int default 10, b int default 100, + c varchar(100) as(hex(aes_encrypt("abc",key ${db}.my_key)))) distributed by hash(a) properties("replication_num"="1");""" + qt_test_aes_encrypt_insert_with_db "insert into test_encrypt_key_gen_col_with_db values(1,2,default)" + qt_test_aes_encrypt_select_with_db "select * from test_encrypt_key_gen_col_with_db" + + qt_describe "describe gencol_refer_gencol" + + //test update + sql "drop table if exists test_gen_col_update" + sql """create table test_gen_col_update (a int, b int, c int as (a+b)) + unique key(a) + distributed by hash(a) properties("replication_num"="1")""" + sql "insert into test_gen_col_update values(1,3,default)" + qt_test_update "update test_gen_col_update set b=20" + qt_test_update_generated_column "select * from test_gen_col_update" + + // test unique table, generated column is not key + sql "drop table if exists test_gen_col_unique_key" + qt_gen_col_unique_key """create table test_gen_col_unique_key(a int,b int,c int generated always as (abs(a+b)) not null) + unique key(a,b) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + qt_gen_col_unique_key_insert "INSERT INTO test_gen_col_unique_key values(6,7,default);" + qt_gen_col_unique_key_insert "INSERT INTO test_gen_col_unique_key values(6,7,default);" + qt_gen_col_unique_key_select "select * from test_gen_col_unique_key order by 1,2,3;" + + // test unique table,generated column is key + sql "drop table if exists test_gen_col_unique_key2" + sql """create table test_gen_col_unique_key2(a int,c int generated always as (abs(a+1)) not null,b int) + unique key(a,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + qt_gen_col_unique_key_insert_is_key "INSERT INTO test_gen_col_unique_key2 values(6,default,7);" + qt_gen_col_unique_key_insert_is_key "INSERT INTO test_gen_col_unique_key2 values(6,default,7);" + qt_gen_col_unique_key_select_is_key "select * from test_gen_col_unique_key2" + + // test aggregate table,generated column is key + sql "drop table if exists test_gen_col_aggregate" + sql """ + create table test_gen_col_aggregate(a int,b int,c int generated always as (abs(a+1)) not null) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + qt_gen_col_aggregate_insert "INSERT INTO test_gen_col_aggregate values(6,7,default);" + qt_gen_col_aggregate_insert "INSERT INTO test_gen_col_aggregate values(6,7,default);" + qt_gen_col_aggregate_select "select * from test_gen_col_aggregate" + + // test drop dependency + sql "drop table if exists gencol_refer_gencol" + qt_gencol_refer_gencol """ + create table gencol_refer_gencol(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + """ + sql "insert into gencol_refer_gencol(a,b) values(3,4)" + test { + sql "alter table gencol_refer_gencol drop column a" + exception "Column 'a' has a generated column dependency on :[c]" + } + test { + sql "alter table gencol_refer_gencol drop column c" + exception "Column 'c' has a generated column dependency on :[d]" + } + sql "alter table gencol_refer_gencol drop column d" + sql "alter table gencol_refer_gencol drop column c" + sql "alter table gencol_refer_gencol drop column b" + qt_test_drop_column "select * from gencol_refer_gencol" + + // test agg table, gen col is the key or replace/replace_if_not_null + multi_sql """ + drop table if exists agg_gen_col_key; + create table agg_gen_col_key(a int,b int, c int as (a+b), d int sum) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + insert into agg_gen_col_key(a,b) values(1,2); + insert into agg_gen_col_key(a,b,d) values(1,2,6); + """ + qt_test_agg_key "select * from agg_gen_col_key order by 1,2,3,4" + + sql "drop table if exists agg_gen_col_replace" + qt_agg_gen_col_replace """create table agg_gen_col_replace(a int,b int, c int as (a+b), d int replace as (c+1)) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + + multi_sql """ + drop table if exists agg_gen_col_replace_if_not_null; + create table agg_gen_col_replace_if_not_null(a int,b int, c int as (a+b), d int REPLACE_IF_NOT_NULL as (c+1), f int sum default 10) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + insert into agg_gen_col_replace_if_not_null(a,b,f) values(1,2,3); + insert into agg_gen_col_replace_if_not_null(a,b) values(1,2); + """ + qt_agg_replace_null "select * from agg_gen_col_replace_if_not_null order by 1,2,3,4" + + test { + sql """create table agg_gen_col_replace(a int,b int, c int as (a+b), d int sum as (c+1)) + aggregate key(a,b,c) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + exception "The generated columns can be key columns, or value columns of replace and replace_if_not_null aggregation type." + } +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy new file mode 100644 index 00000000000000..5a5f696a0b50a7 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_insert_overwrite_generated_column.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_insert_overwrite_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql """ + drop table if exists gen_col_insert_overwrite_src; + CREATE TABLE gen_col_insert_overwrite_src(a int, b int, c int,d int) ENGINE=OLAP + UNIQUE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`a`) BUCKETS 10 properties("replication_num"="1"); + insert into gen_col_insert_overwrite_src values(1,2,3,5),(3,23,5,1); + + drop table if exists gen_col_insert_overwrite; + CREATE TABLE gen_col_insert_overwrite(a int, b int, c int AS(a+b), d int AS (c+1)) + ENGINE=OLAP + UNIQUE KEY(`a`, `b`, `c`) + DISTRIBUTED BY HASH(`a`) BUCKETS 10 properties("replication_num"="1"); + INSERT into gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default); + """ + + qt_overwrite_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) values(3,4)" + qt_overwrite_value "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default);" + qt_overwrite_value_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_multi_value "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,DEFAULT,default),(3,4,DEFAULT,default);" + qt_overwrite_multi_value_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_multi_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) values(1,2),(3,4);" + qt_overwrite_multi_value_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_select_value_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) select 1,2;" + qt_overwrite_select_value_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + qt_overwrite_select_table_partial_column "INSERT overwrite TABLE gen_col_insert_overwrite(a,b) select a,b from gen_col_insert_overwrite_src;" + qt_overwrite_select_table_partial_column_select "select * from gen_col_insert_overwrite order by 1,2,3,4;" + + multi_sql """ + drop table if exists gen_col_insert_overwrite_par; + CREATE TABLE IF NOT EXISTS gen_col_insert_overwrite_par ( + `c1` int NOT NULL DEFAULT "1", + `c2` int NOT NULL DEFAULT "4", + `c3` int as(c1+c2) + ) ENGINE=OLAP + UNIQUE KEY(`c1`) + PARTITION BY LIST (`c1`) + ( + PARTITION p1 VALUES IN ("1","2","3"), + PARTITION p2 VALUES IN ("4","5","6") + ) + DISTRIBUTED BY HASH(`c1`) BUCKETS 3 + properties("replication_num"="1"); + """ + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1,p2) (c1,c2) VALUES (1, 2);" + qt_overwrite_partition_single_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1) (c1,c2) VALUES (1, 2);" + qt_overwrite_partition_partial_single_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1,p2) (c1,c2) VALUES (1, 2), (4, 2 * 2), (5, DEFAULT);" + qt_overwrite_partition_multi_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(p1) (c1) VALUES (1),(2),(3);" + qt_overwrite_partition_partial_multi_value "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + sql "insert into gen_col_insert_overwrite_par values(1,5,default),(4,3,default);" + sql "INSERT OVERWRITE table gen_col_insert_overwrite_par PARTITION(*) (c1) VALUES (1), (2),(3);" + qt_auto_partition_overwrite "select * from gen_col_insert_overwrite_par order by 1,2,3;" + + test { + sql "insert into gen_col_insert_overwrite_par values(1,5,2),(4,3,3);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par values(1,5,2);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) values(1,5,2),(4,3,3);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) values(1,5,2);" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test{ + sql "INSERT overwrite TABLE gen_col_insert_overwrite values(1,2,5,default),(3,4,DEFAULT,3);" + exception "The value specified for generated column 'c' in table 'gen_col_insert_overwrite' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par select 1,2,3;" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par select * from gen_col_insert_overwrite_src" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + test { + sql "insert into gen_col_insert_overwrite_par(c1,c2,c3) select * from gen_col_insert_overwrite_src;" + exception "The value specified for generated column 'c3' in table 'gen_col_insert_overwrite_par' is not allowed." + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy new file mode 100644 index 00000000000000..6b89934f1a2ca4 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_load_delete_generated_column.groovy @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_load_delete_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + multi_sql""" + drop table if exists test_stream_delete; + create table test_stream_delete(a int, b int, c int as (a + b) , d int as (c + 1) ) + unique key(a, b, c) distributed by hash(a) properties("replication_num"="1");; + insert into test_stream_delete(a, b) values(1, 2) , (3, 5 ), (2, 9 ); + """ + + streamLoad { + table 'test_stream_delete' + set 'column_separator', ',' + file 'gen_col_data_delete.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'merge_type', 'DELETE' + } + sql "sync" + qt_test_stream_load_delete "select * from test_stream_delete order by 1,2,3,4" + + streamLoad { + table 'test_stream_delete' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'merge_type', 'MERGE' + set 'delete','a=3' + } + sql "sync" + qt_test_stream_load_merge "select * from test_stream_delete order by 1,2,3,4" + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy new file mode 100644 index 00000000000000..f8cd5cf567adff --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_partial_update_generated_column.groovy @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partial_update_generated_column") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + sql "set enable_unique_key_partial_update=true" + sql "set enable_insert_strict=false" + + sql "drop table if exists test_partial_column_unique_gen_col" + sql """create table test_partial_column_unique_gen_col (a int, b int, c int as(a+b), d int as(c+1), e int) + unique key(a) distributed by hash(a) properties( + "enable_unique_key_merge_on_write" = "true", + "replication_num"="1" + );""" + + multi_sql """ + insert into test_partial_column_unique_gen_col(a,b,e) values(1,2,7); + insert into test_partial_column_unique_gen_col(a,b) select 2,3; + insert into test_partial_column_unique_gen_col(a,b) select 2,4; + insert into test_partial_column_unique_gen_col(a,b) values(1,6),(3,6); + insert into test_partial_column_unique_gen_col(a,b) values(1,5);""" + qt_partial_update_select "select * from test_partial_column_unique_gen_col order by 1,2,3,4,5" + + test { + sql "insert into test_partial_column_unique_gen_col(a) values(3);" + exception "Partial update should include all ordinary columns referenced by generated columns, missing: b" + } + + // test stream load partial update + multi_sql """ + truncate table test_partial_column_unique_gen_col; + insert into test_partial_column_unique_gen_col(a,b,e) values(1,4,0),(3,2,8),(2,9,null); + """ + + streamLoad { + table 'test_partial_column_unique_gen_col' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,b' + time 10000 // limit inflight 10s + set 'partial_columns', 'true' + } + sql "sync" + qt_test_stream_load_refer_gencol "select * from test_partial_column_unique_gen_col order by 1,2,3,4" + + streamLoad { + table 'test_partial_column_unique_gen_col' + set 'column_separator', ',' + file 'gen_col_data.csv' + set 'columns', 'a,e' + time 10000 // limit inflight 10s + set 'partial_columns', 'true' + check {result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + mustContain(json.Message, "Partial update should include all ordinary columns referenced by generated columns, missing: b") + } + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy new file mode 100644 index 00000000000000..edf8ab2532345a --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_routine_load_generated_column.groovy @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_routine_load_generated_column") { + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // define kafka + String topic = "testGenCol"; + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + def producer = new KafkaProducer<>(props) + filepath = getLoalFilePath "gen_col_data.csv" + def txt = new File("${filepath}").text + def lines = txt.readLines(); + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(topic, null, line) + producer.send(record) + } + def tableName = "gencol_refer_gencol_rload" + sql "drop table if exists ${tableName}" + sql """create table ${tableName}(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1"); + ;""" + + sql """ + CREATE ROUTINE LOAD gen_col_routine_load ON ${tableName} + COLUMNS(a,b), + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${topic}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + while (true) { + sleep(1000) + def res = sql "show routine load for gen_col_routine_load" + def state = res[0][8].toString() + if (state == "NEED_SCHEDULE") { + continue; + } + log.info("reason of state changed: ${res[0][17].toString()}".toString()) + assertEquals(res[0][8].toString(), "RUNNING") + break; + } + def count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for gen_col_routine_load" + log.info("routine load state: ${state[0][8].toString()}".toString()) + log.info("routine load statistic: ${state[0][14].toString()}".toString()) + log.info("reason of state changed: ${state[0][17].toString()}".toString()) + if (res[0][0] > 0) { + break + } + if (count >= 120) { + log.error("routine load can not visible for long time") + assertEquals(20, res[0][0]) + break + } + sleep(5000) + count++ + } + + qt_common_default "select * from ${tableName} order by 1,2,3" + + sql "stop routine load for gen_col_routine_load;" + } + +} \ No newline at end of file diff --git a/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy new file mode 100644 index 00000000000000..3ff05817d965ed --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_generated_column/test_update_generated_column.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_generated_column_update") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + + // duplicate table + sql "drop table if exists test_gen_col_update2" + sql """create table test_gen_col_update2(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "insert into test_gen_col_update2 values(1,default,5,default);" + sql "insert into test_gen_col_update2(a,b) values(4,5);" + + sql "drop table if exists test_gen_col_update2" + sql """CREATE TABLE `test_gen_col_update2` ( + `id` INT NULL, + `c1` DECIMAL(10, 4) NULL, + `c2` TEXT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `c1`) + DISTRIBUTED BY RANDOM BUCKETS 10 + PROPERTIES ("replication_num" = "1");""" + sql "insert into test_gen_col_update2 values(1,2,'d'),(2,3,'ds'),(3,4,'aa')" + + test { + sql "update test_gen_col_update2 set a=9,b=10" + exception "Only unique table could be updated." + } + + // unique + sql "drop table if exists test_gen_col_update_unique" + sql """create table test_gen_col_update_unique(a int,c double generated always as (abs(a+b)) not null,b int, d int generated always as(c+1)) + unique key(a) + DISTRIBUTED BY HASH(a) + PROPERTIES("replication_num" = "1");""" + sql "insert into test_gen_col_update_unique values(3,default,5,default);" + sql "insert into test_gen_col_update_unique(a,b) values(4,5);" + qt_update_gen_col "update test_gen_col_update_unique set b=10" + qt_update_select "select * from test_gen_col_update_unique order by a,b,c,d" + test { + sql "update test_gen_col_update_unique set c=100" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } + test { + sql "update test_gen_col_update_unique set c=(select 10)" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } + + qt_update_use_expr "update test_gen_col_update_unique set b=a+100 WHERE a=3;" + qt_update_use_expr_select "select * from test_gen_col_update_unique order by a,b,c,d" + + qt_update_use_other_table_col "update test_gen_col_update_unique t1 set t1.b=test_gen_col_update2.id from test_gen_col_update2 where t1.a=test_gen_col_update2.id;" + qt_update_use_expr_select "select * from test_gen_col_update_unique order by a,b,c,d" + + test{ + sql "update test_gen_col_update_unique t1 set t1.c=test_gen_col_update2.id from test_gen_col_update2 where t1.a=test_gen_col_update2.id;" + exception "The value specified for generated column 'c' in table 'test_gen_col_update_unique' is not allowed." + } +} \ No newline at end of file