diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 7d0d24aa4e5ce5..918128588460a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -199,6 +199,8 @@ public String getValue() { private int clusterKeyId = -1; private Optional generatedColumnInfo = Optional.empty(); private Set generatedColumnsThatReferToThis = new HashSet<>(); + // if add hidden column, must set enableAddHiddenColumn true + private boolean enableAddHiddenColumn = false; public ColumnDef(String name, TypeDef typeDef) { @@ -262,6 +264,10 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr this.visible = visible; } + public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) { + this.enableAddHiddenColumn = enableAddHiddenColumn; + } + public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType nullableType, String comment, Optional generatedColumnInfo) { this(name, typeDef, isKey, null, nullableType, -1, DefaultValue.NOT_SET, @@ -269,50 +275,72 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType } public static ColumnDef newDeleteSignColumnDef() { - return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, - ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), + ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, + ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"), "doris delete flag hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, - ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), + ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, + aggregateType, + ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"), "doris delete flag hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newSequenceColumnDef(Type type) { - return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1, + ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), + false, null, ColumnNullableType.NULLABLE, -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) { - return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE, + ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), + false, aggregateType, ColumnNullableType.NULLABLE, -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""), "doris row store hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newVersionColumnDef() { - return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, + ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newVersionColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newSkipBitmapColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE, "doris skip bitmap hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public boolean isAllowNull() { @@ -375,7 +403,13 @@ public void analyze(boolean isOlap) throws AnalysisException { if (name == null || typeDef == null) { throw new AnalysisException("No column name or column type in column definition."); } - FeNameFormat.checkColumnName(name); + + if (enableAddHiddenColumn) { + FeNameFormat.checkColumnNameBypassHiddenColumn(name); + } else { + FeNameFormat.checkColumnName(name); + } + FeNameFormat.checkColumnCommentLength(comment); typeDef.analyze(null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 47cca3b90bba5f..ab6b423d976ffa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -62,6 +62,7 @@ */ public class Column implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Column.class); + public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_"; // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index a7165c3d815636..c6c751a88c6761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.catalog.Column; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.Role; import org.apache.doris.mysql.privilege.RoleManager; @@ -93,18 +94,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep } public static void checkColumnName(String columnName) throws AnalysisException { + // if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn` + checkColumnNameBypassHiddenColumn(columnName); + checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX); + } + + public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName, getColumnNameRegex()); } - if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, - columnName, getColumnNameRegex()); + checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX); + checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX); + checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX); + } + + private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException { + int prefixLength = prefix.length(); + if (columnName.length() < prefixLength) { + return; } - if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX) - || columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) { + if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) { throw new AnalysisException( - "Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'"); + "Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 367bdff18fc19f..77c712de990883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -69,6 +69,8 @@ public class ColumnDefinition { private int clusterKeyId = -1; private Optional generatedColumnDesc = Optional.empty(); private Set generatedColumnsThatReferToThis = new HashSet<>(); + // if add hidden column, must set enableAddHiddenColumn true + private boolean enableAddHiddenColumn = false; public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable, Optional defaultValue, String comment) { @@ -232,7 +234,14 @@ private void checkKeyColumnType(boolean isOlap) { public void validate(boolean isOlap, Set keysSet, Set clusterKeySet, boolean isEnableMergeOnWrite, KeysType keysType) { try { - FeNameFormat.checkColumnName(name); + // if enableAddHiddenColumn is true, can add hidden column. + // So does not check if the column name starts with __DORIS_ + if (enableAddHiddenColumn) { + FeNameFormat.checkColumnNameBypassHiddenColumn(name); + } else { + FeNameFormat.checkColumnName(name); + } + FeNameFormat.checkColumnCommentLength(comment); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); @@ -689,43 +698,54 @@ public Column translateToCatalogStyle() { return column; } - // hidden column - public static ColumnDefinition newDeleteSignColumnDefinition() { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); - } - + /** + * add hidden column + */ public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); - } + ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris delete flag hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); - public static ColumnDefinition newSequenceColumnDefinition(DataType type) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true, - Optional.empty(), "sequence column hidden column", false); - } - - public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true, - Optional.empty(), "sequence column hidden column", false); + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue("")), "doris row store hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue("")), + "doris row store hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris version hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } - // used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE - // becasue ColumnDefinition.validate() will check that bitmap type column don't set default value - // and then set the default value of that column to bitmap_empty() + /** + * used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE + * becasue ColumnDefinition.validate() will check that bitmap type column don't set default value + * and then set the default value of that column to bitmap_empty() + */ public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) { - return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false, - Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, + aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), + "doris skip bitmap hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } public Optional getGeneratedColumnDesc() { @@ -740,6 +760,10 @@ public void addGeneratedColumnsThatReferToThis(List list) { generatedColumnsThatReferToThis.addAll(list); } + public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) { + this.enableAddHiddenColumn = enableAddHiddenColumn; + } + private void validateGeneratedColumnInfo() { // for generated column if (generatedColumnDesc.isPresent()) {