diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index ead905e9ed2e8a..720dcf2dc1b687 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -58,6 +58,7 @@ */ public class Column implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Column.class); + public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_"; // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index bab89ce85f1330..225798cd13949a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.catalog.Column; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.Role; import org.apache.doris.mysql.privilege.RoleManager; @@ -94,6 +95,12 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep } public static void checkColumnName(String columnName) throws AnalysisException { + // if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn` + checkColumnNameBypassHiddenColumn(columnName); + checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX); + } + + public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName, getColumnNameRegex()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index a8eadf16d3d815..3fe1a52445db3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -67,6 +67,8 @@ public class ColumnDefinition { private int clusterKeyId = -1; private Optional generatedColumnDesc = Optional.empty(); private Set generatedColumnsThatReferToThis = new HashSet<>(); + // if add hidden column, must set enableAddHiddenColumn true + private boolean enableAddHiddenColumn = false; public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable, Optional defaultValue, String comment) { @@ -286,7 +288,14 @@ private void checkKeyColumnType(boolean isOlap) { public void validate(boolean isOlap, Set keysSet, Set clusterKeySet, boolean isEnableMergeOnWrite, KeysType keysType) { try { - FeNameFormat.checkColumnName(name); + // if enableAddHiddenColumn is true, can add hidden column. + // So does not check if the column name starts with __DORIS_ + if (enableAddHiddenColumn) { + FeNameFormat.checkColumnNameBypassHiddenColumn(name); + } else { + FeNameFormat.checkColumnName(name); + } + FeNameFormat.checkColumnCommentLength(comment); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); @@ -523,43 +532,90 @@ public Column translateToCatalogStyleForSchemaChange() { return column; } - // hidden column + /** + * add hidden column + */ public static ColumnDefinition newDeleteSignColumnDefinition() { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, + false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris delete flag hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris delete flag hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newSequenceColumnDefinition(DataType type) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true, - Optional.empty(), "sequence column hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, + true, Optional.empty(), + "sequence column hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true, - Optional.empty(), "sequence column hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, + true, Optional.empty(), + "sequence column hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue("")), "doris row store hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue("")), + "doris row store hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris version hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } - // used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE - // becasue ColumnDefinition.validate() will check that bitmap type column don't set default value - // and then set the default value of that column to bitmap_empty() + /** + * used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE + * becasue ColumnDefinition.validate() will check that bitmap type column don't set default value + * and then set the default value of that column to bitmap_empty() + */ public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) { - return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false, - Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, + aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), + "doris skip bitmap hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } public Optional getGeneratedColumnDesc() { @@ -574,6 +630,10 @@ public void addGeneratedColumnsThatReferToThis(List list) { generatedColumnsThatReferToThis.addAll(list); } + public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) { + this.enableAddHiddenColumn = enableAddHiddenColumn; + } + private void validateGeneratedColumnInfo() { // for generated column if (generatedColumnDesc.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java index de3b01c7865b41..74a216d265173f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java @@ -72,6 +72,9 @@ public void validate(ConnectContext ctx) throws UserException { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, colName, FeNameFormat.getColumnNameRegex()); } + if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + throw new AnalysisException("Do not support drop hidden column"); + } TableIf table = Env.getCurrentEnv().getCatalogMgr() .getCatalogOrDdlException(tableName.getCtl()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java index 8bbdcfc723a123..6774645795f0cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.AlterOpType; import org.apache.doris.analysis.AlterTableClause; import org.apache.doris.analysis.ColumnRenameClause; +import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; @@ -61,6 +62,10 @@ public void validate(ConnectContext ctx) throws UserException { throw new AnalysisException("New column name is not set"); } + if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) { + throw new AnalysisException("Do not support rename hidden column"); + } + FeNameFormat.checkColumnName(newColName); } diff --git a/regression-test/suites/schema_change/test_alter_table_on_hidden_column.groovy b/regression-test/suites/schema_change/test_alter_table_on_hidden_column.groovy new file mode 100644 index 00000000000000..1cd74bd337e71a --- /dev/null +++ b/regression-test/suites/schema_change/test_alter_table_on_hidden_column.groovy @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_alter_table_on_hidden_column", "schema_change") { + def tableName = "alter_table_hidden_column" + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + k1 INT, + value1 INT, + value2 INT, + value3 INT + ) + DUPLICATE KEY (k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + properties( + "replication_num" = "1" + ); + """ + + // rename hidden column + try { + sql "ALTER TABLE ${tableName} RENAME COLUMN __DORIS_VERSION_COL__ test;" + } catch (Exception ex) { + assert("${ex}".contains("Do not support rename hidden column")) + } + + // drop hidden column + try { + sql "ALTER TABLE ${tableName} DROP COLUMN __DORIS_VERSION_COL__;" + } catch (Exception ex) { + assert("${ex}".contains("Do not support drop hidden column")) + } + + // add a column name starting with __DORIS_ + try { + sql "ALTER TABLE ${tableName} ADD COLUMN __DORIS_VERSION_COL__ bigint;" + } catch (Exception ex) { + assert("${ex}".contains("column name can't start with")) + } + + // add columns, a column name starting with __DORIS_ + try { + sql "ALTER TABLE ${tableName} ADD COLUMN (c1 int, __DORIS_VERSION_COL__ bigint);" + } catch (Exception ex) { + assert("${ex}".contains("column name can't start with")) + } + + // modify hidden column comment + try { + sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ COMMENT 'test';" + } catch (Exception ex) { + assert("${ex}".contains("Unknown column")) + } + + // modify hidden column type + try { + sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ VARCHAR(64);" + } catch (Exception ex) { + assert("${ex}".contains("column name can't start with")) + } + +}