[Enhancement] reject schema change on hidden columns

fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java

@@ -58,6 +58,7 @@
  */
 public class Column implements GsonPostProcessable {
     private static final Logger LOG = LogManager.getLogger(Column.class);
+    public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
     // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
     public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";

fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java

@@ -20,6 +20,7 @@
 import org.apache.doris.alter.SchemaChangeHandler;
 import org.apache.doris.analysis.CreateMaterializedViewStmt;
 import org.apache.doris.analysis.ResourceTypeEnum;
+import org.apache.doris.catalog.Column;
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.mysql.privilege.Role;
 import org.apache.doris.mysql.privilege.RoleManager;
@@ -94,6 +95,12 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
     }
 
     public static void checkColumnName(String columnName) throws AnalysisException {
+        // if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
+        checkColumnNameBypassHiddenColumn(columnName);
+        checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
+    }
+
+    public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
         if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
             ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
                     columnName, getColumnNameRegex());

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java

@@ -67,6 +67,8 @@ public class ColumnDefinition {
     private int clusterKeyId = -1;
     private Optional<GeneratedColumnDesc> generatedColumnDesc = Optional.empty();
     private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
+    // if add hidden column, must set enableAddHiddenColumn true
+    private boolean enableAddHiddenColumn = false;
 
     public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
             Optional<DefaultValue> defaultValue, String comment) {
@@ -286,7 +288,14 @@ private void checkKeyColumnType(boolean isOlap) {
     public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
             KeysType keysType) {
         try {
-            FeNameFormat.checkColumnName(name);
+            // if enableAddHiddenColumn is true, can add hidden column.
+            // So does not check if the column name starts with __DORIS_
+            if (enableAddHiddenColumn) {
+                FeNameFormat.checkColumnNameBypassHiddenColumn(name);
+            } else {
+                FeNameFormat.checkColumnName(name);
+            }
+
             FeNameFormat.checkColumnCommentLength(comment);
         } catch (Exception e) {
             throw new AnalysisException(e.getMessage(), e);
@@ -523,43 +532,90 @@ public Column translateToCatalogStyleForSchemaChange() {
         return column;
     }
 
-    // hidden column
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newDeleteSignColumnDefinition() {
-        return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
-                Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null,
+                false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
+                "doris delete flag hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
-        return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
-                Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
+                        aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
+                "doris delete flag hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
-        return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
-                Optional.empty(), "sequence column hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null,
+                true, Optional.empty(),
+                "sequence column hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
-        return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
-                Optional.empty(), "sequence column hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType,
+                true, Optional.empty(),
+                "sequence column hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
-        return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
-                Optional.of(new DefaultValue("")), "doris row store hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
+                        aggregateType, false, Optional.of(new DefaultValue("")),
+                "doris row store hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
+    /**
+     * add hidden column
+     */
     public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
-        return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
-                Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
+                    aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
+                "doris version hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
-    // used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
-    // becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
-    // and then set the default value of that column to bitmap_empty()
+    /**
+     * used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
+     * becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
+     * and then set the default value of that column to bitmap_empty()
+     */
     public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) {
-        return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false,
-                Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false);
+        ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false,
+                aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE),
+                "doris skip bitmap hidden column", false);
+        columnDefinition.setEnableAddHiddenColumn(true);
+
+        return columnDefinition;
     }
 
     public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
@@ -574,6 +630,10 @@ public void addGeneratedColumnsThatReferToThis(List<String> list) {
         generatedColumnsThatReferToThis.addAll(list);
     }
 
+    public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
+        this.enableAddHiddenColumn = enableAddHiddenColumn;
+    }
+
     private void validateGeneratedColumnInfo() {
         // for generated column
         if (generatedColumnDesc.isPresent()) {

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/DropColumnOp.java

@@ -72,6 +72,9 @@ public void validate(ConnectContext ctx) throws UserException {
             ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
                     colName, FeNameFormat.getColumnNameRegex());
         }
+        if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) {
+            throw new AnalysisException("Do not support drop hidden column");
+        }
 
         TableIf table = Env.getCurrentEnv().getCatalogMgr()
                 .getCatalogOrDdlException(tableName.getCtl())

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RenameColumnOp.java

@@ -20,6 +20,7 @@
 import org.apache.doris.alter.AlterOpType;
 import org.apache.doris.analysis.AlterTableClause;
 import org.apache.doris.analysis.ColumnRenameClause;
+import org.apache.doris.catalog.Column;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.UserException;
@@ -61,6 +62,10 @@ public void validate(ConnectContext ctx) throws UserException {
             throw new AnalysisException("New column name is not set");
         }
 
+        if (colName.startsWith(Column.HIDDEN_COLUMN_PREFIX)) {
+            throw new AnalysisException("Do not support rename hidden column");
+        }
+
         FeNameFormat.checkColumnName(newColName);
     }
 

regression-test/suites/schema_change/test_alter_table_on_hidden_column.groovy

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_alter_table_on_hidden_column", "schema_change") {
+    def tableName = "alter_table_hidden_column"
+
+    sql "DROP TABLE IF EXISTS ${tableName}"
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                k1 INT,
+                value1 INT,
+                value2 INT,
+                value3 INT
+            )
+            DUPLICATE KEY (k1)
+            DISTRIBUTED BY HASH(k1) BUCKETS 1 
+            properties(
+                "replication_num" = "1"
+            );
+        """
+
+    // rename hidden column 
+    try {
+        sql "ALTER TABLE ${tableName} RENAME COLUMN __DORIS_VERSION_COL__ test;"
+    } catch (Exception ex) {
+        assert("${ex}".contains("Do not support rename hidden column"))
+    }
+
+    // drop hidden column
+    try {
+        sql "ALTER TABLE ${tableName} DROP COLUMN __DORIS_VERSION_COL__;"
+    } catch (Exception ex) {
+        assert("${ex}".contains("Do not support drop hidden column"))
+    }
+
+    // add a column name starting with __DORIS_ 
+    try {
+        sql "ALTER TABLE ${tableName} ADD COLUMN __DORIS_VERSION_COL__ bigint;"
+    } catch (Exception ex) {
+        assert("${ex}".contains("column name can't start with"))
+    }
+
+    // add columns, a column name starting with __DORIS_ 
+    try {
+        sql "ALTER TABLE ${tableName} ADD COLUMN (c1 int, __DORIS_VERSION_COL__ bigint);"
+    } catch (Exception ex) {
+        assert("${ex}".contains("column name can't start with"))
+    }
+
+    // modify hidden column comment
+    try {
+        sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ COMMENT 'test';"
+    } catch (Exception ex) {
+        assert("${ex}".contains("Unknown column"))
+    }
+
+    // modify hidden column type
+    try {
+        sql "ALTER TABLE ${tableName} MODIFY COLUMN __DORIS_VERSION_COL__ VARCHAR(64);"
+    } catch (Exception ex) {
+        assert("${ex}".contains("column name can't start with"))
+    }
+
+}