Skip to content

branch-3.1: [Enhancement] reject schema change on hidden columns #53376 #55385

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 1 commit into from
Sep 4, 2025
Merged

branch-3.1: [Enhancement] reject schema change on hidden columns #53376 #55385

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 45 additions & 11 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ public String getValue() {
private int clusterKeyId = -1;
private Optional<GeneratedColumnInfo> generatedColumnInfo = Optional.empty();
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
// if add hidden column, must set enableAddHiddenColumn true
private boolean enableAddHiddenColumn = false;


public ColumnDef(String name, TypeDef typeDef) {
Expand Down Expand Up @@ -262,57 +264,83 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, AggregateType aggr
this.visible = visible;
}

public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
this.enableAddHiddenColumn = enableAddHiddenColumn;
}

public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType nullableType, String comment,
Optional<GeneratedColumnInfo> generatedColumnInfo) {
this(name, typeDef, isKey, null, nullableType, -1, DefaultValue.NOT_SET,
comment, true, generatedColumnInfo);
}

public static ColumnDef newDeleteSignColumnDef() {
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null,
ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"),
"doris delete flag hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) {
return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false,
aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new DefaultValue(true, "0"),
"doris delete flag hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newSequenceColumnDef(Type type) {
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1,
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
false, null, ColumnNullableType.NULLABLE, -1,
DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) {
return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE,
ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type),
false, aggregateType, ColumnNullableType.NULLABLE,
-1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false,
Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) {
return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType,
ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING),
false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""),
"doris row store hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newVersionColumnDef() {
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null,
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
"doris version hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newVersionColumnDef(AggregateType aggregateType) {
return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType,
ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT),
false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"),
"doris version hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public static ColumnDef newSkipBitmapColumnDef(AggregateType aggregateType) {
return new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP), false, aggregateType,
ColumnDef columnDef = new ColumnDef(Column.SKIP_BITMAP_COL, TypeDef.create(PrimitiveType.BITMAP),
false, aggregateType,
ColumnNullableType.NOT_NULLABLE, -1, DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE,
"doris skip bitmap hidden column", false, Optional.empty());
columnDef.setEnableAddHiddenColumn(true);
return columnDef;
}

public boolean isAllowNull() {
Expand Down Expand Up @@ -375,7 +403,13 @@ public void analyze(boolean isOlap) throws AnalysisException {
if (name == null || typeDef == null) {
throw new AnalysisException("No column name or column type in column definition.");
}
FeNameFormat.checkColumnName(name);

if (enableAddHiddenColumn) {
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
} else {
FeNameFormat.checkColumnName(name);
}

FeNameFormat.checkColumnCommentLength(comment);

typeDef.analyze(null);
Expand Down
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
*/
public class Column implements GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(Column.class);
public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_";
// NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__";
public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__";
Expand Down
24 changes: 18 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.alter.SchemaChangeHandler;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.ResourceTypeEnum;
import org.apache.doris.catalog.Column;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.mysql.privilege.Role;
import org.apache.doris.mysql.privilege.RoleManager;
Expand Down Expand Up @@ -93,18 +94,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep
}

public static void checkColumnName(String columnName) throws AnalysisException {
// if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn`
checkColumnNameBypassHiddenColumn(columnName);
checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX);
}

public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException {
if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, getColumnNameRegex());
}
if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME,
columnName, getColumnNameRegex());
checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX);
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX);
checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX);
}

private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException {
int prefixLength = prefix.length();
if (columnName.length() < prefixLength) {
return;
}
if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
|| columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) {
throw new AnalysisException(
"Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'");
"Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'");
}
}

Expand Down
78 changes: 51 additions & 27 deletions ...re/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ public class ColumnDefinition {
private int clusterKeyId = -1;
private Optional<GeneratedColumnDesc> generatedColumnDesc = Optional.empty();
private Set<String> generatedColumnsThatReferToThis = new HashSet<>();
// if add hidden column, must set enableAddHiddenColumn true
private boolean enableAddHiddenColumn = false;

public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable,
Optional<DefaultValue> defaultValue, String comment) {
Expand Down Expand Up @@ -232,7 +234,14 @@ private void checkKeyColumnType(boolean isOlap) {
public void validate(boolean isOlap, Set<String> keysSet, Set<String> clusterKeySet, boolean isEnableMergeOnWrite,
KeysType keysType) {
try {
FeNameFormat.checkColumnName(name);
// if enableAddHiddenColumn is true, can add hidden column.
// So does not check if the column name starts with __DORIS_
if (enableAddHiddenColumn) {
FeNameFormat.checkColumnNameBypassHiddenColumn(name);
} else {
FeNameFormat.checkColumnName(name);
}

FeNameFormat.checkColumnCommentLength(comment);
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e);
Expand Down Expand Up @@ -689,43 +698,54 @@ public Column translateToCatalogStyle() {
return column;
}

// hidden column
public static ColumnDefinition newDeleteSignColumnDefinition() {
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false,
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
}

/**
* add hidden column
*/
public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) {
return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false,
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false);
}
ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false,
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris delete flag hidden column", false);
columnDefinition.setEnableAddHiddenColumn(true);

public static ColumnDefinition newSequenceColumnDefinition(DataType type) {
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true,
Optional.empty(), "sequence column hidden column", false);
}

public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) {
return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true,
Optional.empty(), "sequence column hidden column", false);
return columnDefinition;
}

/**
* add hidden column
*/
public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) {
return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false,
Optional.of(new DefaultValue("")), "doris row store hidden column", false);
ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false,
aggregateType, false, Optional.of(new DefaultValue("")),
"doris row store hidden column", false);
columnDefinition.setEnableAddHiddenColumn(true);

return columnDefinition;
}

/**
* add hidden column
*/
public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) {
return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false,
Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false);
ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false,
aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)),
"doris version hidden column", false);
columnDefinition.setEnableAddHiddenColumn(true);

return columnDefinition;
}

// used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
// becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
// and then set the default value of that column to bitmap_empty()
/**
* used in CreateTableInfo.validate(), specify the default value as DefaultValue.NULL_DEFAULT_VALUE
* becasue ColumnDefinition.validate() will check that bitmap type column don't set default value
* and then set the default value of that column to bitmap_empty()
*/
public static ColumnDefinition newSkipBitmapColumnDef(AggregateType aggregateType) {
return new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false, aggregateType, false,
Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE), "doris skip bitmap hidden column", false);
ColumnDefinition columnDefinition = new ColumnDefinition(Column.SKIP_BITMAP_COL, BitmapType.INSTANCE, false,
aggregateType, false, Optional.of(DefaultValue.BITMAP_EMPTY_DEFAULT_VALUE),
"doris skip bitmap hidden column", false);
columnDefinition.setEnableAddHiddenColumn(true);

return columnDefinition;
}

public Optional<GeneratedColumnDesc> getGeneratedColumnDesc() {
Expand All @@ -740,6 +760,10 @@ public void addGeneratedColumnsThatReferToThis(List<String> list) {
generatedColumnsThatReferToThis.addAll(list);
}

public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) {
this.enableAddHiddenColumn = enableAddHiddenColumn;
}

private void validateGeneratedColumnInfo() {
// for generated column
if (generatedColumnDesc.isPresent()) {
Expand Down