Skip to content

[opt](iceberg) support create branch/tag for iceberg #51727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 9 commits into from
Jun 24, 2025
Merged

[opt](iceberg) support create branch/tag for iceberg #51727

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5a91659
fix
wuwenchi Jun 11, 2025
a9b3598
1
morningman Jun 23, 2025
33ad233
add test
morningman Jun 24, 2025
0b6e963
java odc
morningman Jun 24, 2025
0ebe92d
java odc
morningman Jun 24, 2025
04dcab3
fix test
morningman Jun 24, 2025
02d730c
fix error
morningman Jun 24, 2025
51d38d1
rename
morningman Jun 24, 2025
153806a
add ut
morningman Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ...thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run15.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
use demo.test_db;
CREATE TABLE tmp_schema_change_branch (id bigint, data string, col float);
INSERT INTO tmp_schema_change_branch VALUES (1, 'a', 1.0), (2, 'b', 2.0), (3, 'c', 3.0);
ALTER TABLE tmp_schema_change_branch CREATE BRANCH test_branch;
ALTER TABLE tmp_schema_change_branch CREATE TAG test_tag;
ALTER TABLE tmp_schema_change_branch DROP COLUMN col;
ALTER TABLE tmp_schema_change_branch ADD COLUMN new_col date;
INSERT INTO tmp_schema_change_branch VALUES (4, 'd', date('2024-04-04')), (5, 'e', date('2024-05-05'));
2 changes: 1 addition & 1 deletion docker/thirdparties/run-thirdparties-docker.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -739,7 +739,7 @@ if [[ "${RUN_SPARK}" -eq 1 ]]; then
fi

if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
start_iceberg > start_icerberg.log 2>&1 &
start_iceberg > start_iceberg.log 2>&1 &
pids["iceberg"]=$!
fi

Expand Down
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ BITOR: 'BITOR';
BITXOR: 'BITXOR';
BLOB: 'BLOB';
BOOLEAN: 'BOOLEAN';
BRANCH: 'BRANCH';
BRIEF: 'BRIEF';
BROKER: 'BROKER';
BUCKETS: 'BUCKETS';
Expand Down Expand Up @@ -186,6 +187,7 @@ DATEV2: 'DATEV2';
DATETIMEV1: 'DATETIMEV1';
DATEV1: 'DATEV1';
DAY: 'DAY';
DAYS: 'DAYS';
DECIMAL: 'DECIMAL';
DECIMALV2: 'DECIMALV2';
DECIMALV3: 'DECIMALV3';
Expand Down Expand Up @@ -281,6 +283,7 @@ HLL_UNION: 'HLL_UNION';
HOSTNAME: 'HOSTNAME';
HOTSPOT: 'HOTSPOT';
HOUR: 'HOUR';
HOURS: 'HOURS';
HUB: 'HUB';
IDENTIFIED: 'IDENTIFIED';
IF: 'IF';
Expand Down Expand Up @@ -359,6 +362,7 @@ MIGRATIONS: 'MIGRATIONS';
MIN: 'MIN';
MINUS: 'MINUS';
MINUTE: 'MINUTE';
MINUTES: 'MINUTES';
MODIFY: 'MODIFY';
MONTH: 'MONTH';
MTMV: 'MTMV';
Expand Down Expand Up @@ -454,6 +458,8 @@ RESOURCES: 'RESOURCES';
RESTORE: 'RESTORE';
RESTRICTIVE: 'RESTRICTIVE';
RESUME: 'RESUME';
RETAIN: 'RETAIN';
RETENTION: 'RETENTION';
RETURNS: 'RETURNS';
REVOKE: 'REVOKE';
REWRITTEN: 'REWRITTEN';
Expand Down Expand Up @@ -487,6 +493,7 @@ SIGNED: 'SIGNED';
SKEW: 'SKEW';
SMALLINT: 'SMALLINT';
SNAPSHOT: 'SNAPSHOT';
SNAPSHOTS: 'SNAPSHOTS';
SONAME: 'SONAME';
SPLIT: 'SPLIT';
SQL: 'SQL';
Expand All @@ -513,6 +520,7 @@ TABLES: 'TABLES';
TABLESAMPLE: 'TABLESAMPLE';
TABLET: 'TABLET';
TABLETS: 'TABLETS';
TAG: 'TAG';
TASK: 'TASK';
TASKS: 'TASKS';
TEMPORARY: 'TEMPORARY';
Expand Down
46 changes: 46 additions & 0 deletions fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,44 @@ alterTableClause
| ADD TEMPORARY? PARTITIONS
FROM from=partitionValueList TO to=partitionValueList
INTERVAL INTEGER_VALUE unit=identifier? properties=propertyClause? #alterMultiPartitionClause
| createOrReplaceTagClause #createOrReplaceTagClauses
| createOrReplaceBranchClause #createOrReplaceBranchClauses
;

createOrReplaceTagClause
: CREATE TAG (IF NOT EXISTS)? name=identifier ops=tagOptions
| (CREATE OR)? REPLACE TAG name=identifier ops=tagOptions
;

createOrReplaceBranchClause
: CREATE BRANCH (IF NOT EXISTS)? name=identifier ops=branchOptions
| (CREATE OR)? REPLACE BRANCH name=identifier ops=branchOptions
;

tagOptions
: (AS OF VERSION version=INTEGER_VALUE)? (retainTime)?
;

branchOptions
: (AS OF VERSION version=INTEGER_VALUE)? (retainTime)? (retentionSnapshot)?
;

retainTime
: RETAIN timeValueWithUnit
;

retentionSnapshot
: WITH SNAPSHOT RETENTION minSnapshotsToKeep
| WITH SNAPSHOT RETENTION timeValueWithUnit
| WITH SNAPSHOT RETENTION minSnapshotsToKeep timeValueWithUnit
;

minSnapshotsToKeep
: value=INTEGER_VALUE SNAPSHOTS
;

timeValueWithUnit
: timeValue=INTEGER_VALUE timeUnit=(DAYS | HOURS | MINUTES)
;

columnPosition
Expand Down Expand Up @@ -1785,6 +1823,7 @@ nonReserved
| BITXOR
| BLOB
| BOOLEAN
| BRANCH
| BRIEF
| BROKER
| BUCKETS
Expand Down Expand Up @@ -1839,6 +1878,7 @@ nonReserved
| DATEV1
| DATEV2
| DAY
| DAYS
| DECIMAL
| DECIMALV2
| DECIMALV3
Expand Down Expand Up @@ -1896,6 +1936,7 @@ nonReserved
| HOSTNAME
| HOTSPOT
| HOUR
| HOURS
| HUB
| IDENTIFIED
| IGNORE
Expand Down Expand Up @@ -1945,6 +1986,7 @@ nonReserved
| MIGRATIONS
| MIN
| MINUTE
| MINUTES
| MODIFY
| MONTH
| MTMV
Expand Down Expand Up @@ -2011,6 +2053,8 @@ nonReserved
| RESTORE
| RESTRICTIVE
| RESUME
| RETAIN
| RETENTION
| RETURNS
| REWRITTEN
| RIGHT_BRACE
Expand All @@ -2031,6 +2075,7 @@ nonReserved
| SHAPE
| SKEW
| SNAPSHOT
| SNAPSHOTS
| SONAME
| SPLIT
| SQL
Expand All @@ -2048,6 +2093,7 @@ nonReserved
| STRUCT
| SUM
| TABLES
| TAG
| TASK
| TASKS
| TEMPORARY
Expand Down
24 changes: 23 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.apache.doris.analysis.AlterViewStmt;
import org.apache.doris.analysis.ColumnRenameClause;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.CreateOrReplaceBranchClause;
import org.apache.doris.analysis.CreateOrReplaceTagClause;
import org.apache.doris.analysis.DropMaterializedViewStmt;
import org.apache.doris.analysis.DropPartitionClause;
import org.apache.doris.analysis.DropPartitionFromIndexClause;
Expand Down Expand Up @@ -378,6 +380,26 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List<AlterCl
Env.getCurrentEnv().getEditLog().logModifyTableProperties(info);
}

private void processAlterTableForExternalTable(
ExternalTable table,
List<AlterClause> alterClauses) throws UserException {
for (AlterClause alterClause : alterClauses) {
if (alterClause instanceof ModifyTablePropertiesClause) {
setExternalTableAutoAnalyzePolicy(table, alterClauses);
} else if (alterClause instanceof CreateOrReplaceBranchClause) {
table.getCatalog().createOrReplaceBranch(
table.getDbName(), table.getName(),
((CreateOrReplaceBranchClause) alterClause).getBranchInfo());
} else if (alterClause instanceof CreateOrReplaceTagClause) {
table.getCatalog().createOrReplaceTag(
table.getDbName(), table.getName(),
((CreateOrReplaceTagClause) alterClause).getTagInfo());
} else {
throw new UserException("Invalid alter operations for external table: " + alterClauses);
}
}
}

private boolean needChangeMTMVState(List<AlterClause> alterClauses) {
for (AlterClause alterClause : alterClauses) {
if (alterClause.needChangeMTMVState()) {
Expand Down Expand Up @@ -673,7 +695,7 @@ public void processAlterTable(AlterTableCommand command) throws UserException {
case HUDI_EXTERNAL_TABLE:
case TRINO_CONNECTOR_EXTERNAL_TABLE:
alterClauses.addAll(command.getOps());
setExternalTableAutoAnalyzePolicy((ExternalTable) tableIf, alterClauses);
processAlterTableForExternalTable((ExternalTable) tableIf, alterClauses);
return;
default:
throw new DdlException("Do not support alter "
Expand Down
2 changes: 2 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ public enum AlterOpType {
MODIFY_TABLE_COMMENT,
MODIFY_COLUMN_COMMENT,
MODIFY_ENGINE,
ALTER_BRANCH,
ALTER_TAG,
INVALID_OP; // INVALID_OP must be the last one

// true means 2 operations have no conflict.
Expand Down
55 changes: 55 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceBranchClause.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.analysis;

import org.apache.doris.alter.AlterOpType;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;

public class CreateOrReplaceBranchClause extends AlterTableClause {
private final CreateOrReplaceBranchInfo branchInfo;

public CreateOrReplaceBranchClause(CreateOrReplaceBranchInfo branchInfo) {
super(AlterOpType.ALTER_BRANCH);
this.branchInfo = branchInfo;
}

@Override
public boolean allowOpMTMV() {
return false;
}

@Override
public boolean needChangeMTMVState() {
return false;
}

@Override
public void analyze(Analyzer analyzer) throws UserException {

}

@Override
public String toSql() {
return branchInfo.toSql();
}

public CreateOrReplaceBranchInfo getBranchInfo() {
return branchInfo;
}
}
55 changes: 55 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceTagClause.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.analysis;

import org.apache.doris.alter.AlterOpType;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;

public class CreateOrReplaceTagClause extends AlterTableClause {
private final CreateOrReplaceTagInfo tagInfo;

public CreateOrReplaceTagClause(CreateOrReplaceTagInfo tagInfo) {
super(AlterOpType.ALTER_TAG);
this.tagInfo = tagInfo;
}

@Override
public boolean allowOpMTMV() {
return false;
}

@Override
public boolean needChangeMTMVState() {
return false;
}

@Override
public void analyze(Analyzer analyzer) throws UserException {

}

@Override
public String toSql() {
return tagInfo.toSql();
}

public CreateOrReplaceTagInfo getTagInfo() {
return tagInfo;
}
}
17 changes: 17 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand;
import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand;
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;

import com.google.common.base.Strings;
import com.google.common.collect.Lists;
Expand Down Expand Up @@ -220,4 +222,19 @@ default String fromRemoteDatabaseName(String remoteDatabaseName) {
default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) {
return remoteTableName;
}

// Create or replace branch operations, overridden by subclass if necessary
default void createOrReplaceBranch(String db, String tbl, CreateOrReplaceBranchInfo branchInfo)
throws UserException {
throw new UserException("Not support create or replace branch operation");
}

// Create or replace tag operation, overridden by subclass if necessary
default void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo tagInfo) throws UserException {
throw new UserException("Not support create or replace tag operation");
}

default void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) {

}
}
Loading