diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run15.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run15.sql new file mode 100644 index 00000000000000..e65c777e54571f --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run15.sql @@ -0,0 +1,8 @@ +use demo.test_db; +CREATE TABLE tmp_schema_change_branch (id bigint, data string, col float); +INSERT INTO tmp_schema_change_branch VALUES (1, 'a', 1.0), (2, 'b', 2.0), (3, 'c', 3.0); +ALTER TABLE tmp_schema_change_branch CREATE BRANCH test_branch; +ALTER TABLE tmp_schema_change_branch CREATE TAG test_tag; +ALTER TABLE tmp_schema_change_branch DROP COLUMN col; +ALTER TABLE tmp_schema_change_branch ADD COLUMN new_col date; +INSERT INTO tmp_schema_change_branch VALUES (4, 'd', date('2024-04-04')), (5, 'e', date('2024-05-05')); diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index fd45932d883fd2..625de65d9a45e7 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -739,7 +739,7 @@ if [[ "${RUN_SPARK}" -eq 1 ]]; then fi if [[ "${RUN_ICEBERG}" -eq 1 ]]; then - start_iceberg > start_icerberg.log 2>&1 & + start_iceberg > start_iceberg.log 2>&1 & pids["iceberg"]=$! fi diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index adf13f0b8f7017..c9f0fdf976b297 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -119,6 +119,7 @@ BITOR: 'BITOR'; BITXOR: 'BITXOR'; BLOB: 'BLOB'; BOOLEAN: 'BOOLEAN'; +BRANCH: 'BRANCH'; BRIEF: 'BRIEF'; BROKER: 'BROKER'; BUCKETS: 'BUCKETS'; @@ -186,6 +187,7 @@ DATEV2: 'DATEV2'; DATETIMEV1: 'DATETIMEV1'; DATEV1: 'DATEV1'; DAY: 'DAY'; +DAYS: 'DAYS'; DECIMAL: 'DECIMAL'; DECIMALV2: 'DECIMALV2'; DECIMALV3: 'DECIMALV3'; @@ -281,6 +283,7 @@ HLL_UNION: 'HLL_UNION'; HOSTNAME: 'HOSTNAME'; HOTSPOT: 'HOTSPOT'; HOUR: 'HOUR'; +HOURS: 'HOURS'; HUB: 'HUB'; IDENTIFIED: 'IDENTIFIED'; IF: 'IF'; @@ -359,6 +362,7 @@ MIGRATIONS: 'MIGRATIONS'; MIN: 'MIN'; MINUS: 'MINUS'; MINUTE: 'MINUTE'; +MINUTES: 'MINUTES'; MODIFY: 'MODIFY'; MONTH: 'MONTH'; MTMV: 'MTMV'; @@ -454,6 +458,8 @@ RESOURCES: 'RESOURCES'; RESTORE: 'RESTORE'; RESTRICTIVE: 'RESTRICTIVE'; RESUME: 'RESUME'; +RETAIN: 'RETAIN'; +RETENTION: 'RETENTION'; RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; REWRITTEN: 'REWRITTEN'; @@ -487,6 +493,7 @@ SIGNED: 'SIGNED'; SKEW: 'SKEW'; SMALLINT: 'SMALLINT'; SNAPSHOT: 'SNAPSHOT'; +SNAPSHOTS: 'SNAPSHOTS'; SONAME: 'SONAME'; SPLIT: 'SPLIT'; SQL: 'SQL'; @@ -513,6 +520,7 @@ TABLES: 'TABLES'; TABLESAMPLE: 'TABLESAMPLE'; TABLET: 'TABLET'; TABLETS: 'TABLETS'; +TAG: 'TAG'; TASK: 'TASK'; TASKS: 'TASKS'; TEMPORARY: 'TEMPORARY'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 89f86bc9437650..7d0a50b8acce96 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -722,6 +722,44 @@ alterTableClause | ADD TEMPORARY? PARTITIONS FROM from=partitionValueList TO to=partitionValueList INTERVAL INTEGER_VALUE unit=identifier? properties=propertyClause? #alterMultiPartitionClause + | createOrReplaceTagClause #createOrReplaceTagClauses + | createOrReplaceBranchClause #createOrReplaceBranchClauses + ; + +createOrReplaceTagClause + : CREATE TAG (IF NOT EXISTS)? name=identifier ops=tagOptions + | (CREATE OR)? REPLACE TAG name=identifier ops=tagOptions + ; + +createOrReplaceBranchClause + : CREATE BRANCH (IF NOT EXISTS)? name=identifier ops=branchOptions + | (CREATE OR)? REPLACE BRANCH name=identifier ops=branchOptions + ; + +tagOptions + : (AS OF VERSION version=INTEGER_VALUE)? (retainTime)? + ; + +branchOptions + : (AS OF VERSION version=INTEGER_VALUE)? (retainTime)? (retentionSnapshot)? + ; + +retainTime + : RETAIN timeValueWithUnit + ; + +retentionSnapshot + : WITH SNAPSHOT RETENTION minSnapshotsToKeep + | WITH SNAPSHOT RETENTION timeValueWithUnit + | WITH SNAPSHOT RETENTION minSnapshotsToKeep timeValueWithUnit + ; + +minSnapshotsToKeep + : value=INTEGER_VALUE SNAPSHOTS + ; + +timeValueWithUnit + : timeValue=INTEGER_VALUE timeUnit=(DAYS | HOURS | MINUTES) ; columnPosition @@ -1785,6 +1823,7 @@ nonReserved | BITXOR | BLOB | BOOLEAN + | BRANCH | BRIEF | BROKER | BUCKETS @@ -1839,6 +1878,7 @@ nonReserved | DATEV1 | DATEV2 | DAY + | DAYS | DECIMAL | DECIMALV2 | DECIMALV3 @@ -1896,6 +1936,7 @@ nonReserved | HOSTNAME | HOTSPOT | HOUR + | HOURS | HUB | IDENTIFIED | IGNORE @@ -1945,6 +1986,7 @@ nonReserved | MIGRATIONS | MIN | MINUTE + | MINUTES | MODIFY | MONTH | MTMV @@ -2011,6 +2053,8 @@ nonReserved | RESTORE | RESTRICTIVE | RESUME + | RETAIN + | RETENTION | RETURNS | REWRITTEN | RIGHT_BRACE @@ -2031,6 +2075,7 @@ nonReserved | SHAPE | SKEW | SNAPSHOT + | SNAPSHOTS | SONAME | SPLIT | SQL @@ -2048,6 +2093,7 @@ nonReserved | STRUCT | SUM | TABLES + | TAG | TASK | TASKS | TEMPORARY diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index a5f541db9ef36d..97446fef69c24a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -25,6 +25,8 @@ import org.apache.doris.analysis.AlterViewStmt; import org.apache.doris.analysis.ColumnRenameClause; import org.apache.doris.analysis.CreateMaterializedViewStmt; +import org.apache.doris.analysis.CreateOrReplaceBranchClause; +import org.apache.doris.analysis.CreateOrReplaceTagClause; import org.apache.doris.analysis.DropMaterializedViewStmt; import org.apache.doris.analysis.DropPartitionClause; import org.apache.doris.analysis.DropPartitionFromIndexClause; @@ -378,6 +380,26 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List alterClauses) throws UserException { + for (AlterClause alterClause : alterClauses) { + if (alterClause instanceof ModifyTablePropertiesClause) { + setExternalTableAutoAnalyzePolicy(table, alterClauses); + } else if (alterClause instanceof CreateOrReplaceBranchClause) { + table.getCatalog().createOrReplaceBranch( + table.getDbName(), table.getName(), + ((CreateOrReplaceBranchClause) alterClause).getBranchInfo()); + } else if (alterClause instanceof CreateOrReplaceTagClause) { + table.getCatalog().createOrReplaceTag( + table.getDbName(), table.getName(), + ((CreateOrReplaceTagClause) alterClause).getTagInfo()); + } else { + throw new UserException("Invalid alter operations for external table: " + alterClauses); + } + } + } + private boolean needChangeMTMVState(List alterClauses) { for (AlterClause alterClause : alterClauses) { if (alterClause.needChangeMTMVState()) { @@ -673,7 +695,7 @@ public void processAlterTable(AlterTableCommand command) throws UserException { case HUDI_EXTERNAL_TABLE: case TRINO_CONNECTOR_EXTERNAL_TABLE: alterClauses.addAll(command.getOps()); - setExternalTableAutoAnalyzePolicy((ExternalTable) tableIf, alterClauses); + processAlterTableForExternalTable((ExternalTable) tableIf, alterClauses); return; default: throw new DdlException("Do not support alter " diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java index 818ceda2ceb845..06777976ff6020 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java @@ -42,6 +42,8 @@ public enum AlterOpType { MODIFY_TABLE_COMMENT, MODIFY_COLUMN_COMMENT, MODIFY_ENGINE, + ALTER_BRANCH, + ALTER_TAG, INVALID_OP; // INVALID_OP must be the last one // true means 2 operations have no conflict. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceBranchClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceBranchClause.java new file mode 100644 index 00000000000000..29700cb2ab0fe4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceBranchClause.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; + +public class CreateOrReplaceBranchClause extends AlterTableClause { + private final CreateOrReplaceBranchInfo branchInfo; + + public CreateOrReplaceBranchClause(CreateOrReplaceBranchInfo branchInfo) { + super(AlterOpType.ALTER_BRANCH); + this.branchInfo = branchInfo; + } + + @Override + public boolean allowOpMTMV() { + return false; + } + + @Override + public boolean needChangeMTMVState() { + return false; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + + } + + @Override + public String toSql() { + return branchInfo.toSql(); + } + + public CreateOrReplaceBranchInfo getBranchInfo() { + return branchInfo; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceTagClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceTagClause.java new file mode 100644 index 00000000000000..048fbb127c7827 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateOrReplaceTagClause.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; + +public class CreateOrReplaceTagClause extends AlterTableClause { + private final CreateOrReplaceTagInfo tagInfo; + + public CreateOrReplaceTagClause(CreateOrReplaceTagInfo tagInfo) { + super(AlterOpType.ALTER_TAG); + this.tagInfo = tagInfo; + } + + @Override + public boolean allowOpMTMV() { + return false; + } + + @Override + public boolean needChangeMTMVState() { + return false; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + + } + + @Override + public String toSql() { + return tagInfo.toSql(); + } + + public CreateOrReplaceTagInfo getTagInfo() { + return tagInfo; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index de1ce152110ce5..c85a85c96c0589 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -33,6 +33,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -220,4 +222,19 @@ default String fromRemoteDatabaseName(String remoteDatabaseName) { default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { return remoteTableName; } + + // Create or replace branch operations, overridden by subclass if necessary + default void createOrReplaceBranch(String db, String tbl, CreateOrReplaceBranchInfo branchInfo) + throws UserException { + throw new UserException("Not support create or replace branch operation"); + } + + // Create or replace tag operation, overridden by subclass if necessary + default void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo tagInfo) throws UserException { + throw new UserException("Not support create or replace tag operation"); + } + + default void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a00a6c65035ed2..16dca6cfdfead5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -59,10 +59,13 @@ import org.apache.doris.fs.remote.dfs.DFSFileSystem; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; import org.apache.doris.persist.CreateDbInfo; import org.apache.doris.persist.CreateTableInfo; import org.apache.doris.persist.DropDbInfo; import org.apache.doris.persist.DropInfo; +import org.apache.doris.persist.TableBranchOrTagInfo; import org.apache.doris.persist.TruncateTableInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.qe.ConnectContext; @@ -1334,4 +1337,47 @@ public boolean viewExists(String dbName, String viewName) { throw new UnsupportedOperationException("View is not supported."); } + @Override + public void createOrReplaceBranch(String db, String tbl, CreateOrReplaceBranchInfo branchInfo) + throws UserException { + makeSureInitialized(); + if (metadataOps == null) { + throw new DdlException("branching operation is not supported for catalog: " + getName()); + } + try { + metadataOps.createOrReplaceBranch(db, tbl, branchInfo); + TableBranchOrTagInfo info = new TableBranchOrTagInfo(getName(), db, tbl); + Env.getCurrentEnv().getEditLog().logBranchOrTag(info); + } catch (Exception e) { + LOG.warn("Failed to create or replace branch for table {}.{} in catalog {}", + db, tbl, getName(), e); + throw e; + } + } + + @Override + public void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo tagInfo) + throws UserException { + makeSureInitialized(); + if (metadataOps == null) { + throw new DdlException("Tagging operation is not supported for catalog: " + getName()); + } + try { + metadataOps.createOrReplaceTag(db, tbl, tagInfo); + TableBranchOrTagInfo info = new TableBranchOrTagInfo(getName(), db, tbl); + Env.getCurrentEnv().getEditLog().logBranchOrTag(info); + } catch (Exception e) { + LOG.warn("Failed to create or replace tag for table {}.{} in catalog {}", + db, tbl, getName(), e); + throw e; + } + } + + @Override + public void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) { + if (metadataOps != null) { + metadataOps.afterCreateOrReplaceBranchOrTag(dbName, tblName); + } + } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 8e999df2aef03b..d6b41544201c45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -135,7 +135,7 @@ public boolean isView() { return false; } - protected void makeSureInitialized() { + protected synchronized void makeSureInitialized() { try { // getDbOrAnalysisException will call makeSureInitialized in ExternalCatalog. ExternalDatabase db = catalog.getDbOrAnalysisException(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index d1e635f50f4629..4faff646d4b930 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -40,6 +40,8 @@ import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.property.constants.HMSProperties; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; @@ -397,6 +399,18 @@ public void afterTruncateTable(String dbName, String tblName) { } } + @Override + public void createOrReplaceBranchImpl(String dbName, String tblName, CreateOrReplaceBranchInfo branchInfo) + throws UserException { + throw new UserException("Not support create or replace branch in hive catalog."); + } + + @Override + public void createOrReplaceTagImpl(String dbName, String tblName, CreateOrReplaceTagInfo tagInfo) + throws UserException { + throw new UserException("Not support create or replace tag in hive catalog."); + } + @Override public List listTableNames(String dbName) { return client.getAllTables(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 0589f95b55dc8b..904f1f0986d0df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -32,11 +32,18 @@ import org.apache.doris.datasource.DorisTypeVisitor; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; +import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TagOptions; +import org.apache.iceberg.ManageSnapshots; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -385,6 +392,117 @@ public void truncateTableImpl(String dbName, String tblName, List partit throw new UnsupportedOperationException("Truncate Iceberg table is not supported."); } + @Override + public void createOrReplaceBranchImpl(String dbName, String tblName, CreateOrReplaceBranchInfo branchInfo) + throws UserException { + Table icebergTable = IcebergUtils.getIcebergTable(dorisCatalog, dbName, tblName); + BranchOptions branchOptions = branchInfo.getBranchOptions(); + + Long snapshotId = branchOptions.getSnapshotId() + .orElse( + // use current snapshot + Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); + + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + String branchName = branchInfo.getBranchName(); + boolean refExists = null != icebergTable.refs().get(branchName); + boolean create = branchInfo.getCreate(); + boolean replace = branchInfo.getReplace(); + boolean ifNotExists = branchInfo.getIfNotExists(); + + Runnable safeCreateBranch = () -> { + if (snapshotId == null) { + manageSnapshots.createBranch(branchName); + } else { + manageSnapshots.createBranch(branchName, snapshotId); + } + }; + + if (create && replace && !refExists) { + safeCreateBranch.run(); + } else if (replace) { + if (snapshotId == null) { + // Cannot perform a replace operation on an empty table + throw new UserException( + "Cannot complete replace branch operation on " + icebergTable.name() + + " , main has no snapshot"); + } + manageSnapshots.replaceBranch(branchName, snapshotId); + } else { + if (refExists && ifNotExists) { + return; + } + safeCreateBranch.run(); + } + + branchOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxSnapshotAgeMs(branchName, n)); + branchOptions.getNumSnapshots().ifPresent(n -> manageSnapshots.setMinSnapshotsToKeep(branchName, n)); + branchOptions.getRetention().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(branchName, n)); + + try { + preExecutionAuthenticator.execute(() -> manageSnapshots.commit()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create or replace branch: " + branchName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } + } + + @Override + public void afterCreateOrReplaceBranchOrTag(String dbName, String tblName) { + ExternalDatabase db = dorisCatalog.getDbNullable(dbName); + if (db != null) { + ExternalTable tbl = db.getTableNullable(tblName); + if (tbl != null) { + tbl.unsetObjectCreated(); + } + } + } + + @Override + public void createOrReplaceTagImpl(String dbName, String tblName, CreateOrReplaceTagInfo tagInfo) + throws UserException { + Table icebergTable = IcebergUtils.getIcebergTable(dorisCatalog, dbName, tblName); + TagOptions tagOptions = tagInfo.getTagOptions(); + Long snapshotId = tagOptions.getSnapshotId() + .orElse( + // use current snapshot + Optional.ofNullable(icebergTable.currentSnapshot()).map(Snapshot::snapshotId).orElse(null)); + + if (snapshotId == null) { + // Creating tag for empty tables is not allowed + throw new UserException( + "Cannot complete replace branch operation on " + icebergTable.name() + " , main has no snapshot"); + } + + String tagName = tagInfo.getTagName(); + boolean create = tagInfo.getCreate(); + boolean replace = tagInfo.getReplace(); + boolean ifNotExists = tagInfo.getIfNotExists(); + boolean refExists = null != icebergTable.refs().get(tagName); + + ManageSnapshots manageSnapshots = icebergTable.manageSnapshots(); + if (create && replace && !refExists) { + manageSnapshots.createTag(tagName, snapshotId); + } else if (replace) { + manageSnapshots.replaceTag(tagName, snapshotId); + } else { + if (refExists && ifNotExists) { + return; + } + manageSnapshots.createTag(tagName, snapshotId); + } + + tagOptions.getRetain().ifPresent(n -> manageSnapshots.setMaxRefAgeMs(tagName, n)); + try { + preExecutionAuthenticator.execute(() -> manageSnapshots.commit()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to create or replace tag: " + tagName + " in table: " + icebergTable.name() + + ", error message is: " + e.getMessage(), e); + } + } + public PreExecutionAuthenticator getPreExecutionAuthenticator() { return preExecutionAuthenticator; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index 0ac498928ad721..cb69b041f3329b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -24,6 +24,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; import org.apache.iceberg.view.View; @@ -130,6 +132,7 @@ default void afterDropTable(String dbName, String tblName) { } /** + * truncate table in external metastore * * @param dbName * @param tblName @@ -145,6 +148,43 @@ default void truncateTable(String dbName, String tblName, List partition default void afterTruncateTable(String dbName, String tblName) { } + /** + * create or replace branch in external metastore + * + * @param dbName + * @param tblName + * @param branchInfo + * @throws UserException + */ + default void createOrReplaceBranch(String dbName, String tblName, CreateOrReplaceBranchInfo branchInfo) + throws UserException { + createOrReplaceBranchImpl(dbName, tblName, branchInfo); + afterCreateOrReplaceBranchOrTag(dbName, tblName); + } + + void createOrReplaceBranchImpl(String dbName, String tblName, CreateOrReplaceBranchInfo branchInfo) + throws UserException; + + default void afterCreateOrReplaceBranchOrTag(String dbName, String tblName) { + } + + /** + * create or replace tag in external metastore + * + * @param dbName + * @param tblName + * @param tagInfo + * @throws UserException + */ + default void createOrReplaceTag(String dbName, String tblName, CreateOrReplaceTagInfo tagInfo) + throws UserException { + createOrReplaceTagImpl(dbName, tblName, tagInfo); + afterCreateOrReplaceBranchOrTag(dbName, tblName); + } + + void createOrReplaceTagImpl(String dbName, String tblName, CreateOrReplaceTagInfo tagInfo) + throws UserException; + /** * * @return diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index da7aa445b1509e..f46d0b652d08f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -120,6 +120,7 @@ import org.apache.doris.persist.SetTableStatusOperationLog; import org.apache.doris.persist.TableAddOrDropColumnsInfo; import org.apache.doris.persist.TableAddOrDropInvertedIndicesInfo; +import org.apache.doris.persist.TableBranchOrTagInfo; import org.apache.doris.persist.TableInfo; import org.apache.doris.persist.TablePropertyInfo; import org.apache.doris.persist.TableRenameColumnInfo; @@ -984,6 +985,11 @@ public void readFields(DataInput in) throws IOException { isRead = true; break; } + case OperationType.OP_BRANCH_OR_TAG: { + data = TableBranchOrTagInfo.read(in); + isRead = true; + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}", opCode, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index d708e3e3b69fa3..902f57dc25ff8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -841,6 +841,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterUserInfo; import org.apache.doris.nereids.trees.plans.commands.info.AlterViewInfo; +import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; import org.apache.doris.nereids.trees.plans.commands.info.BuildIndexOp; import org.apache.doris.nereids.trees.plans.commands.info.BulkLoadDataDesc; import org.apache.doris.nereids.trees.plans.commands.info.BulkStorageDesc; @@ -852,6 +853,8 @@ import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp; import org.apache.doris.nereids.trees.plans.commands.info.CreateJobInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchOp; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagOp; import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateRoutineLoadInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; @@ -924,6 +927,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.StepPartition; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TagOptions; import org.apache.doris.nereids.trees.plans.commands.info.WarmUpItem; import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; @@ -1023,6 +1027,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -8387,4 +8392,101 @@ public LogicalPlan visitShowIndexTokenizer(ShowIndexTokenizerContext ctx) { public LogicalPlan visitShowIndexTokenFilter(ShowIndexTokenFilterContext ctx) { return new ShowIndexTokenFilterCommand(); } + + @Override + public AlterTableOp visitCreateOrReplaceBranchClauses(DorisParser.CreateOrReplaceBranchClausesContext ctx) { + return visitCreateOrReplaceBranchClause(ctx.createOrReplaceBranchClause()); + } + + @Override + public CreateOrReplaceBranchOp visitCreateOrReplaceBranchClause( + DorisParser.CreateOrReplaceBranchClauseContext ctx) { + BranchOptions branchOptions = visitBranchOptions(ctx.branchOptions()); + return new CreateOrReplaceBranchOp( + ctx.name.getText(), + ctx.CREATE() != null, + ctx.REPLACE() != null, + ctx.EXISTS() != null, + branchOptions); + } + + @Override + public BranchOptions visitBranchOptions(DorisParser.BranchOptionsContext ctx) { + if (ctx == null) { + return BranchOptions.EMPTY; + } + + Optional snapshotId = Optional.empty(); + if (ctx.version != null) { + snapshotId = Optional.of(Long.parseLong(ctx.version.getText())); + } + + Optional retainTime = Optional.empty(); + if (ctx.retainTime() != null) { + DorisParser.TimeValueWithUnitContext time = ctx.retainTime().timeValueWithUnit(); + if (time != null) { + retainTime = Optional.of(visitTimeValueWithUnit(time)); + } + } + + Optional numSnapshots = Optional.empty(); + Optional retention = Optional.empty(); + if (ctx.retentionSnapshot() != null) { + DorisParser.RetentionSnapshotContext retentionSnapshotContext = ctx.retentionSnapshot(); + if (retentionSnapshotContext.minSnapshotsToKeep() != null) { + numSnapshots = Optional.of( + Integer.parseInt(retentionSnapshotContext.minSnapshotsToKeep().value.getText())); + } + if (retentionSnapshotContext.timeValueWithUnit() != null) { + retention = Optional.of(visitTimeValueWithUnit(retentionSnapshotContext.timeValueWithUnit())); + } + } + return new BranchOptions(snapshotId, retainTime, numSnapshots, retention); + } + + @Override + public AlterTableOp visitCreateOrReplaceTagClauses(DorisParser.CreateOrReplaceTagClausesContext ctx) { + return visitCreateOrReplaceTagClause(ctx.createOrReplaceTagClause()); + } + + @Override + public CreateOrReplaceTagOp visitCreateOrReplaceTagClause(DorisParser.CreateOrReplaceTagClauseContext ctx) { + + TagOptions tagOptions = visitTagOptions(ctx.tagOptions()); + return new CreateOrReplaceTagOp( + ctx.name.getText(), + ctx.CREATE() != null, + ctx.REPLACE() != null, + ctx.EXISTS() != null, + tagOptions); + } + + @Override + public TagOptions visitTagOptions(DorisParser.TagOptionsContext ctx) { + if (ctx == null) { + return TagOptions.EMPTY; + } + + Optional snapshotId = Optional.empty(); + if (ctx.version != null) { + snapshotId = Optional.of(Long.parseLong(ctx.version.getText())); + } + + Optional retainTime = Optional.empty(); + if (ctx.retainTime() != null) { + DorisParser.TimeValueWithUnitContext time = ctx.retainTime().timeValueWithUnit(); + if (time != null) { + retainTime = Optional.of(visitTimeValueWithUnit(time)); + } + } + + return new TagOptions(snapshotId, retainTime); + } + + @Override + public Long visitTimeValueWithUnit(DorisParser.TimeValueWithUnitContext ctx) { + return TimeUnit.valueOf(ctx.timeUnit.getText().toUpperCase()) + .toMillis(Long.parseLong(ctx.timeValue.getText())); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java index bea08f0cf8f3e2..47060bd97d8bc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterTableCommand.java @@ -43,6 +43,8 @@ import org.apache.doris.nereids.trees.plans.commands.info.AddRollupOp; import org.apache.doris.nereids.trees.plans.commands.info.AlterTableOp; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchOp; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagOp; import org.apache.doris.nereids.trees.plans.commands.info.DropColumnOp; import org.apache.doris.nereids.trees.plans.commands.info.DropRollupOp; import org.apache.doris.nereids.trees.plans.commands.info.EnableFeatureOp; @@ -242,7 +244,9 @@ private void checkExternalTableOperationAllow(TableIf table) throws UserExceptio || alterClause instanceof ModifyColumnOp || alterClause instanceof ReorderColumnsOp || alterClause instanceof ModifyEngineOp - || alterClause instanceof ModifyTablePropertiesOp) { + || alterClause instanceof ModifyTablePropertiesOp + || alterClause instanceof CreateOrReplaceBranchOp + || alterClause instanceof CreateOrReplaceTagOp) { alterTableOps.add(alterClause); } else { throw new AnalysisException(table.getType().toString() + " [" + table.getName() + "] " diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BranchOptions.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BranchOptions.java new file mode 100644 index 00000000000000..8b3a8725f759ed --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BranchOptions.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import java.util.Optional; + +/** + * Represents options that can be specified for a branch operation in the Nereids module. + *

+ * This class encapsulates optional parameters that control the behavior of branch operations, + * such as specifying a snapshot ID, retention policy, number of snapshots to keep, and retention period. + */ +public class BranchOptions { + public static final BranchOptions EMPTY = new BranchOptions(Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + private final Optional snapshotId; + // retain time in milliseconds + private final Optional retain; + private final Optional numSnapshots; + private final Optional retention; + + public BranchOptions(Optional snapshotId, + Optional retain, + Optional numSnapshots, + Optional retention) { + this.snapshotId = snapshotId; + this.retain = retain; + this.numSnapshots = numSnapshots; + this.retention = retention; + } + + public Optional getSnapshotId() { + return snapshotId; + } + + public Optional getRetain() { + return retain; + } + + public Optional getNumSnapshots() { + return numSnapshots; + } + + public Optional getRetention() { + return retention; + } + + /** + * Generates the SQL representation of the branch options. + */ + public String toSql() { + StringBuilder sb = new StringBuilder(); + if (snapshotId.isPresent()) { + sb.append(" AS OF VERSION ").append(snapshotId.get()); + } + if (retain.isPresent()) { + // "RETAIN", and convert retain time to MINUTES + sb.append(" RETAIN ").append(retain.get() / 1000 / 60).append(" MINUTES"); + } + if (numSnapshots.isPresent() || retention.isPresent()) { + sb.append(" WITH SNAPSHOT RETENTION"); + if (numSnapshots.isPresent()) { + sb.append(" ").append(numSnapshots.get()).append(" SNAPSHOTS"); + } + if (retention.isPresent()) { + sb.append(" ").append(retention.get() / 1000 / 60).append(" MINUTES"); + } + } + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchInfo.java new file mode 100644 index 00000000000000..0b1dd71757fd3d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchInfo.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +/** + * Represents the information required to create or replace a branch in the Nereids module. + *

+ * This class encapsulates the branch name, operation flags (create, replace, ifNotExists), + * and associated branch options that control the behavior of the branch operation. + */ +public class CreateOrReplaceBranchInfo { + + private final String branchName; + private final BranchOptions branchOptions; + private final Boolean create; + private final Boolean replace; + private final Boolean ifNotExists; + + public CreateOrReplaceBranchInfo(String branchName, + boolean create, + boolean replace, + boolean ifNotExists, + BranchOptions branchOptions) { + this.branchName = branchName; + this.create = create; + this.replace = replace; + this.ifNotExists = ifNotExists; + this.branchOptions = branchOptions; + } + + public String getBranchName() { + return branchName; + } + + public BranchOptions getBranchOptions() { + return branchOptions; + } + + public Boolean getCreate() { + return create; + } + + public Boolean getReplace() { + return replace; + } + + public Boolean getIfNotExists() { + return ifNotExists; + } + + /** + * Generates the SQL representation of the create or replace branch command. + */ + public String toSql() { + StringBuilder sb = new StringBuilder(); + if (create && replace) { + sb.append("CREATE OR REPLACE BRANCH"); + } else if (create) { + sb.append("CREATE BRANCH"); + } else if (replace) { + sb.append("REPLACE BRANCH"); + } + if (ifNotExists) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" ").append(branchName); + if (branchOptions != null) { + sb.append(branchOptions.toSql()); + } + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOp.java new file mode 100644 index 00000000000000..11258470ebc043 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOp.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.AlterTableClause; +import org.apache.doris.analysis.CreateOrReplaceBranchClause; + +import java.util.Map; + +/** + * Represents an operation to create or replace a branch within the Nereids module's plan structure. + *

+ * This class extends {@link AlterTableOp} and encapsulates the logic for handling branch creation or replacement, + * including branch name, operation flags (create, replace, ifNotExists), and associated options defined in + * {@link BranchOptions}. + *

+ * It also provides implementations for required methods such as SQL translation and legacy clause conversion. + */ +public class CreateOrReplaceBranchOp extends AlterTableOp { + + private final CreateOrReplaceBranchInfo branchInfo; + + public CreateOrReplaceBranchOp(String branchName, + boolean create, + boolean replace, + boolean ifNotExists, + BranchOptions branchOptions) { + super(AlterOpType.ALTER_BRANCH); + this.branchInfo = new CreateOrReplaceBranchInfo(branchName, create, replace, ifNotExists, branchOptions); + } + + @Override + public boolean allowOpMTMV() { + return false; + } + + @Override + public boolean needChangeMTMVState() { + return false; + } + + @Override + public String toSql() { + return branchInfo.toSql(); + } + + @Override + public Map getProperties() { + return null; + } + + @Override + public AlterTableClause translateToLegacyAlterClause() { + return new CreateOrReplaceBranchClause(branchInfo); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagInfo.java new file mode 100644 index 00000000000000..e5506946f28e3c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagInfo.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +/** + * Represents the information needed to create or replace a tag in the system. + * + */ +public class CreateOrReplaceTagInfo { + + private final String tagName; + private final TagOptions tagOptions; + private final Boolean create; + private final Boolean replace; + private final Boolean ifNotExists; + + public CreateOrReplaceTagInfo(String tagName, + boolean create, + boolean replace, + boolean ifNotExists, + TagOptions tagOptions) { + this.tagName = tagName; + this.create = create; + this.replace = replace; + this.ifNotExists = ifNotExists; + this.tagOptions = tagOptions; + } + + public String getTagName() { + return tagName; + } + + public TagOptions getTagOptions() { + return tagOptions; + } + + public Boolean getCreate() { + return create; + } + + public Boolean getReplace() { + return replace; + } + + public Boolean getIfNotExists() { + return ifNotExists; + } + + /** + * Generates the SQL representation of the create or replace tag command. + * + * @return SQL string for creating or replacing a tag + */ + public String toSql() { + StringBuilder sb = new StringBuilder(); + if (create && replace) { + sb.append("CREATE OR REPLACE TAG"); + } else if (create) { + sb.append("CREATE TAG"); + } else if (replace) { + sb.append("REPLACE TAG"); + } + if (ifNotExists) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" ").append(tagName); + if (tagOptions != null) { + sb.append(tagOptions.toSql()); + } + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagOp.java new file mode 100644 index 00000000000000..e56ccf5cafe5c0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceTagOp.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.alter.AlterOpType; +import org.apache.doris.analysis.AlterTableClause; +import org.apache.doris.analysis.CreateOrReplaceTagClause; + +import java.util.Map; + +/** + * Operation class representing the creation or replacement of a tag in the system. + * This class extends {@link AlterTableOp} and encapsulates the logic for handling + * create or replace tag operations. + */ +public class CreateOrReplaceTagOp extends AlterTableOp { + + private final CreateOrReplaceTagInfo tagInfo; + + public CreateOrReplaceTagOp(String tagName, + boolean create, + boolean replace, + boolean ifNotExists, + TagOptions tagOptions) { + super(AlterOpType.ALTER_TAG); + this.tagInfo = new CreateOrReplaceTagInfo(tagName, create, replace, ifNotExists, tagOptions); + } + + @Override + public boolean allowOpMTMV() { + return false; + } + + @Override + public boolean needChangeMTMVState() { + return false; + } + + @Override + public String toSql() { + return tagInfo.toSql(); + } + + @Override + public Map getProperties() { + return null; + } + + @Override + public AlterTableClause translateToLegacyAlterClause() { + return new CreateOrReplaceTagClause(tagInfo); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TagOptions.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TagOptions.java new file mode 100644 index 00000000000000..291c87bff45a99 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TagOptions.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import java.util.Optional; + +/** + * Represents the options available for managing tags in Iceberg through Doris. + * This class encapsulates optional parameters that can be specified when creating or manipulating tags. + * + *

{@code TagOptions} is typically used in conjunction with commands that interact with Iceberg tables, + * such as creating a tag or specifying retention policies.

+ */ +public class TagOptions { + public static final TagOptions EMPTY = new TagOptions(Optional.empty(), Optional.empty()); + + private final Optional snapshotId; + + private final Optional retain; + + public TagOptions(Optional snapshotId, + Optional retain) { + this.snapshotId = snapshotId; + this.retain = retain; + } + + public Optional getSnapshotId() { + return snapshotId; + } + + public Optional getRetain() { + return retain; + } + + /** + * Generates the SQL representation of the tag options. + */ + public String toSql() { + StringBuilder sb = new StringBuilder(); + if (snapshotId.isPresent()) { + sb.append(" AS OF VERSION ").append(snapshotId.get()); + } + if (retain.isPresent()) { + // "RETAIN", and convert retain time to MINUTES + sb.append(" RETAIN ").append(retain.get() / 1000 / 60).append(" MINUTES"); + } + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 3af06e84ac7351..086d5c9afb3546 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -50,6 +50,7 @@ import org.apache.doris.cooldown.CooldownConfHandler; import org.apache.doris.cooldown.CooldownConfList; import org.apache.doris.cooldown.CooldownDelete; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogLog; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalObjectLog; @@ -1279,6 +1280,14 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { env.getIndexPolicyMgr().replayDropIndexPolicy(log); break; } + case OperationType.OP_BRANCH_OR_TAG: { + TableBranchOrTagInfo info = (TableBranchOrTagInfo) journal.getData(); + CatalogIf ctl = Env.getCurrentEnv().getCatalogMgr().getCatalog(info.getCtlName()); + if (ctl != null) { + ctl.replayCreateOrReplaceBranchOrTag(info.getDbName(), info.getTblName()); + } + break; + } default: { IOException e = new IOException(); LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode, logId, e); @@ -2271,4 +2280,8 @@ public void logDictionaryIncVersion(Dictionary dictionary) { public void logDictionaryDecVersion(Dictionary dictionary) { logEdit(OperationType.OP_DICTIONARY_DEC_VERSION, new DictionaryDecreaseVersionInfo(dictionary)); } + + public void logBranchOrTag(TableBranchOrTagInfo info) { + logEdit(OperationType.OP_BRANCH_OR_TAG, info); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index f2aad4557f3dd7..8205ff2043d3d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -56,6 +56,7 @@ public class OperationType { public static final short OP_REPLACE_TEMP_PARTITION = 210; public static final short OP_BATCH_MODIFY_PARTITION = 211; public static final short OP_REPLACE_TABLE = 212; + public static final short OP_BRANCH_OR_TAG = 213; // 20~29 120~129 220~229 ... @Deprecated diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TableBranchOrTagInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TableBranchOrTagInfo.java new file mode 100644 index 00000000000000..ea1f54607f4887 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TableBranchOrTagInfo.java @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * Represents the information needed for table's branch or tag operation. + * This class is used for serialization and deserialization of table branch or tag information. + */ +public class TableBranchOrTagInfo implements Writable { + @SerializedName(value = "ctl") + private String ctlName; + @SerializedName(value = "db") + private String dbName; + @SerializedName(value = "tbl") + private String tblName; + + public TableBranchOrTagInfo(String ctlName, String dbName, String tblName) { + this.ctlName = ctlName; + this.dbName = dbName; + this.tblName = tblName; + } + + public String getCtlName() { + return ctlName; + } + + public String getDbName() { + return dbName; + } + + public String getTblName() { + return tblName; + } + + public static TableBranchOrTagInfo read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, TableBranchOrTagInfo.class); + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof TableBranchOrTagInfo)) { + return false; + } + TableBranchOrTagInfo other = (TableBranchOrTagInfo) obj; + return ctlName.equals(other.ctlName) && dbName.equals(other.dbName) && tblName.equals(other.tblName); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java new file mode 100644 index 00000000000000..4626b2e7b2d134 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.UserException; +import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo; +import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo; +import org.apache.doris.nereids.trees.plans.commands.info.TagOptions; +import org.apache.doris.persist.EditLog; + +import com.google.common.collect.Lists; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +public class IcebergExternalTableBranchAndTagTest { + + Path tempDirectory; + Table icebergTable; + IcebergExternalCatalog catalog; + IcebergExternalDatabase db; + IcebergExternalTable dorisTable; + HadoopCatalog icebergCatalog; + MockedStatic mockedIcebergUtils; + MockedStatic mockedEnv; + String dbName = "db"; + String tblName = "tbl"; + + @BeforeEach + public void setUp() throws IOException { + HashMap map = new HashMap<>(); + tempDirectory = Files.createTempDirectory(""); + map.put("warehouse", "file://" + tempDirectory.toString()); + map.put("type", "hadoop"); + System.out.println(tempDirectory); + icebergCatalog = + (HadoopCatalog) CatalogUtil.buildIcebergCatalog("iceberg_catalog", map, new Configuration()); + + // init iceberg table + icebergCatalog.createNamespace(Namespace.of(dbName)); + icebergTable = icebergCatalog.createTable( + TableIdentifier.of(dbName, tblName), + new Schema(Types.NestedField.required(1, "level", Types.StringType.get()))); + + // init external table + catalog = Mockito.spy(new IcebergHadoopExternalCatalog(1L, "iceberg", null, map, null)); + catalog.setInitializedForTest(true); + // db = new IcebergExternalDatabase(catalog, 1L, dbName, dbName); + db = Mockito.spy(new IcebergExternalDatabase(catalog, 1L, dbName, dbName)); + dorisTable = Mockito.spy(new IcebergExternalTable(1, tblName, tblName, catalog, db)); + Mockito.doReturn(db).when(catalog).getDbNullable(Mockito.any()); + Mockito.doReturn(dorisTable).when(db).getTableNullable(Mockito.any()); + + // mock IcebergUtils.getIcebergTable to return our test icebergTable + mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); + mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(icebergTable); + + // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing + Env mockEnv = Mockito.mock(Env.class); + EditLog mockEditLog = Mockito.mock(EditLog.class); + mockedEnv = Mockito.mockStatic(Env.class); + mockedEnv.when(Env::getCurrentEnv).thenReturn(mockEnv); + Mockito.when(mockEnv.getEditLog()).thenReturn(mockEditLog); + Mockito.doNothing().when(mockEditLog).logBranchOrTag(Mockito.any()); + } + + @AfterEach + public void tearDown() throws IOException { + if (icebergCatalog != null) { + icebergCatalog.dropTable(TableIdentifier.of("db", "tbl")); + icebergCatalog.dropNamespace(Namespace.of("db")); + } + Files.deleteIfExists(tempDirectory); + + // close the static mock + if (mockedIcebergUtils != null) { + mockedIcebergUtils.close(); + } + if (mockedEnv != null) { + mockedEnv.close(); + } + } + + @Test + public void testCreateTagWithTable() throws UserException, IOException { + String tag1 = "tag1"; + String tag2 = "tag2"; + String tag3 = "tag3"; + + // create a new tag: tag1 + // will fail + CreateOrReplaceTagInfo info = + new CreateOrReplaceTagInfo(tag1, true, false, false, TagOptions.EMPTY); + Assertions.assertThrows( + UserException.class, + () -> catalog.createOrReplaceTag(dbName, tblName, info)); + + // add some data + addSomeDataIntoIcebergTable(); + List snapshots = Lists.newArrayList(icebergTable.snapshots()); + Assertions.assertEquals(1, snapshots.size()); + + // create a new tag: tag1 + catalog.createOrReplaceTag(dbName, tblName, info); + assertSnapshotRef( + icebergTable.refs().get(tag1), + icebergTable.currentSnapshot().snapshotId(), + false, null, null, null); + + // create an existed tag: tag1 + Assertions.assertThrows( + IllegalArgumentException.class, + () -> catalog.createOrReplaceTag(dbName, tblName, info)); + + // create an existed tag with replace + CreateOrReplaceTagInfo info2 = + new CreateOrReplaceTagInfo(tag1, true, true, false, TagOptions.EMPTY); + catalog.createOrReplaceTag(dbName, tblName, info2); + assertSnapshotRef( + icebergTable.refs().get(tag1), + icebergTable.currentSnapshot().snapshotId(), + false, null, null, null); + + // create an existed tag with if not exists + CreateOrReplaceTagInfo info3 = + new CreateOrReplaceTagInfo(tag1, true, false, true, TagOptions.EMPTY); + catalog.createOrReplaceTag(dbName, tblName, info3); + assertSnapshotRef( + icebergTable.refs().get(tag1), + icebergTable.currentSnapshot().snapshotId(), + false, null, null, null); + + // add some data + addSomeDataIntoIcebergTable(); + addSomeDataIntoIcebergTable(); + snapshots = Lists.newArrayList(icebergTable.snapshots()); + Assertions.assertEquals(3, snapshots.size()); + + // create new tag: tag2 with snapshotId + TagOptions tagOps = new TagOptions( + Optional.of(snapshots.get(1).snapshotId()), + Optional.empty()); + CreateOrReplaceTagInfo info4 = + new CreateOrReplaceTagInfo(tag2, true, false, false, tagOps); + catalog.createOrReplaceTag(dbName, tblName, info4); + assertSnapshotRef( + icebergTable.refs().get(tag2), + snapshots.get(1).snapshotId(), + false, null, null, null); + + // update tag2 + TagOptions tagOps2 = new TagOptions( + Optional.empty(), + Optional.of(2L)); + CreateOrReplaceTagInfo info5 = + new CreateOrReplaceTagInfo(tag2, true, true, false, tagOps2); + catalog.createOrReplaceTag(dbName, tblName, info5); + assertSnapshotRef( + icebergTable.refs().get(tag2), + icebergTable.currentSnapshot().snapshotId(), + false, null, null, 2L); + + // create new tag: tag3 + CreateOrReplaceTagInfo info6 = + new CreateOrReplaceTagInfo(tag3, true, false, false, tagOps2); + catalog.createOrReplaceTag(dbName, tblName, info6); + assertSnapshotRef( + icebergTable.refs().get(tag3), + icebergTable.currentSnapshot().snapshotId(), + false, null, null, 2L); + + Assertions.assertEquals(4, icebergTable.refs().size()); + } + + @Test + public void testCreateBranchWithNotEmptyTable() throws UserException, IOException { + + String branch1 = "branch1"; + String branch2 = "branch2"; + String branch3 = "branch3"; + + // create a new branch: branch1 + CreateOrReplaceBranchInfo info = + new CreateOrReplaceBranchInfo(branch1, true, false, false, BranchOptions.EMPTY); + catalog.createOrReplaceBranch(dbName, tblName, info); + List snapshots = Lists.newArrayList(icebergTable.snapshots()); + Assertions.assertEquals(1, snapshots.size()); + assertSnapshotRef( + icebergTable.refs().get(branch1), + snapshots.get(0).snapshotId(), + true, null, null, null); + + // create an existed branch, failed + Assertions.assertThrows( + IllegalArgumentException.class, + () -> catalog.createOrReplaceBranch(dbName, tblName, info)); + + // create or replace an empty branch, will fail + // because cannot perform a replace operation on an empty branch. + CreateOrReplaceBranchInfo info2 = + new CreateOrReplaceBranchInfo(branch1, true, true, false, BranchOptions.EMPTY); + Assertions.assertThrows( + UserException.class, + () -> catalog.createOrReplaceBranch(dbName, tblName, info2)); + + // create an existed branch with ifNotExists + CreateOrReplaceBranchInfo info4 = + new CreateOrReplaceBranchInfo(branch1, true, false, true, BranchOptions.EMPTY); + catalog.createOrReplaceBranch(dbName, tblName, info4); + assertSnapshotRef( + icebergTable.refs().get(branch1), + snapshots.get(0).snapshotId(), + true, null, null, null); + + // add some data + addSomeDataIntoIcebergTable(); + snapshots = Lists.newArrayList(icebergTable.snapshots()); + Assertions.assertEquals(2, snapshots.size()); + + // update branch1 + catalog.createOrReplaceBranch(dbName, tblName, info2); + assertSnapshotRef( + icebergTable.refs().get(branch1), + icebergTable.currentSnapshot().snapshotId(), + true, null, null, null); + + // create or replace a new branch: branch2 + CreateOrReplaceBranchInfo info3 = + new CreateOrReplaceBranchInfo(branch2, true, true, false, BranchOptions.EMPTY); + catalog.createOrReplaceBranch(dbName, tblName, info3); + assertSnapshotRef( + icebergTable.refs().get(branch2), + icebergTable.currentSnapshot().snapshotId(), + true, null, null, null); + + // update branch2 + BranchOptions brOps = new BranchOptions( + Optional.empty(), + Optional.of(1L), + Optional.of(2), + Optional.of(3L)); + CreateOrReplaceBranchInfo info5 = + new CreateOrReplaceBranchInfo(branch2, true, true, false, brOps); + catalog.createOrReplaceBranch(dbName, tblName, info5); + assertSnapshotRef( + icebergTable.refs().get(branch2), + icebergTable.currentSnapshot().snapshotId(), + true, 1L, 2, 3L); + + // total branch: + // 'main','branch1','branch2' + Assertions.assertEquals(3, icebergTable.refs().size()); + + // insert some data + addSomeDataIntoIcebergTable(); + addSomeDataIntoIcebergTable(); + addSomeDataIntoIcebergTable(); + addSomeDataIntoIcebergTable(); + snapshots = Lists.newArrayList(icebergTable.snapshots()); + Assertions.assertEquals(6, snapshots.size()); + + // create a new branch: branch3 + BranchOptions brOps2 = new BranchOptions( + Optional.of(snapshots.get(4).snapshotId()), + Optional.of(1L), + Optional.of(2), + Optional.of(3L)); + CreateOrReplaceBranchInfo info6 = + new CreateOrReplaceBranchInfo(branch3, true, true, false, brOps2); + catalog.createOrReplaceBranch(dbName, tblName, info6); + assertSnapshotRef( + icebergTable.refs().get(branch3), + snapshots.get(4).snapshotId(), + true, 1L, 2, 3L); + + // update branch1 + catalog.createOrReplaceBranch(dbName, tblName, info2); + assertSnapshotRef( + icebergTable.refs().get(branch1), + icebergTable.currentSnapshot().snapshotId(), + true, null, null, null); + + Assertions.assertEquals(4, icebergTable.refs().size()); + } + + private void addSomeDataIntoIcebergTable() throws IOException { + Path fileA = Files.createFile(tempDirectory.resolve(UUID.randomUUID().toString())); + DataFiles.Builder builder = DataFiles.builder(icebergTable.spec()) + .withPath(fileA.toString()) + .withFileSizeInBytes(10) + .withRecordCount(1) + .withFormat("parquet"); + icebergTable.newFastAppend() + .appendFile(builder.build()) + .commit(); + } + + private void assertSnapshotRef( + SnapshotRef ref, + Long snapshotId, + boolean isBranch, + Long maxSnapshotAgeMs, + Integer minSnapshotsToKeep, + Long maxRefAgeMs) { + if (snapshotId != null) { + Assertions.assertEquals(snapshotId, ref.snapshotId()); + } + if (isBranch) { + Assertions.assertTrue(ref.isBranch()); + } else { + Assertions.assertTrue(ref.isTag()); + } + Assertions.assertEquals(maxSnapshotAgeMs, ref.maxSnapshotAgeMs()); + Assertions.assertEquals(minSnapshotsToKeep, ref.minSnapshotsToKeep()); + Assertions.assertEquals(maxRefAgeMs, ref.maxRefAgeMs()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOrTagInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOrTagInfoTest.java new file mode 100644 index 00000000000000..e3d9ad9b23bdfc --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/CreateOrReplaceBranchOrTagInfoTest.java @@ -0,0 +1,281 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Optional; + +public class CreateOrReplaceBranchOrTagInfoTest { + + @Test + public void testCreateBranchToSql() { + // Test CREATE BRANCH with no options + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, null); + String expected = "CREATE BRANCH test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testReplaceBranchToSql() { + // Test REPLACE BRANCH with no options + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", false, true, false, null); + String expected = "REPLACE BRANCH test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateOrReplaceBranchToSql() { + // Test CREATE OR REPLACE BRANCH with no options + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, true, false, null); + String expected = "CREATE OR REPLACE BRANCH test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchIfNotExistsToSql() { + // Test CREATE BRANCH IF NOT EXISTS + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, true, null); + String expected = "CREATE BRANCH IF NOT EXISTS test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateOrReplaceBranchIfNotExistsToSql() { + // Test CREATE OR REPLACE BRANCH IF NOT EXISTS + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, true, true, null); + String expected = "CREATE OR REPLACE BRANCH IF NOT EXISTS test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithSnapshotIdToSql() { + // Test CREATE BRANCH with snapshot ID + BranchOptions options = new BranchOptions( + Optional.of(123456L), Optional.empty(), Optional.empty(), Optional.empty()); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected = "CREATE BRANCH test_branch AS OF VERSION 123456"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithRetainToSql() { + // Test CREATE BRANCH with retain time (1 hour = 3600000ms) + BranchOptions options = new BranchOptions( + Optional.empty(), Optional.of(3600000L), Optional.empty(), Optional.empty()); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected = "CREATE BRANCH test_branch RETAIN 60 MINUTES"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithNumSnapshotsToSql() { + // Test CREATE BRANCH with number of snapshots + BranchOptions options = new BranchOptions( + Optional.empty(), Optional.empty(), Optional.of(5), Optional.empty()); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected = "CREATE BRANCH test_branch WITH SNAPSHOT RETENTION 5 SNAPSHOTS"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithRetentionToSql() { + // Test CREATE BRANCH with retention time (1 day = 86400000ms) + BranchOptions options = new BranchOptions( + Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(86400000L)); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected = "CREATE BRANCH test_branch WITH SNAPSHOT RETENTION 1440 MINUTES"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithNumSnapshotsAndRetentionToSql() { + // Test CREATE BRANCH with both num snapshots and retention + BranchOptions options = new BranchOptions( + Optional.empty(), Optional.empty(), Optional.of(10), Optional.of(86400000L)); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected = "CREATE BRANCH test_branch WITH SNAPSHOT RETENTION 10 SNAPSHOTS 1440 MINUTES"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testCreateBranchWithAllOptionsToSql() { + // Test CREATE BRANCH with all options + BranchOptions options = new BranchOptions( + Optional.of(123456L), Optional.of(3600000L), Optional.of(5), Optional.of(86400000L)); + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, options); + String expected + = "CREATE BRANCH test_branch AS OF VERSION 123456 RETAIN 60 MINUTES WITH SNAPSHOT RETENTION 5 SNAPSHOTS 1440 MINUTES"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testEmptyOptionsToSql() { + // Test with BranchOptions.EMPTY + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "test_branch", true, false, false, BranchOptions.EMPTY); + String expected = "CREATE BRANCH test_branch"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + @Test + public void testBranchNameWithSpecialCharacters() { + // Test branch name with underscores and numbers + CreateOrReplaceBranchInfo branchInfo = new CreateOrReplaceBranchInfo( + "feature_branch_v2_0", true, false, false, null); + String expected = "CREATE BRANCH feature_branch_v2_0"; + Assertions.assertEquals(expected, branchInfo.toSql()); + } + + // ========================== Tag Tests ========================== + + @Test + public void testCreateTagToSql() { + // Test CREATE TAG with no options + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, false, null); + String expected = "CREATE TAG test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testReplaceTagToSql() { + // Test REPLACE TAG with no options + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", false, true, false, null); + String expected = "REPLACE TAG test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateOrReplaceTagToSql() { + // Test CREATE OR REPLACE TAG with no options + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, true, false, null); + String expected = "CREATE OR REPLACE TAG test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateTagIfNotExistsToSql() { + // Test CREATE TAG IF NOT EXISTS + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, true, null); + String expected = "CREATE TAG IF NOT EXISTS test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateOrReplaceTagIfNotExistsToSql() { + // Test CREATE OR REPLACE TAG IF NOT EXISTS + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, true, true, null); + String expected = "CREATE OR REPLACE TAG IF NOT EXISTS test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateTagWithSnapshotIdToSql() { + // Test CREATE TAG with snapshot ID + TagOptions options = new TagOptions(Optional.of(123456L), Optional.empty()); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, false, options); + String expected = "CREATE TAG test_tag AS OF VERSION 123456"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateTagWithRetainToSql() { + // Test CREATE TAG with retain time (1 hour = 3600000ms) + TagOptions options = new TagOptions(Optional.empty(), Optional.of(3600000L)); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, false, options); + String expected = "CREATE TAG test_tag RETAIN 60 MINUTES"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateTagWithAllOptionsToSql() { + // Test CREATE TAG with all options (snapshot ID and retain) + TagOptions options = new TagOptions(Optional.of(123456L), Optional.of(3600000L)); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, false, options); + String expected = "CREATE TAG test_tag AS OF VERSION 123456 RETAIN 60 MINUTES"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testEmptyTagOptionsToSql() { + // Test with TagOptions.EMPTY + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "test_tag", true, false, false, TagOptions.EMPTY); + String expected = "CREATE TAG test_tag"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testTagNameWithSpecialCharacters() { + // Test tag name with underscores and numbers + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "release_tag_v1_0", true, false, false, null); + String expected = "CREATE TAG release_tag_v1_0"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testReplaceTagWithOptionsToSql() { + // Test REPLACE TAG with snapshot ID and retain + TagOptions options = new TagOptions(Optional.of(789012L), Optional.of(7200000L)); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "production_tag", false, true, false, options); + String expected = "REPLACE TAG production_tag AS OF VERSION 789012 RETAIN 120 MINUTES"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateOrReplaceTagWithSnapshotOnlyToSql() { + // Test CREATE OR REPLACE TAG with only snapshot ID + TagOptions options = new TagOptions(Optional.of(555666L), Optional.empty()); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "backup_tag", true, true, false, options); + String expected = "CREATE OR REPLACE TAG backup_tag AS OF VERSION 555666"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } + + @Test + public void testCreateTagIfNotExistsWithRetainToSql() { + // Test CREATE TAG IF NOT EXISTS with retain time + TagOptions options = new TagOptions(Optional.empty(), Optional.of(86400000L)); + CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo( + "daily_tag", true, false, true, options); + String expected = "CREATE TAG IF NOT EXISTS daily_tag RETAIN 1440 MINUTES"; + Assertions.assertEquals(expected, tagInfo.toSql()); + } +} diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out b/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out new file mode 100644 index 00000000000000..a7108fdcc02837 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/iceberg_branch_tag_operate.out @@ -0,0 +1,150 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1 -- + +-- !q2 -- + +-- !q3 -- +1 + +-- !q4 -- +1 +2 + +-- !q5 -- +1 +2 +3 + +-- !q6 -- +1 +2 +3 +4 +5 + +-- !q7 -- +1 + +-- !q8 -- +1 + +-- !q9 -- +1 +2 + +-- !q10 -- +1 +2 +3 +4 +5 + +-- !q11 -- +1 +2 + +-- !q12 -- +1 +2 +3 + +-- !q13 -- +1 +2 +3 +4 +5 + +-- !q14 -- +1 +2 +3 +4 +5 + +-- !q15 -- +1 +2 +3 +4 +5 + +-- !q16 -- +1 +2 +3 +4 +5 + +-- !q20 -- + +-- !q21 -- +1 + +-- !q22 -- +1 +2 + +-- !q23 -- +1 +2 +3 +4 +5 + +-- !q24 -- +1 + +-- !q25 -- +1 + +-- !q26 -- +1 +2 + +-- !q27 -- +1 +2 +3 + +-- !q28 -- +1 +2 +3 +4 +5 + +-- !sc01 -- +1 a \N +2 b \N +3 c \N +4 d 2024-04-04 +5 e 2024-05-05 + +-- !sc02 -- +1 a \N +2 b \N +3 c \N + +-- !sc03 -- +1 a \N +2 b \N +3 c \N + +-- !sc04 -- +1 a 1.0 +2 b 2.0 +3 c 3.0 + +-- !sc05 -- +1 a \N +2 b \N +3 c \N +4 d 2024-04-04 +5 e 2024-05-05 + +-- !sc06 -- +1 a 1.0 +2 b 2.0 +3 c 3.0 + diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy new file mode 100644 index 00000000000000..d96b2a08397d66 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_branch_tag_operate.groovy @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("iceberg_branch_tag_operate", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalog_name = "iceberg_branch_tag_operate" + + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + sql """ use ${catalog_name}.test_db """ + + sql """ drop table if exists test_branch_tag_operate """ + sql """ create table test_branch_tag_operate (id int) """ + + // with empty table + + test { + sql """ alter table test_branch_tag_operate create tag b1 """ + exception "main has no snapshot" + } + + sql """ alter table test_branch_tag_operate create branch b1 """ + sql """ alter table test_branch_tag_operate create branch if not exists b1 """ + + test { + sql """ alter table test_branch_tag_operate create or replace branch b1 """ + exception "main has no snapshot" + } + + test { + sql """ alter table test_branch_tag_operate create branch b1 """ + exception "Ref b1 already exists" + } + + qt_q1 """ select * from test_branch_tag_operate@branch(b1) """ // empty table + + // with some data + sql """ insert into test_branch_tag_operate values (1) """ + sql """ insert into test_branch_tag_operate values (2) """ + sql """ insert into test_branch_tag_operate values (3) """ + sql """ insert into test_branch_tag_operate values (4) """ + sql """ insert into test_branch_tag_operate values (5) """ + + + List> snapshots = sql """ select snapshot_id from iceberg_meta("table" = "${catalog_name}.test_db.test_branch_tag_operate", "query_type" = "snapshots") order by committed_at; """ + String s0 = snapshots.get(0)[0] + String s1 = snapshots.get(1)[0] + String s2 = snapshots.get(2)[0] + String s3 = snapshots.get(3)[0] + String s4 = snapshots.get(4)[0] + + // branch + sql """ alter table test_branch_tag_operate create branch b2 as of version ${s0} """ + qt_q2 """ select * from test_branch_tag_operate@branch(b2) order by id """ // 0 records + + sql """ alter table test_branch_tag_operate create or replace branch b2 AS OF VERSION ${s1} RETAIN 2 days """ + qt_q3 """ select * from test_branch_tag_operate@branch(b2) order by id """ // 1 records + + sql """ alter table test_branch_tag_operate create or replace branch b2 AS OF VERSION ${s2} RETAIN 2 hours WITH SNAPSHOT RETENTION 3 SNAPSHOTS""" + qt_q4 """ select * from test_branch_tag_operate@branch(b2) order by id """ // 2 records + + sql """ alter table test_branch_tag_operate replace branch b2 AS OF VERSION ${s3} RETAIN 2 hours WITH SNAPSHOT RETENTION 4 DAYS """ + qt_q5 """ select * from test_branch_tag_operate@branch(b2) order by id """ // 3 records + + sql """ alter table test_branch_tag_operate create or replace branch b2 RETAIN 2 hours WITH SNAPSHOT RETENTION 3 SNAPSHOTS 4 DAYS """ + qt_q6 """ select * from test_branch_tag_operate@branch(b2) order by id """ // 5 records + + sql """ alter table test_branch_tag_operate create or replace branch b3 AS OF VERSION ${s1} RETAIN 2 days """ + qt_q7 """ select * from test_branch_tag_operate@branch(b3) order by id """ // 1 records + + sql """ alter table test_branch_tag_operate create branch if not exists b3 AS OF VERSION ${s2} RETAIN 2 days """ + qt_q8 """ select * from test_branch_tag_operate@branch(b3) order by id """ // still 1 records + + sql """ alter table test_branch_tag_operate create branch if not exists b4 AS OF VERSION ${s2} RETAIN 2 MINUTES WITH SNAPSHOT RETENTION 3 SNAPSHOTS """ + qt_q9 """ select * from test_branch_tag_operate@branch(b4) order by id """ // 2 records + + sql """ alter table test_branch_tag_operate create branch if not exists b5 """ + qt_q10 """ select * from test_branch_tag_operate@branch(b5) order by id """ // 5 records + + sql """ alter table test_branch_tag_operate create branch if not exists b6 AS OF VERSION ${s2} """ + qt_q11 """ select * from test_branch_tag_operate@branch(b6) order by id """ // 2 records + + sql """ alter table test_branch_tag_operate create or replace branch b6 AS OF VERSION ${s3} """ + qt_q12 """ select * from test_branch_tag_operate@branch(b6) order by id """ // 3 records + + sql """ alter table test_branch_tag_operate create or replace branch b6 """ + qt_q13 """ select * from test_branch_tag_operate@branch(b6) order by id """ // 5 records + + sql """ alter table test_branch_tag_operate create or replace branch b6 """ + qt_q14 """ select * from test_branch_tag_operate@branch(b6) order by id """ // still 5 records + + sql """ alter table test_branch_tag_operate create or replace branch b6 RETAIN 2 DAYS """ + qt_q15 """ select * from test_branch_tag_operate@branch(b6) order by id """ // still 5 records + + sql """ alter table test_branch_tag_operate create branch b7 """ + qt_q16 """ select * from test_branch_tag_operate@branch(b7) order by id """ // 5 records + + test { + sql """ alter table test_branch_tag_operate create branch b7 as of version ${s3} """ + exception "Ref b7 already exists" + } + + test { + sql """ alter table test_branch_tag_operate create branch b8 as of version 11223344 """ + exception "Cannot set b8 to unknown snapshot: 11223344" + } + + + // tag + sql """ alter table test_branch_tag_operate create tag t2 as of version ${s0} """ + qt_q20 """ select * from test_branch_tag_operate@tag(t2) order by id """ // 0 records + + sql """ alter table test_branch_tag_operate create or replace tag t2 as of version ${s1} """ + qt_q21 """ select * from test_branch_tag_operate@tag(t2) order by id """ // 1 records + + sql """ alter table test_branch_tag_operate create or replace tag t2 as of version ${s2} RETAIN 10 MINUTES """ + qt_q22 """ select * from test_branch_tag_operate@tag(t2) order by id """ // 2 records + + sql """ alter table test_branch_tag_operate create or replace tag t2 RETAIN 10 MINUTES """ + qt_q23 """ select * from test_branch_tag_operate@tag(t2) order by id """ // 5 records + + sql """ alter table test_branch_tag_operate create tag if not exists t3 as of version ${s1} """ + qt_q24 """ select * from test_branch_tag_operate@tag(t3) order by id """ // 1 records + + sql """ alter table test_branch_tag_operate create tag if not exists t3 as of version ${s2} """ // still 1 records + qt_q25 """ select * from test_branch_tag_operate@tag(t3) order by id """ + + sql """ alter table test_branch_tag_operate create tag t4 as of version ${s2} """ + qt_q26 """ select * from test_branch_tag_operate@tag(t4) order by id """ // 2 records + + sql """ alter table test_branch_tag_operate create or replace tag t5 as of version ${s3} """ + qt_q27 """ select * from test_branch_tag_operate@tag(t5) order by id """ // 3 records + + sql """ alter table test_branch_tag_operate create tag t6 """ + qt_q28 """ select * from test_branch_tag_operate@tag(t6) order by id """ // 5 records + + test { + sql """ alter table test_branch_tag_operate create tag t6 as of version ${s3} """ + exception "Ref t6 already exists" + } + + test { + sql """ alter table test_branch_tag_operate create branch t7 as of version 11223344 """ + exception "Cannot set t7 to unknown snapshot: 11223344" + } + + // test branch/tag with schema change + qt_sc01 """select * from tmp_schema_change_branch order by id;""" + /// select by branch will use table schema + qt_sc02 """select * from tmp_schema_change_branch@branch(test_branch) order by id;;""" + qt_sc03 """select * from tmp_schema_change_branch for version as of "test_branch" order by id;;""" + List> refs = sql """select * from tmp_schema_change_branch\$refs order by name""" + String s_main = refs.get(0)[2] + String s_test_branch = refs.get(1)[2] + + /// select by version will use branch schema + qt_sc04 """SELECT * FROM tmp_schema_change_branch for VERSION AS OF ${s_test_branch} order by id;""" + qt_sc05 """SELECT * FROM tmp_schema_change_branch for VERSION AS OF ${s_main} order by id;""" + + /// select by tag will use tag schema + qt_sc06 """SELECT * FROM tmp_schema_change_branch@tag(test_tag) order by id;""" +}