From bbb406264a41450ad38e3567c62dd8685ad5c5b8 Mon Sep 17 00:00:00 2001 From: big face cat <731030576@qq.com> Date: Fri, 1 Nov 2024 11:21:03 +0800 Subject: [PATCH] [AMORO-2842] Support list all paimon branch (#3093) * Support list all paimon branch * update * Support list all paimon branch --------- Co-authored-by: huyuanfeng Co-authored-by: ConradJam --- .../MixedAndIcebergTableDescriptor.java | 4 ++- .../dashboard/ServerTableDescriptor.java | 4 +-- .../dashboard/controller/TableController.java | 5 +++- .../descriptor/FormatTableDescriptor.java | 3 ++- .../formats/hudi/HudiTableDescriptor.java | 4 ++- .../formats/paimon/PaimonTableDescriptor.java | 27 ++++++++++++++----- amoro-web/src/services/table.service.ts | 5 ++-- .../src/views/tables/components/Snapshots.vue | 1 + 8 files changed, 39 insertions(+), 14 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java index 792a9170d9..8b58d96dc3 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/MixedAndIcebergTableDescriptor.java @@ -87,6 +87,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -341,7 +343,7 @@ private void collectSnapshots( @Override public List getSnapshotDetail( - AmoroTable amoroTable, String snapshotId) { + AmoroTable amoroTable, String snapshotId, @Nullable String ref) { MixedTable mixedTable = getTable(amoroTable); List result = new ArrayList<>(); long commitId = Long.parseLong(snapshotId); diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java index 521d80f62f..7b8ed7b8a2 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/ServerTableDescriptor.java @@ -81,10 +81,10 @@ public List getSnapshots( } public List getSnapshotDetail( - TableIdentifier tableIdentifier, String snapshotId) { + TableIdentifier tableIdentifier, String snapshotId, String ref) { AmoroTable amoroTable = loadTable(tableIdentifier); FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format()); - return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId); + return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId, ref); } public List getTableOperations(TableIdentifier tableIdentifier) { diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index e009840844..00e2129527 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -416,10 +416,13 @@ public void getSnapshotDetail(Context ctx) { String snapshotId = ctx.pathParam("snapshotId"); Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1); Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20); + String ref = ctx.queryParamAsClass("ref", String.class).getOrDefault(null); List result = tableDescriptor.getSnapshotDetail( - TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(), snapshotId); + TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(), + snapshotId, + ref); int offset = (page - 1) * pageSize; PageResult amsPageResult = PageResult.of(result, offset, pageSize); ctx.json(OkResponse.of(amsPageResult)); diff --git a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java index 3c1126a8f1..1e8bc62c2c 100644 --- a/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java +++ b/amoro-common/src/main/java/org/apache/amoro/table/descriptor/FormatTableDescriptor.java @@ -48,7 +48,8 @@ List getSnapshots( AmoroTable amoroTable, String ref, OperationType operationType); /** Get the snapshot detail information of the {@link AmoroTable}. */ - List getSnapshotDetail(AmoroTable amoroTable, String snapshotId); + List getSnapshotDetail( + AmoroTable amoroTable, String snapshotId, String ref); /** Get the DDL information of the {@link AmoroTable}. */ List getTableOperations(AmoroTable amoroTable); diff --git a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java index 8720958b0b..0a50afb30e 100644 --- a/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java +++ b/amoro-format-hudi/src/main/java/org/apache/amoro/formats/hudi/HudiTableDescriptor.java @@ -74,6 +74,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; @@ -246,7 +248,7 @@ public List getSnapshots( @Override public List getSnapshotDetail( - AmoroTable amoroTable, String snapshotId) { + AmoroTable amoroTable, String snapshotId, @Nullable String ref) { HoodieJavaTable hoodieTable = (HoodieJavaTable) amoroTable.originalTable(); SyncableFileSystemView fileSystemView = hoodieTable.getHoodieView(); Map> ptFsMap = diff --git a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java index 681b7e0951..b217662196 100644 --- a/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java +++ b/amoro-format-paimon/src/main/java/org/apache/amoro/formats/paimon/PaimonTableDescriptor.java @@ -24,7 +24,6 @@ import org.apache.amoro.TableFormat; import org.apache.amoro.api.CommitMetaProducer; import org.apache.amoro.process.ProcessStatus; -import org.apache.amoro.shade.guava32.com.google.common.collect.ImmutableList; import org.apache.amoro.shade.guava32.com.google.common.collect.Lists; import org.apache.amoro.shade.guava32.com.google.common.collect.Maps; import org.apache.amoro.shade.guava32.com.google.common.collect.Streams; @@ -61,7 +60,9 @@ import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.table.DataTable; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.utils.BranchManager; import org.apache.paimon.utils.FileStorePathFactory; +import org.apache.paimon.utils.SnapshotManager; import org.jetbrains.annotations.NotNull; import java.io.IOException; @@ -181,9 +182,10 @@ public List getSnapshots( FileStoreTable table = getTable(amoroTable); List snapshotsOfTables = new ArrayList<>(); Iterator snapshots; - if (PAIMON_MAIN_BRANCH_NAME.equals(ref)) { + if (table.branchManager().branchExists(ref) || BranchManager.isMainBranch(ref)) { + SnapshotManager snapshotManager = table.snapshotManager().copyWithBranch(ref); try { - snapshots = table.snapshotManager().snapshots(); + snapshots = snapshotManager.snapshots(); } catch (IOException e) { throw new RuntimeException(e); } @@ -223,11 +225,17 @@ public List getSnapshots( @Override public List getSnapshotDetail( - AmoroTable amoroTable, String snapshotId) { + AmoroTable amoroTable, String snapshotId, String ref) { FileStoreTable table = getTable(amoroTable); List amsDataFileInfos = new ArrayList<>(); long commitId = Long.parseLong(snapshotId); - Snapshot snapshot = table.snapshotManager().snapshot(commitId); + Snapshot snapshot; + if (BranchManager.isMainBranch(ref) || table.branchManager().branchExists(ref)) { + snapshot = table.snapshotManager().copyWithBranch(ref).snapshot(commitId); + } else { + snapshot = table.tagManager().tag(ref); + } + FileStore store = table.store(); FileStorePathFactory fileStorePathFactory = store.pathFactory(); ManifestList manifestList = store.manifestListFactory().create(); @@ -531,7 +539,14 @@ public List getTableTags(AmoroTable amoroTable) { @Override public List getTableBranches(AmoroTable amoroTable) { - return ImmutableList.of(TagOrBranchInfo.MAIN_BRANCH); + FileStoreTable table = getTable(amoroTable); + List branches = table.branchManager().branches(); + List branchInfos = + branches.stream() + .map(name -> new TagOrBranchInfo(name, -1, -1, 0L, 0L, TagOrBranchInfo.BRANCH)) + .collect(Collectors.toList()); + branchInfos.add(TagOrBranchInfo.MAIN_BRANCH); + return branchInfos; } @Override diff --git a/amoro-web/src/services/table.service.ts b/amoro-web/src/services/table.service.ts index 8b83d71d6b..309a05d476 100644 --- a/amoro-web/src/services/table.service.ts +++ b/amoro-web/src/services/table.service.ts @@ -127,14 +127,15 @@ export function getDetailBySnapshotId( catalog: string db: string table: string + ref: string snapshotId: string page: number pageSize: number token?: string }, ) { - const { catalog, db, table, snapshotId, page, pageSize, token } = params - return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, token } }) + const { catalog, db, table, snapshotId, page, pageSize, ref, token } = params + return request.get(`api/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, ref, token } }) } // get operations export function getOperations( diff --git a/amoro-web/src/views/tables/components/Snapshots.vue b/amoro-web/src/views/tables/components/Snapshots.vue index 219092439b..fef7e57685 100644 --- a/amoro-web/src/views/tables/components/Snapshots.vue +++ b/amoro-web/src/views/tables/components/Snapshots.vue @@ -157,6 +157,7 @@ async function getBreadcrumbTable() { const params = { ...sourceData, snapshotId: snapshotId.value, + ref: tblRef.value, page: breadcrumbPagination.current, pageSize: breadcrumbPagination.pageSize, }