[ARCTIC-1620] Check there are no orphan files in the directory when M…

…ixed Hive Full Optimizing committing (apache#1621) * set delete-untracked-hive-file=true when Mixed Hive Table commit * not check orphan files when move files to old hive location
wangtaohz · Jun 29, 2023 · a5a5b5a · a5a5b5a
1 parent d7c64e9
commit a5a5b5a
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/KeyedTableCommit.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/KeyedTableCommit.java
@@ -4,6 +4,7 @@
 import com.netease.arctic.data.DataFileType;
 import com.netease.arctic.data.IcebergContentFile;
 import com.netease.arctic.data.PrimaryKeyedFile;
+import com.netease.arctic.hive.utils.TableTypeUtil;
 import com.netease.arctic.op.OverwriteBaseFiles;
 import com.netease.arctic.optimizing.RewriteFilesInput;
 import com.netease.arctic.optimizing.RewriteFilesOutput;
@@ -29,6 +30,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static com.netease.arctic.hive.op.UpdateHiveFiles.DELETE_UNTRACKED_HIVE_FILE;
 import static com.netease.arctic.server.ArcticServiceConstants.INVALID_SNAPSHOT_ID;
 
 public class KeyedTableCommit extends UnKeyedTableCommit {
@@ -139,6 +141,9 @@ private void executeCommit(
     addedDataFiles.forEach(overwriteBaseFiles::addFile);
     addedDeleteFiles.forEach(overwriteBaseFiles::addFile);
     removedDataFiles.forEach(overwriteBaseFiles::deleteFile);
+    if (TableTypeUtil.isHive(table) && !needMoveFile2Hive()) {
+      overwriteBaseFiles.set(DELETE_UNTRACKED_HIVE_FILE, "true");
+    }
     overwriteBaseFiles.skipEmptyCommit().commit();
 
     //remove delete files

diff --git a/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java b/ams/server/src/main/java/com/netease/arctic/server/optimizing/UnKeyedTableCommit.java
@@ -25,6 +25,7 @@
 import com.netease.arctic.hive.table.SupportHive;
 import com.netease.arctic.hive.utils.HivePartitionUtil;
 import com.netease.arctic.hive.utils.HiveTableUtil;
+import com.netease.arctic.hive.utils.TableTypeUtil;
 import com.netease.arctic.optimizing.OptimizingInputProperties;
 import com.netease.arctic.optimizing.RewriteFilesOutput;
 import com.netease.arctic.server.ArcticServiceConstants;
@@ -63,6 +64,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import static com.netease.arctic.hive.op.UpdateHiveFiles.DELETE_UNTRACKED_HIVE_FILE;
 import static com.netease.arctic.server.ArcticServiceConstants.INVALID_SNAPSHOT_ID;
 
 public class UnKeyedTableCommit {
@@ -192,6 +194,9 @@ protected void replaceFiles(
           dataFileRewrite.rewriteFiles(removedDataFiles, addedDataFiles);
         }
         dataFileRewrite.set(SnapshotSummary.SNAPSHOT_PRODUCER, CommitMetaProducer.OPTIMIZE.name());
+        if (TableTypeUtil.isHive(table) && !needMoveFile2Hive()) {
+          dataFileRewrite.set(DELETE_UNTRACKED_HIVE_FILE, "true");
+        }
         dataFileRewrite.commit();
       }
       if (CollectionUtils.isNotEmpty(addDeleteFiles)) {