Merge remote-tracking branch 'origin/auto-quota' into auto-quota

apache · Jan 11, 2023 · d9cd84a · d9cd84a
2 parents 5591bcd + 5a56ca0
commit d9cd84a
Showing 1 changed file with 19 additions and 3 deletions.
diff --git a/hive/src/test/java/com/netease/arctic/hive/io/TestIOUtils.java b/hive/src/test/java/com/netease/arctic/hive/io/TestIOUtils.java
@@ -18,6 +18,7 @@
 
 package com.netease.arctic.hive.io;
 
+import com.netease.arctic.data.ChangeAction;
 import com.netease.arctic.data.DefaultKeyedFile;
 import com.netease.arctic.hive.io.reader.AdaptHiveGenericArcticDataReader;
 import com.netease.arctic.hive.io.reader.GenericAdaptHiveIcebergDataReader;
@@ -30,6 +31,7 @@
 import com.netease.arctic.table.ArcticTable;
 import com.netease.arctic.table.LocationKind;
 import com.netease.arctic.table.PrimaryKeySpec;
+import java.util.stream.Stream;
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.Files;
 import org.apache.iceberg.PartitionSpec;
@@ -58,12 +60,13 @@
 public class TestIOUtils {
 
   public static void testWrite(ArcticTable table, LocationKind locationKind, List<Record> records, String pathFeature) throws IOException {
-    testWrite(table, locationKind, records, pathFeature, null, null);
+    testWrite(table, locationKind, records, null, pathFeature, null, null);
   }
 
   public static void testWrite(
-      ArcticTable table, LocationKind locationKind, List<Record> records, String pathFeature,
+      ArcticTable table, LocationKind locationKind, List<Record> records,List<Record> deleteRecords, String pathFeature,
       Expression expression, List<Record> readRecords) throws IOException {
+    //write data records
     AdaptHiveGenericTaskWriterBuilder builder = AdaptHiveGenericTaskWriterBuilder
         .builderFor(table)
         .withTransactionId(table.isKeyedTable() ? 1L : null);
@@ -73,7 +76,20 @@ public static void testWrite(
       changeWrite.write(record);
     }
     WriteResult complete = changeWrite.complete();
-    Arrays.stream(complete.dataFiles()).forEach(s -> Assert.assertTrue(s.path().toString().contains(pathFeature)));
+
+    //write delete records
+    AdaptHiveGenericTaskWriterBuilder deleteBuilder = AdaptHiveGenericTaskWriterBuilder
+        .builderFor(table)
+        .withChangeAction(ChangeAction.DELETE)
+        .withTransactionId(table.isKeyedTable() ? 2L : null);
+
+    TaskWriter<Record> changeDeleteWrite = builder.buildWriter(locationKind);
+    for (Record record: deleteRecords) {
+      changeDeleteWrite.write(record);
+    }
+    WriteResult deleteComplete = changeDeleteWrite.complete();
+
+    Stream.concat(Arrays.stream(complete.dataFiles()), Arrays.stream(deleteComplete.dataFiles())).forEach(s -> Assert.assertTrue(s.path().toString().contains(pathFeature)));
     CloseableIterator<Record> iterator = readParquet(
             table.schema(),
             complete.dataFiles(),