[AMORO-2571] Cannot find data files under partitions with null partit…

…ion values (#2573) [AMORO-2571]Cannot find data files under partitions with null partition values
apache · Feb 26, 2024 · bfd8735 · bfd8735
1 parent 8266f2d
commit bfd8735
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 3 deletions.
diff --git a/core/src/main/java/org/apache/iceberg/IcebergFindFiles.java b/core/src/main/java/org/apache/iceberg/IcebergFindFiles.java
@@ -42,6 +42,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiFunction;
 import java.util.function.Predicate;
@@ -182,9 +183,12 @@ public IcebergFindFiles inPartitions(PartitionSpec spec, List<StructLike> partit
       Expression partFilter = Expressions.alwaysTrue();
       for (int i = 0; i < spec.fields().size(); i += 1) {
         PartitionField field = spec.fields().get(i);
-        partFilter =
-            Expressions.and(
-                partFilter, Expressions.equal(field.name(), partitionData.get(i, Object.class)));
+        Object partitionValue = partitionData.get(i, Object.class);
+        if (Objects.isNull(partitionValue)) {
+          partFilter = Expressions.and(partFilter, Expressions.isNull(field.name()));
+        } else {
+          partFilter = Expressions.and(partFilter, Expressions.equal(field.name(), partitionValue));
+        }
       }
       partitionSetFilter = Expressions.or(partitionSetFilter, partFilter);
     }

diff --git a/core/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java b/core/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
@@ -134,6 +134,17 @@ public void testInPartitionForRemovePartitionField() {
         .commit();
     table.updateSpec().removeField(Expressions.bucket("data", 16)).commit();
 
+    DataFile FILE_DROP_PARTITION_FIELD =
+        DataFiles.builder(table.spec())
+            .withPath("/path/to/data-drop-partition.parquet")
+            .withFileSizeInBytes(10L)
+            .withRecordCount(1L)
+            .build();
+    table
+        .newAppend()
+        .appendFile(FILE_DROP_PARTITION_FIELD) // without partition field
+        .commit();
+
     Iterable<ContentFile<?>> files =
         transform(
             new IcebergFindFiles(table)
@@ -143,6 +154,14 @@ public void testInPartitionForRemovePartitionField() {
                 .entries());
 
     Assert.assertEquals(pathSet(FILE_B, FILE_C), pathSet(files));
+
+    Iterable<ContentFile<?>> files2 =
+        transform(
+            new IcebergFindFiles(table)
+                .inPartitions(table.spec(), StaticDataTask.Row.of(new Object[] {null}))
+                .entries());
+
+    Assert.assertEquals(pathSet(FILE_DROP_PARTITION_FIELD), pathSet(files2));
   }
 
   @Test