add parquet2 ci task

delta-io · Aug 28, 2022 · 3dc6132 · 3dc6132
1 parent 215ac78
commit 3dc6132
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 8 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -86,3 +86,17 @@ jobs:
       - name: Run tests
         run: |
           cargo test --features s3,datafusion-ext
+
+  paruqet2_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install minimal stable with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: default
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v1
+      - name: Run tests
+        run: cargo test --no-default-features --features=parquet2 --features=arrow2
diff --git a/rust/src/action/parquet2_read/mod.rs b/rust/src/action/parquet2_read/mod.rs
@@ -808,7 +808,7 @@ mod tests {
             }
             match &actions[1] {
                 Action::metaData(meta_data) => {
-                    assert_eq!(meta_data.id, "e3501f3e-ca63-4521-84f2-5901b5b66ac1");
+                    assert_eq!(meta_data.id, "94ba8468-c676-4468-b326-adde3ab9dcd2");
                     assert_eq!(meta_data.name, None);
                     assert_eq!(meta_data.description, None);
                     assert_eq!(
@@ -820,17 +820,17 @@ mod tests {
                         r#"{"type":"struct","fields":[{"name":"id","type":"integer","nullable":true,"metadata":{}},{"name":"color","type":"string","nullable":true,"metadata":{}}]}"#
                     );
                     assert_eq!(meta_data.partition_columns, vec!["color"]);
-                    assert_eq!(meta_data.created_time, Some(1655607917641));
+                    assert_eq!(meta_data.created_time, Some(1661662807027));
                     assert_eq!(meta_data.configuration, HashMap::new());
                 }
                 _ => panic!("expect txn action, got: {:?}", &actions[1]),
             }
 
             match &actions[2] {
                 Action::add(add_action) => {
-                    assert_eq!(add_action.path, "9160473b-59b0-4d51-b442-e495f1d9965f");
+                    assert_eq!(add_action.path, "f62d8868-d952-4f9d-8bb2-fd4e011ebf36");
                     assert_eq!(add_action.size, 100);
-                    assert_eq!(add_action.modification_time, 1655607917660);
+                    assert_eq!(add_action.modification_time, 1661662807080);
                     assert_eq!(add_action.partition_values.len(), 1);
                     assert_eq!(
                         add_action.partition_values.get("color").unwrap(),
@@ -844,9 +844,9 @@ mod tests {
             }
             match &actions[3] {
                 Action::add(add_action) => {
-                    assert_eq!(add_action.path, "4cf2c035-9fa7-4d9a-b09f-e13aa7aceb3d");
+                    assert_eq!(add_action.path, "8ac7d8e1-daab-48ef-9d05-ec22fb4b0d2f");
                     assert_eq!(add_action.size, 100);
-                    assert_eq!(add_action.modification_time, 1655607917674);
+                    assert_eq!(add_action.modification_time, 1661662807097);
                     assert_eq!(add_action.partition_values.len(), 1);
                     assert_eq!(add_action.partition_values.get("color").unwrap(), &None);
                     assert_eq!(add_action.data_change, false);

diff --git a/rust/src/action/parquet2_read/primitive.rs b/rust/src/action/parquet2_read/primitive.rs
@@ -173,7 +173,13 @@ where
                 set_fn(ActType::try_mut_from_action(a)?, value);
             }
         }
-        _ => todo!(),
+        _ => {
+            return Err(ParseError::InvalidAction(format!(
+                "unsupported page encoding type for primitive column: {:?}",
+                page.encoding()
+            )));
+        }
     }
+
     Ok(())
 }
diff --git a/rust/src/action/parquet2_read/string.rs b/rust/src/action/parquet2_read/string.rs
@@ -299,7 +299,7 @@ where
             return Err(ParseError::InvalidAction(format!(
                 "unsupported page encoding type for string column: {:?}",
                 page.encoding()
-            )))
+            )));
         }
     }
 

diff --git a/rust/src/action/parquet2_read/validity.rs b/rust/src/action/parquet2_read/validity.rs
@@ -101,13 +101,15 @@ impl<'a> Iterator for ValidityRepeatedRowIndexIter<'a> {
                         }
                     } else {
                         if self.repeat_count >= 1 {
+                            // current row is None, emit previous row
                             let row_idx = self.row_idx;
                             let item_count = self.repeat_count;
                             self.row_idx += 1;
                             // set to 0 becauze def_lvl not at max def level
                             self.repeat_count = 0;
                             return Some(Ok((row_idx, item_count)));
                         } else {
+                            // both previous and current row are None, proceed to the next row
                             self.row_idx += 1;
                             continue;
                         }
@@ -122,6 +124,7 @@ impl<'a> Iterator for ValidityRepeatedRowIndexIter<'a> {
             }
         }
 
+        // end of iteration, emit the last row
         if self.repeat_count >= 1 {
             let item_count = self.repeat_count;
             // set repeat count to 0 so we can end the iteration
-Original file line number
+Diff line change
@@ Expand Up / @@ -299,7 +299,7 @@ where @@
                 return Err(ParseError::InvalidAction(format!(
                     "unsupported page encoding type for string column: {:?}",
                     page.encoding()
-                )))
+                )));
             }
         }
@@ Expand Down @@