Parse timestamp partition value to micro seconds.

delta-io · Sep 15, 2021 · a4f71c8 · a4f71c8
1 parent 081df68
commit a4f71c8
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 10 deletions.
diff --git a/rust/src/checkpoints.rs b/rust/src/checkpoints.rs
@@ -316,12 +316,9 @@ fn typed_partition_value_from_string(
                 let ts =
                     chrono::naive::NaiveDateTime::parse_from_str(string_value, "%Y-%m-%d %H:%M:%S")
                         .map_err(|_| {
-                            CheckPointWriterError::PartitionValueNotParseable(
-                                string_value.to_owned(),
-                            )
+                            CheckpointError::PartitionValueNotParseable(string_value.to_owned())
                         })?;
-                // Use nanosecond precision for timestamp: https://github.com/delta-io/delta-rs/pull/194
-                Ok(ts.timestamp_nanos().into())
+                Ok((ts.timestamp_millis() * 1000).into())
             }
             s => unimplemented!(
                 "Primitive type {} is not supported for partition column values.",
@@ -475,11 +472,11 @@ mod tests {
         }
 
         for (s, v) in [
-            ("2021-08-08 01:00:01", 1628384401000000000i64),
-            ("1970-01-02 12:59:59", 133199000000000i64),
-            ("1970-01-01 13:00:01", 46801000000000i64),
-            ("1969-12-31 00:00:00", -86400000000000i64),
-            ("1677-09-21 00:12:44", -9223372036000000000i64),
+            ("2021-08-08 01:00:01", 1628384401000000i64),
+            ("1970-01-02 12:59:59", 133199000000i64),
+            ("1970-01-01 13:00:01", 46801000000i64),
+            ("1969-12-31 00:00:00", -86400000000i64),
+            ("1677-09-21 00:12:44", -9223372036000000i64),
         ] {
             let timestamp_value: Value = v.into();
             assert_eq!(

diff --git a/rust/src/delta_arrow.rs b/rust/src/delta_arrow.rs
@@ -85,6 +85,7 @@ impl TryFrom<&schema::SchemaDataType> for ArrowDataType {
                     "float" => Ok(ArrowDataType::Float32),
                     "double" => Ok(ArrowDataType::Float64),
                     "boolean" => Ok(ArrowDataType::Boolean),
+                    // Change to binary type after https://github.com/apache/arrow-rs/pull/702 is released
                     "binary" => Ok(ArrowDataType::Utf8),
                     decimal if DECIMAL_REGEX.is_match(decimal) => {
                         let extract = DECIMAL_REGEX.captures(decimal).ok_or_else(|| {