diff --git a/python/src/schema.rs b/python/src/schema.rs index 77e5f0d4da..bd35e9f0d4 100644 --- a/python/src/schema.rs +++ b/python/src/schema.rs @@ -66,6 +66,9 @@ fn schema_type_to_python(schema_type: SchemaDataType, py: Python) -> PyResult Err(PyErr::new::( + "Encountered schema field not meant to be written", + )), } } diff --git a/rust/src/schema/arrow_convert.rs b/rust/src/schema/arrow_convert.rs index 2b37b05c4a..abcb1dd049 100644 --- a/rust/src/schema/arrow_convert.rs +++ b/rust/src/schema/arrow_convert.rs @@ -128,6 +128,9 @@ impl TryFrom<&schema::SchemaDataType> for ArrowDataType { ))), } } + schema::SchemaDataType::timestamp(_) => { + Ok(ArrowDataType::Timestamp(TimeUnit::Microsecond, None)) + } schema::SchemaDataType::r#struct(s) => Ok(ArrowDataType::Struct( s.get_fields() .iter() @@ -248,6 +251,14 @@ impl TryFrom<&ArrowDataType> for schema::SchemaDataType { { Ok(schema::SchemaDataType::primitive("timestamp".to_string())) } + ArrowDataType::Timestamp(TimeUnit::Nanosecond, None) => { + Ok(schema::SchemaDataType::timestamp(true)) + } + ArrowDataType::Timestamp(TimeUnit::Nanosecond, Some(tz)) + if tz.eq_ignore_ascii_case("utc") => + { + Ok(schema::SchemaDataType::timestamp(true)) + } ArrowDataType::Struct(fields) => { let converted_fields: Result, _> = fields .iter() @@ -830,6 +841,15 @@ mod tests { ); } + #[test] + fn test_delta_from_arrow_timestamp_nano_type() { + let timestamp_field = ArrowDataType::Timestamp(TimeUnit::Nanosecond, None); + assert_eq!( + >::try_from(×tamp_field).unwrap(), + crate::SchemaDataType::timestamp(true) + ); + } + #[test] fn test_delta_from_arrow_timestamp_type_with_tz() { let timestamp_field = diff --git a/rust/src/schema/mod.rs b/rust/src/schema/mod.rs index a853725fc6..8a227932dc 100644 --- a/rust/src/schema/mod.rs +++ b/rust/src/schema/mod.rs @@ -298,6 +298,10 @@ pub enum SchemaDataType { /// * timestamp: Microsecond precision timestamp without a timezone /// * decimal: Signed decimal number with fixed precision (maximum number of digits) and scale (number of digits on right side of dot), where the precision and scale can be up to 38 primitive(String), + /// Variant for timestamps that specifies whether timestamps must be converted from nanoseconds + /// This is only for an initial convert to delta call, after initial parquet is converted it would + /// go back to primitive usage above, so this type should never be written back out + timestamp(bool), /// Variant representing a struct. r#struct(SchemaTypeStruct), /// Variant representing an array.