Add serialization of ScalarValue::Binary and `ScalarValue::LargeBin…

…ary`, `ScalarValue::Time64`
apache · Sep 19, 2022 · 813b56b · 813b56b
1 parent a6fbf24
commit 813b56b
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 12 deletions.
diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto
@@ -759,6 +759,9 @@ message ScalarValue{
         int64 interval_daytime_value = 25;
         ScalarTimestampValue timestamp_value = 26;
         ScalarDictionaryValue dictionary_value = 27;
+        bytes binary_value = 28;
+        bytes large_binary_value = 29;
+        int64 time64_value = 30;
     }
 }
 
@@ -796,6 +799,9 @@ enum PrimitiveScalarType{
     TIME_MILLISECOND = 22;
     INTERVAL_YEARMONTH = 23;
     INTERVAL_DAYTIME = 24;
+
+    BINARY = 25;
+    LARGE_BINARY = 26;
 }
 
 message ScalarType{

diff --git a/datafusion/proto/src/from_proto.rs b/datafusion/proto/src/from_proto.rs
@@ -218,6 +218,8 @@ impl From<protobuf::PrimitiveScalarType> for DataType {
             protobuf::PrimitiveScalarType::Float64 => DataType::Float64,
             protobuf::PrimitiveScalarType::Utf8 => DataType::Utf8,
             protobuf::PrimitiveScalarType::LargeUtf8 => DataType::LargeUtf8,
+            protobuf::PrimitiveScalarType::Binary => DataType::Binary,
+            protobuf::PrimitiveScalarType::LargeBinary => DataType::LargeBinary,
             protobuf::PrimitiveScalarType::Date32 => DataType::Date32,
             protobuf::PrimitiveScalarType::TimeMicrosecond => {
                 DataType::Time64(TimeUnit::Microsecond)
@@ -643,6 +645,8 @@ impl TryFrom<&protobuf::PrimitiveScalarType> for ScalarValue {
             PrimitiveScalarType::Float64 => Self::Float64(None),
             PrimitiveScalarType::Utf8 => Self::Utf8(None),
             PrimitiveScalarType::LargeUtf8 => Self::LargeUtf8(None),
+            PrimitiveScalarType::Binary => Self::Binary(None),
+            PrimitiveScalarType::LargeBinary => Self::LargeBinary(None),
             PrimitiveScalarType::Date32 => Self::Date32(None),
             PrimitiveScalarType::TimeMicrosecond => {
                 Self::TimestampMicrosecond(None, None)
@@ -749,6 +753,7 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {
                 )
             }
             Value::Date64Value(v) => Self::Date64(Some(*v)),
+            Value::Time64Value(v) => Self::Time64(Some(*v)),
             Value::IntervalYearmonthValue(v) => Self::IntervalYearMonth(Some(*v)),
             Value::IntervalDaytimeValue(v) => Self::IntervalDayTime(Some(*v)),
             Value::TimestampValue(v) => {
@@ -792,6 +797,8 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {
 
                 Self::Dictionary(Box::new(index_type), Box::new(value))
             }
+            Value::BinaryValue(v) => Self::Binary(Some(v.clone())),
+            Value::LargeBinaryValue(v) => Self::LargeBinary(Some(v.clone())),
         })
     }
 }
@@ -1496,6 +1503,8 @@ fn typechecked_scalar_value_conversion(
                     PrimitiveScalarType::IntervalDaytime => {
                         ScalarValue::IntervalDayTime(None)
                     }
+                    PrimitiveScalarType::Binary => ScalarValue::Binary(None),
+                    PrimitiveScalarType::LargeBinary => ScalarValue::LargeBinary(None),
                 };
                 scalar_value
             } else {

diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs
@@ -402,6 +402,10 @@ mod roundtrip_tests {
             ScalarValue::LargeUtf8(Some(String::from("Test Large utf8"))),
             ScalarValue::Date32(Some(0)),
             ScalarValue::Date32(Some(i32::MAX)),
+            ScalarValue::Date32(None),
+            ScalarValue::Time64(Some(0)),
+            ScalarValue::Time64(Some(i64::MAX)),
+            ScalarValue::Time64(None),
             ScalarValue::TimestampNanosecond(Some(0), None),
             ScalarValue::TimestampNanosecond(Some(i64::MAX), None),
             ScalarValue::TimestampNanosecond(Some(0), Some("UTC".to_string())),
@@ -459,14 +463,20 @@ mod roundtrip_tests {
                 Box::new(DataType::Int32),
                 Box::new(ScalarValue::Utf8(None)),
             ),
+            ScalarValue::Binary(Some(b"bar".to_vec())),
+            ScalarValue::Binary(None),
+            ScalarValue::LargeBinary(Some(b"bar".to_vec())),
+            ScalarValue::LargeBinary(None),
         ];
 
         for test_case in should_pass.into_iter() {
             let proto: super::protobuf::ScalarValue = (&test_case)
                 .try_into()
                 .expect("failed conversion to protobuf");
-            let _roundtrip: ScalarValue =
-                (&proto).try_into().expect("failed conversion to protobuf");
+
+            let _roundtrip: ScalarValue = (&proto)
+                .try_into()
+                .expect("failed conversion from protobuf");
         }
     }
 

diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs
@@ -1180,19 +1180,21 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue {
                 value: Some(Value::NullValue(PrimitiveScalarType::Null as i32)),
             },
 
-            datafusion::scalar::ScalarValue::Binary(_) => {
-                // not yet implemented (TODO file ticket)
-                return Err(Error::invalid_scalar_value(val));
+            scalar::ScalarValue::Binary(val) => {
+                create_proto_scalar(val, PrimitiveScalarType::Binary, |s| {
+                    Value::BinaryValue(s.to_owned())
+                })
             }
-
-            datafusion::scalar::ScalarValue::LargeBinary(_) => {
-                // not yet implemented (TODO file ticket)
-                return Err(Error::invalid_scalar_value(val));
+            scalar::ScalarValue::LargeBinary(val) => {
+                create_proto_scalar(val, PrimitiveScalarType::LargeBinary, |s| {
+                    Value::LargeBinaryValue(s.to_owned())
+                })
             }
 
-            datafusion::scalar::ScalarValue::Time64(_) => {
-                // not yet implemented (TODO file ticket)
-                return Err(Error::invalid_scalar_value(val));
+            datafusion::scalar::ScalarValue::Time64(v) => {
+                create_proto_scalar(v, PrimitiveScalarType::TimeNanosecond, |v| {
+                    Value::Time64Value(*v)
+                })
             }
 
             datafusion::scalar::ScalarValue::IntervalMonthDayNano(_) => {