From 6b56441fce8e1273e1b14ca4f750e82099507407 Mon Sep 17 00:00:00 2001 From: jayzhan211 Date: Sat, 11 Nov 2023 14:17:02 +0800 Subject: [PATCH] remove general conversion Signed-off-by: jayzhan211 --- datafusion/common/src/scalar.rs | 37 ------------------- .../src/aggregate/array_agg_distinct.rs | 14 +++++-- 2 files changed, 11 insertions(+), 40 deletions(-) diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index 65fd6c18a6b13..e8efac919ffd0 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -2110,43 +2110,6 @@ impl ScalarValue { } } - /// Retrieve ScalarValue for each row in `array` - /// - /// Example - /// ``` - /// use datafusion_common::ScalarValue; - /// use arrow::array::ListArray; - /// use arrow::datatypes::{DataType, Int32Type}; - /// - /// let list_arr = ListArray::from_iter_primitive::(vec![ - /// Some(vec![Some(1), Some(2), Some(3)]), - /// None, - /// Some(vec![Some(4), Some(5)]) - /// ]); - /// - /// let scalar_vec = ScalarValue::convert_array_to_scalar_vec(&list_arr).unwrap(); - /// - /// let expected = vec![ - /// vec![ - /// ScalarValue::Int32(Some(1)), - /// ScalarValue::Int32(Some(2)), - /// ScalarValue::Int32(Some(3)), - /// ], - /// vec![], - /// vec![ScalarValue::Int32(Some(4)), ScalarValue::Int32(Some(5))] - /// ]; - /// - /// assert_eq!(scalar_vec, expected); - /// ``` - pub fn convert_array_to_scalar_vec(array: &dyn Array) -> Result>> { - let data_type = array.data_type().to_owned(); - - match data_type { - DataType::List(_) => Self::convert_list_array_to_scalar_vec(array), - _ => Ok(vec![Self::convert_non_list_array_to_scalars(array)?]), - } - } - // TODO: Support more types after other ScalarValue is wrapped with ArrayRef /// Get raw data (inner array) inside ScalarValue pub fn raw_data(&self) -> Result { diff --git a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs index 4b2f65670832b..f905e63728a04 100644 --- a/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs +++ b/datafusion/physical-expr/src/aggregate/array_agg_distinct.rs @@ -137,9 +137,17 @@ impl Accumulator for DistinctArrayAggAccumulator { assert_eq!(values.len(), 1, "batch input should only include 1 column!"); let array = &values[0]; - let scalars = ScalarValue::convert_array_to_scalar_vec(array)?; - for scalar in scalars { - self.values.extend(scalar) + match array.data_type() { + DataType::List(_) => { + let scalar_vec = ScalarValue::convert_list_array_to_scalar_vec(array)?; + for scalars in scalar_vec { + self.values.extend(scalars); + } + } + _ => { + let scalars = ScalarValue::convert_non_list_array_to_scalars(array)?; + self.values.extend(scalars); + } } Ok(()) }