diff --git a/bench-vortex/src/datasets/struct_list_of_ints.rs b/bench-vortex/src/datasets/struct_list_of_ints.rs index c97cb6cd1a2..5d38d97c2df 100644 --- a/bench-vortex/src/datasets/struct_list_of_ints.rs +++ b/bench-vortex/src/datasets/struct_list_of_ints.rs @@ -71,7 +71,7 @@ impl Dataset for StructListOfInts { .collect::>>()?; Ok(StructArray::try_new( names.clone(), - fields, + fields.into(), chunk_row_count, Validity::NonNullable, )? diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index 73c36405403..92bfbe4b5f8 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -21,7 +21,7 @@ impl CanonicalVTable for FSSTVTable { unsafe { Canonical::VarBinView(VarBinViewArray::new_unchecked( views, - Arc::new([buffer]), + Arc::from([buffer]), array.dtype().clone(), array.codes().validity().clone(), )) diff --git a/encodings/sparse/src/canonical.rs b/encodings/sparse/src/canonical.rs index ed9fd1e76a8..c64b681099a 100644 --- a/encodings/sparse/src/canonical.rs +++ b/encodings/sparse/src/canonical.rs @@ -628,7 +628,7 @@ mod test { let patch_values_b = PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(3)]).into_array(); let patch_values = StructArray::try_new_with_dtype( - vec![patch_values_a, patch_values_b], + vec![patch_values_a, patch_values_b].into(), struct_fields.clone(), 4, Validity::Array( @@ -669,7 +669,7 @@ mod test { })); let expected = StructArray::try_new_with_dtype( - vec![expected_a.into_array(), expected_b.into_array()], + vec![expected_a.into_array(), expected_b.into_array()].into(), struct_fields, len, // NB: patch indices: [0, 1, 7, 8]; patch validity: [Valid, Valid, Valid, Invalid]; ergo 8 is Invalid. @@ -706,7 +706,7 @@ mod test { let patch_values_b = PrimitiveArray::from_option_iter([Some(1i32), Some(2), None, Some(3)]).into_array(); let patch_values = StructArray::try_new_with_dtype( - vec![patch_values_a, patch_values_b], + vec![patch_values_a, patch_values_b].into(), struct_fields.clone(), 4, Validity::Array( @@ -744,7 +744,7 @@ mod test { })); let expected = StructArray::try_new_with_dtype( - vec![expected_a.into_array(), expected_b.into_array()], + vec![expected_a.into_array(), expected_b.into_array()].into(), struct_fields, len, // NB: patch indices: [0, 1, 7, 8]; patch validity: [Valid, Valid, Valid, Invalid]; ergo 0, 1, 7 are valid. diff --git a/fuzz/src/array/filter.rs b/fuzz/src/array/filter.rs index 3225898d19a..fdb3e23ba67 100644 --- a/fuzz/src/array/filter.rs +++ b/fuzz/src/array/filter.rs @@ -92,7 +92,7 @@ pub fn filter_canonical_array(array: &dyn Array, filter: &[bool]) -> VortexResul .collect::>>()?; StructArray::try_new_with_dtype( - filtered_children, + filtered_children.into(), struct_array.struct_fields().clone(), filter.iter().filter(|b| **b).map(|b| *b as usize).sum(), validity, diff --git a/fuzz/src/array/slice.rs b/fuzz/src/array/slice.rs index e90c1171456..0573d775cea 100644 --- a/fuzz/src/array/slice.rs +++ b/fuzz/src/array/slice.rs @@ -58,7 +58,7 @@ pub fn slice_canonical_array( .map(|c| slice_canonical_array(c, start, stop)) .collect::>>()?; StructArray::try_new_with_dtype( - sliced_children, + sliced_children.into(), struct_array.struct_fields().clone(), stop - start, validity, diff --git a/fuzz/src/array/take.rs b/fuzz/src/array/take.rs index 9a29c82e257..6e7ff6fed7d 100644 --- a/fuzz/src/array/take.rs +++ b/fuzz/src/array/take.rs @@ -107,7 +107,7 @@ pub fn take_canonical_array( StructArray::try_new( struct_array.names().clone(), - taken_children, + taken_children.into(), indices_slice_non_opt.len(), validity, ) diff --git a/java/testfiles/src/main.rs b/java/testfiles/src/main.rs index 59bf884aeb7..47e5d5e9f6f 100644 --- a/java/testfiles/src/main.rs +++ b/java/testfiles/src/main.rs @@ -63,7 +63,7 @@ fn main() { // Make the struct array let rows = StructArray::try_new( ["Name", "Salary", "State"].into(), - vec![names, salary, states], + vec![names, salary, states].into(), 10, Validity::NonNullable, ) diff --git a/vortex-array/benches/scalar_at_struct.rs b/vortex-array/benches/scalar_at_struct.rs index 3cff5b7fc3b..906cd877dd3 100644 --- a/vortex-array/benches/scalar_at_struct.rs +++ b/vortex-array/benches/scalar_at_struct.rs @@ -33,7 +33,7 @@ fn scalar_at_struct_simple(bencher: Bencher) { let struct_array = StructArray::try_new( FieldNames::from(["value"]), - vec![field], + vec![field].into(), ARRAY_SIZE, Validity::NonNullable, ) @@ -71,8 +71,13 @@ fn scalar_at_struct_wide(bencher: Bencher) { "field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", ]); - let struct_array = - StructArray::try_new(field_names, fields, ARRAY_SIZE, Validity::NonNullable).unwrap(); + let struct_array = StructArray::try_new( + field_names, + fields.into(), + ARRAY_SIZE, + Validity::NonNullable, + ) + .unwrap(); let indices: Vec = (0..NUM_ACCESSES) .map(|_| rng.random_range(0..ARRAY_SIZE)) diff --git a/vortex-array/benches/take_struct.rs b/vortex-array/benches/take_struct.rs index db8d5680718..5737b020033 100644 --- a/vortex-array/benches/take_struct.rs +++ b/vortex-array/benches/take_struct.rs @@ -34,7 +34,7 @@ fn take_struct_simple(bencher: Bencher) { let struct_array = StructArray::try_new( FieldNames::from(["value"]), - vec![field], + vec![field].into(), ARRAY_SIZE, Validity::NonNullable, ) @@ -70,8 +70,13 @@ fn take_struct_wide(bencher: Bencher, width: usize) { "field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", ]); - let struct_array = - StructArray::try_new(field_names, fields, ARRAY_SIZE, Validity::NonNullable).unwrap(); + let struct_array = StructArray::try_new( + field_names, + fields.into(), + ARRAY_SIZE, + Validity::NonNullable, + ) + .unwrap(); let indices: Buffer = (0..TAKE_SIZE) .map(|_| rng.random_range(0..ARRAY_SIZE) as u64) @@ -98,7 +103,7 @@ fn take_struct_sequential_indices(bencher: Bencher) { let struct_array = StructArray::try_new( FieldNames::from(["value"]), - vec![field], + vec![field].into(), ARRAY_SIZE, Validity::NonNullable, ) diff --git a/vortex-array/src/array/display/mod.rs b/vortex-array/src/array/display/mod.rs index a0d583b2583..644c2cb78cb 100644 --- a/vortex-array/src/array/display/mod.rs +++ b/vortex-array/src/array/display/mod.rs @@ -331,7 +331,7 @@ mod test { fn test_empty_struct() { let s = StructArray::try_new( FieldNames::empty(), - vec![], + vec![].into(), 3, Validity::Array(BoolArray::from_iter([true, false, true]).into_array()), ) diff --git a/vortex-array/src/arrays/arbitrary.rs b/vortex-array/src/arrays/arbitrary.rs index abee85f5a3b..5efd5e86970 100644 --- a/vortex-array/src/arrays/arbitrary.rs +++ b/vortex-array/src/arrays/arbitrary.rs @@ -133,7 +133,7 @@ fn random_array_chunk( .collect::>>()?; Ok(StructArray::try_new( sdt.names().clone(), - children, + children.into(), resolved_len, random_validity(u, *n, resolved_len)?, ) diff --git a/vortex-array/src/arrays/chunked/compute/take.rs b/vortex-array/src/arrays/chunked/compute/take.rs index 06e3327240e..fd02c99893f 100644 --- a/vortex-array/src/arrays/chunked/compute/take.rs +++ b/vortex-array/src/arrays/chunked/compute/take.rs @@ -106,9 +106,13 @@ mod test { #[test] fn test_take_nullability() { - let struct_array = - StructArray::try_new(FieldNames::default(), vec![], 100, Validity::NonNullable) - .unwrap(); + let struct_array = StructArray::try_new( + FieldNames::default(), + vec![].into(), + 100, + Validity::NonNullable, + ) + .unwrap(); let arr = ChunkedArray::from_iter(vec![struct_array.to_array(), struct_array.to_array()]); @@ -120,7 +124,7 @@ mod test { let expect = StructArray::try_new( FieldNames::default(), - vec![], + vec![].into(), 3, Validity::Array(BoolArray::from_iter(vec![true, false, true]).to_array()), ) diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index 9623e536285..bf849d7a0fb 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -134,7 +134,7 @@ fn scalar_at_empty_children_leading() { pub fn pack_nested_structs() { let struct_array = StructArray::try_new( ["a"].into(), - vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()], + vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()].into(), 4, Validity::NonNullable, ) diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index f40ee447b46..b126f757400 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -82,7 +82,7 @@ fn pack_struct_chunks( // SAFETY: field_arrays are built from corresponding chunks of same length, dtypes match by // construction. - unsafe { StructArray::new_unchecked(field_arrays, struct_dtype.clone(), len, validity) } + unsafe { StructArray::new_unchecked(field_arrays.into(), struct_dtype.clone(), len, validity) } } /// Packs [`ListViewArray`]s together into a chunked `ListViewArray`. @@ -184,7 +184,7 @@ mod tests { pub fn pack_nested_structs() { let struct_array = StructArray::try_new( ["a"].into(), - vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()], + vec![VarBinViewArray::from_iter_str(["foo", "bar", "baz", "quak"]).into_array()].into(), 4, Validity::NonNullable, ) diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index bde1d00787e..ccc2fcf835d 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -138,7 +138,12 @@ impl CanonicalVTable for ConstantVTable { // SAFETY: Fields are constructed from the same struct scalar, all have same // length, dtypes match by construction. Canonical::Struct(unsafe { - StructArray::new_unchecked(fields, struct_dtype.clone(), array.len(), validity) + StructArray::new_unchecked( + fields.into(), + struct_dtype.clone(), + array.len(), + validity, + ) }) } DType::List(..) => Canonical::List(constant_canonical_list_array(scalar, array.len())), @@ -199,7 +204,7 @@ fn constant_canonical_byte_view( unsafe { VarBinViewArray::new_unchecked( views, - Arc::from(buffers), + Arc::from(buffers.into_boxed_slice()), dtype.clone(), Validity::from(dtype.nullability()), ) diff --git a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs index 3300db33b1f..1a72d39a9bc 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/nested.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/nested.rs @@ -401,7 +401,7 @@ fn test_fsl_of_struct() { let struct_array = StructArray::try_new( struct_fields.names().clone(), - vec![a_values, b_values], + vec![a_values, b_values].into(), fsl_len * fsl_size as usize, Validity::NonNullable, ) @@ -447,7 +447,7 @@ fn test_fsl_of_nullable_struct() { let struct_validity = Validity::from_iter([true, false, true, true, false, true]); let struct_array = StructArray::try_new( struct_fields.names().clone(), - vec![x_values.into_array(), y_values.into_array()], + vec![x_values.into_array(), y_values.into_array()].into(), fsl_len * fsl_size as usize, struct_validity, ) @@ -482,7 +482,7 @@ fn test_fsl_with_empty_struct() { let struct_array = StructArray::try_new( struct_fields.names().clone(), - vec![], + vec![].into(), fsl_len * fsl_size as usize, Validity::NonNullable, ) @@ -527,7 +527,7 @@ fn test_struct_of_fsl() { let struct_array = StructArray::try_new( struct_fields.names().clone(), - vec![fsl1.into_array(), fsl2.into_array()], + vec![fsl1.into_array(), fsl2.into_array()].into(), 3, Validity::NonNullable, ) diff --git a/vortex-array/src/arrays/fixed_size_list/tests/take.rs b/vortex-array/src/arrays/fixed_size_list/tests/take.rs index d9bf4a0df83..8c588e9a28a 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/take.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/take.rs @@ -210,7 +210,7 @@ fn test_take_nullable_arrays_fsl_specific( builder .append_value( Scalar::list( - DType::Primitive(PType::I32, Nullability::NonNullable), + DType::Primitive(PType::I32, Nullability::NonNullable).into(), scalars, Nullability::NonNullable, ) diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs index 9291e5789d6..8cb95c6857e 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/operations.rs @@ -45,9 +45,10 @@ impl OperationsVTable for FixedSizeListVTable { let children_elements: Vec = (0..list.len()).map(|i| list.scalar_at(i)).collect(); debug_assert_eq!(children_elements.len(), array.list_size() as usize); + let element_dtype = list.dtype().clone().into(); Scalar::fixed_size_list( - list.dtype().clone(), + element_dtype, children_elements, array.dtype().nullability(), ) diff --git a/vortex-array/src/arrays/list/compute/is_constant.rs b/vortex-array/src/arrays/list/compute/is_constant.rs index 96e35d961a2..ddcc190256e 100644 --- a/vortex-array/src/arrays/list/compute/is_constant.rs +++ b/vortex-array/src/arrays/list/compute/is_constant.rs @@ -91,7 +91,7 @@ mod tests { let struct_of_lists = StructArray::try_new( FieldNames::from(["xs"]), - vec![xs.into_array()], + vec![xs.into_array()].into(), 2, Validity::NonNullable, ) diff --git a/vortex-array/src/arrays/listview/conversion.rs b/vortex-array/src/arrays/listview/conversion.rs index 018d323dac6..9cdaa449284 100644 --- a/vortex-array/src/arrays/listview/conversion.rs +++ b/vortex-array/src/arrays/listview/conversion.rs @@ -165,7 +165,7 @@ pub fn recursive_list_from_list_view(array: ArrayRef) -> ArrayRef { if any_changed { StructArray::try_new( struct_array.names().clone(), - converted_fields, + converted_fields.into(), struct_array.len(), struct_array.validity().clone(), ) @@ -464,7 +464,7 @@ mod tests { let struct_array = StructArray::try_new( FieldNames::from(["lists", "values"]), - vec![listview_field, primitive_field], + vec![listview_field, primitive_field].into(), 4, Validity::NonNullable, ) @@ -521,7 +521,7 @@ mod tests { let struct_array = StructArray::try_new( FieldNames::from(["inner_lists"]), - vec![innermost_listview.into_array()], + vec![innermost_listview.into_array()].into(), 2, Validity::NonNullable, ) @@ -559,7 +559,7 @@ mod tests { let struct_array = StructArray::try_new( FieldNames::from(["listview_field", "list_field"]), - vec![listview.into_array(), list.into_array()], + vec![listview.into_array(), list.into_array()].into(), 4, Validity::NonNullable, ) diff --git a/vortex-array/src/arrays/listview/tests/nested.rs b/vortex-array/src/arrays/listview/tests/nested.rs index 21cbc528b07..e0b4e6c17a6 100644 --- a/vortex-array/src/arrays/listview/tests/nested.rs +++ b/vortex-array/src/arrays/listview/tests/nested.rs @@ -303,7 +303,7 @@ fn test_listview_of_struct_with_nulls() { let struct_array = StructArray::try_new( struct_fields.names().clone(), - vec![id_values, value_values], + vec![id_values, value_values].into(), 6, struct_validity, ) diff --git a/vortex-array/src/arrays/struct_/array.rs b/vortex-array/src/arrays/struct_/array.rs index 9e98aa45af2..c2b97df5613 100644 --- a/vortex-array/src/arrays/struct_/array.rs +++ b/vortex-array/src/arrays/struct_/array.rs @@ -3,9 +3,10 @@ use std::fmt::Debug; use std::iter::once; +use std::ops::Index; use std::sync::Arc; -use vortex_dtype::{DType, FieldName, FieldNames, StructFields}; +use vortex_dtype::{DType, FieldName, FieldNames, Nullability, StructFields}; use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_err}; use crate::stats::ArrayStats; @@ -46,7 +47,7 @@ use crate::{Array, ArrayRef, IntoArray}; /// buffer![1i32, 2i32].into_array(), // non-null field a /// buffer![10i32, 20i32].into_array(), // non-null field b /// buffer![100i32, 200i32].into_array(), // non-null field c -/// ], +/// ].into(), /// 2, /// Validity::Array(BoolArray::from_iter([true, false]).into_array()), // row 1 is null /// ).unwrap(); @@ -78,7 +79,7 @@ use crate::{Array, ArrayRef, IntoArray}; /// vec![ /// buffer![1i32, 2i32].into_array(), // first "data" /// buffer![3i32, 4i32].into_array(), // second "data" -/// ], +/// ].into(), /// 2, /// Validity::NonNullable, /// ).unwrap(); @@ -119,7 +120,7 @@ use crate::{Array, ArrayRef, IntoArray}; /// // Create struct array with named fields /// let struct_array = StructArray::try_new( /// FieldNames::from(["id", "score"]), -/// vec![ids.into_array(), names.into_array()], +/// vec![ids.into_array(), names.into_array()].into(), /// 3, /// Validity::NonNullable, /// ).unwrap(); @@ -135,13 +136,13 @@ use crate::{Array, ArrayRef, IntoArray}; pub struct StructArray { pub(super) len: usize, pub(super) dtype: DType, - pub(super) fields: Arc<[ArrayRef]>, + pub(super) fields: StructArrays, pub(super) validity: Validity, pub(super) stats_set: ArrayStats, } impl StructArray { - pub fn fields(&self) -> &Arc<[ArrayRef]> { + pub fn fields(&self) -> &StructArrays { &self.fields } @@ -175,13 +176,13 @@ impl StructArray { /// Create a new `StructArray` with the given length, but without any fields. pub fn new_fieldless_with_len(len: usize) -> Self { - Self::try_new( - FieldNames::default(), - Vec::new(), + Self { len, - Validity::NonNullable, - ) - .vortex_expect("StructArray::new_with_len should not fail") + dtype: DType::Struct(StructFields::empty(), Nullability::NonNullable), + fields: Default::default(), + validity: Validity::NonNullable, + stats_set: Default::default(), + } } /// Creates a new [`StructArray`]. @@ -190,12 +191,7 @@ impl StructArray { /// /// Panics if the provided components do not satisfy the invariants documented /// in [`StructArray::new_unchecked`]. - pub fn new( - names: FieldNames, - fields: impl Into>, - length: usize, - validity: Validity, - ) -> Self { + pub fn new(names: FieldNames, fields: StructArrays, length: usize, validity: Validity) -> Self { Self::try_new(names, fields, length, validity) .vortex_expect("StructArray construction failed") } @@ -210,15 +206,14 @@ impl StructArray { /// [`StructArray::new_unchecked`]. pub fn try_new( names: FieldNames, - fields: impl Into>, + fields: StructArrays, length: usize, validity: Validity, ) -> VortexResult { - let fields = fields.into(); let field_dtypes: Vec<_> = fields.iter().map(|d| d.dtype()).cloned().collect(); let dtype = StructFields::new(names, field_dtypes); - Self::validate(&fields, &dtype, length, &validity)?; + Self::validate(fields.as_ref(), &dtype, length, &validity)?; // SAFETY: validate ensures all invariants are met. Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) @@ -250,15 +245,13 @@ impl StructArray { /// /// - If `validity` is [`Validity::Array`], its length must exactly equal `length`. pub unsafe fn new_unchecked( - fields: impl Into>, + fields: StructArrays, dtype: StructFields, length: usize, validity: Validity, ) -> Self { - let fields = fields.into(); - #[cfg(debug_assertions)] - Self::validate(&fields, &dtype, length, &validity) + Self::validate(fields.as_ref(), &dtype, length, &validity) .vortex_expect("[Debug Assertion]: Invalid `StructArray` parameters"); Self { @@ -324,13 +317,12 @@ impl StructArray { } pub fn try_new_with_dtype( - fields: impl Into>, + fields: StructArrays, dtype: StructFields, length: usize, validity: Validity, ) -> VortexResult { - let fields = fields.into(); - Self::validate(&fields, &dtype, length, &validity)?; + Self::validate(fields.as_ref(), &dtype, length, &validity)?; // SAFETY: validate ensures all invariants are met. Ok(unsafe { Self::new_unchecked(fields, dtype, length, validity) }) @@ -357,7 +349,7 @@ impl StructArray { .map(|f| f.len()) .ok_or_else(|| vortex_err!("StructArray cannot be constructed from an empty slice of arrays because the length is unspecified"))?; - Self::try_new(FieldNames::from_iter(names), fields, len, validity) + Self::try_new(FieldNames::from_iter(names), fields.into(), len, validity) } pub fn try_from_iter, A: IntoArray, T: IntoIterator>( @@ -392,7 +384,7 @@ impl StructArray { StructArray::try_new( FieldNames::from(names.as_slice()), - children, + children.into(), self.len(), self.validity().clone(), ) @@ -402,7 +394,6 @@ impl StructArray { /// If the column does not exist, returns `None`. pub fn remove_column(&mut self, name: impl Into) -> Option { let name = name.into(); - let struct_dtype = self.struct_fields().clone(); let position = struct_dtype @@ -410,17 +401,17 @@ impl StructArray { .iter() .position(|field_name| field_name.as_ref() == name.as_ref())?; - let field = self.fields[position].clone(); - let new_fields: Arc<[ArrayRef]> = self - .fields - .iter() - .enumerate() - .filter(|(i, _)| *i != position) - .map(|(_, f)| f.clone()) - .collect(); - if let Ok(new_dtype) = struct_dtype.without_field(position) { - self.fields = new_fields; + let field = self.fields[position].clone(); + let new_fields = self + .fields + .iter() + .enumerate() + .filter(|(i, _)| *i != position) + .map(|(_, f)| f.clone()) + .collect::>(); + + self.fields = new_fields.into(); self.dtype = DType::Struct(new_dtype, self.dtype.nullability()); return Some(field); } @@ -436,8 +427,85 @@ impl StructArray { let types = struct_dtype.fields().chain(once(array.dtype().clone())); let new_fields = StructFields::new(names.collect(), types.collect()); - let children: Arc<[ArrayRef]> = self.fields.iter().cloned().chain(once(array)).collect(); + let children = self + .fields + .iter() + .cloned() + .chain(once(array)) + .collect::>(); + + Self::try_new_with_dtype(children.into(), new_fields, self.len, self.validity.clone()) + } +} + +#[derive(Clone, Debug, Default)] +pub struct StructArrays(Arc>); + +impl StructArrays { + #[inline] + pub fn len(&self) -> usize { + self.0.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Returns an iterator over the field arrays. + #[inline] + pub fn iter(&self) -> std::slice::Iter<'_, ArrayRef> { + self.0.iter() + } + + /// Returns a reference to the field array at the given index, or `None` if out of bounds. + #[inline] + pub fn get(&self, index: usize) -> Option<&ArrayRef> { + self.0.get(index) + } + + pub fn to_vec(&self) -> Vec { + self.0.to_vec() + } +} + +impl AsRef<[ArrayRef]> for StructArrays { + fn as_ref(&self) -> &[ArrayRef] { + &self.0 + } +} + +impl Index for StructArrays { + type Output = ArrayRef; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl<'a> IntoIterator for &'a StructArrays { + type Item = &'a ArrayRef; + type IntoIter = std::slice::Iter<'a, ArrayRef>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl From> for StructArrays { + fn from(fields: Vec) -> Self { + Self(Arc::new(fields.into_boxed_slice())) + } +} + +impl From> for StructArrays { + fn from(fields: Box<[ArrayRef]>) -> Self { + Self(Arc::new(fields)) + } +} - Self::try_new_with_dtype(children, new_fields, self.len, self.validity.clone()) +impl From>> for StructArrays { + fn from(fields: Arc>) -> Self { + Self(fields) } } diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index 3ecae4f46f2..8053cbbe926 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -37,7 +37,8 @@ impl CastKernel for StructVTable { .iter() .zip_eq(target_sdtype.fields()) .map(|(field, dtype)| cast(field, &dtype)) - .collect::, _>>()?, + .collect::, _>>()? + .into(), array.len(), validity, ) @@ -79,7 +80,7 @@ mod tests { StructArray::try_new( names, - vec![a, b], + vec![a, b].into(), 3, if nullable { Validity::AllValid @@ -96,9 +97,10 @@ mod tests { let x = buffer![1.0f32, 2.0, 3.0].into_array(); let y = buffer![4.0f32, 5.0, 6.0].into_array(); - let inner_struct = StructArray::try_new(inner_names, vec![x, y], 3, Validity::NonNullable) - .unwrap() - .into_array(); + let inner_struct = + StructArray::try_new(inner_names, vec![x, y].into(), 3, Validity::NonNullable) + .unwrap() + .into_array(); // Create outer struct with inner struct as a field let outer_names: FieldNames = ["id", "point"].into(); @@ -108,7 +110,7 @@ mod tests { StructArray::try_new( outer_names, - vec![ids, inner_struct], + vec![ids, inner_struct].into(), 3, Validity::NonNullable, ) @@ -121,14 +123,14 @@ mod tests { let values = buffer![42u8].into_array(); - StructArray::try_new(names, vec![values], 1, Validity::NonNullable).unwrap() + StructArray::try_new(names, vec![values].into(), 1, Validity::NonNullable).unwrap() } #[test] fn cast_nullable_all_invalid() { let empty_struct = StructArray::try_new( FieldNames::from(["a"]), - vec![PrimitiveArray::new::(buffer![], Validity::AllInvalid).to_array()], + vec![PrimitiveArray::new::(buffer![], Validity::AllInvalid).to_array()].into(), 0, Validity::AllInvalid, ) diff --git a/vortex-array/src/arrays/struct_/compute/filter.rs b/vortex-array/src/arrays/struct_/compute/filter.rs index bd81a714346..efddee9ca61 100644 --- a/vortex-array/src/arrays/struct_/compute/filter.rs +++ b/vortex-array/src/arrays/struct_/compute/filter.rs @@ -24,8 +24,13 @@ impl FilterKernel for StructVTable { .map(|a| a.len()) .unwrap_or_else(|| mask.true_count()); - StructArray::try_new_with_dtype(fields, array.struct_fields().clone(), length, validity) - .map(|a| a.into_array()) + StructArray::try_new_with_dtype( + fields.into(), + array.struct_fields().clone(), + length, + validity, + ) + .map(|a| a.into_array()) } } diff --git a/vortex-array/src/arrays/struct_/compute/mod.rs b/vortex-array/src/arrays/struct_/compute/mod.rs index 2cec31846d0..88cc0f6cfe3 100644 --- a/vortex-array/src/arrays/struct_/compute/mod.rs +++ b/vortex-array/src/arrays/struct_/compute/mod.rs @@ -30,8 +30,13 @@ mod tests { #[test] fn filter_empty_struct() { - let struct_arr = - StructArray::try_new(FieldNames::empty(), vec![], 10, Validity::NonNullable).unwrap(); + let struct_arr = StructArray::try_new( + FieldNames::empty(), + vec![].into(), + 10, + Validity::NonNullable, + ) + .unwrap(); let mask = vec![ false, true, false, true, false, true, false, true, false, true, ]; @@ -41,8 +46,13 @@ mod tests { #[test] fn take_empty_struct() { - let struct_arr = - StructArray::try_new(FieldNames::empty(), vec![], 10, Validity::NonNullable).unwrap(); + let struct_arr = StructArray::try_new( + FieldNames::empty(), + vec![].into(), + 10, + Validity::NonNullable, + ) + .unwrap(); let indices = PrimitiveArray::from_option_iter([Some(1), None]); let taken = take(struct_arr.as_ref(), indices.as_ref()).unwrap(); assert_eq!(taken.len(), 2); @@ -86,7 +96,8 @@ mod tests { #[test] fn filter_empty_struct_with_empty_filter() { let struct_arr = - StructArray::try_new(FieldNames::empty(), vec![], 0, Validity::NonNullable).unwrap(); + StructArray::try_new(FieldNames::empty(), vec![].into(), 0, Validity::NonNullable) + .unwrap(); let filtered = filter(struct_arr.as_ref(), &Mask::from_iter::<[bool; 0]>([])).unwrap(); assert_eq!(filtered.len(), 0); } @@ -94,7 +105,7 @@ mod tests { #[test] fn test_mask_empty_struct() { test_mask_conformance( - StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) + StructArray::try_new(FieldNames::empty(), vec![].into(), 5, Validity::NonNullable) .unwrap() .as_ref(), ); @@ -117,7 +128,7 @@ mod tests { vec![ StructArray::try_new( ["left", "right"].into(), - vec![xs.clone(), xs], + vec![xs.clone(), xs].into(), 5, Validity::NonNullable, ) @@ -125,7 +136,8 @@ mod tests { .into_array(), ys, zs, - ], + ] + .into(), 5, Validity::NonNullable, ) @@ -137,7 +149,7 @@ mod tests { #[test] fn test_filter_empty_struct() { test_filter_conformance( - StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) + StructArray::try_new(FieldNames::empty(), vec![].into(), 5, Validity::NonNullable) .unwrap() .as_ref(), ); @@ -160,7 +172,7 @@ mod tests { vec![ StructArray::try_new( ["left", "right"].into(), - vec![xs.clone(), xs], + vec![xs.clone(), xs].into(), 5, Validity::NonNullable, ) @@ -168,7 +180,8 @@ mod tests { .into_array(), ys, zs, - ], + ] + .into(), 5, Validity::NonNullable, ) @@ -179,9 +192,14 @@ mod tests { #[test] fn test_cast_empty_struct() { - let array = StructArray::try_new(FieldNames::default(), vec![], 5, Validity::NonNullable) - .unwrap() - .into_array(); + let array = StructArray::try_new( + FieldNames::default(), + vec![].into(), + 5, + Validity::NonNullable, + ) + .unwrap() + .into_array(); let non_nullable_dtype = DType::Struct( StructFields::new(FieldNames::default(), vec![]), NonNullable, @@ -203,7 +221,8 @@ mod tests { buffer![1u8].into_array(), buffer![1u8].into_array(), buffer![1u8].into_array(), - ], + ] + .into(), 1, Validity::NonNullable, ) @@ -243,7 +262,7 @@ mod tests { vec![ StructArray::try_new( ["left", "right"].into(), - vec![xs.to_array(), xs.to_array()], + vec![xs.to_array(), xs.to_array()].into(), 5, Validity::AllValid, ) @@ -251,7 +270,8 @@ mod tests { .into_array(), ys.into_array(), zs.into_array(), - ], + ] + .into(), 5, Validity::AllValid, ) @@ -319,7 +339,7 @@ mod tests { #[test] fn test_take_empty_struct_conformance() { test_take_conformance( - StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable) + StructArray::try_new(FieldNames::empty(), vec![].into(), 5, Validity::NonNullable) .unwrap() .as_ref(), ); @@ -335,9 +355,14 @@ mod tests { .into_array(); test_take_conformance( - StructArray::try_new(["xs", "ys"].into(), vec![xs, ys], 5, Validity::NonNullable) - .unwrap() - .as_ref(), + StructArray::try_new( + ["xs", "ys"].into(), + vec![xs, ys].into(), + 5, + Validity::NonNullable, + ) + .unwrap() + .as_ref(), ); } @@ -353,7 +378,7 @@ mod tests { test_take_conformance( StructArray::try_new( ["xs", "ys"].into(), - vec![xs.into_array(), ys.into_array()], + vec![xs.into_array(), ys.into_array()].into(), 5, Validity::NonNullable, ) @@ -369,7 +394,7 @@ mod tests { let inner_ys = buffer![100i32, 200, 300, 400, 500].into_array(); let inner_struct = StructArray::try_new( ["x", "y"].into(), - vec![inner_xs, inner_ys], + vec![inner_xs, inner_ys].into(), 5, Validity::NonNullable, ) @@ -381,7 +406,7 @@ mod tests { test_take_conformance( StructArray::try_new( ["inner", "z"].into(), - vec![inner_struct, outer_zs], + vec![inner_struct, outer_zs].into(), 5, Validity::NonNullable, ) @@ -396,9 +421,14 @@ mod tests { let ys = VarBinArray::from_iter(["hello"].map(Some), DType::Utf8(NonNullable)).into_array(); test_take_conformance( - StructArray::try_new(["xs", "ys"].into(), vec![xs, ys], 1, Validity::NonNullable) - .unwrap() - .as_ref(), + StructArray::try_new( + ["xs", "ys"].into(), + vec![xs, ys].into(), + 1, + Validity::NonNullable, + ) + .unwrap() + .as_ref(), ); } @@ -416,7 +446,7 @@ mod tests { test_take_conformance( StructArray::try_new( ["xs", "ys", "zs"].into(), - vec![xs, ys, zs], + vec![xs, ys, zs].into(), 100, Validity::NonNullable, ) @@ -436,7 +466,7 @@ mod tests { ); StructArray::try_new( ["xs", "ys"].into(), - vec![xs.into_array(), ys.into_array()], + vec![xs.into_array(), ys.into_array()].into(), 5, Validity::NonNullable, ) @@ -450,17 +480,17 @@ mod tests { ); StructArray::try_new( ["xs", "ys"].into(), - vec![xs.into_array(), ys.into_array()], + vec![xs.into_array(), ys.into_array()].into(), 5, Validity::NonNullable, ) .unwrap() })] // Additional test cases - #[case::empty_struct(StructArray::try_new(FieldNames::empty(), vec![], 5, Validity::NonNullable).unwrap())] + #[case::empty_struct(StructArray::try_new(FieldNames::empty(), vec![].into(), 5, Validity::NonNullable).unwrap())] #[case::single_field({ let xs = buffer![42i64].into_array(); - StructArray::try_new(["xs"].into(), vec![xs], 1, Validity::NonNullable).unwrap() + StructArray::try_new(["xs"].into(), vec![xs].into(), 1, Validity::NonNullable).unwrap() })] #[case::large_struct({ let xs = buffer![0..100i64].into_array(); @@ -468,7 +498,7 @@ mod tests { (0..100).map(|i| format!("value_{i}")).map(Some), DType::Utf8(NonNullable), ).into_array(); - StructArray::try_new(["xs", "ys"].into(), vec![xs, ys], 100, Validity::NonNullable).unwrap() + StructArray::try_new(["xs", "ys"].into(), vec![xs, ys].into(), 100, Validity::NonNullable).unwrap() })] fn test_struct_consistency(#[case] array: StructArray) { test_array_consistency(array.as_ref()); diff --git a/vortex-array/src/arrays/struct_/compute/take.rs b/vortex-array/src/arrays/struct_/compute/take.rs index ee94694a7e2..78e142eef45 100644 --- a/vortex-array/src/arrays/struct_/compute/take.rs +++ b/vortex-array/src/arrays/struct_/compute/take.rs @@ -34,7 +34,8 @@ impl TakeKernel for StructVTable { .fields() .iter() .map(|field| compute::take(field, inner_indices)) - .collect::, _>>()?, + .collect::, _>>()? + .into(), array.struct_fields().clone(), indices.len(), array.validity().take(indices)?, diff --git a/vortex-array/src/arrays/struct_/compute/zip.rs b/vortex-array/src/arrays/struct_/compute/zip.rs index 77c9ff74f54..0585a10467b 100644 --- a/vortex-array/src/arrays/struct_/compute/zip.rs +++ b/vortex-array/src/arrays/struct_/compute/zip.rs @@ -58,8 +58,13 @@ impl ZipKernel for StructVTable { }; Ok(Some( - StructArray::try_new(if_true.names().clone(), fields, if_true.len(), validity)? - .to_array(), + StructArray::try_new( + if_true.names().clone(), + fields.into(), + if_true.len(), + validity, + )? + .to_array(), )) } } @@ -81,7 +86,7 @@ mod tests { // Both structs have Validity::Array let if_true = StructArray::new( FieldNames::from_iter(["field"]), - vec![PrimitiveArray::from_iter([1, 2, 3, 4]).into_array()], + vec![PrimitiveArray::from_iter([1, 2, 3, 4]).into_array()].into(), 4, Validity::from_iter([true, false, true, false]), ) @@ -89,7 +94,7 @@ mod tests { let if_false = StructArray::new( FieldNames::from_iter(["field"]), - vec![PrimitiveArray::from_iter([10, 20, 30, 40]).into_array()], + vec![PrimitiveArray::from_iter([10, 20, 30, 40]).into_array()].into(), 4, Validity::from_iter([false, true, false, true]), ) @@ -118,7 +123,7 @@ mod tests { fn test_validity_zip_allvalid_and_array() { let if_true = StructArray::new( FieldNames::from_iter(["a"]), - vec![PrimitiveArray::from_iter([1, 2, 3, 4]).into_array()], + vec![PrimitiveArray::from_iter([1, 2, 3, 4]).into_array()].into(), 4, Validity::AllValid, ) @@ -126,7 +131,7 @@ mod tests { let if_false = StructArray::new( FieldNames::from_iter(["a"]), - vec![PrimitiveArray::from_iter([10, 20, 30, 40]).into_array()], + vec![PrimitiveArray::from_iter([10, 20, 30, 40]).into_array()].into(), 4, Validity::from_iter([false, false, true, true]), ) diff --git a/vortex-array/src/arrays/struct_/tests.rs b/vortex-array/src/arrays/struct_/tests.rs index c03fe5a725c..ff911fa2e0d 100644 --- a/vortex-array/src/arrays/struct_/tests.rs +++ b/vortex-array/src/arrays/struct_/tests.rs @@ -22,7 +22,7 @@ fn test_project() { let struct_a = StructArray::try_new( FieldNames::from(["xs", "ys", "zs"]), - vec![xs.into_array(), ys.into_array(), zs.into_array()], + vec![xs.into_array(), ys.into_array(), zs.into_array()].into(), 5, Validity::NonNullable, ) @@ -55,7 +55,7 @@ fn test_remove_column() { let mut struct_a = StructArray::try_new( FieldNames::from(["xs", "ys"]), - vec![xs.into_array(), ys.into_array()], + vec![xs.into_array(), ys.into_array()].into(), 5, Validity::NonNullable, ) @@ -98,7 +98,7 @@ fn test_duplicate_field_names() { // Create struct with duplicate field names - "value" appears twice let struct_array = StructArray::try_new( FieldNames::from(["value", "other", "value"]), - vec![field1, field2, field3], + vec![field1, field2, field3].into(), 3, Validity::NonNullable, ) @@ -130,7 +130,7 @@ fn test_duplicate_field_names() { fn test_uncompressed_size_in_bytes() { let struct_array = StructArray::new( FieldNames::from(["integers"]), - vec![ConstantArray::new(5, 1000).into_array()], + vec![ConstantArray::new(5, 1000).into_array()].into(), 1000, Validity::NonNullable, ); @@ -143,3 +143,106 @@ fn test_uncompressed_size_in_bytes() { assert_eq!(canonical_size, 2); assert_eq!(uncompressed_size, Some(4000)); } + +#[test] +fn test_struct_arrays_iterator() { + // Create a struct with multiple fields + let field1 = buffer![1i32, 2, 3].into_array(); + let field2 = buffer![10u64, 20, 30].into_array(); + let field3 = buffer![100i16, 200, 300].into_array(); + + let struct_array = StructArray::try_new( + FieldNames::from(["a", "b", "c"]), + vec![field1, field2, field3].into(), + 3, + Validity::NonNullable, + ) + .unwrap(); + + // Test IntoIterator for &StructArrays + let fields = struct_array.fields(); + let mut iter = fields.into_iter(); + + // Verify we can iterate over all fields + let first = iter.next().unwrap(); + assert_eq!(first.to_primitive().as_slice::(), [1i32, 2, 3]); + + let second = iter.next().unwrap(); + assert_eq!(second.to_primitive().as_slice::(), [10u64, 20, 30]); + + let third = iter.next().unwrap(); + assert_eq!(third.to_primitive().as_slice::(), [100i16, 200, 300]); + + assert!(iter.next().is_none()); +} + +#[test] +fn test_struct_arrays_for_loop() { + // Test that we can use StructArrays in a for loop + let field1 = buffer![1i32, 2].into_array(); + let field2 = buffer![10i32, 20].into_array(); + + let struct_array = StructArray::try_new( + FieldNames::from(["x", "y"]), + vec![field1, field2].into(), + 2, + Validity::NonNullable, + ) + .unwrap(); + + let mut count = 0; + for field in struct_array.fields() { + assert_eq!(field.len(), 2); + count += 1; + } + assert_eq!(count, 2); +} + +#[test] +fn test_struct_arrays_index() { + // Test Index trait implementation + let field1 = buffer![1i32, 2, 3].into_array(); + let field2 = buffer![10i32, 20, 30].into_array(); + + let struct_array = StructArray::try_new( + FieldNames::from(["a", "b"]), + vec![field1, field2].into(), + 3, + Validity::NonNullable, + ) + .unwrap(); + + let fields = struct_array.fields(); + + // Direct indexing should work + assert_eq!(fields[0].to_primitive().as_slice::(), [1i32, 2, 3]); + assert_eq!(fields[1].to_primitive().as_slice::(), [10i32, 20, 30]); +} + +#[test] +fn test_struct_arrays_collect() { + // Test that we can collect fields into a Vec + let field1 = buffer![1i32, 2].into_array(); + let field2 = buffer![10i32, 20].into_array(); + let field3 = buffer![100i32, 200].into_array(); + + let struct_array = StructArray::try_new( + FieldNames::from(["a", "b", "c"]), + vec![field1, field2, field3].into(), + 2, + Validity::NonNullable, + ) + .unwrap(); + + // Collect all fields + let collected: Vec<_> = struct_array.fields().into_iter().collect(); + assert_eq!(collected.len(), 3); + + // Verify dtypes + for field in collected { + assert_eq!( + field.dtype(), + &DType::Primitive(PType::I32, Nullability::NonNullable) + ); + } +} diff --git a/vortex-array/src/arrays/struct_/vtable/operations.rs b/vortex-array/src/arrays/struct_/vtable/operations.rs index 7996f60b11b..87fd7791894 100644 --- a/vortex-array/src/arrays/struct_/vtable/operations.rs +++ b/vortex-array/src/arrays/struct_/vtable/operations.rs @@ -24,7 +24,7 @@ impl OperationsVTable for StructVTable { // - Validity length matches array length (both sliced to same range) unsafe { StructArray::new_unchecked( - fields, + fields.into(), array.struct_fields().clone(), range.len(), array.validity().slice(range), diff --git a/vortex-array/src/arrays/struct_/vtable/pipeline.rs b/vortex-array/src/arrays/struct_/vtable/pipeline.rs index f2b4d42c9c1..e0a7842ead2 100644 --- a/vortex-array/src/arrays/struct_/vtable/pipeline.rs +++ b/vortex-array/src/arrays/struct_/vtable/pipeline.rs @@ -179,7 +179,7 @@ impl BatchExecution for StructExecution { .vortex_expect("Struct dtype must have fields") .names() .clone(), - children, + children.into(), self.len, // self.validity, Validity::AllValid, diff --git a/vortex-array/src/arrays/struct_/vtable/serde.rs b/vortex-array/src/arrays/struct_/vtable/serde.rs index 39ab29a6e99..9f41724cb19 100644 --- a/vortex-array/src/arrays/struct_/vtable/serde.rs +++ b/vortex-array/src/arrays/struct_/vtable/serde.rs @@ -55,6 +55,6 @@ impl SerdeVTable for StructVTable { }) .try_collect()?; - StructArray::try_new_with_dtype(children, struct_dtype.clone(), len, validity) + StructArray::try_new_with_dtype(children.into(), struct_dtype.clone(), len, validity) } } diff --git a/vortex-array/src/arrays/validation_tests.rs b/vortex-array/src/arrays/validation_tests.rs index 1eecc512421..2d391cde9fc 100644 --- a/vortex-array/src/arrays/validation_tests.rs +++ b/vortex-array/src/arrays/validation_tests.rs @@ -161,7 +161,7 @@ mod tests { let views = Buffer::from_iter([view1, view2]); let result = VarBinViewArray::try_new( views, - Arc::new([]), + Arc::from([]), DType::Utf8(Nullability::NonNullable), Validity::NonNullable, ); @@ -178,7 +178,7 @@ mod tests { let view = BinaryView::make_view(data, 1, 0); // Buffer index 1. let views = Buffer::from_iter([view]); - let buffers = Arc::new([ByteBuffer::from(data.to_vec())]); + let buffers = Arc::from([ByteBuffer::from(data.to_vec())]); let result = VarBinViewArray::try_new( views, @@ -196,7 +196,7 @@ mod tests { let field2 = buffer![4.0f64, 5.0, 6.0].into_array(); let fields = vec![field1, field2]; let names = ["a", "b"]; - let result = StructArray::try_new(names.into(), fields, 3, Validity::NonNullable); + let result = StructArray::try_new(names.into(), fields.into(), 3, Validity::NonNullable); assert!(result.is_ok()); } @@ -207,7 +207,7 @@ mod tests { let field2 = buffer![4.0f64, 5.0].into_array(); // Length 2, not 3. let fields = vec![field1, field2]; let names = ["a", "b"]; - let result = StructArray::try_new(names.into(), fields, 3, Validity::NonNullable); + let result = StructArray::try_new(names.into(), fields.into(), 3, Validity::NonNullable); assert!(result.is_err()); } } diff --git a/vortex-array/src/arrays/varbinview/vtable/serde.rs b/vortex-array/src/arrays/varbinview/vtable/serde.rs index 7ed7d3115ff..66f1d51adf6 100644 --- a/vortex-array/src/arrays/varbinview/vtable/serde.rs +++ b/vortex-array/src/arrays/varbinview/vtable/serde.rs @@ -51,6 +51,11 @@ impl SerdeVTable for VarBinViewVTable { vortex_bail!("Expected 0 or 1 children, got {}", children.len()); }; - VarBinViewArray::try_new(views, Arc::from(buffers), dtype.clone(), validity) + VarBinViewArray::try_new( + views, + Arc::from(buffers.into_boxed_slice()), + dtype.clone(), + validity, + ) } } diff --git a/vortex-array/src/arrow/compute/to_arrow/canonical.rs b/vortex-array/src/arrow/compute/to_arrow/canonical.rs index 8b44a95771d..70a29dbd06f 100644 --- a/vortex-array/src/arrow/compute/to_arrow/canonical.rs +++ b/vortex-array/src/arrow/compute/to_arrow/canonical.rs @@ -722,7 +722,7 @@ mod tests { let struct_a = StructArray::try_new( FieldNames::from(["xs"]), - vec![xs.into_array()], + vec![xs.into_array()].into(), 5, Validity::AllValid, ) @@ -741,7 +741,7 @@ mod tests { let struct_a = StructArray::try_new( FieldNames::from(["xs"]), - vec![xs.into_array()], + vec![xs.into_array()].into(), 5, Validity::AllValid, ) @@ -759,7 +759,7 @@ mod tests { let struct_a = StructArray::try_new( FieldNames::from(["xs"]), - vec![xs.into_array()], + vec![xs.into_array()].into(), 5, Validity::AllValid, ) diff --git a/vortex-array/src/arrow/convert.rs b/vortex-array/src/arrow/convert.rs index 6fd4b9c8e26..f195be3ff2b 100644 --- a/vortex-array/src/arrow/convert.rs +++ b/vortex-array/src/arrow/convert.rs @@ -239,7 +239,8 @@ impl FromArrowArray<&GenericByteViewArray> for ArrayRef { .data_buffers() .iter() .map(|b| ByteBuffer::from_arrow_buffer(b.clone(), Alignment::of::())) - .collect::>(), + .collect::>() + .into_boxed_slice(), ), dtype, nulls(value.nulls(), nullable), @@ -322,7 +323,8 @@ impl FromArrowArray<&ArrowStructArray> for ArrayRef { Self::from_arrow(c.as_ref(), field.is_nullable()) } }) - .collect::>(), + .collect::>() + .into(), value.len(), nulls(value.nulls(), nullable), ) diff --git a/vortex-array/src/arrow/record_batch.rs b/vortex-array/src/arrow/record_batch.rs index b458831808a..61a94e39ac6 100644 --- a/vortex-array/src/arrow/record_batch.rs +++ b/vortex-array/src/arrow/record_batch.rs @@ -58,7 +58,7 @@ mod tests { ); xs.append_scalar(&Scalar::list( - xs.element_dtype().clone(), + xs.element_dtype().clone().into(), vec![1i32.into(), 2i32.into(), 3i32.into()], Nullability::Nullable, )) diff --git a/vortex-array/src/builders/struct_.rs b/vortex-array/src/builders/struct_.rs index f1a90eb144f..a63f929f2c6 100644 --- a/vortex-array/src/builders/struct_.rs +++ b/vortex-array/src/builders/struct_.rs @@ -96,7 +96,7 @@ impl StructBuilder { let validity = self.nulls.finish_with_nullability(self.dtype.nullability()); - StructArray::try_new_with_dtype(fields, self.struct_fields().clone(), len, validity) + StructArray::try_new_with_dtype(fields.into(), self.struct_fields().clone(), len, validity) .vortex_expect("Fields must all have same length.") } diff --git a/vortex-array/src/builders/varbinview.rs b/vortex-array/src/builders/varbinview.rs index 140a3157918..c00ea65b0ea 100644 --- a/vortex-array/src/builders/varbinview.rs +++ b/vortex-array/src/builders/varbinview.rs @@ -403,7 +403,7 @@ impl CompletedBuffers { fn finish(self) -> Arc<[ByteBuffer]> { match self { - Self::Default(buffers) => Arc::from(buffers), + Self::Default(buffers) => Arc::from(buffers.into_boxed_slice()), Self::Deduplicated(buffers) => buffers.finish(), } } @@ -454,7 +454,7 @@ impl DeduplicatedBuffers { } fn finish(self) -> Arc<[ByteBuffer]> { - Arc::from(self.buffers) + Arc::from(self.buffers.into_boxed_slice()) } } @@ -574,7 +574,7 @@ impl BuffersWithOffsets { .map(|(b, _)| b.vortex_expect("already checked for rewrite")) .collect(); Self::AllKept { - buffers: Arc::from(buffers), + buffers: Arc::from(buffers.into_boxed_slice()), offsets: None, } } @@ -594,7 +594,7 @@ impl BuffersWithOffsets { }) .collect(); Self::AllKept { - buffers: Arc::from(buffers), + buffers: Arc::from(buffers.into_boxed_slice()), offsets: Some(offsets), } } diff --git a/vortex-array/src/compute/conformance/cast.rs b/vortex-array/src/compute/conformance/cast.rs index 26d7959f5d9..762aca3d1d2 100644 --- a/vortex-array/src/compute/conformance/cast.rs +++ b/vortex-array/src/compute/conformance/cast.rs @@ -616,9 +616,13 @@ mod tests { ) .into_array(); - let array = - StructArray::try_new(names, vec![a, b], 3, crate::validity::Validity::NonNullable) - .unwrap(); + let array = StructArray::try_new( + names, + vec![a, b].into(), + 3, + crate::validity::Validity::NonNullable, + ) + .unwrap(); test_cast_conformance(array.as_ref()); } diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index 4ab3183a97d..7ba7d78f6a0 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -250,7 +250,7 @@ pub struct ArrayParts { flatbuffer: FlatBuffer, // The location of the current fb::ArrayNode flatbuffer_loc: usize, - buffers: Arc<[ByteBuffer]>, + buffers: Arc>, } impl Debug for ArrayParts { @@ -434,7 +434,7 @@ impl TryFrom for ArrayParts { let fb_root = fb_array.root().vortex_expect("Array must have a root node"); let mut offset = 0; - let buffers: Arc<[ByteBuffer]> = fb_array + let buffers = fb_array .buffers() .unwrap_or_default() .iter() @@ -452,7 +452,9 @@ impl TryFrom for ArrayParts { offset += buffer_len; buffer }) - .collect(); + .collect::>() + .into_boxed_slice() + .into(); Ok(ArrayParts { flatbuffer: fb_buffer.clone(), diff --git a/vortex-btrblocks/src/lib.rs b/vortex-btrblocks/src/lib.rs index 0d3a916cb04..157d3c40c66 100644 --- a/vortex-btrblocks/src/lib.rs +++ b/vortex-btrblocks/src/lib.rs @@ -392,7 +392,7 @@ impl BtrBlocksCompressor { Ok(StructArray::try_new( struct_array.names().clone(), - fields, + fields.into(), struct_array.len(), struct_array.validity().clone(), )? diff --git a/vortex-datafusion/examples/vortex_table.rs b/vortex-datafusion/examples/vortex_table.rs index cb48e415da3..4ad3e218528 100644 --- a/vortex-datafusion/examples/vortex_table.rs +++ b/vortex-datafusion/examples/vortex_table.rs @@ -34,7 +34,7 @@ async fn main() -> anyhow::Result<()> { let st = StructArray::try_new( ["strings", "numbers"].into(), - vec![strings, numbers], + vec![strings, numbers].into(), 8, Validity::NonNullable, )?; diff --git a/vortex-datafusion/src/convert/exprs.rs b/vortex-datafusion/src/convert/exprs.rs index e1072bfa1d5..2c855fd1cdb 100644 --- a/vortex-datafusion/src/convert/exprs.rs +++ b/vortex-datafusion/src/convert/exprs.rs @@ -95,7 +95,7 @@ impl TryFromDataFusion for ExprRef { .try_collect()?; let list = Scalar::list( - list_elements[0].dtype().clone(), + list_elements[0].dtype().clone().into(), list_elements, Nullability::Nullable, ); diff --git a/vortex-datafusion/src/persistent/mod.rs b/vortex-datafusion/src/persistent/mod.rs index d3f9e886562..6364cf35468 100644 --- a/vortex-datafusion/src/persistent/mod.rs +++ b/vortex-datafusion/src/persistent/mod.rs @@ -79,7 +79,7 @@ mod tests { let st = StructArray::try_new( ["strings", "numbers"].into(), - vec![strings, numbers], + vec![strings, numbers].into(), 8, Validity::NonNullable, )?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 9c5830781a8..8a80aa48c0b 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -126,7 +126,7 @@ mod tests { .into_array(); let arr = StructArray::try_new( ["col1", "col2"].into(), - vec![prim, strings], + vec![prim, strings].into(), 10, Validity::from(BooleanBuffer::from_iter([ true, true, true, false, false, false, true, true, true, true, @@ -167,7 +167,7 @@ mod tests { .into_array(); let arr = StructArray::try_new( ["col1", "col2"].into(), - vec![prim, strings], + vec![prim, strings].into(), 10, Validity::from(BooleanBuffer::from_iter([ true, true, true, false, false, false, true, true, true, true, diff --git a/vortex-expr/src/exprs/get_item.rs b/vortex-expr/src/exprs/get_item.rs index 6a1474cff84..2b0564a5f59 100644 --- a/vortex-expr/src/exprs/get_item.rs +++ b/vortex-expr/src/exprs/get_item.rs @@ -230,7 +230,7 @@ mod tests { fn get_nullable_field() { let st = StructArray::try_new( FieldNames::from(["a"]), - vec![buffer![1i32].into_array()], + vec![buffer![1i32].into_array()].into(), 1, Validity::AllInvalid, ) diff --git a/vortex-expr/src/exprs/merge.rs b/vortex-expr/src/exprs/merge.rs index eec4b42dfc5..49df4a195cb 100644 --- a/vortex-expr/src/exprs/merge.rs +++ b/vortex-expr/src/exprs/merge.rs @@ -109,7 +109,7 @@ impl VTable for MergeVTable { // TODO(DK): When children are allowed to be nullable, this needs to change. let validity = Validity::NonNullable; Ok( - StructArray::try_new(FieldNames::from(field_names), arrays, len, validity)? + StructArray::try_new(FieldNames::from(field_names), arrays.into(), len, validity)? .into_array(), ) } diff --git a/vortex-expr/src/exprs/pack.rs b/vortex-expr/src/exprs/pack.rs index dce5e7c05ec..c42bd4361bb 100644 --- a/vortex-expr/src/exprs/pack.rs +++ b/vortex-expr/src/exprs/pack.rs @@ -117,7 +117,10 @@ impl VTable for PackVTable { Nullability::NonNullable => Validity::NonNullable, Nullability::Nullable => Validity::AllValid, }; - Ok(StructArray::try_new(expr.names.clone(), value_arrays, len, validity)?.into_array()) + Ok( + StructArray::try_new(expr.names.clone(), value_arrays.into(), len, validity)? + .into_array(), + ) } fn return_dtype(expr: &Self::Expr, scope: &DType) -> VortexResult { diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 53140c76e24..39da4ec9785 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -262,7 +262,7 @@ mod tests { let ages = PrimitiveArray::new(buffer![30u8, 25u8, 35u8], Validity::NonNullable); let struct_array = StructArray::try_new( ["name", "age"].into(), - vec![names.into_array(), ages.into_array()], + vec![names.into_array(), ages.into_array()].into(), 3, Validity::NonNullable, ) diff --git a/vortex-file/src/pruning.rs b/vortex-file/src/pruning.rs index 005ff7ad707..5f95afe9fef 100644 --- a/vortex-file/src/pruning.rs +++ b/vortex-file/src/pruning.rs @@ -19,8 +19,13 @@ pub fn extract_relevant_file_stats_as_struct_row( struct_dtype: &StructFields, ) -> VortexResult> { if access.is_empty() { - return StructArray::try_new(FieldNames::default(), vec![], 1, Validity::NonNullable) - .map(|s| Some(s.to_array())); + return StructArray::try_new( + FieldNames::default(), + vec![].into(), + 1, + Validity::NonNullable, + ) + .map(|s| Some(s.to_array())); } let mut columns: Vec<(FieldName, ArrayRef)> = Vec::with_capacity(access.len() * 2); diff --git a/vortex-file/src/tests.rs b/vortex-file/src/tests.rs index 1e6eef05e49..882180079dc 100644 --- a/vortex-file/src/tests.rs +++ b/vortex-file/src/tests.rs @@ -335,7 +335,7 @@ async fn write_chunked() { .into_array(); let st = StructArray::try_new( ["strings", "numbers"].into(), - vec![strings_chunked, numbers_chunked], + vec![strings_chunked, numbers_chunked].into(), 16, Validity::NonNullable, ) @@ -408,7 +408,7 @@ async fn filter_string() { PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]).into_array(); let st = StructArray::try_new( ["name", "age"].into(), - vec![names_orig, ages_orig], + vec![names_orig, ages_orig].into(), 5, Validity::NonNullable, ) @@ -458,7 +458,7 @@ async fn filter_or() { let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]); let st = StructArray::try_new( ["name", "age"].into(), - vec![names.into_array(), ages.into_array()], + vec![names.into_array(), ages.into_array()].into(), 5, Validity::NonNullable, ) @@ -520,7 +520,7 @@ async fn filter_and() { let ages = PrimitiveArray::from_option_iter([Some(25), Some(31), None, Some(57), None]); let st = StructArray::try_new( ["name", "age"].into(), - vec![names.into_array(), ages.into_array()], + vec![names.into_array(), ages.into_array()].into(), 5, Validity::NonNullable, ) @@ -1115,7 +1115,7 @@ async fn write_nullable_top_level_struct() { let array = StructArray::try_new( ["age"].into(), - vec![ages.into_array()], + vec![ages.into_array()].into(), 5, Validity::AllValid, ) @@ -1163,7 +1163,7 @@ async fn write_nullable_nested_struct() -> VortexResult<()> { let array = StructArray::try_new( ["struct"].into(), - vec![struct_.into_array()], + vec![struct_.into_array()].into(), 3, Validity::NonNullable, )? diff --git a/vortex-layout/src/layouts/compact.rs b/vortex-layout/src/layouts/compact.rs index 40f20bba4e2..4a063921b7a 100644 --- a/vortex-layout/src/layouts/compact.rs +++ b/vortex-layout/src/layouts/compact.rs @@ -126,7 +126,7 @@ impl CompactCompressor { StructArray::try_new( struct_array.names().clone(), - fields, + fields.into(), struct_array.len(), struct_array.validity().clone(), )? @@ -221,7 +221,7 @@ mod tests { let n_rows = columns[0].len(); let struct_array = StructArray::try_new( field_names.clone().into(), - columns.clone(), + columns.clone().into(), n_rows, Validity::NonNullable, ) diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 3e94da639a0..bf593635610 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -288,13 +288,14 @@ mod tests { vec![ StructArray::try_new( FieldNames::from([FieldName::from("one"), FieldName::from("two")]), - vec![array.clone(), array], + vec![array.clone(), array].into(), 9, Validity::NonNullable, ) .unwrap() .into_array(), - ], + ] + .into(), 9, Validity::NonNullable, ) diff --git a/vortex-layout/src/layouts/flat/writer.rs b/vortex-layout/src/layouts/flat/writer.rs index bdb5f04eca7..1c21a9b4232 100644 --- a/vortex-layout/src/layouts/flat/writer.rs +++ b/vortex-layout/src/layouts/flat/writer.rs @@ -276,7 +276,8 @@ mod tests { vec![ buffer![1_u64, 2].into_array(), buffer![3_u64, 4].into_array(), - ], + ] + .into(), 2, validity, ) diff --git a/vortex-layout/src/layouts/partitioned.rs b/vortex-layout/src/layouts/partitioned.rs index 1cdc7b6e670..a9950dbfad3 100644 --- a/vortex-layout/src/layouts/partitioned.rs +++ b/vortex-layout/src/layouts/partitioned.rs @@ -75,7 +75,7 @@ impl PartitionedExprEval

for PartitionedExpr

{ let root_scope = StructArray::try_new( self.partition_names.clone(), - field_arrays, + field_arrays.into(), mask.len(), Validity::NonNullable, )? @@ -111,7 +111,7 @@ impl PartitionedExprEval

for PartitionedExpr

{ let root_scope = StructArray::try_new( self.partition_names.clone(), - field_arrays, + field_arrays.into(), mask.true_count(), Validity::NonNullable, )? diff --git a/vortex-layout/src/layouts/struct_/writer.rs b/vortex-layout/src/layouts/struct_/writer.rs index bfaf5a58e40..6c8afae364a 100644 --- a/vortex-layout/src/layouts/struct_/writer.rs +++ b/vortex-layout/src/layouts/struct_/writer.rs @@ -214,7 +214,7 @@ mod tests { segments, StructArray::try_new( ["a"].into(), - vec![buffer![1, 2, 3].into_array()], + vec![buffer![1, 2, 3].into_array()].into(), 3, Validity::Array(BoolArray::from_iter(vec![true, true, false]).into_array()), ) @@ -243,12 +243,22 @@ mod tests { ctx, segments, ChunkedArray::from_iter([ - StructArray::try_new(FieldNames::default(), vec![], 3, Validity::NonNullable) - .unwrap() - .into_array(), - StructArray::try_new(FieldNames::default(), vec![], 5, Validity::NonNullable) - .unwrap() - .into_array(), + StructArray::try_new( + FieldNames::default(), + vec![].into(), + 3, + Validity::NonNullable, + ) + .unwrap() + .into_array(), + StructArray::try_new( + FieldNames::default(), + vec![].into(), + 5, + Validity::NonNullable, + ) + .unwrap() + .into_array(), ]) .into_array() .to_array_stream() diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 8946f8733b0..61690d3c1e6 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -232,8 +232,13 @@ impl StatsAccumulator { } Some(ZoneMap { - array: StructArray::try_new(names.into(), fields, self.length, Validity::NonNullable) - .vortex_expect("Failed to create zone map"), + array: StructArray::try_new( + names.into(), + fields.into(), + self.length, + Validity::NonNullable, + ) + .vortex_expect("Failed to create zone map"), stats: stats.into(), }) } diff --git a/vortex-scalar/src/list.rs b/vortex-scalar/src/list.rs index 19df27543ee..008117deab7 100644 --- a/vortex-scalar/src/list.rs +++ b/vortex-scalar/src/list.rs @@ -201,13 +201,11 @@ enum ListKind { /// Helper functions to create a [`ListScalar`] as a [`Scalar`]. impl Scalar { fn create_list( - element_dtype: impl Into>, + element_dtype: Arc, children: Vec, nullability: Nullability, list_kind: ListKind, ) -> Self { - let element_dtype = element_dtype.into(); - let children: Arc<[ScalarValue]> = children .into_iter() .map(|child| { @@ -241,7 +239,7 @@ impl Scalar { /// Panics if any child scalar has a different type than the element type, or if there are too /// many children. pub fn list( - element_dtype: impl Into>, + element_dtype: Arc, children: Vec, nullability: Nullability, ) -> Self { @@ -260,7 +258,7 @@ impl Scalar { /// Panics if any child scalar has a different type than the element type, or if there are too /// many children. pub fn fixed_size_list( - element_dtype: impl Into>, + element_dtype: Arc, children: Vec, nullability: Nullability, ) -> Self {