From d24bae34d731b9f9080b45b99dea9866c8b54333 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 13 Apr 2022 17:48:35 -0700 Subject: [PATCH 1/3] Fix incorrect buffers for UnionArray --- arrow/src/array/data.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arrow/src/array/data.rs b/arrow/src/array/data.rs index 551bf536cb33..043e4af76f88 100644 --- a/arrow/src/array/data.rs +++ b/arrow/src/array/data.rs @@ -219,8 +219,14 @@ pub(crate) fn into_buffers( DataType::Utf8 | DataType::Binary | DataType::LargeUtf8 - | DataType::LargeBinary - | DataType::Union(_, _) => vec![buffer1.into(), buffer2.into()], + | DataType::LargeBinary => vec![buffer1.into(), buffer2.into()], + DataType::Union(_, mode) => { + match mode { + // Based on Union's DataTypeLayout + UnionMode::Sparse => vec![buffer1.into()], + UnionMode::Dense => vec![buffer1.into(), buffer2.into()], + } + } _ => vec![buffer1.into()], } } From ee9ec378649369763bb55715cd8ff2faf595f42e Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Apr 2022 09:30:41 -0700 Subject: [PATCH 2/3] Add test --- arrow/src/array/data.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arrow/src/array/data.rs b/arrow/src/array/data.rs index 043e4af76f88..fe8020a04a18 100644 --- a/arrow/src/array/data.rs +++ b/arrow/src/array/data.rs @@ -2600,4 +2600,21 @@ mod tests { assert_eq!(&struct_array_slice, &cloned); } + + #[test] + fn test_into_buffers() { + let data_types = vec![ + DataType::Union(vec![], UnionMode::Dense), + DataType::Union(vec![], UnionMode::Sparse), + ]; + + for data_type in data_types { + let buffers = new_buffers(&data_type, 0); + let [buffer1, buffer2] = buffers; + let buffers = into_buffers(&data_type, buffer1, buffer2); + + let layout = layout(&data_type); + assert_eq!(buffers.len(), layout.buffers.len()); + } + } } From 9f78ddffc0555d2685f27f641e116df6425aee61 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 14 Apr 2022 10:45:07 -0700 Subject: [PATCH 3/3] Re-enable test_filter_union_array_sparse --- arrow/src/compute/kernels/filter.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arrow/src/compute/kernels/filter.rs b/arrow/src/compute/kernels/filter.rs index 4ab5b180bf5a..df59ba63c79d 100644 --- a/arrow/src/compute/kernels/filter.rs +++ b/arrow/src/compute/kernels/filter.rs @@ -1692,9 +1692,6 @@ mod tests { } #[test] - // Fails when validation enabled - // https://github.com/apache/arrow-rs/issues/1547 - #[cfg(not(feature = "force_validate"))] fn test_filter_union_array_sparse() { let mut builder = UnionBuilder::new_sparse(3); builder.append::("A", 1).unwrap(); @@ -1706,9 +1703,6 @@ mod tests { } #[test] - // Fails when validation enabled - // https://github.com/apache/arrow-rs/issues/1547 - #[cfg(not(feature = "force_validate"))] fn test_filter_union_array_sparse_with_nulls() { let mut builder = UnionBuilder::new_sparse(4); builder.append::("A", 1).unwrap();