Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use Arc<[Buffer]> instead of raw Vec<Buffer> in GenericByteViewArray for faster slice #6427

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Use Arc<[Buffer]> instead of raw Vec<Buffer> in GenericByteViewArray for faster slice #6427

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18e3113
Use `Arc<[Buffer]>` instead of raw `Vec<Buffer>` in `GenericByteViewA…
ShiKaiWi Sep 20, 2024
33eae4b
add benchmark case about view array slice
ShiKaiWi Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion arrow-array/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ name = "occupancy"
harness = false

[[bench]]
name = "gc_view_types"
name = "view_types"
harness = false

[[bench]]
Expand Down
6 changes: 6 additions & 0 deletions arrow-array/benches/gc_view_types.rs → arrow-array/benches/view_types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ fn criterion_benchmark(c: &mut Criterion) {
black_box(sliced.gc());
});
});

c.bench_function("view types slice", |b| {
b.iter(|| {
black_box(array.slice(0, 100_000 / 2));
});
});
}

criterion_group!(benches, criterion_benchmark);
Expand Down
28 changes: 18 additions & 10 deletions arrow-array/src/array/byte_view_array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ use super::ByteArrayType;
pub struct GenericByteViewArray<T: ByteViewType + ?Sized> {
data_type: DataType,
views: ScalarBuffer<u128>,
buffers: Vec<Buffer>,
buffers: Arc<[Buffer]>,
phantom: PhantomData<T>,
nulls: Option<NullBuffer>,
}
Expand Down Expand Up @@ -178,7 +178,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
Ok(Self {
data_type: T::DATA_TYPE,
views,
buffers,
buffers: buffers.into(),
nulls,
phantom: Default::default(),
})
Expand All @@ -191,14 +191,14 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
/// Safe if [`Self::try_new`] would not error
pub unsafe fn new_unchecked(
views: ScalarBuffer<u128>,
buffers: Vec<Buffer>,
buffers: impl Into<Arc<[Buffer]>>,
nulls: Option<NullBuffer>,
) -> Self {
Self {
data_type: T::DATA_TYPE,
phantom: Default::default(),
views,
buffers,
buffers: buffers.into(),
nulls,
}
}
Expand All @@ -208,7 +208,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
Self {
data_type: T::DATA_TYPE,
views: vec![0; len].into(),
buffers: vec![],
buffers: vec![].into(),
nulls: Some(NullBuffer::new_null(len)),
phantom: Default::default(),
}
Expand All @@ -234,7 +234,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
}

/// Deconstruct this array into its constituent parts
pub fn into_parts(self) -> (ScalarBuffer<u128>, Vec<Buffer>, Option<NullBuffer>) {
pub fn into_parts(self) -> (ScalarBuffer<u128>, Arc<[Buffer]>, Option<NullBuffer>) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is also a breaking change

(self.views, self.buffers, self.nulls)
}

Expand Down Expand Up @@ -516,7 +516,7 @@ impl<T: ByteViewType + ?Sized> From<ArrayData> for GenericByteViewArray<T> {
Self {
data_type: T::DATA_TYPE,
views,
buffers,
buffers: buffers.into(),
nulls: value.nulls().cloned(),
phantom: Default::default(),
}
Expand Down Expand Up @@ -569,12 +569,20 @@ where
}

impl<T: ByteViewType + ?Sized> From<GenericByteViewArray<T>> for ArrayData {
fn from(mut array: GenericByteViewArray<T>) -> Self {
fn from(array: GenericByteViewArray<T>) -> Self {
let len = array.len();
array.buffers.insert(0, array.views.into_inner());
let new_buffers = {
let mut buffers = Vec::with_capacity(array.buffers.len() + 1);
buffers.push(array.views.into_inner());
for buffer in array.buffers.iter() {
buffers.push(buffer.clone());
}
buffers
};

let builder = ArrayDataBuilder::new(T::DATA_TYPE)
.len(len)
.buffers(array.buffers)
.buffers(new_buffers)
.nulls(array.nulls);

unsafe { builder.build_unchecked() }
Expand Down
Loading