Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preallocate buffers for FixedSizeBinary array creation #3793

Merged
merged 4 commits into from
Mar 3, 2023
Merged

Preallocate buffers for FixedSizeBinary array creation #3793

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8c082b2
Preallocate buffers for FixedSizeBinary array creation
maxburke Mar 2, 2023
d94ad69
Merge branch 'master' into fsb-buffer-reserve
maxburke Mar 2, 2023
a2ecdfd
fix build
maxburke Mar 2, 2023
f792e44
code review fixup
maxburke Mar 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions arrow-array/src/array/fixed_size_binary_array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,11 @@ impl FixedSizeBinaryArray {
let mut len = 0;
let mut size = None;
let mut byte = 0;
let mut null_buf = MutableBuffer::from_len_zeroed(0);
let mut buffer = MutableBuffer::from_len_zeroed(0);

let iter_size_hint = iter.size_hint().0;
let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
let mut buffer = MutableBuffer::new(0);

let mut prepend = 0;
iter.try_for_each(|item| -> Result<(), ArrowError> {
// extend null bitmask by one byte per each 8 items
Expand All @@ -163,7 +166,12 @@ impl FixedSizeBinaryArray {
)));
}
} else {
size = Some(slice.len());
let len = slice.len();
size = Some(len);
// Now that we know how large each element is we can reserve
// sufficient capacity in the underlying mutable buffer for
// the data.
buffer.reserve(iter_size_hint * len);
buffer.extend_zeros(slice.len() * prepend);
}
bit_util::set_bit(null_buf.as_slice_mut(), len);
Expand Down Expand Up @@ -234,8 +242,10 @@ impl FixedSizeBinaryArray {
{
let mut len = 0;
let mut byte = 0;
let mut null_buf = MutableBuffer::from_len_zeroed(0);
let mut buffer = MutableBuffer::from_len_zeroed(0);

let iter_size_hint = iter.size_hint().0;
let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8));
let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize));

iter.try_for_each(|item| -> Result<(), ArrowError> {
// extend null bitmask by one byte per each 8 items
Expand Down Expand Up @@ -304,7 +314,9 @@ impl FixedSizeBinaryArray {
{
let mut len = 0;
let mut size = None;
let mut buffer = MutableBuffer::from_len_zeroed(0);
let iter_size_hint = iter.size_hint().0;
let mut buffer = MutableBuffer::new(0);

iter.try_for_each(|item| -> Result<(), ArrowError> {
let slice = item.as_ref();
if let Some(size) = size {
Expand All @@ -316,8 +328,11 @@ impl FixedSizeBinaryArray {
)));
}
} else {
size = Some(slice.len());
let len = slice.len();
size = Some(len);
buffer.reserve(iter_size_hint * len);
}

buffer.extend_from_slice(slice);

len += 1;
Expand Down