Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[BugFix]streamvbyte decode should use RawVectorPad16 to correctly use avx #13062

Merged
merged 4 commits into from
Nov 11, 2022
Merged

[BugFix]streamvbyte decode should use RawVectorPad16 to correctly use avx #13062

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9955930
streamvbyte decode should use RawVectorPad16 to correctly use avx
fzhedu Nov 7, 2022
6506389
fix build bug
fzhedu Nov 7, 2022
7159470
change log level from warn to info
fzhedu Nov 8, 2022
e4115d4
change log level from warn to debug
fzhedu Nov 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions be/src/serde/column_array_serde.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,19 @@ const uint8_t* read_raw(const uint8_t* buff, void* target, size_t size) {
return buff + size;
}

inline size_t upper_int32(size_t size) {
return (3 + size) / 4.0;
}

template <bool sorted_32ints>
uint8_t* encode_integers(const void* data, size_t size, uint8_t* buff, int encode_level) {
uint64_t encode_size = 0;
if (sorted_32ints) { // only support sorted 32-bit integers
encode_size = streamvbyte_delta_encode(reinterpret_cast<const uint32_t*>(data), (3 + size) * 1.0 / 4.0,
encode_size = streamvbyte_delta_encode(reinterpret_cast<const uint32_t*>(data), upper_int32(size),
buff + sizeof(uint64_t), 0);
} else {
encode_size = streamvbyte_encode(reinterpret_cast<const uint32_t*>(data), (3 + size) * 1.0 / 4.0,
buff + sizeof(uint64_t));
encode_size =
streamvbyte_encode(reinterpret_cast<const uint32_t*>(data), upper_int32(size), buff + sizeof(uint64_t));
}
buff = write_little_endian_64(encode_size, buff);

Expand All @@ -82,9 +86,9 @@ const uint8_t* decode_integers(const uint8_t* buff, void* target, size_t size) {
buff = read_little_endian_64(buff, &encode_size);
uint64_t decode_size = 0;
if (sorted_32ints) {
decode_size = streamvbyte_delta_decode(buff, (uint32_t*)target, (3 + size) * 1.0 / 4.0, 0);
decode_size = streamvbyte_delta_decode(buff, (uint32_t*)target, upper_int32(size), 0);
} else {
decode_size = streamvbyte_decode(buff, (uint32_t*)target, (3 + size) * 1.0 / 4.0);
decode_size = streamvbyte_decode(buff, (uint32_t*)target, upper_int32(size));
}
if (encode_size != decode_size) {
throw std::runtime_error(fmt::format(
Expand Down Expand Up @@ -133,7 +137,7 @@ class FixedLengthColumnSerde {
uint32_t size = sizeof(T) * column.size();
if ((encode_level & 2) && size >= ENCODE_SIZE_LIMIT) {
return sizeof(uint32_t) + sizeof(uint64_t) +
std::max((int64_t)size, (int64_t)streamvbyte_max_compressedbytes((size + 3) / 4.0));
std::max((int64_t)size, (int64_t)streamvbyte_max_compressedbytes(upper_int32(size)));
} else {
return sizeof(uint32_t) + size;
}
Expand All @@ -160,14 +164,15 @@ class FixedLengthColumnSerde {
uint32_t size = 0;
buff = read_little_endian_32(buff, &size);
std::vector<T>& data = column->get_data();
raw::make_room(&data, size / sizeof(T));
if ((encode_level & 2) && size >= ENCODE_SIZE_LIMIT) {
raw::make_room_pad16(&data, size / sizeof(T));
if (sizeof(T) == 4 && sorted) { // only support sorted 32-bit integers
buff = decode_integers<true>(buff, data.data(), size);
} else {
buff = decode_integers<false>(buff, data.data(), size);
}
} else {
raw::make_room(&data, size / sizeof(T));
buff = read_raw(buff, data.data(), size);
}
return buff;
Expand All @@ -184,7 +189,7 @@ class BinaryColumnSerde {
int64_t offsets_size = offsets.size() * sizeof(typename vectorized::BinaryColumnBase<T>::Offset);
if ((encode_level & 2) && offsets_size >= ENCODE_SIZE_LIMIT) {
res += sizeof(uint64_t) +
std::max((int64_t)offsets_size, (int64_t)streamvbyte_max_compressedbytes((offsets_size + 3) / 4.0));
std::max((int64_t)offsets_size, (int64_t)streamvbyte_max_compressedbytes(upper_int32(offsets_size)));
} else {
res += offsets_size;
}
Expand Down Expand Up @@ -254,14 +259,17 @@ class BinaryColumnSerde {
} else {
buff = read_little_endian_64(buff, &offsets_size);
}
raw::make_room(&column->get_offset(), offsets_size / sizeof(typename vectorized::BinaryColumnBase<T>::Offset));
if ((encode_level & 2) && offsets_size >= ENCODE_SIZE_LIMIT) {
raw::make_room_pad16(&column->get_offset(),
offsets_size / sizeof(typename vectorized::BinaryColumnBase<T>::Offset));
if (sizeof(T) == 4) { // only support sorted 32-bit integers
buff = decode_integers<true>(buff, column->get_offset().data(), offsets_size);
} else {
buff = decode_integers<false>(buff, column->get_offset().data(), offsets_size);
}
} else {
raw::make_room(&column->get_offset(),
offsets_size / sizeof(typename vectorized::BinaryColumnBase<T>::Offset));
buff = read_raw(buff, column->get_offset().data(), offsets_size);
}
return buff;
Expand Down
7 changes: 7 additions & 0 deletions be/src/util/raw_container.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,13 @@ inline void make_room(std::vector<T>* v, size_t n) {
v->swap(reinterpret_cast<std::vector<T>&>(rv));
}

template <class T>
inline void make_room_pad16(std::vector<T>* v, size_t n) {
RawVectorPad16<T> rv;
rv.resize(n);
v->swap(reinterpret_cast<std::vector<T>&>(rv));
}

inline void make_room(std::string* s, size_t n) {
RawStringPad16 rs;
rs.resize(n);
Expand Down