Skip to content

Commit

Permalink
[Fix](orc-reader) Fix StringRef nullptr data in orc-reader. (#41073)
Browse files Browse the repository at this point in the history 
## Proposed changes

backport #40857
  • Loading branch information
@kaka11chen
kaka11chen authored Sep 20, 2024
1 parent dbd72de commit 7e0d498
Showing 1 changed file with 28 additions and 14 deletions.
42 changes: 28 additions & 14 deletions be/src/vec/exec/format/orc/vorc_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1140,8 +1140,9 @@ Status OrcReader::_decode_string_non_dict_encoded_column(const std::string& col_
if (cvb->hasNulls) {
for (int i = 0; i < num_values; ++i) {
if (cvb->notNull[i]) {
string_values.emplace_back(cvb->data[i],
trim_right(cvb->data[i], cvb->length[i]));
size_t length = trim_right(cvb->data[i], cvb->length[i]);
string_values.emplace_back((length > 0) ? cvb->data[i] : empty_string.data(),
length);
} else {
// Orc doesn't fill null values in new batch, but the former batch has been release.
// Other types like int/long/timestamp... are flat types without pointer in them,
Expand All @@ -1151,21 +1152,26 @@ Status OrcReader::_decode_string_non_dict_encoded_column(const std::string& col_
}
} else {
for (int i = 0; i < num_values; ++i) {
string_values.emplace_back(cvb->data[i], trim_right(cvb->data[i], cvb->length[i]));
size_t length = trim_right(cvb->data[i], cvb->length[i]);
string_values.emplace_back((length > 0) ? cvb->data[i] : empty_string.data(),
length);
}
}
} else {
if (cvb->hasNulls) {
for (int i = 0; i < num_values; ++i) {
if (cvb->notNull[i]) {
string_values.emplace_back(cvb->data[i], cvb->length[i]);
string_values.emplace_back(
(cvb->length[i] > 0) ? cvb->data[i] : empty_string.data(),
cvb->length[i]);
} else {
string_values.emplace_back(empty_string.data(), 0);
}
}
} else {
for (int i = 0; i < num_values; ++i) {
string_values.emplace_back(cvb->data[i], cvb->length[i]);
string_values.emplace_back(
(cvb->length[i] > 0) ? cvb->data[i] : empty_string.data(), cvb->length[i]);
}
}
}
Expand Down Expand Up @@ -1204,7 +1210,8 @@ Status OrcReader::_decode_string_dict_encoded_column(const std::string& col_name
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
} else {
// Orc doesn't fill null values in new batch, but the former batch has been release.
// Other types like int/long/timestamp... are flat types without pointer in them,
Expand All @@ -1227,7 +1234,8 @@ Status OrcReader::_decode_string_dict_encoded_column(const std::string& col_name
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
}
}
} else {
Expand All @@ -1246,7 +1254,8 @@ Status OrcReader::_decode_string_dict_encoded_column(const std::string& col_name
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
} else {
string_values.emplace_back(EMPTY_STRING_FOR_OVERFLOW, 0);
}
Expand All @@ -1265,7 +1274,8 @@ Status OrcReader::_decode_string_dict_encoded_column(const std::string& col_name
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
}
}
}
Expand Down Expand Up @@ -2068,7 +2078,7 @@ Status OrcReader::on_string_dicts_loaded(
char* val_ptr;
int64_t length;
dict->getValueByIndex(i, val_ptr, length);
StringRef dict_value(val_ptr, length);
StringRef dict_value((length > 0) ? val_ptr : "", length);
if (length > max_value_length) {
max_value_length = length;
}
Expand Down Expand Up @@ -2329,7 +2339,8 @@ MutableColumnPtr OrcReader::_convert_dict_column_to_string_column(
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
} else {
// Orc doesn't fill null values in new batch, but the former batch has been release.
// Other types like int/long/timestamp... are flat types without pointer in them,
Expand All @@ -2347,7 +2358,8 @@ MutableColumnPtr OrcReader::_convert_dict_column_to_string_column(
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
}
}
} else {
Expand All @@ -2362,7 +2374,8 @@ MutableColumnPtr OrcReader::_convert_dict_column_to_string_column(
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
} else {
string_values.emplace_back(EMPTY_STRING_FOR_OVERFLOW, 0);
}
Expand All @@ -2376,7 +2389,8 @@ MutableColumnPtr OrcReader::_convert_dict_column_to_string_column(
if (length > max_value_length) {
max_value_length = length;
}
string_values.emplace_back(val_ptr, length);
string_values.emplace_back((length > 0) ? val_ptr : EMPTY_STRING_FOR_OVERFLOW,
length);
}
}
}
Expand Down

0 comments on commit 7e0d498

Please sign in to comment.