Skip to content

[Enhancement] Improve the performance of query with IN predicate #3694

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 9 commits into from
Jun 4, 2020
Merged

[Enhancement] Improve the performance of query with IN predicate #3694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f37b29e
first
morningman-cmy May 25, 2020
e736343
test
morningman-cmy May 25, 2020
981e37e
change config name
morningman-cmy May 26, 2020
bc06670
add session variables
morningman May 26, 2020
f49b469
add doc
morningman May 26, 2020
89e2fa7
fix comment
morningman May 26, 2020
d6872a9
fix bug
morningman-cmy May 27, 2020
e6e3e9d
change name
morningman May 27, 2020
6466650
fix bug
morningman May 27, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions be/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ namespace config {
CONF_mInt32(doris_scanner_row_num, "16384");
// number of max scan keys
CONF_mInt32(doris_max_scan_key_num, "1024");
// the max number of push down values of a single column.
// if exceed, no conditions will be pushed down for that column.
CONF_mInt32(max_pushdown_conditions_per_column, "1024");
// return_row / total_row
CONF_mInt32(doris_max_pushdown_conjuncts_return_rate, "90");
// (Advanced) Maximum size of per-query receive-side buffer
Expand Down
12 changes: 6 additions & 6 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ class OlapScanKeys {
_is_convertible(true) {}

template<class T>
Status extend_scan_key(ColumnValueRange<T>& range);
Status extend_scan_key(ColumnValueRange<T>& range, int32_t max_scan_key_num);

Status get_key_range(std::vector<std::unique_ptr<OlapScanRange>>* key_range);

Expand Down Expand Up @@ -615,7 +615,7 @@ bool ColumnValueRange<T>::has_intersection(ColumnValueRange<T>& range) {
}

template<class T>
Status OlapScanKeys::extend_scan_key(ColumnValueRange<T>& range) {
Status OlapScanKeys::extend_scan_key(ColumnValueRange<T>& range, int32_t max_scan_key_num) {
using namespace std;
typedef typename set<T>::const_iterator const_iterator_type;

Expand All @@ -636,8 +636,8 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange<T>& range) {
bool has_converted = false;

if (range.is_fixed_value_range()) {
if ((_begin_scan_keys.empty() && range.get_fixed_value_size() > config::doris_max_scan_key_num)
|| range.get_fixed_value_size() * _begin_scan_keys.size() > config::doris_max_scan_key_num) {
if ((_begin_scan_keys.empty() && range.get_fixed_value_size() > max_scan_key_num)
|| range.get_fixed_value_size() * _begin_scan_keys.size() > max_scan_key_num) {
if (range.is_range_value_convertible()) {
range.convert_to_range_value();
} else {
Expand All @@ -647,7 +647,7 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange<T>& range) {
} else {
if (range.is_fixed_value_convertible() && _is_convertible) {
if (_begin_scan_keys.empty()) {
if (range.get_convertible_fixed_value_size() < config::doris_max_scan_key_num) {
if (range.get_convertible_fixed_value_size() < max_scan_key_num) {
if (range.is_low_value_mininum() && range.is_high_value_maximum()) {
has_converted = true;
}
Expand All @@ -656,7 +656,7 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange<T>& range) {
}
} else {
if (range.get_convertible_fixed_value_size() * _begin_scan_keys.size()
< config::doris_max_scan_key_num) {
< max_scan_key_num) {
if (range.is_low_value_mininum() && range.is_high_value_maximum()) {
has_converted = true;
}
Expand Down
Loading