Skip to content

[refactor](predicate) Simplify ValueRange #58832

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel39 merged 2 commits into from
Dec 9, 2025
Merged

[refactor](predicate) Simplify ValueRange #58832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
694fea2
[refactor](predicate) Simplify ValueRange
Gabriel39 Dec 9, 2025
86f9845
update
Gabriel39 Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 2 additions & 208 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,9 @@ class ColumnValueRange {

ColumnValueRange();

ColumnValueRange(std::string col_name);

ColumnValueRange(std::string col_name, const CppType& min, const CppType& max,
bool contain_null);

ColumnValueRange(std::string col_name, int precision, int scale);

ColumnValueRange(std::string col_name, bool is_nullable_col, int precision, int scale);

ColumnValueRange(std::string col_name, const CppType& min, const CppType& max,
Expand All @@ -131,8 +127,6 @@ class ColumnValueRange {

bool is_range_value_convertible() const;

void convert_to_fixed_value();

void convert_to_range_value();

bool convert_to_avg_range_value(std::vector<OlapTuple>& begin_scan_keys,
Expand All @@ -145,8 +139,6 @@ class ColumnValueRange {

constexpr bool is_reject_split_type() const { return _is_reject_split_type; }

bool has_intersection(ColumnValueRange<primitive_type>& range);

void intersection(ColumnValueRange<primitive_type>& range);

void set_empty_value_range() {
Expand All @@ -172,12 +164,8 @@ class ColumnValueRange {

bool is_low_value_minimum() const { return Compare::equal(_low_value, TYPE_MIN); }

bool is_low_value_maximum() const { return Compare::equal(_low_value, TYPE_MAX); }

bool is_high_value_maximum() const { return Compare::equal(_high_value, TYPE_MAX); }

bool is_high_value_minimum() const { return Compare::equal(_high_value, TYPE_MIN); }

bool is_begin_include() const { return _low_op == FILTER_LARGER_OR_EQUAL; }

bool is_end_include() const { return _high_op == FILTER_LESS_OR_EQUAL; }
Expand All @@ -186,99 +174,10 @@ class ColumnValueRange {

const std::string& column_name() const { return _column_name; }

bool is_nullable_col() const { return _is_nullable_col; }

bool contain_null() const { return _contain_null; }

size_t get_fixed_value_size() const { return _fixed_values.size(); }

void to_olap_filter(std::vector<FilterOlapParam<TCondition>>& filters) const {
if (is_fixed_value_range()) {
// 1. convert to in filter condition
to_in_condition(filters, true);
} else if (Compare::less(_low_value, _high_value)) {
// 2. convert to min max filter condition
TCondition null_pred;
if (Compare::equal(TYPE_MAX, _high_value) && _high_op == FILTER_LESS_OR_EQUAL &&
Compare::equal(TYPE_MIN, _low_value) && _low_op == FILTER_LARGER_OR_EQUAL &&
_is_nullable_col && !contain_null()) {
null_pred.__set_column_name(_column_name);
null_pred.__set_condition_op("is");
null_pred.condition_values.emplace_back("not null");
}

if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter,
_predicate_always_true_rows_counter);
return;
}

TCondition low;
if (Compare::not_equal(TYPE_MIN, _low_value) || FILTER_LARGER_OR_EQUAL != _low_op) {
low.__set_column_name(_column_name);
low.__set_condition_op((_low_op == FILTER_LARGER_OR_EQUAL ? ">=" : ">>"));
low.condition_values.push_back(
cast_to_string<primitive_type, CppType>(_low_value, _scale));
}

if (low.condition_values.size() != 0) {
filters.emplace_back(
_column_name, low, _runtime_filter_id, _predicate_filtered_rows_counter,
_predicate_input_rows_counter, _predicate_always_true_rows_counter);
}

TCondition high;
if (Compare::not_equal(TYPE_MAX, _high_value) || FILTER_LESS_OR_EQUAL != _high_op) {
high.__set_column_name(_column_name);
high.__set_condition_op((_high_op == FILTER_LESS_OR_EQUAL ? "<=" : "<<"));
high.condition_values.push_back(
cast_to_string<primitive_type, CppType>(_high_value, _scale));
}

if (high.condition_values.size() != 0) {
filters.emplace_back(
_column_name, high, _runtime_filter_id, _predicate_filtered_rows_counter,
_predicate_input_rows_counter, _predicate_always_true_rows_counter);
}
} else {
// 3. convert to is null and is not null filter condition
TCondition null_pred;
if (Compare::equal(TYPE_MAX, _low_value) && Compare::equal(TYPE_MIN, _high_value) &&
_is_nullable_col && contain_null()) {
null_pred.__set_column_name(_column_name);
null_pred.__set_condition_op("is");
null_pred.condition_values.emplace_back("null");
}

if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred, _runtime_filter_id,
_predicate_filtered_rows_counter,
_predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
}
}

void to_in_condition(std::vector<FilterOlapParam<TCondition>>& filters,
bool is_in = true) const {
TCondition condition;
condition.__set_column_name(_column_name);
condition.__set_condition_op(is_in ? "*=" : "!*=");

for (const auto& value : _fixed_values) {
condition.condition_values.push_back(
cast_to_string<primitive_type, CppType>(value, _scale));
}

if (condition.condition_values.size() != 0) {
filters.emplace_back(_column_name, condition, _runtime_filter_id,
_predicate_filtered_rows_counter, _predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
}

void set_whole_value_range() {
_fixed_values.clear();
_low_value = TYPE_MIN;
Expand Down Expand Up @@ -355,14 +254,8 @@ class ColumnValueRange {
static ColumnValueRange<primitive_type> create_empty_column_value_range(bool is_nullable_col,
int precision,
int scale) {
return ColumnValueRange<primitive_type>::create_empty_column_value_range(
"", is_nullable_col, precision, scale);
}

static ColumnValueRange<primitive_type> create_empty_column_value_range(
const std::string& col_name, bool is_nullable_col, int precision, int scale) {
return ColumnValueRange<primitive_type>(col_name, TYPE_MAX, TYPE_MIN, is_nullable_col,
false, precision, scale);
return ColumnValueRange<primitive_type>("", TYPE_MAX, TYPE_MIN, is_nullable_col, false,
precision, scale);
}

protected:
Expand Down Expand Up @@ -457,23 +350,8 @@ class OlapScanKeys {
return _begin_scan_keys.size();
}

void set_begin_include(bool begin_include) { _begin_include = begin_include; }

bool begin_include() const { return _begin_include; }

void set_end_include(bool end_include) { _end_include = end_include; }

bool end_include() const { return _end_include; }

void set_is_convertible(bool is_convertible) { _is_convertible = is_convertible; }

// now, only use in UT
static std::string to_print_key(const OlapTuple& scan_keys) {
std::stringstream sstream;
sstream << scan_keys;
return sstream.str();
}

private:
std::vector<OlapTuple> _begin_scan_keys;
std::vector<OlapTuple> _end_scan_keys;
Expand Down Expand Up @@ -508,10 +386,6 @@ template <PrimitiveType primitive_type>
ColumnValueRange<primitive_type>::ColumnValueRange()
: _column_type(INVALID_TYPE), _precision(-1), _scale(-1) {}

template <PrimitiveType primitive_type>
ColumnValueRange<primitive_type>::ColumnValueRange(std::string col_name)
: ColumnValueRange(std::move(col_name), TYPE_MIN, TYPE_MAX, true) {}

template <PrimitiveType primitive_type>
ColumnValueRange<primitive_type>::ColumnValueRange(std::string col_name, const CppType& min,
const CppType& max, bool contain_null)
Expand Down Expand Up @@ -541,10 +415,6 @@ ColumnValueRange<primitive_type>::ColumnValueRange(std::string col_name, const C
_precision(precision),
_scale(scale) {}

template <PrimitiveType primitive_type>
ColumnValueRange<primitive_type>::ColumnValueRange(std::string col_name, int precision, int scale)
: ColumnValueRange(std::move(col_name), TYPE_MIN, TYPE_MAX, true, true, precision, scale) {}

template <PrimitiveType primitive_type>
ColumnValueRange<primitive_type>::ColumnValueRange(std::string col_name, bool is_nullable_col,
int precision, int scale)
Expand Down Expand Up @@ -973,77 +843,6 @@ void ColumnValueRange<primitive_type>::intersection(ColumnValueRange<primitive_t
}
}

template <PrimitiveType primitive_type>
bool ColumnValueRange<primitive_type>::has_intersection(ColumnValueRange<primitive_type>& range) {
// 1. return false if column type not match
if (_column_type != range._column_type) {
return false;
}

// 2. return false if any range is empty
if (is_empty_value_range() || range.is_empty_value_range()) {
return false;
}

// 3.1 return false if two int fixedRange has no intersection
if (is_fixed_value_range() && range.is_fixed_value_range()) {
SetType result_values;
set_intersection(_fixed_values.begin(), _fixed_values.end(), range._fixed_values.begin(),
range._fixed_values.end(),
std::inserter(result_values, result_values.begin()));

if (result_values.size() != 0) {
return true;
} else {
return false;
}
} // 3.2
else if (is_fixed_value_range() && !range.is_fixed_value_range()) {
IteratorType iter = _fixed_values.begin();

while (iter != _fixed_values.end()) {
if (range.is_in_range(*iter)) {
return true;
}

++iter;
}

return false;
} else if (!is_fixed_value_range() && range.is_fixed_value_range()) {
IteratorType iter = range._fixed_values.begin();

while (iter != range._fixed_values.end()) {
if (this->is_in_range(*iter)) {
return true;
}

++iter;
}

return false;
} else {
if (Compare::greater(_low_value, range._high_value) ||
Compare::greater(range._low_value, _high_value)) {
return false;
} else if (Compare::equal(_low_value, range._high_value)) {
if (FILTER_LARGER_OR_EQUAL == _low_op && FILTER_LESS_OR_EQUAL == range._high_op) {
return true;
} else {
return false;
}
} else if (Compare::equal(range._low_value, _high_value)) {
if (FILTER_LARGER_OR_EQUAL == range._low_op && FILTER_LESS_OR_EQUAL == _high_op) {
return true;
} else {
return false;
}
} else {
return true;
}
}
}

template <PrimitiveType primitive_type>
Status OlapScanKeys::extend_scan_key(ColumnValueRange<primitive_type>& range,
int32_t max_scan_key_num, bool* exact_value, bool* eos,
Expand Down Expand Up @@ -1198,10 +997,5 @@ Status OlapScanKeys::extend_scan_key(ColumnValueRange<primitive_type>& range,
return Status::OK();
}

struct ScanPredicate {
TCondition condition;
PrimitiveType primitiveType;
};

#include "common/compile_check_end.h"
} // namespace doris
2 changes: 1 addition & 1 deletion be/src/pipeline/exec/olap_scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,7 +887,7 @@ Status OlapScanLocalState::_build_key_ranges_and_filters() {
break;
}
DCHECK(_slot_id_to_predicates.count(iter->first) > 0);
const auto& value_range = iter->second.second;
const auto& value_range = iter->second;

RETURN_IF_ERROR(std::visit(
[&](auto&& range) {
Expand Down
6 changes: 3 additions & 3 deletions be/src/pipeline/exec/scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
ColumnValueRange<TYPE_##NAME> range(slot->col_name(), slot->is_nullable(), \
cast_set<int>(type_desc->get_precision()), \
cast_set<int>(type_desc->get_scale())); \
_slot_id_to_value_range[slot->id()] = std::pair {slot, range}; \
_slot_id_to_value_range[slot->id()] = std::move(range); \
break; \
}
#define APPLY_FOR_PRIMITIVE_TYPE(M) \
Expand Down Expand Up @@ -289,7 +289,7 @@ Status ScanLocalState<Derived>::_normalize_conjuncts(RuntimeState* state) {
_scan_dependency->set_ready();
}
},
it.second.second);
it.second);
}

return Status::OK();
Expand Down Expand Up @@ -542,7 +542,7 @@ bool ScanLocalState<Derived>::_is_predicate_acting_on_slot(
if (_slot_id_to_value_range.end() == sid_to_range) {
return false;
}
*range = &(sid_to_range->second.second);
*range = &(sid_to_range->second);
SlotDescriptor* src_slot_desc = p._slot_id_to_slot_desc[slot_ref->slot_id()];
DCHECK(child_contains_slot != nullptr);
if (child_contains_slot->data_type()->get_primitive_type() !=
Expand Down
3 changes: 1 addition & 2 deletions be/src/pipeline/exec/scan_operator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,8 +324,7 @@ class ScanLocalState : public ScanLocalStateBase {

// slot id -> ColumnValueRange
// Parsed from conjuncts
phmap::flat_hash_map<int, std::pair<SlotDescriptor*, ColumnValueRangeType>>
_slot_id_to_value_range;
phmap::flat_hash_map<int, ColumnValueRangeType> _slot_id_to_value_range;
phmap::flat_hash_map<int, std::vector<std::shared_ptr<ColumnPredicate>>> _slot_id_to_predicates;

std::atomic<bool> _eos = false;
Expand Down
Loading
Loading