Skip to content

[refactor](field) Remove nearest type in field #59488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Gabriel39 wants to merge 4 commits into
base: master
Choose a base branch
from
Open

[refactor](field) Remove nearest type in field #59488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8234515
[refactor](field) Remove nearest type in field
Gabriel39 Dec 30, 2025
993897e
update
Gabriel39 Dec 30, 2025
e269fdb
update
Gabriel39 Dec 31, 2025
a96cd3c
update
Gabriel39 Dec 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 12 additions & 15 deletions be/src/exec/es/es_scroll_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ Status get_int_value(const rapidjson::Value& col, PrimitiveType type, void* slot

template <PrimitiveType T>
Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool is_date_str,
typename PrimitiveTypeTraits<T>::ColumnItemType* slot,
typename PrimitiveTypeTraits<T>::CppType* slot,
const cctz::time_zone& time_zone) {
constexpr bool is_datetime_v1 = T == TYPE_DATE || T == TYPE_DATETIME;
typename PrimitiveTypeTraits<T>::CppType dt_val;
Expand Down Expand Up @@ -271,16 +271,13 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool
}
}

*reinterpret_cast<typename PrimitiveTypeTraits<T>::ColumnItemType*>(slot) =
binary_cast<typename PrimitiveTypeTraits<T>::CppType,
typename PrimitiveTypeTraits<T>::ColumnItemType>(
*reinterpret_cast<typename PrimitiveTypeTraits<T>::CppType*>(&dt_val));
*slot = *reinterpret_cast<typename PrimitiveTypeTraits<T>::CppType*>(&dt_val);
return Status::OK();
}

template <PrimitiveType T>
Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_doc_value,
typename PrimitiveTypeTraits<T>::ColumnItemType* slot,
typename PrimitiveTypeTraits<T>::CppType* slot,
const cctz::time_zone& time_zone) {
// this would happend just only when `enable_docvalue_scan = false`, and field has timestamp format date from _source
if (col.IsNumber()) {
Expand Down Expand Up @@ -309,7 +306,7 @@ Status get_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_d
template <PrimitiveType T>
Status fill_date_int(const rapidjson::Value& col, PrimitiveType type, bool pure_doc_value,
vectorized::IColumn* col_ptr, const cctz::time_zone& time_zone) {
typename PrimitiveTypeTraits<T>::ColumnItemType data;
typename PrimitiveTypeTraits<T>::CppType data;
RETURN_IF_ERROR((get_date_int<T>(col, type, pure_doc_value, &data, time_zone)));
col_ptr->insert_data(const_cast<const char*>(reinterpret_cast<char*>(&data)), 0);
return Status::OK();
Expand Down Expand Up @@ -426,11 +423,11 @@ Status insert_int_value(const rapidjson::Value& col, PrimitiveType type,

template <PrimitiveType T>
Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value,
typename PrimitiveTypeTraits<T>::ColumnItemType& val) {
typename PrimitiveTypeTraits<T>::CppType& val) {
if constexpr (T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || T == TYPE_BIGINT ||
T == TYPE_LARGEINT) {
RETURN_IF_ERROR(get_int_value<typename PrimitiveTypeTraits<T>::ColumnItemType>(
col, sub_type, &val, pure_doc_value));
RETURN_IF_ERROR(get_int_value<typename PrimitiveTypeTraits<T>::CppType>(col, sub_type, &val,
pure_doc_value));
return Status::OK();
}
if constexpr (T == TYPE_FLOAT) {
Expand All @@ -457,7 +454,7 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu
}

if (col.IsNumber()) {
val = static_cast<typename PrimitiveTypeTraits<T>::ColumnItemType>(col.GetInt());
val = static_cast<typename PrimitiveTypeTraits<T>::CppType>(col.GetInt());
return Status::OK();
}

Expand Down Expand Up @@ -485,7 +482,7 @@ Status handle_value(const rapidjson::Value& col, PrimitiveType sub_type, bool pu
template <PrimitiveType T>
Status process_single_column(const rapidjson::Value& col, PrimitiveType sub_type,
bool pure_doc_value, vectorized::Array& array) {
typename PrimitiveTypeTraits<T>::ColumnItemType val;
typename PrimitiveTypeTraits<T>::CppType val;
RETURN_IF_ERROR(handle_value<T>(col, sub_type, pure_doc_value, val));
array.push_back(vectorized::Field::create_field<T>(val));
return Status::OK();
Expand Down Expand Up @@ -514,12 +511,12 @@ template <PrimitiveType T>
Status process_date_column(const rapidjson::Value& col, PrimitiveType sub_type, bool pure_doc_value,
vectorized::Array& array, const cctz::time_zone& time_zone) {
if (!col.IsArray()) {
typename PrimitiveTypeTraits<T>::ColumnItemType data;
typename PrimitiveTypeTraits<T>::CppType data;
RETURN_IF_ERROR((get_date_int<T>(col, sub_type, pure_doc_value, &data, time_zone)));
array.push_back(vectorized::Field::create_field<T>(data));
} else {
for (const auto& sub_col : col.GetArray()) {
typename PrimitiveTypeTraits<T>::ColumnItemType data;
typename PrimitiveTypeTraits<T>::CppType data;
RETURN_IF_ERROR((get_date_int<T>(sub_col, sub_type, pure_doc_value, &data, time_zone)));
array.push_back(vectorized::Field::create_field<T>(data));
}
Expand All @@ -533,7 +530,7 @@ Status process_jsonb_column(const rapidjson::Value& col, PrimitiveType sub_type,
JsonBinaryValue jsonb_value;
RETURN_IF_ERROR(jsonb_value.from_json_string(json_value_to_string(col)));
vectorized::JsonbField json(jsonb_value.value(), jsonb_value.size());
array.push_back(vectorized::Field::create_field<TYPE_JSONB>(json));
array.push_back(vectorized::Field::create_field<TYPE_JSONB>(std::move(json)));
} else {
for (const auto& sub_col : col.GetArray()) {
JsonBinaryValue jsonb_value;
Expand Down
10 changes: 6 additions & 4 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,9 @@ std::string cast_to_string(T value, int scale) {
template <PrimitiveType primitive_type>
class ColumnValueRange {
public:
using CppType = std::conditional_t<primitive_type == TYPE_HLL, StringRef,
typename PrimitiveTypeTraits<primitive_type>::CppType>;
using CppType =
std::conditional_t<primitive_type == TYPE_HLL || is_string_type(primitive_type),
StringRef, typename PrimitiveTypeTraits<primitive_type>::CppType>;
using SetType = std::set<CppType, doris::Less<CppType>>;
using IteratorType = typename SetType::iterator;

Expand Down Expand Up @@ -814,8 +815,9 @@ template <PrimitiveType primitive_type>
Status OlapScanKeys::extend_scan_key(ColumnValueRange<primitive_type>& range,
int32_t max_scan_key_num, bool* exact_value, bool* eos,
bool* should_break) {
using CppType = std::conditional_t<primitive_type == TYPE_HLL, StringRef,
typename PrimitiveTypeTraits<primitive_type>::CppType>;
using CppType =
std::conditional_t<primitive_type == TYPE_HLL || is_string_type(primitive_type),
StringRef, typename PrimitiveTypeTraits<primitive_type>::CppType>;
using ConstIterator = typename ColumnValueRange<primitive_type>::SetType::const_iterator;

// 1. clear ScanKey if some column range is empty
Expand Down
5 changes: 2 additions & 3 deletions be/src/exprs/create_predicate_function.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,15 @@ class HybridSetTraits {
using BasePtr = HybridSetBase*;
template <PrimitiveType type, size_t N>
static BasePtr get_function(bool null_aware) {
using CppType = typename PrimitiveTypeTraits<type>::CppType;
if constexpr (N >= 1 && N <= FIXED_CONTAINER_MAX_SIZE) {
using Set = std::conditional_t<
std::is_same_v<CppType, StringRef>, StringSet<>,
is_string_type(type), StringSet<>,
HybridSet<type,
FixedContainer<typename PrimitiveTypeTraits<type>::CppType, N>>>;
return new Set(null_aware);
} else {
using Set = std::conditional_t<
std::is_same_v<CppType, StringRef>, StringSet<>,
is_string_type(type), StringSet<>,
HybridSet<type, DynamicContainer<typename PrimitiveTypeTraits<type>::CppType>>>;
return new Set(null_aware);
}
Expand Down
18 changes: 4 additions & 14 deletions be/src/olap/comparison_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ template <PrimitiveType Type, PredicateType PT>
class ComparisonPredicateBase final : public ColumnPredicate {
public:
ENABLE_FACTORY_CREATOR(ComparisonPredicateBase);
using T = typename PrimitiveTypeTraits<Type>::CppType;
using T = std::conditional_t<is_string_type(Type), StringRef,
typename PrimitiveTypeTraits<Type>::CppType>;
ComparisonPredicateBase(uint32_t column_id, const T& value, bool opposite = false)
: ColumnPredicate(column_id, Type, opposite), _value(value) {}
ComparisonPredicateBase(const ComparisonPredicateBase<Type, PT>& other, uint32_t col_id)
Expand Down Expand Up @@ -166,19 +167,8 @@ class ComparisonPredicateBase final : public ColumnPredicate {
*/

bool camp_field(const vectorized::Field& min_field, const vectorized::Field& max_field) const {
T min_value;
T max_value;
if constexpr (is_int_or_bool(Type) || is_float_or_double(Type)) {
min_value =
(typename PrimitiveTypeTraits<Type>::CppType)min_field
.template get<typename PrimitiveTypeTraits<Type>::NearestFieldType>();
max_value =
(typename PrimitiveTypeTraits<Type>::CppType)max_field
.template get<typename PrimitiveTypeTraits<Type>::NearestFieldType>();
} else {
min_value = min_field.template get<typename PrimitiveTypeTraits<Type>::CppType>();
max_value = max_field.template get<typename PrimitiveTypeTraits<Type>::CppType>();
}
T min_value = min_field.template get<T>();
T max_value = max_field.template get<T>();

if constexpr (PT == PredicateType::EQ) {
return Compare::less_equal(min_value, _value) &&
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/delete_handler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,8 @@ Status parse_to_predicate(const uint32_t index, const vectorized::DataTypePtr& t
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
RETURN_IF_ERROR(convert<TYPE_STRING>(type, res.value_str.front(), arena, v));
auto tmp = vectorized::String(v.data, v.size);
RETURN_IF_ERROR(convert<TYPE_STRING>(type, res.value_str.front(), arena, tmp));
switch (res.condition_op) {
case PredicateType::EQ:
predicate =
Expand Down
18 changes: 4 additions & 14 deletions be/src/olap/in_list_predicate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ template <PrimitiveType Type, PredicateType PT, int N>
class InListPredicateBase final : public ColumnPredicate {
public:
ENABLE_FACTORY_CREATOR(InListPredicateBase);
using T = typename PrimitiveTypeTraits<Type>::CppType;
using T = std::conditional_t<is_string_type(Type), StringRef,
typename PrimitiveTypeTraits<Type>::CppType>;
using HybridSetType = std::conditional_t<
N >= 1 && N <= FIXED_CONTAINER_MAX_SIZE,
std::conditional_t<
Expand Down Expand Up @@ -276,19 +277,8 @@ class InListPredicateBase final : public ColumnPredicate {
}

bool camp_field(const vectorized::Field& min_field, const vectorized::Field& max_field) const {
T min_value;
T max_value;
if constexpr (is_int_or_bool(Type) || is_float_or_double(Type)) {
min_value =
(typename PrimitiveTypeTraits<Type>::CppType)min_field
.template get<typename PrimitiveTypeTraits<Type>::NearestFieldType>();
max_value =
(typename PrimitiveTypeTraits<Type>::CppType)max_field
.template get<typename PrimitiveTypeTraits<Type>::NearestFieldType>();
} else {
min_value = min_field.template get<typename PrimitiveTypeTraits<Type>::CppType>();
max_value = max_field.template get<typename PrimitiveTypeTraits<Type>::CppType>();
}
T min_value = min_field.template get<T>();
T max_value = max_field.template get<T>();

if constexpr (PT == PredicateType::IN_LIST) {
return (Compare::less_equal(min_value, _max_value) &&
Expand Down
4 changes: 1 addition & 3 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,9 +405,7 @@ class InvertedIndexQueryParamFactory {
} else {
CPP_TYPE cpp_val;
if constexpr (std::is_same_v<ValueType, doris::vectorized::Field>) {
auto field_val =
doris::vectorized::get<doris::vectorized::NearestFieldType<CPP_TYPE>>(
*value);
auto field_val = value->template get<typename PrimitiveTypeTraits<PT>::CppType>();
cpp_val = static_cast<CPP_TYPE>(field_val);
} else {
cpp_val = static_cast<CPP_TYPE>(*value);
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/rowset/segment_v2/zone_map_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ void TypedZoneMapIndexWriter<Type>::add_values(const void* values, size_t count)
if (count > 0) {
_page_zone_map.has_not_null = true;
}
using ValType = PrimitiveTypeTraits<Type>::StorageFieldType;
using ValType = std::conditional_t<is_string_type(Type), StringRef,
typename PrimitiveTypeTraits<Type>::StorageFieldType>;
const auto* vals = reinterpret_cast<const ValType*>(values);
if constexpr (Type == TYPE_FLOAT || Type == TYPE_DOUBLE) {
ValType min = std::numeric_limits<ValType>::max();
Expand Down
15 changes: 10 additions & 5 deletions be/src/pipeline/exec/scan_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,17 +938,15 @@ Status ScanLocalState<Derived>::_change_value_range(ColumnValueRange<PrimitiveTy
func(temp_range, to_olap_filter_type(fn_name, slot_ref_child),
reinterpret_cast<const StringRef*>(value));
}
} else if constexpr ((PrimitiveType == TYPE_DECIMALV2) || (PrimitiveType == TYPE_CHAR) ||
(PrimitiveType == TYPE_VARCHAR) || (PrimitiveType == TYPE_DATETIMEV2) ||
} else if constexpr ((PrimitiveType == TYPE_DECIMALV2) || (PrimitiveType == TYPE_DATETIMEV2) ||
(PrimitiveType == TYPE_TINYINT) || (PrimitiveType == TYPE_SMALLINT) ||
(PrimitiveType == TYPE_INT) || (PrimitiveType == TYPE_BIGINT) ||
(PrimitiveType == TYPE_LARGEINT) || (PrimitiveType == TYPE_FLOAT) ||
(PrimitiveType == TYPE_DOUBLE) || (PrimitiveType == TYPE_IPV4) ||
(PrimitiveType == TYPE_IPV6) || (PrimitiveType == TYPE_DECIMAL32) ||
(PrimitiveType == TYPE_DECIMAL64) || (PrimitiveType == TYPE_DECIMAL128I) ||
(PrimitiveType == TYPE_DECIMAL256) || (PrimitiveType == TYPE_STRING) ||
(PrimitiveType == TYPE_BOOLEAN) || (PrimitiveType == TYPE_DATEV2) ||
(PrimitiveType == TYPE_TIMESTAMPTZ)) {
(PrimitiveType == TYPE_DECIMAL256) || (PrimitiveType == TYPE_BOOLEAN) ||
(PrimitiveType == TYPE_DATEV2) || (PrimitiveType == TYPE_TIMESTAMPTZ)) {
if constexpr (IsFixed) {
func(temp_range,
reinterpret_cast<const typename PrimitiveTypeTraits<PrimitiveType>::CppType*>(
Expand All @@ -958,6 +956,13 @@ Status ScanLocalState<Derived>::_change_value_range(ColumnValueRange<PrimitiveTy
reinterpret_cast<const typename PrimitiveTypeTraits<PrimitiveType>::CppType*>(
value));
}
} else if constexpr (is_string_type(PrimitiveType)) {
if constexpr (IsFixed) {
func(temp_range, reinterpret_cast<const StringRef*>(value));
} else {
func(temp_range, to_olap_filter_type(fn_name, slot_ref_child),
reinterpret_cast<const StringRef*>(value));
}
} else {
static_assert(always_false_v<PrimitiveType>);
}
Expand Down
Loading
Loading