[refactor](predicate) Refine column predicates on OLAP table

be/src/exec/olap_common.h

@@ -425,6 +425,7 @@ class OlapScanKeys {
               _end_include(true),
               _is_convertible(true) {}
 
+    // TODO(gabriel): use ColumnPredicate to extend scan key
     template <PrimitiveType primitive_type>
     Status extend_scan_key(ColumnValueRange<primitive_type>& range, int32_t max_scan_key_num,
                            bool* exact_value, bool* eos, bool* should_break);

be/src/exprs/bitmapfilter_predicate.h

@@ -19,6 +19,7 @@
 
 #include <algorithm>
 
+#include "common/cast_set.h"
 #include "runtime/define_primitive_type.h"
 #include "runtime/primitive_type.h"
 #include "runtime_filter/runtime_filter_definitions.h"
@@ -67,7 +68,8 @@ class BitmapFilterFunc : public BitmapFilterFuncBase {
         if (right < 0) {
             return false;
         }
-        return _bitmap_value->contains_any(std::max(left, (CppType)0), right);
+        return _bitmap_value->contains_any(cast_set<uint64_t>(std::max(left, (CppType)0)),
+                                           cast_set<uint64_t>(right));
     }
 
 private:

be/src/exprs/create_predicate_function.h

@@ -231,55 +231,56 @@ inline auto create_bitmap_filter(PrimitiveType type) {
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<BloomFilterFuncBase>& filter,
-                                              const TabletColumn*, bool null_aware) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<BloomFilterFuncBase>& filter, const TabletColumn*,
+        bool null_aware) {
     std::shared_ptr<BloomFilterFuncBase> filter_olap;
     filter_olap.reset(create_bloom_filter(PT, null_aware));
     filter_olap->light_copy(filter.get());
     // create a new filter to match the input filter and PT. For example, filter may be varchar, but PT is char
-    return new BloomFilterColumnPredicate<PT>(column_id, filter_olap);
+    return BloomFilterColumnPredicate<PT>::create_shared(column_id, filter_olap);
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<BitmapFilterFuncBase>& filter,
-                                              const TabletColumn*, bool) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<BitmapFilterFuncBase>& filter,
+        const TabletColumn*, bool) {
     if constexpr (PT == TYPE_TINYINT || PT == TYPE_SMALLINT || PT == TYPE_INT ||
                   PT == TYPE_BIGINT) {
-        return new BitmapFilterColumnPredicate<PT>(column_id, filter);
+        return BitmapFilterColumnPredicate<PT>::create_shared(column_id, filter);
     } else {
         throw Exception(ErrorCode::INTERNAL_ERROR, "bitmap filter do not support type {}", PT);
     }
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<HybridSetBase>& filter,
-                                              const TabletColumn* column, bool) {
+std::shared_ptr<const ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<HybridSetBase>& filter,
+        const TabletColumn* column, bool) {
     return create_in_list_predicate<PT, PredicateType::IN_LIST>(column_id, filter,
                                                                 column->length());
 }
 
 template <PrimitiveType PT>
-ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
-                                              const std::shared_ptr<FunctionFilter>& filter,
-                                              const TabletColumn* column, bool) {
+std::shared_ptr<ColumnPredicate> create_olap_column_predicate(
+        uint32_t column_id, const std::shared_ptr<FunctionFilter>& filter,
+        const TabletColumn* column, bool) {
     // currently only support like predicate
     if constexpr (PT == TYPE_CHAR) {
-        return new LikeColumnPredicate<TYPE_CHAR>(filter->_opposite, column_id, filter->_fn_ctx,
-                                                  filter->_string_param);
+        return LikeColumnPredicate<TYPE_CHAR>::create_shared(
+                filter->_opposite, column_id, filter->_fn_ctx, filter->_string_param);
     } else if constexpr (PT == TYPE_VARCHAR || PT == TYPE_STRING) {
-        return new LikeColumnPredicate<TYPE_STRING>(filter->_opposite, column_id, filter->_fn_ctx,
-                                                    filter->_string_param);
+        return LikeColumnPredicate<TYPE_STRING>::create_shared(
+                filter->_opposite, column_id, filter->_fn_ctx, filter->_string_param);
     }
     throw Exception(ErrorCode::INTERNAL_ERROR, "function filter do not support type {}", PT);
 }
 
 template <typename T>
-ColumnPredicate* create_column_predicate(uint32_t column_id, const std::shared_ptr<T>& filter,
-                                         FieldType type, const TabletColumn* column,
-                                         bool null_aware = false) {
+std::shared_ptr<ColumnPredicate> create_column_predicate(uint32_t column_id,
+                                                         const std::shared_ptr<T>& filter,
+                                                         FieldType type, const TabletColumn* column,
+                                                         bool null_aware = false) {
     switch (type) {
 #define M(NAME)                                                                           \
     case FieldType::OLAP_FIELD_##NAME: {                                                  \

be/src/olap/accept_null_predicate.h

@@ -40,8 +40,27 @@ class AcceptNullPredicate : public ColumnPredicate {
     ENABLE_FACTORY_CREATOR(AcceptNullPredicate);
 
 public:
-    AcceptNullPredicate(ColumnPredicate* nested)
-            : ColumnPredicate(nested->column_id(), nested->opposite()), _nested {nested} {}
+    AcceptNullPredicate(const std::shared_ptr<ColumnPredicate>& nested)
+            : ColumnPredicate(nested->column_id(), nested->primitive_type(), nested->opposite()),
+              _nested {nested} {}
+    AcceptNullPredicate(const AcceptNullPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _nested(assert_cast<const AcceptNullPredicate&>(other)._nested
+                              ? assert_cast<const AcceptNullPredicate&>(other)._nested->clone(
+                                        col_id)
+                              : nullptr) {}
+    AcceptNullPredicate(const AcceptNullPredicate& other) = delete;
+    ~AcceptNullPredicate() override = default;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return AcceptNullPredicate::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        auto n = _nested;
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "AcceptNullPredicate({}, nested={})",
+                       ColumnPredicate::debug_string(), n ? n->debug_string() : "null");
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return _nested->type(); }
 
@@ -173,11 +192,7 @@ class AcceptNullPredicate : public ColumnPredicate {
         return _nested->evaluate(column, sel, size);
     }
 
-    std::string _debug_string() const override {
-        return "passnull predicate for " + _nested->debug_string();
-    }
-
-    std::unique_ptr<ColumnPredicate> _nested;
+    std::shared_ptr<ColumnPredicate> _nested;
 };
 
 } //namespace doris

be/src/olap/bitmap_filter_predicate.h

@@ -27,17 +27,32 @@
 
 namespace doris {
 template <PrimitiveType T>
-class BitmapFilterColumnPredicate : public ColumnPredicate {
+class BitmapFilterColumnPredicate final : public ColumnPredicate {
 public:
+    ENABLE_FACTORY_CREATOR(BitmapFilterColumnPredicate);
     using CppType = typename PrimitiveTypeTraits<T>::CppType;
     using SpecificFilter = BitmapFilterFunc<T>;
 
     BitmapFilterColumnPredicate(uint32_t column_id,
                                 const std::shared_ptr<BitmapFilterFuncBase>& filter)
-            : ColumnPredicate(column_id),
+            : ColumnPredicate(column_id, T),
               _filter(filter),
               _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
     ~BitmapFilterColumnPredicate() override = default;
+    BitmapFilterColumnPredicate(const BitmapFilterColumnPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _filter(other._filter),
+              _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
+    BitmapFilterColumnPredicate(const BitmapFilterColumnPredicate& other) = delete;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return BitmapFilterColumnPredicate<T>::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "BitmapFilterColumnPredicate({})",
+                       ColumnPredicate::debug_string());
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return PredicateType::BITMAP_FILTER; }
 
@@ -85,10 +100,6 @@ class BitmapFilterColumnPredicate : public ColumnPredicate {
         return new_size;
     }
 
-    std::string _debug_string() const override {
-        return "BitmapFilterColumnPredicate(" + type_to_string(T) + ")";
-    }
-
     std::shared_ptr<BitmapFilterFuncBase> _filter;
     SpecificFilter* _specific_filter; // owned by _filter
 

be/src/olap/block_column_predicate.h

@@ -60,7 +60,7 @@ class BlockColumnPredicate {
     virtual void get_all_column_ids(std::set<ColumnId>& column_id_set) const = 0;
 
     virtual void get_all_column_predicate(
-            std::set<const ColumnPredicate*>& predicate_set) const = 0;
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const = 0;
 
     virtual uint16_t evaluate(vectorized::MutableColumns& block, uint16_t* sel,
                               uint16_t selected_size) const {
@@ -118,13 +118,15 @@ class SingleColumnBlockPredicate : public BlockColumnPredicate {
     ENABLE_FACTORY_CREATOR(SingleColumnBlockPredicate);
 
 public:
-    explicit SingleColumnBlockPredicate(const ColumnPredicate* pre) : _predicate(pre) {}
+    explicit SingleColumnBlockPredicate(const std::shared_ptr<const ColumnPredicate>& pre)
+            : _predicate(pre) {}
 
     void get_all_column_ids(std::set<ColumnId>& column_id_set) const override {
         column_id_set.insert(_predicate->column_id());
     }
 
-    void get_all_column_predicate(std::set<const ColumnPredicate*>& predicate_set) const override {
+    void get_all_column_predicate(
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const override {
         predicate_set.insert(_predicate);
     }
 
@@ -154,7 +156,7 @@ class SingleColumnBlockPredicate : public BlockColumnPredicate {
     }
 
 private:
-    const ColumnPredicate* _predicate = nullptr;
+    const std::shared_ptr<const ColumnPredicate> _predicate = nullptr;
 };
 
 class MutilColumnBlockPredicate : public BlockColumnPredicate {
@@ -185,7 +187,8 @@ class MutilColumnBlockPredicate : public BlockColumnPredicate {
         }
     }
 
-    void get_all_column_predicate(std::set<const ColumnPredicate*>& predicate_set) const override {
+    void get_all_column_predicate(
+            std::set<std::shared_ptr<const ColumnPredicate>>& predicate_set) const override {
         for (auto& child_block_predicate : _block_column_predicate_vec) {
             child_block_predicate->get_all_column_predicate(predicate_set);
         }

be/src/olap/bloom_filter_predicate.h

@@ -30,16 +30,31 @@
 namespace doris {
 
 template <PrimitiveType T>
-class BloomFilterColumnPredicate : public ColumnPredicate {
+class BloomFilterColumnPredicate final : public ColumnPredicate {
 public:
+    ENABLE_FACTORY_CREATOR(BloomFilterColumnPredicate);
     using SpecificFilter = BloomFilterFunc<T>;
 
     BloomFilterColumnPredicate(uint32_t column_id,
                                const std::shared_ptr<BloomFilterFuncBase>& filter)
-            : ColumnPredicate(column_id),
+            : ColumnPredicate(column_id, T),
               _filter(filter),
               _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
     ~BloomFilterColumnPredicate() override = default;
+    BloomFilterColumnPredicate(const BloomFilterColumnPredicate& other, uint32_t col_id)
+            : ColumnPredicate(other, col_id),
+              _filter(other._filter),
+              _specific_filter(assert_cast<SpecificFilter*>(_filter.get())) {}
+    BloomFilterColumnPredicate(const BloomFilterColumnPredicate& other) = delete;
+    std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const override {
+        return BloomFilterColumnPredicate<T>::create_shared(*this, col_id);
+    }
+    std::string debug_string() const override {
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer, "BloomFilterColumnPredicate({})",
+                       ColumnPredicate::debug_string());
+        return fmt::to_string(debug_string_buffer);
+    }
 
     PredicateType type() const override { return PredicateType::BF; }
 
@@ -76,8 +91,6 @@ class BloomFilterColumnPredicate : public ColumnPredicate {
         return new_size;
     }
 
-    std::string _debug_string() const override { return "BloomFilter(" + type_to_string(T) + ")"; }
-
     std::shared_ptr<BloomFilterFuncBase> _filter;
     SpecificFilter* _specific_filter; // owned by _filter
 };

be/src/olap/column_predicate.h

@@ -160,14 +160,20 @@ struct PredicateTypeTraits {
 
 class ColumnPredicate {
 public:
-    explicit ColumnPredicate(uint32_t column_id, bool opposite = false)
-            : _column_id(column_id), _opposite(opposite) {
+    explicit ColumnPredicate(uint32_t column_id, PrimitiveType primitive_type,
+                             bool opposite = false)
+            : _column_id(column_id), _primitive_type(primitive_type), _opposite(opposite) {
         reset_judge_selectivity();
     }
+    ColumnPredicate(const ColumnPredicate& other, uint32_t col_id) : ColumnPredicate(other) {
+        _column_id = col_id;
+    }
 
     virtual ~ColumnPredicate() = default;
 
     virtual PredicateType type() const = 0;
+    virtual PrimitiveType primitive_type() const { return _primitive_type; }
+    virtual std::shared_ptr<ColumnPredicate> clone(uint32_t col_id) const = 0;
 
     //evaluate predicate on inverted
     virtual Status evaluate(const vectorized::IndexFieldNameAndTypePair& name_with_type,
@@ -178,6 +184,16 @@ class ColumnPredicate {
     }
 
     virtual double get_ignore_threshold() const { return 0; }
+    // Return the size of value set for IN/NOT IN predicates and 0 for others.
+    virtual std::string debug_string() const {
+        fmt::memory_buffer debug_string_buffer;
+        fmt::format_to(debug_string_buffer,
+                       "Column ID: {}, Data Type: {}, PredicateType: {}, opposite: {}, Runtime "
+                       "Filter ID: {}",
+                       _column_id, type_to_string(primitive_type()), pred_type_string(type()),
+                       _opposite, _runtime_filter_id);
+        return fmt::to_string(debug_string_buffer);
+    }
 
     // evaluate predicate on IColumn
     // a short circuit eval way
@@ -266,14 +282,6 @@ class ColumnPredicate {
 
     bool opposite() const { return _opposite; }
 
-    std::string debug_string() const {
-        return _debug_string() +
-               fmt::format(", column_id={}, opposite={}, can_ignore={}, runtime_filter_id={}",
-                           _column_id, _opposite, _can_ignore(), _runtime_filter_id);
-    }
-
-    int get_runtime_filter_id() const { return _runtime_filter_id; }
-
     void attach_profile_counter(
             int filter_id, std::shared_ptr<RuntimeProfile::Counter> predicate_filtered_rows_counter,
             std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter,
@@ -347,7 +355,6 @@ class ColumnPredicate {
     virtual bool is_runtime_filter() const { return _can_ignore(); }
 
 protected:
-    virtual std::string _debug_string() const = 0;
     virtual bool _can_ignore() const { return _runtime_filter_id != -1; }
     virtual uint16_t _evaluate_inner(const vectorized::IColumn& column, uint16_t* sel,
                                      uint16_t size) const {
@@ -377,6 +384,7 @@ class ColumnPredicate {
     }
 
     uint32_t _column_id;
+    PrimitiveType _primitive_type;
     // TODO: the value is only in delete condition, better be template value
     bool _opposite;
     int _runtime_filter_id = -1;
@@ -399,6 +407,9 @@ class ColumnPredicate {
             std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
     std::shared_ptr<RuntimeProfile::Counter> _predicate_always_true_rows_counter =
             std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
+
+private:
+    ColumnPredicate(const ColumnPredicate& other) = default;
 };
 
 } //namespace doris