Merge branch 'master' into nereids_refresh_catalog

vinlee19 · Nov 19, 2024 · 13387c6 · 13387c6
2 parents 8251753 + 20473c1
commit 13387c6
Show file tree

Hide file tree

Showing 292 changed files with 5,604 additions and 2,535 deletions.
diff --git a/.github/workflows/auto-cherry-pick.yml b/.github/workflows/auto-cherry-pick.yml
@@ -21,6 +21,7 @@ on:
   pull_request_target:
     types:
       - closed
+      - labeled
     branches:
       - master
 permissions:
@@ -30,7 +31,7 @@ permissions:
 jobs:
   auto_cherry_pick:
     runs-on: ubuntu-latest
-    if: ${{ (contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x')) && github.event.pull_request.merged == true }}
+    if: ${{(contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') || contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') ||github.event.label.name == 'dev/3.0.x' || github.event.label.name == 'dev/2.1.x') && github.event.pull_request.merged == true }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -54,15 +55,15 @@ jobs:
               echo "SHA matches: $calculated_sha"
             fi
       - name: Auto cherry-pick to branch-3.0
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/3.0.x') }}
+        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/3.0.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/3.0.x')) }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO_NAME: ${{ github.repository }}
           CONFLICT_LABEL: cherry-pick-conflict-in-3.0
         run: |
           python tools/auto-pick-script.py ${{ github.event.pull_request.number }} branch-3.0  
       - name: Auto cherry-pick to branch-2.1
-        if: ${{ contains(github.event.pull_request.labels.*.name, 'dev/2.1.x') }}
+        if: ${{ ((github.event.action == 'labeled' && github.event.label.name == 'dev/2.1.x'))|| ((github.event_name == 'pull_request_target' && github.event.action == 'closed') && contains(github.event.pull_request.labels.*.name, 'dev/2.1.x')) }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO_NAME: ${{ github.repository }}

diff --git a/be/src/clucene b/be/src/clucene
diff --git a/be/src/common/status.h b/be/src/common/status.h
@@ -489,6 +489,7 @@ class [[nodiscard]] Status {
     ERROR_CTOR_NOSTACK(NeedSendAgain, NEED_SEND_AGAIN)
     ERROR_CTOR_NOSTACK(CgroupError, CGROUP_ERROR)
     ERROR_CTOR_NOSTACK(ObtainLockFailed, OBTAIN_LOCK_FAILED)
+    ERROR_CTOR_NOSTACK(NetworkError, NETWORK_ERROR)
 #undef ERROR_CTOR
 
     template <int code>

diff --git a/be/src/exec/lzo_decompressor.cpp b/be/src/exec/lzo_decompressor.cpp
@@ -103,6 +103,7 @@ Status LzopDecompressor::decompress(uint8_t* input, size_t input_len, size_t* in
     ptr = get_uint32(ptr, &uncompressed_size);
     left_input_len -= sizeof(uint32_t);
     if (uncompressed_size == 0) {
+        *input_bytes_read += sizeof(uint32_t);
         *stream_end = true;
         return Status::OK();
     }

diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
@@ -17,6 +17,7 @@
 
 #include "exec/tablet_info.h"
 
+#include <butil/logging.h>
 #include <gen_cpp/Descriptors_types.h>
 #include <gen_cpp/Exprs_types.h>
 #include <gen_cpp/Partitions_types.h>
@@ -180,6 +181,17 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) {
                 auto it = slots_map.find(to_lower(pcolumn_desc.name()) + "+" + data_type_str +
                                          is_null_str);
                 if (it == std::end(slots_map)) {
+                    std::string keys {};
+                    for (const auto& [key, _] : slots_map) {
+                        keys += fmt::format("{},", key);
+                    }
+                    LOG_EVERY_SECOND(WARNING) << fmt::format(
+                            "[OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema)]: "
+                            "unknown index column, column={}, type={}, data_type_str={}, "
+                            "is_null_str={}, slots_map.keys()=[{}], {}\npschema={}",
+                            pcolumn_desc.name(), pcolumn_desc.type(), data_type_str, is_null_str,
+                            keys, debug_string(), pschema.ShortDebugString());
+
                     return Status::InternalError("unknown index column, column={}, type={}",
                                                  pcolumn_desc.name(), pcolumn_desc.type());
                 }
@@ -286,6 +298,18 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) {
                 auto it = slots_map.find(to_lower(tcolumn_desc.column_name) + "+" + data_type_str +
                                          is_null_str);
                 if (it == slots_map.end()) {
+                    std::stringstream ss;
+                    ss << tschema;
+                    std::string keys {};
+                    for (const auto& [key, _] : slots_map) {
+                        keys += fmt::format("{},", key);
+                    }
+                    LOG_EVERY_SECOND(WARNING) << fmt::format(
+                            "[OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema)]: "
+                            "unknown index column, column={}, type={}, data_type_str={}, "
+                            "is_null_str={}, slots_map.keys()=[{}], {}\ntschema={}",
+                            tcolumn_desc.column_name, tcolumn_desc.column_type.type, data_type_str,
+                            is_null_str, keys, debug_string(), ss.str());
                     return Status::InternalError("unknown index column, column={}, type={}",
                                                  tcolumn_desc.column_name,
                                                  tcolumn_desc.column_type.type);

diff --git a/be/src/exprs/bitmapfilter_predicate.h b/be/src/exprs/bitmapfilter_predicate.h
@@ -30,11 +30,7 @@ namespace doris {
 // only used in Runtime Filter
 class BitmapFilterFuncBase : public RuntimeFilterFuncBase {
 public:
-    virtual void insert(const void* data) = 0;
     virtual void insert_many(const std::vector<const BitmapValue*>& bitmaps) = 0;
-    virtual bool empty() = 0;
-    virtual Status assign(BitmapValue* bitmap_value) = 0;
-    virtual void light_copy(BitmapFilterFuncBase* other) { _not_in = other->_not_in; }
     virtual uint16_t find_fixed_len_olap_engine(const char* data, const uint8* nullmap,
                                                 uint16_t* offsets, int number) = 0;
     virtual void find_batch(const char* data, const uint8* nullmap, size_t number,
@@ -58,8 +54,6 @@ class BitmapFilterFunc : public BitmapFilterFuncBase {
 
     ~BitmapFilterFunc() override = default;
 
-    void insert(const void* data) override;
-
     void insert_many(const std::vector<const BitmapValue*>& bitmaps) override;
 
     uint16_t find_fixed_len_olap_engine(const char* data, const uint8* nullmap, uint16_t* offsets,
@@ -68,45 +62,21 @@ class BitmapFilterFunc : public BitmapFilterFuncBase {
     void find_batch(const char* data, const uint8* nullmap, size_t number,
                     uint8* results) const override;
 
-    bool empty() override { return _bitmap_value->empty(); }
-
-    Status assign(BitmapValue* bitmap_value) override {
-        *_bitmap_value = *bitmap_value;
-        return Status::OK();
-    }
-
-    void light_copy(BitmapFilterFuncBase* bitmapfilter_func) override;
-
     size_t size() const override { return _bitmap_value->cardinality(); }
 
-    uint64_t max() { return _bitmap_value->max(nullptr); }
-
-    uint64_t min() { return _bitmap_value->min(nullptr); }
-
     bool contains_any(CppType left, CppType right) {
         if (right < 0) {
             return false;
         }
         return _bitmap_value->contains_any(std::max(left, (CppType)0), right);
     }
 
-    std::shared_ptr<BitmapValue> get_inner_bitmap() { return _bitmap_value; }
-
 private:
     std::shared_ptr<BitmapValue> _bitmap_value;
 
     bool find(CppType data) const { return _not_in ^ (data >= 0 && _bitmap_value->contains(data)); }
 };
 
-template <PrimitiveType type>
-void BitmapFilterFunc<type>::insert(const void* data) {
-    if (data == nullptr) {
-        return;
-    }
-
-    *_bitmap_value |= *reinterpret_cast<const BitmapValue*>(data);
-}
-
 template <PrimitiveType type>
 void BitmapFilterFunc<type>::insert_many(const std::vector<const BitmapValue*>& bitmaps) {
     if (bitmaps.empty()) {
@@ -147,12 +117,4 @@ void BitmapFilterFunc<type>::find_batch(const char* data, const uint8* nullmap,
     }
 }
 
-template <PrimitiveType type>
-void BitmapFilterFunc<type>::light_copy(BitmapFilterFuncBase* bitmapfilter_func) {
-    BitmapFilterFuncBase::light_copy(bitmapfilter_func);
-    auto other_func = reinterpret_cast<BitmapFilterFunc*>(bitmapfilter_func);
-    _bitmap_value = other_func->_bitmap_value;
-    set_filter_id(bitmapfilter_func->get_filter_id());
-}
-
 } // namespace doris
diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include "common/exception.h"
+#include "common/status.h"
 #include "exprs/hybrid_set.h"
 #include "exprs/minmax_predicate.h"
 #include "function_filter.h"
@@ -244,12 +246,9 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
                                               int be_exec_version, const TabletColumn*) {
     if constexpr (PT == TYPE_TINYINT || PT == TYPE_SMALLINT || PT == TYPE_INT ||
                   PT == TYPE_BIGINT) {
-        std::shared_ptr<BitmapFilterFuncBase> filter_olap;
-        filter_olap.reset(create_bitmap_filter(PT));
-        filter_olap->light_copy(filter.get());
         return new BitmapFilterColumnPredicate<PT>(column_id, filter, be_exec_version);
     } else {
-        return nullptr;
+        throw Exception(ErrorCode::INTERNAL_ERROR, "bitmap filter do not support type {}", PT);
     }
 }
 
@@ -266,17 +265,14 @@ ColumnPredicate* create_olap_column_predicate(uint32_t column_id,
                                               const std::shared_ptr<FunctionFilter>& filter, int,
                                               const TabletColumn* column = nullptr) {
     // currently only support like predicate
-    if constexpr (PT == TYPE_CHAR || PT == TYPE_VARCHAR || PT == TYPE_STRING) {
-        if constexpr (PT == TYPE_CHAR) {
-            return new LikeColumnPredicate<TYPE_CHAR>(filter->_opposite, column_id, filter->_fn_ctx,
-                                                      filter->_string_param);
-        } else {
-            return new LikeColumnPredicate<TYPE_STRING>(filter->_opposite, column_id,
-                                                        filter->_fn_ctx, filter->_string_param);
-        }
-    } else {
-        return nullptr;
+    if constexpr (PT == TYPE_CHAR) {
+        return new LikeColumnPredicate<TYPE_CHAR>(filter->_opposite, column_id, filter->_fn_ctx,
+                                                  filter->_string_param);
+    } else if constexpr (PT == TYPE_VARCHAR || PT == TYPE_STRING) {
+        return new LikeColumnPredicate<TYPE_STRING>(filter->_opposite, column_id, filter->_fn_ctx,
+                                                    filter->_string_param);
     }
+    throw Exception(ErrorCode::INTERNAL_ERROR, "function filter do not support type {}", PT);
 }
 
 template <typename T>
+4 −4		src/core/CLucene/index/IndexReader.cpp
+4 −4		src/core/CLucene/index/IndexReader.h
+4 −2		src/core/CLucene/index/MultiReader.cpp
+2 −2		src/core/CLucene/index/MultiReader.h
+10 −4		src/core/CLucene/index/MultiSegmentReader.cpp
+8 −4		src/core/CLucene/index/SegmentReader.cpp
+11 −2		src/core/CLucene/index/SegmentTermDocs.cpp
+6 −0		src/core/CLucene/index/SegmentTermEnum.cpp
+5 −0		src/core/CLucene/index/SegmentTermPositions.cpp
+6 −0		src/core/CLucene/index/SkipListReader.cpp
+5 −1		src/core/CLucene/index/TermInfosReader.cpp
+2 −0		src/core/CLucene/index/Terms.h
+7 −2		src/core/CLucene/index/_MultiSegmentReader.h
+8 −3		src/core/CLucene/index/_SegmentHeader.h
+2 −0		src/core/CLucene/index/_SegmentTermEnum.h
+2 −0		src/core/CLucene/index/_SkipListReader.h
+2 −1		src/core/CLucene/index/_TermInfosReader.h
+5 −0		src/core/CLucene/store/IndexInput.h