diff --git a/.asf.yaml b/.asf.yaml
index 83a80db5ff6353..821947fa1a0451 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -55,6 +55,7 @@ github:
- P1 Regression (Doris Regression)
- External Regression (Doris External Regression)
- cloud_p1 (Doris Cloud Regression)
+ - cloud_p0 (Doris Cloud Regression)
- FE UT (Doris FE UT)
- BE UT (Doris BE UT)
- Build Broker
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 15b12fc843eaff..2757578827c2bb 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -16,3 +16,4 @@
#
be/src/io/* @platoneko @gavinchou @dataroaring
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
+**/pom.xml @CalvinKirs @morningman
diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml
index bd3e29dbb0c8e8..5adadaba205d6e 100644
--- a/.github/workflows/comment-to-trigger-teamcity.yml
+++ b/.github/workflows/comment-to-trigger-teamcity.yml
@@ -150,6 +150,7 @@ jobs:
set -x
if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
"${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
"${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_fe_ut }}" \
@@ -158,7 +159,7 @@ jobs:
"feut" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run feut"
+ echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run feut"
trigger_or_skip_build \
"false" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -175,6 +176,7 @@ jobs:
set -x
if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
"${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
"${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_be_ut }}" \
@@ -183,7 +185,7 @@ jobs:
"beut" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run beut"
+ echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run beut"
trigger_or_skip_build \
"false" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -197,7 +199,8 @@ jobs:
run: |
source ./regression-test/pipeline/common/teamcity-utils.sh
set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_cloud_ut }}" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -205,7 +208,7 @@ jobs:
"cloudut" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch is not master, skip run cloudut"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloudut"
fi
- name: "Trigger or Skip compile"
@@ -279,8 +282,9 @@ jobs:
echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
fi
set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
- echo "PR target branch in (master), need run cloud_p0"
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+ echo "PR target branch is in (master, branch-3.0), need run cloud_p0"
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_cloud_p0 }}" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -288,7 +292,7 @@ jobs:
"cloud_p0" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch not in (master), skip run cloud_p0"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloud_p0"
trigger_or_skip_build \
"false" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -305,8 +309,9 @@ jobs:
echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
fi
set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
- echo "PR target branch in (master), need run cloud_p1"
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+ echo "PR target branch is in (master, branch-3.0), need run cloud_p1"
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_cloud_p1 }}" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -314,7 +319,7 @@ jobs:
"cloud_p1" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch not in (master), skip run cloud_p1"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloud_p1"
trigger_or_skip_build \
"false" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -341,8 +346,9 @@ jobs:
source ./regression-test/pipeline/common/teamcity-utils.sh
set -x
if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
"${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
- echo "PR target branch in (master, branch-2.0), need run performance"
+ echo "PR target branch is in (master, branch-2.0, branch-3.0), need run performance"
trigger_or_skip_build \
"${{ steps.changes.outputs.changed_performance }}" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -350,7 +356,7 @@ jobs:
"performance" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
else
- echo "PR target branch not in (master, branch-2.0), skip run performance"
+ echo "PR target branch is not in (master, branch-2.0, branch-3.0), skip run performance"
trigger_or_skip_build \
"false" \
"${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
diff --git a/.github/workflows/labeler/scope-label-conf.yml b/.github/workflows/labeler/scope-label-conf.yml
index 5a88d046a630ba..2afd31bc1ef7bd 100644
--- a/.github/workflows/labeler/scope-label-conf.yml
+++ b/.github/workflows/labeler/scope-label-conf.yml
@@ -23,3 +23,6 @@ meta-change:
- fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
- gensrc/thrift/*
- gensrc/proto/*
+
+doing:
+ - '**'
diff --git a/.github/workflows/lfs-warning.yml b/.github/workflows/lfs-warning.yml
new file mode 100644
index 00000000000000..4db79ceded9eea
--- /dev/null
+++ b/.github/workflows/lfs-warning.yml
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Check Large File'
+
+on: [push, pull_request_target]
+
+jobs:
+ large-file-checker:
+ name: "Check large file"
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ submodules: recursive
+
+ - name: "Checkout lfs-warning commit"
+ run: |
+ rm -rf ./.github/actions/lfs-warning
+ git clone https://github.com/ppremk/lfs-warning .github/actions/lfs-warning
+ pushd .github/actions/lfs-warning &>/dev/null
+ git checkout 4b98a8a5e6c429c23c34eee02d71553bca216425
+ popd &>/dev/null
+
+ - name: "Check Large File"
+ uses: ./.github/actions/lfs-warning
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ filesizelimit: 1MB
+
diff --git a/README.md b/README.md
index 0630ce6389b25f..c999651ddee68d 100644
--- a/README.md
+++ b/README.md
@@ -24,17 +24,12 @@ under the License.
[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
[![GitHub release](https://img.shields.io/github/release/apache/doris.svg)](https://github.com/apache/doris/releases)
[![OSSRank](https://shields.io/endpoint?url=https://ossrank.com/shield/516)](https://ossrank.com/p/516)
-[![Jenkins Vec](https://img.shields.io/jenkins/tests?compact_message&jobUrl=https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized&label=VectorizedEngine)](https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized)
-[![Total Line](https://img.shields.io/badge/Total_Line-GitHub-blue)]((https://github.com/apache/doris))
-[![Join the chat at https://gitter.im/apache-doris/Lobby](https://badges.gitter.im/apache-doris/Lobby.svg)](https://gitter.im/apache-doris/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Commit activity](https://img.shields.io/github/commit-activity/m/apache/doris)](https://github.com/apache/doris/commits/master/)
[![EN doc](https://img.shields.io/badge/Docs-English-blue.svg)](https://doris.apache.org/docs/get-starting/quick-start)
[![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://doris.apache.org/zh-CN/docs/get-starting/quick-start/)
-
-
-
[![Official Website](
)](https://doris.apache.org/)
[![Quick Download]()](https://doris.apache.org/download)
@@ -64,10 +59,10 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
-🎉 Version 2.1.0 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.0) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
+🎉 Version 2.1.4 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.4) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
-🎉 Version 2.0.6 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.0.6) here.
+🎉 Version 2.0.12 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/2.0/releasenotes/release-2.0.12) here.
👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 51cf86fe4083cf..f554ba6053a5e6 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -157,11 +157,20 @@ set(BOOST_VERSION "1.81.0")
if (NOT APPLE)
find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time)
+ find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system container)
else()
find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time)
find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
endif()
+# Set if use libazure or not
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+ add_definitions(-DUSE_AZURE)
+endif()
+
+
set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
include (cmake/thirdparty.cmake)
@@ -290,12 +299,11 @@ if (COMPILER_CLANG)
-Wno-implicit-float-conversion
-Wno-implicit-int-conversion
-Wno-sign-conversion
+ -Wno-missing-field-initializers
+ -Wno-unused-const-variable
-Wno-shorten-64-to-32)
if (USE_LIBCPP)
add_compile_options($<$:-stdlib=libc++>)
- if (NOT OS_MACOSX)
- add_compile_options($<$:-lstdc++>)
- endif()
add_definitions(-DUSE_LIBCPP)
endif()
endif ()
@@ -358,10 +366,6 @@ if (USE_UNWIND)
endif()
endif()
-if (ENABLE_STACKTRACE)
- add_definitions(-DENABLE_STACKTRACE)
-endif()
-
if (USE_DWARF)
add_compile_options(-gdwarf-5)
endif()
@@ -369,29 +373,29 @@ endif()
# For CMAKE_BUILD_TYPE=Debug
if (OS_MACOSX AND ARCH_ARM)
# Using -O0 may meet ARM64 branch out of range errors when linking with tcmalloc.
- set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Og")
+ set(CXX_FLAGS_DEBUG "-Og")
else()
- set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -O0")
+ set(CXX_FLAGS_DEBUG "-O0")
endif()
# For CMAKE_BUILD_TYPE=Release
# -O3: Enable all compiler optimizations
# -DNDEBUG: Turn off dchecks/asserts/debug only code.
-set(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG")
-set(CXX_FLAGS_ASAN "${CXX_GCC_FLAGS} -O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
-set(CXX_FLAGS_LSAN "${CXX_GCC_FLAGS} -O0 -fsanitize=leak -DLEAK_SANITIZER")
+set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
+set(CXX_FLAGS_ASAN "-O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
+set(CXX_FLAGS_LSAN "-O0 -fsanitize=leak -DLEAK_SANITIZER")
## Use for BE-UT
-set(CXX_FLAGS_ASAN_UT "${CXX_GCC_FLAGS} -O0 -fsanitize=address -DADDRESS_SANITIZER")
+set(CXX_FLAGS_ASAN_UT "-O0 -fsanitize=address -DADDRESS_SANITIZER")
# Set the flags to the undefined behavior sanitizer, also known as "ubsan"
# Turn on sanitizer and debug symbols to get stack traces:
-set(CXX_FLAGS_UBSAN "${CXX_GCC_FLAGS} -O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
+set(CXX_FLAGS_UBSAN "-O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
# Set the flags to the thread sanitizer, also known as "tsan"
# Turn on sanitizer and debug symbols to get stack traces:
# Use -Wno-builtin-declaration-mismatch to mute warnings like "new declaration ‘__tsan_atomic16 __tsan_atomic16_fetch_nand(..."
# If use -O0 to compile, BE will stack overflow when start. https://github.com/apache/doris/issues/8868
-set(CXX_FLAGS_TSAN "${CXX_GCC_FLAGS} -O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
+set(CXX_FLAGS_TSAN "-O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
# Set compile flags based on the build type.
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
@@ -513,6 +517,7 @@ find_package(absl)
# add it here first.
set(COMMON_THIRDPARTY
Boost::date_time
+ Boost::container
${COMMON_THIRDPARTY}
)
@@ -555,7 +560,6 @@ endif()
if (OS_MACOSX)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
- Boost::container
bfd
iberty
intl
@@ -599,9 +603,11 @@ if (NOT OS_MACOSX)
${DORIS_DEPENDENCIES}
-static-libstdc++
-static-libgcc
- -lstdc++fs
-lresolv
)
+ if (NOT (USE_LIBCPP AND COMPILER_CLANG))
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs)
+ endif()
else()
set(DORIS_LINK_LIBS
${DORIS_LINK_LIBS}
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 502a22bbdf4532..19f2a00012aafb 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -141,10 +141,12 @@ if (NOT OS_MACOSX)
add_thirdparty(aws-s2n LIBNAME "lib/libs2n.a")
endif()
-add_thirdparty(azure-core)
-add_thirdparty(azure-identity)
-add_thirdparty(azure-storage-blobs)
-add_thirdparty(azure-storage-common)
+if(BUILD_AZURE STREQUAL "ON")
+ add_thirdparty(azure-core)
+ add_thirdparty(azure-identity)
+ add_thirdparty(azure-storage-blobs)
+ add_thirdparty(azure-storage-common)
+endif()
add_thirdparty(minizip LIB64)
add_thirdparty(simdjson LIB64)
diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h
index ec6ddf497ec084..a55e26f7ba4493 100644
--- a/be/src/agent/be_exec_version_manager.h
+++ b/be/src/agent/be_exec_version_manager.h
@@ -80,8 +80,9 @@ class BeExecVersionManager {
* b. clear old version of version 3->4
* c. change FunctionIsIPAddressInRange from AlwaysNotNullable to DependOnArguments
* d. change some agg function nullable property: PR #37215
+ * e. change variant serde to fix PR #38413
*/
-constexpr inline int BeExecVersionManager::max_be_exec_version = 5;
+constexpr inline int BeExecVersionManager::max_be_exec_version = 6;
constexpr inline int BeExecVersionManager::min_be_exec_version = 0;
/// functional
@@ -89,5 +90,6 @@ constexpr inline int BITMAP_SERDE = 3;
constexpr inline int USE_NEW_SERDE = 4; // release on DORIS version 2.1
constexpr inline int OLD_WAL_SERDE = 3; // use to solve compatibility issues, see pr #32299
constexpr inline int AGG_FUNCTION_NULLABLE = 5; // change some agg nullable property: PR #37215
+constexpr inline int VARIANT_SERDE = 6; // change variant serde to fix PR #38413
} // namespace doris
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 0e851fba17a035..7bbd602f571ede 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -40,6 +40,7 @@
#include
#include
#include
+#include
#include
#include
@@ -103,6 +104,10 @@ std::unordered_map> s_task_signatur
std::atomic_ulong s_report_version(time(nullptr) * 10000);
+void increase_report_version() {
+ s_report_version.fetch_add(1, std::memory_order_relaxed);
+}
+
// FIXME(plat1ko): Paired register and remove task info
bool register_task_info(const TTaskType::type task_type, int64_t signature) {
if (task_type == TTaskType::type::PUSH_STORAGE_POLICY ||
@@ -213,7 +218,7 @@ void alter_tablet(StorageEngine& engine, const TAgentTaskRequest& agent_task_req
}
if (status.ok()) {
- s_report_version.fetch_add(1, std::memory_order_relaxed);
+ increase_report_version();
}
// Return result to fe
@@ -289,7 +294,7 @@ void alter_cloud_tablet(CloudStorageEngine& engine, const TAgentTaskRequest& age
}
if (status.ok()) {
- s_report_version.fetch_add(1, std::memory_order_relaxed);
+ increase_report_version();
}
// Return result to fe
@@ -1384,9 +1389,12 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr
auto client = static_cast(existed_fs.get())->client_holder();
auto new_s3_conf = S3Conf::get_s3_conf(param.s3_storage_param);
S3ClientConf conf {
+ .endpoint {},
+ .region {},
.ak = std::move(new_s3_conf.client_conf.ak),
.sk = std::move(new_s3_conf.client_conf.sk),
.token = std::move(new_s3_conf.client_conf.token),
+ .bucket {},
.provider = new_s3_conf.client_conf.provider,
};
st = client->reset(conf);
@@ -1530,7 +1538,7 @@ void create_tablet_callback(StorageEngine& engine, const TAgentTaskRequest& req)
.tag("tablet_id", create_tablet_req.tablet_id)
.error(status);
} else {
- s_report_version.fetch_add(1, std::memory_order_relaxed);
+ increase_report_version();
// get path hash of the created tablet
TabletSharedPtr tablet;
{
@@ -1625,7 +1633,7 @@ void push_callback(StorageEngine& engine, const TAgentTaskRequest& req) {
.tag("signature", req.signature)
.tag("tablet_id", push_req.tablet_id)
.tag("push_type", push_req.push_type);
- ++s_report_version;
+ increase_report_version();
finish_task_request.__set_finish_tablet_infos(tablet_infos);
} else {
LOG_WARNING("failed to execute push task")
@@ -1671,7 +1679,7 @@ void cloud_push_callback(CloudStorageEngine& engine, const TAgentTaskRequest& re
.tag("signature", req.signature)
.tag("tablet_id", push_req.tablet_id)
.tag("push_type", push_req.push_type);
- ++s_report_version;
+ increase_report_version();
auto& tablet_info = finish_task_request.finish_tablet_infos.emplace_back();
// Just need tablet_id
tablet_info.tablet_id = push_req.tablet_id;
@@ -1789,7 +1797,7 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
if (tablet->exceed_version_limit(config::max_tablet_version_num * 2 / 3) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
- tablet, CompactionType::CUMULATIVE_COMPACTION, true);
+ tablet, CompactionType::CUMULATIVE_COMPACTION, true, false);
if (!st.ok()) [[unlikely]] {
LOG(WARNING) << "trigger compaction failed, tablet_id=" << tablet_id
<< ", published=" << published_count << " : " << st;
@@ -1968,6 +1976,10 @@ void clone_callback(StorageEngine& engine, const TMasterInfo& master_info,
LOG_INFO("successfully clone tablet")
.tag("signature", req.signature)
.tag("tablet_id", clone_req.tablet_id);
+ if (engine_task.is_new_tablet()) {
+ increase_report_version();
+ finish_task_request.__set_report_version(s_report_version);
+ }
finish_task_request.__set_finish_tablet_infos(tablet_infos);
}
diff --git a/be/src/agent/workload_group_listener.cpp b/be/src/agent/workload_group_listener.cpp
index 61af4543196ecc..f0f57869f2545a 100644
--- a/be/src/agent/workload_group_listener.cpp
+++ b/be/src/agent/workload_group_listener.cpp
@@ -61,6 +61,9 @@ void WorkloadGroupListener::handle_topic_info(const std::vector& topi
// 4 create and update task scheduler
wg->upsert_task_scheduler(&workload_group_info, _exec_env);
+ // 5 upsert io throttle
+ wg->upsert_scan_io_throttle(&workload_group_info);
+
LOG(INFO) << "[topic_publish_wg]update workload group finish, wg info="
<< wg->debug_string() << ", enable_cpu_hard_limit="
<< (_exec_env->workload_group_mgr()->enable_cpu_hard_limit() ? "true" : "false")
diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp
index f576b60045d54d..d91e9e416b81a1 100644
--- a/be/src/cloud/cloud_backend_service.cpp
+++ b/be/src/cloud/cloud_backend_service.cpp
@@ -29,6 +29,8 @@
#include "common/status.h"
#include "io/cache/block_file_cache_downloader.h"
#include "io/cache/block_file_cache_factory.h"
+#include "runtime/stream_load/stream_load_context.h"
+#include "runtime/stream_load/stream_load_recorder.h"
#include "util/brpc_client_cache.h" // BrpcClientCache
#include "util/thrift_server.h"
@@ -186,4 +188,10 @@ void CloudBackendService::check_warm_up_cache_async(TCheckWarmUpCacheAsyncRespon
response.status = t_status;
}
+void CloudBackendService::get_stream_load_record(TStreamLoadRecordResult& result,
+ int64_t last_stream_record_time) {
+ BaseBackendService::get_stream_load_record(result, last_stream_record_time,
+ _engine.get_stream_load_recorder());
+}
+
} // namespace doris
diff --git a/be/src/cloud/cloud_backend_service.h b/be/src/cloud/cloud_backend_service.h
index 88f0099fe73f09..358cb4d1f0b2ec 100644
--- a/be/src/cloud/cloud_backend_service.h
+++ b/be/src/cloud/cloud_backend_service.h
@@ -53,6 +53,9 @@ class CloudBackendService final : public BaseBackendService {
void check_warm_up_cache_async(TCheckWarmUpCacheAsyncResponse& response,
const TCheckWarmUpCacheAsyncRequest& request) override;
+ void get_stream_load_record(TStreamLoadRecordResult& result,
+ int64_t last_stream_record_time) override;
+
private:
CloudStorageEngine& _engine;
};
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index fc56f971cad522..b8c4ee20cb2077 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -268,6 +268,10 @@ int32_t CloudTimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
continue;
}
return transient_size;
+ } else if (
+ *compaction_score >=
+ config::compaction_max_rowset_count) { // If the number of rowsets is too large: FDB_ERROR_CODE_TXN_TOO_LARGE
+ return transient_size;
}
}
diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp
index b1b455d2007e1f..12cfbb7c0035fd 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -38,6 +38,7 @@
#include "cloud/cloud_warm_up_manager.h"
#include "cloud/config.h"
#include "io/cache/block_file_cache_downloader.h"
+#include "io/cache/block_file_cache_factory.h"
#include "io/cache/file_cache_common.h"
#include "io/fs/file_system.h"
#include "io/fs/hdfs_file_system.h"
@@ -48,6 +49,7 @@
#include "olap/memtable_flush_executor.h"
#include "olap/storage_policy.h"
#include "runtime/memory/cache_manager.h"
+#include "util/parse_util.h"
namespace doris {
@@ -180,14 +182,21 @@ Status CloudStorageEngine::open() {
// TODO(plat1ko): DeleteBitmapTxnManager
_memtable_flush_executor = std::make_unique();
- // TODO(plat1ko): Use file cache disks number?
- _memtable_flush_executor->init(1);
+ // Use file cache disks number
+ _memtable_flush_executor->init(io::FileCacheFactory::instance()->get_cache_instance_size());
_calc_delete_bitmap_executor = std::make_unique();
_calc_delete_bitmap_executor->init();
- _txn_delete_bitmap_cache =
- std::make_unique(config::delete_bitmap_agg_cache_capacity);
+ // The default cache is set to 100MB, use memory limit to dynamic adjustment
+ bool is_percent = false;
+ int64_t delete_bitmap_agg_cache_cache_limit =
+ ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit,
+ MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent);
+ _txn_delete_bitmap_cache = std::make_unique(
+ delete_bitmap_agg_cache_cache_limit > config::delete_bitmap_agg_cache_capacity
+ ? delete_bitmap_agg_cache_cache_limit
+ : config::delete_bitmap_agg_cache_capacity);
RETURN_IF_ERROR(_txn_delete_bitmap_cache->init());
_file_cache_block_downloader = std::make_unique(*this);
@@ -196,6 +205,10 @@ Status CloudStorageEngine::open() {
_tablet_hotspot = std::make_unique();
+ RETURN_NOT_OK_STATUS_WITH_WARN(
+ init_stream_load_recorder(ExecEnv::GetInstance()->store_paths()[0].path),
+ "init StreamLoadRecorder failed");
+
return ThreadPoolBuilder("SyncLoadForTabletsThreadPool")
.set_max_threads(config::sync_load_for_tablets_thread)
.set_min_threads(config::sync_load_for_tablets_thread)
diff --git a/be/src/cloud/cloud_stream_load_executor.cpp b/be/src/cloud/cloud_stream_load_executor.cpp
index a87f37a5188600..1b8167c96ebd48 100644
--- a/be/src/cloud/cloud_stream_load_executor.cpp
+++ b/be/src/cloud/cloud_stream_load_executor.cpp
@@ -60,7 +60,10 @@ Status CloudStreamLoadExecutor::operate_txn_2pc(StreamLoadContext* ctx) {
Status st = Status::InternalError("impossible branch reached, " + op_info);
if (ctx->txn_operation.compare("commit") == 0) {
- if (topt == TxnOpParamType::WITH_TXN_ID) {
+ if (!config::enable_stream_load_commit_txn_on_be) {
+ VLOG_DEBUG << "2pc commit stream load txn with FE support: " << op_info;
+ st = StreamLoadExecutor::operate_txn_2pc(ctx);
+ } else if (topt == TxnOpParamType::WITH_TXN_ID) {
VLOG_DEBUG << "2pc commit stream load txn directly: " << op_info;
st = _exec_env->storage_engine().to_cloud().meta_mgr().commit_txn(*ctx, true);
} else if (topt == TxnOpParamType::WITH_LABEL) {
@@ -93,12 +96,9 @@ Status CloudStreamLoadExecutor::operate_txn_2pc(StreamLoadContext* ctx) {
}
Status CloudStreamLoadExecutor::commit_txn(StreamLoadContext* ctx) {
- if (ctx->load_type == TLoadType::ROUTINE_LOAD) {
- return StreamLoadExecutor::commit_txn(ctx);
- }
-
// forward to fe to excute commit transaction for MoW table
- if (ctx->is_mow_table()) {
+ if (ctx->is_mow_table() || !config::enable_stream_load_commit_txn_on_be ||
+ ctx->load_type == TLoadType::ROUTINE_LOAD) {
Status st;
int retry_times = 0;
while (retry_times < config::mow_stream_load_commit_retry_times) {
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index d2596d8a7d2d0c..17ec1fe22b0d85 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -26,7 +26,9 @@
#include
#include
+#include
#include
+#include
#include "cloud/cloud_meta_mgr.h"
#include "cloud/cloud_storage_engine.h"
@@ -42,8 +44,10 @@
#include "olap/rowset/rowset_writer.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
#include "olap/storage_policy.h"
+#include "olap/tablet_schema.h"
#include "olap/txn_manager.h"
#include "util/debug_points.h"
+#include "vec/common/schema_util.h"
namespace doris {
using namespace ErrorCode;
@@ -131,6 +135,19 @@ Status CloudTablet::sync_rowsets(int64_t query_version, bool warmup_delta_data)
return st;
}
+TabletSchemaSPtr CloudTablet::merged_tablet_schema() const {
+ std::shared_lock rdlock(_meta_lock);
+ TabletSchemaSPtr target_schema;
+ std::vector schemas;
+ for (const auto& [_, rowset] : _rs_version_map) {
+ schemas.push_back(rowset->tablet_schema());
+ }
+ // get the max version schema and merge all schema
+ static_cast(
+ vectorized::schema_util::get_least_common_schema(schemas, nullptr, target_schema));
+ return target_schema;
+}
+
// Sync tablet meta and all rowset meta if not running.
// This could happen when BE didn't finish schema change job and another BE committed this schema change job.
// It should be a quite rare situation.
@@ -227,6 +244,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_
{
.expiration_time = expiration_time,
},
+ .download_done {},
});
}
#endif
@@ -463,6 +481,7 @@ int64_t CloudTablet::get_cloud_base_compaction_score() const {
if (_tablet_meta->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY) {
bool has_delete = false;
int64_t point = cumulative_layer_point();
+ std::shared_lock rlock(_meta_lock);
for (const auto& rs_meta : _tablet_meta->all_rs_metas()) {
if (rs_meta->start_version() >= point) {
continue;
diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h
index ca05759cdbf83e..10ff1835e6c830 100644
--- a/be/src/cloud/cloud_tablet.h
+++ b/be/src/cloud/cloud_tablet.h
@@ -147,18 +147,6 @@ class CloudTablet final : public BaseTablet {
std::vector pick_candidate_rowsets_to_base_compaction();
- void traverse_rowsets(std::function visitor,
- bool include_stale = false) {
- std::shared_lock rlock(_meta_lock);
- for (auto& [v, rs] : _rs_version_map) {
- visitor(rs);
- }
- if (!include_stale) return;
- for (auto& [v, rs] : _stale_rs_version_map) {
- visitor(rs);
- }
- }
-
inline Version max_version() const {
std::shared_lock rdlock(_meta_lock);
return _tablet_meta->max_version();
@@ -206,6 +194,9 @@ class CloudTablet final : public BaseTablet {
int64_t last_cumu_compaction_success_time_ms = 0;
int64_t last_cumu_no_suitable_version_ms = 0;
+ // Return merged extended schema
+ TabletSchemaSPtr merged_tablet_schema() const override;
+
private:
// FIXME(plat1ko): No need to record base size if rowsets are ordered by version
void update_base_size(const Rowset& rs);
diff --git a/be/src/cloud/cloud_tablet_hotspot.cpp b/be/src/cloud/cloud_tablet_hotspot.cpp
index ae8b3a54d2b6cf..dd197268646fbc 100644
--- a/be/src/cloud/cloud_tablet_hotspot.cpp
+++ b/be/src/cloud/cloud_tablet_hotspot.cpp
@@ -89,20 +89,20 @@ void TabletHotspot::get_top_n_hot_partition(std::vector* hot_t
hot_partition.qpd = std::max(hot_partition.qpd, counter->qpd());
hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw());
hot_partition.last_access_time =
- std::max(hot_partition.last_access_time,
- std::chrono::duration_cast(
- counter->last_access_time.time_since_epoch())
- .count());
+ std::max(hot_partition.last_access_time,
+ std::chrono::duration_cast(
+ counter->last_access_time.time_since_epoch())
+ .count());
} else if (counter->qpw() != 0) {
auto& hot_partition = week_hot_partitions[std::make_pair(
counter->table_id, counter->index_id)][counter->partition_id];
hot_partition.qpd = 0;
hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw());
hot_partition.last_access_time =
- std::max(hot_partition.last_access_time,
- std::chrono::duration_cast(
- counter->last_access_time.time_since_epoch())
- .count());
+ std::max(hot_partition.last_access_time,
+ std::chrono::duration_cast(
+ counter->last_access_time.time_since_epoch())
+ .count());
}
}
});
diff --git a/be/src/cloud/cloud_tablets_channel.cpp b/be/src/cloud/cloud_tablets_channel.cpp
index e063ab68116bb2..85b8e3ea33a865 100644
--- a/be/src/cloud/cloud_tablets_channel.cpp
+++ b/be/src/cloud/cloud_tablets_channel.cpp
@@ -59,15 +59,20 @@ Status CloudTabletsChannel::add_batch(const PTabletWriterAddBlockRequest& reques
_build_tablet_to_rowidxs(request, &tablet_to_rowidxs);
std::unordered_set partition_ids;
- for (auto& [tablet_id, _] : tablet_to_rowidxs) {
- auto tablet_writer_it = _tablet_writers.find(tablet_id);
- if (tablet_writer_it == _tablet_writers.end()) {
- return Status::InternalError("unknown tablet to append data, tablet={}", tablet_id);
+ {
+ // add_batch may concurrency with inc_open but not under _lock.
+ // so need to protect it with _tablet_writers_lock.
+ std::lock_guard l(_tablet_writers_lock);
+ for (auto& [tablet_id, _] : tablet_to_rowidxs) {
+ auto tablet_writer_it = _tablet_writers.find(tablet_id);
+ if (tablet_writer_it == _tablet_writers.end()) {
+ return Status::InternalError("unknown tablet to append data, tablet={}", tablet_id);
+ }
+ partition_ids.insert(tablet_writer_it->second->partition_id());
+ }
+ if (!partition_ids.empty()) {
+ RETURN_IF_ERROR(_init_writers_by_partition_ids(partition_ids));
}
- partition_ids.insert(tablet_writer_it->second->partition_id());
- }
- if (!partition_ids.empty()) {
- RETURN_IF_ERROR(_init_writers_by_partition_ids(partition_ids));
}
return _write_block_data(request, cur_seq, tablet_to_rowidxs, response);
@@ -124,7 +129,7 @@ Status CloudTabletsChannel::close(LoadChannel* parent, const PTabletWriterAddBlo
_state = kFinished;
// All senders are closed
- // 1. close all delta writers
+ // 1. close all delta writers. under _lock.
std::vector writers_to_commit;
writers_to_commit.reserve(_tablet_writers.size());
bool success = true;
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 80522759b84b44..82c466120e94fb 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -35,7 +35,7 @@ DEFINE_Int64(tablet_cache_shards, "16");
DEFINE_mInt32(tablet_sync_interval_s, "1800");
DEFINE_mInt64(min_compaction_failure_interval_ms, "5000");
-DEFINE_mInt64(base_compaction_freeze_interval_s, "86400");
+DEFINE_mInt64(base_compaction_freeze_interval_s, "7200");
DEFINE_mInt64(cu_compaction_freeze_interval_s, "1200");
DEFINE_mInt64(cumu_compaction_interval_s, "1800");
@@ -48,6 +48,7 @@ DEFINE_mDouble(cumu_compaction_thread_num_factor, "0.5");
DEFINE_mInt32(check_auto_compaction_interval_seconds, "5");
DEFINE_mInt32(max_base_compaction_task_num_per_disk, "2");
DEFINE_mBool(prioritize_query_perf_in_compaction, "false");
+DEFINE_mInt32(compaction_max_rowset_count, "10000");
DEFINE_mInt32(refresh_s3_info_interval_s, "60");
DEFINE_mInt32(vacuum_stale_rowsets_interval_s, "300");
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index bf041ba0fa6fc5..02e7014801e566 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -79,6 +79,7 @@ DECLARE_mDouble(cumu_compaction_thread_num_factor);
DECLARE_mInt32(check_auto_compaction_interval_seconds);
DECLARE_mInt32(max_base_compaction_task_num_per_disk);
DECLARE_mBool(prioritize_query_perf_in_compaction);
+DECLARE_mInt32(compaction_max_rowset_count);
// CloudStorageEngine config
DECLARE_mInt32(refresh_s3_info_interval_s);
diff --git a/be/src/clucene b/be/src/clucene
index 5db9db68e448b8..fdbf2204031128 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit 5db9db68e448b8ccfd360d02666bbac44e6f8d1a
+Subproject commit fdbf2204031128b2bd8505fc73c06403b7c1a815
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 28c19df70bcb80..a68b4c60c01ad6 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -135,9 +135,9 @@ DEFINE_mBool(enable_query_memory_overcommit, "true");
DEFINE_mBool(disable_memory_gc, "false");
-DEFINE_mBool(enable_stacktrace_in_allocator_check_failed, "false");
+DEFINE_mBool(enable_stacktrace, "true");
-DEFINE_mInt64(large_memory_check_bytes, "2147483648");
+DEFINE_mInt64(stacktrace_in_alloc_large_memory_bytes, "2147483648");
DEFINE_mBool(enable_memory_orphan_check, "false");
@@ -248,6 +248,7 @@ DEFINE_Validator(doris_scanner_thread_pool_thread_num, [](const int config) -> b
}
return true;
});
+DEFINE_Int32(doris_scanner_min_thread_pool_thread_num, "8");
DEFINE_Int32(remote_split_source_batch_size, "10240");
DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1");
// number of olap scanner thread pool queue size
@@ -482,6 +483,8 @@ DEFINE_mInt32(migration_remaining_size_threshold_mb, "10");
// If the task runs longer than this time, the task will be terminated, in seconds.
// timeout = std::max(migration_task_timeout_secs, tablet size / 1MB/s)
DEFINE_mInt32(migration_task_timeout_secs, "300");
+// timeout for try_lock migration lock
+DEFINE_Int64(migration_lock_timeout_ms, "1000");
// Port to start debug webserver on
DEFINE_Int32(webserver_port, "8040");
@@ -539,6 +542,8 @@ DEFINE_mInt32(stream_load_record_batch_size, "50");
DEFINE_Int32(stream_load_record_expire_time_secs, "28800");
// time interval to clean expired stream load records
DEFINE_mInt64(clean_stream_load_record_interval_secs, "1800");
+// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
+DEFINE_mBool(enable_stream_load_commit_txn_on_be, "false");
// The buffer size to store stream table function schema info
DEFINE_Int64(stream_tvf_buffer_size, "1048576"); // 1MB
@@ -594,8 +599,7 @@ DEFINE_mInt32(memory_gc_sleep_time_ms, "1000");
// Sleep time in milliseconds between memtbale flush mgr refresh iterations
DEFINE_mInt64(memtable_mem_tracker_refresh_interval_ms, "5");
-// Sleep time in milliseconds between refresh iterations of workload group memory statistics
-DEFINE_mInt64(wg_mem_refresh_interval_ms, "50");
+DEFINE_mInt64(wg_weighted_memory_ratio_refresh_interval_ms, "50");
// percent of (active memtables size / all memtables size) when reach hard limit
DEFINE_mInt32(memtable_hard_limit_active_percent, "50");
@@ -628,6 +632,8 @@ DEFINE_Int32(load_process_safe_mem_permit_percent, "5");
// result buffer cancelled time (unit: second)
DEFINE_mInt32(result_buffer_cancelled_interval_time, "300");
+DEFINE_mInt32(arrow_flight_result_sink_buffer_size_rows, "32768");
+
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
DEFINE_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
@@ -930,7 +936,8 @@ DEFINE_mInt32(cold_data_compaction_interval_sec, "1800");
DEFINE_String(tmp_file_dir, "tmp");
-DEFINE_Int32(s3_transfer_executor_pool_size, "2");
+DEFINE_Int32(min_s3_file_system_thread_num, "16");
+DEFINE_Int32(max_s3_file_system_thread_num, "64");
DEFINE_Bool(enable_time_lut, "true");
DEFINE_mBool(enable_simdjson_reader, "true");
@@ -1001,6 +1008,7 @@ DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true");
DEFINE_mBool(variant_enable_flatten_nested, "false");
DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1");
DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "1000");
+DEFINE_mBool(variant_throw_exeception_on_invalid_json, "false");
// block file cache
DEFINE_Bool(enable_file_cache, "false");
@@ -1062,8 +1070,6 @@ DEFINE_mInt64(max_tablet_io_errors, "-1");
DEFINE_Int32(tablet_path_check_interval_seconds, "-1");
DEFINE_mInt32(tablet_path_check_batch_size, "1000");
-// Page size of row column, default 4KB
-DEFINE_mInt64(row_column_page_size, "4096");
// it must be larger than or equal to 5MB
DEFINE_mInt64(s3_write_buffer_size, "5242880");
// Log interval when doing s3 upload task
@@ -1234,6 +1240,13 @@ DEFINE_mInt32(s3_read_base_wait_time_ms, "100");
DEFINE_mInt32(s3_read_max_wait_time_ms, "800");
DEFINE_mBool(enable_s3_rate_limiter, "false");
+DEFINE_mInt64(s3_get_bucket_tokens, "1000000000000000000");
+DEFINE_mInt64(s3_get_token_per_second, "1000000000000000000");
+DEFINE_mInt64(s3_get_token_limit, "0");
+
+DEFINE_mInt64(s3_put_bucket_tokens, "1000000000000000000");
+DEFINE_mInt64(s3_put_token_per_second, "1000000000000000000");
+DEFINE_mInt64(s3_put_token_limit, "0");
DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors");
@@ -1331,6 +1344,8 @@ DEFINE_mBool(ignore_not_found_file_in_external_table, "true");
DEFINE_mBool(enable_hdfs_mem_limiter, "true");
+DEFINE_mInt16(topn_agg_limit_multiplier, "2");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index dd44f56fd300a9..3c43ed66593e51 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -183,13 +183,14 @@ DECLARE_mBool(enable_query_memory_overcommit);
// default gc strategy is conservative, if you want to exclude the interference of gc, let it be true
DECLARE_mBool(disable_memory_gc);
-// Allocator check failed log stacktrace if not catch exception
-DECLARE_mBool(enable_stacktrace_in_allocator_check_failed);
+// if false, turn off all stacktrace
+DECLARE_mBool(enable_stacktrace);
-// malloc or new large memory larger than large_memory_check_bytes, default 2G,
-// will print a warning containing the stacktrace, but not prevent memory alloc.
-// If is -1, disable large memory check.
-DECLARE_mInt64(large_memory_check_bytes);
+// when alloc memory larger than stacktrace_in_alloc_large_memory_bytes, default 2G,
+// if alloc successful, will print a warning with stacktrace, but not prevent memory alloc.
+// if alloc failed using Doris Allocator, will print stacktrace in error log.
+// if is -1, disable print stacktrace when alloc large memory.
+DECLARE_mInt64(stacktrace_in_alloc_large_memory_bytes);
// default is true. if any memory tracking in Orphan mem tracker will report error.
DECLARE_mBool(enable_memory_orphan_check);
@@ -296,6 +297,7 @@ DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
// number of scanner thread pool size for olap table
// and the min thread num of remote scanner thread pool
DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
+DECLARE_mInt32(doris_scanner_min_thread_pool_thread_num);
// number of batch size to fetch the remote split source
DECLARE_mInt32(remote_split_source_batch_size);
// max number of remote scanner thread pool size
@@ -531,6 +533,8 @@ DECLARE_mInt32(migration_remaining_size_threshold_mb);
// If the task runs longer than this time, the task will be terminated, in seconds.
// timeout = std::max(migration_task_timeout_secs, tablet size / 1MB/s)
DECLARE_mInt32(migration_task_timeout_secs);
+// timeout for try_lock migration lock
+DECLARE_Int64(migration_lock_timeout_ms);
// Port to start debug webserver on
DECLARE_Int32(webserver_port);
@@ -592,6 +596,8 @@ DECLARE_mInt32(stream_load_record_batch_size);
DECLARE_Int32(stream_load_record_expire_time_secs);
// time interval to clean expired stream load records
DECLARE_mInt64(clean_stream_load_record_interval_secs);
+// enable stream load commit txn on BE directly, bypassing FE. Only for cloud.
+DECLARE_mBool(enable_stream_load_commit_txn_on_be);
// The buffer size to store stream table function schema info
DECLARE_Int64(stream_tvf_buffer_size);
@@ -650,8 +656,8 @@ DECLARE_mInt32(memory_gc_sleep_time_ms);
// Sleep time in milliseconds between memtbale flush mgr memory refresh iterations
DECLARE_mInt64(memtable_mem_tracker_refresh_interval_ms);
-// Sleep time in milliseconds between refresh iterations of workload group memory statistics
-DECLARE_mInt64(wg_mem_refresh_interval_ms);
+// Sleep time in milliseconds between refresh iterations of workload group weighted memory ratio
+DECLARE_mInt64(wg_weighted_memory_ratio_refresh_interval_ms);
// percent of (active memtables size / all memtables size) when reach hard limit
DECLARE_mInt32(memtable_hard_limit_active_percent);
@@ -684,6 +690,9 @@ DECLARE_Int32(load_process_safe_mem_permit_percent);
// result buffer cancelled time (unit: second)
DECLARE_mInt32(result_buffer_cancelled_interval_time);
+// arrow flight result sink buffer rows size, default 4096 * 8
+DECLARE_mInt32(arrow_flight_result_sink_buffer_size_rows);
+
// the increased frequency of priority for remaining tasks in BlockingPriorityQueue
DECLARE_mInt32(priority_queue_remaining_tasks_increased_frequency);
@@ -984,7 +993,8 @@ DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
DECLARE_Int32(cold_data_compaction_thread_num);
DECLARE_mInt32(cold_data_compaction_interval_sec);
-DECLARE_Int32(s3_transfer_executor_pool_size);
+DECLARE_Int32(min_s3_file_system_thread_num);
+DECLARE_Int32(max_s3_file_system_thread_num);
DECLARE_Bool(enable_time_lut);
DECLARE_mBool(enable_simdjson_reader);
@@ -1111,8 +1121,6 @@ DECLARE_mInt64(max_tablet_io_errors);
DECLARE_Int32(tablet_path_check_interval_seconds);
DECLARE_mInt32(tablet_path_check_batch_size);
-// Page size of row column, default 4KB
-DECLARE_mInt64(row_column_page_size);
// it must be larger than or equal to 5MB
DECLARE_mInt64(s3_write_buffer_size);
// Log interval when doing s3 upload task
@@ -1191,6 +1199,8 @@ DECLARE_mDouble(variant_ratio_of_defaults_as_sparse_column);
// Threshold to estimate a column is sparsed
// Notice: TEST ONLY
DECLARE_mInt64(variant_threshold_rows_to_estimate_sparse_column);
+// Treat invalid json format str as string, instead of throwing exception if false
+DECLARE_mBool(variant_throw_exeception_on_invalid_json);
DECLARE_mBool(enable_merge_on_write_correctness_check);
// USED FOR DEBUGING
@@ -1315,6 +1325,13 @@ DECLARE_Int32(spill_io_thread_pool_queue_size);
DECLARE_mBool(check_segment_when_build_rowset_meta);
DECLARE_mBool(enable_s3_rate_limiter);
+DECLARE_mInt64(s3_get_bucket_tokens);
+DECLARE_mInt64(s3_get_token_per_second);
+DECLARE_mInt64(s3_get_token_limit);
+
+DECLARE_mInt64(s3_put_bucket_tokens);
+DECLARE_mInt64(s3_put_token_per_second);
+DECLARE_mInt64(s3_put_token_limit);
// max s3 client retry times
DECLARE_mInt32(max_s3_client_retry);
// When meet s3 429 error, the "get" request will
@@ -1421,6 +1438,10 @@ DECLARE_mBool(ignore_not_found_file_in_external_table);
DECLARE_mBool(enable_hdfs_mem_limiter);
+// Define how many percent data in hashtable bigger than limit
+// we should do agg limit opt
+DECLARE_mInt16(topn_agg_limit_multiplier);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index c97904f5677b44..d8245f4045ce81 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -228,6 +228,7 @@ void Daemon::memory_maintenance_thread() {
DorisMetrics::instance()->system_metrics()->update_allocator_metrics();
}
#endif
+ MemInfo::refresh_memory_bvar();
// Update and print memory stat when the memory changes by 256M.
if (abs(last_print_proc_mem - PerfCounters::get_vm_rss()) > 268435456) {
@@ -387,16 +388,34 @@ void Daemon::je_purge_dirty_pages_thread() const {
if (_stop_background_threads_latch.count() == 0) {
break;
}
+ if (config::disable_memory_gc) {
+ continue;
+ }
doris::MemInfo::je_purge_all_arena_dirty_pages();
doris::MemInfo::je_purge_dirty_pages_notify.store(false, std::memory_order_relaxed);
} while (true);
}
-void Daemon::wg_mem_used_refresh_thread() {
- // Refresh memory usage and limit of workload groups
+void Daemon::cache_prune_stale_thread() {
+ int32_t interval = config::cache_periodic_prune_stale_sweep_sec;
+ while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
+ if (interval <= 0) {
+ LOG(WARNING) << "config of cache clean interval is illegal: [" << interval
+ << "], force set to 3600 ";
+ interval = 3600;
+ }
+ if (config::disable_memory_gc) {
+ continue;
+ }
+ CacheManager::instance()->for_each_cache_prune_stale();
+ }
+}
+
+void Daemon::wg_weighted_memory_ratio_refresh_thread() {
+ // Refresh weighted memory ratio of workload groups
while (!_stop_background_threads_latch.wait_for(
- std::chrono::milliseconds(config::wg_mem_refresh_interval_ms))) {
- doris::ExecEnv::GetInstance()->workload_group_mgr()->refresh_wg_memory_info();
+ std::chrono::milliseconds(config::wg_weighted_memory_ratio_refresh_interval_ms))) {
+ doris::ExecEnv::GetInstance()->workload_group_mgr()->refresh_wg_weighted_memory_limit();
}
}
@@ -435,13 +454,19 @@ void Daemon::start() {
st = Thread::create(
"Daemon", "je_purge_dirty_pages_thread",
[this]() { this->je_purge_dirty_pages_thread(); }, &_threads.emplace_back());
+ CHECK(st.ok()) << st;
+ st = Thread::create(
+ "Daemon", "cache_prune_stale_thread", [this]() { this->cache_prune_stale_thread(); },
+ &_threads.emplace_back());
+ CHECK(st.ok()) << st;
st = Thread::create(
"Daemon", "query_runtime_statistics_thread",
[this]() { this->report_runtime_query_statistics_thread(); }, &_threads.emplace_back());
CHECK(st.ok()) << st;
st = Thread::create(
- "Daemon", "wg_mem_refresh_thread", [this]() { this->wg_mem_used_refresh_thread(); },
+ "Daemon", "wg_weighted_memory_ratio_refresh_thread",
+ [this]() { this->wg_weighted_memory_ratio_refresh_thread(); },
&_threads.emplace_back());
if (config::enable_be_proc_monitor) {
diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h
index 9dfb079b904ad4..64c9f0c8993ae3 100644
--- a/be/src/common/daemon.h
+++ b/be/src/common/daemon.h
@@ -43,8 +43,9 @@ class Daemon {
void memtable_memory_refresh_thread();
void calculate_metrics_thread();
void je_purge_dirty_pages_thread() const;
+ void cache_prune_stale_thread();
void report_runtime_query_statistics_thread();
- void wg_mem_used_refresh_thread();
+ void wg_weighted_memory_ratio_refresh_thread();
void be_proc_monitor_thread();
CountDownLatch _stop_background_threads_latch;
diff --git a/be/src/common/exception.cpp b/be/src/common/exception.cpp
index c6139c0f995fa3..48e1229d44e83b 100644
--- a/be/src/common/exception.cpp
+++ b/be/src/common/exception.cpp
@@ -32,23 +32,4 @@ Exception::Exception(int code, const std::string_view& msg) {
LOG(FATAL) << "[ExitOnException] error code: " << code << ", message: " << msg;
}
}
-
-Exception::Exception(const Exception& nested, int code, const std::string_view& msg) {
- _code = code;
- _err_msg = std::make_unique();
- _err_msg->_msg = msg;
- if (ErrorCode::error_states[abs(code)].stacktrace) {
- _err_msg->_stack = get_stack_trace();
- }
- _nested_excption = std::make_unique();
- _nested_excption->_code = nested._code;
- _nested_excption->_err_msg = std::make_unique();
- _nested_excption->_err_msg->_msg = nested._err_msg->_msg;
- _nested_excption->_err_msg->_stack = nested._err_msg->_stack;
-
- if (config::exit_on_exception) {
- LOG(FATAL) << "[ExitOnException] error code: " << code << ", message: " << msg;
- }
-}
-
} // namespace doris
\ No newline at end of file
diff --git a/be/src/common/exception.h b/be/src/common/exception.h
index ce44e6587499b4..37d6afe937e0a2 100644
--- a/be/src/common/exception.h
+++ b/be/src/common/exception.h
@@ -19,8 +19,8 @@
#include
#include
-#include
+#include
#include
#include
#include
@@ -39,9 +39,6 @@ class Exception : public std::exception {
Exception() : _code(ErrorCode::OK) {}
Exception(int code, const std::string_view& msg);
Exception(const Status& status) : Exception(status.code(), status.msg()) {}
- // add nested exception as first param, or the template may could not find
- // the correct method for ...args
- Exception(const Exception& nested, int code, const std::string_view& msg);
// Format message with fmt::format, like the logging functions.
template
@@ -63,7 +60,6 @@ class Exception : public std::exception {
std::string _stack;
};
std::unique_ptr _err_msg;
- std::unique_ptr _nested_excption;
mutable std::string _cache_string;
};
@@ -71,16 +67,12 @@ inline const std::string& Exception::to_string() const {
if (!_cache_string.empty()) {
return _cache_string;
}
- std::stringstream ostr;
- ostr << "[E" << _code << "] ";
- ostr << (_err_msg ? _err_msg->_msg : "");
+ fmt::memory_buffer buf;
+ fmt::format_to(buf, "[E{}] {}", _code, _err_msg ? _err_msg->_msg : "");
if (_err_msg && !_err_msg->_stack.empty()) {
- ostr << '\n' << _err_msg->_stack;
+ fmt::format_to(buf, "\n{}", _err_msg->_stack);
}
- if (_nested_excption != nullptr) {
- ostr << '\n' << "Caused by:" << _nested_excption->to_string();
- }
- _cache_string = ostr.str();
+ _cache_string = fmt::to_string(buf);
return _cache_string;
}
@@ -139,3 +131,26 @@ inline const std::string& Exception::to_string() const {
} \
} \
} while (0);
+
+#define HANDLE_EXCEPTION_IF_CATCH_EXCEPTION(stmt, exception_handler) \
+ do { \
+ try { \
+ doris::enable_thread_catch_bad_alloc++; \
+ Defer defer {[&]() { doris::enable_thread_catch_bad_alloc--; }}; \
+ { \
+ Status _status_ = (stmt); \
+ if (UNLIKELY(!_status_.ok())) { \
+ exception_handler(doris::Exception()); \
+ return _status_; \
+ } \
+ } \
+ } catch (const doris::Exception& e) { \
+ exception_handler(e); \
+ if (e.code() == doris::ErrorCode::MEM_ALLOC_FAILED) { \
+ return Status::MemoryLimitExceeded(fmt::format( \
+ "PreCatch error code:{}, {}, __FILE__:{}, __LINE__:{}, __FUNCTION__:{}", \
+ e.code(), e.to_string(), __FILE__, __LINE__, __PRETTY_FUNCTION__)); \
+ } \
+ return Status::Error(e.code(), e.to_string()); \
+ } \
+ } while (0);
diff --git a/be/src/common/status.h b/be/src/common/status.h
index c7b815e6b4e03f..11c7c42ac99496 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -17,11 +17,9 @@
#include
#include "common/compiler_util.h" // IWYU pragma: keep
-#ifdef ENABLE_STACKTRACE
-#include "util/stack_util.h"
-#endif
-
+#include "common/config.h"
#include "common/expected.h"
+#include "util/stack_util.h"
namespace doris {
@@ -363,9 +361,9 @@ class [[nodiscard]] Status {
Status(int code, std::string msg, std::string stack = "") : _code(code) {
_err_msg = std::make_unique();
_err_msg->_msg = std::move(msg);
-#ifdef ENABLE_STACKTRACE
- _err_msg->_stack = std::move(stack);
-#endif
+ if (config::enable_stacktrace) {
+ _err_msg->_stack = std::move(stack);
+ }
}
// copy c'tor makes copy of error detail so Status can be returned by value
@@ -416,13 +414,12 @@ class [[nodiscard]] Status {
} else {
status._err_msg->_msg = fmt::format(msg, std::forward(args)...);
}
-#ifdef ENABLE_STACKTRACE
- if (stacktrace && ErrorCode::error_states[abs(code)].stacktrace) {
+ if (stacktrace && ErrorCode::error_states[abs(code)].stacktrace &&
+ config::enable_stacktrace) {
// Delete the first one frame pointers, which are inside the status.h
status._err_msg->_stack = get_stack_trace(1);
LOG(WARNING) << "meet error status: " << status; // may print too many stacks.
}
-#endif
return status;
}
@@ -436,12 +433,11 @@ class [[nodiscard]] Status {
} else {
status._err_msg->_msg = fmt::format(msg, std::forward(args)...);
}
-#ifdef ENABLE_STACKTRACE
- if (stacktrace && ErrorCode::error_states[abs(code)].stacktrace) {
+ if (stacktrace && ErrorCode::error_states[abs(code)].stacktrace &&
+ config::enable_stacktrace) {
status._err_msg->_stack = get_stack_trace(1);
LOG(WARNING) << "meet error status: " << status; // may print too many stacks.
}
-#endif
return status;
}
@@ -545,9 +541,7 @@ class [[nodiscard]] Status {
int _code;
struct ErrMsg {
std::string _msg;
-#ifdef ENABLE_STACKTRACE
std::string _stack;
-#endif
};
std::unique_ptr _err_msg;
@@ -604,11 +598,9 @@ class AtomicStatus {
inline std::ostream& operator<<(std::ostream& ostr, const Status& status) {
ostr << '[' << status.code_as_string() << ']';
ostr << status.msg();
-#ifdef ENABLE_STACKTRACE
- if (status._err_msg && !status._err_msg->_stack.empty()) {
+ if (status._err_msg && !status._err_msg->_stack.empty() && config::enable_stacktrace) {
ostr << '\n' << status._err_msg->_stack;
}
-#endif
return ostr;
}
diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h
index 778444346d3d4c..c2199cd6be9360 100644
--- a/be/src/exec/olap_common.h
+++ b/be/src/exec/olap_common.h
@@ -44,6 +44,7 @@
#include "vec/io/io_helper.h"
#include "vec/runtime/ipv4_value.h"
#include "vec/runtime/ipv6_value.h"
+#include "vec/runtime/time_value.h"
#include "vec/runtime/vdatetime_value.h"
namespace doris {
@@ -70,6 +71,8 @@ std::string cast_to_string(T value, int scale) {
std::stringstream ss;
ss << buf;
return ss.str();
+ } else if constexpr (primitive_type == TYPE_TIMEV2) {
+ return TimeValue::to_string(value, scale);
} else if constexpr (primitive_type == TYPE_IPV4) {
return IPv4Value::to_string(value);
} else if constexpr (primitive_type == TYPE_IPV6) {
diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp
index de9857bad2ce83..2ddb3db295b487 100644
--- a/be/src/exec/schema_scanner.cpp
+++ b/be/src/exec/schema_scanner.cpp
@@ -48,10 +48,14 @@
#include "exec/schema_scanner/schema_user_scanner.h"
#include "exec/schema_scanner/schema_variables_scanner.h"
#include "exec/schema_scanner/schema_views_scanner.h"
+#include "exec/schema_scanner/schema_workload_group_privileges.h"
#include "exec/schema_scanner/schema_workload_groups_scanner.h"
#include "exec/schema_scanner/schema_workload_sched_policy_scanner.h"
#include "olap/hll.h"
+#include "pipeline/dependency.h"
#include "runtime/define_primitive_type.h"
+#include "runtime/fragment_mgr.h"
+#include "runtime/types.h"
#include "util/string_util.h"
#include "util/types.h"
#include "vec/columns/column.h"
@@ -65,6 +69,7 @@
#include "vec/core/column_with_type_and_name.h"
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_factory.hpp"
namespace doris {
class ObjectPool;
@@ -85,7 +90,60 @@ Status SchemaScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaScanner::get_next_block(RuntimeState* state, vectorized::Block* block, bool* eos) {
+ if (_data_block == nullptr) {
+ return Status::InternalError("No data left!");
+ }
+ DCHECK(_async_thread_running == false);
+ RETURN_IF_ERROR(_scanner_status.status());
+ for (size_t i = 0; i < block->columns(); i++) {
+ std::move(*block->get_by_position(i).column)
+ .mutate()
+ ->insert_range_from(*_data_block->get_by_position(i).column, 0,
+ _data_block->rows());
+ }
+ _data_block->clear_column_data();
+ *eos = _eos;
+ if (!*eos) {
+ RETURN_IF_ERROR(get_next_block_async(state));
+ }
+ return Status::OK();
+}
+
+Status SchemaScanner::get_next_block_async(RuntimeState* state) {
+ _dependency->block();
+ auto task_ctx = state->get_task_execution_context();
+ RETURN_IF_ERROR(ExecEnv::GetInstance()->fragment_mgr()->get_thread_pool()->submit_func(
+ [this, task_ctx, state]() {
+ DCHECK(_async_thread_running == false);
+ auto task_lock = task_ctx.lock();
+ if (task_lock == nullptr) {
+ _scanner_status.update(Status::InternalError("Task context not exists!"));
+ return;
+ }
+ SCOPED_ATTACH_TASK(state);
+ _dependency->block();
+ _async_thread_running = true;
+ _finish_dependency->block();
+ if (!_opened) {
+ _data_block = vectorized::Block::create_unique();
+ _init_block(_data_block.get());
+ _scanner_status.update(start(state));
+ _opened = true;
+ }
+ bool eos = false;
+ _scanner_status.update(get_next_block_internal(_data_block.get(), &eos));
+ _eos = eos;
+ _async_thread_running = false;
+ _dependency->set_ready();
+ if (eos) {
+ _finish_dependency->set_ready();
+ }
+ }));
+ return Status::OK();
+}
+
+Status SchemaScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("used before initialized.");
}
@@ -170,12 +228,24 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type
return SchemaWorkloadSchedulePolicyScanner::create_unique();
case TSchemaTableType::SCH_TABLE_OPTIONS:
return SchemaTableOptionsScanner::create_unique();
+ case TSchemaTableType::SCH_WORKLOAD_GROUP_PRIVILEGES:
+ return SchemaWorkloadGroupPrivilegesScanner::create_unique();
default:
return SchemaDummyScanner::create_unique();
break;
}
}
+void SchemaScanner::_init_block(vectorized::Block* src_block) {
+ const std::vector& columns_desc(get_column_desc());
+ for (int i = 0; i < columns_desc.size(); ++i) {
+ TypeDescriptor descriptor(columns_desc[i].type);
+ auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true);
+ src_block->insert(vectorized::ColumnWithTypeAndName(data_type->create_column(), data_type,
+ columns_desc[i].name));
+ }
+}
+
Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_t pos,
const std::vector& datas) {
const ColumnDesc& col_desc = _columns[pos];
diff --git a/be/src/exec/schema_scanner.h b/be/src/exec/schema_scanner.h
index a23706ac6a440a..da61d58b943fc4 100644
--- a/be/src/exec/schema_scanner.h
+++ b/be/src/exec/schema_scanner.h
@@ -22,6 +22,7 @@
#include
#include
+#include
#include
#include
#include
@@ -43,6 +44,10 @@ namespace vectorized {
class Block;
}
+namespace pipeline {
+class Dependency;
+}
+
struct SchemaScannerCommonParam {
SchemaScannerCommonParam()
: db(nullptr),
@@ -64,6 +69,7 @@ struct SchemaScannerCommonParam {
int32_t port; // frontend thrift port
int64_t thread_id;
const std::string* catalog = nullptr;
+ std::set fe_addr_list;
};
// scanner parameter from frontend
@@ -94,15 +100,23 @@ class SchemaScanner {
// init object need information, schema etc.
virtual Status init(SchemaScannerParam* param, ObjectPool* pool);
+ Status get_next_block(RuntimeState* state, vectorized::Block* block, bool* eos);
// Start to work
virtual Status start(RuntimeState* state);
- virtual Status get_next_block(vectorized::Block* block, bool* eos);
+ virtual Status get_next_block_internal(vectorized::Block* block, bool* eos);
const std::vector& get_column_desc() const { return _columns; }
// factory function
static std::unique_ptr create(TSchemaTableType::type type);
TSchemaTableType::type type() const { return _schema_table_type; }
+ void set_dependency(std::shared_ptr dep,
+ std::shared_ptr fin_dep) {
+ _dependency = dep;
+ _finish_dependency = fin_dep;
+ }
+ Status get_next_block_async(RuntimeState* state);
protected:
+ void _init_block(vectorized::Block* src_block);
Status fill_dest_column_for_range(vectorized::Block* block, size_t pos,
const std::vector& datas);
@@ -125,6 +139,15 @@ class SchemaScanner {
RuntimeProfile::Counter* _get_table_timer = nullptr;
RuntimeProfile::Counter* _get_describe_timer = nullptr;
RuntimeProfile::Counter* _fill_block_timer = nullptr;
+
+ std::shared_ptr _dependency = nullptr;
+ std::shared_ptr _finish_dependency = nullptr;
+
+ std::unique_ptr _data_block;
+ AtomicStatus _scanner_status;
+ std::atomic _eos = false;
+ std::atomic _opened = false;
+ std::atomic _async_thread_running = false;
};
} // namespace doris
diff --git a/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp b/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp
index 2115a38a6ebce3..46522a36242fc1 100644
--- a/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp
@@ -137,7 +137,7 @@ Status SchemaActiveQueriesScanner::_get_active_queries_block_from_fe() {
return Status::OK();
}
-Status SchemaActiveQueriesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaActiveQueriesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_active_queries_scanner.h b/be/src/exec/schema_scanner/schema_active_queries_scanner.h
index 1df5b1f9d7402d..7e9ae4b8034083 100644
--- a/be/src/exec/schema_scanner/schema_active_queries_scanner.h
+++ b/be/src/exec/schema_scanner/schema_active_queries_scanner.h
@@ -36,7 +36,7 @@ class SchemaActiveQueriesScanner : public SchemaScanner {
~SchemaActiveQueriesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp b/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp
index f1155796ed434d..b35e84a9f9c9f4 100644
--- a/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp
+++ b/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp
@@ -51,7 +51,8 @@ Status SchemaBackendActiveTasksScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaBackendActiveTasksScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaBackendActiveTasksScanner::get_next_block_internal(vectorized::Block* block,
+ bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_backend_active_tasks.h b/be/src/exec/schema_scanner/schema_backend_active_tasks.h
index d8a2a1ffa3f96a..43819818b57f69 100644
--- a/be/src/exec/schema_scanner/schema_backend_active_tasks.h
+++ b/be/src/exec/schema_scanner/schema_backend_active_tasks.h
@@ -36,7 +36,7 @@ class SchemaBackendActiveTasksScanner : public SchemaScanner {
~SchemaBackendActiveTasksScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
index 534f045341b7e3..d06cd8fa745634 100644
--- a/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_charsets_scanner.cpp
@@ -48,7 +48,7 @@ SchemaCharsetsScanner::SchemaCharsetsScanner()
SchemaCharsetsScanner::~SchemaCharsetsScanner() {}
-Status SchemaCharsetsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaCharsetsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("call this before initial.");
}
diff --git a/be/src/exec/schema_scanner/schema_charsets_scanner.h b/be/src/exec/schema_scanner/schema_charsets_scanner.h
index 1f01070875ccf6..d5089c62826b0b 100644
--- a/be/src/exec/schema_scanner/schema_charsets_scanner.h
+++ b/be/src/exec/schema_scanner/schema_charsets_scanner.h
@@ -36,7 +36,7 @@ class SchemaCharsetsScanner : public SchemaScanner {
SchemaCharsetsScanner();
~SchemaCharsetsScanner() override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
struct CharsetStruct {
diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.cpp b/be/src/exec/schema_scanner/schema_collations_scanner.cpp
index 9d50b5216303d8..8592eb7575c387 100644
--- a/be/src/exec/schema_scanner/schema_collations_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_collations_scanner.cpp
@@ -50,7 +50,7 @@ SchemaCollationsScanner::SchemaCollationsScanner()
SchemaCollationsScanner::~SchemaCollationsScanner() {}
-Status SchemaCollationsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaCollationsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("call this before initial.");
}
diff --git a/be/src/exec/schema_scanner/schema_collations_scanner.h b/be/src/exec/schema_scanner/schema_collations_scanner.h
index f0f60538cacce0..2fe200da78d04d 100644
--- a/be/src/exec/schema_scanner/schema_collations_scanner.h
+++ b/be/src/exec/schema_scanner/schema_collations_scanner.h
@@ -36,7 +36,7 @@ class SchemaCollationsScanner : public SchemaScanner {
SchemaCollationsScanner();
~SchemaCollationsScanner() override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
struct CollationStruct {
diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
index deda8af7d8de58..f4e15d2aef0af2 100644
--- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp
@@ -347,7 +347,7 @@ Status SchemaColumnsScanner::_get_new_table() {
return Status::OK();
}
-Status SchemaColumnsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaColumnsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("use this class before inited.");
}
diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.h b/be/src/exec/schema_scanner/schema_columns_scanner.h
index 2499db7ed82a2b..99150c36d109a2 100644
--- a/be/src/exec/schema_scanner/schema_columns_scanner.h
+++ b/be/src/exec/schema_scanner/schema_columns_scanner.h
@@ -38,7 +38,7 @@ class SchemaColumnsScanner : public SchemaScanner {
SchemaColumnsScanner();
~SchemaColumnsScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp
index 1d5956f390ea26..9e3a703d9fb5d6 100644
--- a/be/src/exec/schema_scanner/schema_dummy_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_dummy_scanner.cpp
@@ -40,7 +40,7 @@ Status SchemaDummyScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaDummyScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaDummyScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
*eos = true;
return Status::OK();
}
diff --git a/be/src/exec/schema_scanner/schema_dummy_scanner.h b/be/src/exec/schema_scanner/schema_dummy_scanner.h
index a67f6fa25c1648..0c5e4aabe357e4 100644
--- a/be/src/exec/schema_scanner/schema_dummy_scanner.h
+++ b/be/src/exec/schema_scanner/schema_dummy_scanner.h
@@ -33,7 +33,7 @@ class SchemaDummyScanner : public SchemaScanner {
SchemaDummyScanner();
~SchemaDummyScanner() override;
Status start(RuntimeState* state = nullptr) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
};
} // namespace doris
diff --git a/be/src/exec/schema_scanner/schema_files_scanner.cpp b/be/src/exec/schema_scanner/schema_files_scanner.cpp
index 55b7a338c319e8..20aa07fa69116c 100644
--- a/be/src/exec/schema_scanner/schema_files_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_files_scanner.cpp
@@ -113,7 +113,7 @@ Status SchemaFilesScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaFilesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaFilesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_files_scanner.h b/be/src/exec/schema_scanner/schema_files_scanner.h
index 6805a04be4aacc..bb3b2d68493147 100644
--- a/be/src/exec/schema_scanner/schema_files_scanner.h
+++ b/be/src/exec/schema_scanner/schema_files_scanner.h
@@ -38,7 +38,7 @@ class SchemaFilesScanner : public SchemaScanner {
~SchemaFilesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
int _db_index;
int _table_index;
diff --git a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp
index 928567a2e4a99e..1267c32c8d8dfb 100644
--- a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.cpp
@@ -225,7 +225,7 @@ Status SchemaMetadataNameIdsScanner::_fill_block_impl(vectorized::Block* block)
return Status::OK();
}
-Status SchemaMetadataNameIdsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaMetadataNameIdsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h
index 9981d441d856aa..c3beea7769754d 100644
--- a/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h
+++ b/be/src/exec/schema_scanner/schema_metadata_name_ids_scanner.h
@@ -39,7 +39,7 @@ class SchemaMetadataNameIdsScanner : public SchemaScanner {
~SchemaMetadataNameIdsScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
index f1ad1f594f883f..ea7394e15e12d2 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.cpp
@@ -101,7 +101,7 @@ Status SchemaPartitionsScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaPartitionsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaPartitionsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_partitions_scanner.h b/be/src/exec/schema_scanner/schema_partitions_scanner.h
index 47e1d1fcf87d15..87e55db984a3de 100644
--- a/be/src/exec/schema_scanner/schema_partitions_scanner.h
+++ b/be/src/exec/schema_scanner/schema_partitions_scanner.h
@@ -38,7 +38,7 @@ class SchemaPartitionsScanner : public SchemaScanner {
~SchemaPartitionsScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
int _db_index;
int _table_index;
diff --git a/be/src/exec/schema_scanner/schema_processlist_scanner.cpp b/be/src/exec/schema_scanner/schema_processlist_scanner.cpp
index 0f270a6a8c1777..185ef2ab44237f 100644
--- a/be/src/exec/schema_scanner/schema_processlist_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_processlist_scanner.cpp
@@ -56,14 +56,19 @@ Status SchemaProcessListScanner::start(RuntimeState* state) {
TShowProcessListRequest request;
request.__set_show_full_sql(true);
- RETURN_IF_ERROR(SchemaHelper::show_process_list(*(_param->common_param->ip),
- _param->common_param->port, request,
- &_process_list_result));
+ for (const auto& fe_addr : _param->common_param->fe_addr_list) {
+ TShowProcessListResult tmp_ret;
+ RETURN_IF_ERROR(
+ SchemaHelper::show_process_list(fe_addr.hostname, fe_addr.port, request, &tmp_ret));
+ _process_list_result.process_list.insert(_process_list_result.process_list.end(),
+ tmp_ret.process_list.begin(),
+ tmp_ret.process_list.end());
+ }
return Status::OK();
}
-Status SchemaProcessListScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaProcessListScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("call this before initial.");
}
diff --git a/be/src/exec/schema_scanner/schema_processlist_scanner.h b/be/src/exec/schema_scanner/schema_processlist_scanner.h
index 8aae87e1ef6d0f..c0b0a47f6154ee 100644
--- a/be/src/exec/schema_scanner/schema_processlist_scanner.h
+++ b/be/src/exec/schema_scanner/schema_processlist_scanner.h
@@ -40,7 +40,7 @@ class SchemaProcessListScanner : public SchemaScanner {
~SchemaProcessListScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_processlist_columns;
diff --git a/be/src/exec/schema_scanner/schema_profiling_scanner.cpp b/be/src/exec/schema_scanner/schema_profiling_scanner.cpp
index 2f71eb96f2613a..0a2a64330bb018 100644
--- a/be/src/exec/schema_scanner/schema_profiling_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_profiling_scanner.cpp
@@ -88,7 +88,7 @@ Status SchemaProfilingScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaProfilingScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaProfilingScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_profiling_scanner.h b/be/src/exec/schema_scanner/schema_profiling_scanner.h
index 5399cb14eb43f5..6b969a478aca69 100644
--- a/be/src/exec/schema_scanner/schema_profiling_scanner.h
+++ b/be/src/exec/schema_scanner/schema_profiling_scanner.h
@@ -38,7 +38,7 @@ class SchemaProfilingScanner : public SchemaScanner {
~SchemaProfilingScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
};
diff --git a/be/src/exec/schema_scanner/schema_routine_scanner.cpp b/be/src/exec/schema_scanner/schema_routine_scanner.cpp
index 3d55addee6c093..8c263c99d2d6c8 100644
--- a/be/src/exec/schema_scanner/schema_routine_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_routine_scanner.cpp
@@ -141,7 +141,7 @@ Status SchemaRoutinesScanner::get_block_from_fe() {
return Status::OK();
}
-Status SchemaRoutinesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaRoutinesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_routine_scanner.h b/be/src/exec/schema_scanner/schema_routine_scanner.h
index 543f9e8e8f684a..c60d72340e1104 100644
--- a/be/src/exec/schema_scanner/schema_routine_scanner.h
+++ b/be/src/exec/schema_scanner/schema_routine_scanner.h
@@ -36,7 +36,7 @@ class SchemaRoutinesScanner : public SchemaScanner {
~SchemaRoutinesScanner() override = default;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
index 6ece8e22331e38..16d5f2daba61e7 100644
--- a/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.cpp
@@ -97,7 +97,7 @@ Status SchemaRowsetsScanner::_get_all_rowsets() {
return Status::OK();
}
-Status SchemaRowsetsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaRowsetsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_rowsets_scanner.h b/be/src/exec/schema_scanner/schema_rowsets_scanner.h
index b975cc4231bc20..cad34fc04945e4 100644
--- a/be/src/exec/schema_scanner/schema_rowsets_scanner.h
+++ b/be/src/exec/schema_scanner/schema_rowsets_scanner.h
@@ -40,7 +40,7 @@ class SchemaRowsetsScanner : public SchemaScanner {
~SchemaRowsetsScanner() override = default;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_all_rowsets();
diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
index 9789b6c72d6f30..f529821e5a54e2 100644
--- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.cpp
@@ -82,7 +82,7 @@ Status SchemaSchemaPrivilegesScanner::_get_new_table() {
return Status::OK();
}
-Status SchemaSchemaPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaSchemaPrivilegesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h
index af2ad49634bd49..9522fba908bb2a 100644
--- a/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h
+++ b/be/src/exec/schema_scanner/schema_schema_privileges_scanner.h
@@ -38,7 +38,7 @@ class SchemaSchemaPrivilegesScanner : public SchemaScanner {
~SchemaSchemaPrivilegesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
index 1854e4f2b54af1..618e831c90e219 100644
--- a/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_schemata_scanner.cpp
@@ -81,7 +81,7 @@ Status SchemaSchemataScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaSchemataScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaSchemataScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before Initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_schemata_scanner.h b/be/src/exec/schema_scanner/schema_schemata_scanner.h
index 46fad31af1fd5e..39a5ddda495bdd 100644
--- a/be/src/exec/schema_scanner/schema_schemata_scanner.h
+++ b/be/src/exec/schema_scanner/schema_schemata_scanner.h
@@ -38,7 +38,7 @@ class SchemaSchemataScanner : public SchemaScanner {
~SchemaSchemataScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _fill_block_impl(vectorized::Block* block);
diff --git a/be/src/exec/schema_scanner/schema_table_options_scanner.cpp b/be/src/exec/schema_scanner/schema_table_options_scanner.cpp
index 604da59b6377b6..e0481599a2f004 100644
--- a/be/src/exec/schema_scanner/schema_table_options_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_table_options_scanner.cpp
@@ -103,7 +103,7 @@ Status SchemaTableOptionsScanner::get_block_from_fe() {
return Status::OK();
}
-Status SchemaTableOptionsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaTableOptionsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_table_options_scanner.h b/be/src/exec/schema_scanner/schema_table_options_scanner.h
index d40f1b73c633c5..95c8bdb89076d7 100644
--- a/be/src/exec/schema_scanner/schema_table_options_scanner.h
+++ b/be/src/exec/schema_scanner/schema_table_options_scanner.h
@@ -36,7 +36,7 @@ class SchemaTableOptionsScanner : public SchemaScanner {
~SchemaTableOptionsScanner() override = default;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
index fe8aa725b73b80..cdeac2b70dcadd 100644
--- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.cpp
@@ -84,7 +84,7 @@ Status SchemaTablePrivilegesScanner::_get_new_table() {
return Status::OK();
}
-Status SchemaTablePrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaTablePrivilegesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h
index aa79c88304b7c5..4cfcc16d3583ce 100644
--- a/be/src/exec/schema_scanner/schema_table_privileges_scanner.h
+++ b/be/src/exec/schema_scanner/schema_table_privileges_scanner.h
@@ -38,7 +38,7 @@ class SchemaTablePrivilegesScanner : public SchemaScanner {
~SchemaTablePrivilegesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.cpp b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
index 093acf9cecbcb1..23710b81971c15 100644
--- a/be/src/exec/schema_scanner/schema_tables_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_tables_scanner.cpp
@@ -342,7 +342,7 @@ Status SchemaTablesScanner::_fill_block_impl(vectorized::Block* block) {
return Status::OK();
}
-Status SchemaTablesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaTablesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_tables_scanner.h b/be/src/exec/schema_scanner/schema_tables_scanner.h
index 11a96bf65d5271..7f8eb11f397e06 100644
--- a/be/src/exec/schema_scanner/schema_tables_scanner.h
+++ b/be/src/exec/schema_scanner/schema_tables_scanner.h
@@ -39,7 +39,7 @@ class SchemaTablesScanner : public SchemaScanner {
~SchemaTablesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
index 6a12d846fbd560..3eeabc0e4a0917 100644
--- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.cpp
@@ -81,7 +81,7 @@ Status SchemaUserPrivilegesScanner::_get_new_table() {
return Status::OK();
}
-Status SchemaUserPrivilegesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaUserPrivilegesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h
index eb8f3c63f1433b..ffc3840db676c4 100644
--- a/be/src/exec/schema_scanner/schema_user_privileges_scanner.h
+++ b/be/src/exec/schema_scanner/schema_user_privileges_scanner.h
@@ -38,7 +38,7 @@ class SchemaUserPrivilegesScanner : public SchemaScanner {
~SchemaUserPrivilegesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_user_scanner.cpp b/be/src/exec/schema_scanner/schema_user_scanner.cpp
index 9b153414380350..e56f18f05aea93 100644
--- a/be/src/exec/schema_scanner/schema_user_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_user_scanner.cpp
@@ -76,7 +76,7 @@ Status SchemaUserScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaUserScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaUserScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("call this before initial.");
}
diff --git a/be/src/exec/schema_scanner/schema_user_scanner.h b/be/src/exec/schema_scanner/schema_user_scanner.h
index c55f216804d5dd..bdc618eb5a0332 100644
--- a/be/src/exec/schema_scanner/schema_user_scanner.h
+++ b/be/src/exec/schema_scanner/schema_user_scanner.h
@@ -40,7 +40,7 @@ class SchemaUserScanner : public SchemaScanner {
~SchemaUserScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_user_columns;
diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.cpp b/be/src/exec/schema_scanner/schema_variables_scanner.cpp
index 546a0a471cfb01..ad4d5d072cb03f 100644
--- a/be/src/exec/schema_scanner/schema_variables_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_variables_scanner.cpp
@@ -40,7 +40,8 @@ std::vector SchemaVariablesScanner::_s_vars_columns =
// name, type, size
{"VARIABLE_NAME", TYPE_VARCHAR, sizeof(StringRef), false},
{"VARIABLE_VALUE", TYPE_VARCHAR, sizeof(StringRef), false},
-};
+ {"DEFAULT_VALUE", TYPE_VARCHAR, sizeof(StringRef), false},
+ {"CHANGED", TYPE_VARCHAR, sizeof(StringRef), false}};
SchemaVariablesScanner::SchemaVariablesScanner(TVarType::type type)
: SchemaScanner(_s_vars_columns, TSchemaTableType::SCH_VARIABLES), _type(type) {}
@@ -70,7 +71,7 @@ Status SchemaVariablesScanner::start(RuntimeState* state) {
return Status::OK();
}
-Status SchemaVariablesScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaVariablesScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("call this before initial.");
}
@@ -94,7 +95,7 @@ Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) {
std::vector strs(row_num);
int idx = 0;
for (auto& it : _var_result.variables) {
- strs[idx] = StringRef(it.first.c_str(), it.first.size());
+ strs[idx] = StringRef(it[0].c_str(), it[0].size());
datas[idx] = strs.data() + idx;
++idx;
}
@@ -105,12 +106,34 @@ Status SchemaVariablesScanner::_fill_block_impl(vectorized::Block* block) {
std::vector strs(row_num);
int idx = 0;
for (auto& it : _var_result.variables) {
- strs[idx] = StringRef(it.second.c_str(), it.second.size());
+ strs[idx] = StringRef(it[1].c_str(), it[1].size());
datas[idx] = strs.data() + idx;
++idx;
}
RETURN_IF_ERROR(fill_dest_column_for_range(block, 1, datas));
}
+ // default value
+ {
+ std::vector strs(row_num);
+ int idx = 0;
+ for (auto& it : _var_result.variables) {
+ strs[idx] = StringRef(it[2].c_str(), it[2].size());
+ datas[idx] = strs.data() + idx;
+ ++idx;
+ }
+ RETURN_IF_ERROR(fill_dest_column_for_range(block, 2, datas));
+ }
+ // changed
+ {
+ std::vector strs(row_num);
+ int idx = 0;
+ for (auto& it : _var_result.variables) {
+ strs[idx] = StringRef(it[3].c_str(), it[3].size());
+ datas[idx] = strs.data() + idx;
+ ++idx;
+ }
+ RETURN_IF_ERROR(fill_dest_column_for_range(block, 3, datas));
+ }
return Status::OK();
}
diff --git a/be/src/exec/schema_scanner/schema_variables_scanner.h b/be/src/exec/schema_scanner/schema_variables_scanner.h
index 2d207ff8b2e6c2..31bbacf713be0f 100644
--- a/be/src/exec/schema_scanner/schema_variables_scanner.h
+++ b/be/src/exec/schema_scanner/schema_variables_scanner.h
@@ -40,7 +40,7 @@ class SchemaVariablesScanner : public SchemaScanner {
~SchemaVariablesScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
struct VariableStruct {
diff --git a/be/src/exec/schema_scanner/schema_views_scanner.cpp b/be/src/exec/schema_scanner/schema_views_scanner.cpp
index 6c3b5f2e21bc3a..f47766ef3567ad 100644
--- a/be/src/exec/schema_scanner/schema_views_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_views_scanner.cpp
@@ -113,7 +113,7 @@ Status SchemaViewsScanner::_get_new_table() {
return Status::OK();
}
-Status SchemaViewsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaViewsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_views_scanner.h b/be/src/exec/schema_scanner/schema_views_scanner.h
index bc473057905a12..b86ad922e5e76a 100644
--- a/be/src/exec/schema_scanner/schema_views_scanner.h
+++ b/be/src/exec/schema_scanner/schema_views_scanner.h
@@ -38,7 +38,7 @@ class SchemaViewsScanner : public SchemaScanner {
~SchemaViewsScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
private:
Status _get_new_table();
diff --git a/be/src/exec/schema_scanner/schema_workload_group_privileges.cpp b/be/src/exec/schema_scanner/schema_workload_group_privileges.cpp
new file mode 100644
index 00000000000000..a1d4568d9053cd
--- /dev/null
+++ b/be/src/exec/schema_scanner/schema_workload_group_privileges.cpp
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/schema_scanner/schema_workload_group_privileges.h"
+
+#include "runtime/client_cache.h"
+#include "runtime/exec_env.h"
+#include "runtime/runtime_state.h"
+#include "util/thrift_rpc_helper.h"
+#include "vec/common/string_ref.h"
+#include "vec/core/block.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+namespace doris {
+std::vector SchemaWorkloadGroupPrivilegesScanner::_s_tbls_columns = {
+ {"GRANTEE", TYPE_VARCHAR, sizeof(StringRef), true},
+ {"WORKLOAD_GROUP_NAME", TYPE_VARCHAR, sizeof(StringRef), true},
+ {"PRIVILEGE_TYPE", TYPE_VARCHAR, sizeof(StringRef), true},
+ {"IS_GRANTABLE", TYPE_VARCHAR, sizeof(StringRef), true},
+};
+
+SchemaWorkloadGroupPrivilegesScanner::SchemaWorkloadGroupPrivilegesScanner()
+ : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_WORKLOAD_GROUPS) {}
+
+SchemaWorkloadGroupPrivilegesScanner::~SchemaWorkloadGroupPrivilegesScanner() {}
+
+Status SchemaWorkloadGroupPrivilegesScanner::start(RuntimeState* state) {
+ _block_rows_limit = state->batch_size();
+ _rpc_timeout = state->execution_timeout() * 1000;
+ return Status::OK();
+}
+
+Status SchemaWorkloadGroupPrivilegesScanner::_get_workload_group_privs_block_from_fe() {
+ TNetworkAddress master_addr = ExecEnv::GetInstance()->master_info()->network_address;
+
+ TSchemaTableRequestParams schema_table_request_params;
+ for (int i = 0; i < _s_tbls_columns.size(); i++) {
+ schema_table_request_params.__isset.columns_name = true;
+ schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name);
+ }
+ schema_table_request_params.__set_current_user_ident(*_param->common_param->current_user_ident);
+
+ TFetchSchemaTableDataRequest request;
+ request.__set_schema_table_name(TSchemaTableName::WORKLOAD_GROUP_PRIVILEGES);
+ request.__set_schema_table_params(schema_table_request_params);
+
+ TFetchSchemaTableDataResult result;
+
+ RETURN_IF_ERROR(ThriftRpcHelper::rpc(
+ master_addr.hostname, master_addr.port,
+ [&request, &result](FrontendServiceConnection& client) {
+ client->fetchSchemaTableData(result, request);
+ },
+ _rpc_timeout));
+
+ Status status(Status::create(result.status));
+ if (!status.ok()) {
+ LOG(WARNING) << "fetch workload group privileges from FE failed, errmsg=" << status;
+ return status;
+ }
+ std::vector result_data = result.data_batch;
+
+ _workload_groups_privs_block = vectorized::Block::create_unique();
+ for (int i = 0; i < _s_tbls_columns.size(); ++i) {
+ TypeDescriptor descriptor(_s_tbls_columns[i].type);
+ auto data_type = vectorized::DataTypeFactory::instance().create_data_type(descriptor, true);
+ _workload_groups_privs_block->insert(vectorized::ColumnWithTypeAndName(
+ data_type->create_column(), data_type, _s_tbls_columns[i].name));
+ }
+
+ if (result_data.size() > 0) {
+ int col_size = result_data[0].column_value.size();
+ if (col_size != _s_tbls_columns.size()) {
+ return Status::InternalError(
+ "workload group privileges schema is not match for FE and BE");
+ }
+ }
+
+ _workload_groups_privs_block->reserve(result_data.size());
+
+ for (int i = 0; i < result_data.size(); i++) {
+ TRow row = result_data[i];
+
+ for (int j = 0; j < _s_tbls_columns.size(); j++) {
+ RETURN_IF_ERROR(insert_block_column(row.column_value[j], j,
+ _workload_groups_privs_block.get(),
+ _s_tbls_columns[j].type));
+ }
+ }
+ return Status::OK();
+}
+
+Status SchemaWorkloadGroupPrivilegesScanner::get_next_block_internal(vectorized::Block* block,
+ bool* eos) {
+ if (!_is_init) {
+ return Status::InternalError("Used before initialized.");
+ }
+
+ if (nullptr == block || nullptr == eos) {
+ return Status::InternalError("input pointer is nullptr.");
+ }
+
+ if (_workload_groups_privs_block == nullptr) {
+ RETURN_IF_ERROR(_get_workload_group_privs_block_from_fe());
+ _total_rows = _workload_groups_privs_block->rows();
+ }
+
+ if (_row_idx == _total_rows) {
+ *eos = true;
+ return Status::OK();
+ }
+
+ int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx);
+ vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block);
+ RETURN_IF_ERROR(
+ mblock.add_rows(_workload_groups_privs_block.get(), _row_idx, current_batch_rows));
+ _row_idx += current_batch_rows;
+
+ *eos = _row_idx == _total_rows;
+ return Status::OK();
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/exec/schema_scanner/schema_workload_group_privileges.h b/be/src/exec/schema_scanner/schema_workload_group_privileges.h
new file mode 100644
index 00000000000000..0a7bf1258eed1f
--- /dev/null
+++ b/be/src/exec/schema_scanner/schema_workload_group_privileges.h
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include
+
+#include "common/status.h"
+#include "exec/schema_scanner.h"
+
+namespace doris {
+class RuntimeState;
+namespace vectorized {
+class Block;
+} // namespace vectorized
+
+class SchemaWorkloadGroupPrivilegesScanner : public SchemaScanner {
+ ENABLE_FACTORY_CREATOR(SchemaWorkloadGroupPrivilegesScanner);
+
+public:
+ SchemaWorkloadGroupPrivilegesScanner();
+ ~SchemaWorkloadGroupPrivilegesScanner() override;
+
+ Status start(RuntimeState* state) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
+
+ static std::vector _s_tbls_columns;
+
+private:
+ Status _get_workload_group_privs_block_from_fe();
+
+ int _block_rows_limit = 4096;
+ int _row_idx = 0;
+ int _total_rows = 0;
+ std::unique_ptr _workload_groups_privs_block = nullptr;
+ int _rpc_timeout = 3000;
+};
+}; // namespace doris
\ No newline at end of file
diff --git a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp
index def52df531df7b..dd81a3ecb267da 100644
--- a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp
@@ -42,6 +42,8 @@ std::vector SchemaWorkloadGroupsScanner::_s_tbls_colu
{"SPILL_THRESHOLD_LOW_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true},
{"SPILL_THRESHOLD_HIGH_WATERMARK", TYPE_VARCHAR, sizeof(StringRef), true},
{"TAG", TYPE_VARCHAR, sizeof(StringRef), true},
+ {"READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true},
+ {"REMOTE_READ_BYTES_PER_SECOND", TYPE_BIGINT, sizeof(int64_t), true},
};
SchemaWorkloadGroupsScanner::SchemaWorkloadGroupsScanner()
@@ -114,7 +116,7 @@ Status SchemaWorkloadGroupsScanner::_get_workload_groups_block_from_fe() {
return Status::OK();
}
-Status SchemaWorkloadGroupsScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaWorkloadGroupsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_workload_groups_scanner.h b/be/src/exec/schema_scanner/schema_workload_groups_scanner.h
index bf7a103526dc80..3121c4dbac149e 100644
--- a/be/src/exec/schema_scanner/schema_workload_groups_scanner.h
+++ b/be/src/exec/schema_scanner/schema_workload_groups_scanner.h
@@ -36,7 +36,7 @@ class SchemaWorkloadGroupsScanner : public SchemaScanner {
~SchemaWorkloadGroupsScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp
index 035d3bfe217aec..2d91f151f5f2bb 100644
--- a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp
@@ -106,7 +106,8 @@ Status SchemaWorkloadSchedulePolicyScanner::_get_workload_schedule_policy_block_
return Status::OK();
}
-Status SchemaWorkloadSchedulePolicyScanner::get_next_block(vectorized::Block* block, bool* eos) {
+Status SchemaWorkloadSchedulePolicyScanner::get_next_block_internal(vectorized::Block* block,
+ bool* eos) {
if (!_is_init) {
return Status::InternalError("Used before initialized.");
}
diff --git a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.h b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.h
index 5284975fe66b31..da8d9f15c4989e 100644
--- a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.h
+++ b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.h
@@ -36,7 +36,7 @@ class SchemaWorkloadSchedulePolicyScanner : public SchemaScanner {
~SchemaWorkloadSchedulePolicyScanner() override;
Status start(RuntimeState* state) override;
- Status get_next_block(vectorized::Block* block, bool* eos) override;
+ Status get_next_block_internal(vectorized::Block* block, bool* eos) override;
static std::vector _s_tbls_columns;
diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h
index a831395a5eabf1..e88f692a23db8a 100644
--- a/be/src/exprs/bloom_filter_func.h
+++ b/be/src/exprs/bloom_filter_func.h
@@ -100,8 +100,12 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
virtual ~BloomFilterFuncBase() = default;
void init_params(const RuntimeFilterParams* params) {
- _bloom_filter_length = params->bloom_filter_size;
+ _bloom_filter_length =
+ params->runtime_bloom_filter_min_size > 0
+ ? std::max(params->bloom_filter_size, params->runtime_bloom_filter_min_size)
+ : params->bloom_filter_size;
_build_bf_exactly = params->build_bf_exactly;
+ _runtime_bloom_filter_min_size = params->runtime_bloom_filter_min_size;
_null_aware = params->null_aware;
_bloom_filter_size_calculated_by_ndv = params->bloom_filter_size_calculated_by_ndv;
}
@@ -124,9 +128,16 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
// if FE do use ndv stat to predict the bf size, BE only use the row count. FE have more
// exactly row count stat. which one is min is more correctly.
if (_bloom_filter_size_calculated_by_ndv) {
- _bloom_filter_length = std::min(be_calculate_size, _bloom_filter_length);
+ _bloom_filter_length =
+ _runtime_bloom_filter_min_size > 0
+ ? std::max(_runtime_bloom_filter_min_size,
+ std::min(be_calculate_size, _bloom_filter_length))
+ : std::min(be_calculate_size, _bloom_filter_length);
} else {
- _bloom_filter_length = be_calculate_size;
+ _bloom_filter_length =
+ _runtime_bloom_filter_min_size > 0
+ ? std::max(_runtime_bloom_filter_min_size, be_calculate_size)
+ : be_calculate_size;
}
}
return init_with_fixed_length(_bloom_filter_length);
@@ -221,8 +232,9 @@ class BloomFilterFuncBase : public RuntimeFilterFuncBase {
// bloom filter size
int32_t _bloom_filter_alloced;
std::shared_ptr _bloom_filter;
- bool _inited {};
+ bool _inited = false;
int64_t _bloom_filter_length;
+ int64_t _runtime_bloom_filter_min_size;
bool _build_bf_exactly = false;
bool _bloom_filter_size_calculated_by_ndv = false;
};
diff --git a/be/src/exprs/create_predicate_function.h b/be/src/exprs/create_predicate_function.h
index 11889ff2ec349b..4808caa00f37d0 100644
--- a/be/src/exprs/create_predicate_function.h
+++ b/be/src/exprs/create_predicate_function.h
@@ -34,7 +34,9 @@ class MinmaxFunctionTraits {
using BasePtr = MinMaxFuncBase*;
template
static BasePtr get_function() {
- return new MinMaxNumFunc::CppType>();
+ using CppType = typename PrimitiveTypeTraits::CppType;
+ return new MinMaxNumFunc<
+ std::conditional_t, std::string, CppType>>();
}
};
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 205ee5a5d20b92..5e3fb136929595 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -353,4 +353,9 @@ void JsonFunctions::merge_objects(rapidjson::Value& dst_object, rapidjson::Value
}
}
+// root path "$."
+bool JsonFunctions::is_root_path(const std::vector& json_path) {
+ return json_path.size() == 2 && json_path[0].key == "$" && json_path[1].key.empty();
+}
+
} // namespace doris
diff --git a/be/src/exprs/json_functions.h b/be/src/exprs/json_functions.h
index 72aa522ff374fa..11970eb8c46c56 100644
--- a/be/src/exprs/json_functions.h
+++ b/be/src/exprs/json_functions.h
@@ -116,6 +116,8 @@ class JsonFunctions {
static std::string print_json_value(const rapidjson::Value& value);
+ static bool is_root_path(const std::vector& json_path);
+
private:
static rapidjson::Value* match_value(const std::vector& parsed_paths,
rapidjson::Value* document,
diff --git a/be/src/exprs/minmax_predicate.h b/be/src/exprs/minmax_predicate.h
index b4291e2edb7e6b..377b33696c82b9 100644
--- a/be/src/exprs/minmax_predicate.h
+++ b/be/src/exprs/minmax_predicate.h
@@ -26,6 +26,7 @@
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
#include "vec/common/assert_cast.h"
+#include "vec/common/string_ref.h"
namespace doris {
// only used in Runtime Filter
@@ -75,19 +76,22 @@ class MinMaxNumFunc : public MinMaxFuncBase {
for (size_t i = start; i < size; i++) {
if (nullmap == nullptr || !nullmap[i]) {
if constexpr (NeedMin) {
- _min = std::min(_min, column_string.get_data_at(i));
+ if (column_string.get_data_at(i) < StringRef(_min)) {
+ _min = column_string.get_data_at(i).to_string();
+ }
}
if constexpr (NeedMax) {
- _max = std::max(_max, column_string.get_data_at(i));
+ if (column_string.get_data_at(i) > StringRef(_max)) {
+ _max = column_string.get_data_at(i).to_string();
+ }
}
}
}
- store_string_ref();
}
void update_batch(const vectorized::ColumnPtr& column, size_t start) {
const auto size = column->size();
- if constexpr (std::is_same_v) {
+ if constexpr (std::is_same_v) {
if (column->is_column_string64()) {
_update_batch_string(assert_cast(*column),
nullptr, start, size);
@@ -111,7 +115,7 @@ class MinMaxNumFunc : public MinMaxFuncBase {
void update_batch(const vectorized::ColumnPtr& column, const vectorized::NullMap& nullmap,
size_t start) {
const auto size = column->size();
- if constexpr (std::is_same_v) {
+ if constexpr (std::is_same_v) {
if (column->is_column_string64()) {
_update_batch_string(assert_cast(*column),
nullmap.data(), start, size);
@@ -135,26 +139,15 @@ class MinMaxNumFunc : public MinMaxFuncBase {
}
Status merge(MinMaxFuncBase* minmax_func) override {
- if constexpr (std::is_same_v) {
- auto* other_minmax = static_cast*>(minmax_func);
- if constexpr (NeedMin) {
- _min = std::min(_min, other_minmax->_min);
- }
- if constexpr (NeedMax) {
- _max = std::max(_max, other_minmax->_max);
- }
- store_string_ref();
- } else {
- auto* other_minmax = static_cast*>(minmax_func);
- if constexpr (NeedMin) {
- if (other_minmax->_min < _min) {
- _min = other_minmax->_min;
- }
+ auto* other_minmax = static_cast*>(minmax_func);
+ if constexpr (NeedMin) {
+ if (other_minmax->_min < _min) {
+ _min = other_minmax->_min;
}
- if constexpr (NeedMax) {
- if (other_minmax->_max > _max) {
- _max = other_minmax->_max;
- }
+ }
+ if constexpr (NeedMax) {
+ if (other_minmax->_max > _max) {
+ _max = other_minmax->_max;
}
}
@@ -172,28 +165,9 @@ class MinMaxNumFunc : public MinMaxFuncBase {
return Status::OK();
}
- void store_string_ref() {
- if constexpr (std::is_same_v) {
- if constexpr (NeedMin) {
- if (_min.data != _stored_min.data()) {
- _stored_min = _min.to_string();
- _min = StringRef(_stored_min);
- }
- }
- if constexpr (NeedMax) {
- if (_max.data != _stored_max.data()) {
- _stored_max = _max.to_string();
- _max = StringRef(_stored_max);
- }
- }
- }
- }
-
protected:
T _max = type_limit::min();
T _min = type_limit::max();
- std::string _stored_min;
- std::string _stored_max;
};
template
diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index e69ff714d32978..d7fc2ff7490761 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -35,7 +35,6 @@
#include "agent/be_exec_version_manager.h"
#include "common/logging.h"
-#include "common/object_pool.h"
#include "common/status.h"
#include "exprs/bitmapfilter_predicate.h"
#include "exprs/bloom_filter_func.h"
@@ -281,15 +280,13 @@ Status create_vbin_predicate(const TypeDescriptor& type, TExprOpcode::type opcod
// This class is a wrapper of runtime predicate function
class RuntimePredicateWrapper {
public:
- RuntimePredicateWrapper(ObjectPool* pool, const RuntimeFilterParams* params)
- : RuntimePredicateWrapper(pool, params->column_return_type, params->filter_type,
+ RuntimePredicateWrapper(const RuntimeFilterParams* params)
+ : RuntimePredicateWrapper(params->column_return_type, params->filter_type,
params->filter_id) {};
// for a 'tmp' runtime predicate wrapper
// only could called assign method or as a param for merge
- RuntimePredicateWrapper(ObjectPool* pool, PrimitiveType column_type, RuntimeFilterType type,
- uint32_t filter_id)
- : _pool(pool),
- _column_return_type(column_type),
+ RuntimePredicateWrapper(PrimitiveType column_type, RuntimeFilterType type, uint32_t filter_id)
+ : _column_return_type(column_type),
_filter_type(type),
_context(new RuntimeFilterContext()),
_filter_id(filter_id) {}
@@ -566,51 +563,45 @@ class RuntimePredicateWrapper {
switch (type) {
case TYPE_BOOLEAN: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
bool bool_val = column.boolval();
set->insert(&bool_val);
});
break;
}
case TYPE_TINYINT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- int8_t int_val = static_cast(column.intval());
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ auto int_val = static_cast(column.intval());
set->insert(&int_val);
});
break;
}
case TYPE_SMALLINT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- int16_t int_val = static_cast(column.intval());
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ auto int_val = static_cast(column.intval());
set->insert(&int_val);
});
break;
}
case TYPE_INT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
int32_t int_val = column.intval();
set->insert(&int_val);
});
break;
}
case TYPE_BIGINT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
int64_t long_val = column.longval();
set->insert(&long_val);
});
break;
}
case TYPE_LARGEINT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
auto string_val = column.stringval();
StringParser::ParseResult result;
- int128_t int128_val = StringParser::string_to_int(
+ auto int128_val = StringParser::string_to_int(
string_val.c_str(), string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
set->insert(&int128_val);
@@ -618,32 +609,28 @@ class RuntimePredicateWrapper {
break;
}
case TYPE_FLOAT: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- float float_val = static_cast(column.doubleval());
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ auto float_val = static_cast(column.doubleval());
set->insert(&float_val);
});
break;
}
case TYPE_DOUBLE: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
double double_val = column.doubleval();
set->insert(&double_val);
});
break;
}
case TYPE_DATEV2: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
auto date_v2_val = column.intval();
set->insert(&date_v2_val);
});
break;
}
case TYPE_DATETIMEV2: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
auto date_v2_val = column.longval();
set->insert(&date_v2_val);
});
@@ -651,9 +638,8 @@ class RuntimePredicateWrapper {
}
case TYPE_DATETIME:
case TYPE_DATE: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- auto& string_val_ref = column.stringval();
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ const auto& string_val_ref = column.stringval();
VecDateTimeValue datetime_val;
datetime_val.from_date_str(string_val_ref.c_str(), string_val_ref.length());
set->insert(&datetime_val);
@@ -661,36 +647,32 @@ class RuntimePredicateWrapper {
break;
}
case TYPE_DECIMALV2: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- auto& string_val_ref = column.stringval();
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ const auto& string_val_ref = column.stringval();
DecimalV2Value decimal_val(string_val_ref);
set->insert(&decimal_val);
});
break;
}
case TYPE_DECIMAL32: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
int32_t decimal_32_val = column.intval();
set->insert(&decimal_32_val);
});
break;
}
case TYPE_DECIMAL64: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
int64_t decimal_64_val = column.longval();
set->insert(&decimal_64_val);
});
break;
}
case TYPE_DECIMAL128I: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
auto string_val = column.stringval();
StringParser::ParseResult result;
- int128_t int128_val = StringParser::string_to_int(
+ auto int128_val = StringParser::string_to_int(
string_val.c_str(), string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
set->insert(&int128_val);
@@ -698,8 +680,7 @@ class RuntimePredicateWrapper {
break;
}
case TYPE_DECIMAL256: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
auto string_val = column.stringval();
StringParser::ParseResult result;
auto int_val = StringParser::string_to_int(
@@ -712,12 +693,9 @@ class RuntimePredicateWrapper {
case TYPE_VARCHAR:
case TYPE_CHAR:
case TYPE_STRING: {
- batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column,
- ObjectPool* pool) {
- auto& string_val_ref = column.stringval();
- auto val_ptr = pool->add(new std::string(string_val_ref));
- StringRef string_val(val_ptr->c_str(), val_ptr->length());
- set->insert(&string_val);
+ batch_assign(in_filter, [](std::shared_ptr& set, PColumnValue& column) {
+ const auto& string_val_ref = column.stringval();
+ set->insert(&string_val_ref);
});
break;
}
@@ -761,13 +739,13 @@ class RuntimePredicateWrapper {
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_TINYINT: {
- int8_t min_val = static_cast(minmax_filter->min_val().intval());
- int8_t max_val = static_cast(minmax_filter->max_val().intval());
+ auto min_val = static_cast(minmax_filter->min_val().intval());
+ auto max_val = static_cast(minmax_filter->max_val().intval());
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_SMALLINT: {
- int16_t min_val = static_cast(minmax_filter->min_val().intval());
- int16_t max_val = static_cast(minmax_filter->max_val().intval());
+ auto min_val = static_cast(minmax_filter->min_val().intval());
+ auto max_val = static_cast(minmax_filter->max_val().intval());
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_INT: {
@@ -784,22 +762,22 @@ class RuntimePredicateWrapper {
auto min_string_val = minmax_filter->min_val().stringval();
auto max_string_val = minmax_filter->max_val().stringval();
StringParser::ParseResult result;
- int128_t min_val = StringParser::string_to_int(
- min_string_val.c_str(), min_string_val.length(), &result);
+ auto min_val = StringParser::string_to_int(min_string_val.c_str(),
+ min_string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
- int128_t max_val = StringParser::string_to_int(
- max_string_val.c_str(), max_string_val.length(), &result);
+ auto max_val = StringParser::string_to_int(max_string_val.c_str(),
+ max_string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_FLOAT: {
- float min_val = static_cast(minmax_filter->min_val().doubleval());
- float max_val = static_cast(minmax_filter->max_val().doubleval());
+ auto min_val = static_cast(minmax_filter->min_val().doubleval());
+ auto max_val = static_cast(minmax_filter->max_val().doubleval());
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_DOUBLE: {
- double min_val = static_cast(minmax_filter->min_val().doubleval());
- double max_val = static_cast(minmax_filter->max_val().doubleval());
+ auto min_val = static_cast(minmax_filter->min_val().doubleval());
+ auto max_val = static_cast(minmax_filter->max_val().doubleval());
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_DATEV2: {
@@ -814,8 +792,8 @@ class RuntimePredicateWrapper {
}
case TYPE_DATETIME:
case TYPE_DATE: {
- auto& min_val_ref = minmax_filter->min_val().stringval();
- auto& max_val_ref = minmax_filter->max_val().stringval();
+ const auto& min_val_ref = minmax_filter->min_val().stringval();
+ const auto& max_val_ref = minmax_filter->max_val().stringval();
VecDateTimeValue min_val;
VecDateTimeValue max_val;
min_val.from_date_str(min_val_ref.c_str(), min_val_ref.length());
@@ -823,8 +801,8 @@ class RuntimePredicateWrapper {
return _context->minmax_func->assign(&min_val, &max_val);
}
case TYPE_DECIMALV2: {
- auto& min_val_ref = minmax_filter->min_val().stringval();
- auto& max_val_ref = minmax_filter->max_val().stringval();
+ const auto& min_val_ref = minmax_filter->min_val().stringval();
+ const auto& max_val_ref = minmax_filter->max_val().stringval();
DecimalV2Value min_val(min_val_ref);
DecimalV2Value max_val(max_val_ref);
return _context->minmax_func->assign(&min_val, &max_val);
@@ -843,11 +821,11 @@ class RuntimePredicateWrapper {
auto min_string_val = minmax_filter->min_val().stringval();
auto max_string_val = minmax_filter->max_val().stringval();
StringParser::ParseResult result;
- int128_t min_val = StringParser::string_to_int(
- min_string_val.c_str(), min_string_val.length(), &result);
+ auto min_val = StringParser::string_to_int(min_string_val.c_str(),
+ min_string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
- int128_t max_val = StringParser::string_to_int(
- max_string_val.c_str(), max_string_val.length(), &result);
+ auto max_val = StringParser::string_to_int(max_string_val.c_str(),
+ max_string_val.length(), &result);
DCHECK(result == StringParser::PARSE_SUCCESS);
return _context->minmax_func->assign(&min_val, &max_val);
}
@@ -866,13 +844,9 @@ class RuntimePredicateWrapper {
case TYPE_VARCHAR:
case TYPE_CHAR:
case TYPE_STRING: {
- auto& min_val_ref = minmax_filter->min_val().stringval();
- auto& max_val_ref = minmax_filter->max_val().stringval();
- auto min_val_ptr = _pool->add(new std::string(min_val_ref));
- auto max_val_ptr = _pool->add(new std::string(max_val_ref));
- StringRef min_val(min_val_ptr->c_str(), min_val_ptr->length());
- StringRef max_val(max_val_ptr->c_str(), max_val_ptr->length());
- return _context->minmax_func->assign(&min_val, &max_val);
+ auto min_val_ref = minmax_filter->min_val().stringval();
+ auto max_val_ref = minmax_filter->max_val().stringval();
+ return _context->minmax_func->assign(&min_val_ref, &max_val_ref);
}
default:
break;
@@ -915,10 +889,10 @@ class RuntimePredicateWrapper {
void batch_assign(const PInFilter* filter,
void (*assign_func)(std::shared_ptr& _hybrid_set,
- PColumnValue&, ObjectPool*)) {
+ PColumnValue&)) {
for (int i = 0; i < filter->values_size(); ++i) {
PColumnValue column = filter->values(i);
- assign_func(_context->hybrid_set, column, _pool);
+ assign_func(_context->hybrid_set, column);
}
}
@@ -945,27 +919,25 @@ class RuntimePredicateWrapper {
}
private:
- ObjectPool* _pool;
-
// When a runtime filter received from remote and it is a bloom filter, _column_return_type will be invalid.
PrimitiveType _column_return_type; // column type
RuntimeFilterType _filter_type;
int32_t _max_in_num = -1;
- SharedRuntimeFilterContext _context;
+ RuntimeFilterContextSPtr _context;
uint32_t _filter_id;
};
-Status IRuntimeFilter::create(RuntimeFilterParamsContext* state, ObjectPool* pool,
- const TRuntimeFilterDesc* desc, const TQueryOptions* query_options,
- const RuntimeFilterRole role, int node_id, IRuntimeFilter** res,
+Status IRuntimeFilter::create(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc,
+ const TQueryOptions* query_options, const RuntimeFilterRole role,
+ int node_id, std::shared_ptr* res,
bool build_bf_exactly, bool need_local_merge) {
- *res = pool->add(new IRuntimeFilter(state, pool, desc, need_local_merge));
+ *res = std::make_shared(state, desc, need_local_merge);
(*res)->set_role(role);
return (*res)->init_with_desc(desc, query_options, node_id, build_bf_exactly);
}
-SharedRuntimeFilterContext& IRuntimeFilter::get_shared_context_ref() {
+RuntimeFilterContextSPtr& IRuntimeFilter::get_shared_context_ref() {
return _wrapper->_context;
}
@@ -983,12 +955,12 @@ Status IRuntimeFilter::publish(bool publish_local) {
RETURN_IF_ERROR(_state->runtime_filter_mgr->get_merge_addr(&addr));
return filter->push_to_remote(&addr);
};
- auto send_to_local = [&](RuntimePredicateWrapper* wrapper) {
- std::vector filters;
+ auto send_to_local = [&](std::shared_ptr wrapper) {
+ std::vector> filters;
RETURN_IF_ERROR(_state->runtime_filter_mgr->get_consume_filters(_filter_id, filters));
DCHECK(!filters.empty());
// push down
- for (auto* filter : filters) {
+ for (auto filter : filters) {
filter->_wrapper = wrapper;
filter->update_runtime_filter_type_to_profile();
filter->signal();
@@ -1000,13 +972,13 @@ Status IRuntimeFilter::publish(bool publish_local) {
RETURN_IF_ERROR(_state->runtime_filter_mgr->get_local_merge_producer_filters(
_filter_id, &local_merge_filters));
std::lock_guard l(*local_merge_filters->lock);
- RETURN_IF_ERROR(local_merge_filters->filters[0]->merge_from(_wrapper));
+ RETURN_IF_ERROR(local_merge_filters->filters[0]->merge_from(_wrapper.get()));
local_merge_filters->merge_time--;
if (local_merge_filters->merge_time == 0) {
if (_has_local_target) {
RETURN_IF_ERROR(send_to_local(local_merge_filters->filters[0]->_wrapper));
} else {
- RETURN_IF_ERROR(send_to_remote(local_merge_filters->filters[0]));
+ RETURN_IF_ERROR(send_to_remote(local_merge_filters->filters[0].get()));
}
}
return Status::OK();
@@ -1033,14 +1005,18 @@ Status IRuntimeFilter::publish(bool publish_local) {
class SyncSizeClosure : public AutoReleaseClosure> {
std::shared_ptr _dependency;
- IRuntimeFilter* _filter;
+ // Should use weak ptr here, because when query context deconstructs, should also delete runtime filter
+ // context, it not the memory is not released. And rpc is in another thread, it will hold rf context
+ // after query context because the rpc is not returned.
+ std::weak_ptr _rf_context;
+ std::string _rf_debug_info;
using Base =
AutoReleaseClosure>;
ENABLE_FACTORY_CREATOR(SyncSizeClosure);
void _process_if_rpc_failed() override {
((pipeline::CountedFinishDependency*)_dependency.get())->sub();
- LOG(WARNING) << "sync filter size meet rpc error, filter=" << _filter->debug_string();
+ LOG(WARNING) << "sync filter size meet rpc error, filter=" << _rf_debug_info;
Base::_process_if_rpc_failed();
}
@@ -1048,10 +1024,15 @@ class SyncSizeClosure : public AutoReleaseClosuresub();
if (status.is()) {
// rf merger backend may finished before rf's send_filter_size, we just ignore filter in this case.
- _filter->set_ignored();
+ auto ctx = _rf_context.lock();
+ if (ctx) {
+ ctx->ignored = true;
+ } else {
+ LOG(WARNING) << "sync filter size returned but context is released, filter="
+ << _rf_debug_info;
+ }
} else {
- LOG(WARNING) << "sync filter size meet error status, filter="
- << _filter->debug_string();
+ LOG(WARNING) << "sync filter size meet error status, filter=" << _rf_debug_info;
Base::_process_if_meet_error_status(status);
}
}
@@ -1059,8 +1040,12 @@ class SyncSizeClosure : public AutoReleaseClosure req,
std::shared_ptr> callback,
- std::shared_ptr dependency, IRuntimeFilter* filter)
- : Base(req, callback), _dependency(std::move(dependency)), _filter(filter) {}
+ std::shared_ptr dependency,
+ RuntimeFilterContextSPtr rf_context, std::string_view rf_debug_info)
+ : Base(req, callback),
+ _dependency(std::move(dependency)),
+ _rf_context(rf_context),
+ _rf_debug_info(rf_debug_info) {}
};
Status IRuntimeFilter::send_filter_size(RuntimeState* state, uint64_t local_filter_size) {
@@ -1077,7 +1062,7 @@ Status IRuntimeFilter::send_filter_size(RuntimeState* state, uint64_t local_filt
return Status::OK();
} else {
if (_has_local_target) {
- for (auto* filter : local_merge_filters->filters) {
+ for (auto filter : local_merge_filters->filters) {
filter->set_synced_size(local_merge_filters->local_merged_size);
}
return Status::OK();
@@ -1103,7 +1088,10 @@ Status IRuntimeFilter::send_filter_size(RuntimeState* state, uint64_t local_filt
auto request = std::make_shared();
auto callback = DummyBrpcCallback::create_shared();
- auto closure = SyncSizeClosure::create_unique(request, callback, _dependency, this);
+ // IRuntimeFilter maybe deconstructed before the rpc finished, so that could not use
+ // a raw pointer in closure. Has to use the context's shared ptr.
+ auto closure = SyncSizeClosure::create_unique(request, callback, _dependency,
+ _wrapper->_context, this->debug_string());
auto* pquery_id = request->mutable_query_id();
pquery_id->set_hi(_state->query_id.hi());
pquery_id->set_lo(_state->query_id.lo());
@@ -1193,33 +1181,6 @@ Status IRuntimeFilter::get_push_expr_ctxs(std::listexecution_timeout * 1000;
- auto runtime_filter_wait_time_ms = _state->runtime_filter_wait_time_ms;
- // bitmap filter is precise filter and only filter once, so it must be applied.
- int64_t wait_times_ms = _wrapper->get_real_type() == RuntimeFilterType::BITMAP_FILTER
- ? execution_timeout
- : runtime_filter_wait_time_ms;
- auto expected = _rf_state_atomic.load(std::memory_order_acquire);
- if (expected == RuntimeFilterState::NOT_READY) {
- if (!_rf_state_atomic.compare_exchange_strong(
- expected,
- MonotonicMillis() - registration_time_ >= wait_times_ms
- ? RuntimeFilterState::TIME_OUT
- : RuntimeFilterState::NOT_READY,
- std::memory_order_acq_rel)) {
- DCHECK(expected == RuntimeFilterState::READY ||
- expected == RuntimeFilterState::TIME_OUT);
- return (expected == RuntimeFilterState::READY);
- }
- return false;
- } else if (expected == RuntimeFilterState::TIME_OUT) {
- return false;
- }
- return true;
-}
-
void IRuntimeFilter::update_state() {
DCHECK(is_consumer());
auto execution_timeout = _state->execution_timeout * 1000;
@@ -1325,6 +1286,9 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
params.filter_type = _runtime_filter_type;
params.column_return_type = build_ctx->root()->type().type;
params.max_in_num = options->runtime_filter_max_in_num;
+ params.runtime_bloom_filter_min_size = options->__isset.runtime_bloom_filter_min_size
+ ? options->runtime_bloom_filter_min_size
+ : 0;
// We build runtime filter by exact distinct count iff three conditions are met:
// 1. Only 1 join key
// 2. Do not have remote target (e.g. do not need to merge), or broadcast join
@@ -1372,7 +1336,7 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
_probe_expr = iter->second;
}
- _wrapper = _pool->add(new RuntimePredicateWrapper(_pool, ¶ms));
+ _wrapper = std::make_shared(¶ms);
return _wrapper->init(¶ms);
}
@@ -1388,22 +1352,22 @@ Status IRuntimeFilter::serialize(PPublishFilterRequestV2* request, void** data,
return serialize_impl(request, data, len);
}
-Status IRuntimeFilter::create_wrapper(const MergeRuntimeFilterParams* param, ObjectPool* pool,
+Status IRuntimeFilter::create_wrapper(const MergeRuntimeFilterParams* param,
std::unique_ptr* wrapper) {
- return _create_wrapper(param, pool, wrapper);
+ return _create_wrapper(param, wrapper);
}
-Status IRuntimeFilter::create_wrapper(const UpdateRuntimeFilterParams* param, ObjectPool* pool,
+Status IRuntimeFilter::create_wrapper(const UpdateRuntimeFilterParams* param,
std::unique_ptr* wrapper) {
- return _create_wrapper(param, pool, wrapper);
+ return _create_wrapper(param, wrapper);
}
Status IRuntimeFilter::create_wrapper(const UpdateRuntimeFilterParamsV2* param,
- RuntimePredicateWrapper** wrapper) {
+ std::shared_ptr* wrapper) {
auto filter_type = param->request->filter_type();
PrimitiveType column_type = param->column_type;
- *wrapper = param->pool->add(new RuntimePredicateWrapper(
- param->pool, column_type, get_type(filter_type), param->request->filter_id()));
+ *wrapper = std::make_shared(column_type, get_type(filter_type),
+ param->request->filter_id());
if (param->request->has_ignored() && param->request->ignored()) {
(*wrapper)->set_ignored();
@@ -1441,7 +1405,7 @@ Status IRuntimeFilter::init_bloom_filter(const size_t build_bf_cardinality) {
}
template
-Status IRuntimeFilter::_create_wrapper(const T* param, ObjectPool* pool,
+Status IRuntimeFilter::_create_wrapper(const T* param,
std::unique_ptr* wrapper) {
int filter_type = param->request->filter_type();
PrimitiveType column_type = PrimitiveType::INVALID_TYPE;
@@ -1451,7 +1415,7 @@ Status IRuntimeFilter::_create_wrapper(const T* param, ObjectPool* pool,
if (param->request->has_column_type()) {
column_type = to_primitive_type(param->request->column_type());
}
- *wrapper = std::make_unique(pool, column_type, get_type(filter_type),
+ *wrapper = std::make_unique(column_type, get_type(filter_type),
param->request->filter_id());
if (param->request->has_ignored() && param->request->ignored()) {
@@ -1552,7 +1516,7 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) {
auto column_type = _wrapper->column_type();
filter->set_column_type(to_proto(column_type));
- auto it = _wrapper->get_in_filter_iterator();
+ auto* it = _wrapper->get_in_filter_iterator();
DCHECK(it != nullptr);
switch (column_type) {
@@ -1663,8 +1627,8 @@ void IRuntimeFilter::to_protobuf(PInFilter* filter) {
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
- batch_copy(filter, it, [](PColumnValue* column, const StringRef* value) {
- column->set_stringval(std::string(value->data, value->size));
+ batch_copy(filter, it, [](PColumnValue* column, const std::string* value) {
+ column->set_stringval(*value);
});
return;
}
@@ -1684,8 +1648,8 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) {
switch (_wrapper->column_type()) {
case TYPE_BOOLEAN: {
- filter->mutable_min_val()->set_boolval(*reinterpret_cast(min_data));
- filter->mutable_max_val()->set_boolval(*reinterpret_cast(max_data));
+ filter->mutable_min_val()->set_boolval(*reinterpret_cast(min_data));
+ filter->mutable_max_val()->set_boolval(*reinterpret_cast(max_data));
return;
}
case TYPE_TINYINT: {
@@ -1778,12 +1742,10 @@ void IRuntimeFilter::to_protobuf(PMinMaxFilter* filter) {
case TYPE_CHAR:
case TYPE_VARCHAR:
case TYPE_STRING: {
- const StringRef* min_string_value = reinterpret_cast(min_data);
- filter->mutable_min_val()->set_stringval(
- std::string(min_string_value->data, min_string_value->size));
- const StringRef* max_string_value = reinterpret_cast(max_data);
- filter->mutable_max_val()->set_stringval(
- std::string(max_string_value->data, max_string_value->size));
+ const auto* min_string_value = reinterpret_cast(min_data);
+ filter->mutable_min_val()->set_stringval(*min_string_value);
+ const auto* max_string_value = reinterpret_cast(max_data);
+ filter->mutable_max_val()->set_stringval(*max_string_value);
break;
}
default: {
@@ -1810,7 +1772,7 @@ Status IRuntimeFilter::update_filter(const UpdateRuntimeFilterParams* param) {
set_ignored();
} else {
std::unique_ptr wrapper;
- RETURN_IF_ERROR(IRuntimeFilter::create_wrapper(param, _pool, &wrapper));
+ RETURN_IF_ERROR(IRuntimeFilter::create_wrapper(param, &wrapper));
RETURN_IF_ERROR(_wrapper->merge(wrapper.get()));
update_runtime_filter_type_to_profile();
}
@@ -1819,8 +1781,8 @@ Status IRuntimeFilter::update_filter(const UpdateRuntimeFilterParams* param) {
return Status::OK();
}
-void IRuntimeFilter::update_filter(RuntimePredicateWrapper* wrapper, int64_t merge_time,
- int64_t start_apply) {
+void IRuntimeFilter::update_filter(std::shared_ptr wrapper,
+ int64_t merge_time, int64_t start_apply) {
_profile->add_info_string("UpdateTime",
std::to_string(MonotonicMillis() - start_apply) + " ms");
_profile->add_info_string("MergeTime", std::to_string(merge_time) + " ms");
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 390a61bfe1a628..f199e173e84cd4 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -48,7 +48,6 @@ class IOBufAsZeroCopyInputStream;
}
namespace doris {
-class ObjectPool;
class RuntimePredicateWrapper;
class PPublishFilterRequest;
class PPublishFilterRequestV2;
@@ -65,7 +64,7 @@ class TQueryOptions;
namespace vectorized {
class VExpr;
class VExprContext;
-struct SharedRuntimeFilterContext;
+struct RuntimeFilterContextSPtr;
} // namespace vectorized
namespace pipeline {
@@ -128,6 +127,7 @@ struct RuntimeFilterParams {
// used in bloom filter
int64_t bloom_filter_size;
int32_t max_in_num;
+ int64_t runtime_bloom_filter_min_size;
int32_t filter_id;
bool bitmap_filter_not_in;
bool build_bf_exactly;
@@ -157,17 +157,15 @@ struct RuntimeFilterFuncBase {
struct UpdateRuntimeFilterParams {
UpdateRuntimeFilterParams(const PPublishFilterRequest* req,
- butil::IOBufAsZeroCopyInputStream* data_stream, ObjectPool* obj_pool)
- : request(req), data(data_stream), pool(obj_pool) {}
+ butil::IOBufAsZeroCopyInputStream* data_stream)
+ : request(req), data(data_stream) {}
const PPublishFilterRequest* request = nullptr;
butil::IOBufAsZeroCopyInputStream* data = nullptr;
- ObjectPool* pool = nullptr;
};
struct UpdateRuntimeFilterParamsV2 {
const PPublishFilterRequestV2* request;
butil::IOBufAsZeroCopyInputStream* data;
- ObjectPool* pool = nullptr;
PrimitiveType column_type = INVALID_TYPE;
};
@@ -192,10 +190,9 @@ enum RuntimeFilterState {
/// that can be pushed down to node based on the results of the right table.
class IRuntimeFilter {
public:
- IRuntimeFilter(RuntimeFilterParamsContext* state, ObjectPool* pool,
- const TRuntimeFilterDesc* desc, bool need_local_merge = false)
+ IRuntimeFilter(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc,
+ bool need_local_merge = false)
: _state(state),
- _pool(pool),
_filter_id(desc->filter_id),
_is_broadcast_join(true),
_has_remote_target(false),
@@ -215,12 +212,12 @@ class IRuntimeFilter {
~IRuntimeFilter() = default;
- static Status create(RuntimeFilterParamsContext* state, ObjectPool* pool,
- const TRuntimeFilterDesc* desc, const TQueryOptions* query_options,
- const RuntimeFilterRole role, int node_id, IRuntimeFilter** res,
+ static Status create(RuntimeFilterParamsContext* state, const TRuntimeFilterDesc* desc,
+ const TQueryOptions* query_options, const RuntimeFilterRole role,
+ int node_id, std::shared_ptr* res,
bool build_bf_exactly = false, bool need_local_merge = false);
- SharedRuntimeFilterContext& get_shared_context_ref();
+ RuntimeFilterContextSPtr& get_shared_context_ref();
// insert data to build filter
void insert_batch(vectorized::ColumnPtr column, size_t start);
@@ -257,12 +254,6 @@ class IRuntimeFilter {
void set_role(const RuntimeFilterRole role) { _role = role; }
int expr_order() const { return _expr_order; }
- // only used for consumer
- // if filter is not ready for filter data scan_node
- // will wait util it ready or timeout
- // This function will wait at most config::runtime_filter_shuffle_wait_time_ms
- // if return true , filter is ready to use
- bool await();
void update_state();
// this function will be called if a runtime filter sent by rpc
// it will notify all wait threads
@@ -281,17 +272,17 @@ class IRuntimeFilter {
Status merge_from(const RuntimePredicateWrapper* wrapper);
- static Status create_wrapper(const MergeRuntimeFilterParams* param, ObjectPool* pool,
+ static Status create_wrapper(const MergeRuntimeFilterParams* param,
std::unique_ptr* wrapper);
- static Status create_wrapper(const UpdateRuntimeFilterParams* param, ObjectPool* pool,
+ static Status create_wrapper(const UpdateRuntimeFilterParams* param,
std::unique_ptr* wrapper);
static Status create_wrapper(const UpdateRuntimeFilterParamsV2* param,
- RuntimePredicateWrapper** wrapper);
+ std::shared_ptr* wrapper);
Status change_to_bloom_filter();
Status init_bloom_filter(const size_t build_bf_cardinality);
Status update_filter(const UpdateRuntimeFilterParams* param);
- void update_filter(RuntimePredicateWrapper* filter_wrapper, int64_t merge_time,
+ void update_filter(std::shared_ptr filter_wrapper, int64_t merge_time,
int64_t start_apply);
void set_ignored();
@@ -381,7 +372,7 @@ class IRuntimeFilter {
Status serialize_impl(T* request, void** data, int* len);
template
- static Status _create_wrapper(const T* param, ObjectPool* pool,
+ static Status _create_wrapper(const T* param,
std::unique_ptr* wrapper);
void _set_push_down(bool push_down) { _is_push_down = push_down; }
@@ -395,10 +386,8 @@ class IRuntimeFilter {
}
RuntimeFilterParamsContext* _state = nullptr;
- ObjectPool* _pool = nullptr;
// _wrapper is a runtime filter function wrapper
- // _wrapper should alloc from _pool
- RuntimePredicateWrapper* _wrapper = nullptr;
+ std::shared_ptr _wrapper;
// runtime filter id
int _filter_id;
// Specific types BoardCast or Shuffle
diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h
index ebda4b56fcc30e..c0a249cd6b063d 100644
--- a/be/src/exprs/runtime_filter_slots.h
+++ b/be/src/exprs/runtime_filter_slots.h
@@ -34,10 +34,10 @@ class VRuntimeFilterSlots {
public:
VRuntimeFilterSlots(
const std::vector>& build_expr_ctxs,
- const std::vector& runtime_filters)
+ const std::vector>& runtime_filters)
: _build_expr_context(build_expr_ctxs), _runtime_filters(runtime_filters) {
- for (auto* runtime_filter : _runtime_filters) {
- _runtime_filters_map[runtime_filter->expr_order()].push_back(runtime_filter);
+ for (auto runtime_filter : _runtime_filters) {
+ _runtime_filters_map[runtime_filter->expr_order()].push_back(runtime_filter.get());
}
}
@@ -46,14 +46,14 @@ class VRuntimeFilterSlots {
if (_runtime_filters.empty()) {
return Status::OK();
}
- for (auto* runtime_filter : _runtime_filters) {
+ for (auto runtime_filter : _runtime_filters) {
if (runtime_filter->need_sync_filter_size()) {
runtime_filter->set_dependency(dependency);
}
}
// send_filter_size may call dependency->sub(), so we call set_dependency firstly for all rf to avoid dependency set_ready repeatedly
- for (auto* runtime_filter : _runtime_filters) {
+ for (auto runtime_filter : _runtime_filters) {
if (runtime_filter->need_sync_filter_size()) {
RETURN_IF_ERROR(runtime_filter->send_filter_size(state, hash_table_size));
}
@@ -70,7 +70,7 @@ class VRuntimeFilterSlots {
Status ignore_filters(RuntimeState* state) {
// process ignore duplicate IN_FILTER
std::unordered_set has_in_filter;
- for (auto* filter : _runtime_filters) {
+ for (auto filter : _runtime_filters) {
if (filter->get_ignored()) {
continue;
}
@@ -85,7 +85,7 @@ class VRuntimeFilterSlots {
}
// process ignore filter when it has IN_FILTER on same expr, and init bloom filter size
- for (auto* filter : _runtime_filters) {
+ for (auto filter : _runtime_filters) {
if (filter->get_ignored()) {
continue;
}
@@ -100,12 +100,13 @@ class VRuntimeFilterSlots {
Status init_filters(RuntimeState* state, uint64_t local_hash_table_size) {
// process IN_OR_BLOOM_FILTER's real type
- for (auto* filter : _runtime_filters) {
+ for (auto filter : _runtime_filters) {
if (filter->get_ignored()) {
continue;
}
if (filter->type() == RuntimeFilterType::IN_OR_BLOOM_FILTER &&
- get_real_size(filter, local_hash_table_size) > state->runtime_filter_max_in_num()) {
+ get_real_size(filter.get(), local_hash_table_size) >
+ state->runtime_filter_max_in_num()) {
RETURN_IF_ERROR(filter->change_to_bloom_filter());
}
@@ -114,8 +115,8 @@ class VRuntimeFilterSlots {
return Status::InternalError("sync filter size meet error, filter: {}",
filter->debug_string());
}
- RETURN_IF_ERROR(
- filter->init_bloom_filter(get_real_size(filter, local_hash_table_size)));
+ RETURN_IF_ERROR(filter->init_bloom_filter(
+ get_real_size(filter.get(), local_hash_table_size)));
}
}
return Status::OK();
@@ -175,7 +176,7 @@ class VRuntimeFilterSlots {
private:
const std::vector>& _build_expr_context;
- std::vector _runtime_filters;
+ std::vector> _runtime_filters;
// prob_contition index -> [IRuntimeFilter]
std::map> _runtime_filters_map;
};
diff --git a/be/src/exprs/runtime_filter_slots_cross.h b/be/src/exprs/runtime_filter_slots_cross.h
index 1d496ddf5571e6..01ae21a75992de 100644
--- a/be/src/exprs/runtime_filter_slots_cross.h
+++ b/be/src/exprs/runtime_filter_slots_cross.h
@@ -17,6 +17,7 @@
#pragma once
+#include
#include
#include "common/status.h"
@@ -34,14 +35,14 @@ namespace doris {
// this class used in cross join node
class VRuntimeFilterSlotsCross {
public:
- VRuntimeFilterSlotsCross(const std::vector