Merge branch 'master' into fix_kafka_topic_change

.asf.yaml

@@ -55,6 +55,7 @@ github:
  - P1 Regression (Doris Regression)
  - External Regression (Doris External Regression)
  - cloud_p1 (Doris Cloud Regression)
+ - cloud_p0 (Doris Cloud Regression)
  - FE UT (Doris FE UT)
  - BE UT (Doris BE UT)
  - Build Broker

.github/CODEOWNERS

@@ -16,3 +16,4 @@
 #
 be/src/io/* @platoneko @gavinchou @dataroaring
 fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @dataroaring @CalvinKirs @morningman
+**/pom.xml @CalvinKirs @morningman

.github/workflows/comment-to-trigger-teamcity.yml

@@ -150,6 +150,7 @@ jobs:
  set -x
  if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
  "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
  "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_fe_ut }}" \
@@ -158,7 +159,7 @@ jobs:
  "feut" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run feut"
+ echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run feut"
  trigger_or_skip_build \
  "false" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -175,6 +176,7 @@ jobs:
  set -x
  if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
  "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
  "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.1'" ]]; then
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_be_ut }}" \
@@ -183,7 +185,7 @@ jobs:
  "beut" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch is not in (master, branch-2.0, branch-2.1), skip run beut"
+ echo "PR target branch is not in (master, branch-2.0, branch-2.1, branch-3.0), skip run beut"
  trigger_or_skip_build \
  "false" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -197,15 +199,16 @@ jobs:
  run: |
  source ./regression-test/pipeline/common/teamcity-utils.sh
  set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_cloud_ut }}" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
  "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
  "cloudut" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch is not master, skip run cloudut"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloudut"
  fi
 
  - name: "Trigger or Skip compile"
@@ -279,16 +282,17 @@ jobs:
  echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit
  fi
  set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
- echo "PR target branch in (master), need run cloud_p0"
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+ echo "PR target branch is in (master, branch-3.0), need run cloud_p0"
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_cloud_p0 }}" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
  "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
  "cloud_p0" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch not in (master), skip run cloud_p0"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloud_p0"
  trigger_or_skip_build \
  "false" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -305,16 +309,17 @@ jobs:
  echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit
  fi
  set -x
- if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
- echo "PR target branch in (master), need run cloud_p1"
+ if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ]]; then
+ echo "PR target branch is in (master, branch-3.0), need run cloud_p1"
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_cloud_p1 }}" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
  "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
  "cloud_p1" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch not in (master), skip run cloud_p1"
+ echo "PR target branch is not in (master, branch-3.0), skip run cloud_p1"
  trigger_or_skip_build \
  "false" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
@@ -341,16 +346,17 @@ jobs:
  source ./regression-test/pipeline/common/teamcity-utils.sh
  set -x
  if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-3.0'" ||
  "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
- echo "PR target branch in (master, branch-2.0), need run performance"
+ echo "PR target branch is in (master, branch-2.0, branch-3.0), need run performance"
  trigger_or_skip_build \
  "${{ steps.changes.outputs.changed_performance }}" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
  "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
  "performance" \
  "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
  else
- echo "PR target branch not in (master, branch-2.0), skip run performance"
+ echo "PR target branch is not in (master, branch-2.0, branch-3.0), skip run performance"
  trigger_or_skip_build \
  "false" \
  "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \

.github/workflows/labeler/scope-label-conf.yml

@@ -23,3 +23,6 @@ meta-change:
  - fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
  - gensrc/thrift/*
  - gensrc/proto/*
+
+doing:
+ - '**'

.github/workflows/lfs-warning.yml

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: 'Check Large File'
+
+on: [push, pull_request_target]
+
+jobs:
+ large-file-checker:
+ name: "Check large file"
+ runs-on: ubuntu-latest
+ steps:
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ submodules: recursive
+
+ - name: "Checkout lfs-warning commit"
+ run: |
+ rm -rf ./.github/actions/lfs-warning
+ git clone https://github.com/ppremk/lfs-warning .github/actions/lfs-warning
+ pushd .github/actions/lfs-warning &>/dev/null
+ git checkout 4b98a8a5e6c429c23c34eee02d71553bca216425
+ popd &>/dev/null
+
+ - name: "Check Large File"
+ uses: ./.github/actions/lfs-warning
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
+ filesizelimit: 1MB
+

README.md

@@ -24,17 +24,12 @@ under the License.
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
 [![GitHub release](https://img.shields.io/github/release/apache/doris.svg)](https://github.com/apache/doris/releases)
 [![OSSRank](https://shields.io/endpoint?url=https://ossrank.com/shield/516)](https://ossrank.com/p/516)
-[![Jenkins Vec](https://img.shields.io/jenkins/tests?compact_message&jobUrl=https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized&label=VectorizedEngine)](https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized)
-[![Total Line](https://img.shields.io/badge/Total_Line-GitHub-blue)]((https://github.com/apache/doris))
-[![Join the chat at https://gitter.im/apache-doris/Lobby](https://badges.gitter.im/apache-doris/Lobby.svg)](https://gitter.im/apache-doris/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![Commit activity](https://img.shields.io/github/commit-activity/m/apache/doris)](https://github.com/apache/doris/commits/master/)
 [![EN doc](https://img.shields.io/badge/Docs-English-blue.svg)](https://doris.apache.org/docs/get-starting/quick-start)
 [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://doris.apache.org/zh-CN/docs/get-starting/quick-start/)
 
-
-
 <div>
 
-
 [![Official Website](<https://img.shields.io/badge/-Visit%20the%20Official%20Website%20%E2%86%92-rgb(15,214,106)?style=for-the-badge>)](https://doris.apache.org/)
 [![Quick Download](<https://img.shields.io/badge/-Quick%20%20Download%20%E2%86%92-rgb(66,56,255)?style=for-the-badge>)](https://doris.apache.org/download)
 
@@ -64,10 +59,10 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
-🎉 Version 2.1.0 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.0) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
+🎉 Version 2.1.4 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.4) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
 
 
-🎉 Version 2.0.6 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.0.6) here. 
+🎉 Version 2.0.12 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/2.0/releasenotes/release-2.0.12) here. 
 
 👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
 

be/CMakeLists.txt

@@ -157,11 +157,20 @@ set(BOOST_VERSION "1.81.0")
 
 if (NOT APPLE)
  find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time)
+ find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system container)
 else()
  find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time)
  find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
 endif()
 
+# Set if use libazure or not
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+ add_definitions(-DUSE_AZURE)
+endif()
+
+
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
 
 include (cmake/thirdparty.cmake)
@@ -290,12 +299,11 @@ if (COMPILER_CLANG)
  -Wno-implicit-float-conversion
  -Wno-implicit-int-conversion
  -Wno-sign-conversion
+ -Wno-missing-field-initializers
+ -Wno-unused-const-variable
  -Wno-shorten-64-to-32)
  if (USE_LIBCPP)
  add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
- if (NOT OS_MACOSX)
- add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-lstdc++>)
- endif()
  add_definitions(-DUSE_LIBCPP)
  endif()
 endif ()
@@ -358,40 +366,36 @@ if (USE_UNWIND)
  endif()
 endif()
 
-if (ENABLE_STACKTRACE)
- add_definitions(-DENABLE_STACKTRACE)
-endif()
-
 if (USE_DWARF)
  add_compile_options(-gdwarf-5)
 endif()
 
 # For CMAKE_BUILD_TYPE=Debug
 if (OS_MACOSX AND ARCH_ARM)
  # Using -O0 may meet ARM64 branch out of range errors when linking with tcmalloc.
- set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Og")
+ set(CXX_FLAGS_DEBUG "-Og")
 else()
- set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -O0")
+ set(CXX_FLAGS_DEBUG "-O0")
 endif()
 
 # For CMAKE_BUILD_TYPE=Release
 # -O3: Enable all compiler optimizations
 # -DNDEBUG: Turn off dchecks/asserts/debug only code.
-set(CXX_FLAGS_RELEASE "${CXX_GCC_FLAGS} -O3 -DNDEBUG")
-set(CXX_FLAGS_ASAN "${CXX_GCC_FLAGS} -O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
-set(CXX_FLAGS_LSAN "${CXX_GCC_FLAGS} -O0 -fsanitize=leak -DLEAK_SANITIZER")
+set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
+set(CXX_FLAGS_ASAN "-O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
+set(CXX_FLAGS_LSAN "-O0 -fsanitize=leak -DLEAK_SANITIZER")
 ## Use for BE-UT
-set(CXX_FLAGS_ASAN_UT "${CXX_GCC_FLAGS} -O0 -fsanitize=address -DADDRESS_SANITIZER")
+set(CXX_FLAGS_ASAN_UT "-O0 -fsanitize=address -DADDRESS_SANITIZER")
 
 # Set the flags to the undefined behavior sanitizer, also known as "ubsan"
 # Turn on sanitizer and debug symbols to get stack traces:
-set(CXX_FLAGS_UBSAN "${CXX_GCC_FLAGS} -O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
+set(CXX_FLAGS_UBSAN "-O0 -fno-wrapv -mcmodel=medium -fsanitize=undefined -DUNDEFINED_BEHAVIOR_SANITIZER")
 
 # Set the flags to the thread sanitizer, also known as "tsan"
 # Turn on sanitizer and debug symbols to get stack traces:
 # Use -Wno-builtin-declaration-mismatch to mute warnings like "new declaration ‘__tsan_atomic16 __tsan_atomic16_fetch_nand(..."
 # If use -O0 to compile, BE will stack overflow when start. https://github.com/apache/doris/issues/8868
-set(CXX_FLAGS_TSAN "${CXX_GCC_FLAGS} -O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
+set(CXX_FLAGS_TSAN "-O1 -fsanitize=thread -DTHREAD_SANITIZER -Wno-missing-declarations")
 
 # Set compile flags based on the build type.
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
@@ -513,6 +517,7 @@ find_package(absl)
 # add it here first.
 set(COMMON_THIRDPARTY
  Boost::date_time
+ Boost::container
  ${COMMON_THIRDPARTY}
 )
 
@@ -555,7 +560,6 @@ endif()
 if (OS_MACOSX)
  set(COMMON_THIRDPARTY
  ${COMMON_THIRDPARTY}
- Boost::container
  bfd
  iberty
  intl
@@ -599,9 +603,11 @@ if (NOT OS_MACOSX)
  ${DORIS_DEPENDENCIES}
  -static-libstdc++
  -static-libgcc
- -lstdc++fs
  -lresolv
  )
+ if (NOT (USE_LIBCPP AND COMPILER_CLANG))
+ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs)
+ endif()
 else()
  set(DORIS_LINK_LIBS
  ${DORIS_LINK_LIBS}

be/cmake/thirdparty.cmake

@@ -141,10 +141,12 @@ if (NOT OS_MACOSX)
  add_thirdparty(aws-s2n LIBNAME "lib/libs2n.a")
 endif()
 
-add_thirdparty(azure-core)
-add_thirdparty(azure-identity)
-add_thirdparty(azure-storage-blobs)
-add_thirdparty(azure-storage-common)
+if(BUILD_AZURE STREQUAL "ON")
+ add_thirdparty(azure-core)
+ add_thirdparty(azure-identity)
+ add_thirdparty(azure-storage-blobs)
+ add_thirdparty(azure-storage-common)
+endif()
 
 add_thirdparty(minizip LIB64)
 add_thirdparty(simdjson LIB64)

be/src/agent/be_exec_version_manager.h

@@ -80,14 +80,16 @@ class BeExecVersionManager {
  * b. clear old version of version 3->4
  * c. change FunctionIsIPAddressInRange from AlwaysNotNullable to DependOnArguments
  * d. change some agg function nullable property: PR #37215
+ * e. change variant serde to fix PR #38413
  */
-constexpr inline int BeExecVersionManager::max_be_exec_version = 5;
+constexpr inline int BeExecVersionManager::max_be_exec_version = 6;
 constexpr inline int BeExecVersionManager::min_be_exec_version = 0;
 
 /// functional
 constexpr inline int BITMAP_SERDE = 3;
 constexpr inline int USE_NEW_SERDE = 4; // release on DORIS version 2.1
 constexpr inline int OLD_WAL_SERDE = 3; // use to solve compatibility issues, see pr #32299
 constexpr inline int AGG_FUNCTION_NULLABLE = 5; // change some agg nullable property: PR #37215
+constexpr inline int VARIANT_SERDE = 6; // change variant serde to fix PR #38413
 
 } // namespace doris