[deps](jindofs) add jindofs for 2.1 (#49259)(#49371)(#49374)(#49502)(#49503)(#49886)(#49903)(#49963)

.github/workflows/build-thirdparty.yml

@@ -87,7 +87,7 @@ jobs:
             'autoconf'
             'libtool-bin'
             'pkg-config'
-            'cmake'
+            'cmake=3.22.1-1ubuntu1.22.04.2'
             'ninja-build'
             'ccache'
             'python-is-python3'
@@ -107,6 +107,7 @@ jobs:
           )
 
           sudo apt update
+          sudo apt-cache policy cmake
           sudo DEBIAN_FRONTEND=noninteractive apt install --yes "${packages[@]}"
 
           mkdir -p "${DEFAULT_DIR}"
@@ -153,7 +154,6 @@ jobs:
             'coreutils'
             'gnu-getopt'
             'python@3'
-            'cmake'
             'ninja'
             'ccache'
             'bison'
@@ -167,8 +167,16 @@ jobs:
             'llvm@16'
           )
 
+          # Install packages except cmake
           brew install "${packages[@]}" || true
 
+          # Install specific version of cmake
+          brew unlink cmake || true
+          wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
+          tar -xzf cmake-3.22.1-macos-universal.tar.gz
+          sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
+          cmake --version
+
       - name: Build
         run: |
           export MACOSX_DEPLOYMENT_TARGET=12.0
@@ -204,7 +212,6 @@ jobs:
             'coreutils'
             'gnu-getopt'
             'python@3'
-            'cmake'
             'ninja'
             'ccache'
             'bison'
@@ -218,8 +225,16 @@ jobs:
             'llvm@16'
           )
 
+          # Install packages except cmake
           brew install "${packages[@]}" || true
 
+          # Install specific version of cmake
+          brew unlink cmake || true
+          wget https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-macos-universal.tar.gz
+          tar -xzf cmake-3.22.1-macos-universal.tar.gz
+          sudo cp -r cmake-3.22.1-macos-universal/CMake.app/Contents/* /usr/local/
+          cmake --version
+
       - name: Build
         run: |
           export MACOSX_DEPLOYMENT_TARGET=12.0

build.sh

@@ -36,6 +36,10 @@ fi
 export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
 export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
 
+TARGET_SYSTEM="$(uname -s)"
+TARGET_ARCH="$(uname -m)"
+echo "Target system: ${TARGET_SYSTEM}; Target arch: ${TARGET_ARCH}"
+
 . "${DORIS_HOME}/env.sh"
 
 # Check args
@@ -344,7 +348,7 @@ if [[ -z "${WITH_MYSQL}" ]]; then
     WITH_MYSQL='OFF'
 fi
 if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
-    if [[ "$(uname -s)" != 'Darwin' ]]; then
+    if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
         GLIBC_COMPATIBILITY='ON'
     else
         GLIBC_COMPATIBILITY='OFF'
@@ -354,7 +358,7 @@ if [[ -z "${USE_AVX2}" ]]; then
     USE_AVX2='ON'
 fi
 if [[ -z "${USE_LIBCPP}" ]]; then
-    if [[ "$(uname -s)" != 'Darwin' ]]; then
+    if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
         USE_LIBCPP='OFF'
     else
         USE_LIBCPP='ON'
@@ -364,7 +368,7 @@ if [[ -z "${STRIP_DEBUG_INFO}" ]]; then
     STRIP_DEBUG_INFO='OFF'
 fi
 if [[ -z "${USE_MEM_TRACKER}" ]]; then
-    if [[ "$(uname -s)" != 'Darwin' ]]; then
+    if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
         USE_MEM_TRACKER='ON'
     else
         USE_MEM_TRACKER='OFF'
@@ -399,7 +403,7 @@ if [[ -z "${USE_DWARF}" ]]; then
 fi
 
 if [[ -z "${USE_UNWIND}" ]]; then
-    if [[ "$(uname -s)" != 'Darwin' ]]; then
+    if [[ "${TARGET_SYSTEM}" != 'Darwin' ]]; then
         USE_UNWIND='ON'
     else
         USE_UNWIND='OFF'
@@ -460,7 +464,7 @@ if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then
     RECORD_COMPILER_SWITCHES='OFF'
 fi
 
-if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then
+if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "${TARGET_SYSTEM}" == 'Darwin' ]]; then
     if [[ -z "${JAVA_HOME}" ]]; then
         CAUSE='the environment variable JAVA_HOME is not set'
     else
@@ -746,6 +750,17 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
     cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/
     #cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/
+
+    # copy jindofs jars, only support for Linux x64 or arm
+    if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/
+    elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib"/
+    fi
+
     cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/
     cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/
 
@@ -876,6 +891,16 @@ EOF
         fi
     done
 
+    # copy jindofs jars, only support for Linux x64 or arm
+    if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+    elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+        cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/
+    fi
+
     cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/
     copy_common_files "${DORIS_OUTPUT}/be/"
     mkdir -p "${DORIS_OUTPUT}/be/log"

fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java

@@ -318,11 +318,12 @@ private static Map<String, String> convertToOSSProperties(Map<String, String> pr
             endpoint = endpoint.replace(OssProperties.OSS_PREFIX, "");
         }
         ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY, endpoint);
-        ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss"));
         boolean hdfsEnabled = Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED, "false"));
         if (LocationPath.isHdfsOnOssEndpoint(endpoint) || hdfsEnabled) {
             // use endpoint or enable hdfs
             rewriteHdfsOnOssProperties(ossProperties, endpoint);
+        } else {
+            ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss"));
         }
         if (credential.isWhole()) {
             ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID, credential.getAccessKey());

thirdparty/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}`
 
+## 20250408
+
+- Modified: jindofs 6.3.4 -> 6.8.2
+
 ## 20250402
 
 - Added: icu 75-1, develop ICU tokenizer based on ICU library.

thirdparty/build-thirdparty.sh

@@ -1854,8 +1854,18 @@ build_icu() {
     make install
 }
 
+# jindofs
+build_jindofs() {
+    check_if_source_exist "${JINDOFS_SOURCE}"
+
+    rm -rf "${TP_INSTALL_DIR}/jindofs_libs/"
+    mkdir -p "${TP_INSTALL_DIR}/jindofs_libs/"
+    cp -r ${TP_SOURCE_DIR}/${JINDOFS_SOURCE}/* "${TP_INSTALL_DIR}/jindofs_libs/"
+}
+
 if [[ "${#packages[@]}" -eq 0 ]]; then
     packages=(
+        jindofs
         odbc
         openssl
         libevent

thirdparty/vars.sh

@@ -526,6 +526,12 @@ ICU_NAME=release-69-1.tar.gz
 ICU_SOURCE=icu-release-69-1
 ICU_MD5SUM="135125f633864285d637db5c01e0388b"
 
+# jindofs
+JINDOFS_DOWNLOAD="https://github.com/apache/doris-thirdparty/releases/download/alibabacloud-jindodata-releases/jindofs-6.8.2-libs-0.1.tar.gz"
+JINDOFS_NAME=jindofs-6.8.2-libs-0.1.tar.gz
+JINDOFS_SOURCE=jindofs-6.8.2-libs-0.1
+JINDOFS_MD5SUM="0e5b0f71e636b8ed3f09e0bf16208fd1"
+
 # all thirdparties which need to be downloaded is set in array TP_ARCHIVES
 export TP_ARCHIVES=(
     'LIBEVENT'
@@ -604,6 +610,7 @@ export TP_ARCHIVES=(
     'AZURE'
     'DRAGONBOX'
     'ICU'
+    'JINDOFS'
 )
 
 if [[ "$(uname -s)" == 'Darwin' ]]; then