From 19d9d6d8119e49bc51ed7456a48626ab42f06596 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Mar 2025 12:25:03 +0800 Subject: [PATCH 1/3] [opt](jindofs) remove unused hadoop-aliyun --- build.sh | 1 + fe/fe-common/pom.xml | 4 ---- .../doris/datasource/property/PropertyConverter.java | 3 ++- fe/pom.xml | 12 ------------ 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/build.sh b/build.sh index f6e974affff592..3ddc158a805c4f 100755 --- a/build.sh +++ b/build.sh @@ -769,6 +769,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then # copy jindofs jars, only support for Linux x64 or arm if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-6.3.4.jar" "${DORIS_OUTPUT}/fe/lib"/ + cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-6.3.4.jar" "${DORIS_OUTPUT}/fe/lib"/ cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-sdk-6.3.4.jar" "${DORIS_OUTPUT}/fe/lib"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-linux-el7-aarch64-6.3.4.jar" "${DORIS_OUTPUT}/fe/lib"/ diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index 55934fd4e246a5..9d356d35008a55 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -137,10 +137,6 @@ under the License. aliyun-sdk-oss ${aliyun-sdk-oss.version} - - org.apache.hadoop - hadoop-aliyun - com.esotericsoftware kryo-shaded diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index cbdf3d4ce6af06..47ae235e3e1606 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -318,11 +318,12 @@ private static Map convertToOSSProperties(Map pr endpoint = endpoint.replace(OssProperties.OSS_PREFIX, ""); } ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ENDPOINT_KEY, endpoint); - ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss")); boolean hdfsEnabled = Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED, "false")); if (LocationPath.isHdfsOnOssEndpoint(endpoint) || hdfsEnabled) { // use endpoint or enable hdfs rewriteHdfsOnOssProperties(ossProperties, endpoint); + } else { + ossProperties.put("fs.oss.impl", getHadoopFSImplByScheme("oss")); } if (credential.isWhole()) { ossProperties.put(org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID, credential.getAccessKey()); diff --git a/fe/pom.xml b/fe/pom.xml index 55d2763ac52303..697a8724ac1441 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -654,18 +654,6 @@ under the License. - - org.apache.hadoop - hadoop-aliyun - ${hadoop.version} - - - - org.ini4j - ini4j - - - org.apache.doris hive-catalog-shade From e7bcfa32991363f50fb60b420276874850d0efd5 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Mar 2025 22:03:08 +0800 Subject: [PATCH 2/3] 1 --- fe/fe-common/pom.xml | 4 ++++ fe/pom.xml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index 9d356d35008a55..55934fd4e246a5 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -137,6 +137,10 @@ under the License. aliyun-sdk-oss ${aliyun-sdk-oss.version} + + org.apache.hadoop + hadoop-aliyun + com.esotericsoftware kryo-shaded diff --git a/fe/pom.xml b/fe/pom.xml index 697a8724ac1441..55d2763ac52303 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -654,6 +654,18 @@ under the License. + + org.apache.hadoop + hadoop-aliyun + ${hadoop.version} + + + + org.ini4j + ini4j + + + org.apache.doris hive-catalog-shade From 27256d3f49982211a73d122a0927bbeb43c5615c Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Mar 2025 22:04:20 +0800 Subject: [PATCH 3/3] 1 --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 3ddc158a805c4f..3774803c5332af 100755 --- a/build.sh +++ b/build.sh @@ -917,6 +917,7 @@ EOF # copy jindofs jars, only support for Linux x64 or arm if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-6.3.4.jar" "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ + cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-6.3.4.jar" "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-sdk-6.3.4.jar" "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/ elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then cp -r -p "${DORIS_THIRDPARTY}/installed/jindofs_libs/jindo-core-linux-el7-aarch64-6.3.4.jar" "${DORIS_OUTPUT}/be/lib/java_extensions/preload-extensions"/