From 84463919a48be52b579af8bdd22e5a27fa9915aa Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 1 Dec 2025 15:47:14 +0800 Subject: [PATCH 1/2] [fix](S3)Downgrade the AWS SDK version to avoid impacting other S3-compatible services. This PR addresses an issue related to AWS SDK for Java v2 S3 client behavior described in: https://github.com/aws/aws-sdk-java-v2/issues/5805 Starting from newer versions of the SDK, the S3 client introduces unexpected behavior that causes our application to fail during normal S3 operations. Due to this problem, we were forced to downgrade aws-s3 from the latest version to 2.29.x, which restored correct functionality. The regression affects stability in our production environment and prevents us from upgrading the SDK. --- fe/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/pom.xml b/fe/pom.xml index ef6b50f3272d17..cb307dca7e7026 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -390,7 +390,8 @@ under the License. 12.22.0 5.3.0 3.15.0 - 2.37.2 + + 2.29.52 0.1.4 4.11.0 From c613ff3595b839b33d781d07109262956802f5e2 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 1 Dec 2025 16:08:02 +0800 Subject: [PATCH 2/2] 1 --- .../AbstractS3CompatibleProperties.java | 30 ------------------- .../property/storage/OSSPropertiesTest.java | 10 ------- 2 files changed, 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index ed03e1f0fcd09d..4b8997b6d56b9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -29,7 +29,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.SdkSystemSetting; import java.util.HashMap; import java.util.Map; @@ -251,7 +250,6 @@ public void initializeHadoopStorageConfig() { // storage is OSS, OBS, etc., users may still configure the schema as "s3://". // To ensure backward compatibility, we append S3-related properties by default. appendS3HdfsProperties(hadoopStorageConfig); - setDefaultRequestChecksum(); } private void appendS3HdfsProperties(Configuration hadoopStorageConfig) { @@ -277,34 +275,6 @@ private void appendS3HdfsProperties(Configuration hadoopStorageConfig) { hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle()); } - /** - * Sets the AWS request checksum calculation property to "WHEN_REQUIRED" - * only if it has not been explicitly set by the user. - * - *

- * Background: - * AWS SDK for Java v2 uses the system property - * {@link SdkSystemSetting#AWS_REQUEST_CHECKSUM_CALCULATION} to determine - * whether request payloads should have a checksum calculated. - *

- * According to the official AWS discussion: - * https://github.com/aws/aws-sdk-java-v2/discussions/5802 - * - Default SDK behavior may calculate checksums automatically if the property is not set. - * - Automatic calculation can affect performance or cause unexpected behavior for large requests. - *

- * This method ensures: - * 1. The property is set to "WHEN_REQUIRED" only if the user has not already set it. - * 2. User-specified settings are never overridden. - * 3. Aligns with AWS SDK recommended best practices. - *

- */ - public static void setDefaultRequestChecksum() { - String key = SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property(); - if (System.getProperty(key) == null) { - System.setProperty(key, "WHEN_REQUIRED"); - } - } - @Override public String getStorageName() { return "S3"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java index 5d6c5dae12f31f..0bc5e823f0ecee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.SdkSystemSetting; import java.util.HashMap; import java.util.Map; @@ -269,13 +268,4 @@ public void testS3DisableHadoopCache() throws UserException { Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache", false)); } - @Test - public void testResuestCheckSum() throws UserException { - Map props = Maps.newHashMap(); - props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); - Assertions.assertEquals("WHEN_REQUIRED", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property())); - System.setProperty("aws.requestChecksumCalculation", "ALWAYS"); - Assertions.assertEquals("ALWAYS", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property())); - } - }