diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index 4b8997b6d56b9e..ed03e1f0fcd09d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -29,6 +29,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; import java.util.HashMap; import java.util.Map; @@ -250,6 +251,7 @@ public void initializeHadoopStorageConfig() { // storage is OSS, OBS, etc., users may still configure the schema as "s3://". // To ensure backward compatibility, we append S3-related properties by default. appendS3HdfsProperties(hadoopStorageConfig); + setDefaultRequestChecksum(); } private void appendS3HdfsProperties(Configuration hadoopStorageConfig) { @@ -275,6 +277,34 @@ private void appendS3HdfsProperties(Configuration hadoopStorageConfig) { hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle()); } + /** + * Sets the AWS request checksum calculation property to "WHEN_REQUIRED" + * only if it has not been explicitly set by the user. + * + *

+ * Background: + * AWS SDK for Java v2 uses the system property + * {@link SdkSystemSetting#AWS_REQUEST_CHECKSUM_CALCULATION} to determine + * whether request payloads should have a checksum calculated. + *

+ * According to the official AWS discussion: + * https://github.com/aws/aws-sdk-java-v2/discussions/5802 + * - Default SDK behavior may calculate checksums automatically if the property is not set. + * - Automatic calculation can affect performance or cause unexpected behavior for large requests. + *

+ * This method ensures: + * 1. The property is set to "WHEN_REQUIRED" only if the user has not already set it. + * 2. User-specified settings are never overridden. + * 3. Aligns with AWS SDK recommended best practices. + *

+ */ + public static void setDefaultRequestChecksum() { + String key = SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property(); + if (System.getProperty(key) == null) { + System.setProperty(key, "WHEN_REQUIRED"); + } + } + @Override public String getStorageName() { return "S3"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java index 0bc5e823f0ecee..5d6c5dae12f31f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; import java.util.HashMap; import java.util.Map; @@ -268,4 +269,13 @@ public void testS3DisableHadoopCache() throws UserException { Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache", false)); } + @Test + public void testResuestCheckSum() throws UserException { + Map props = Maps.newHashMap(); + props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com"); + Assertions.assertEquals("WHEN_REQUIRED", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property())); + System.setProperty("aws.requestChecksumCalculation", "ALWAYS"); + Assertions.assertEquals("ALWAYS", System.getProperty(SdkSystemSetting.AWS_REQUEST_CHECKSUM_CALCULATION.property())); + } + }