From 5346e7da00cf986762a139ca28206238f7389ca3 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 2 Sep 2025 11:51:18 +0800 Subject: [PATCH 1/2] [Fix](oss-hdfs)Fix insert failure on Hive partitioned table with oss-hdfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When inserting into a Hive partitioned table stored on oss-hdfs, the following issue occurs: First insert succeeds: Since the partition does not exist yet, HiveTableSink#setPartitionValues does not set storage-related information for the partition. Subsequent inserts fail: Once the partition exists, the system tries to resolve the partition’s storage information. At this stage, oss-hdfs is incorrectly treated as s3 instead of being recognized as hdfs, leading to insert failure. This PR fixes the storage type handling logic so that oss-hdfs partitions are correctly recognized as hdfs. --- .../org/apache/doris/common/util/LocationPath.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java index 798e13d321fe88..ad16ce1c74c079 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java @@ -137,19 +137,7 @@ public LocationPath(String originLocation, Map props, boolean co tmpLocation = convertPath ? convertToS3(tmpLocation) : tmpLocation; break; case FeConstants.FS_PREFIX_OSS: - String endpoint = ""; - if (props.containsKey(OssProperties.ENDPOINT)) { - endpoint = props.get(OssProperties.ENDPOINT); - if (endpoint.startsWith(OssProperties.OSS_PREFIX)) { - // may use oss.oss-cn-beijing.aliyuncs.com - endpoint = endpoint.replace(OssProperties.OSS_PREFIX, ""); - } - } else if (props.containsKey(S3Properties.ENDPOINT)) { - endpoint = props.get(S3Properties.ENDPOINT); - } else if (props.containsKey(S3Properties.Env.ENDPOINT)) { - endpoint = props.get(S3Properties.Env.ENDPOINT); - } - if (isHdfsOnOssEndpoint(endpoint)) { + if (isHdfsOnOssEndpoint(originLocation)) { this.scheme = Scheme.OSS_HDFS; } else { if (useS3EndPoint(props)) { From 002d64dd54e4d36973b41bfd0ae896526f47f8c6 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 2 Sep 2025 13:06:11 +0800 Subject: [PATCH 2/2] check style --- .../java/org/apache/doris/fs/remote/RemoteFileSystemTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java index 3fc15ab8e374fa..2b94dbf8bad92e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java @@ -68,7 +68,7 @@ public void testFilesystemAndAuthType() throws UserException { dfsPaths.add("ofs://a/b/c"); dfsPaths.add("gfs://a/b/c"); dfsPaths.add("hdfs://a/b/c"); - dfsPaths.add("oss://a/b/c"); // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem + dfsPaths.add("oss://emr-ssss-oss.cn-beijing.oss-dls.aliyuncs.com/tmp/"); // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem new MockUp(UserGroupInformation.class) { @Mock