[feat](catalog)Support for Loading Catalog Credentials via AwsCredentialsProviderChain

fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java

@@ -22,10 +22,12 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider;
 import com.amazonaws.glue.catalog.util.AWSGlueConfig;
 import com.amazonaws.util.StringUtils;
+import org.apache.doris.common.Config;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
 import org.apache.hadoop.conf.Configuration;
 
 public class ConfigurationAWSCredentialsProvider implements AWSCredentialsProvider {
@@ -47,9 +49,9 @@ public AWSCredentials getCredentials() {
             return (StringUtils.isNullOrEmpty(sessionToken) ? new BasicAWSCredentials(accessKey,
                     secretKey) : new BasicSessionCredentials(accessKey, secretKey, sessionToken));
         }
-
-        AWSCredentialsProvider longLivedProvider = new DefaultAWSCredentialsProviderChain();
-
+        String credentialsProviderModeString = StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE));
+        AwsCredentialsProviderMode credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString);
+        AWSCredentialsProvider longLivedProvider = AwsCredentialsProviderFactory.createV1(credentialsProviderMode);
         if (!StringUtils.isNullOrEmpty(roleArn)) {
             STSAssumeRoleSessionCredentialsProvider.Builder builder =
                     new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session")
@@ -61,6 +63,9 @@ public AWSCredentials getCredentials() {
             STSAssumeRoleSessionCredentialsProvider provider = builder.build();
             return provider.getCredentials();
         }
+        if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+            return longLivedProvider.getCredentials();
+        }
         throw new SdkClientException("Unable to load AWS credentials from any provider in the chain");
 
     }

fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java

@@ -63,4 +63,5 @@ private AWSGlueConfig() {
     public static final String AWS_GLUE_SESSION_TOKEN = "aws.glue.session-token";
     public static final String AWS_GLUE_ROLE_ARN = "aws.glue.role-arn";
     public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external-id";
+    public static final String AWS_CREDENTIALS_PROVIDER_MODE = "aws.credentials.provider.mode";
 }

fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java

@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Copied from
+// https://github.com/awslabs/aws-glue-data-catalog-client-for-apache-hive-metastore/blob/branch-3.4.0/
+//
+
+package org.apache.doris.datasource.property.common;
+
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
+import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public final class AwsCredentialsProviderFactory {
+
+    private AwsCredentialsProviderFactory() {
+    }
+
+    /* =========================
+     * AWS SDK V1
+     * ========================= */
+
+    public static com.amazonaws.auth.AWSCredentialsProvider createV1(
+            AwsCredentialsProviderMode mode) {
+
+        switch (mode) {
+            case ENV:
+                return new com.amazonaws.auth.EnvironmentVariableCredentialsProvider();
+            case SYSTEM_PROPERTIES:
+                return new com.amazonaws.auth.SystemPropertiesCredentialsProvider();
+            case WEB_IDENTITY:
+                return com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create();
+            case CONTAINER:
+                return new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper();
+            case ANONYMOUS:
+                throw new UnsupportedOperationException(
+                        "AWS SDK V1 does not support anonymous credentials provider.");
+            case INSTANCE_PROFILE:
+                return new com.amazonaws.auth.InstanceProfileCredentialsProvider();
+            case DEFAULT:
+                return createDefaultV1();
+            default:
+                throw new UnsupportedOperationException(
+                        "AWS SDK V1 does not support credentials provider mode: " + mode);
+        }
+    }
+
+    private static com.amazonaws.auth.AWSCredentialsProvider createDefaultV1() {
+        List<com.amazonaws.auth.AWSCredentialsProvider> providers = new ArrayList<>();
+        providers.add(new com.amazonaws.auth.InstanceProfileCredentialsProvider());
+        //lazy + env
+        providers.add(com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create());
+        providers.add(new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper());
+        providers.add(new com.amazonaws.auth.EnvironmentVariableCredentialsProvider());
+        providers.add(new com.amazonaws.auth.SystemPropertiesCredentialsProvider());
+        return new com.amazonaws.auth.AWSCredentialsProviderChain(
+                providers.toArray(new com.amazonaws.auth.AWSCredentialsProvider[0]));
+    }
+
+    /* =========================
+     * AWS SDK V2
+     * ========================= */
+
+    public static AwsCredentialsProvider createV2(
+            AwsCredentialsProviderMode mode,
+            boolean includeAnonymousInDefault) {
+        switch (mode) {
+            case ENV:
+                return EnvironmentVariableCredentialsProvider.create();
+            case SYSTEM_PROPERTIES:
+                return SystemPropertyCredentialsProvider.create();
+            case WEB_IDENTITY:
+                return WebIdentityTokenFileCredentialsProvider.create();
+            case CONTAINER:
+                return ContainerCredentialsProvider.create();
+            case INSTANCE_PROFILE:
+                return InstanceProfileCredentialsProvider.create();
+            case ANONYMOUS:
+                return AnonymousCredentialsProvider.create();
+            case DEFAULT:
+                return createDefaultV2(includeAnonymousInDefault);
+            default:
+                throw new UnsupportedOperationException(
+                        "AWS SDK V2 does not support credentials provider mode: " + mode);
+        }
+    }
+
+    private static AwsCredentialsProvider createDefaultV2(
+            boolean includeAnonymous) {
+
+        List<AwsCredentialsProvider> providers = new ArrayList<>();
+        providers.add(InstanceProfileCredentialsProvider.create());
+        providers.add(WebIdentityTokenFileCredentialsProvider.create());
+        providers.add(ContainerCredentialsProvider.create());
+        providers.add(EnvironmentVariableCredentialsProvider.create());
+        providers.add(SystemPropertyCredentialsProvider.create());
+        if (includeAnonymous) {
+            providers.add(AnonymousCredentialsProvider.create());
+        }
+        return AwsCredentialsProviderChain.builder()
+                .credentialsProviders(providers)
+                .build();
+    }
+
+    public static String getV2ClassName(AwsCredentialsProviderMode mode, boolean includeAnonymousInDefault) {
+        switch (mode) {
+            case ENV:
+                return EnvironmentVariableCredentialsProvider.class.getName();
+            case SYSTEM_PROPERTIES:
+                return SystemPropertyCredentialsProvider.class.getName();
+            case WEB_IDENTITY:
+                return WebIdentityTokenFileCredentialsProvider.class.getName();
+            case CONTAINER:
+                return ContainerCredentialsProvider.class.getName();
+            case INSTANCE_PROFILE:
+                return InstanceProfileCredentialsProvider.class.getName();
+            case ANONYMOUS:
+                return AnonymousCredentialsProvider.class.getName();
+            case DEFAULT:
+                List<String> providers = new ArrayList<>();
+                providers.add(EnvironmentVariableCredentialsProvider.class.getName());
+                providers.add(SystemPropertyCredentialsProvider.class.getName());
+                providers.add(WebIdentityTokenFileCredentialsProvider.class.getName());
+                providers.add(ContainerCredentialsProvider.class.getName());
+                providers.add(InstanceProfileCredentialsProvider.class.getName());
+                if (includeAnonymousInDefault) {
+                    providers.add(AnonymousCredentialsProvider.class.getName());
+                }
+                return String.join("+", providers);
+            default:
+                throw new UnsupportedOperationException(
+                        "AWS SDK V2 does not support credentials provider mode: " + mode);
+        }
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.common;
+
+public enum AwsCredentialsProviderMode {
+
+    DEFAULT("DEFAULT"),
+
+    ENV("ENV"),
+
+    SYSTEM_PROPERTIES("SYSTEM_PROPERTIES"),
+
+    WEB_IDENTITY("WEB_IDENTITY"),
+
+    CONTAINER("CONTAINER"),
+
+    INSTANCE_PROFILE("INSTANCE_PROFILE"),
+
+    ANONYMOUS("ANONYMOUS");
+
+    private final String mode;
+
+    AwsCredentialsProviderMode(String mode) {
+        this.mode = mode;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+
+
+    public static AwsCredentialsProviderMode fromString(String value) {
+        if (value == null || value.isEmpty()) {
+            return DEFAULT;
+        }
+
+        String normalized = value.trim().toUpperCase().replace('-', '_');
+
+        switch (normalized) {
+            case "ENV":
+                return ENV;
+            case "SYSTEM_PROPERTIES":
+                return SYSTEM_PROPERTIES;
+            case "WEB_IDENTITY":
+                return WEB_IDENTITY;
+            case "CONTAINER":
+                return CONTAINER;
+            case "INSTANCE_PROFILE":
+                return INSTANCE_PROFILE;
+            case "ANONYMOUS":
+                return ANONYMOUS;
+            case "DEFAULT":
+                return DEFAULT;
+            default:
+                throw new IllegalArgumentException(
+                        "Unsupported AWS credentials provider mode: " + value);
+        }
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java

@@ -114,8 +114,6 @@ private ParamRules buildRules() {
 
         return new ParamRules().requireTogether(new String[]{glueAccessKey, glueSecretKey},
                         "glue.access_key and glue.secret_key must be set together")
-                .requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole},
-                        "At least one of glue.access_key or glue.role_arn must be set")
                 .require(glueEndpoint, "glue.endpoint must be set")
                 .check(() -> StringUtils.isNotBlank(glueEndpoint) && !glueEndpoint.startsWith("https://"),
                         "glue.endpoint must use https protocol,please set glue.endpoint to https://...");
@@ -137,6 +135,22 @@ private void checkAndInit() {
         }
     }
 
+    /**
+     * Validate that at least one Glue credential (an access key or an IAM role) is explicitly provided.
+     *
+     * Purpose: Some catalog implementations (for example, Iceberg) do not support obtaining credentials
+     * from the default credential chain (instance metadata, environment variables, etc.). In addition,
+     * the configuration or UI may only expose two options: {@code glue.access_key} and {@code glue.role_arn}.
+     * In such cases, at least one of these must be explicitly set. If neither is provided, an
+     * {@link IllegalArgumentException} is thrown to prompt the user to complete the configuration.
+     */
+    protected void requireExplicitGlueCredentials() {
+        if (StringUtils.isNotBlank(glueAccessKey) || StringUtils.isNotBlank(glueIAMRole)) {
+            return;
+        }
+        throw new IllegalArgumentException("At least one of glue.access_key or glue.role_arn must be set");
+    }
+
     private String extractRegionFromEndpoint(Matcher matcher) {
         for (int i = 1; i <= matcher.groupCount(); i++) {
             String group = matcher.group(i);

fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java

@@ -18,6 +18,7 @@
 package org.apache.doris.datasource.property.metastore;
 
 import org.apache.doris.datasource.property.ConnectorProperty;
+import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode;
 
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.glue.catalog.util.AWSGlueConfig;
@@ -75,6 +76,14 @@ public class HiveGlueMetaStoreProperties extends AbstractHiveProperties {
             description = "Catalog separator character for AWS Glue.")
     protected String awsGlueCatalogSeparator = "";
 
+    @ConnectorProperty(names = {"glue.credentials_provider_type"},
+            required = false,
+            description = "The credentials provider type of S3. "
+                    + "Options are: DEFAULT, ASSUME_ROLE, ANONYMOUS, ENVIRONMENT, SYSTEM_PROPERTIES, "
+                    + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. "
+                    + "If not set, it will use the default provider chain of AWS SDK.")
+    protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name();
+
     // ========== Constructor ==========
 
     /**
@@ -115,6 +124,8 @@ private void initHiveConf() {
         setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, baseProperties.glueSessionToken);
         setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_ROLE_ARN, baseProperties.glueIAMRole);
         setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_EXTERNAL_ID, baseProperties.glueExternalId);
+        setHiveConfPropertiesIfNotNull(hiveConf,
+                AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE, awsCredentialsProviderType);
     }
 
     private static void setHiveConfPropertiesIfNotNull(HiveConf hiveConf, String key, String value) {

fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java

@@ -55,6 +55,7 @@ public String getIcebergCatalogType() {
     public void initNormalizeAndCheckProps() {
         super.initNormalizeAndCheckProps();
         glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
+        glueProperties.requireExplicitGlueCredentials();
         s3Properties = S3Properties.of(origProps);
     }