diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java index 49bea54c38e5b0..23ba972fdfe1aa 100644 --- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java +++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/credentials/ConfigurationAWSCredentialsProvider.java @@ -22,10 +22,12 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; import com.amazonaws.glue.catalog.util.AWSGlueConfig; import com.amazonaws.util.StringUtils; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.hadoop.conf.Configuration; public class ConfigurationAWSCredentialsProvider implements AWSCredentialsProvider { @@ -47,9 +49,9 @@ public AWSCredentials getCredentials() { return (StringUtils.isNullOrEmpty(sessionToken) ? new BasicAWSCredentials(accessKey, secretKey) : new BasicSessionCredentials(accessKey, secretKey, sessionToken)); } - - AWSCredentialsProvider longLivedProvider = new DefaultAWSCredentialsProviderChain(); - + String credentialsProviderModeString = StringUtils.lowerCase(conf.get(AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE)); + AwsCredentialsProviderMode credentialsProviderMode=AwsCredentialsProviderMode.fromString(credentialsProviderModeString); + AWSCredentialsProvider longLivedProvider = AwsCredentialsProviderFactory.createV1(credentialsProviderMode); if (!StringUtils.isNullOrEmpty(roleArn)) { STSAssumeRoleSessionCredentialsProvider.Builder builder = new STSAssumeRoleSessionCredentialsProvider.Builder(roleArn, "local-session") @@ -61,6 +63,9 @@ public AWSCredentials getCredentials() { STSAssumeRoleSessionCredentialsProvider provider = builder.build(); return provider.getCredentials(); } + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + return longLivedProvider.getCredentials(); + } throw new SdkClientException("Unable to load AWS credentials from any provider in the chain"); } diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java index 4296eee0e94364..7b0bd3ef979130 100644 --- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java +++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/util/AWSGlueConfig.java @@ -63,4 +63,5 @@ private AWSGlueConfig() { public static final String AWS_GLUE_SESSION_TOKEN = "aws.glue.session-token"; public static final String AWS_GLUE_ROLE_ARN = "aws.glue.role-arn"; public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external-id"; + public static final String AWS_CREDENTIALS_PROVIDER_MODE = "aws.credentials.provider.mode"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java new file mode 100644 index 00000000000000..f56aed0533e619 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderFactory.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +// Copied from +// https://github.com/awslabs/aws-glue-data-catalog-client-for-apache-hive-metastore/blob/branch-3.4.0/ +// + +package org.apache.doris.datasource.property.common; + + +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; +import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; +import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; + +import java.util.ArrayList; +import java.util.List; + +public final class AwsCredentialsProviderFactory { + + private AwsCredentialsProviderFactory() { + } + + /* ========================= + * AWS SDK V1 + * ========================= */ + + public static com.amazonaws.auth.AWSCredentialsProvider createV1( + AwsCredentialsProviderMode mode) { + + switch (mode) { + case ENV: + return new com.amazonaws.auth.EnvironmentVariableCredentialsProvider(); + case SYSTEM_PROPERTIES: + return new com.amazonaws.auth.SystemPropertiesCredentialsProvider(); + case WEB_IDENTITY: + return com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create(); + case CONTAINER: + return new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper(); + case ANONYMOUS: + throw new UnsupportedOperationException( + "AWS SDK V1 does not support anonymous credentials provider."); + case INSTANCE_PROFILE: + return new com.amazonaws.auth.InstanceProfileCredentialsProvider(); + case DEFAULT: + return createDefaultV1(); + default: + throw new UnsupportedOperationException( + "AWS SDK V1 does not support credentials provider mode: " + mode); + } + } + + private static com.amazonaws.auth.AWSCredentialsProvider createDefaultV1() { + List providers = new ArrayList<>(); + providers.add(new com.amazonaws.auth.InstanceProfileCredentialsProvider()); + //lazy + env + providers.add(com.amazonaws.auth.WebIdentityTokenCredentialsProvider.create()); + providers.add(new com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper()); + providers.add(new com.amazonaws.auth.EnvironmentVariableCredentialsProvider()); + providers.add(new com.amazonaws.auth.SystemPropertiesCredentialsProvider()); + return new com.amazonaws.auth.AWSCredentialsProviderChain( + providers.toArray(new com.amazonaws.auth.AWSCredentialsProvider[0])); + } + + /* ========================= + * AWS SDK V2 + * ========================= */ + + public static AwsCredentialsProvider createV2( + AwsCredentialsProviderMode mode, + boolean includeAnonymousInDefault) { + switch (mode) { + case ENV: + return EnvironmentVariableCredentialsProvider.create(); + case SYSTEM_PROPERTIES: + return SystemPropertyCredentialsProvider.create(); + case WEB_IDENTITY: + return WebIdentityTokenFileCredentialsProvider.create(); + case CONTAINER: + return ContainerCredentialsProvider.create(); + case INSTANCE_PROFILE: + return InstanceProfileCredentialsProvider.create(); + case ANONYMOUS: + return AnonymousCredentialsProvider.create(); + case DEFAULT: + return createDefaultV2(includeAnonymousInDefault); + default: + throw new UnsupportedOperationException( + "AWS SDK V2 does not support credentials provider mode: " + mode); + } + } + + private static AwsCredentialsProvider createDefaultV2( + boolean includeAnonymous) { + + List providers = new ArrayList<>(); + providers.add(InstanceProfileCredentialsProvider.create()); + providers.add(WebIdentityTokenFileCredentialsProvider.create()); + providers.add(ContainerCredentialsProvider.create()); + providers.add(EnvironmentVariableCredentialsProvider.create()); + providers.add(SystemPropertyCredentialsProvider.create()); + if (includeAnonymous) { + providers.add(AnonymousCredentialsProvider.create()); + } + return AwsCredentialsProviderChain.builder() + .credentialsProviders(providers) + .build(); + } + + public static String getV2ClassName(AwsCredentialsProviderMode mode, boolean includeAnonymousInDefault) { + switch (mode) { + case ENV: + return EnvironmentVariableCredentialsProvider.class.getName(); + case SYSTEM_PROPERTIES: + return SystemPropertyCredentialsProvider.class.getName(); + case WEB_IDENTITY: + return WebIdentityTokenFileCredentialsProvider.class.getName(); + case CONTAINER: + return ContainerCredentialsProvider.class.getName(); + case INSTANCE_PROFILE: + return InstanceProfileCredentialsProvider.class.getName(); + case ANONYMOUS: + return AnonymousCredentialsProvider.class.getName(); + case DEFAULT: + List providers = new ArrayList<>(); + providers.add(EnvironmentVariableCredentialsProvider.class.getName()); + providers.add(SystemPropertyCredentialsProvider.class.getName()); + providers.add(WebIdentityTokenFileCredentialsProvider.class.getName()); + providers.add(ContainerCredentialsProvider.class.getName()); + providers.add(InstanceProfileCredentialsProvider.class.getName()); + if (includeAnonymousInDefault) { + providers.add(AnonymousCredentialsProvider.class.getName()); + } + return String.join("+", providers); + default: + throw new UnsupportedOperationException( + "AWS SDK V2 does not support credentials provider mode: " + mode); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java new file mode 100644 index 00000000000000..63481ca5a7a482 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/common/AwsCredentialsProviderMode.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.common; + +public enum AwsCredentialsProviderMode { + + DEFAULT("DEFAULT"), + + ENV("ENV"), + + SYSTEM_PROPERTIES("SYSTEM_PROPERTIES"), + + WEB_IDENTITY("WEB_IDENTITY"), + + CONTAINER("CONTAINER"), + + INSTANCE_PROFILE("INSTANCE_PROFILE"), + + ANONYMOUS("ANONYMOUS"); + + private final String mode; + + AwsCredentialsProviderMode(String mode) { + this.mode = mode; + } + + public String getMode() { + return mode; + } + + + public static AwsCredentialsProviderMode fromString(String value) { + if (value == null || value.isEmpty()) { + return DEFAULT; + } + + String normalized = value.trim().toUpperCase().replace('-', '_'); + + switch (normalized) { + case "ENV": + return ENV; + case "SYSTEM_PROPERTIES": + return SYSTEM_PROPERTIES; + case "WEB_IDENTITY": + return WEB_IDENTITY; + case "CONTAINER": + return CONTAINER; + case "INSTANCE_PROFILE": + return INSTANCE_PROFILE; + case "ANONYMOUS": + return ANONYMOUS; + case "DEFAULT": + return DEFAULT; + default: + throw new IllegalArgumentException( + "Unsupported AWS credentials provider mode: " + value); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java index 445b8311f85f67..b17504fbca7bb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java @@ -114,8 +114,6 @@ private ParamRules buildRules() { return new ParamRules().requireTogether(new String[]{glueAccessKey, glueSecretKey}, "glue.access_key and glue.secret_key must be set together") - .requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole}, - "At least one of glue.access_key or glue.role_arn must be set") .require(glueEndpoint, "glue.endpoint must be set") .check(() -> StringUtils.isNotBlank(glueEndpoint) && !glueEndpoint.startsWith("https://"), "glue.endpoint must use https protocol,please set glue.endpoint to https://..."); @@ -137,6 +135,22 @@ private void checkAndInit() { } } + /** + * Validate that at least one Glue credential (an access key or an IAM role) is explicitly provided. + * + * Purpose: Some catalog implementations (for example, Iceberg) do not support obtaining credentials + * from the default credential chain (instance metadata, environment variables, etc.). In addition, + * the configuration or UI may only expose two options: {@code glue.access_key} and {@code glue.role_arn}. + * In such cases, at least one of these must be explicitly set. If neither is provided, an + * {@link IllegalArgumentException} is thrown to prompt the user to complete the configuration. + */ + protected void requireExplicitGlueCredentials() { + if (StringUtils.isNotBlank(glueAccessKey) || StringUtils.isNotBlank(glueIAMRole)) { + return; + } + throw new IllegalArgumentException("At least one of glue.access_key or glue.role_arn must be set"); + } + private String extractRegionFromEndpoint(Matcher matcher) { for (int i = 1; i <= matcher.groupCount(); i++) { String group = matcher.group(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java index 5a90a3af7dc94f..06cd55b431b195 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HiveGlueMetaStoreProperties.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.metastore; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import com.amazonaws.ClientConfiguration; import com.amazonaws.glue.catalog.util.AWSGlueConfig; @@ -75,6 +76,14 @@ public class HiveGlueMetaStoreProperties extends AbstractHiveProperties { description = "Catalog separator character for AWS Glue.") protected String awsGlueCatalogSeparator = ""; + @ConnectorProperty(names = {"glue.credentials_provider_type"}, + required = false, + description = "The credentials provider type of S3. " + + "Options are: DEFAULT, ASSUME_ROLE, ANONYMOUS, ENVIRONMENT, SYSTEM_PROPERTIES, " + + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. " + + "If not set, it will use the default provider chain of AWS SDK.") + protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name(); + // ========== Constructor ========== /** @@ -115,6 +124,8 @@ private void initHiveConf() { setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_SESSION_TOKEN, baseProperties.glueSessionToken); setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_ROLE_ARN, baseProperties.glueIAMRole); setHiveConfPropertiesIfNotNull(hiveConf, AWSGlueConfig.AWS_GLUE_EXTERNAL_ID, baseProperties.glueExternalId); + setHiveConfPropertiesIfNotNull(hiveConf, + AWSGlueConfig.AWS_CREDENTIALS_PROVIDER_MODE, awsCredentialsProviderType); } private static void setHiveConfPropertiesIfNotNull(HiveConf hiveConf, String key, String value) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java index ba40940005b5fa..3039a96ea9d3f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java @@ -55,6 +55,7 @@ public String getIcebergCatalogType() { public void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps); + glueProperties.requireExplicitGlueCredentials(); s3Properties = S3Properties.of(origProps); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index ffa6e6c7450dd4..3452d759395b2a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -24,6 +24,8 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.property.ConnectorPropertiesUtils; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory; +import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.doris.thrift.TCredProviderType; import org.apache.doris.thrift.TS3StorageParam; @@ -33,15 +35,11 @@ import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain; -import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider; -import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; -import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; @@ -57,6 +55,8 @@ public class S3Properties extends AbstractS3CompatibleProperties { + private static final Logger LOG = LogManager.getLogger(S3Properties.class); + public static final String USE_PATH_STYLE = "use_path_style"; private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { @@ -170,6 +170,16 @@ public class S3Properties extends AbstractS3CompatibleProperties { description = "The external id of S3.") protected String s3ExternalId = ""; + @ConnectorProperty(names = {"s3.credentials_provider_type", "glue.credentials_provider_type"}, + required = false, + description = "The credentials provider type of S3. " + + "Options are: DEFAULT, ASSUME_ROLE, ENVIRONMENT, SYSTEM_PROPERTIES, " + + "WEB_IDENTITY_TOKEN_FILE, INSTANCE_PROFILE. " + + "If not set, it will use the default provider chain of AWS SDK.") + protected String awsCredentialsProviderType = AwsCredentialsProviderMode.DEFAULT.name(); + + private AwsCredentialsProviderMode awsCredentialsProviderMode; + public static S3Properties of(Map properties) { S3Properties propertiesObj = new S3Properties(properties); ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties); @@ -215,6 +225,7 @@ public void initNormalizeAndCheckProps() { throw new IllegalArgumentException("s3.external_id must be used with s3.role_arn"); } convertGlueToS3EndpointIfNeeded(); + awsCredentialsProviderMode = AwsCredentialsProviderMode.fromString(awsCredentialsProviderType); } /** @@ -310,15 +321,7 @@ private AwsCredentialsProvider getAwsCredentialsProviderV1() { }).build(); } // For anonymous access (no credentials required) - //fixme: should return AwsCredentialsProviderChain - if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) { - return AnonymousCredentialsProvider.create(); - } - return AwsCredentialsProviderChain.of(SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - WebIdentityTokenFileCredentialsProvider.create(), - ProfileCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create()); + return AnonymousCredentialsProvider.create(); } private AwsCredentialsProvider getAwsCredentialsProviderV2() { @@ -329,15 +332,10 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { if (StringUtils.isNotBlank(s3IAMRole)) { StsClient stsClient = StsClient.builder() .region(Region.of(region)) - .credentialsProvider(AwsCredentialsProviderChain.of( - WebIdentityTokenFileCredentialsProvider.create(), - ContainerCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create(), - SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - ProfileCredentialsProvider.create())) + .credentialsProvider(AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + false)) .build(); - return StsAssumeRoleCredentialsProvider.builder() .stsClient(stsClient) .refreshRequest(builder -> { @@ -347,13 +345,9 @@ private AwsCredentialsProvider getAwsCredentialsProviderV2() { } }).build(); } - return AwsCredentialsProviderChain.of( - WebIdentityTokenFileCredentialsProvider.create(), - ContainerCredentialsProvider.create(), - InstanceProfileCredentialsProvider.create(), - SystemPropertyCredentialsProvider.create(), - EnvironmentVariableCredentialsProvider.create(), - ProfileCredentialsProvider.create()); + return AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + true); } @Override @@ -374,12 +368,26 @@ public void initializeHadoopStorageConfig() { hadoopStorageConfig.set("fs.s3a.assumed.role.arn", s3IAMRole); hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider"); - hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", - "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,com.amazonaws.auth.EnvironmentVar" - + "iableCredentialsProvider,com.amazonaws.auth.InstanceProfileCredentialsProvider"); + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", + AwsCredentialsProviderFactory.getV2ClassName( + awsCredentialsProviderMode, + false)); + } else { + hadoopStorageConfig.set("fs.s3a.assumed.role.credentials.provider", + InstanceProfileCredentialsProvider.class.getName()); + } + if (StringUtils.isNotBlank(s3ExternalId)) { hadoopStorageConfig.set("fs.s3a.assumed.role.external.id", s3ExternalId); } + return; + } + if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) { + hadoopStorageConfig.set("fs.s3a.aws.credentials.provider", + AwsCredentialsProviderFactory.createV2( + awsCredentialsProviderMode, + true).getClass().getName()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index f4ea2fcb381a0c..b6a0285f6dae32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -222,9 +222,10 @@ public void testS3IamRoleWithExternalId() throws UserException { Assertions.assertEquals("arn:aws:iam::123456789012:role/MyTestRole", backendProperties.get("AWS_ROLE_ARN")); } + @Test public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsClientBuilder mockBuilder, - @Mocked StsClient mockStsClient, @Mocked InstanceProfileCredentialsProvider mockInstanceCreds) { + @Mocked StsClient mockStsClient) { new Expectations() { { @@ -234,8 +235,6 @@ public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsCli result = mockBuilder; mockBuilder.build(); result = mockStsClient; - InstanceProfileCredentialsProvider.create(); - result = mockInstanceCreds; } }; @@ -247,6 +246,29 @@ public void testGetAwsCredentialsProviderWithIamRoleAndExternalId(@Mocked StsCli AwsCredentialsProvider provider = s3Props.getAwsCredentialsProvider(); Assertions.assertNotNull(provider); Assertions.assertTrue(provider instanceof StsAssumeRoleCredentialsProvider); + origProps.put("s3.credentials_provider_type", "instance_profile"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertEquals(StsAssumeRoleCredentialsProvider.class.getName(), provider.getClass().getName()); + Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(), s3Props.getHadoopStorageConfig().get("fs.s3a.assumed.role.credentials.provider")); + origProps.remove("s3.external_id"); + origProps.remove("s3.role_arn"); + origProps.remove("s3.credentials_provider_type"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertNotNull(provider); + Assertions.assertTrue(provider instanceof AwsCredentialsProviderChain); + origProps.put("s3.credentials_provider_type", "instance_profile"); + s3Props = (S3Properties) StorageProperties.createPrimary(origProps); + provider = s3Props.getAwsCredentialsProvider(); + Assertions.assertNotNull(provider); + Assertions.assertTrue(provider instanceof InstanceProfileCredentialsProvider); + Assertions.assertEquals(InstanceProfileCredentialsProvider.class.getName(), s3Props.getHadoopStorageConfig().get("fs.s3a.aws.credentials.provider")); + origProps.put("s3.credentials_provider_type", "static"); + ExceptionChecker.expectThrowsWithMsg(IllegalArgumentException.class, "Unsupported AWS credentials provider mode: static", () -> StorageProperties.createPrimary(origProps)); + origProps.put("s3.credentials_provider_type", "anonymous"); + Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); + } @Test diff --git a/fe/pom.xml b/fe/pom.xml index a648bef6e1832c..794d32ae01d74b 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -230,7 +230,7 @@ under the License. 3.1.9 compile compile - compile + provided 2.0.1 8.2.7 5.6.211 diff --git a/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy new file mode 100644 index 00000000000000..f8877929b969b5 --- /dev/null +++ b/regression-test/suites/aws_iam_role_p0/test_catalog_instance_profile.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.base.Strings; + +suite("test_catalog_instance_profile_with_role") { + + if (Strings.isNullOrEmpty(context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName"))) { + return + } + + String hiveGlueQueryTableName = context.config.otherConfigs.get("hiveGlueInstanceProfileQueryTableName") + String hiveGlueExpectCounts = context.config.otherConfigs.get("hiveGlueInstanceProfileExpectCounts") + String icebergFsQueryTableName = context.config.otherConfigs.get("icebergFsInstanceProfileQueryTableName") + String icebergFsExpectCounts = context.config.otherConfigs.get("icebergFsInstanceProfileExpectCounts") + String icebergFsWarehouse = context.config.otherConfigs.get("icebergFsInstanceProfileWarehouse") + String region = context.config.otherConfigs.get("awsInstanceProfileRegion") + //query method + def createCatalogAndQuery = { catalogProps, catalogName, queryTableName, expectCounts -> + sql """drop catalog if exists ${catalogName}""" + sql """ + ${catalogProps} + """ + def result = sql """ + select count(1) from ${catalogName}.${queryTableName}; + """ + println("result: ${result}") + def countValue = result[0][0] + assertTrue(countValue == expectCounts.toInteger()) + sql """drop catalog if exists ${catalogName}""" + } + def assertCatalogAndQueryException = { catalogProps, catalogName, errMsg -> + sql """drop catalog if exists ${catalogName}""" + sql """ + ${catalogProps} + """ + try { + sql """ + switch ${catalogName}; + """ + sql """ + show databases; + """ + throw new Exception("Expected exception was not thrown") + }catch (Exception e){ + assertTrue(e.getMessage().contains(errMsg)) + } + } + String hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog", hiveGlueQueryTableName, hiveGlueExpectCounts) + hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.credentials_provider_type"="INSTANCE_PROFILE", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(hiveGlueCatalogProps, "hive_glue_catalog", hiveGlueQueryTableName, hiveGlueExpectCounts) + hiveGlueCatalogProps = """ + create catalog hive_glue_catalog properties( + "type"="hms", + "hive.metastore.type"="glue", + "glue.credentials_provider_type"="CONTAINER", + "glue.region"="${region}", + "glue.endpoint" = "https://glue.${region}.amazonaws.com" + ); + """ + assertCatalogAndQueryException(hiveGlueCatalogProps,"hive_glue_catalog", "The environment variable AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + String icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog", icebergFsQueryTableName, icebergFsExpectCounts) + icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.credentials_provider_type"="INSTANCE_PROFILE", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + createCatalogAndQuery(icebergFsCatalogProps, "iceberg_fs_catalog", icebergFsQueryTableName, icebergFsExpectCounts) + icebergFsCatalogProps = """ + create catalog iceberg_fs_catalog properties( + "type"="iceberg", + "warehouse"="${icebergFsWarehouse}", + "iceberg.catalog.type"="hadoop", + "s3.credentials_provider_type"="CONTAINER", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com" + ); + """ + assertCatalogAndQueryException(icebergFsCatalogProps,"iceberg_fs_catalog", "No AWS Credentials provided by ContainerCredentialsProvider") + +}