diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index 54a2593f4c6f4..ff6e2148fab37 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -131,6 +131,9 @@ String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2") String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs") String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs") +String s3EKSBucket = System.getenv("amazon_s3_bucket_eks") +String s3EKSBasePath = System.getenv("amazon_s3_base_path_eks") + boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean() // If all these variables are missing then we are testing against the internal fixture instead, which has the following @@ -160,13 +163,15 @@ if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3T throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present") } -if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) { +if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath && !s3EKSBucket && !s3EKSBasePath) { s3EC2Bucket = 'ec2_bucket' s3EC2BasePath = 'ec2_base_path' s3ECSBucket = 'ecs_bucket' s3ECSBasePath = 'ecs_base_path' -} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) { - throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present") + s3EKSBucket = 'eks_bucket' + s3EKSBasePath = 'eks_base_path' +} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath || !s3EKSBucket || !s3EKSBasePath) { + throw new IllegalArgumentException("not all options specified to run EC2/ECS/EKS tests are present") } processYamlRestTestResources { @@ -179,7 +184,9 @@ processYamlRestTestResources { 'ec2_base_path': s3EC2BasePath, 'ecs_bucket': s3ECSBucket, 'ecs_base_path': s3ECSBasePath, - 'disable_chunked_encoding': s3DisableChunkedEncoding, + 'eks_bucket': s3EKSBucket, + 'eks_base_path': s3EKSBasePath, + 'disable_chunked_encoding': s3DisableChunkedEncoding ] inputs.properties(expansions) MavenFilteringHack.filter(it, expansions) @@ -198,7 +205,8 @@ yamlRestTest { [ 'repository_s3/30_repository_temporary_credentials/*', 'repository_s3/40_repository_ec2_credentials/*', - 'repository_s3/50_repository_ecs_credentials/*' + 'repository_s3/50_repository_ecs_credentials/*', + 'repository_s3/60_repository_eks_credentials/*' ] ).join(",") } @@ -215,6 +223,7 @@ testClusters.yamlRestTest { testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture') testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-session-token') testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-ec2') + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-eks') normalization { runtimeClasspath { @@ -223,12 +232,21 @@ testClusters.yamlRestTest { } } + keystore 's3.client.integration_test_eks.role_arn', "arn:aws:iam::000000000000:role/test" + keystore 's3.client.integration_test_eks.role_session_name', "s3-test" + keystore 's3.client.integration_test_eks.access_key', "access_key" + keystore 's3.client.integration_test_eks.secret_key', "secret_key" + setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture', '80')}" }, IGNORE_VALUE setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-session-token', '80')}" }, IGNORE_VALUE setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE + setting 's3.client.integration_test_eks.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}" }, IGNORE_VALUE + setting 's3.client.integration_test_eks.region', { "us-east-2" }, IGNORE_VALUE // to redirect InstanceProfileCredentialsProvider to custom auth point systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-ec2', '80')}" }, IGNORE_VALUE + // to redirect AWSSecurityTokenServiceClient to custom auth point + systemProperty "com.amazonaws.sdk.stsEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}/eks_credentials_endpoint" }, IGNORE_VALUE } else { println "Using an external service to test the repository-s3 plugin" } @@ -250,7 +268,8 @@ if (useFixture) { systemProperty 'tests.rest.denylist', [ 'repository_s3/30_repository_temporary_credentials/*', 'repository_s3/40_repository_ec2_credentials/*', - 'repository_s3/50_repository_ecs_credentials/*' + 'repository_s3/50_repository_ecs_credentials/*', + 'repository_s3/60_repository_eks_credentials/*' ].join(",") } check.dependsOn(yamlRestTestMinio) @@ -277,7 +296,8 @@ if (useFixture) { 'repository_s3/10_basic/*', 'repository_s3/20_repository_permanent_credentials/*', 'repository_s3/30_repository_temporary_credentials/*', - 'repository_s3/40_repository_ec2_credentials/*' + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/60_repository_eks_credentials/*' ].join(",") } check.dependsOn(yamlRestTestECS) @@ -289,6 +309,41 @@ if (useFixture) { } } +// EKS +if (useFixture) { + testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-eks') + task yamlRestTestEKS(type: RestIntegTestTask.class) { + description = "Runs tests using the EKS repository." + dependsOn('bundlePlugin') + SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); + SourceSet yamlRestTestSourceSet = sourceSets.getByName(YamlRestTestPlugin.SOURCE_SET_NAME) + setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs()) + setClasspath(yamlRestTestSourceSet.getRuntimeClasspath()) + systemProperty 'tests.rest.denylist', [ + 'repository_s3/10_basic/*', + 'repository_s3/20_repository_permanent_credentials/*', + 'repository_s3/30_repository_temporary_credentials/*', + 'repository_s3/40_repository_ec2_credentials/*', + 'repository_s3/50_repository_ecs_credentials/*' + ].join(",") + } + check.dependsOn(yamlRestTestEKS) + + testClusters.yamlRestTestEKS { + keystore 's3.client.integration_test_eks.role_arn', "arn:aws:iam::000000000000:role/test" + keystore 's3.client.integration_test_eks.role_session_name', "s3-test" + keystore 's3.client.integration_test_eks.access_key', "access_key" + keystore 's3.client.integration_test_eks.secret_key', "secret_key" + + setting 's3.client.integration_test_eks.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}" }, IGNORE_VALUE + setting 's3.client.integration_test_eks.region', { "us-east-2" }, IGNORE_VALUE + plugin tasks.bundlePlugin.archiveFile + + // to redirect AWSSecurityTokenServiceClient to custom auth point + systemProperty "com.amazonaws.sdk.stsEndpointOverride", { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-eks', '80')}/eks_credentials_endpoint" }, IGNORE_VALUE + } +} + // 3rd Party Tests TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) { SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class); diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java index 6919549874445..18bb62944dede 100644 --- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java +++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java @@ -41,6 +41,7 @@ import com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider; import com.amazonaws.auth.STSAssumeRoleWithWebIdentitySessionCredentialsProvider; import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; import com.amazonaws.http.IdleConnectionReaper; import com.amazonaws.http.SystemPropertyTlsKeyManagersProvider; import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; @@ -82,6 +83,8 @@ class S3Service implements Closeable { private static final Logger logger = LogManager.getLogger(S3Service.class); + private static final String STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY = "com.amazonaws.sdk.stsEndpointOverride"; + private volatile Map clientsCache = emptyMap(); /** @@ -280,13 +283,25 @@ static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings c AWSSecurityTokenService securityTokenService = null; final String region = Strings.hasLength(clientSettings.region) ? clientSettings.region : null; + if (region != null || basicCredentials != null) { - securityTokenService = SocketAccess.doPrivileged( - () -> AWSSecurityTokenServiceClientBuilder.standard() - .withCredentials((basicCredentials != null) ? new AWSStaticCredentialsProvider(basicCredentials) : null) - .withRegion(region) - .build() - ); + securityTokenService = SocketAccess.doPrivileged(() -> { + AWSSecurityTokenServiceClientBuilder builder = AWSSecurityTokenServiceClientBuilder.standard(); + + // Use similar approach to override STS endpoint as SDKGlobalConfiguration.EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY + final String stsEndpoint = System.getProperty(STS_ENDPOINT_OVERRIDE_SYSTEM_PROPERTY); + if (region != null && stsEndpoint != null) { + builder = builder.withEndpointConfiguration(new EndpointConfiguration(stsEndpoint, region)); + } else { + builder = builder.withRegion(region); + } + + if (basicCredentials != null) { + builder = builder.withCredentials(new AWSStaticCredentialsProvider(basicCredentials)); + } + + return builder.build(); + }); } if (irsaCredentials.getIdentityTokenFile() == null) { diff --git a/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_eks_credentials.yml b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_eks_credentials.yml new file mode 100644 index 0000000000000..15f2c9612a2ba --- /dev/null +++ b/plugins/repository-s3/src/yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_eks_credentials.yml @@ -0,0 +1,268 @@ +# Integration tests for repository-s3 + +--- +setup: + + # Register repository with eks credentials + - do: + snapshot.create_repository: + repository: repository_eks + body: + type: s3 + settings: + bucket: ${eks_bucket} + client: integration_test_eks + base_path: "${eks_base_path}" + canned_acl: private + storage_class: standard + disable_chunked_encoding: ${disable_chunked_encoding} + +--- +"Snapshot and Restore with repository-s3 using eks credentials": + + # Get repository + - do: + snapshot.get_repository: + repository: repository_eks + + - match: { repository_eks.settings.bucket : ${eks_bucket} } + - match: { repository_eks.settings.client : "integration_test_eks" } + - match: { repository_eks.settings.base_path : "${eks_base_path}" } + - match: { repository_eks.settings.canned_acl : "private" } + - match: { repository_eks.settings.storage_class : "standard" } + - is_false: repository_eks.settings.access_key + - is_false: repository_eks.settings.secret_key + - is_false: repository_eks.settings.session_token + - is_false: repository_eks.settings.role_arn + - is_false: repository_eks.settings.role_session_name + - is_false: repository_eks.settings.identity_token_file + + # Index documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _id: 1 + - snapshot: one + - index: + _index: docs + _id: 2 + - snapshot: one + - index: + _index: docs + _id: 3 + - snapshot: one + + - do: + count: + index: docs + + - match: {count: 3} + + # Create a first snapshot + - do: + snapshot.create: + repository: repository_eks + snapshot: snapshot-one + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-one } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.include_global_state: true } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.status: + repository: repository_eks + snapshot: snapshot-one + + - is_true: snapshots + - match: { snapshots.0.snapshot: snapshot-one } + - match: { snapshots.0.state : SUCCESS } + + # Index more documents + - do: + bulk: + refresh: true + body: + - index: + _index: docs + _id: 4 + - snapshot: two + - index: + _index: docs + _id: 5 + - snapshot: two + - index: + _index: docs + _id: 6 + - snapshot: two + - index: + _index: docs + _id: 7 + - snapshot: two + + - do: + count: + index: docs + + - match: {count: 7} + + # Create a second snapshot + - do: + snapshot.create: + repository: repository_eks + snapshot: snapshot-two + wait_for_completion: true + + - match: { snapshot.snapshot: snapshot-two } + - match: { snapshot.state : SUCCESS } + - match: { snapshot.shards.failed : 0 } + + - do: + snapshot.get: + repository: repository_eks + snapshot: snapshot-one,snapshot-two + + - is_true: snapshots + - match: { snapshots.0.state : SUCCESS } + - match: { snapshots.1.state : SUCCESS } + + # Delete the index + - do: + indices.delete: + index: docs + + # Restore the second snapshot + - do: + snapshot.restore: + repository: repository_eks + snapshot: snapshot-two + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 7} + + # Delete the index again + - do: + indices.delete: + index: docs + + # Restore the first snapshot + - do: + snapshot.restore: + repository: repository_eks + snapshot: snapshot-one + wait_for_completion: true + + - do: + count: + index: docs + + - match: {count: 3} + + # Remove the snapshots + - do: + snapshot.delete: + repository: repository_eks + snapshot: snapshot-two + + - do: + snapshot.delete: + repository: repository_eks + snapshot: snapshot-one + +--- +"Register a repository with a non existing bucket": + + - do: + catch: /repository_verification_exception/ + snapshot.create_repository: + repository: repository_eks + body: + type: s3 + settings: + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test_eks + +--- +"Register a repository with a non existing client": + + - do: + catch: /illegal_argument_exception/ + snapshot.create_repository: + repository: repository_eks + body: + type: s3 + settings: + bucket: repository_eks + client: unknown + +--- +"Register a read-only repository with a non existing bucket": + +- do: + catch: /repository_verification_exception/ + snapshot.create_repository: + repository: repository_eks + body: + type: s3 + settings: + readonly: true + bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE + client: integration_test_eks + +--- +"Register a read-only repository with a non existing client": + +- do: + catch: /illegal_argument_exception/ + snapshot.create_repository: + repository: repository_eks + body: + type: s3 + settings: + readonly: true + bucket: repository_eks + client: unknown + +--- +"Get a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.get: + repository: repository_eks + snapshot: missing + +--- +"Delete a non existing snapshot": + + - do: + catch: /snapshot_missing_exception/ + snapshot.delete: + repository: repository_eks + snapshot: missing + +--- +"Restore a non existing snapshot": + + - do: + catch: /snapshot_restore_exception/ + snapshot.restore: + repository: repository_eks + snapshot: missing + wait_for_completion: true + +--- +teardown: + + # Remove our repository + - do: + snapshot.delete_repository: + repository: repository_eks diff --git a/test/fixtures/s3-fixture/Dockerfile.eks b/test/fixtures/s3-fixture/Dockerfile.eks new file mode 100644 index 0000000000000..d03960472a6a8 --- /dev/null +++ b/test/fixtures/s3-fixture/Dockerfile.eks @@ -0,0 +1,25 @@ +FROM ubuntu:18.04 + +RUN apt-get update -qqy +RUN apt-get install -qqy openjdk-11-jre-headless + +ARG fixtureClass +ARG port +ARG bucket +ARG basePath +ARG accessKey +ARG roleArn +ARG roleSessionName + +ENV S3_FIXTURE_CLASS=${fixtureClass} +ENV S3_FIXTURE_PORT=${port} +ENV S3_FIXTURE_BUCKET=${bucket} +ENV S3_FIXTURE_BASE_PATH=${basePath} +ENV S3_FIXTURE_ACCESS_KEY=${accessKey} +ENV S3_FIXTURE_ROLE_ARN=${roleArn} +ENV S3_FIXTURE_ROLE_SESSION_NAME=${roleSessionName} + +ENTRYPOINT exec java -classpath "/fixture/shared/*" \ + $S3_FIXTURE_CLASS 0.0.0.0 "$S3_FIXTURE_PORT" "$S3_FIXTURE_BUCKET" "$S3_FIXTURE_BASE_PATH" "$S3_FIXTURE_ACCESS_KEY" "$S3_FIXTURE_ROLE_ARN" "$S3_FIXTURE_ROLE_SESSION_NAME" + +EXPOSE $port diff --git a/test/fixtures/s3-fixture/docker-compose.yml b/test/fixtures/s3-fixture/docker-compose.yml index 22d101f41c318..d2b44f13c9530 100644 --- a/test/fixtures/s3-fixture/docker-compose.yml +++ b/test/fixtures/s3-fixture/docker-compose.yml @@ -92,3 +92,20 @@ services: - ./testfixtures_shared/shared:/fixture/shared ports: - "80" + + s3-fixture-with-eks: + build: + context: . + args: + fixtureClass: fixture.s3.S3HttpFixtureWithEKS + port: 80 + bucket: "eks_bucket" + basePath: "eks_base_path" + accessKey: "eks_access_key" + roleArn: "eks_role_arn" + roleSessionName: "eks_role_session_name" + dockerfile: Dockerfile.eks + volumes: + - ./testfixtures_shared/shared:/fixture/shared + ports: + - "80" diff --git a/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java new file mode 100644 index 0000000000000..b26c82a3cb7d4 --- /dev/null +++ b/test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixtureWithEKS.java @@ -0,0 +1,103 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package fixture.s3; + +import com.sun.net.httpserver.HttpHandler; +import org.opensearch.rest.RestStatus; + +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Objects; + +public class S3HttpFixtureWithEKS extends S3HttpFixture { + + private S3HttpFixtureWithEKS(final String[] args) throws Exception { + super(args); + } + + @Override + protected HttpHandler createHandler(final String[] args) { + final String accessKey = Objects.requireNonNull(args[4]); + final String eksRoleArn = Objects.requireNonNull(args[5]); + final HttpHandler delegate = super.createHandler(args); + + return exchange -> { + // https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html + if ("POST".equals(exchange.getRequestMethod()) && exchange.getRequestURI().getPath().startsWith("/eks_credentials_endpoint")) { + final byte[] response = buildCredentialResponse(eksRoleArn, accessKey).getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/xml"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + exchange.close(); + return; + } + + delegate.handle(exchange); + }; + } + + protected String buildCredentialResponse(final String roleArn, final String accessKey) { + // See please: https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + return "\n" + + " \n" + + " amzn1.account.AF6RHO7KZU5XRVQJGXK6HB56KR2A\n" + + " client.5498841531868486423.1548@apps.example.com\n" + + " \n" + + " " + roleArn + "\n" + + " AROACLKWSDQRAOEXAMPLE:s3\n" + + " \n" + + " \n" + + " AQoDYXdzEE0a8ANXXXXXXXXNO1ewxE5TijQyp+IEXAMPLE\n" + + " wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY\n" + + " " + LocalDateTime.now().plusMonths(1).atZone(ZoneId.of("UTC")).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + "\n" + + " " + accessKey + "\n" + + " \n" + + " SourceIdentityValue\n" + + " www.amazon.com\n" + + " \n" + + " \n" + + " ad4156e9-bce1-11e2-82e6-6b6efEXAMPLE\n" + + " \n" + + ""; + } + + public static void main(final String[] args) throws Exception { + if (args == null || args.length < 6) { + throw new IllegalArgumentException("S3HttpFixtureWithEKS expects 6 arguments " + + "[address, port, bucket, base path, role arn, role session name]"); + } + final S3HttpFixtureWithEKS fixture = new S3HttpFixtureWithEKS(args); + fixture.start(); + } +}