From 867c296e1c9b02214602a9213dc297a1c4932c0a Mon Sep 17 00:00:00 2001 From: Zoe Wang <33073555+zoewangg@users.noreply.github.com> Date: Wed, 24 Mar 2021 14:19:49 -0700 Subject: [PATCH] Update APIs --- services-custom/s3-transfermanager/pom.xml | 11 +- .../custom/s3/transfer/DownloadRequest.java | 41 ++++- .../custom/s3/transfer/TransferRequest.java | 9 +- .../custom/s3/transfer/UploadRequest.java | 2 +- .../internal/DefaultS3TransferManager.java | 15 +- .../transfer/util/S3IntegrationTestBase.java | 141 ++++++++++++++++++ .../services/s3/S3CrtAsyncClientBuilder.java | 3 + .../s3crt/DefaultS3CrtClientBuilder.java | 2 + 8 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java diff --git a/services-custom/s3-transfermanager/pom.xml b/services-custom/s3-transfermanager/pom.xml index ee4b7a4fc57c..3341ad242ec3 100644 --- a/services-custom/s3-transfermanager/pom.xml +++ b/services-custom/s3-transfermanager/pom.xml @@ -66,7 +66,16 @@ annotations ${awsjavasdk.version} - + + software.amazon.awssdk + regions + ${awsjavasdk.version} + + + software.amazon.awssdk + aws-core + ${awsjavasdk.version} + software.amazon.awssdk service-test-utils diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java index ac564c495430..df6a57061874 100644 --- a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/DownloadRequest.java @@ -17,6 +17,8 @@ import java.nio.file.Path; import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.Validate; import software.amazon.awssdk.utils.builder.CopyableBuilder; import software.amazon.awssdk.utils.builder.ToCopyableBuilder; @@ -25,14 +27,17 @@ */ @SdkPublicApi public final class DownloadRequest implements TransferRequest, ToCopyableBuilder { - private final String bucket; - private final String key; private final Path destination; + private final GetObjectRequest getObjectRequest; private DownloadRequest(BuilderImpl builder) { - this.bucket = builder.bucket; - this.key = builder.key; + Validate.isTrue((builder.bucket != null && builder.key != null) ^ builder.getObjectRequest != null, + "Bucket key pair and the getObjectRequest can't both be configured"); this.destination = builder.destination; + this.getObjectRequest = builder.getObjectRequest == null ? GetObjectRequest.builder() + .bucket(builder.bucket) + .key(builder.key) + .build() : builder.getObjectRequest; } /** @@ -49,13 +54,12 @@ public Builder toBuilder() { @Override public String bucket() { - return bucket; + return getObjectRequest.bucket(); } - @Override public String key() { - return key; + return getObjectRequest.key(); } /** @@ -68,7 +72,12 @@ public Path destination() { return destination; } - public interface Builder extends TransferRequest.Builder, CopyableBuilder { + public GetObjectRequest toApiRequest() { + return getObjectRequest; + } + + public interface Builder extends TransferRequest.Builder, CopyableBuilder { /** * The {@link Path} to file that response contents will be written to. The file must not exist or this method @@ -79,6 +88,15 @@ public interface Builder extends TransferRequest.Builder, CopyableBuilder { /** * The bucket name containing the object. * * @return Returns a reference to this object so that method calls can be chained together. */ - Builder bucket(String bucket); + BuilderT bucket(String bucket); /** * The Key of the object to transfer. * * @return Returns a reference to this object so that method calls can be chained together. */ - Builder key(String key); + BuilderT key(String key); + + TypeToBuildT build(); } } diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java index af24e1a79c53..6ef8d1232553 100644 --- a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/UploadRequest.java @@ -64,7 +64,7 @@ public Builder toBuilder() { } - public interface Builder extends TransferRequest.Builder, CopyableBuilder { + public interface Builder extends TransferRequest.Builder, CopyableBuilder { /** * The {@link Path} to file containing data to send to the service. File will be read entirely and may be read diff --git a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java index b892a31f7ef2..0eb077997dcb 100644 --- a/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java +++ b/services-custom/s3-transfermanager/src/main/java/software/amazon/awssdk/custom/s3/transfer/internal/DefaultS3TransferManager.java @@ -15,22 +15,31 @@ package software.amazon.awssdk.custom.s3.transfer.internal; +import java.util.ArrayList; +import java.util.List; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.custom.s3.transfer.S3TransferManager; import software.amazon.awssdk.services.s3.S3CrtAsyncClient; +import software.amazon.awssdk.utils.SdkAutoCloseable; @SdkInternalApi public final class DefaultS3TransferManager implements S3TransferManager { private final S3CrtAsyncClient s3CrtAsyncClient; + private final List closables = new ArrayList<>(); public DefaultS3TransferManager(DefaultBuilder builder) { - //TODO: create a managed S3CrtAsyncClient if it's not provided - this.s3CrtAsyncClient = builder.s3CrtAsyncClient; + if (builder.s3CrtAsyncClient == null) { + s3CrtAsyncClient = S3CrtAsyncClient.builder() + .build(); + closables.add(s3CrtAsyncClient); + } else { + s3CrtAsyncClient = builder.s3CrtAsyncClient; + } } @Override public void close() { - s3CrtAsyncClient.close(); + closables.forEach(SdkAutoCloseable::close); } public static Builder builder() { diff --git a/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java new file mode 100644 index 000000000000..cd49b24486a4 --- /dev/null +++ b/services-custom/s3-transfermanager/src/test/java/software/amazon/awssdk/custom/s3/transfer/util/S3IntegrationTestBase.java @@ -0,0 +1,141 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.custom.s3.transfer.util; + +import org.junit.BeforeClass; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.ObjectVersion; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.testutils.service.AwsTestBase; + +/** + * Base class for S3 integration tests. Loads AWS credentials from a properties + * file and creates an S3 client for callers to use. + */ +public class S3IntegrationTestBase extends AwsTestBase { + + protected static final Region DEFAULT_REGION = Region.US_WEST_2; + /** + * The S3 client for all tests to use. + */ + protected static S3Client s3; + + protected static S3AsyncClient s3Async; + + /** + * Loads the AWS account info for the integration tests and creates an S3 + * client for tests to use. + */ + @BeforeClass + public static void setUp() throws Exception { + s3 = s3ClientBuilder().build(); + s3Async = s3AsyncClientBuilder().build(); + } + + protected static S3ClientBuilder s3ClientBuilder() { + return S3Client.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static S3AsyncClientBuilder s3AsyncClientBuilder() { + return S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN); + } + + protected static void createBucket(String bucketName) { + createBucket(bucketName, 0); + } + + private static void createBucket(String bucketName, int retryCount) { + try { + s3.createBucket( + CreateBucketRequest.builder() + .bucket(bucketName) + .createBucketConfiguration( + CreateBucketConfiguration.builder() + .locationConstraint(BucketLocationConstraint.US_WEST_2) + .build()) + .build()); + } catch (S3Exception e) { + System.err.println("Error attempting to create bucket: " + bucketName); + if (e.awsErrorDetails().errorCode().equals("BucketAlreadyOwnedByYou")) { + System.err.printf("%s bucket already exists, likely leaked by a previous run\n", bucketName); + } else if (e.awsErrorDetails().errorCode().equals("TooManyBuckets")) { + System.err.println("Printing all buckets for debug:"); + s3.listBuckets().buckets().forEach(System.err::println); + if (retryCount < 2) { + System.err.println("Retrying..."); + createBucket(bucketName, retryCount + 1); + } else { + throw e; + } + } else { + throw e; + } + } + } + + protected static void deleteBucketAndAllContents(String bucketName) { + System.out.println("Deleting S3 bucket: " + bucketName); + ListObjectsResponse response = s3.listObjects(ListObjectsRequest.builder().bucket(bucketName).build()); + + while (true) { + if (response.contents() == null) { + break; + } + for (S3Object objectSummary : response.contents()) { + s3.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(objectSummary.key()).build()); + } + + if (response.isTruncated()) { + response = s3.listObjects(ListObjectsRequest.builder().marker(response.nextMarker()).build()); + } else { + break; + } + } + + ListObjectVersionsResponse versionsResponse = s3 + .listObjectVersions(ListObjectVersionsRequest.builder().bucket(bucketName).build()); + if (versionsResponse.versions() != null) { + for (ObjectVersion s : versionsResponse.versions()) { + s3.deleteObject(DeleteObjectRequest.builder() + .bucket(bucketName) + .key(s.key()) + .versionId(s.versionId()) + .build()); + } + } + + s3.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build()); + } + +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java index fbd646dbb332..294f61124f50 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/S3CrtAsyncClientBuilder.java @@ -27,4 +27,7 @@ @SdkPublicApi public interface S3CrtAsyncClientBuilder extends AwsClientBuilder { + S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes); + + S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java index 540f279e2043..a3bb3a8c0b78 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/s3crt/DefaultS3CrtClientBuilder.java @@ -77,11 +77,13 @@ public S3CrtAsyncClientBuilder endpointOverride(URI endpointOverride) { throw new UnsupportedOperationException(); } + @Override public S3CrtAsyncClientBuilder partSizeBytes(long partSizeBytes) { this.partSizeBytes = partSizeBytes; return this; } + @Override public S3CrtAsyncClientBuilder maxThroughputGbps(double maxThroughputGbps) { this.maxThroughputGbps = maxThroughputGbps; return this;