diff --git a/docs/src/main/asciidoc/s3.adoc b/docs/src/main/asciidoc/s3.adoc index 8a33e2660..de6e00e0d 100644 --- a/docs/src/main/asciidoc/s3.adoc +++ b/docs/src/main/asciidoc/s3.adoc @@ -48,6 +48,22 @@ public class S3ClientSample { } } ---- +=== Using S3TransferManager + +[CAUTION] +==== +S3 Transfer Manager provided by AWS is in developer preview phase and should not be used in production. +==== + +AWS launched https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/[a high level file transfer utility], called Transfer Manager. The starter automatically configures and registers an `software.amazon.awssdk.transfer.s3.S3TransferManager` bean if it finds the following is added to the project: + +[source,xml] +---- + + software.amazon.awssdk + s3-transfer-manager + +---- === Using Cross-Region S3 client @@ -143,10 +159,11 @@ try (OutputStream outputStream = s3Resource.getOutputStream()) { // e.getPath contains a file location in temporary folder } ---- +If you are using the `S3TransferManager`, the default implementation will switch to `io.awspring.cloud.s3.TransferManagerS3OutputStream`. This OutputStream also uses a temporary file to write it on disk before uploading it to S3, but it may be faster as it uses a multi-part upload under the hood. If `DiskBufferingS3OutputStream` behavior does not fit your needs, you can implement custom `S3OutputStream` and provide a bean of type `io.awspring.cloud.s3.S3OutputStreamProvider` that is responsible for creating stream from `S3Resource`. -Possible alternative implementations can use multi-part upload (for example with https://github.com/CI-CMG/aws-s3-outputstream[aws-s3-outputstream library)] or https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/transfer/s3/S3TransferManager.html[S3TransferManager]. +Possible alternative implementations can use multi-part upload (for example with https://github.com/CI-CMG/aws-s3-outputstream[aws-s3-outputstream library]). === Using S3Template diff --git a/pom.xml b/pom.xml index d46bab9bd..2cd917009 100644 --- a/pom.xml +++ b/pom.xml @@ -34,6 +34,7 @@ 3.1.0 1.17.1 2.22.0 + 2.17.172 diff --git a/spring-cloud-aws-autoconfigure/pom.xml b/spring-cloud-aws-autoconfigure/pom.xml index 97a474a82..251a82470 100644 --- a/spring-cloud-aws-autoconfigure/pom.xml +++ b/spring-cloud-aws-autoconfigure/pom.xml @@ -59,6 +59,11 @@ spring-cloud-aws-s3 true + + software.amazon.awssdk + s3-transfer-manager + true + io.awspring.cloud spring-cloud-aws-s3-cross-region-client diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/core/AwsClientBuilderConfigurer.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/core/AwsClientBuilderConfigurer.java index bbd18415e..e5d9fd6a7 100644 --- a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/core/AwsClientBuilderConfigurer.java +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/core/AwsClientBuilderConfigurer.java @@ -16,6 +16,7 @@ package io.awspring.cloud.autoconfigure.core; import io.awspring.cloud.autoconfigure.AwsClientProperties; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; import io.awspring.cloud.core.SpringCloudClientConfiguration; import java.util.Optional; import org.springframework.util.StringUtils; @@ -23,6 +24,7 @@ import software.amazon.awssdk.awscore.client.builder.AwsClientBuilder; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.transfer.s3.S3ClientConfiguration; /** * Provides a convenience method to apply common configuration to any {@link AwsClientBuilder}. @@ -43,12 +45,24 @@ public class AwsClientBuilderConfigurer { } public AwsClientBuilder configure(AwsClientBuilder builder, AwsClientProperties clientProperties) { - Region region = StringUtils.hasLength(clientProperties.getRegion()) ? Region.of(clientProperties.getRegion()) - : regionProvider.getRegion(); - builder.credentialsProvider(credentialsProvider).region(region) + builder.credentialsProvider(credentialsProvider).region(resolveRegion(clientProperties)) .overrideConfiguration(SpringCloudClientConfiguration.clientOverrideConfiguration()); Optional.ofNullable(awsProperties.getEndpoint()).ifPresent(builder::endpointOverride); Optional.ofNullable(clientProperties.getEndpoint()).ifPresent(builder::endpointOverride); return builder; } + + public S3ClientConfiguration.Builder configure(S3ClientConfiguration.Builder builder, + S3Properties clientProperties) { + builder.credentialsProvider(credentialsProvider).region(resolveRegion(clientProperties)); + // TODO: how to set client override configuration? + Optional.ofNullable(awsProperties.getEndpoint()).ifPresent(builder::endpointOverride); + Optional.ofNullable(clientProperties.getEndpoint()).ifPresent(builder::endpointOverride); + return builder; + } + + private Region resolveRegion(AwsClientProperties clientProperties) { + return StringUtils.hasLength(clientProperties.getRegion()) ? Region.of(clientProperties.getRegion()) + : regionProvider.getRegion(); + } } diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfiguration.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfiguration.java index 7184541f0..52d228053 100644 --- a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfiguration.java +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfiguration.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.awspring.cloud.autoconfigure.core.AwsClientBuilderConfigurer; import io.awspring.cloud.autoconfigure.core.AwsProperties; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; import io.awspring.cloud.s3.DiskBufferingS3OutputStreamProvider; import io.awspring.cloud.s3.Jackson2JsonS3ObjectConverter; import io.awspring.cloud.s3.PropertiesS3ObjectContentTypeResolver; @@ -71,14 +72,6 @@ S3ClientBuilder s3ClientBuilder(AwsClientBuilderConfigurer awsClientBuilderConfi return builder; } - @Bean - @ConditionalOnMissingBean - S3OutputStreamProvider s3OutputStreamProvider(S3Client s3Client, - Optional contentTypeResolver) { - return new DiskBufferingS3OutputStreamProvider(s3Client, - contentTypeResolver.orElseGet(PropertiesS3ObjectContentTypeResolver::new)); - } - @Bean @ConditionalOnMissingBean(S3Operations.class) @ConditionalOnBean(S3ObjectConverter.class) @@ -135,4 +128,12 @@ S3ObjectConverter s3ObjectConverter(Optional objectMapper) { } } + @Bean + @ConditionalOnMissingBean + S3OutputStreamProvider diskBufferingS3StreamProvider(S3Client s3Client, + Optional contentTypeResolver) { + return new DiskBufferingS3OutputStreamProvider(s3Client, + contentTypeResolver.orElseGet(PropertiesS3ObjectContentTypeResolver::new)); + } + } diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfiguration.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfiguration.java new file mode 100644 index 000000000..490baacef --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfiguration.java @@ -0,0 +1,103 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.s3; + +import io.awspring.cloud.autoconfigure.core.AwsClientBuilderConfigurer; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; +import io.awspring.cloud.autoconfigure.s3.properties.S3TransferManagerProperties; +import io.awspring.cloud.s3.PropertiesS3ObjectContentTypeResolver; +import io.awspring.cloud.s3.S3ObjectContentTypeResolver; +import io.awspring.cloud.s3.S3OutputStreamProvider; +import io.awspring.cloud.s3.TransferManagerS3OutputStreamProvider; +import java.util.Optional; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.transfer.s3.S3ClientConfiguration; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.S3TransferManagerOverrideConfiguration; +import software.amazon.awssdk.transfer.s3.UploadDirectoryOverrideConfiguration; + +/** + * {@link EnableAutoConfiguration} for {@link S3TransferManager} + * + * @author Anton Perez + * @since 3.0 + */ +@ConditionalOnClass({ S3TransferManager.class, S3OutputStreamProvider.class }) +@EnableConfigurationProperties({ S3Properties.class }) +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(name = "spring.cloud.aws.s3.enabled", havingValue = "true", matchIfMissing = true) +@AutoConfigureBefore(S3AutoConfiguration.class) +public class S3TransferManagerAutoConfiguration { + + private final S3Properties properties; + + public S3TransferManagerAutoConfiguration(S3Properties properties) { + this.properties = properties; + } + + @Bean + @ConditionalOnMissingBean + S3TransferManager s3TransferManager(AwsClientBuilderConfigurer awsClientBuilderConfigurer) { + return S3TransferManager.builder().s3ClientConfiguration(s3ClientConfiguration(awsClientBuilderConfigurer)) + .transferConfiguration(extractUploadDirectoryOverrideConfiguration()).build(); + } + + @Bean + @ConditionalOnMissingBean + S3OutputStreamProvider transferManagerS3StreamProvider(S3TransferManager s3TransferManager, + Optional contentTypeResolver) { + return new TransferManagerS3OutputStreamProvider(s3TransferManager, + contentTypeResolver.orElseGet(PropertiesS3ObjectContentTypeResolver::new)); + } + + private S3ClientConfiguration s3ClientConfiguration(AwsClientBuilderConfigurer awsClientBuilderConfigurer) { + S3ClientConfiguration.Builder builder = awsClientBuilderConfigurer.configure(S3ClientConfiguration.builder(), + properties); + if (properties.getTransferManager() != null) { + S3TransferManagerProperties transferManagerProperties = properties.getTransferManager(); + PropertyMapper propertyMapper = PropertyMapper.get(); + propertyMapper.from(transferManagerProperties::getMaxConcurrency).whenNonNull().to(builder::maxConcurrency); + propertyMapper.from(transferManagerProperties::getTargetThroughputInGbps).whenNonNull() + .to(builder::targetThroughputInGbps); + propertyMapper.from(transferManagerProperties::getMinimumPartSizeInBytes).whenNonNull() + .to(builder::minimumPartSizeInBytes); + } + return builder.build(); + } + + private S3TransferManagerOverrideConfiguration extractUploadDirectoryOverrideConfiguration() { + UploadDirectoryOverrideConfiguration.Builder config = UploadDirectoryOverrideConfiguration.builder(); + if (properties.getTransferManager() != null && properties.getTransferManager().getUploadDirectory() != null) { + S3TransferManagerProperties.S3UploadDirectoryProperties s3UploadDirectoryProperties = properties + .getTransferManager().getUploadDirectory(); + PropertyMapper propertyMapper = PropertyMapper.get(); + propertyMapper.from(s3UploadDirectoryProperties::getMaxDepth).whenNonNull().to(config::maxDepth); + propertyMapper.from(s3UploadDirectoryProperties::getRecursive).whenNonNull().to(config::recursive); + propertyMapper.from(s3UploadDirectoryProperties::getFollowSymbolicLinks).whenNonNull() + .to(config::followSymbolicLinks); + } + return S3TransferManagerOverrideConfiguration.builder().uploadDirectoryConfiguration(config.build()).build(); + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3Properties.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java similarity index 86% rename from spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3Properties.java rename to spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java index 2d66bc3b8..6beedaf52 100644 --- a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/S3Properties.java +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3Properties.java @@ -13,11 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.awspring.cloud.autoconfigure.s3; +package io.awspring.cloud.autoconfigure.s3.properties; import io.awspring.cloud.autoconfigure.AwsClientProperties; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; import org.springframework.lang.Nullable; +import software.amazon.awssdk.transfer.s3.S3TransferManager; /** * Properties related to AWS S3. @@ -75,6 +77,13 @@ public class S3Properties extends AwsClientProperties { @Nullable private Boolean useArnRegionEnabled; + /** + * Configuration properties for {@link S3TransferManager} integration. + */ + @Nullable + @NestedConfigurationProperty + private S3TransferManagerProperties transferManager; + @Nullable public Boolean getAccelerateModeEnabled() { return accelerateModeEnabled; @@ -129,4 +138,12 @@ public void setUseArnRegionEnabled(@Nullable Boolean useArnRegionEnabled) { this.useArnRegionEnabled = useArnRegionEnabled; } + @Nullable + public S3TransferManagerProperties getTransferManager() { + return transferManager; + } + + public void setTransferManager(@Nullable S3TransferManagerProperties transferManager) { + this.transferManager = transferManager; + } } diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3TransferManagerProperties.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3TransferManagerProperties.java new file mode 100644 index 000000000..aa35a4057 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/S3TransferManagerProperties.java @@ -0,0 +1,112 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.s3.properties; + +import org.springframework.lang.Nullable; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + +/** + * Properties related to AWS S3 {@link S3TransferManager}. + * + * @author Anton Perez + * @since 3.0 + */ +public class S3TransferManagerProperties { + @Nullable + private Double targetThroughputInGbps; + + @Nullable + private Integer maxConcurrency; + + @Nullable + private Long minimumPartSizeInBytes; + + @Nullable + private S3UploadDirectoryProperties uploadDirectory; + + @Nullable + public Double getTargetThroughputInGbps() { + return targetThroughputInGbps; + } + + public void setTargetThroughputInGbps(@Nullable Double targetThroughputInGbps) { + this.targetThroughputInGbps = targetThroughputInGbps; + } + + @Nullable + public S3UploadDirectoryProperties getUploadDirectory() { + return uploadDirectory; + } + + public void setUploadDirectory(@Nullable S3UploadDirectoryProperties uploadDirectory) { + this.uploadDirectory = uploadDirectory; + } + + @Nullable + public Integer getMaxConcurrency() { + return maxConcurrency; + } + + public void setMaxConcurrency(@Nullable Integer maxConcurrency) { + this.maxConcurrency = maxConcurrency; + } + + @Nullable + public Long getMinimumPartSizeInBytes() { + return minimumPartSizeInBytes; + } + + public void setMinimumPartSizeInBytes(@Nullable Long minimumPartSizeInBytes) { + this.minimumPartSizeInBytes = minimumPartSizeInBytes; + } + + public static class S3UploadDirectoryProperties { + @Nullable + private Boolean recursive; + @Nullable + private Boolean followSymbolicLinks; + @Nullable + private Integer maxDepth; + + @Nullable + public Boolean getRecursive() { + return recursive; + } + + public void setRecursive(@Nullable Boolean recursive) { + this.recursive = recursive; + } + + @Nullable + public Boolean getFollowSymbolicLinks() { + return followSymbolicLinks; + } + + public void setFollowSymbolicLinks(@Nullable Boolean followSymbolicLinks) { + this.followSymbolicLinks = followSymbolicLinks; + } + + @Nullable + public Integer getMaxDepth() { + return maxDepth; + } + + public void setMaxDepth(@Nullable Integer maxDepth) { + this.maxDepth = maxDepth; + } + } + +} diff --git a/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/package-info.java b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/package-info.java new file mode 100644 index 000000000..ddd324ac4 --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/main/java/io/awspring/cloud/autoconfigure/s3/properties/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Properties for auto-configuration + */ +@org.springframework.lang.NonNullApi +@org.springframework.lang.NonNullFields +package io.awspring.cloud.autoconfigure.s3.properties; diff --git a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories index 6beb840ba..df80d6757 100644 --- a/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories +++ b/spring-cloud-aws-autoconfigure/src/main/resources/META-INF/spring.factories @@ -3,6 +3,7 @@ io.awspring.cloud.autoconfigure.core.AwsAutoConfiguration,\ io.awspring.cloud.autoconfigure.core.CredentialsProviderAutoConfiguration,\ io.awspring.cloud.autoconfigure.core.RegionProviderAutoConfiguration,\ io.awspring.cloud.autoconfigure.ses.SesAutoConfiguration,\ +io.awspring.cloud.autoconfigure.s3.S3TransferManagerAutoConfiguration,\ io.awspring.cloud.autoconfigure.s3.S3AutoConfiguration,\ io.awspring.cloud.autoconfigure.sns.SnsAutoConfiguration diff --git a/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfigurationTests.java b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfigurationTests.java index 361d56ab9..a8243d5b8 100644 --- a/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfigurationTests.java +++ b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3AutoConfigurationTests.java @@ -23,6 +23,7 @@ import io.awspring.cloud.autoconfigure.core.AwsAutoConfiguration; import io.awspring.cloud.autoconfigure.core.CredentialsProviderAutoConfiguration; import io.awspring.cloud.autoconfigure.core.RegionProviderAutoConfiguration; +import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; import io.awspring.cloud.s3.DiskBufferingS3OutputStreamProvider; import io.awspring.cloud.s3.ObjectMetadata; import io.awspring.cloud.s3.S3ObjectConverter; @@ -114,8 +115,9 @@ void createsStandardClientWhenCrossRegionModuleIsNotInClasspath() { @Nested class OutputStreamProviderTests { + @Test - void byDefaultCreatesDiskBufferingS3OutputStreamProvider() { + void createsDiskBufferingS3OutputStreamProviderWhenBeanDoesNotExistYet() { contextRunner.run(context -> assertThat(context).hasSingleBean(DiskBufferingS3OutputStreamProvider.class)); } diff --git a/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfigurationTests.java b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfigurationTests.java new file mode 100644 index 000000000..9adf3511d --- /dev/null +++ b/spring-cloud-aws-autoconfigure/src/test/java/io/awspring/cloud/autoconfigure/s3/S3TransferManagerAutoConfigurationTests.java @@ -0,0 +1,131 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.autoconfigure.s3; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.awspring.cloud.autoconfigure.ConfiguredAwsClient; +import io.awspring.cloud.autoconfigure.core.AwsAutoConfiguration; +import io.awspring.cloud.autoconfigure.core.CredentialsProviderAutoConfiguration; +import io.awspring.cloud.autoconfigure.core.RegionProviderAutoConfiguration; +import io.awspring.cloud.s3.TransferManagerS3OutputStreamProvider; +import java.net.URI; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.test.context.FilteredClassLoader; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.lang.NonNull; +import org.springframework.test.util.ReflectionTestUtils; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + +/** + * Tests for {@link S3TransferManagerAutoConfigurationTests}. + * + * @author Maciej Walkowiak + * @author Anton Perez + */ +class S3TransferManagerAutoConfigurationTests { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withPropertyValues("spring.cloud.aws.region.static:eu-west-1") + .withConfiguration(AutoConfigurations.of(AwsAutoConfiguration.class, RegionProviderAutoConfiguration.class, + CredentialsProviderAutoConfiguration.class, S3TransferManagerAutoConfiguration.class)); + + @Nested + class TransferManagerTests { + @Test + void createsS3TransferManagerBeanWhenInClassPath() { + contextRunner.run(context -> assertThat(context).hasSingleBean(S3TransferManager.class)); + } + + @Test + void usesExistingS3TransferManagerBeanWhenExists() { + S3TransferManager customDefinedS3TransferManager = Mockito.mock(S3TransferManager.class); + contextRunner.withBean("s3transferManager", S3TransferManager.class, () -> customDefinedS3TransferManager) + .run(context -> assertThat(context.getBean(S3TransferManager.class)) + .isEqualTo(customDefinedS3TransferManager)); + } + + @Test + void doesNotCreateS3TransferManagerBeanWhenNotInClassPath() { + contextRunner.withClassLoader(new FilteredClassLoader(S3TransferManager.class)).run(context -> { + assertThat(context).doesNotHaveBean(S3TransferManager.class); + }); + } + } + + @Nested + class TransferManagerEndpointConfigurationTests { + @Test + void withCustomEndpoint() { + contextRunner.withPropertyValues("spring.cloud.aws.s3.endpoint:http://localhost:8090").run(context -> { + S3TransferManager transferManager = context.getBean(S3TransferManager.class); + ConfiguredAwsClient client = new ConfiguredAwsClient(resolveS3Client(transferManager)); + assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090")); + assertThat(client.isEndpointOverridden()).isTrue(); + }); + } + + @Test + void withCustomGlobalEndpoint() { + contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090").run(context -> { + S3TransferManager transferManager = context.getBean(S3TransferManager.class); + ConfiguredAwsClient client = new ConfiguredAwsClient(resolveS3Client(transferManager)); + assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:8090")); + assertThat(client.isEndpointOverridden()).isTrue(); + }); + } + + @Test + void withCustomGlobalEndpointAndS3Endpoint() { + contextRunner.withPropertyValues("spring.cloud.aws.endpoint:http://localhost:8090", + "spring.cloud.aws.s3.endpoint:http://localhost:9999").run(context -> { + S3TransferManager transferManager = context.getBean(S3TransferManager.class); + ConfiguredAwsClient client = new ConfiguredAwsClient(resolveS3Client(transferManager)); + assertThat(client.getEndpoint()).isEqualTo(URI.create("http://localhost:9999")); + assertThat(client.isEndpointOverridden()).isTrue(); + }); + } + } + + @Nested + class OutputStreamProviderTests { + + @Test + void whenS3TransferManagerInClassPathCreatesTransferManagerSS3OutputStreamProvider() { + contextRunner + .run(context -> assertThat(context).hasSingleBean(TransferManagerS3OutputStreamProvider.class)); + } + + @Test + void customS3OutputStreamProviderCanBeConfigured() { + contextRunner + .withUserConfiguration(S3AutoConfigurationTests.CustomS3OutputStreamProviderConfiguration.class) + .run(context -> assertThat(context) + .hasSingleBean(S3AutoConfigurationTests.CustomS3OutputStreamProvider.class)); + } + } + + @NonNull + private static S3AsyncClient resolveS3Client(S3TransferManager builder) { + return (S3AsyncClient) ReflectionTestUtils.getField(ReflectionTestUtils.getField(builder, "s3CrtAsyncClient"), + "s3AsyncClient"); + } + +} diff --git a/spring-cloud-aws-dependencies/pom.xml b/spring-cloud-aws-dependencies/pom.xml index 15665ba83..713cd367b 100644 --- a/spring-cloud-aws-dependencies/pom.xml +++ b/spring-cloud-aws-dependencies/pom.xml @@ -25,6 +25,7 @@ 3.24.0 2.22.0 + 2.17.172 @@ -32,7 +33,7 @@ software.amazon.awssdk bom - 2.17.172 + ${awssdk.version} pom import @@ -45,6 +46,13 @@ import + + software.amazon.awssdk + s3-transfer-manager + ${awssdk.version}-PREVIEW + true + + io.awspring.cloud spring-cloud-aws-core diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/pom.xml b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/pom.xml index f37a06532..ae4be02c1 100644 --- a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/pom.xml +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/pom.xml @@ -26,6 +26,11 @@ software.amazon.awssdk s3 + + software.amazon.awssdk + s3-transfer-manager + true + com.fasterxml.jackson.core jackson-databind diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/AbstractTempFileS3OutputStream.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/AbstractTempFileS3OutputStream.java new file mode 100644 index 000000000..ba267d4d1 --- /dev/null +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/AbstractTempFileS3OutputStream.java @@ -0,0 +1,156 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; + +/** + * {@link AbstractTempFileS3OutputStream} defines the common behaviour for implementations that use a temporary file for + * the {@link S3OutputStream}. + * + * @author Maciej Walkowiak + * @author Anton Perez + * @since 3.0 + */ +abstract class AbstractTempFileS3OutputStream extends S3OutputStream { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected final Location location; + + /** + * The local file that will be uploaded when the stream is closed. + */ + protected final File file; + + /** + * The outputstream to a local file where the file will be buffered until closed. + */ + protected OutputStream localOutputStream; + + /** + * The MD5 hash of the file. + */ + @Nullable + protected MessageDigest hash; + + @Nullable + protected final ObjectMetadata objectMetadata; + + /** + * Flag to indicate this stream has been closed, to ensure close is only done once. + */ + protected boolean closed; + + @Nullable + protected final S3ObjectContentTypeResolver contentTypeResolver; + + AbstractTempFileS3OutputStream(Location location, @Nullable ObjectMetadata objectMetadata) throws IOException { + this(location, objectMetadata, null); + } + + AbstractTempFileS3OutputStream(Location location, @Nullable ObjectMetadata objectMetadata, + @Nullable S3ObjectContentTypeResolver contentTypeResolver) throws IOException { + Assert.notNull(location, "Location must not be null."); + this.location = location; + this.objectMetadata = objectMetadata; + this.contentTypeResolver = contentTypeResolver; + this.file = File.createTempFile("TempFileS3OutputStream", UUID.randomUUID().toString()); + try { + hash = MessageDigest.getInstance("MD5"); + localOutputStream = new BufferedOutputStream(new DigestOutputStream(new FileOutputStream(file), hash)); + } + catch (NoSuchAlgorithmException e) { + getLogger().warn("Algorithm not available for MD5 hash.", e); + hash = null; + localOutputStream = new BufferedOutputStream(new FileOutputStream(file)); + } + closed = false; + } + + @Override + public void write(int b) throws IOException { + localOutputStream.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + localOutputStream.write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + localOutputStream.write(b, off, len); + } + + @Override + public void flush() throws IOException { + localOutputStream.flush(); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + localOutputStream.close(); + closed = true; + try { + PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(location.getBucket()) + .key(location.getObject()).contentLength(file.length()); + if (objectMetadata != null) { + objectMetadata.apply(builder); + } + if (hash != null) { + String contentMD5 = new String(Base64.getEncoder().encode(hash.digest())); + builder = builder.contentMD5(contentMD5); + } + if (contentTypeResolver != null && (objectMetadata == null || objectMetadata.getContentType() == null)) { + String contentType = contentTypeResolver.resolveContentType(location.getObject()); + if (contentType != null) { + builder.contentType(contentType); + } + } + this.upload(builder.build()); + file.delete(); + } + catch (Exception se) { + getLogger().error( + String.format("Failed to upload %s. Temporary file @%s", location.getObject(), file.getPath())); + throw new UploadFailedException(file.getPath(), se); + } + } + + protected abstract void upload(PutObjectRequest putObjectRequest); + + protected Logger getLogger() { + return this.logger; + } +} diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/DiskBufferingS3OutputStream.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/DiskBufferingS3OutputStream.java index 56f8fb101..7733708f2 100644 --- a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/DiskBufferingS3OutputStream.java +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/DiskBufferingS3OutputStream.java @@ -15,20 +15,8 @@ */ package io.awspring.cloud.s3; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.security.DigestOutputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.Base64; -import java.util.UUID; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.lang.Nullable; -import org.springframework.util.Assert; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -43,41 +31,10 @@ * @author Maciej Walkowiak * @since 3.0 */ -class DiskBufferingS3OutputStream extends S3OutputStream { - - private static final Logger LOG = LoggerFactory.getLogger(DiskBufferingS3OutputStream.class); - - private final Location location; - - /** - * The local file that will be uploaded when the stream is closed. - */ - private final File file; +class DiskBufferingS3OutputStream extends AbstractTempFileS3OutputStream { private final S3Client s3Client; - /** - * The outputstream to a local file where the file will be buffered until closed. - */ - private OutputStream localOutputStream; - - /** - * The MD5 hash of the file. - */ - @Nullable - private MessageDigest hash; - - @Nullable - private final ObjectMetadata objectMetadata; - - /** - * Flag to indicate this stream has been closed, to ensure close is only done once. - */ - private boolean closed; - - @Nullable - private final S3ObjectContentTypeResolver contentTypeResolver; - DiskBufferingS3OutputStream(Location location, S3Client s3Client, @Nullable ObjectMetadata objectMetadata) throws IOException { this(location, s3Client, objectMetadata, null); @@ -85,74 +42,12 @@ class DiskBufferingS3OutputStream extends S3OutputStream { DiskBufferingS3OutputStream(Location location, S3Client client, @Nullable ObjectMetadata objectMetadata, @Nullable S3ObjectContentTypeResolver contentTypeResolver) throws IOException { - Assert.notNull(location, "Location must not be null."); - this.location = location; + super(location, objectMetadata, contentTypeResolver); this.s3Client = client; - this.objectMetadata = objectMetadata; - this.contentTypeResolver = contentTypeResolver; - this.file = File.createTempFile("DiskBufferingS3OutputStream", UUID.randomUUID().toString()); - try { - hash = MessageDigest.getInstance("MD5"); - localOutputStream = new BufferedOutputStream(new DigestOutputStream(new FileOutputStream(file), hash)); - } - catch (NoSuchAlgorithmException e) { - LOG.warn("Algorithm not available for MD5 hash.", e); - hash = null; - localOutputStream = new BufferedOutputStream(new FileOutputStream(file)); - } - closed = false; - } - - @Override - public void write(int b) throws IOException { - localOutputStream.write(b); } @Override - public void write(byte[] b) throws IOException { - localOutputStream.write(b, 0, b.length); + protected void upload(PutObjectRequest putObjectRequest) { + s3Client.putObject(putObjectRequest, RequestBody.fromFile(file)); } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - localOutputStream.write(b, off, len); - } - - @Override - public void flush() throws IOException { - localOutputStream.flush(); - } - - @Override - public void close() throws IOException { - if (closed) { - return; - } - localOutputStream.close(); - closed = true; - try { - PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(location.getBucket()) - .key(location.getObject()).contentLength(file.length()); - if (objectMetadata != null) { - objectMetadata.apply(builder); - } - if (hash != null) { - String contentMD5 = new String(Base64.getEncoder().encode(hash.digest())); - builder = builder.contentMD5(contentMD5); - } - if (contentTypeResolver != null && (objectMetadata == null || objectMetadata.getContentType() == null)) { - String contentType = contentTypeResolver.resolveContentType(location.getObject()); - if (contentType != null) { - builder.contentType(contentType); - } - } - s3Client.putObject(builder.build(), RequestBody.fromFile(file)); - file.delete(); - } - catch (Exception se) { - LOG.error(String.format("Failed to upload %s. Temporary file @%s", location.getObject(), file.getPath())); - throw new UploadFailedException(file.getPath(), se); - } - } - } diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Resource.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Resource.java index 3a7619319..25f0e17e7 100644 --- a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Resource.java +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Resource.java @@ -25,7 +25,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; - import org.springframework.core.io.AbstractResource; import org.springframework.core.io.WritableResource; import org.springframework.lang.Nullable; diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStream.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStream.java new file mode 100644 index 000000000..f2af52e00 --- /dev/null +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStream.java @@ -0,0 +1,55 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3; + +import java.io.IOException; +import org.springframework.lang.Nullable; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; + +/** + * {@link S3OutputStream} implementation, that uses the TransferManager from AWS (in preview). + * + * Transfer manager has been announced here + * https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/ + * + * @author Anton Perez + * @since 3.0 + */ +class TransferManagerS3OutputStream extends AbstractTempFileS3OutputStream { + + private final S3TransferManager s3TransferManager; + + TransferManagerS3OutputStream(Location location, S3TransferManager s3TransferManager, + @Nullable ObjectMetadata objectMetadata) throws IOException { + this(location, s3TransferManager, objectMetadata, null); + } + + TransferManagerS3OutputStream(Location location, S3TransferManager s3TransferManager, + @Nullable ObjectMetadata objectMetadata, @Nullable S3ObjectContentTypeResolver contentTypeResolver) + throws IOException { + super(location, objectMetadata, contentTypeResolver); + this.s3TransferManager = s3TransferManager; + } + + @Override + protected void upload(PutObjectRequest putObjectRequest) { + s3TransferManager + .uploadFile(UploadFileRequest.builder().putObjectRequest(putObjectRequest).source(file).build()) + .completionFuture().join(); + } +} diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamProvider.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamProvider.java new file mode 100644 index 000000000..eef8d7fdc --- /dev/null +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamProvider.java @@ -0,0 +1,46 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3; + +import java.io.IOException; +import org.springframework.lang.Nullable; +import software.amazon.awssdk.transfer.s3.S3TransferManager; + +/** + * Creates {@link TransferManagerS3OutputStream}. + * + * @author Anton Perez + * @since 3.0 + */ +public class TransferManagerS3OutputStreamProvider implements S3OutputStreamProvider { + + private final S3TransferManager s3TransferManager; + @Nullable + private final S3ObjectContentTypeResolver contentTypeResolver; + + public TransferManagerS3OutputStreamProvider(S3TransferManager s3TransferManager, + @Nullable S3ObjectContentTypeResolver contentTypeResolver) { + this.s3TransferManager = s3TransferManager; + this.contentTypeResolver = contentTypeResolver; + } + + @Override + public S3OutputStream create(String bucket, String key, @Nullable ObjectMetadata metadata) throws IOException { + return new TransferManagerS3OutputStream(new Location(bucket, key, null), s3TransferManager, metadata, + contentTypeResolver); + } + +} diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/S3ResourceIntegrationTests.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/S3ResourceIntegrationTests.java index 354d0c1e2..acbf4832a 100644 --- a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/S3ResourceIntegrationTests.java +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/S3ResourceIntegrationTests.java @@ -23,13 +23,19 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.containers.localstack.LocalStackContainer.Service; import org.testcontainers.junit.jupiter.Container; @@ -44,11 +50,13 @@ import software.amazon.awssdk.services.s3.model.NoSuchKeyException; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.transfer.s3.S3TransferManager; /** * Integration tests for {@link S3Resource}. * * @author Maciej Walkowiak + * @author Anton Perez */ @Testcontainers class S3ResourceIntegrationTests { @@ -58,6 +66,14 @@ class S3ResourceIntegrationTests { DockerImageName.parse("localstack/localstack:0.14.2")).withServices(Service.S3).withReuse(true); private static S3Client client; + private static S3TransferManager s3TransferManager; + + // Required for the @TestAvailableOutputStreamProviders annotation + private static Stream availableS3OutputStreamProviders() { + return Stream.of(new DiskBufferingS3OutputStreamProvider(client, new PropertiesS3ObjectContentTypeResolver()), + new TransferManagerS3OutputStreamProvider(s3TransferManager, + new PropertiesS3ObjectContentTypeResolver())); + } @BeforeAll static void beforeAll() { @@ -68,77 +84,82 @@ static void beforeAll() { .create(localstackCredentials.getAWSAccessKeyId(), localstackCredentials.getAWSSecretKey())); client = S3Client.builder().region(Region.of(localstack.getRegion())).credentialsProvider(credentialsProvider) .endpointOverride(localstack.getEndpointOverride(Service.S3)).build(); + s3TransferManager = S3TransferManager.builder() + .s3ClientConfiguration( + b -> b.region(Region.of(localstack.getRegion())).credentialsProvider(credentialsProvider) + .endpointOverride(localstack.getEndpointOverride(Service.S3)).build()) + .build(); client.createBucket(request -> request.bucket("first-bucket")); } - @Test - void readsFileFromS3() throws IOException { + @TestAvailableOutputStreamProviders + void readsFileFromS3(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { client.putObject(PutObjectRequest.builder().bucket("first-bucket").key("test-file.txt").build(), RequestBody.fromString("test-file-content")); - S3Resource resource = s3Resource("s3://first-bucket/test-file.txt"); + S3Resource resource = s3Resource("s3://first-bucket/test-file.txt", s3OutputStreamProvider); String content = retrieveContent(resource); assertThat(content).isEqualTo("test-file-content"); } - @Test - void existsReturnsTrueWhenKeyExists() { + @TestAvailableOutputStreamProviders + void existsReturnsTrueWhenKeyExists(S3OutputStreamProvider s3OutputStreamProvider) { client.putObject(PutObjectRequest.builder().bucket("first-bucket").key("test-file.txt").build(), RequestBody.fromString("test-file-content")); - S3Resource resource = s3Resource("s3://first-bucket/test-file.txt"); + S3Resource resource = s3Resource("s3://first-bucket/test-file.txt", s3OutputStreamProvider); assertThat(resource.exists()).isTrue(); } - @Test - void existsReturnsFalseWhenObjectDoesNotExist() { - S3Resource resource = s3Resource("s3://first-bucket/non-existing-file.txt"); + @TestAvailableOutputStreamProviders + void existsReturnsFalseWhenObjectDoesNotExist(S3OutputStreamProvider s3OutputStreamProvider) { + S3Resource resource = s3Resource("s3://first-bucket/non-existing-file.txt", s3OutputStreamProvider); assertThat(resource.exists()).isFalse(); } - @Test - void objectHasContentLength() throws IOException { + @TestAvailableOutputStreamProviders + void objectHasContentLength(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { String contents = "test-file-content"; client.putObject(PutObjectRequest.builder().bucket("first-bucket").key("test-file.txt").build(), RequestBody.fromString(contents)); - S3Resource resource = s3Resource("s3://first-bucket/test-file.txt"); + S3Resource resource = s3Resource("s3://first-bucket/test-file.txt", s3OutputStreamProvider); assertThat(resource.contentLength()).isEqualTo(contents.length()); } - @Test - void objectHasContentType() { + @TestAvailableOutputStreamProviders + void objectHasContentType(S3OutputStreamProvider s3OutputStreamProvider) { String contents = "{\"foo\":\"bar\"}"; client.putObject(PutObjectRequest.builder().bucket("first-bucket").key("test-file.json") .contentType("application/json").build(), RequestBody.fromString(contents)); - S3Resource resource = s3Resource("s3://first-bucket/test-file.json"); + S3Resource resource = s3Resource("s3://first-bucket/test-file.json", s3OutputStreamProvider); assertThat(resource.contentType()).isEqualTo("application/json"); } - @Test - void contentLengthThrowsWhenResourceDoesNotExist() { - S3Resource resource = s3Resource("s3://first-bucket/non-existing-file.txt"); + @TestAvailableOutputStreamProviders + void contentLengthThrowsWhenResourceDoesNotExist(S3OutputStreamProvider s3OutputStreamProvider) { + S3Resource resource = s3Resource("s3://first-bucket/non-existing-file.txt", s3OutputStreamProvider); assertThatThrownBy(resource::contentLength).isInstanceOf(NoSuchKeyException.class); } - @Test - void returnsResourceUrl() throws IOException { - S3Resource resource = s3Resource("s3://first-bucket/a-file.txt"); + @TestAvailableOutputStreamProviders + void returnsResourceUrl(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { + S3Resource resource = s3Resource("s3://first-bucket/a-file.txt", s3OutputStreamProvider); assertThat(resource.getURL().toString()).isEqualTo("https://first-bucket.s3.amazonaws.com/a-file.txt"); } - @Test - void returnsEncodedResourceUrlAndUri() throws IOException, URISyntaxException { - S3Resource resource = s3Resource("s3://first-bucket/some/[objectName]"); + @TestAvailableOutputStreamProviders + void returnsEncodedResourceUrlAndUri(S3OutputStreamProvider s3OutputStreamProvider) + throws IOException, URISyntaxException { + S3Resource resource = s3Resource("s3://first-bucket/some/[objectName]", s3OutputStreamProvider); assertThat(resource.getURL().toString()) .isEqualTo("https://first-bucket.s3.amazonaws.com/some/%5BobjectName%5D"); - assertThat(resource.getURI()) - .isEqualTo(new URI("https://first-bucket.s3.amazonaws.com/some/%5BobjectName%5D")); + assertThat(resource.getURI()).isEqualTo(new URI("https://first-bucket.s3.amazonaws.com/some/%5BobjectName%5D")); } - @Test - void resourceIsWritableWithDiskBuffering() throws IOException { + @TestAvailableOutputStreamProviders + void resourceIsWritableWithDiskBuffering(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { client.putObject(PutObjectRequest.builder().bucket("first-bucket").key("test-file.txt").build(), RequestBody.fromString("test-file-content")); - S3Resource resource = s3Resource("s3://first-bucket/test-file.txt", s3OutputStreamProvider()); + S3Resource resource = s3Resource("s3://first-bucket/test-file.txt", s3OutputStreamProvider); try (OutputStream outputStream = resource.getOutputStream()) { outputStream.write("overwritten with buffering".getBytes(StandardCharsets.UTF_8)); @@ -146,13 +167,9 @@ void resourceIsWritableWithDiskBuffering() throws IOException { assertThat(retrieveContent(resource)).isEqualTo("overwritten with buffering"); } - private DiskBufferingS3OutputStreamProvider s3OutputStreamProvider() { - return new DiskBufferingS3OutputStreamProvider(client, new PropertiesS3ObjectContentTypeResolver()); - } - - @Test - void objectMetadataCanBeSetOnWriting() throws IOException { - S3Resource resource = s3Resource("s3://first-bucket/new-file.txt", s3OutputStreamProvider()); + @TestAvailableOutputStreamProviders + void objectMetadataCanBeSetOnWriting(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { + S3Resource resource = s3Resource("s3://first-bucket/new-file.txt", s3OutputStreamProvider); ObjectMetadata objectMetadata = ObjectMetadata.builder().storageClass(StorageClass.ONEZONE_IA.name()) .metadata("key", "value").contentLanguage("en").build(); @@ -168,9 +185,9 @@ void objectMetadataCanBeSetOnWriting() throws IOException { assertThat(result.metadata()).containsEntry("key", "value"); } - @Test - void contentTypeCanBeResolved() throws IOException { - S3Resource resource = s3Resource("s3://first-bucket/new-file.txt", s3OutputStreamProvider()); + @TestAvailableOutputStreamProviders + void contentTypeCanBeResolved(S3OutputStreamProvider s3OutputStreamProvider) throws IOException { + S3Resource resource = s3Resource("s3://first-bucket/new-file.txt", s3OutputStreamProvider); try (OutputStream outputStream = resource.getOutputStream()) { outputStream.write("content".getBytes(StandardCharsets.UTF_8)); @@ -180,11 +197,6 @@ void contentTypeCanBeResolved() throws IOException { assertThat(result.contentType()).isEqualTo("text/plain"); } - @NotNull - private S3Resource s3Resource(String location) { - return new S3Resource(location, client, s3OutputStreamProvider()); - } - @NotNull private S3Resource s3Resource(String location, S3OutputStreamProvider s3OutputStreamProvider) { return new S3Resource(location, client, s3OutputStreamProvider); @@ -213,4 +225,10 @@ public String toString() { } } + @Target(ElementType.METHOD) + @Retention(RetentionPolicy.RUNTIME) + @ParameterizedTest + @MethodSource("availableS3OutputStreamProviders") + @interface TestAvailableOutputStreamProviders { + } } diff --git a/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamTests.java b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamTests.java new file mode 100644 index 000000000..1256b058c --- /dev/null +++ b/spring-cloud-aws-s3-parent/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/TransferManagerS3OutputStreamTests.java @@ -0,0 +1,73 @@ +/* + * Copyright 2013-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.awspring.cloud.s3; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.UploadFileRequest; + +/** + * Unit tests for {@link TransferManagerS3OutputStream}. + * + * @author Anton Perez + */ +class TransferManagerS3OutputStreamTests { + + @Test + void setsMd5hash() throws IOException { + S3TransferManager s3TransferManager = mock(S3TransferManager.class, Answers.RETURNS_DEEP_STUBS); + + try (TransferManagerS3OutputStream transferManagerS3OutputStream = new TransferManagerS3OutputStream( + new Location("bucket", "key"), s3TransferManager, null)) { + transferManagerS3OutputStream.write("hello".getBytes(StandardCharsets.UTF_8)); + } + + ArgumentCaptor captor = ArgumentCaptor.forClass(UploadFileRequest.class); + verify(s3TransferManager).uploadFile(captor.capture()); + + assertThat(captor.getValue().putObjectRequest().contentMD5()).isNotNull(); + } + + @Test + void throwsExceptionWhenUploadFails() throws IOException { + S3TransferManager s3TransferManager = mock(S3TransferManager.class); + when(s3TransferManager.uploadFile(any(UploadFileRequest.class))).thenThrow(S3Exception.class); + + try { + try (TransferManagerS3OutputStream transferManagerS3OutputStream = new TransferManagerS3OutputStream( + new Location("bucket", "key"), s3TransferManager, null)) { + transferManagerS3OutputStream.write("hello".getBytes(StandardCharsets.UTF_8)); + } + fail("UploadFailedException should be thrown"); + } + catch (UploadFailedException e) { + assertThat(e.getPath()).isNotNull(); + } + } + +} diff --git a/spring-cloud-aws-samples/spring-cloud-aws-s3-sample/app/pom.xml b/spring-cloud-aws-samples/spring-cloud-aws-s3-sample/app/pom.xml index 64ce9ec4d..d9da197ed 100644 --- a/spring-cloud-aws-samples/spring-cloud-aws-s3-sample/app/pom.xml +++ b/spring-cloud-aws-samples/spring-cloud-aws-s3-sample/app/pom.xml @@ -17,7 +17,6 @@ org.springframework.boot spring-boot-starter-web - io.awspring.cloud spring-cloud-aws-starter-s3