diff --git a/aws-runtime/aws-http/api/aws-http.api b/aws-runtime/aws-http/api/aws-http.api index fea5bdc046c..332992d53e3 100644 --- a/aws-runtime/aws-http/api/aws-http.api +++ b/aws-runtime/aws-http/api/aws-http.api @@ -171,6 +171,9 @@ public final class aws/sdk/kotlin/runtime/http/interceptors/IgnoreCompositeFlexi public final class aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric : java/lang/Enum, aws/smithy/kotlin/runtime/businessmetrics/BusinessMetric { public static final field DDB_MAPPER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static final field S3_EXPRESS_BUCKET Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_DOWNLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; + public static final field S3_TRANSFER_UPLOAD_DIRECTORY Laws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetric; public static fun getEntries ()Lkotlin/enums/EnumEntries; public fun getIdentifier ()Ljava/lang/String; public fun toString ()Ljava/lang/String; diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt index 239686ee3df..bcee46a0020 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/interceptors/businessmetrics/AwsBusinessMetricsUtils.kt @@ -62,6 +62,9 @@ internal fun formatMetrics(metrics: MutableSet, logger: Logger): public enum class AwsBusinessMetric(public override val identifier: String) : BusinessMetric { S3_EXPRESS_BUCKET("J"), DDB_MAPPER("d"), + S3_TRANSFER("G"), + S3_TRANSFER_UPLOAD_DIRECTORY("9"), + S3_TRANSFER_DOWNLOAD_DIRECTORY("+"), ; @InternalApi diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bc5a6c3c571..93fa5b883f2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -92,6 +92,8 @@ smithy-kotlin-telemetry-provider-micrometer = { module = "aws.smithy.kotlin:tele smithy-kotlin-telemetry-provider-otel = { module = "aws.smithy.kotlin:telemetry-provider-otel", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-test-suite = { module = "aws.smithy.kotlin:test-suite", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-testing = { module = "aws.smithy.kotlin:testing", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-test-jvm = { module = "aws.smithy.kotlin:http-test-jvm", version.ref = "smithy-kotlin-runtime-version" } +smithy-kotlin-testing-jvm = { module = "aws.smithy.kotlin:testing-jvm", version.ref = "smithy-kotlin-runtime-version" } smithy-kotlin-codegen = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen", version.ref = "smithy-kotlin-codegen-version" } smithy-kotlin-codegen-testutils = { module = "software.amazon.smithy.kotlin:smithy-kotlin-codegen-testutils", version.ref = "smithy-kotlin-codegen-version" } diff --git a/hll/build.gradle.kts b/hll/build.gradle.kts index e6c5661b02a..a2e924f4cba 100644 --- a/hll/build.gradle.kts +++ b/hll/build.gradle.kts @@ -45,7 +45,7 @@ val hllPreviewVersion = if (sdkVersion.contains("-SNAPSHOT")) { // e.g. 1.3.29-b subprojects { group = "aws.sdk.kotlin" - version = hllPreviewVersion + version = if (name == "s3-transfer-manager") sdkVersion else hllPreviewVersion // TODO Use configurePublishing when migrating to Sonatype Publisher API / JReleaser configurePublishing("aws-sdk-kotlin") } diff --git a/hll/s3-transfer-manager/api/s3-transfer-manager.api b/hll/s3-transfer-manager/api/s3-transfer-manager.api new file mode 100644 index 00000000000..fe4b3332c5c --- /dev/null +++ b/hll/s3-transfer-manager/api/s3-transfer-manager.api @@ -0,0 +1,303 @@ +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/S3Client;JJLaws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;Ljava/util/List;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getInterceptors ()Ljava/util/List; + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J + public final fun uploadFile (Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; + public final fun uploadFile (Lkotlin/jvm/functions/Function1;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Builder { + public fun ()V + public final fun getClient ()Laws/sdk/kotlin/services/s3/S3Client; + public final fun getInterceptors ()Ljava/util/List; + public final fun getMultipartDownloadType ()Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType; + public final fun getMultipartUploadThresholdBytes ()J + public final fun getPartSizeBytes ()J + public final fun setClient (Laws/sdk/kotlin/services/s3/S3Client;)V + public final fun setInterceptors (Ljava/util/List;)V + public final fun setMultipartDownloadType (Laws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType;)V + public final fun setMultipartUploadThresholdBytes (J)V + public final fun setPartSizeBytes (J)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/S3TransferManager; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/TransferContext { + public fun ()V + public fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)V + public synthetic fun (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()Ljava/lang/Object; + public final fun component2 ()Ljava/lang/Object; + public final fun component3 ()Ljava/lang/Long; + public final fun component4 ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun component5 ()Ljava/lang/Long; + public final fun component6 ()Ljava/lang/Long; + public final fun component7 ()Ljava/lang/String; + public final fun component8 ()Ljava/lang/Long; + public final fun copy (Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static synthetic fun copy$default (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Long;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/String;Ljava/lang/Long;ILjava/lang/Object;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun equals (Ljava/lang/Object;)Z + public final fun getCurrentBytes ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getCurrentFile ()Ljava/lang/String; + public final fun getRequest ()Ljava/lang/Object; + public final fun getResponse ()Ljava/lang/Object; + public final fun getTransferableBytes ()Ljava/lang/Long; + public final fun getTransferableFiles ()Ljava/lang/Long; + public final fun getTransferredBytes ()Ljava/lang/Long; + public final fun getTransferredFiles ()Ljava/lang/Long; + public fun hashCode ()I + public final fun setCurrentBytes (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public final fun setCurrentFile (Ljava/lang/String;)V + public final fun setRequest (Ljava/lang/Object;)V + public final fun setResponse (Ljava/lang/Object;)V + public final fun setTransferableBytes (Ljava/lang/Long;)V + public final fun setTransferableFiles (Ljava/lang/Long;)V + public final fun setTransferredBytes (Ljava/lang/Long;)V + public final fun setTransferredFiles (Ljava/lang/Long;)V + public fun toString ()Ljava/lang/String; +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor { + public fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor$DefaultImpls { + public static fun modifyAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun modifyBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)Laws/sdk/kotlin/hll/s3transfermanager/TransferContext; + public static fun readAfterBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readAfterTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeBytesTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeFileTransferred (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeTransferCompleted (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V + public static fun readBeforeTransferInitiated (Laws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor;Laws/sdk/kotlin/hll/s3transfermanager/TransferContext;)V +} + +public abstract interface class aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Part : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Part; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/Range : aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType { + public static final field INSTANCE Laws/sdk/kotlin/hll/s3transfermanager/model/Range; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion; + public synthetic fun (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;Laws/smithy/kotlin/runtime/content/ByteStream;Ljava/lang/String;Ljava/lang/Boolean;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/smithy/kotlin/runtime/time/Instant;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;Laws/sdk/kotlin/services/s3/model/ObjectLockMode;Laws/smithy/kotlin/runtime/time/Instant;Laws/sdk/kotlin/services/s3/model/RequestPayer;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Laws/sdk/kotlin/services/s3/model/StorageClass;Ljava/lang/String;Ljava/lang/String;Lkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Builder { + public fun ()V + public final fun build ()Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; + public final fun getAcl ()Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl; + public final fun getBody ()Laws/smithy/kotlin/runtime/content/ByteStream; + public final fun getBucket ()Ljava/lang/String; + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getCacheControl ()Ljava/lang/String; + public final fun getChecksumAlgorithm ()Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getContentDisposition ()Ljava/lang/String; + public final fun getContentEncoding ()Ljava/lang/String; + public final fun getContentLanguage ()Ljava/lang/String; + public final fun getContentType ()Ljava/lang/String; + public final fun getExpectedBucketOwner ()Ljava/lang/String; + public final fun getExpires ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getGrantFullControl ()Ljava/lang/String; + public final fun getGrantRead ()Ljava/lang/String; + public final fun getGrantReadAcp ()Ljava/lang/String; + public final fun getGrantWriteAcp ()Ljava/lang/String; + public final fun getIfMatch ()Ljava/lang/String; + public final fun getIfNoneMatch ()Ljava/lang/String; + public final fun getKey ()Ljava/lang/String; + public final fun getMetadata ()Ljava/util/Map; + public final fun getObjectLockLegalHoldStatus ()Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus; + public final fun getObjectLockMode ()Laws/sdk/kotlin/services/s3/model/ObjectLockMode; + public final fun getObjectLockRetainUntilDate ()Laws/smithy/kotlin/runtime/time/Instant; + public final fun getRequestPayer ()Laws/sdk/kotlin/services/s3/model/RequestPayer; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKey ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getStorageClass ()Laws/sdk/kotlin/services/s3/model/StorageClass; + public final fun getTagging ()Ljava/lang/String; + public final fun getWebsiteRedirectLocation ()Ljava/lang/String; + public final fun setAcl (Laws/sdk/kotlin/services/s3/model/ObjectCannedAcl;)V + public final fun setBody (Laws/smithy/kotlin/runtime/content/ByteStream;)V + public final fun setBucket (Ljava/lang/String;)V + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setCacheControl (Ljava/lang/String;)V + public final fun setChecksumAlgorithm (Laws/sdk/kotlin/services/s3/model/ChecksumAlgorithm;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setContentDisposition (Ljava/lang/String;)V + public final fun setContentEncoding (Ljava/lang/String;)V + public final fun setContentLanguage (Ljava/lang/String;)V + public final fun setContentType (Ljava/lang/String;)V + public final fun setExpectedBucketOwner (Ljava/lang/String;)V + public final fun setExpires (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setGrantFullControl (Ljava/lang/String;)V + public final fun setGrantRead (Ljava/lang/String;)V + public final fun setGrantReadAcp (Ljava/lang/String;)V + public final fun setGrantWriteAcp (Ljava/lang/String;)V + public final fun setIfMatch (Ljava/lang/String;)V + public final fun setIfNoneMatch (Ljava/lang/String;)V + public final fun setKey (Ljava/lang/String;)V + public final fun setMetadata (Ljava/util/Map;)V + public final fun setObjectLockLegalHoldStatus (Laws/sdk/kotlin/services/s3/model/ObjectLockLegalHoldStatus;)V + public final fun setObjectLockMode (Laws/sdk/kotlin/services/s3/model/ObjectLockMode;)V + public final fun setObjectLockRetainUntilDate (Laws/smithy/kotlin/runtime/time/Instant;)V + public final fun setRequestPayer (Laws/sdk/kotlin/services/s3/model/RequestPayer;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKey (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setStorageClass (Laws/sdk/kotlin/services/s3/model/StorageClass;)V + public final fun setTagging (Ljava/lang/String;)V + public final fun setWebsiteRedirectLocation (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse { + public static final field Companion Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion; + public fun (Ljava/lang/Boolean;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ChecksumType;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/RequestCharged;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;Ljava/lang/String;)V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Builder { + public fun ()V + public final fun getBucketKeyEnabled ()Ljava/lang/Boolean; + public final fun getChecksumCrc32 ()Ljava/lang/String; + public final fun getChecksumCrc32C ()Ljava/lang/String; + public final fun getChecksumCrc64Nvme ()Ljava/lang/String; + public final fun getChecksumSha1 ()Ljava/lang/String; + public final fun getChecksumSha256 ()Ljava/lang/String; + public final fun getChecksumType ()Laws/sdk/kotlin/services/s3/model/ChecksumType; + public final fun getETag ()Ljava/lang/String; + public final fun getExpiration ()Ljava/lang/String; + public final fun getRequestCharged ()Laws/sdk/kotlin/services/s3/model/RequestCharged; + public final fun getServerSideEncryption ()Laws/sdk/kotlin/services/s3/model/ServerSideEncryption; + public final fun getSseCustomerAlgorithm ()Ljava/lang/String; + public final fun getSseCustomerKeyMd5 ()Ljava/lang/String; + public final fun getSsekmsEncryptionContext ()Ljava/lang/String; + public final fun getSsekmsKeyId ()Ljava/lang/String; + public final fun getVersionId ()Ljava/lang/String; + public final fun setBucketKeyEnabled (Ljava/lang/Boolean;)V + public final fun setChecksumCrc32 (Ljava/lang/String;)V + public final fun setChecksumCrc32C (Ljava/lang/String;)V + public final fun setChecksumCrc64Nvme (Ljava/lang/String;)V + public final fun setChecksumSha1 (Ljava/lang/String;)V + public final fun setChecksumSha256 (Ljava/lang/String;)V + public final fun setChecksumType (Laws/sdk/kotlin/services/s3/model/ChecksumType;)V + public final fun setETag (Ljava/lang/String;)V + public final fun setExpiration (Ljava/lang/String;)V + public final fun setRequestCharged (Laws/sdk/kotlin/services/s3/model/RequestCharged;)V + public final fun setServerSideEncryption (Laws/sdk/kotlin/services/s3/model/ServerSideEncryption;)V + public final fun setSseCustomerAlgorithm (Ljava/lang/String;)V + public final fun setSseCustomerKeyMd5 (Ljava/lang/String;)V + public final fun setSsekmsEncryptionContext (Ljava/lang/String;)V + public final fun setSsekmsKeyId (Ljava/lang/String;)V + public final fun setVersionId (Ljava/lang/String;)V +} + +public final class aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse$Companion { + public final fun invoke (Lkotlin/jvm/functions/Function1;)Laws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse; +} + diff --git a/hll/s3-transfer-manager/build.gradle.kts b/hll/s3-transfer-manager/build.gradle.kts new file mode 100644 index 00000000000..bb636ac3db6 --- /dev/null +++ b/hll/s3-transfer-manager/build.gradle.kts @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.sourceSets + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +description = "S3 Transfer Manager for the AWS SDK for Kotlin" +extra["displayName"] = "AWS :: SDK :: Kotlin :: HLL :: S3 Transfer Manager" +extra["moduleName"] = "aws.sdk.kotlin.hll.s3transfermanager" + +kotlin { + sourceSets { + commonMain { + dependencies { + implementation(project(":aws-runtime:aws-http")) + implementation(project(":services:s3")) + } + } + jvmTest { + dependencies { + implementation(libs.smithy.kotlin.test.jvm) + implementation(libs.smithy.kotlin.testing.jvm) + } + } + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt new file mode 100644 index 00000000000..edc27f86449 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt @@ -0,0 +1,269 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.hll.s3transfermanager.model.MultipartDownloadType +import aws.sdk.kotlin.hll.s3transfermanager.model.Part +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.hll.s3transfermanager.utils.S3TransferManagerException +import aws.sdk.kotlin.hll.s3transfermanager.utils.buildCompleteMultipartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.buildUploadPartRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.ceilDiv +import aws.sdk.kotlin.hll.s3transfermanager.utils.getNextPart +import aws.sdk.kotlin.hll.s3transfermanager.utils.resolvePartSize +import aws.sdk.kotlin.hll.s3transfermanager.utils.toCreateMultiPartUploadRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.toPutObjectRequest +import aws.sdk.kotlin.hll.s3transfermanager.utils.toUploadFileResponse +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.abortMultipartUpload +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.sdk.kotlin.services.s3.model.UploadPartResponse +import aws.sdk.kotlin.services.s3.withConfig +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.telemetry.logging.logger +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + +/** + * High level utility for managing transfers to Amazon S3. + */ +public class S3TransferManager private constructor( + public val client: S3Client, + public val partSizeBytes: Long, + public val multipartUploadThresholdBytes: Long, + public val multipartDownloadType: MultipartDownloadType, + public val interceptors: MutableList, +) { + internal var context: TransferContext = TransferContext() + + public companion object { + public operator fun invoke(block: Builder.() -> Unit): S3TransferManager = + Builder().apply(block).build() + } + + public class Builder { + public var client: S3Client? = null + public var partSizeBytes: Long = 8_000_000 + public var multipartUploadThresholdBytes: Long = 16_000_000L + public var multipartDownloadType: MultipartDownloadType = Part + public var interceptors: MutableList = mutableListOf() + + internal fun build(): S3TransferManager = + S3TransferManager( + client = client?.withConfig { interceptors += S3TransferManagerBusinessMetricInterceptor } ?: error("client must be set"), + partSizeBytes = partSizeBytes, + multipartUploadThresholdBytes = multipartUploadThresholdBytes, + multipartDownloadType = multipartDownloadType, + interceptors = interceptors, + ) + } + + /** + * Executes a sequence of operations around a hook. + * + * The execution flow is as follows: + * 1. Runs all interceptors scheduled to execute **before** the hook. + * 2. Executes the main hook logic. + * 3. Runs all interceptors scheduled to execute **after** the hook. + */ + private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) { + when (hook) { + is TransferInitiated -> { + interceptors.forEach { it.readBeforeTransferInitiated(context) } + interceptors.forEach { context = it.modifyBeforeTransferInitiated(context) } + block.invoke() + interceptors.forEach { it.readAfterTransferInitiated(context) } + interceptors.forEach { context = it.modifyAfterTransferInitiated(context) } + } + is BytesTransferred -> { + interceptors.forEach { it.readBeforeBytesTransferred(context) } + interceptors.forEach { context = it.modifyBeforeBytesTransferred(context) } + block.invoke() + interceptors.forEach { it.readAfterBytesTransferred(context) } + interceptors.forEach { context = it.modifyAfterBytesTransferred(context) } + } + is FileTransferred -> { + interceptors.forEach { it.readBeforeFileTransferred(context) } + interceptors.forEach { context = it.modifyBeforeFileTransferred(context) } + block.invoke() + interceptors.forEach { it.readAfterFileTransferred(context) } + interceptors.forEach { context = it.modifyAfterFileTransferred(context) } + } + is TransferCompleted -> { + interceptors.forEach { it.readBeforeTransferCompleted(context) } + interceptors.forEach { context = it.modifyBeforeTransferCompleted(context) } + block.invoke() + interceptors.forEach { it.readAfterTransferCompleted(context) } + interceptors.forEach { context = it.modifyAfterTransferCompleted(context) } + } + else -> error("TransferHook not implemented: ${hook::class.simpleName}") + } + } + + /** + * Uploads a byte stream to Amazon S3, automatically using multipart uploads + * for large objects as needed. + * + * This function handles the complexity of splitting the data into parts, + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], + * a standard single-part upload is performed automatically. + * + * If the specified [partSizeBytes] for multipart uploads is too small to allow + * all parts to fit within S3's limit of 10,000 parts, the part size will be + * automatically increased so that exactly 10,000 parts are uploaded. + */ + public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred = coroutineScope { + val contentLength = uploadFileRequest.body?.contentLength ?: throw S3TransferManagerException("UploadFileRequest.body.contentLength must be set") + val multiPartUpload = contentLength >= multipartUploadThresholdBytes + val uploadedParts = mutableListOf() + lateinit var mpuUploadId: String + + val logger = coroutineContext.logger() + + /* + Handles transfer initiated hook + */ + suspend fun transferInitiated(multiPartUpload: Boolean) { + context.transferredBytes = 0L + context.transferableBytes = contentLength + context.request = if (multiPartUpload) { + uploadFileRequest.toCreateMultiPartUploadRequest() + } else { + uploadFileRequest.toPutObjectRequest() + } + operationHook(TransferInitiated) { + if (multiPartUpload) { + context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest) + mpuUploadId = (context.response as CreateMultipartUploadResponse).uploadId ?: throw S3TransferManagerException("Missing upload id in create multipart upload response") + } + } + } + + /* + Handles bytes transferred hook + */ + suspend fun transferBytes(multiPartUpload: Boolean) { + if (multiPartUpload) { + try { + val partSize = resolvePartSize(contentLength, this@S3TransferManager, logger) + val numberOfParts = ceilDiv(contentLength, partSize) + val partSource = when (uploadFileRequest.body) { + is ByteStream.Buffer -> uploadFileRequest.body.bytes() + is ByteStream.ChannelStream -> uploadFileRequest.body.readFrom() + is ByteStream.SourceStream -> uploadFileRequest.body.readFrom() + else -> throw S3TransferManagerException("Unhandled body type: ${uploadFileRequest.body?.let { it::class.simpleName } ?: "null"}") + } + val partBuffer = SdkBuffer() + var currentPartNumber = 1L + + while (context.transferredBytes!! < context.transferableBytes!!) { + partBuffer.getNextPart(partSource, partSize, this@S3TransferManager) + if (currentPartNumber != numberOfParts) { + if (partBuffer.size != partSize) { + throw S3TransferManagerException("Part #$currentPartNumber size mismatch detected. Expected $partSize, actual: ${partBuffer.size}") + } + } + + context.request = + buildUploadPartRequest( + uploadFileRequest, + partBuffer, + currentPartNumber, + mpuUploadId, + ) + + operationHook(BytesTransferred) { + context.response = client.uploadPart(context.request as UploadPartRequest) + context.transferredBytes = context.transferredBytes!! + partSize + } + + uploadedParts += CompletedPart { + partNumber = currentPartNumber.toInt() + eTag = (context.response as UploadPartResponse).eTag + } + currentPartNumber += 1 + } + + if (uploadedParts.size != numberOfParts.toInt()) { + throw S3TransferManagerException("The number of uploaded parts does not match the expected count. Expected $numberOfParts, actual: ${uploadedParts.size}") + } + } catch (uploadPartThrowable: Throwable) { + try { + client.abortMultipartUpload { + bucket = uploadFileRequest.bucket + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + uploadId = mpuUploadId + } + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). One or more parts could not be uploaded", uploadPartThrowable) + } catch (abortThrowable: Throwable) { + throw S3TransferManagerException("Multipart upload failed (ID: $mpuUploadId). Unable to abort multipart upload.", abortThrowable) + } + } + } else { + operationHook(BytesTransferred) { + context.response = client.putObject(context.request as PutObjectRequest) + context.transferredBytes = context.transferableBytes + } + } + } + + /* + Handles transfer completed hook + */ + suspend fun transferComplete(multiPartUpload: Boolean) { + if (multiPartUpload) { + context.request = buildCompleteMultipartUploadRequest(uploadFileRequest, mpuUploadId, uploadedParts) + } + operationHook(TransferCompleted) { + if (multiPartUpload) { + try { + context.response = client.completeMultipartUpload(context.request as CompleteMultipartUploadRequest) + } catch (t: Throwable) { + throw S3TransferManagerException("Unable to complete multipart upload with ID: $mpuUploadId", t) + } + } + } + } + + async { + transferInitiated(multiPartUpload) + transferBytes(multiPartUpload) + transferComplete(multiPartUpload) + + when (context.response) { + is PutObjectResponse -> (context.response as PutObjectResponse).toUploadFileResponse() + is CompleteMultipartUploadResponse -> (context.response as CompleteMultipartUploadResponse).toUploadFileResponse() + else -> throw S3TransferManagerException("Unexpected response: ${context.response?.let { it::class.simpleName } ?: "null"}") + } + } + } + + /** + * Uploads a byte stream to Amazon S3, automatically using multipart uploads + * for large objects as needed. + * + * This function handles the complexity of splitting the data into parts, + * uploading each part, and completing the multipart upload. For object smaller than [multipartUploadThresholdBytes], + * a standard single-part upload is performed automatically. + * + * If the specified [partSizeBytes] for multipart uploads is too small to allow + * all parts to fit within S3's limit of 10,000 parts, the part size will be + * automatically increased so that exactly 10,000 parts are uploaded. + */ + public suspend inline fun uploadFile(crossinline block: UploadFileRequest.Builder.() -> Unit): Deferred = uploadFile(UploadFileRequest.Builder().apply(block).build()) +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt new file mode 100644 index 00000000000..16331343cbb --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricInterceptor.kt @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric +import aws.smithy.kotlin.runtime.client.RequestInterceptorContext +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor + +/** + * An interceptor that emits the S3 Transfer Manager business metric + */ +internal object S3TransferManagerBusinessMetricInterceptor : HttpInterceptor { + override suspend fun modifyBeforeSerialization(context: RequestInterceptorContext): Any { + context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER) + return context.request + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt new file mode 100644 index 00000000000..fca50b465fa --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.smithy.kotlin.runtime.content.ByteStream + +// TODO: KDocs + +public data class TransferContext( + // Req/Resp + var request: Any? = null, + var response: Any? = null, + + // Byte transfers + var transferableBytes: Long? = null, + var currentBytes: ByteStream? = null, + var transferredBytes: Long? = null, + + // File transfers + var transferableFiles: Long? = null, + var currentFile: String? = null, + var transferredFiles: Long? = null, +) + +public interface TransferInterceptor { + // Transfer initialization hooks + public fun readBeforeTransferInitiated(context: TransferContext) {} + public fun modifyBeforeTransferInitiated(context: TransferContext): TransferContext = context + public fun readAfterTransferInitiated(context: TransferContext) {} + public fun modifyAfterTransferInitiated(context: TransferContext): TransferContext = context + + // Byte transferring hooks + public fun readBeforeBytesTransferred(context: TransferContext) {} + public fun modifyBeforeBytesTransferred(context: TransferContext): TransferContext = context + public fun readAfterBytesTransferred(context: TransferContext) {} + public fun modifyAfterBytesTransferred(context: TransferContext): TransferContext = context + + // File transfer hooks + public fun readBeforeFileTransferred(context: TransferContext) {} + public fun modifyBeforeFileTransferred(context: TransferContext): TransferContext = context + public fun readAfterFileTransferred(context: TransferContext) {} + public fun modifyAfterFileTransferred(context: TransferContext): TransferContext = context + + // Transfer completion hooks + public fun readBeforeTransferCompleted(context: TransferContext) {} + public fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext = context + public fun readAfterTransferCompleted(context: TransferContext) {} + public fun modifyAfterTransferCompleted(context: TransferContext): TransferContext = context +} + +internal interface TransferHook +internal object TransferInitiated : TransferHook +internal object BytesTransferred : TransferHook +internal object FileTransferred : TransferHook +internal object TransferCompleted : TransferHook diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt new file mode 100644 index 00000000000..4b872c292da --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultipartDownloadType.kt @@ -0,0 +1,12 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +// TODO: KDocs + +public sealed interface MultipartDownloadType +public object Range : MultipartDownloadType +public object Part : MultipartDownloadType diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt new file mode 100644 index 00000000000..041f781dab8 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileRequest.kt @@ -0,0 +1,145 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm +import aws.sdk.kotlin.services.s3.model.ObjectCannedAcl +import aws.sdk.kotlin.services.s3.model.ObjectLockLegalHoldStatus +import aws.sdk.kotlin.services.s3.model.ObjectLockMode +import aws.sdk.kotlin.services.s3.model.RequestPayer +import aws.sdk.kotlin.services.s3.model.ServerSideEncryption +import aws.sdk.kotlin.services.s3.model.StorageClass +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.time.Instant + +public class UploadFileRequest private constructor( + public val acl: ObjectCannedAcl?, + public val body: ByteStream?, + public val bucket: String?, + public val bucketKeyEnabled: Boolean?, + public val cacheControl: String?, + public val checksumAlgorithm: ChecksumAlgorithm?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val contentDisposition: String?, + public val contentEncoding: String?, + public val contentLanguage: String?, + public val contentType: String?, + public val expectedBucketOwner: String?, + public val expires: Instant?, + public val grantFullControl: String?, + public val grantRead: String?, + public val grantReadAcp: String?, + public val grantWriteAcp: String?, + public val ifMatch: String?, + public val ifNoneMatch: String?, + public val key: String?, + public val metadata: Map?, + public val objectLockLegalHoldStatus: ObjectLockLegalHoldStatus?, + public val objectLockMode: ObjectLockMode?, + public val objectLockRetainUntilDate: Instant?, + public val requestPayer: RequestPayer?, + public val sseCustomerAlgorithm: String?, + public val sseCustomerKey: String?, + public val sseCustomerKeyMd5: String?, + public val ssekmsEncryptionContext: String?, + public val ssekmsKeyId: String?, + public val serverSideEncryption: ServerSideEncryption?, + public val storageClass: StorageClass?, + public val tagging: String?, + public val websiteRedirectLocation: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileRequest = + Builder().apply(block).build() + } + + public class Builder { + public var acl: ObjectCannedAcl? = null + public var body: ByteStream? = null + public var bucket: String? = null + public var bucketKeyEnabled: Boolean? = null + public var cacheControl: String? = null + public var checksumAlgorithm: ChecksumAlgorithm? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var contentDisposition: String? = null + public var contentEncoding: String? = null + public var contentLanguage: String? = null + public var contentType: String? = null + public var expectedBucketOwner: String? = null + public var expires: Instant? = null + public var grantFullControl: String? = null + public var grantRead: String? = null + public var grantReadAcp: String? = null + public var grantWriteAcp: String? = null + public var ifMatch: String? = null + public var ifNoneMatch: String? = null + public var key: String? = null + public var metadata: Map? = null + public var objectLockLegalHoldStatus: ObjectLockLegalHoldStatus? = null + public var objectLockMode: ObjectLockMode? = null + public var objectLockRetainUntilDate: Instant? = null + public var requestPayer: RequestPayer? = null + public var sseCustomerAlgorithm: String? = null + public var sseCustomerKey: String? = null + public var sseCustomerKeyMd5: String? = null + public var ssekmsEncryptionContext: String? = null + public var ssekmsKeyId: String? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var storageClass: StorageClass? = null + public var tagging: String? = null + public var websiteRedirectLocation: String? = null + + public fun build(): UploadFileRequest = + UploadFileRequest( + acl, + body, + bucket, + bucketKeyEnabled, + cacheControl, + checksumAlgorithm, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + contentDisposition, + contentEncoding, + contentLanguage, + contentType, + expectedBucketOwner, + expires, + grantFullControl, + grantRead, + grantReadAcp, + grantWriteAcp, + ifMatch, + ifNoneMatch, + key, + metadata, + objectLockLegalHoldStatus, + objectLockMode, + objectLockRetainUntilDate, + requestPayer, + sseCustomerAlgorithm, + sseCustomerKey, + sseCustomerKeyMd5, + ssekmsEncryptionContext, + ssekmsKeyId, + serverSideEncryption, + storageClass, + tagging, + websiteRedirectLocation, + ) + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt new file mode 100644 index 00000000000..0d51e82e6c7 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/UploadFileResponse.kt @@ -0,0 +1,74 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.model + +import aws.sdk.kotlin.services.s3.model.ChecksumType +import aws.sdk.kotlin.services.s3.model.RequestCharged +import aws.sdk.kotlin.services.s3.model.ServerSideEncryption +import kotlin.String + +public class UploadFileResponse( + public val bucketKeyEnabled: Boolean?, + public val checksumCrc32: String?, + public val checksumCrc32C: String?, + public val checksumCrc64Nvme: String?, + public val checksumSha1: String?, + public val checksumSha256: String?, + public val checksumType: ChecksumType?, + public val eTag: String?, + public val expiration: String?, + public val requestCharged: RequestCharged?, + public val sseCustomerAlgorithm: String?, + public val sseCustomerKeyMd5: String?, + public val ssekmsEncryptionContext: String?, + public val ssekmsKeyId: String?, + public val serverSideEncryption: ServerSideEncryption?, + public val versionId: String?, +) { + public companion object { + public operator fun invoke(block: Builder.() -> Unit): UploadFileResponse = + Builder().apply(block).build() + } + + public class Builder { + public var bucketKeyEnabled: Boolean? = null + public var checksumCrc32: String? = null + public var checksumCrc32C: String? = null + public var checksumCrc64Nvme: String? = null + public var checksumSha1: String? = null + public var checksumSha256: String? = null + public var checksumType: ChecksumType? = null + public var eTag: String? = null + public var expiration: String? = null + public var requestCharged: RequestCharged? = null + public var sseCustomerAlgorithm: String? = null + public var sseCustomerKeyMd5: String? = null + public var ssekmsEncryptionContext: String? = null + public var ssekmsKeyId: String? = null + public var serverSideEncryption: ServerSideEncryption? = null + public var versionId: String? = null + + internal fun build(): UploadFileResponse = + UploadFileResponse( + bucketKeyEnabled, + checksumCrc32, + checksumCrc32C, + checksumCrc64Nvme, + checksumSha1, + checksumSha256, + checksumType, + eTag, + expiration, + requestCharged, + sseCustomerAlgorithm, + sseCustomerKeyMd5, + ssekmsEncryptionContext, + ssekmsKeyId, + serverSideEncryption, + versionId, + ) + } +} diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt new file mode 100644 index 00000000000..14b6b442466 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Conversions.kt @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadResponse +import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse + +internal fun PutObjectResponse.toUploadFileResponse(): UploadFileResponse = + UploadFileResponse { + bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled + checksumCrc32 = this@toUploadFileResponse.checksumCrc32 + checksumCrc32C = this@toUploadFileResponse.checksumCrc32C + checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme + checksumSha1 = this@toUploadFileResponse.checksumSha1 + checksumSha256 = this@toUploadFileResponse.checksumSha256 + checksumType = this@toUploadFileResponse.checksumType + eTag = this@toUploadFileResponse.eTag + expiration = this@toUploadFileResponse.expiration + requestCharged = this@toUploadFileResponse.requestCharged + sseCustomerAlgorithm = this@toUploadFileResponse.sseCustomerAlgorithm + sseCustomerKeyMd5 = this@toUploadFileResponse.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toUploadFileResponse.ssekmsEncryptionContext + ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption + versionId = this@toUploadFileResponse.versionId + } + +internal fun CompleteMultipartUploadResponse.toUploadFileResponse(): UploadFileResponse = + UploadFileResponse { + bucketKeyEnabled = this@toUploadFileResponse.bucketKeyEnabled + checksumCrc32 = this@toUploadFileResponse.checksumCrc32 + checksumCrc32C = this@toUploadFileResponse.checksumCrc32C + checksumCrc64Nvme = this@toUploadFileResponse.checksumCrc64Nvme + checksumSha1 = this@toUploadFileResponse.checksumSha1 + checksumSha256 = this@toUploadFileResponse.checksumSha256 + checksumType = this@toUploadFileResponse.checksumType + eTag = this@toUploadFileResponse.eTag + expiration = this@toUploadFileResponse.expiration + requestCharged = this@toUploadFileResponse.requestCharged + ssekmsKeyId = this@toUploadFileResponse.ssekmsKeyId + serverSideEncryption = this@toUploadFileResponse.serverSideEncryption + versionId = this@toUploadFileResponse.versionId + } + +internal fun UploadFileRequest.toPutObjectRequest(): PutObjectRequest = + PutObjectRequest { + acl = this@toPutObjectRequest.acl + body = this@toPutObjectRequest.body + bucket = this@toPutObjectRequest.bucket + bucketKeyEnabled = this@toPutObjectRequest.bucketKeyEnabled + cacheControl = this@toPutObjectRequest.cacheControl + checksumAlgorithm = this@toPutObjectRequest.checksumAlgorithm + checksumCrc32 = this@toPutObjectRequest.checksumCrc32 + checksumCrc32C = this@toPutObjectRequest.checksumCrc32C + checksumCrc64Nvme = this@toPutObjectRequest.checksumCrc64Nvme + checksumSha1 = this@toPutObjectRequest.checksumSha1 + checksumSha256 = this@toPutObjectRequest.checksumSha256 + contentDisposition = this@toPutObjectRequest.contentDisposition + contentEncoding = this@toPutObjectRequest.contentEncoding + contentLanguage = this@toPutObjectRequest.contentLanguage + contentLength = this@toPutObjectRequest.body?.contentLength + contentType = this@toPutObjectRequest.contentType + expectedBucketOwner = this@toPutObjectRequest.expectedBucketOwner + expires = this@toPutObjectRequest.expires + grantFullControl = this@toPutObjectRequest.grantFullControl + grantRead = this@toPutObjectRequest.grantRead + grantReadAcp = this@toPutObjectRequest.grantReadAcp + grantWriteAcp = this@toPutObjectRequest.grantWriteAcp + ifMatch = this@toPutObjectRequest.ifMatch + ifNoneMatch = this@toPutObjectRequest.ifNoneMatch + key = this@toPutObjectRequest.key + metadata = this@toPutObjectRequest.metadata + objectLockLegalHoldStatus = this@toPutObjectRequest.objectLockLegalHoldStatus + objectLockMode = this@toPutObjectRequest.objectLockMode + objectLockRetainUntilDate = this@toPutObjectRequest.objectLockRetainUntilDate + requestPayer = this@toPutObjectRequest.requestPayer + sseCustomerAlgorithm = this@toPutObjectRequest.sseCustomerAlgorithm + sseCustomerKey = this@toPutObjectRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toPutObjectRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toPutObjectRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toPutObjectRequest.ssekmsKeyId + serverSideEncryption = this@toPutObjectRequest.serverSideEncryption + storageClass = this@toPutObjectRequest.storageClass + tagging = this@toPutObjectRequest.tagging + websiteRedirectLocation = this@toPutObjectRequest.websiteRedirectLocation + } + +internal fun UploadFileRequest.toCreateMultiPartUploadRequest(): CreateMultipartUploadRequest = + CreateMultipartUploadRequest { + acl = this@toCreateMultiPartUploadRequest.acl + bucket = this@toCreateMultiPartUploadRequest.bucket + bucketKeyEnabled = this@toCreateMultiPartUploadRequest.bucketKeyEnabled + cacheControl = this@toCreateMultiPartUploadRequest.cacheControl + checksumAlgorithm = this@toCreateMultiPartUploadRequest.checksumAlgorithm + contentDisposition = this@toCreateMultiPartUploadRequest.contentDisposition + contentEncoding = this@toCreateMultiPartUploadRequest.contentEncoding + contentLanguage = this@toCreateMultiPartUploadRequest.contentLanguage + contentType = this@toCreateMultiPartUploadRequest.contentType + expectedBucketOwner = this@toCreateMultiPartUploadRequest.expectedBucketOwner + expires = this@toCreateMultiPartUploadRequest.expires + grantFullControl = this@toCreateMultiPartUploadRequest.grantFullControl + grantRead = this@toCreateMultiPartUploadRequest.grantRead + grantReadAcp = this@toCreateMultiPartUploadRequest.grantReadAcp + grantWriteAcp = this@toCreateMultiPartUploadRequest.grantWriteAcp + key = this@toCreateMultiPartUploadRequest.key + metadata = this@toCreateMultiPartUploadRequest.metadata + objectLockLegalHoldStatus = this@toCreateMultiPartUploadRequest.objectLockLegalHoldStatus + objectLockMode = this@toCreateMultiPartUploadRequest.objectLockMode + objectLockRetainUntilDate = this@toCreateMultiPartUploadRequest.objectLockRetainUntilDate + requestPayer = this@toCreateMultiPartUploadRequest.requestPayer + sseCustomerAlgorithm = this@toCreateMultiPartUploadRequest.sseCustomerAlgorithm + sseCustomerKey = this@toCreateMultiPartUploadRequest.sseCustomerKey + sseCustomerKeyMd5 = this@toCreateMultiPartUploadRequest.sseCustomerKeyMd5 + ssekmsEncryptionContext = this@toCreateMultiPartUploadRequest.ssekmsEncryptionContext + ssekmsKeyId = this@toCreateMultiPartUploadRequest.ssekmsKeyId + serverSideEncryption = this@toCreateMultiPartUploadRequest.serverSideEncryption + storageClass = this@toCreateMultiPartUploadRequest.storageClass + tagging = this@toCreateMultiPartUploadRequest.tagging + websiteRedirectLocation = this@toCreateMultiPartUploadRequest.websiteRedirectLocation + } diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt new file mode 100644 index 00000000000..b4eded45c30 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/Exceptions.kt @@ -0,0 +1,8 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +internal class S3TransferManagerException(message: String, cause: Throwable? = null) : Exception(message, cause) diff --git a/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt new file mode 100644 index 00000000000..bf4bf649115 --- /dev/null +++ b/hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/utils/UploadFile.kt @@ -0,0 +1,140 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager.utils + +import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager +import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.CompletedPart +import aws.sdk.kotlin.services.s3.model.UploadPartRequest +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.io.SdkBuffer +import aws.smithy.kotlin.runtime.io.SdkByteReadChannel +import aws.smithy.kotlin.runtime.io.SdkSource +import aws.smithy.kotlin.runtime.telemetry.logging.Logger + +// S3 imposed limit for parts in a multipart upload +private const val MAX_NUMBER_PARTS = 10_000L + +/** + * Determines the actual part size to use for a multipart S3 upload. + * + * This function calculates the part size based on the total size + * of the file and the requested part size. If the requested part size is + * too small to allow the upload to fit within S3's 10,000-part limit, the + * part size will be automatically increased so that exactly 10,000 parts + * are uploaded. + */ +internal fun resolvePartSize(contentLength: Long, tm: S3TransferManager, logger: Logger): Long { + val targetNumberOfParts = contentLength / tm.partSizeBytes + return if (targetNumberOfParts > MAX_NUMBER_PARTS) { + ceilDiv(contentLength, MAX_NUMBER_PARTS).also { + logger.debug { "Target part size is too small to meet the 10,000 S3 part limit. Increasing part size to $it" } + } + } else { + tm.partSizeBytes + } +} + +/** + * Retrieves the next part of a multipart upload from the given part source. + */ +internal suspend fun SdkBuffer.getNextPart(partSource: Any, partSize: Long, tm: S3TransferManager) { + when (partSource) { + is ByteArray -> { + this.write( + partSource.sliceArray( + tm.context.transferredBytes!!.toInt().. { + var readBytes = 0L + while (readBytes < partSize) { + readBytes += partSource.read(this, partSize - readBytes) + } + } + is SdkSource -> { + var readBytes = 0L + while (readBytes < partSize) { + readBytes += partSource.read(this, partSize - readBytes) + } + } + } +} + +/** + * Builds a low-level S3 upload part request from a high-level upload file request + * and data from the S3 Transfer Manager. + */ +internal fun buildUploadPartRequest( + uploadFileRequest: UploadFileRequest, + currentPart: SdkBuffer, + currentPartNumber: Long, + mpuUploadId: String, +): UploadPartRequest = + UploadPartRequest { + // From high-level request + bucket = uploadFileRequest.bucket + checksumAlgorithm = uploadFileRequest.checksumAlgorithm + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm + sseCustomerKey = uploadFileRequest.sseCustomerKey + sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 + + // From transfer manager + uploadId = mpuUploadId + body = ByteStream.fromInputStream(currentPart.inputStream(), currentPart.size) + partNumber = currentPartNumber.toInt() + } + +/** + * Builds a low-level S3 complete multipart upload request from a high-level upload file request + * and data from the S3 Transfer Manager. + */ +internal fun buildCompleteMultipartUploadRequest( + uploadFileRequest: UploadFileRequest, + mpuUploadId: String, + uploadedParts: List, +): CompleteMultipartUploadRequest = + CompleteMultipartUploadRequest { + // From high-level request + bucket = uploadFileRequest.bucket + checksumCrc32 = uploadFileRequest.checksumCrc32 + checksumCrc32C = uploadFileRequest.checksumCrc32C + checksumCrc64Nvme = uploadFileRequest.checksumCrc64Nvme + checksumSha1 = uploadFileRequest.checksumSha1 + checksumSha256 = uploadFileRequest.checksumSha256 + expectedBucketOwner = uploadFileRequest.expectedBucketOwner + ifMatch = uploadFileRequest.ifMatch + ifNoneMatch = uploadFileRequest.ifNoneMatch + key = uploadFileRequest.key + requestPayer = uploadFileRequest.requestPayer + sseCustomerAlgorithm = uploadFileRequest.sseCustomerAlgorithm + sseCustomerKey = uploadFileRequest.sseCustomerKey + sseCustomerKeyMd5 = uploadFileRequest.sseCustomerKeyMd5 + + // From transfer manager + uploadId = mpuUploadId + multipartUpload { + parts = uploadedParts + } + } + +/** + * Returns the ceiling of the division + * + * This means the result is rounded up to the nearest integer if the dividend is not + * evenly divisible by the divisor + */ +internal fun ceilDiv(dividend: Long, divisor: Long): Long { + val div = dividend / divisor + val remainder = dividend % divisor + return if (remainder != 0L) div + 1 else div +} diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt new file mode 100644 index 00000000000..ed3c5720b3f --- /dev/null +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManagerBusinessMetricsTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.businessmetrics.containsBusinessMetric +import aws.smithy.kotlin.runtime.client.ProtocolResponseInterceptorContext +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.test.Test + +class S3TransferManagerBusinessMetricsTest { + @Test + fun s3Transfer(): Unit = runBlocking { + val message = "Hello World" + val testInterceptor = object : HttpInterceptor { + override fun readAfterTransmit(context: ProtocolResponseInterceptorContext) { + assert(context.executionContext.containsBusinessMetric(AwsBusinessMetric.S3_TRANSFER)) + } + } + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + interceptors += testInterceptor + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager { + client = s3Client + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } +} diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt new file mode 100644 index 00000000000..8199295bedd --- /dev/null +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptorTest.kt @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.CompleteMultipartUploadRequest +import aws.sdk.kotlin.services.s3.model.PutObjectRequest +import aws.sdk.kotlin.services.s3.model.PutObjectResponse +import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.httptest.TestEngine +import kotlinx.coroutines.runBlocking +import kotlin.collections.plusAssign +import kotlin.test.Test + +class TransferInterceptorTest { + @Test + fun interceptorsCanReadAndModify(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + httpClient = TestEngine() + credentialsProvider = StaticCredentialsProvider(Credentials("akid", "secret")) + }.use { s3Client -> + S3TransferManager { + client = s3Client + interceptors += object : TransferInterceptor { + // Test reads + override fun readBeforeTransferInitiated(context: TransferContext) { + assert(context.transferredBytes == 0L) + assert(context.request is PutObjectRequest) + } + + override fun readBeforeTransferCompleted(context: TransferContext) { + assert(context.transferredBytes == message.length.toLong()) + assert(context.response is PutObjectResponse) + } + + // Test modifications + override fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext { + context.request = CompleteMultipartUploadRequest {} + context.transferredBytes = message.length.toLong() * 10 + return context + } + + override fun readAfterTransferCompleted(context: TransferContext) { + assert(context.request is CompleteMultipartUploadRequest) + assert(context.transferredBytes == message.length.toLong() * 10) + } + } + }.uploadFile { + bucket = "b" + key = "k" + body = ByteStream.fromString(message) + } + } + } +} diff --git a/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt new file mode 100644 index 00000000000..893b9ab5bd7 --- /dev/null +++ b/hll/s3-transfer-manager/jvm/test/aws/sdk/kotlin/hll/s3transfermanager/UploadFileTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package aws.sdk.kotlin.hll.s3transfermanager + +import aws.sdk.kotlin.services.s3.S3Client +import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromInputStream +import aws.smithy.kotlin.runtime.testing.RandomTempFile +import kotlinx.coroutines.runBlocking +import kotlin.test.Ignore +import kotlin.test.Test + +// TODO: Setup e2e test environment - can't run these every build and in CI +class UploadFileTest { + @Ignore + @Test + fun singleObjectUpload(): Unit = runBlocking { + val message = "Hello World" + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager { + client = s3Client + }.uploadFile { + bucket = "aoperez" + key = "k" + body = ByteStream.fromString(message) + } + } + } + + @Ignore + @Test + fun multiplePartUpload(): Unit = runBlocking { + val messageLength = 10L * 1024L * 1024L // 10 MB + val file = RandomTempFile(messageLength) + + S3Client { + region = "us-west-2" + }.use { s3Client -> + S3TransferManager { + client = s3Client + multipartUploadThresholdBytes = 1 + partSizeBytes = 5L * 1024L * 1024L // 5 MB + }.uploadFile { + bucket = "aoperez" + key = "mpuK" + body = ByteStream.fromInputStream(file.inputStream(), messageLength) + } + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 826adedd84d..665e55cfb82 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -91,6 +91,12 @@ if ("dynamodb".isBootstrappedService) { logger.warn(":services:dynamodb is not bootstrapped, skipping :hll:dynamodb-mapper and subprojects") } +if ("s3".isBootstrappedService) { + include(":hll:s3-transfer-manager") +} else { + logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects") +} + // Service benchmarks project val benchmarkServices = listOf( // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts