Skip to content

Commit

Permalink
Merge pull request #1828 from adobe/1827-validate-checksums-for-uploa…
Browse files Browse the repository at this point in the history
…ds-and-parts

Verify checksums on upload
  • Loading branch information
afranken authored May 1, 2024
2 parents 434092e + 73ce586 commit 48a64aa
Show file tree
Hide file tree
Showing 29 changed files with 1,086 additions and 733 deletions.
19 changes: 12 additions & 7 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
* [Planned changes](#planned-changes)
* [CURRENT - 3.x - THIS VERSION IS UNDER ACTIVE DEVELOPMENT](#current---3x---this-version-is-under-active-development)
* [3.8.0 - PLANNED](#380---planned)
* [3.7.1 - PLANNED](#371---planned)
* [3.7.3 - PLANNED](#373---planned)
* [3.7.2](#372)
* [3.7.1](#371)
* [3.7.0](#370)
* [3.6.0](#360)
* [3.5.2](#352)
Expand Down Expand Up @@ -132,15 +134,18 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
* Version updates
* TBD

## 3.7.2 - PLANNED
## 3.7.3 - PLANNED
3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

* Features and fixes
* Calculate and validate checksums on upload
* Refactorings
* TBD
* Version updates
* TBD
* Support large, chunked, unsigned, asynchronous uploads (fixes #1818)

## 3.7.2
3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

* Features and fixes
* Calculate and validate checksums on upload (fixes #1827)
* UploadPart API now also returns checksums, if available.

## 3.7.1
3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,30 @@ internal class AclIT : S3TestBase() {
val sourceKey = UPLOAD_FILE_NAME
val (bucketName, _) = givenBucketAndObjectV2(testInfo, sourceKey)

val putAclResponse = s3ClientV2.putObjectAcl(
s3ClientV2.putObjectAcl(
PutObjectAclRequest
.builder()
.bucket(bucketName)
.key(sourceKey)
.acl(ObjectCannedACL.PRIVATE)
.build()
)
assertThat(putAclResponse.sdkHttpResponse().isSuccessful).isTrue()
).also {
assertThat(it.sdkHttpResponse().isSuccessful).isTrue()
}

val getAclResponse = s3ClientV2.getObjectAcl(
s3ClientV2.getObjectAcl(
GetObjectAclRequest
.builder()
.bucket(bucketName)
.key(sourceKey)
.build()
)
assertThat(getAclResponse.sdkHttpResponse().isSuccessful).isTrue()
assertThat(getAclResponse.owner().id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(getAclResponse.owner().displayName()).isEqualTo(DEFAULT_OWNER.displayName)
assertThat(getAclResponse.grants().size).isEqualTo(1)
assertThat(getAclResponse.grants()[0].permission()).isEqualTo(FULL_CONTROL)
).also {
assertThat(it.sdkHttpResponse().isSuccessful).isTrue()
assertThat(it.owner().id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(it.owner().displayName()).isEqualTo(DEFAULT_OWNER.displayName)
assertThat(it.grants().size).isEqualTo(1)
assertThat(it.grants()[0].permission()).isEqualTo(FULL_CONTROL)
}
}

@Test
Expand All @@ -78,21 +80,22 @@ internal class AclIT : S3TestBase() {
.build()
)

val owner = acl.owner()
assertThat(owner.id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(owner.displayName()).isEqualTo(DEFAULT_OWNER.displayName)

val grants = acl.grants()
assertThat(grants).hasSize(1)
acl.owner().also { owner ->
assertThat(owner.id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(owner.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
}

val grants = acl.grants().also {
assertThat(it).hasSize(1)
}
val grant = grants[0]
assertThat(grant.permission()).isEqualTo(FULL_CONTROL)

val grantee = grant.grantee()
assertThat(grantee).isNotNull
assertThat(grantee.id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(grantee.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
assertThat(grantee.type()).isEqualTo(CANONICAL_USER)
grant.grantee().also {
assertThat(it).isNotNull
assertThat(it.id()).isEqualTo(DEFAULT_OWNER.id)
assertThat(it.displayName()).isEqualTo(DEFAULT_OWNER.displayName)
assertThat(it.type()).isEqualTo(CANONICAL_USER)
}
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,22 +62,25 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() {
RequestBody.fromFile(uploadFile)
)

val putChecksum = putObjectResponse.checksumSHA256()
assertThat(putChecksum).isNotBlank
assertThat(putChecksum).isEqualTo(expectedChecksum)
putObjectResponse.checksumSHA256().also {
assertThat(it).isNotBlank
assertThat(it).isEqualTo(expectedChecksum)
}

val getObjectResponse = s3ClientV2.getObject(
s3ClientV2.getObject(
GetObjectRequest.builder()
.bucket(bucket)
.key(UPLOAD_FILE_NAME)
.build()
)
assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
).also { getObjectResponse ->
assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())

val getChecksum = getObjectResponse.response().checksumSHA256()
assertThat(getChecksum).isNotBlank
assertThat(getChecksum).isEqualTo(expectedChecksum)
getObjectResponse.response().checksumSHA256().also {
assertThat(it).isNotBlank
assertThat(it).isEqualTo(expectedChecksum)
}
}
}

/**
Expand All @@ -104,13 +107,14 @@ internal class AwsChunkedEndcodingITV2 : S3TestBase() {
RequestBody.fromFile(uploadFile)
)

val getObjectResponse = s3ClientV2.getObject(
s3ClientV2.getObject(
GetObjectRequest.builder()
.bucket(bucket)
.key(UPLOAD_FILE_NAME)
.build()
)
assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
).also {
assertThat(it.response().eTag()).isEqualTo(expectedEtag)
assertThat(it.response().contentLength()).isEqualTo(uploadFile.length())
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,10 @@ internal class BucketV1IT : S3TestBase() {
val createdBucket = buckets[0]
assertThat(createdBucket.creationDate).isAfterOrEqualTo(creationDate)

val bucketOwner = createdBucket.owner
assertThat(bucketOwner.displayName).isEqualTo("s3-mock-file-store")
assertThat(bucketOwner.id).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
createdBucket.owner.also {
assertThat(it.displayName).isEqualTo("s3-mock-file-store")
assertThat(it.id).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
}
}

@Test
Expand All @@ -81,8 +82,9 @@ internal class BucketV1IT : S3TestBase() {
s3Client.headBucket(HeadBucketRequest(bucketName))
s3Client.deleteBucket(bucketName)

val doesBucketExist = s3Client.doesBucketExistV2(bucketName)
assertThat(doesBucketExist).isFalse
s3Client.doesBucketExistV2(bucketName).also {
assertThat(it).isFalse
}
}

@Test
Expand All @@ -104,8 +106,9 @@ internal class BucketV1IT : S3TestBase() {
val bucketName = bucketName(testInfo)
s3Client.createBucket(bucketName)

val doesBucketExist = s3Client.doesBucketExistV2(bucketName)
assertThat(doesBucketExist).isTrue
s3Client.doesBucketExistV2(bucketName).also {
assertThat(it).isTrue
}
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,10 @@ internal class BucketV2IT : S3TestBase() {
s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
val bucketDeleted = s3ClientV2.waiter()
.waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())
val bucketDeletedResponse = bucketDeleted.matched().exception().get()
assertThat(bucketDeletedResponse).isNotNull
assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
bucketDeleted.matched().exception().get().also {
assertThat(it).isNotNull
assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
}
}

@Test
Expand All @@ -86,8 +87,9 @@ internal class BucketV2IT : S3TestBase() {

val bucketCreated = s3ClientV2.waiter()
.waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
val bucketCreatedResponse = bucketCreated.matched().response().get()
assertThat(bucketCreatedResponse).isNotNull
bucketCreated.matched().response().get().also {
assertThat(it).isNotNull
}

assertThatThrownBy {
s3ClientV2.createBucket(CreateBucketRequest.builder().bucket(bucketName).build())
Expand All @@ -103,9 +105,10 @@ internal class BucketV2IT : S3TestBase() {
val bucketDeleted = s3ClientV2.waiter()
.waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())

val bucketDeletedResponse = bucketDeleted.matched().exception().get()
assertThat(bucketDeletedResponse).isNotNull
assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
bucketDeleted.matched().exception().get().also {
assertThat(it).isNotNull
assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
}
}

@Test
Expand All @@ -116,15 +119,17 @@ internal class BucketV2IT : S3TestBase() {

val bucketCreated = s3ClientV2.waiter()
.waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
val bucketCreatedResponse = bucketCreated.matched().response().get()
assertThat(bucketCreatedResponse).isNotNull
bucketCreated.matched().response().get().also {
assertThat(it).isNotNull
}

s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
val bucketDeleted = s3ClientV2.waiter()
.waitUntilBucketNotExists(HeadBucketRequest.builder().bucket(bucketName).build())
val bucketDeletedResponse = bucketDeleted.matched().exception().get()
assertThat(bucketDeletedResponse).isNotNull
assertThat(bucketDeletedResponse).isInstanceOf(NoSuchBucketException::class.java)
bucketDeleted.matched().exception().get().also {
assertThat(it).isNotNull
assertThat(it).isInstanceOf(NoSuchBucketException::class.java)
}

assertThatThrownBy {
s3ClientV2.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build())
Expand Down Expand Up @@ -169,8 +174,9 @@ internal class BucketV2IT : S3TestBase() {

val bucketCreated = s3ClientV2.waiter()
.waitUntilBucketExists(HeadBucketRequest.builder().bucket(bucketName).build())
val bucketCreatedResponse = bucketCreated.matched().response()!!.get()
assertThat(bucketCreatedResponse).isNotNull
bucketCreated.matched().response()!!.get().also {
assertThat(it).isNotNull
}

val configuration = BucketLifecycleConfiguration
.builder()
Expand Down Expand Up @@ -206,20 +212,21 @@ internal class BucketV2IT : S3TestBase() {
.build()
)

val configurationResponse = s3ClientV2.getBucketLifecycleConfiguration(
s3ClientV2.getBucketLifecycleConfiguration(
GetBucketLifecycleConfigurationRequest
.builder()
.bucket(bucketName)
.build()
)

assertThat(configurationResponse.rules()[0]).isEqualTo(configuration.rules()[0])
).also {
assertThat(it.rules()[0]).isEqualTo(configuration.rules()[0])
}

val deleteBucketLifecycle = s3ClientV2.deleteBucketLifecycle(
s3ClientV2.deleteBucketLifecycle(
DeleteBucketLifecycleRequest.builder().bucket(bucketName).build()
)
).also {
assertThat(it.sdkHttpResponse().statusCode()).isEqualTo(204)
}

assertThat(deleteBucketLifecycle.sdkHttpResponse().statusCode()).isEqualTo(204)

// give AWS time to actually delete the lifecycleConfiguration, otherwise the following call
// will not fail as expected...
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,31 +61,35 @@ internal class ConcurrencyIT : S3TestBase() {
inner class Runner(val bucketName: String, val key: String) : Callable<Boolean> {
override fun call(): Boolean {
LATCH.countDown()
val putObjectResponse = s3ClientV2.putObject(
s3ClientV2.putObject(
PutObjectRequest
.builder()
.bucket(bucketName)
.key(key)
.build(), RequestBody.empty()
)
assertThat(putObjectResponse.eTag()).isNotBlank
val getObjectResponse = s3ClientV2.getObject(
).also {
assertThat(it.eTag()).isNotBlank
}

s3ClientV2.getObject(
GetObjectRequest
.builder()
.bucket(bucketName)
.key(key)
.build()
)
assertThat(getObjectResponse.response().eTag()).isNotBlank
).also {
assertThat(it.response().eTag()).isNotBlank
}

val deleteObjectResponse = s3ClientV2.deleteObject(
s3ClientV2.deleteObject(
DeleteObjectRequest
.builder()
.bucket(bucketName)
.key(key)
.build()
)
assertThat(deleteObjectResponse.deleteMarker()).isTrue
).also {
assertThat(it.deleteMarker()).isTrue
}
DONE.incrementAndGet()
return true
}
Expand Down
Loading

0 comments on commit 48a64aa

Please sign in to comment.