Add support for checksums for putObject / getObject

Both "ChecksumAlgorithm" where the SDK calculates the checksum and embeds it into the request body, and checksum headers are supported. Fixes #1123
adobe · Jul 2, 2023 · 2a819cd · 2a819cd
1 parent 7f9bf72
commit 2a819cd
Show file tree

Hide file tree

Showing 24 changed files with 683 additions and 33 deletions.
diff --git a/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt b/integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt
@@ -23,6 +23,7 @@ import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInfo
 import org.springframework.http.ContentDisposition
 import software.amazon.awssdk.core.sync.RequestBody
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
 import software.amazon.awssdk.services.s3.model.GetObjectRequest
 import software.amazon.awssdk.services.s3.model.HeadObjectRequest
 import software.amazon.awssdk.services.s3.model.PutObjectRequest
@@ -49,6 +50,240 @@ internal class GetPutDeleteObjectV2IT : S3TestBase() {
     assertThat(eTag).isEqualTo(expectedEtag)
   }
 
+  @Test
+  fun testPutObject_checksumAlgorithm_sha1(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "+AXXQmKfnxMv0B57SJutbNpZBww="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumAlgorithm(ChecksumAlgorithm.SHA1)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumSHA1()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumSHA1()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checkSum_sha1(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "+AXXQmKfnxMv0B57SJutbNpZBww="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumSHA1(expectedChecksum)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumSHA1()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumSHA1()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checksumAlgorithm_sha256(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumAlgorithm(ChecksumAlgorithm.SHA256)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumSHA256()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumSHA256()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checkSum_sha256(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumSHA256(expectedChecksum)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumSHA256()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumSHA256()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checksumAlgorithm_crc32(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "I6zdvg=="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumAlgorithm(ChecksumAlgorithm.CRC32)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumCRC32()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumCRC32()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checkSum_crc32(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "I6zdvg=="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName).key(UPLOAD_FILE_NAME)
+        .checksumCRC32(expectedChecksum)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumCRC32()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumCRC32()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checksumAlgorithm_crc32c(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "/Ho1Kg=="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .checksumAlgorithm(ChecksumAlgorithm.CRC32_C)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumCRC32C()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumCRC32C()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  @Test
+  fun testPutObject_checkSum_crc32c(testInfo: TestInfo) {
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val expectedChecksum = "/Ho1Kg=="
+    val bucketName = givenBucketV2(testInfo)
+
+    val putObjectResponse = s3ClientV2.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .checksumCRC32C(expectedChecksum)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val putChecksum = putObjectResponse.checksumCRC32C()
+        assertThat(putChecksum).isNotBlank
+        assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = s3ClientV2.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucketName)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    val getChecksum = getObjectResponse.response().checksumCRC32C()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2022)
   fun testPutGetDeleteObject_twoBuckets(testInfo: TestInfo) {

diff --git a/pom.xml b/pom.xml
@@ -258,6 +258,21 @@
         <artifactId>aws-crt-client</artifactId>
         <version>${aws-v2.version}</version>
       </dependency>
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>regions</artifactId>
+        <version>${aws-v2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>utils</artifactId>
+        <version>${aws-v2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>software.amazon.awssdk</groupId>
+        <artifactId>auth</artifactId>
+        <version>${aws-v2.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

diff --git a/server/pom.xml b/server/pom.xml
@@ -50,17 +50,18 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>regions</artifactId>
-      <version>${aws-v2.version}</version>
     </dependency>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>utils</artifactId>
-      <version>${aws-v2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
     </dependency>
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>auth</artifactId>
-      <version>${aws-v2.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

diff --git a/server/src/main/java/com/adobe/testing/s3mock/MultipartController.java b/server/src/main/java/com/adobe/testing/s3mock/MultipartController.java
@@ -19,11 +19,16 @@
 import static com.adobe.testing.s3mock.dto.Owner.DEFAULT_OWNER;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.NOT_X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.NOT_X_AMZ_COPY_SOURCE_RANGE;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC32;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_CRC32C;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA1;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CONTENT_SHA256;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_MATCH;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_NONE_MATCH;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_RANGE;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_SDK_CHECKSUM_ALGORITHM;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_LIFECYCLE;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.PART_NUMBER;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.UPLOADS;
@@ -35,6 +40,7 @@
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
 import static org.springframework.http.MediaType.APPLICATION_XML_VALUE;
 
+import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
 import com.adobe.testing.s3mock.dto.CompleteMultipartUpload;
 import com.adobe.testing.s3mock.dto.CompleteMultipartUploadResult;
 import com.adobe.testing.s3mock.dto.CopyPartResult;
@@ -212,6 +218,12 @@ public ResponseEntity<Void> uploadPart(@PathVariable String bucketName,
       @RequestParam String uploadId,
       @RequestParam String partNumber,
       @RequestHeader(value = X_AMZ_CONTENT_SHA256, required = false) String sha256Header,
+      @RequestHeader(value = X_AMZ_SDK_CHECKSUM_ALGORITHM, required = false)
+                                           ChecksumAlgorithm checksumAlgorithm,
+      @RequestHeader(value = X_AMZ_CHECKSUM_CRC32, required = false) String checksumCrc32,
+      @RequestHeader(value = X_AMZ_CHECKSUM_CRC32C, required = false) String checksumCrc32c,
+      @RequestHeader(value = X_AMZ_CHECKSUM_SHA1, required = false) String checksumSha1,
+      @RequestHeader(value = X_AMZ_CHECKSUM_SHA256, required = false) String checksumSha256,
       @RequestHeader HttpHeaders httpHeaders,
       InputStream inputStream) {
     bucketService.verifyBucketExists(bucketName);