Correctly handle unsigned chunked uploads

Write incoming streams directly to a temp file, removing chunk
information, optionally read checksum information.

Fixes #1662

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/GetPutDeleteObjectV2IT.kt

@@ -195,15 +195,16 @@ internal class GetPutDeleteObjectV2IT : S3TestBase() {
       assertThat(getChecksum).isEqualTo(expectedChecksum)
     }
 
-    val headObjectResponse = s3ClientV2.headObject(
+    s3ClientV2.headObject(
       HeadObjectRequest.builder()
         .bucket(bucketName)
         .key(UPLOAD_FILE_NAME)
         .build()
-    )
-    val headChecksum = headObjectResponse.checksumSHA1()
-    assertThat(headChecksum).isNotBlank
-    assertThat(headChecksum).isEqualTo(expectedChecksum)
+    ).also {
+      val headChecksum = it.checksumSHA1()
+      assertThat(headChecksum).isNotBlank
+      assertThat(headChecksum).isEqualTo(expectedChecksum)
+    }
   }
 
   /**

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/MultiPartUploadV2IT.kt

@@ -16,24 +16,26 @@
 package com.adobe.testing.s3mock.its
 
 import com.adobe.testing.s3mock.S3Exception.PRECONDITION_FAILED
+import com.adobe.testing.s3mock.util.DigestUtil
 import com.adobe.testing.s3mock.util.DigestUtil.hexDigest
 import org.apache.commons.codec.digest.DigestUtils
 import org.apache.commons.lang3.ArrayUtils
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatThrownBy
 import org.assertj.core.api.InstanceOfAssertFactories
 import org.assertj.core.util.Files
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.TestInfo
 import org.springframework.web.util.UriUtils
 import software.amazon.awssdk.awscore.exception.AwsErrorDetails
 import software.amazon.awssdk.awscore.exception.AwsServiceException
 import software.amazon.awssdk.core.async.AsyncRequestBody
+import software.amazon.awssdk.core.checksums.Algorithm
 import software.amazon.awssdk.core.sync.RequestBody
 import software.amazon.awssdk.services.s3.S3AsyncClient
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest
 import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload
 import software.amazon.awssdk.services.s3.model.CompletedPart
@@ -65,34 +67,25 @@ internal class MultiPartUploadV2IT : S3TestBase() {
   val autoS3CrtAsyncClientV2: S3AsyncClient = createAutoS3CrtAsyncClientV2()
   val transferManagerV2: S3TransferManager = createTransferManagerV2()
 
-  private fun lorem(): String {
-    return "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet."
-  }
-
-  @Disabled("This test currently fails. Must debug")
   @Test
   @S3VerifiedTodo
   fun testMultipartUpload_asyncClient(testInfo: TestInfo) {
-    //TODO: this could be related - trailing headers for chunks
-    // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming-trailers.html
     val bucketName = givenBucketV2(testInfo)
-    val uploadFile = Files.newTemporaryFile()
-    java.nio.file.Files.newOutputStream(uploadFile.toPath()).use {
-      for(i in 0.. 10000) {
-        it.write(lorem().toByteArray())
-      }
-    }
-
-    s3AsyncClientV2.putObject(
-      PutObjectRequest
-        .builder()
-        .bucket(bucketName)
-        .key(uploadFile.name)
-        .build(),
-      AsyncRequestBody.fromFile(uploadFile)
-    ).join()
+    val uploadFile = File(UPLOAD_FILE_NAME)
+  s3CrtAsyncClientV2.putObject(
+    PutObjectRequest
+      .builder()
+      .bucket(bucketName)
+      .key(uploadFile.name)
+      .checksumAlgorithm(ChecksumAlgorithm.CRC32)
+      .build(),
+    AsyncRequestBody.fromFile(uploadFile)
+  ).join().also {
+    assertThat(it.checksumCRC32()).isEqualTo(DigestUtil.checksumFor(uploadFile.toPath(), Algorithm.CRC32))
+  }
 
-    s3AsyncClientV2.waiter().waitUntilObjectExists(
+  s3AsyncClientV2.waiter()
+      .waitUntilObjectExists(
       HeadObjectRequest
         .builder()
         .bucket(bucketName)
@@ -107,10 +100,12 @@ internal class MultiPartUploadV2IT : S3TestBase() {
         .key(uploadFile.name)
         .build()
     ).use {
-      val uploadFileIs = java.nio.file.Files.newInputStream(uploadFile.toPath())
       val uploadDigest = hexDigest(uploadFile)
-      val downloadedDigest = hexDigest(it)
-      uploadFileIs.close()
+      val newTemporaryFile = Files.newTemporaryFile()
+      it.transferTo(java.nio.file.Files.newOutputStream(newTemporaryFile.toPath()))
+      assertThat(newTemporaryFile).hasSize(uploadFile.length())
+      assertThat(newTemporaryFile).hasSameBinaryContentAs(uploadFile)
+      val downloadedDigest = hexDigest(newTemporaryFile)
       assertThat(uploadDigest).isEqualTo(downloadedDigest)
     }
   }

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/S3TestBase.kt

@@ -551,7 +551,7 @@ internal abstract class S3TestBase {
   companion object {
     val INITIAL_BUCKET_NAMES: Collection<String> = listOf("bucket-a", "bucket-b")
     const val TEST_ENC_KEY_ID = "valid-test-key-id"
-    const val UPLOAD_FILE_NAME = "src/test/resources/sampleFile.txt"
+    const val UPLOAD_FILE_NAME = "src/test/resources/sampleFile_large.txt"
     const val TEST_WRONG_KEY_ID = "key-ID-WRONGWRONGWRONG"
     const val _1MB = 1024 * 1024
     const val _5MB = 5L * _1MB

server/src/main/java/com/adobe/testing/s3mock/MultipartController.java

@@ -19,7 +19,6 @@
 import static com.adobe.testing.s3mock.dto.Owner.DEFAULT_OWNER;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.NOT_X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.NOT_X_AMZ_COPY_SOURCE_RANGE;
-import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CONTENT_SHA256;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_MATCH;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_NONE_MATCH;
@@ -29,10 +28,9 @@
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.PART_NUMBER;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.UPLOADS;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.UPLOAD_ID;
-import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFrom;
+import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromHeader;
 import static com.adobe.testing.s3mock.util.HeaderUtil.checksumFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.encryptionHeadersFrom;
-import static com.adobe.testing.s3mock.util.HeaderUtil.isV4ChunkedWithSigningEnabled;
 import static com.adobe.testing.s3mock.util.HeaderUtil.storeHeadersFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.userMetadataFrom;
 import static org.springframework.http.HttpHeaders.CONTENT_TYPE;
@@ -54,6 +52,7 @@
 import java.io.InputStream;
 import java.util.List;
 import java.util.UUID;
+import org.apache.commons.io.FileUtils;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpRange;
 import org.springframework.http.ResponseEntity;
@@ -205,28 +204,27 @@ public ResponseEntity<Void> uploadPart(@PathVariable String bucketName,
       @PathVariable ObjectKey key,
       @RequestParam String uploadId,
       @RequestParam String partNumber,
-      @RequestHeader(value = X_AMZ_CONTENT_SHA256, required = false) String sha256Header,
       @RequestHeader HttpHeaders httpHeaders,
       InputStream inputStream) {
 
-    final var input = multipartService.toTempFile(inputStream);
-
+    final var tempFileAndChecksum = objectService.toTempFile(inputStream, httpHeaders);
     bucketService.verifyBucketExists(bucketName);
     multipartService.verifyMultipartUploadExists(uploadId);
     multipartService.verifyPartNumberLimits(partNumber);
 
     var checksum = checksumFrom(httpHeaders);
-    var checksumAlgorithm = checksumAlgorithmFrom(httpHeaders);
+    var checksumAlgorithm = checksumAlgorithmFromHeader(httpHeaders);
 
     //persist checksum per part
     var etag = multipartService.putPart(bucketName,
         key.key(),
         uploadId,
         partNumber,
-        input,
-        isV4ChunkedWithSigningEnabled(sha256Header),
+        tempFileAndChecksum.getLeft(),
         encryptionHeadersFrom(httpHeaders));
 
+    FileUtils.deleteQuietly(tempFileAndChecksum.getLeft().toFile());
+
     //return checksum headers
     //return encryption headers
     return ResponseEntity.ok().eTag("\"" + etag + "\"").build();
@@ -310,7 +308,7 @@ public ResponseEntity<InitiateMultipartUploadResult> createMultipartUpload(
     bucketService.verifyBucketExists(bucketName);
 
     var checksum = checksumFrom(httpHeaders);
-    var checksumAlgorithm = checksumAlgorithmFrom(httpHeaders);
+    var checksumAlgorithm = checksumAlgorithmFromHeader(httpHeaders);
 
     var uploadId = UUID.randomUUID().toString();
     var result =

server/src/main/java/com/adobe/testing/s3mock/ObjectController.java

@@ -22,7 +22,6 @@
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.NOT_X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.RANGE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_ACL;
-import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CONTENT_SHA256;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_MATCH;
 import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_COPY_SOURCE_IF_NONE_MATCH;
@@ -45,11 +44,11 @@
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_UPLOAD_ID;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.RETENTION;
 import static com.adobe.testing.s3mock.util.AwsHttpParameters.TAGGING;
-import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFrom;
+import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromHeader;
+import static com.adobe.testing.s3mock.util.HeaderUtil.checksumAlgorithmFromSdk;
 import static com.adobe.testing.s3mock.util.HeaderUtil.checksumFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.checksumHeaderFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.encryptionHeadersFrom;
-import static com.adobe.testing.s3mock.util.HeaderUtil.isV4ChunkedWithSigningEnabled;
 import static com.adobe.testing.s3mock.util.HeaderUtil.mediaTypeFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.overrideHeadersFrom;
 import static com.adobe.testing.s3mock.util.HeaderUtil.storeHeadersFrom;
@@ -64,6 +63,7 @@
 import static org.springframework.http.MediaType.APPLICATION_XML_VALUE;
 
 import com.adobe.testing.s3mock.dto.AccessControlPolicy;
+import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
 import com.adobe.testing.s3mock.dto.CopyObjectResult;
 import com.adobe.testing.s3mock.dto.CopySource;
 import com.adobe.testing.s3mock.dto.Delete;
@@ -90,6 +90,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.io.input.BoundedInputStream;
 import org.springframework.http.HttpHeaders;
@@ -592,31 +593,48 @@ public ResponseEntity<Void> putObject(@PathVariable String bucketName,
       @RequestHeader(name = X_AMZ_TAGGING, required = false) List<Tag> tags,
       @RequestHeader(value = CONTENT_TYPE, required = false) String contentType,
       @RequestHeader(value = CONTENT_MD5, required = false) String contentMd5,
-      @RequestHeader(value = X_AMZ_CONTENT_SHA256, required = false) String sha256Header,
       @RequestHeader(value = X_AMZ_STORAGE_CLASS, required = false, defaultValue = "STANDARD")
                                           StorageClass storageClass,
       @RequestHeader HttpHeaders httpHeaders,
       InputStream inputStream) {
-    var input = objectService.toTempFile(inputStream);
+
+    String checksum = null;
+    ChecksumAlgorithm checksumAlgorithm = null;
+
+    var tempFileAndChecksum = objectService.toTempFile(inputStream, httpHeaders);
+
+    //TODO: check checksum against incoming headers
+    ChecksumAlgorithm algorithmFromSdk = checksumAlgorithmFromSdk(httpHeaders);
+    if (algorithmFromSdk != null) {
+      checksum = tempFileAndChecksum.getRight();
+      checksumAlgorithm = algorithmFromSdk;
+    }
+    ChecksumAlgorithm algorithmFromHeader = checksumAlgorithmFromHeader(httpHeaders);
+    if (algorithmFromHeader != null) {
+      checksum = checksumFrom(httpHeaders);
+      checksumAlgorithm = algorithmFromHeader;
+    }
     bucketService.verifyBucketExists(bucketName);
-    var stream = objectService.verifyMd5(input, contentMd5, sha256Header);
+    objectService.verifyMd5(tempFileAndChecksum.getLeft(), contentMd5);
+
     //TODO: need to extract owner from headers
     var owner = Owner.DEFAULT_OWNER;
     var s3ObjectMetadata =
         objectService.putS3Object(bucketName,
             key.key(),
             mediaTypeFrom(contentType).toString(),
             storeHeadersFrom(httpHeaders),
-            stream,
-            isV4ChunkedWithSigningEnabled(sha256Header),
+            tempFileAndChecksum.getLeft(),
             userMetadataFrom(httpHeaders),
             encryptionHeadersFrom(httpHeaders),
             tags,
-            checksumAlgorithmFrom(httpHeaders),
-            checksumFrom(httpHeaders),
+            checksumAlgorithm,
+            checksum,
             owner,
             storageClass);
 
+    FileUtils.deleteQuietly(tempFileAndChecksum.getLeft().toFile());
+
     //return version id
     return ResponseEntity
         .ok()

server/src/main/java/com/adobe/testing/s3mock/S3Exception.java

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2017-2023 Adobe.
+ *  Copyright 2017-2024 Adobe.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -99,6 +99,10 @@ public class S3Exception extends RuntimeException {
   public static final S3Exception BAD_REQUEST_CONTENT =
       new S3Exception(BAD_REQUEST.value(), "UnexpectedContent",
           "This request contains unsupported content.");
+  public static final S3Exception BAD_DIGEST =
+      new S3Exception(BAD_REQUEST.value(), "BadDigest",
+          "The Content-MD5 or checksum value that you specified did "
+              + "not match what the server received.");
   private final int status;
   private final String code;
   private final String message;

server/src/main/java/com/adobe/testing/s3mock/dto/ChecksumAlgorithm.java

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2017-2023 Adobe.
+ *  Copyright 2017-2024 Adobe.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -18,6 +18,7 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
+import software.amazon.awssdk.core.checksums.Algorithm;
 
 public enum ChecksumAlgorithm {
   CRC32("CRC32"),
@@ -42,6 +43,15 @@ public static ChecksumAlgorithm fromString(String value) {
     };
   }
 
+  public Algorithm toAlgorithm() {
+    return switch (this) {
+      case CRC32 -> Algorithm.CRC32;
+      case CRC32C -> Algorithm.CRC32C;
+      case SHA1 -> Algorithm.SHA1;
+      case SHA256 -> Algorithm.SHA256;
+    };
+  }
+
   @Override
   @JsonValue
   public String toString() {