Add support for checksums with signing for putObject / getObject

Signed requests including checksums need special handling.

Fixes #1123

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEndcodingITV2.kt

@@ -33,37 +33,83 @@ import java.io.InputStream
  */
 internal class AwsChunkedEndcodingITV2 : S3TestBase() {
 
-    private val client = createS3ClientV2(serviceEndpointHttp)
+  private val client = createS3ClientV2(serviceEndpointHttp)
 
-    /**
-     * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding
-     * was actually used for the putObject request.
-     * This was manually validated through the debugger.
-     */
-    @Test
-    @S3VerifiedFailure(year = 2023,
-        reason = "Only works with http endpoints")
-    fun testPutObject_etagCreation(testInfo: TestInfo) {
-        val bucket = givenBucketV2(testInfo)
-        val uploadFile = File(UPLOAD_FILE_NAME)
-        val uploadFileIs: InputStream = FileInputStream(uploadFile)
-        val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\""
+  /**
+   * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding
+   * was actually used for the putObject request.
+   * This was manually validated through the debugger.
+   */
+  @Test
+  @S3VerifiedFailure(
+    year = 2023,
+    reason = "Only works with http endpoints"
+  )
+  fun testPutObject_checksum(testInfo: TestInfo) {
+    val bucket = givenBucketV2(testInfo)
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val uploadFileIs: InputStream = FileInputStream(uploadFile)
+    val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\""
+    val expectedChecksum = "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE="
 
-        client.putObject(
-            PutObjectRequest.builder()
-                .bucket(bucket)
-                .key(UPLOAD_FILE_NAME)
-              .checksumAlgorithm(ChecksumAlgorithm.SHA256)
-                .build(),
-            RequestBody.fromFile(uploadFile))
+    val putObjectResponse = client.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .checksumAlgorithm(ChecksumAlgorithm.SHA256)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
 
-        val getObjectResponse = client.getObject(
-            GetObjectRequest.builder()
-                .bucket(bucket)
-                .key(UPLOAD_FILE_NAME)
-                .build()
-        )
-        assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
-        assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
-    }
+    val putChecksum = putObjectResponse.checksumSHA256()
+    assertThat(putChecksum).isNotBlank
+    assertThat(putChecksum).isEqualTo(expectedChecksum)
+
+    val getObjectResponse = client.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
+    assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
+
+    val getChecksum = getObjectResponse.response().checksumSHA256()
+    assertThat(getChecksum).isNotBlank
+    assertThat(getChecksum).isEqualTo(expectedChecksum)
+  }
+
+  /**
+   * Unfortunately the S3 API does not persist or return data that would let us verify if signed and chunked encoding
+   * was actually used for the putObject request.
+   * This was manually validated through the debugger.
+   */
+  @Test
+  @S3VerifiedFailure(
+    year = 2023,
+    reason = "Only works with http endpoints"
+  )
+  fun testPutObject_etagCreation(testInfo: TestInfo) {
+    val bucket = givenBucketV2(testInfo)
+    val uploadFile = File(UPLOAD_FILE_NAME)
+    val uploadFileIs: InputStream = FileInputStream(uploadFile)
+    val expectedEtag = "\"${DigestUtil.hexDigest(uploadFileIs)}\""
+
+    client.putObject(
+      PutObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .build(),
+      RequestBody.fromFile(uploadFile)
+    )
+
+    val getObjectResponse = client.getObject(
+      GetObjectRequest.builder()
+        .bucket(bucket)
+        .key(UPLOAD_FILE_NAME)
+        .build()
+    )
+    assertThat(getObjectResponse.response().eTag()).isEqualTo(expectedEtag)
+    assertThat(getObjectResponse.response().contentLength()).isEqualTo(uploadFile.length())
+  }
 }

server/src/main/java/com/adobe/testing/s3mock/store/MultipartStore.java

@@ -184,7 +184,7 @@ public String putPart(BucketMetadata bucket,
       boolean useV4ChunkedWithSigningFormat,
       Map<String, String> encryptionHeaders) {
     File file = objectStore.inputStreamToFile(
-        objectStore.wrapStream(inputStream, useV4ChunkedWithSigningFormat, null, null),
+        objectStore.wrapStream(inputStream, useV4ChunkedWithSigningFormat, false),
         getPartPath(bucket, id, uploadId, partNumber)
     );
 

server/src/main/java/com/adobe/testing/s3mock/store/ObjectStore.java

@@ -32,6 +32,7 @@
 import com.adobe.testing.s3mock.dto.Retention;
 import com.adobe.testing.s3mock.dto.Tag;
 import com.adobe.testing.s3mock.util.AwsChecksumInputStream;
+import com.adobe.testing.s3mock.util.AwsChunkedDecodingChecksumInputStream;
 import com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.File;
@@ -129,8 +130,9 @@ public S3ObjectMetadata storeS3ObjectMetadata(BucketMetadata bucket,
     lockStore.putIfAbsent(id, new Object());
     synchronized (lockStore.get(id)) {
       createObjectRootFolder(bucket, id);
-      InputStream inputStream = wrapStream(dataStream, useV4ChunkedWithSigningFormat,
-          checksum, checksumAlgorithm);
+      boolean checksumEmbedded = checksumAlgorithm != null && checksum == null;
+      InputStream inputStream =
+          wrapStream(dataStream, useV4ChunkedWithSigningFormat, checksumEmbedded);
       File dataFile =
           inputStreamToFile(inputStream,
               getDataFilePath(bucket, id));
@@ -383,10 +385,12 @@ File inputStreamToFile(InputStream inputStream, Path filePath) {
   }
 
   InputStream wrapStream(InputStream dataStream, boolean useV4ChunkedWithSigningFormat,
-                         String checksum, ChecksumAlgorithm checksumAlgorithm) {
-    if (useV4ChunkedWithSigningFormat) {
+                         boolean checksumEbedded) {
+    if (useV4ChunkedWithSigningFormat && checksumEbedded) {
+      return new AwsChunkedDecodingChecksumInputStream(dataStream);
+    } else if (useV4ChunkedWithSigningFormat) {
       return new AwsChunkedDecodingInputStream(dataStream);
-    } else if (checksumAlgorithm != null && checksum == null) {
+    } else if (checksumEbedded) {
       return new AwsChecksumInputStream(dataStream);
     } else {
       return dataStream;

server/src/main/java/com/adobe/testing/s3mock/util/AwsChecksumInputStream.java

@@ -70,6 +70,12 @@ public int read() throws IOException {
     }
 
     //read the last lines which contain the algorithm and the checksum
+    extractAlgorithmAndChecksum();
+
+    return -1;
+  }
+
+  protected void extractAlgorithmAndChecksum() throws IOException {
     if (algorithm == null && checksum == null) {
       readUntil(CHECKSUM_HEADER);
       byte[] typeAndChecksum = readUntil(CRLF);
@@ -79,8 +85,6 @@ public int read() throws IOException {
       algorithm = ChecksumAlgorithm.fromString(type);
       checksum = split[1];
     }
-
-    return -1;
   }
 
   public String getChecksum() {

server/src/main/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStream.java

@@ -0,0 +1,50 @@
+/*
+ *  Copyright 2017-2023 Adobe.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class AwsChunkedDecodingChecksumInputStream extends AwsChecksumInputStream {
+
+  public AwsChunkedDecodingChecksumInputStream(InputStream source) {
+    super(source);
+  }
+
+  @Override
+  public int read() throws IOException {
+    if (payloadLength == 0L) {
+      final byte[] hexLengthBytes = readUntil(DELIMITER);
+      if (hexLengthBytes.length == 0) {
+        return -1;
+      }
+
+      setPayloadLength(hexLengthBytes);
+
+      if (payloadLength == 0L) {
+        extractAlgorithmAndChecksum();
+        return -1;
+      }
+
+      readUntil(CRLF);
+    }
+
+    payloadLength--;
+
+    return source.read();
+  }
+}

server/src/main/java/com/adobe/testing/s3mock/util/HeaderUtil.java

@@ -48,6 +48,8 @@ public final class HeaderUtil {
   private static final String HEADER_X_AMZ_META_PREFIX = "x-amz-meta-";
   private static final String STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD =
       "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
+  private static final String STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD_TRAILER =
+      "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
   private static final MediaType FALLBACK_MEDIA_TYPE = new MediaType("binary", "octet-stream");
 
   /**
@@ -127,7 +129,9 @@ && isNotBlank(entry.getValue().get(0))) {
   }
 
   public static boolean isV4ChunkedWithSigningEnabled(final String sha256Header) {
-    return sha256Header != null && sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD);
+    return sha256Header != null
+        && (sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD)
+            || sha256Header.equals(STREAMING_AWS_4_HMAC_SHA_256_PAYLOAD_TRAILER));
   }
 
   public static MediaType parseMediaType(final String contentType) {

server/src/test/java/com/adobe/testing/s3mock/util/AwsChunkedDecodingChecksumInputStreamTest.java

@@ -0,0 +1,68 @@
+/*
+ *  Copyright 2017-2023 Adobe.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.adobe.testing.s3mock.util;
+
+import static com.adobe.testing.s3mock.dto.ChecksumAlgorithm.SHA256;
+import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_CHECKSUM_SHA256;
+import static com.adobe.testing.s3mock.util.TestUtil.getFileFromClasspath;
+import static com.adobe.testing.s3mock.util.TestUtil.getPayloadFile;
+import static com.adobe.testing.s3mock.util.TestUtil.getTestFile;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import com.adobe.testing.s3mock.dto.ChecksumAlgorithm;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsS3V4ChunkSigner;
+import software.amazon.awssdk.auth.signer.internal.chunkedencoding.AwsSignedChunkedEncodingInputStream;
+import software.amazon.awssdk.core.checksums.Algorithm;
+import software.amazon.awssdk.core.checksums.SdkChecksum;
+
+class AwsChunkedDecodingChecksumInputStreamTest {
+
+  @Test
+  void testDecode(TestInfo testInfo) throws IOException {
+    doTest(testInfo, "sampleFile.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+        "1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=", SHA256);
+    doTest(testInfo, "sampleFile_large.txt", X_AMZ_CHECKSUM_SHA256, Algorithm.SHA256,
+        "BNNY15scjxN9jPa+AEJ6p08Cfm4BhYOXkhladr4QSTs=", SHA256);
+  }
+
+  void doTest(TestInfo testInfo, String fileName, String header, Algorithm algorithm,
+              String checksum, ChecksumAlgorithm checksumAlgorithm) throws IOException {
+    File sampleFile = getFileFromClasspath(testInfo, fileName);
+    InputStream chunkedEncodingInputStream = AwsSignedChunkedEncodingInputStream
+        .builder()
+        .inputStream(Files.newInputStream(sampleFile.toPath()))
+        .sdkChecksum(SdkChecksum.forAlgorithm(algorithm))
+        .checksumHeaderForTrailer(header)
+        .awsChunkSigner(new AwsS3V4ChunkSigner("signingKey".getBytes(),
+            "dateTime",
+            "keyPath"))
+        .build();
+    AwsChunkedDecodingChecksumInputStream iut = new
+        AwsChunkedDecodingChecksumInputStream(chunkedEncodingInputStream);
+    assertThat(iut).hasSameContentAs(Files.newInputStream(sampleFile.toPath()));
+    assertThat(iut.getChecksum()).isEqualTo(checksum);
+    assertThat(iut.getAlgorithm()).isEqualTo(checksumAlgorithm);
+  }
+}

server/src/test/resources/com/adobe/testing/s3mock/util/sampleFile_chunkEncoded_checksumSha256

@@ -0,0 +1,8 @@
+24;chunk-signature=2ce83f44c5a2bf95d759e5ec2515c2114b523212e3b9014b7b0080ec3ab67b83
+## sample test file ##
+
+demo=content
+0;chunk-signature=e77862ed6e82662dba1a08d16bfc43bfd218b5d4c319e99c3e4c4d45a156556b
+x-amz-checksum-sha256:1VcEifAruhjVvjzul4sC0B1EmlUdzqvsp6BP0KSVdTE=
+x-amz-trailer-signature:603d2d124500656602e098897dc70f427d508559be18aee70d7fc5102a3dd276
+