diff --git a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java index b748f50d7..9f7a13e1e 100644 --- a/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3AsyncEncryptionClient.java @@ -79,7 +79,7 @@ public class S3AsyncEncryptionClient extends DelegatingS3AsyncClient { private final boolean _enableDelayedAuthenticationMode; private final boolean _enableMultipartPutObject; private final long _bufferSize; - private InstructionFileConfig _instructionFileConfig; + private final InstructionFileConfig _instructionFileConfig; private S3AsyncEncryptionClient(Builder builder) { super(builder._wrappedClient); @@ -151,6 +151,7 @@ public CompletableFuture putObject(PutObjectRequest putObject .s3AsyncClient(_wrappedClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); return pipeline.putObject(putObjectRequest, requestBody); @@ -169,6 +170,7 @@ private CompletableFuture multipartPutObject(PutObjectRequest .s3AsyncClient(mpuClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); // Ensures parts are not retried to avoid corrupting ciphertext AsyncRequestBody noRetryBody = new NoRetriesAsyncRequestBody(requestBody); @@ -289,6 +291,7 @@ public void close() { _instructionFileConfig.closeClient(); } + // This is very similar to the S3EncryptionClient builder // Make sure to keep both clients in mind when adding new builder options public static class Builder implements S3AsyncClientBuilder { diff --git a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java index 3aae26a80..0c7f12a7b 100644 --- a/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java +++ b/src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java @@ -202,6 +202,7 @@ public PutObjectResponse putObject(PutObjectRequest putObjectRequest, RequestBod .s3AsyncClient(_wrappedAsyncClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); @@ -1164,6 +1165,7 @@ public S3EncryptionClient build() { .s3AsyncClient(_wrappedAsyncClient) .cryptoMaterialsManager(_cryptoMaterialsManager) .secureRandom(_secureRandom) + .instructionFileConfig(_instructionFileConfig) .build(); return new S3EncryptionClient(this); diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java index cde853a48..0ab055873 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataDecodingStrategy.java @@ -169,7 +169,6 @@ private ContentMetadata readFromMap(Map metadata, GetObjectRespo public ContentMetadata decode(GetObjectRequest request, GetObjectResponse response) { Map metadata = response.metadata(); - ContentMetadataDecodingStrategy strategy; if (metadata != null && metadata.containsKey(MetadataKeyConstants.CONTENT_IV) && (metadata.containsKey(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V1) diff --git a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java index 3045ae0eb..cf4e86fa7 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ContentMetadataEncodingStrategy.java @@ -2,13 +2,94 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.encryption.s3.internal; +import software.amazon.awssdk.protocols.jsoncore.JsonWriter; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.encryption.s3.S3EncryptionClientException; +import software.amazon.encryption.s3.materials.EncryptedDataKey; import software.amazon.encryption.s3.materials.EncryptionMaterials; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; import java.util.Map; -@FunctionalInterface -public interface ContentMetadataEncodingStrategy { +public class ContentMetadataEncodingStrategy { - Map encodeMetadata(EncryptionMaterials materials, byte[] iv, - Map metadata); + private static final Base64.Encoder ENCODER = Base64.getEncoder(); + private final InstructionFileConfig _instructionFileConfig; + + public ContentMetadataEncodingStrategy(InstructionFileConfig instructionFileConfig) { + _instructionFileConfig = instructionFileConfig; + } + + public PutObjectRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, PutObjectRequest putObjectRequest) { + if (_instructionFileConfig.isInstructionFilePutEnabled()) { + final String metadataString = metadataToString(materials, iv); + _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString); + // the original request object is returned as-is + return putObjectRequest; + } else { + Map newMetadata = addMetadataToMap(putObjectRequest.metadata(), materials, iv); + return putObjectRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + } + + public CreateMultipartUploadRequest encodeMetadata(EncryptionMaterials materials, byte[] iv, CreateMultipartUploadRequest createMultipartUploadRequest) { + if(_instructionFileConfig.isInstructionFilePutEnabled()) { + final String metadataString = metadataToString(materials, iv); + PutObjectRequest putObjectRequest = ConvertSDKRequests.convertRequest(createMultipartUploadRequest); + _instructionFileConfig.putInstructionFile(putObjectRequest, metadataString); + // the original request object is returned as-is + return createMultipartUploadRequest; + } else { + Map newMetadata = addMetadataToMap(createMultipartUploadRequest.metadata(), materials, iv); + return createMultipartUploadRequest.toBuilder() + .metadata(newMetadata) + .build(); + } + } + private String metadataToString(EncryptionMaterials materials, byte[] iv) { + // this is just the metadata map serialized as JSON + // so first get the Map + final Map metadataMap = addMetadataToMap(new HashMap<>(), materials, iv); + // then serialize it + try (JsonWriter jsonWriter = JsonWriter.create()) { + jsonWriter.writeStartObject(); + for (Map.Entry entry : metadataMap.entrySet()) { + jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); + } + jsonWriter.writeEndObject(); + + return new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); + } catch (JsonWriter.JsonGenerationException e) { + throw new S3EncryptionClientException("Cannot serialize materials to JSON.", e); + } + } + + private Map addMetadataToMap(Map map, EncryptionMaterials materials, byte[] iv) { + Map metadata = new HashMap<>(map); + EncryptedDataKey edk = materials.encryptedDataKeys().get(0); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey())); + metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv)); + metadata.put(MetadataKeyConstants.CONTENT_CIPHER, materials.algorithmSuite().cipherName()); + metadata.put(MetadataKeyConstants.CONTENT_CIPHER_TAG_LENGTH, Integer.toString(materials.algorithmSuite().cipherTagLengthBits())); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_ALGORITHM, new String(edk.keyProviderInfo(), StandardCharsets.UTF_8)); + + try (JsonWriter jsonWriter = JsonWriter.create()) { + jsonWriter.writeStartObject(); + for (Map.Entry entry : materials.encryptionContext().entrySet()) { + jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); + } + jsonWriter.writeEndObject(); + + String jsonEncryptionContext = new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); + metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, jsonEncryptionContext); + } catch (JsonWriter.JsonGenerationException e) { + throw new S3EncryptionClientException("Cannot serialize encryption context to JSON.", e); + } + return metadata; + } } diff --git a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java index 55d86ec3f..e4c72a2a6 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java +++ b/src/main/java/software/amazon/encryption/s3/internal/ConvertSDKRequests.java @@ -4,7 +4,6 @@ import java.util.Map; import org.apache.commons.logging.LogFactory; -import software.amazon.awssdk.services.s3.model.ChecksumType; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -12,8 +11,149 @@ public class ConvertSDKRequests { + /** + * Converts a CreateMultipartUploadRequest to a PutObjectRequest. This conversion is necessary when + * Instruction File PutObject is enabled and a multipart upload is performed.The method copies all the + * relevant fields from the CreateMultipartUploadRequest to the PutObjectRequest. + * @param request The CreateMultipartUploadRequest to convert + * @return The converted PutObjectRequest + * @throws IllegalArgumentException if the request contains an invalid field + */ + public static PutObjectRequest convertRequest(CreateMultipartUploadRequest request) { + final PutObjectRequest.Builder output = PutObjectRequest.builder(); + request + .toBuilder() + .sdkFields() + .forEach(f -> { + final Object value = f.getValueOrDefault(request); + if (value != null) { + switch (f.memberName()) { + case "ACL": + output.acl((String) value); + break; + case "Bucket": + output.bucket((String) value); + break; + case "BucketKeyEnabled": + output.bucketKeyEnabled((Boolean) value); + break; + case "CacheControl": + output.cacheControl((String) value); + break; + case "ChecksumAlgorithm": + output.checksumAlgorithm((String) value); + break; + case "ContentDisposition": + assert value instanceof String; + output.contentDisposition((String) value); + break; + case "ContentEncoding": + output.contentEncoding((String) value); + break; + case "ContentLanguage": + output.contentLanguage((String) value); + break; + case "ContentType": + output.contentType((String) value); + break; + case "ExpectedBucketOwner": + output.expectedBucketOwner((String) value); + break; + case "Expires": + output.expires((Instant) value); + break; + case "GrantFullControl": + output.grantFullControl((String) value); + break; + case "GrantRead": + output.grantRead((String) value); + break; + case "GrantReadACP": + output.grantReadACP((String) value); + break; + case "GrantWriteACP": + output.grantWriteACP((String) value); + break; + case "Key": + output.key((String) value); + break; + case "Metadata": + if (!isStringStringMap(value)) { + throw new IllegalArgumentException("Metadata must be a Map"); + } + @SuppressWarnings("unchecked") + Map metadata = (Map) value; + output.metadata(metadata); + break; + case "ObjectLockLegalHoldStatus": + output.objectLockLegalHoldStatus((String) value); + break; + case "ObjectLockMode": + output.objectLockMode((String) value); + break; + case "ObjectLockRetainUntilDate": + output.objectLockRetainUntilDate((Instant) value); + break; + case "RequestPayer": + output.requestPayer((String) value); + break; + case "ServerSideEncryption": + output.serverSideEncryption((String) value); + break; + case "SSECustomerAlgorithm": + output.sseCustomerAlgorithm((String) value); + break; + case "SSECustomerKey": + output.sseCustomerKey((String) value); + break; + case "SSECustomerKeyMD5": + output.sseCustomerKeyMD5((String) value); + break; + case "SSEKMSEncryptionContext": + output.ssekmsEncryptionContext((String) value); + break; + case "SSEKMSKeyId": + output.ssekmsKeyId((String) value); + break; + case "StorageClass": + output.storageClass((String) value); + break; + case "Tagging": + output.tagging((String) value); + break; + case "WebsiteRedirectLocation": + output.websiteRedirectLocation((String) value); + break; + default: + // Rather than silently dropping the value, + // we loudly signal that we don't know how to handle this field. + throw new IllegalArgumentException( + f.memberName() + " is an unknown field. " + + "The S3 Encryption Client does not recognize this option and cannot set it on the PutObjectRequest." + + "This may be a new S3 feature." + + "Please report this to the Amazon S3 Encryption Client for Java: " + + "https://github.com/aws/amazon-s3-encryption-client-java/issues." + + "To work around this issue, you can disable Instruction File on PutObject or disable" + + "multi part upload, or use the Async client, or not set this value on PutObject." + + "You may be able to update this value after the PutObject request completes." + ); + } + } + }); + return output + // OverrideConfiguration is not as SDKField but still needs to be supported + .overrideConfiguration(request.overrideConfiguration().orElse(null)) + .build(); + } + /** + * Converts a PutObjectRequest to CreateMultipartUploadRequest.This conversion is necessary to convert an + * original PutObjectRequest into a CreateMultipartUploadRequest to initiate the + * multipart upload while maintaining the original request's configuration. + * @param request The PutObjectRequest to convert + * @return The converted CreateMultipartUploadRequest + * @throws IllegalArgumentException if the request contains an invalid field + */ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest request) { - final CreateMultipartUploadRequest.Builder output = CreateMultipartUploadRequest.builder(); request .toBuilder() @@ -37,8 +177,6 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque case "ChecksumAlgorithm": output.checksumAlgorithm((String) value); break; - case "ChecksumType": - output.checksumType((ChecksumType) value); case "ContentDisposition": assert value instanceof String; output.contentDisposition((String) value); @@ -107,6 +245,9 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque case "SSECustomerKey": output.sseCustomerKey((String) value); break; + case "SSECustomerKeyMD5": + output.sseCustomerKeyMD5((String) value); + break; case "SSEKMSEncryptionContext": output.ssekmsEncryptionContext((String) value); break; @@ -126,7 +267,7 @@ public static CreateMultipartUploadRequest convertRequest(PutObjectRequest reque // Rather than silently dropping the value, // we loudly signal that we don't know how to handle this field. throw new IllegalArgumentException( - f.locationName() + " is an unknown field. " + + f.memberName() + " is an unknown field. " + "The S3 Encryption Client does not recognize this option and cannot set it on the CreateMultipartUploadRequest." + "This may be a new S3 feature." + "Please report this to the Amazon S3 Encryption Client for Java: " + diff --git a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java index 4b70a15d1..59cccc788 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java +++ b/src/main/java/software/amazon/encryption/s3/internal/InstructionFileConfig.java @@ -1,13 +1,23 @@ package software.amazon.encryption.s3.internal; import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.encryption.s3.S3EncryptionClientException; +import java.util.HashMap; +import java.util.Map; + +import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX; +import static software.amazon.encryption.s3.internal.MetadataKeyConstants.INSTRUCTION_FILE; + /** * Provides configuration options for instruction file behaviors. */ @@ -16,13 +26,14 @@ public class InstructionFileConfig { final private InstructionFileClientType _clientType; final private S3AsyncClient _s3AsyncClient; final private S3Client _s3Client; + final private boolean _enableInstructionFilePut; private InstructionFileConfig(final Builder builder) { _clientType = builder._clientType; _s3Client = builder._s3Client; _s3AsyncClient = builder._s3AsyncClient; + _enableInstructionFilePut = builder._enableInstructionFilePut; } - public static Builder builder() { return new Builder(); } @@ -33,6 +44,42 @@ public enum InstructionFileClientType { ASYNC } + boolean isInstructionFilePutEnabled() { + return _enableInstructionFilePut; + } + + PutObjectResponse putInstructionFile(PutObjectRequest request, String instructionFileContent) { + // This shouldn't happen in practice because the metadata strategy will evaluate + // if instruction file Puts are enabled before calling this method; check again anyway for robustness + if (!_enableInstructionFilePut) { + throw new S3EncryptionClientException("Enable Instruction File Put must be set to true in order to call PutObject with an instruction file!"); + } + + // Instruction file DOES NOT contain the same metadata as the actual object + Map instFileMetadata = new HashMap<>(1); + // It contains a key with no value identifying it as an instruction file + instFileMetadata.put(INSTRUCTION_FILE, ""); + + // In a future release, non-default suffixes will be supported. + // Use toBuilder to keep all other fields the same as the actual request + final PutObjectRequest instPutRequest = request.toBuilder() + .key(request.key() + INSTRUCTION_FILE_SUFFIX) + .metadata(instFileMetadata) + .build(); + switch (_clientType) { + case SYNCHRONOUS: + return _s3Client.putObject(instPutRequest, RequestBody.fromString(instructionFileContent)); + case ASYNC: + return _s3AsyncClient.putObject(instPutRequest, AsyncRequestBody.fromString(instructionFileContent)).join(); + case DISABLED: + // this should never happen because we check enablePut first + throw new S3EncryptionClientException("Instruction File has been disabled!"); + default: + // this should never happen + throw new S3EncryptionClientException("Unknown Instruction File Type"); + } + } + ResponseInputStream getInstructionFile(GetObjectRequest request) { switch (_clientType) { case SYNCHRONOUS: @@ -64,6 +111,7 @@ public static class Builder { private boolean _disableInstructionFile; private S3AsyncClient _s3AsyncClient; private S3Client _s3Client; + private boolean _enableInstructionFilePut; /** * When set to true, the S3 Encryption Client will not attempt to get instruction files. @@ -75,6 +123,11 @@ public Builder disableInstructionFile(boolean disableInstructionFile) { return this; } + public Builder enableInstructionFilePutObject(boolean enableInstructionFilePutObject) { + _enableInstructionFilePut = enableInstructionFilePutObject; + return this; + } + /** * Sets the S3 client to use to retrieve instruction files. * @param instructionFileClient @@ -94,6 +147,7 @@ public Builder instructionFileAsyncClient(S3AsyncClient instructionFileAsyncClie _s3AsyncClient = instructionFileAsyncClient; return this; } + public InstructionFileConfig build() { if ((_s3AsyncClient != null || _s3Client != null) && _disableInstructionFile) { throw new S3EncryptionClientException("Instruction Files have been disabled but a client has been passed!"); @@ -101,6 +155,9 @@ public InstructionFileConfig build() { if (_disableInstructionFile) { // We know both clients are null, so carry on. this._clientType = InstructionFileClientType.DISABLED; + if (_enableInstructionFilePut) { + throw new S3EncryptionClientException("Instruction Files must be enabled to enable Instruction Files for PutObject."); + } return new InstructionFileConfig(this); } if (_s3Client != null && _s3AsyncClient != null) { diff --git a/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java b/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java index 12c0c856f..722d3c48e 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MetadataKeyConstants.java @@ -13,4 +13,6 @@ public class MetadataKeyConstants { // This is usually an actual Java cipher e.g. AES/GCM/NoPadding public static final String CONTENT_CIPHER = "x-amz-cek-alg"; public static final String CONTENT_CIPHER_TAG_LENGTH = "x-amz-tag-len"; + // Only used in instruction files to identify them as such + public static final String INSTRUCTION_FILE = "x-amz-crypto-instr-file"; } diff --git a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java index b24d6d499..cb4dde03a 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/MultipartUploadObjectPipeline.java @@ -42,6 +42,7 @@ public class MultipartUploadObjectPipeline { final private CryptographicMaterialsManager _cryptoMaterialsManager; final private MultipartContentEncryptionStrategy _contentEncryptionStrategy; final private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; + final private InstructionFileConfig _instructionFileConfig; /** * Map of data about in progress encrypted multipart uploads. */ @@ -53,6 +54,7 @@ private MultipartUploadObjectPipeline(Builder builder) { this._contentEncryptionStrategy = builder._contentEncryptionStrategy; this._contentMetadataEncodingStrategy = builder._contentMetadataEncodingStrategy; this._multipartUploadMaterials = builder._multipartUploadMaterials; + this._instructionFileConfig = builder._instructionFileConfig; } public static Builder builder() { @@ -67,11 +69,10 @@ public CreateMultipartUploadResponse createMultipartUpload(CreateMultipartUpload MultipartEncryptedContent encryptedContent = _contentEncryptionStrategy.initMultipartEncryption(materials); - Map metadata = new HashMap<>(request.metadata()); - metadata = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), metadata); - request = request.toBuilder() + CreateMultipartUploadRequest createMpuRequest = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), request); + request = createMpuRequest.toBuilder() .overrideConfiguration(API_NAME_INTERCEPTOR) - .metadata(metadata).build(); + .build(); CreateMultipartUploadResponse response = _s3AsyncClient.createMultipartUpload(request).join(); @@ -217,12 +218,13 @@ public void putLocalObject(RequestBody requestBody, String uploadId, OutputStrea public static class Builder { private final Map _multipartUploadMaterials = Collections.synchronizedMap(new HashMap<>()); - private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ObjectMetadataEncodingStrategy(); + private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; private S3AsyncClient _s3AsyncClient; private CryptographicMaterialsManager _cryptoMaterialsManager; private SecureRandom _secureRandom; // To Create Cipher which is used in during uploadPart requests. private MultipartContentEncryptionStrategy _contentEncryptionStrategy; + private InstructionFileConfig _instructionFileConfig; private Builder() { } @@ -247,6 +249,11 @@ public Builder secureRandom(SecureRandom secureRandom) { return this; } + public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) { + this._instructionFileConfig = instructionFileConfig; + return this; + } + public MultipartUploadObjectPipeline build() { // Default to AesGcm since it is the only active (non-legacy) content encryption strategy if (_contentEncryptionStrategy == null) { @@ -255,6 +262,10 @@ public MultipartUploadObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + if(_instructionFileConfig == null) { + _instructionFileConfig = InstructionFileConfig.builder().build(); + } + _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); return new MultipartUploadObjectPipeline(this); } } diff --git a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java b/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java deleted file mode 100644 index c0f0d5a0c..000000000 --- a/src/main/java/software/amazon/encryption/s3/internal/ObjectMetadataEncodingStrategy.java +++ /dev/null @@ -1,41 +0,0 @@ -package software.amazon.encryption.s3.internal; - -import software.amazon.awssdk.protocols.jsoncore.JsonWriter; -import software.amazon.encryption.s3.S3EncryptionClientException; -import software.amazon.encryption.s3.materials.EncryptedDataKey; -import software.amazon.encryption.s3.materials.EncryptionMaterials; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Map; - -public class ObjectMetadataEncodingStrategy implements ContentMetadataEncodingStrategy { - - private static final Base64.Encoder ENCODER = Base64.getEncoder(); - - @Override - public Map encodeMetadata(EncryptionMaterials materials, byte[] iv, - Map metadata) { - EncryptedDataKey edk = materials.encryptedDataKeys().get(0); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_V2, ENCODER.encodeToString(edk.encryptedDatakey())); - metadata.put(MetadataKeyConstants.CONTENT_IV, ENCODER.encodeToString(iv)); - metadata.put(MetadataKeyConstants.CONTENT_CIPHER, materials.algorithmSuite().cipherName()); - metadata.put(MetadataKeyConstants.CONTENT_CIPHER_TAG_LENGTH, Integer.toString(materials.algorithmSuite().cipherTagLengthBits())); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_ALGORITHM, new String(edk.keyProviderInfo(), StandardCharsets.UTF_8)); - - try (JsonWriter jsonWriter = JsonWriter.create()) { - jsonWriter.writeStartObject(); - for (Map.Entry entry : materials.encryptionContext().entrySet()) { - jsonWriter.writeFieldName(entry.getKey()).writeValue(entry.getValue()); - } - jsonWriter.writeEndObject(); - - String jsonEncryptionContext = new String(jsonWriter.getBytes(), StandardCharsets.UTF_8); - metadata.put(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, jsonEncryptionContext); - } catch (JsonWriter.JsonGenerationException e) { - throw new S3EncryptionClientException("Cannot serialize encryption context to JSON.", e); - } - return metadata; - } - -} diff --git a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java index 3f5f29979..a311e210c 100644 --- a/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java +++ b/src/main/java/software/amazon/encryption/s3/internal/PutEncryptedObjectPipeline.java @@ -14,8 +14,6 @@ import software.amazon.encryption.s3.materials.EncryptionMaterialsRequest; import java.security.SecureRandom; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static software.amazon.encryption.s3.internal.ApiNameVersion.API_NAME_INTERCEPTOR; @@ -26,7 +24,7 @@ public class PutEncryptedObjectPipeline { final private CryptographicMaterialsManager _cryptoMaterialsManager; final private AsyncContentEncryptionStrategy _asyncContentEncryptionStrategy; final private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; - + final private InstructionFileConfig _instructionFileConfig; public static Builder builder() { return new Builder(); } @@ -36,6 +34,7 @@ private PutEncryptedObjectPipeline(Builder builder) { this._cryptoMaterialsManager = builder._cryptoMaterialsManager; this._asyncContentEncryptionStrategy = builder._asyncContentEncryptionStrategy; this._contentMetadataEncodingStrategy = builder._contentMetadataEncodingStrategy; + this._instructionFileConfig = builder._instructionFileConfig; } public CompletableFuture putObject(PutObjectRequest request, AsyncRequestBody requestBody) { @@ -70,12 +69,10 @@ public CompletableFuture putObject(PutObjectRequest request, EncryptedContent encryptedContent = _asyncContentEncryptionStrategy.encryptContent(materials, requestBody); - Map metadata = new HashMap<>(request.metadata()); - metadata = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), metadata); - PutObjectRequest encryptedPutRequest = request.toBuilder() + PutObjectRequest modifiedRequest = _contentMetadataEncodingStrategy.encodeMetadata(materials, encryptedContent.getIv(), request); + PutObjectRequest encryptedPutRequest = modifiedRequest.toBuilder() .overrideConfiguration(API_NAME_INTERCEPTOR) .contentLength(encryptedContent.getCiphertextLength()) - .metadata(metadata) .build(); return _s3AsyncClient.putObject(encryptedPutRequest, encryptedContent.getAsyncCiphertext()); } @@ -85,7 +82,8 @@ public static class Builder { private CryptographicMaterialsManager _cryptoMaterialsManager; private SecureRandom _secureRandom; private AsyncContentEncryptionStrategy _asyncContentEncryptionStrategy; - private final ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy = new ObjectMetadataEncodingStrategy(); + private InstructionFileConfig _instructionFileConfig; + private ContentMetadataEncodingStrategy _contentMetadataEncodingStrategy; private Builder() { } @@ -110,6 +108,11 @@ public Builder secureRandom(SecureRandom secureRandom) { return this; } + public Builder instructionFileConfig(InstructionFileConfig instructionFileConfig) { + this._instructionFileConfig = instructionFileConfig; + return this; + } + public PutEncryptedObjectPipeline build() { // Default to AesGcm since it is the only active (non-legacy) content encryption strategy if (_asyncContentEncryptionStrategy == null) { @@ -118,6 +121,11 @@ public PutEncryptedObjectPipeline build() { .secureRandom(_secureRandom) .build(); } + if(_instructionFileConfig == null) { + _instructionFileConfig = InstructionFileConfig.builder().build(); + } + _contentMetadataEncodingStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig); + return new PutEncryptedObjectPipeline(this); } } diff --git a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java index 698b95b8d..2668918b0 100644 --- a/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3AsyncEncryptionClientTest.java @@ -39,8 +39,10 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.StorageClass; import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.encryption.s3.internal.InstructionFileConfig; import software.amazon.encryption.s3.materials.KmsKeyring; @@ -66,6 +68,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -829,6 +832,98 @@ public void testAsyncInstructionFileConfig() { s3ClientDisabledInstructionFile.close(); s3Client.close(); } + @Test + public void testAsyncInstructionFileConfigMultipart() { + final String objectKey = appendTestSuffix("test-multipart-async-instruction-file-config"); + final String input = "SimpleTestOfV3EncryptionClient"; + + AwsCredentialsProvider credentials = DefaultCredentialsProvider.create(); + S3Client wrappedClient = S3Client.create(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .credentialsProvider(credentials) + .build(); + PutObjectRequest request = PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .build(); + CompletableFuture putObjectResponse = v3Client.putObject(request, AsyncRequestBody.fromString(input)); + putObjectResponse.join(); + + assertNotNull(putObjectResponse); + + ResponseBytes instructionFileResponse = wrappedClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertNotNull(instructionFileResponse); + assertTrue(instructionFileResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + CompletableFuture> futureGetObj = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncResponseTransformer.toBytes()); + ResponseBytes getResponse = futureGetObj.join(); + assertNotNull(getResponse); + assertEquals(input, getResponse.asUtf8String()); + + deleteObject(BUCKET, objectKey, v3Client); + + v3Client.close(); + } + @Test + public void testAsyncInstructionFileConfigMultipartWithOptions() { + final String objectKey = appendTestSuffix("test-multipart-async-instruction-file-config-options"); + final String input = "SimpleTestOfV3EncryptionClient"; + final StorageClass storageClass = StorageClass.STANDARD_IA; + + S3Client wrappedClient = S3Client.create(); + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .build(); + PutObjectRequest putObjectRequest = PutObjectRequest.builder() + .bucket(BUCKET) + .key(objectKey) + .storageClass(storageClass) + .build(); + + CompletableFuture putObjectResponse = v3Client.putObject(putObjectRequest, AsyncRequestBody.fromString(input)); + putObjectResponse.join(); + + ResponseBytes instructionFileResponse = wrappedClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + assertNotNull(instructionFileResponse); + Map metadata = instructionFileResponse.response().metadata(); + assertTrue(metadata.containsKey("x-amz-crypto-instr-file")); + + assertEquals(storageClass.toString(), instructionFileResponse.response().storageClassAsString()); + + CompletableFuture> futureGetObj = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), AsyncResponseTransformer.toBytes()); + ResponseBytes getResponse = futureGetObj.join(); + assertNotNull(getResponse); + assertEquals(input, getResponse.asUtf8String()); + + assertEquals(getResponse.response().storageClassAsString(), storageClass.toString()); + + deleteObject(BUCKET, objectKey, v3Client); + v3Client.close(); + + } @Test public void wrappedClientMultipartUploadThrowsException() throws IOException { diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java index 281880504..fb8dad2e9 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientCompatibilityTest.java @@ -15,9 +15,15 @@ import com.amazonaws.services.s3.model.EncryptedPutObjectRequest; import com.amazonaws.services.s3.model.EncryptionMaterials; import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; import com.amazonaws.services.s3.model.KMSEncryptionMaterials; import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.StorageClass; +import com.amazonaws.services.s3.model.UploadObjectRequest; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseBytes; @@ -28,20 +34,25 @@ import software.amazon.awssdk.services.s3.model.MetadataDirective; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.utils.BoundedInputStream; import javax.crypto.KeyGenerator; import javax.crypto.SecretKey; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ExecutionException; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static software.amazon.encryption.s3.S3EncryptionClient.builder; import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; @@ -563,6 +574,7 @@ public void KmsV1toV3() { CryptoConfiguration v1Config = new CryptoConfiguration(CryptoMode.AuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile) .withAwsKmsRegion(KMS_REGION); AmazonS3Encryption v1Client = AmazonS3EncryptionClient.encryptionBuilder() @@ -598,8 +610,10 @@ public void KmsContextV2toV3() { // V2 Client EncryptionMaterialsProvider materialsProvider = new KMSEncryptionMaterialsProvider(KMS_KEY_ID); + CryptoConfigurationV2 config = new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption); AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() .withEncryptionMaterialsProvider(materialsProvider) + .withCryptoConfiguration(config) .build(); // V3 Client diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java new file mode 100644 index 000000000..9ca5b8352 --- /dev/null +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientInstructionFileTest.java @@ -0,0 +1,462 @@ +package software.amazon.encryption.s3; + +import com.amazonaws.services.s3.AmazonS3EncryptionClientV2; +import com.amazonaws.services.s3.AmazonS3EncryptionV2; +import com.amazonaws.services.s3.model.CryptoConfigurationV2; +import com.amazonaws.services.s3.model.CryptoMode; +import com.amazonaws.services.s3.model.CryptoStorageMode; +import com.amazonaws.services.s3.model.EncryptionMaterials; +import com.amazonaws.services.s3.model.EncryptionMaterialsProvider; +import com.amazonaws.services.s3.model.KMSEncryptionMaterials; +import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.SdkPartType; +import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; +import software.amazon.encryption.s3.internal.InstructionFileConfig; +import software.amazon.encryption.s3.utils.BoundedInputStream; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static software.amazon.encryption.s3.S3EncryptionClient.withAdditionalConfiguration; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.BUCKET; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.KMS_KEY_ID; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.appendTestSuffix; +import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.deleteObject; + +public class S3EncryptionClientInstructionFileTest { + + @Test + public void testInstructionFileExists() { + final String objectKey = appendTestSuffix("instruction-file-put-object"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + // Ensure its metadata identifies it as such + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + // Ensure decryption succeeds + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + defaultClient.close(); + } + + @Test + public void testDisabledClientFails() { + final String objectKey = appendTestSuffix("instruction-file-put-object-disabled-fails"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + // Put with Instruction File + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Disabled client should fail + S3Client s3ClientDisabledInstructionFile = S3EncryptionClient.builder() + .wrappedClient(wrappedClient) + .instructionFileConfig(InstructionFileConfig.builder() + .disableInstructionFile(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + try { + s3ClientDisabledInstructionFile.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + fail("expected exception"); + } catch (S3EncryptionClientException exception) { + assertTrue(exception.getMessage().contains("Exception encountered while fetching Instruction File.")); + } + + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + s3ClientDisabledInstructionFile.close(); + } + + + /** + * This test is somewhat redundant given deletion itself is tested in + * e.g. deleteObjectWithInstructionFileSuccess, but is included anyway to be thorough + */ + @Test + public void testInstructionFileDelete() { + final String objectKey = appendTestSuffix("instruction-file-put-object-delete"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + // Get the instruction file separately using a default client + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + // Ensure its metadata identifies it as such + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + + // Ensure decryption succeeds + ResponseBytes objectResponse = s3Client.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build()); + String output = objectResponse.asUtf8String(); + assertEquals(input, output); + + deleteObject(BUCKET, objectKey, s3Client); + + try { + defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(objectKey + ".instruction") + .build()); + fail("expected exception!"); + } catch (NoSuchKeyException e) { + // expected + } + + s3Client.close(); + defaultClient.close(); + } + + @Test + public void testPutWithInstructionFileV3ToV2Kms() { + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-kms"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new KMSEncryptionMaterials(KMS_KEY_ID)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); + + // Cleanup + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } + + @Test + public void testPutWithInstructionFileV3ToV2Aes() throws NoSuchAlgorithmException { + KeyGenerator keyGen = KeyGenerator.getInstance("AES"); + keyGen.init(256); + SecretKey aesKey = keyGen.generateKey(); + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-aes"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .aesKey(aesKey) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(aesKey)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); + + // Cleanup + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } + + @Test + public void testPutWithInstructionFileV3ToV2Rsa() throws NoSuchAlgorithmException { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair rsaKey = keyPairGen.generateKeyPair(); + + final String objectKey = appendTestSuffix("instruction-file-put-object-v3-to-v2-rsa"); + final String input = "SimpleTestOfV3EncryptionClient"; + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .rsaKeyPair(rsaKey) + .build(); + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .key(objectKey) + .build(), RequestBody.fromString(input)); + + EncryptionMaterialsProvider materialsProvider = + new StaticEncryptionMaterialsProvider(new EncryptionMaterials(rsaKey)); + CryptoConfigurationV2 cryptoConfig = + new CryptoConfigurationV2(CryptoMode.StrictAuthenticatedEncryption) + .withStorageMode(CryptoStorageMode.InstructionFile); + + AmazonS3EncryptionV2 v2Client = AmazonS3EncryptionClientV2.encryptionBuilder() + .withCryptoConfiguration(cryptoConfig) + .withEncryptionMaterialsProvider(materialsProvider) + .build(); + + String result = v2Client.getObjectAsString(BUCKET, objectKey); + assertEquals(input, result); + + // Cleanup + deleteObject(BUCKET, objectKey, s3Client); + s3Client.close(); + } + + @Test + public void testMultipartPutWithInstructionFile() throws IOException { + final String object_key = appendTestSuffix("test-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; //50 MB + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; + + S3Client wrappedClient = S3Client.create(); + S3Client s3Client = S3EncryptionClient.builder() + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .kmsKeyId(KMS_KEY_ID) + .enableMultipartPutObject(true) + .build(); + + Map encryptionContext = new HashMap<>(); + encryptionContext.put("test-key", "test-value"); + + + s3Client.putObject(builder -> builder + .bucket(BUCKET) + .storageClass(storageClass) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key), RequestBody.fromInputStream(inputStream, fileSizeLimit)); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); + + ResponseInputStream getResponse = s3Client.getObject(builder -> builder + .bucket(BUCKET) + .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, s3Client); + s3Client.close(); + + } + + @Test + public void testLowLevelMultipartPutWithInstructionFile() throws NoSuchAlgorithmException, IOException { + final String object_key = appendTestSuffix("test-low-level-multipart-put-instruction-file"); + + final long fileSizeLimit = 1024 * 1024 * 50; + final int PART_SIZE = 10 * 1024 * 1024; + final InputStream inputStream = new BoundedInputStream(fileSizeLimit); + final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + final StorageClass storageClass = StorageClass.STANDARD_IA; + + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(2048); + KeyPair rsaKey = keyPairGen.generateKeyPair(); + + S3Client wrappedClient = S3Client.create(); + + S3Client v3Client = S3EncryptionClient.builder() + .rsaKeyPair(rsaKey) + .instructionFileConfig(InstructionFileConfig.builder() + .instructionFileClient(wrappedClient) + .enableInstructionFilePutObject(true) + .build()) + .enableDelayedAuthenticationMode(true) + .build(); + + + CreateMultipartUploadResponse initiateResult = v3Client.createMultipartUpload(builder -> + builder.bucket(BUCKET).key(object_key).storageClass(storageClass)); + + List partETags = new ArrayList<>(); + + int bytesRead, bytesSent = 0; + byte[] partData = new byte[PART_SIZE]; + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + int partsSent = 1; + while ((bytesRead = inputStream.read(partData, 0, partData.length)) != -1) { + outputStream.write(partData, 0, bytesRead); + if (bytesSent < PART_SIZE) { + bytesSent += bytesRead; + continue; + } + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .build(); + + final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); + + UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, + RequestBody.fromInputStream(partInputStream, partInputStream.available())); + + partETags.add(CompletedPart.builder() + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); + outputStream.reset(); + bytesSent = 0; + partsSent++; + } + inputStream.close(); + UploadPartRequest uploadPartRequest = UploadPartRequest.builder() + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .partNumber(partsSent) + .sdkPartType(SdkPartType.LAST) + .build(); + final InputStream partInputStream = new ByteArrayInputStream(outputStream.toByteArray()); + UploadPartResponse uploadPartResult = v3Client.uploadPart(uploadPartRequest, + RequestBody.fromInputStream(partInputStream, partInputStream.available())); + partETags.add(CompletedPart.builder() + .partNumber(partsSent) + .eTag(uploadPartResult.eTag()) + .build()); + v3Client.completeMultipartUpload(builder -> builder + .bucket(BUCKET) + .key(object_key) + .uploadId(initiateResult.uploadId()) + .multipartUpload(partBuilder -> partBuilder.parts(partETags))); + + S3Client defaultClient = S3Client.create(); + ResponseBytes directInstGetResponse = defaultClient.getObjectAsBytes(builder -> builder + .bucket(BUCKET) + .key(object_key + ".instruction") + .build()); + assertTrue(directInstGetResponse.response().metadata().containsKey("x-amz-crypto-instr-file")); + assertEquals(storageClass.toString(), directInstGetResponse.response().storageClassAsString()); + + ResponseInputStream getResponse = v3Client.getObject(builder -> builder + .bucket(BUCKET) + .key(object_key)); + + assertTrue(IOUtils.contentEquals(objectStreamForResult, getResponse)); + + deleteObject(BUCKET, object_key, v3Client); + v3Client.close(); + } + +} + diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java index 3f61fa2fb..d6535d525 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientMultipartUploadTest.java @@ -67,6 +67,8 @@ public void multipartPutObjectAsync() throws IOException { final InputStream inputStream = new BoundedInputStream(fileSizeLimit); final InputStream objectStreamForResult = new BoundedInputStream(fileSizeLimit); + + S3AsyncClient v3Client = S3AsyncEncryptionClient.builder() .kmsKeyId(KMS_KEY_ID) .enableMultipartPutObject(true) @@ -78,11 +80,12 @@ public void multipartPutObjectAsync() throws IOException { encryptionContext.put("user-metadata-key", "user-metadata-value-v3-to-v3"); ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor(); - + CompletableFuture futurePut = v3Client.putObject(builder -> builder .bucket(BUCKET) .overrideConfiguration(withAdditionalConfiguration(encryptionContext)) .key(objectKey), AsyncRequestBody.fromInputStream(inputStream, fileSizeLimit, singleThreadExecutor)); + futurePut.join(); singleThreadExecutor.shutdown(); diff --git a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java index 182fc4841..d2520c6b2 100644 --- a/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java +++ b/src/test/java/software/amazon/encryption/s3/S3EncryptionClientTest.java @@ -1124,4 +1124,4 @@ private void simpleV3RoundTrip(final S3Client v3Client, final String objectKey) String output = objectResponse.asUtf8String(); assertEquals(input, output); } -} +} \ No newline at end of file diff --git a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java index 2f7757d3c..fc167d502 100644 --- a/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java +++ b/src/test/java/software/amazon/encryption/s3/internal/ConvertSDKRequestsTest.java @@ -1,4 +1,5 @@ package software.amazon.encryption.s3.internal; + import org.junit.jupiter.api.Test; import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.services.s3.model.*; @@ -7,6 +8,7 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; + import static org.junit.jupiter.api.Assertions.*; class ConvertSDKRequestsTest { @@ -427,24 +429,24 @@ void testConvertPutObjectRequest_OverrideConfiguration() { public void testConvertResponse() { // Create a CompleteMultipartUploadResponse with various fields set CompleteMultipartUploadResponse completeResponse = CompleteMultipartUploadResponse.builder() - .eTag("test-etag") - .expiration("test-expiration") - .checksumCRC32("test-crc32") - .checksumCRC32C("test-crc32c") - .checksumCRC64NVME("test-crc64") - .checksumSHA1("test-sha1") - .checksumSHA256("test-sha256") - .checksumType(ChecksumType.COMPOSITE) - .serverSideEncryption(ServerSideEncryption.AWS_KMS) - .versionId("test-version-id") - .ssekmsKeyId("test-kms-key-id") - .bucketKeyEnabled(true) - .requestCharged("requester") - // Fields that should be ignored - .location("test-location") - .bucket("test-bucket") - .key("test-key") - .build(); + .eTag("test-etag") + .expiration("test-expiration") + .checksumCRC32("test-crc32") + .checksumCRC32C("test-crc32c") + .checksumCRC64NVME("test-crc64") + .checksumSHA1("test-sha1") + .checksumSHA256("test-sha256") + .checksumType(ChecksumType.COMPOSITE) + .serverSideEncryption(ServerSideEncryption.AWS_KMS) + .versionId("test-version-id") + .ssekmsKeyId("test-kms-key-id") + .bucketKeyEnabled(true) + .requestCharged("requester") + // Fields that should be ignored + .location("test-location") + .bucket("test-bucket") + .key("test-key") + .build(); // Convert the response PutObjectResponse putResponse = ConvertSDKRequests.convertResponse(completeResponse); @@ -470,4 +472,120 @@ public void testConvertResponse() { assertNull(putResponse.ssekmsEncryptionContext()); assertNull(putResponse.size()); } + + @Test + public void testBasicConvertMultipartUploadRequest() { + // Create a MultipartUploadRequest with various fields set + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-bucket", result.bucket()); + assertEquals("test-key", result.key()); + assertNotNull(result); + } + + @Test + public void testConversionAllFieldsMultipartUploadRequestToPutObjectRequest() { + Map metadata = new HashMap(); + metadata.put("test-key-1", "test-value-1"); + metadata.put("test-key-2", "test-value-2"); + metadata.put("test-key-3", "test-value-3"); + + Instant expires = Instant.now(); + Instant retainUntilDate = Instant.now(); + + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .acl("test-acl") + .bucket("test-bucket") + .bucketKeyEnabled(true) + .cacheControl("test-cache-control") + .checksumAlgorithm("test-checksum-algorithm") + .contentDisposition("test-content-disposition") + .contentEncoding("test-content-encoding") + .contentLanguage("test-content-language") + .contentType("test-content-type") + .expectedBucketOwner("test-bucket-owner") + .expires(expires) + .grantFullControl("test-grant-full-control") + .grantRead("test-grant-read") + .grantReadACP("test-grant-read-acp") + .grantWriteACP("test-grant-write-acp") + .key("test-key") + .metadata(metadata) + .objectLockLegalHoldStatus(ObjectLockLegalHoldStatus.OFF) + .objectLockMode(ObjectLockMode.COMPLIANCE) + .objectLockRetainUntilDate(retainUntilDate) + .requestPayer(RequestPayer.REQUESTER) + .serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE) + .sseCustomerAlgorithm("test-sse-customer-algorithm") + .sseCustomerKey("test-sse-customer-key") + .ssekmsEncryptionContext("test-ssekms-encryption-context") + .ssekmsKeyId("test-ssekms-key-id") + .storageClass(StorageClass.SNOW) + .tagging("test-tagging") + .websiteRedirectLocation("test-website-redirect-location") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-acl", result.aclAsString()); + assertEquals("test-bucket", result.bucket()); + assertEquals(true, result.bucketKeyEnabled()); + assertEquals("test-cache-control", result.cacheControl()); + assertEquals("test-checksum-algorithm", result.checksumAlgorithmAsString()); + assertEquals("test-content-disposition", result.contentDisposition()); + assertEquals("test-content-encoding", result.contentEncoding()); + assertEquals("test-content-language", result.contentLanguage()); + assertEquals("test-content-type", result.contentType()); + assertEquals("test-bucket-owner", result.expectedBucketOwner()); + assertEquals(expires, result.expires()); + assertEquals("test-grant-full-control", result.grantFullControl()); + assertEquals("test-grant-read", result.grantRead()); + assertEquals("test-grant-read-acp", result.grantReadACP()); + assertEquals("test-grant-write-acp", result.grantWriteACP()); + assertEquals("test-key", result.key()); + assertEquals(metadata, result.metadata()); + assertEquals(ObjectLockLegalHoldStatus.OFF.toString(), result.objectLockLegalHoldStatusAsString()); + assertEquals(ObjectLockMode.COMPLIANCE.toString(), result.objectLockModeAsString()); + assertEquals(retainUntilDate, result.objectLockRetainUntilDate()); + assertEquals(RequestPayer.REQUESTER.toString(), result.requestPayerAsString()); + assertEquals(ServerSideEncryption.AWS_KMS_DSSE.toString(), result.serverSideEncryptionAsString()); + assertEquals("test-sse-customer-algorithm", result.sseCustomerAlgorithm()); + assertEquals("test-sse-customer-key", result.sseCustomerKey()); + assertEquals("test-ssekms-encryption-context", result.ssekmsEncryptionContext()); + assertEquals("test-ssekms-key-id", result.ssekmsKeyId()); + assertEquals(StorageClass.SNOW.toString(), result.storageClassAsString()); + assertEquals("test-tagging", result.tagging()); + assertEquals("test-website-redirect-location", result.websiteRedirectLocation()); + } + + @Test + public void testConvertMultipartUploadRequestWithNullValues() { + CreateMultipartUploadRequest request = CreateMultipartUploadRequest.builder() + .bucket("test-bucket") + .key("test-key") + .tagging("test-tagging") + .objectLockMode(ObjectLockMode.COMPLIANCE) + .contentLanguage("test-content-language") + .grantReadACP("test-grant-read-acp") + .build(); + PutObjectRequest result = ConvertSDKRequests.convertRequest(request); + assertEquals("test-bucket", result.bucket()); + assertEquals("test-key", result.key()); + assertEquals("test-tagging", result.tagging()); + assertEquals(ObjectLockMode.COMPLIANCE.toString(), result.objectLockModeAsString()); + assertEquals("test-content-language", result.contentLanguage()); + assertEquals("test-grant-read-acp", result.grantReadACP()); + + assertNull(result.aclAsString()); + assertNull(result.grantFullControl()); + assertNull(result.grantRead()); + assertNull(result.storageClass()); + assertNull(result.websiteRedirectLocation()); + assertTrue(result.metadata().isEmpty()); + + } } + + +