feat: reEncryptInstructionFile Implementation (#478)

Implemented the following feature: re-encrypting an instruction file with a new keyring while preserving the original encrypted object in S3. This enables:

Key rotation by updating instruction file metadata without re-encrypting object content
Sharing encrypted objects with partners by creating new instruction files with a custom suffix using their public keys

---------

Co-authored-by: Anirav Kareddy <aniravk@amazon.com>

Author: akareddy04

.prettierignore

@@ -0,0 +1,3 @@
+# Ignore artifacts:
+build
+coverage

src/examples/java/software/amazon/encryption/s3/examples/ReEncryptInstructionFileExample.java

@@ -47,20 +47,30 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
+import software.amazon.encryption.s3.internal.ContentMetadata;
+import software.amazon.encryption.s3.internal.ContentMetadataDecodingStrategy;
+import software.amazon.encryption.s3.internal.ContentMetadataEncodingStrategy;
 import software.amazon.encryption.s3.internal.ConvertSDKRequests;
 import software.amazon.encryption.s3.internal.GetEncryptedObjectPipeline;
 import software.amazon.encryption.s3.internal.InstructionFileConfig;
 import software.amazon.encryption.s3.internal.MultiFileOutputStream;
 import software.amazon.encryption.s3.internal.MultipartUploadObjectPipeline;
 import software.amazon.encryption.s3.internal.PutEncryptedObjectPipeline;
+import software.amazon.encryption.s3.internal.ReEncryptInstructionFileRequest;
+import software.amazon.encryption.s3.internal.ReEncryptInstructionFileResponse;
 import software.amazon.encryption.s3.internal.UploadObjectObserver;
 import software.amazon.encryption.s3.materials.AesKeyring;
 import software.amazon.encryption.s3.materials.CryptographicMaterialsManager;
+import software.amazon.encryption.s3.materials.DecryptMaterialsRequest;
+import software.amazon.encryption.s3.materials.DecryptionMaterials;
 import software.amazon.encryption.s3.materials.DefaultCryptoMaterialsManager;
+import software.amazon.encryption.s3.materials.EncryptedDataKey;
+import software.amazon.encryption.s3.materials.EncryptionMaterials;
 import software.amazon.encryption.s3.materials.Keyring;
 import software.amazon.encryption.s3.materials.KmsKeyring;
 import software.amazon.encryption.s3.materials.MultipartConfiguration;
 import software.amazon.encryption.s3.materials.PartialRsaKeyPair;
+import software.amazon.encryption.s3.materials.RawKeyring;
 import software.amazon.encryption.s3.materials.RsaKeyring;
 import software.amazon.encryption.s3.materials.S3Keyring;
 
@@ -71,6 +81,7 @@
 import java.security.Provider;
 import java.security.SecureRandom;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -83,7 +94,8 @@
 import java.util.function.Consumer;
 
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_BUFFER_SIZE_BYTES;
-import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
+
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MAX_ALLOWED_BUFFER_SIZE_BYTES;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.MIN_ALLOWED_BUFFER_SIZE_BYTES;
 import static software.amazon.encryption.s3.S3EncryptionClientUtilities.instructionFileKeysToDelete;
@@ -99,6 +111,9 @@ public class S3EncryptionClient extends DelegatingS3Client {
     public static final ExecutionAttribute<Map<String, String>> ENCRYPTION_CONTEXT = new ExecutionAttribute<>("EncryptionContext");
     public static final ExecutionAttribute<MultipartConfiguration> CONFIGURATION = new ExecutionAttribute<>("MultipartConfiguration");
 
+    //Used for specifying custom instruction file suffix on a per-request basis
+    public static final ExecutionAttribute<String> CUSTOM_INSTRUCTION_FILE_SUFFIX = new ExecutionAttribute<>("CustomInstructionFileSuffix");
+
     private final S3Client _wrappedClient;
     private final S3AsyncClient _wrappedAsyncClient;
     private final CryptographicMaterialsManager _cryptoMaterialsManager;
@@ -145,6 +160,18 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
                 builder.putExecutionAttribute(S3EncryptionClient.ENCRYPTION_CONTEXT, encryptionContext);
     }
 
+    /**
+     * Attaches a custom instruction file suffix to a request. Must be used as a parameter to
+     * {@link S3Request#overrideConfiguration()} in the request.
+     * This allows specifying a custom suffix for the instruction file on a per-request basis.
+     * @param customInstructionFileSuffix the custom suffix to use for the instruction file.
+     * @return Consumer for use in overrideConfiguration()
+     */
+    public static Consumer<AwsRequestOverrideConfiguration.Builder> withCustomInstructionFileSuffix(String customInstructionFileSuffix) {
+        return builder ->
+                builder.putExecutionAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX, customInstructionFileSuffix);
+    }
+
     /**
      * Attaches multipart configuration to a request. Must be used as a parameter to
      * {@link S3Request#overrideConfiguration()} in the request.
@@ -156,7 +183,6 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
                 builder.putExecutionAttribute(S3EncryptionClient.CONFIGURATION, multipartConfiguration);
     }
 
-
     /**
      * Attaches encryption context and multipart configuration to a request.
      * * Must be used as a parameter to
@@ -174,6 +200,102 @@ public static Consumer<AwsRequestOverrideConfiguration.Builder> withAdditionalCo
                         .putExecutionAttribute(S3EncryptionClient.CONFIGURATION, multipartConfiguration);
     }
 
+    /**
+     * Re-encrypts an instruction file with a new keyring while preserving the original encrypted object in S3.
+     * This enables:
+     * 1. Key rotation by updating instruction file metadata without re-encrypting object content
+     * 2. Sharing encrypted objects with partners by creating new instruction files with a custom suffix using their public keys
+     * <p>
+     * Key rotation scenarios:
+     * - Legacy to V3: Can rotate same wrapping key from legacy wrapping algorithms to fully supported wrapping algorithms
+     * - Within V3: When rotating the wrapping key, the new keyring must be different from the current keyring
+     * - Enforce Rotation: When enabled, ensures old keyring cannot decrypt data encrypted by new keyring
+     *
+     * @param reEncryptInstructionFileRequest the request containing bucket, object key, new keyring, and optional instruction file suffix
+     * @return ReEncryptInstructionFileResponse containing the bucket, object key, and instruction file suffix used
+     * @throws S3EncryptionClientException if the new keyring has the same materials description as the current one
+     */
+    public ReEncryptInstructionFileResponse reEncryptInstructionFile(ReEncryptInstructionFileRequest reEncryptInstructionFileRequest) {
+        if (!_instructionFileConfig.isInstructionFilePutEnabled()) {
+            throw new S3EncryptionClientException("Instruction file put operations must be enabled to re-encrypt instruction files");
+        }
+
+        //Build request to retrieve the encrypted object and its associated instruction file
+        final GetObjectRequest request = GetObjectRequest.builder()
+          .bucket(reEncryptInstructionFileRequest.bucket())
+          .key(reEncryptInstructionFileRequest.key())
+          .build();
+
+        ResponseInputStream<GetObjectResponse> response = this.getObject(request);
+        ContentMetadataDecodingStrategy decodingStrategy = new ContentMetadataDecodingStrategy(_instructionFileConfig);
+        ContentMetadata contentMetadata = decodingStrategy.decode(request, response.response());
+
+        //Extract cryptographic parameters from the current instruction file that MUST be preserved during re-encryption
+        final AlgorithmSuite algorithmSuite = contentMetadata.algorithmSuite();
+        final EncryptedDataKey originalEncryptedDataKey = contentMetadata.encryptedDataKey();
+        final Map<String, String> currentKeyringMaterialsDescription = contentMetadata.encryptedDataKeyMatDescOrContext();
+        final byte[] iv = contentMetadata.contentIv();
+
+        //Decrypt the data key using the current keyring
+        DecryptionMaterials decryptedMaterials = this._cryptoMaterialsManager.decryptMaterials(
+          DecryptMaterialsRequest.builder()
+            .algorithmSuite(algorithmSuite)
+            .encryptedDataKeys(Collections.singletonList(originalEncryptedDataKey))
+            .s3Request(request)
+            .build()
+        );
+
+        final byte[] plaintextDataKey = decryptedMaterials.plaintextDataKey();
+
+        //Prepare encryption materials with the decrypted data key
+        EncryptionMaterials encryptionMaterials = EncryptionMaterials.builder()
+          .algorithmSuite(algorithmSuite)
+          .plaintextDataKey(plaintextDataKey)
+          .s3Request(request)
+          .build();
+
+        //Re-encrypt the data key with the new keyring while preserving other cryptographic parameters
+        RawKeyring newKeyring = reEncryptInstructionFileRequest.newKeyring();
+        EncryptionMaterials encryptedMaterials = newKeyring.onEncrypt(encryptionMaterials);
+
+        final Map<String, String> newMaterialsDescription = encryptedMaterials.materialsDescription().getMaterialsDescription();
+        //Validate that the new keyring has different materials description than the old keyring
+        if (newMaterialsDescription.equals(currentKeyringMaterialsDescription)) {
+            throw new S3EncryptionClientException("New keyring must have new materials description!");
+        }
+
+        // If enforceRotation is set to true, ensure that the old keyring cannot decrypt the newly encrypted data key
+        if (reEncryptInstructionFileRequest.enforceRotation()) {
+            enforceRotation(encryptedMaterials, request);
+        }
+      
+        //Create or update instruction file with the re-encrypted metadata while preserving IV
+        ContentMetadataEncodingStrategy encodeStrategy = new ContentMetadataEncodingStrategy(_instructionFileConfig);
+        encodeStrategy.encodeMetadata(encryptedMaterials, iv, PutObjectRequest.builder()
+          .bucket(reEncryptInstructionFileRequest.bucket())
+          .key(reEncryptInstructionFileRequest.key())
+          .build(), reEncryptInstructionFileRequest.instructionFileSuffix());
+
+        return new ReEncryptInstructionFileResponse(reEncryptInstructionFileRequest.bucket(),
+            reEncryptInstructionFileRequest.key(), reEncryptInstructionFileRequest.instructionFileSuffix(), reEncryptInstructionFileRequest.enforceRotation());
+
+    }
+
+    private void enforceRotation(EncryptionMaterials newEncryptionMaterials, GetObjectRequest request) {
+        try {
+            DecryptionMaterials decryptedMaterials = this._cryptoMaterialsManager.decryptMaterials(
+              DecryptMaterialsRequest.builder()
+                .algorithmSuite(newEncryptionMaterials.algorithmSuite())
+                .encryptedDataKeys(newEncryptionMaterials.encryptedDataKeys())
+                .s3Request(request)
+                .build()
+            );
+        } catch (S3EncryptionClientException e) {
+            return;
+        }
+        throw new S3EncryptionClientException("Re-encryption failed due to enforced rotation! Old keyring is still able to decrypt the newly encrypted data key");
+    }
+
     /**
      * See {@link S3EncryptionClient#putObject(PutObjectRequest, RequestBody)}.
      * <p>
@@ -382,7 +504,7 @@ public DeleteObjectResponse deleteObject(DeleteObjectRequest deleteObjectRequest
             // Delete the object
             DeleteObjectResponse deleteObjectResponse = _wrappedAsyncClient.deleteObject(actualRequest).join();
             // If Instruction file exists, delete the instruction file as well.
-            String instructionObjectKey = deleteObjectRequest.key() + INSTRUCTION_FILE_SUFFIX;
+            String instructionObjectKey = deleteObjectRequest.key() + DEFAULT_INSTRUCTION_FILE_SUFFIX;
             _wrappedAsyncClient.deleteObject(builder -> builder
                     .overrideConfiguration(API_NAME_INTERCEPTOR)
                     .bucket(deleteObjectRequest.bucket())

src/main/java/software/amazon/encryption/s3/S3EncryptionClient.java

@@ -15,7 +15,7 @@
  */
 public class S3EncryptionClientUtilities {
 
-    public static final String INSTRUCTION_FILE_SUFFIX = ".instruction";
+    public static final String DEFAULT_INSTRUCTION_FILE_SUFFIX = ".instruction";
     public static final long MIN_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherBlockSizeBytes();
     public static final long MAX_ALLOWED_BUFFER_SIZE_BYTES = AlgorithmSuite.ALG_AES_256_GCM_IV12_TAG16_NO_KDF.cipherMaxContentLengthBytes();
 
@@ -32,7 +32,7 @@ public class S3EncryptionClientUtilities {
      */
     static List<ObjectIdentifier> instructionFileKeysToDelete(final DeleteObjectsRequest request) {
         return request.delete().objects().stream()
-                .map(o -> o.toBuilder().key(o.key() + INSTRUCTION_FILE_SUFFIX).build())
+                .map(o -> o.toBuilder().key(o.key() + DEFAULT_INSTRUCTION_FILE_SUFFIX).build())
                 .collect(Collectors.toList());
     }
 }

src/main/java/software/amazon/encryption/s3/S3EncryptionClientUtilities.java

@@ -15,7 +15,13 @@ public class ContentMetadata {
 
     private final EncryptedDataKey _encryptedDataKey;
     private final String _encryptedDataKeyAlgorithm;
-    private final Map<String, String> _encryptedDataKeyContext;
+
+    /**
+     * This field stores either encryption context or material description.
+     * We use a single field to store both in order to maintain backwards
+     * compatibility with V2, which treated both as the same.
+     */
+    private final Map<String, String> _encryptionContextOrMatDesc;
 
     private final byte[] _contentIv;
     private final String _contentCipher;
@@ -27,7 +33,7 @@ private ContentMetadata(Builder builder) {
 
         _encryptedDataKey = builder._encryptedDataKey;
         _encryptedDataKeyAlgorithm = builder._encryptedDataKeyAlgorithm;
-        _encryptedDataKeyContext = builder._encryptedDataKeyContext;
+        _encryptionContextOrMatDesc = builder._encryptionContextOrMatDesc;
 
         _contentIv = builder._contentIv;
         _contentCipher = builder._contentCipher;
@@ -51,14 +57,15 @@ public String encryptedDataKeyAlgorithm() {
         return _encryptedDataKeyAlgorithm;
     }
 
+
     /**
      * Note that the underlying implementation uses a Collections.unmodifiableMap which is
      * immutable.
      */
     @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "False positive; underlying"
         + " implementation is immutable")
-    public Map<String, String> encryptedDataKeyContext() {
-        return _encryptedDataKeyContext;
+    public Map<String, String> encryptedDataKeyMatDescOrContext() {
+        return _encryptionContextOrMatDesc;
     }
 
     public byte[] contentIv() {
@@ -85,7 +92,7 @@ public static class Builder {
 
         private EncryptedDataKey _encryptedDataKey;
         private String _encryptedDataKeyAlgorithm;
-        private Map<String, String> _encryptedDataKeyContext;
+        private Map<String, String> _encryptionContextOrMatDesc;
 
         private byte[] _contentIv;
         private String _contentCipher;
@@ -111,8 +118,8 @@ public Builder encryptedDataKeyAlgorithm(String encryptedDataKeyAlgorithm) {
             return this;
         }
 
-        public Builder encryptedDataKeyContext(Map<String, String> encryptedDataKeyContext) {
-            _encryptedDataKeyContext = Collections.unmodifiableMap(encryptedDataKeyContext);
+        public Builder encryptionContextOrMatDesc(Map<String, String> encryptionContextOrMatDesc) {
+            _encryptionContextOrMatDesc = Collections.unmodifiableMap(encryptionContextOrMatDesc);
             return this;
         }
 

src/main/java/software/amazon/encryption/s3/internal/ContentMetadata.java

@@ -9,6 +9,7 @@
 import software.amazon.awssdk.services.s3.model.GetObjectRequest;
 import software.amazon.awssdk.services.s3.model.GetObjectResponse;
 import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.encryption.s3.S3EncryptionClient;
 import software.amazon.encryption.s3.S3EncryptionClientException;
 import software.amazon.encryption.s3.algorithms.AlgorithmSuite;
 import software.amazon.encryption.s3.materials.EncryptedDataKey;
@@ -24,7 +25,7 @@
 import java.util.Map;
 import java.util.concurrent.CompletionException;
 
-import static software.amazon.encryption.s3.S3EncryptionClientUtilities.INSTRUCTION_FILE_SUFFIX;
+import static software.amazon.encryption.s3.S3EncryptionClientUtilities.DEFAULT_INSTRUCTION_FILE_SUFFIX;
 
 public class ContentMetadataDecodingStrategy {
 
@@ -136,8 +137,8 @@ private ContentMetadata readFromMap(Map<String, String> metadata, GetObjectRespo
                 .keyProviderInfo(keyProviderInfo.getBytes(StandardCharsets.UTF_8))
                 .build();
 
-        // Get encrypted data key encryption context
-        final Map<String, String> encryptionContext = new HashMap<>();
+        // Get encrypted data key encryption context or materials description (depending on the keyring)
+        final Map<String, String> encryptionContextOrMatDesc = new HashMap<>();
         // The V2 client treats null value here as empty, do the same to avoid incompatibility
         String jsonEncryptionContext = metadata.getOrDefault(MetadataKeyConstants.ENCRYPTED_DATA_KEY_CONTEXT, "{}");
         // When the encryption context contains non-US-ASCII characters,
@@ -149,7 +150,7 @@ private ContentMetadata readFromMap(Map<String, String> metadata, GetObjectRespo
             JsonNode objectNode = parser.parse(decodedJsonEncryptionContext);
 
             for (Map.Entry<String, JsonNode> entry : objectNode.asObject().entrySet()) {
-                encryptionContext.put(entry.getKey(), entry.getValue().asString());
+                encryptionContextOrMatDesc.put(entry.getKey(), entry.getValue().asString());
             }
         } catch (Exception e) {
             throw new RuntimeException(e);
@@ -161,7 +162,7 @@ private ContentMetadata readFromMap(Map<String, String> metadata, GetObjectRespo
         return ContentMetadata.builder()
                 .algorithmSuite(algorithmSuite)
                 .encryptedDataKey(edk)
-                .encryptedDataKeyContext(encryptionContext)
+                .encryptionContextOrMatDesc(encryptionContextOrMatDesc)
                 .contentIv(iv)
                 .contentRange(contentRange)
                 .build();
@@ -224,9 +225,13 @@ private ContentMetadata decodeFromObjectMetadata(GetObjectRequest request, GetOb
     }
 
     private ContentMetadata decodeFromInstructionFile(GetObjectRequest request, GetObjectResponse response) {
+       String instructionFileSuffix = request.overrideConfiguration()
+         .flatMap(config -> config.executionAttributes().getOptionalAttribute(S3EncryptionClient.CUSTOM_INSTRUCTION_FILE_SUFFIX))
+         .orElse(DEFAULT_INSTRUCTION_FILE_SUFFIX);
+
         GetObjectRequest instructionGetObjectRequest = GetObjectRequest.builder()
                 .bucket(request.bucket())
-                .key(request.key() + INSTRUCTION_FILE_SUFFIX)
+                .key(request.key() + instructionFileSuffix)
                 .build();
 
         ResponseInputStream<GetObjectResponse> instruction;