ESDK v1.7 custom changes

Signed-off-by: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com>

libs/encryption-sdk/build.gradle

@@ -24,7 +24,7 @@ dependencies {
   implementation 'commons-logging:commons-logging:1.2'
 
   // Encryption
-  implementation "com.amazonaws:aws-encryption-sdk-java:2.4.0"
+  implementation "com.amazonaws:aws-encryption-sdk-java:1.7.0"
   implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
   implementation "org.apache.commons:commons-lang3:${versions.commonslang}"
 

libs/encryption-sdk/licenses/aws-encryption-sdk-java-1.7.0.jar.sha1

@@ -0,0 +1 @@
+51704a672e65456d37f444c5992c079feff31218

libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java

@@ -12,6 +12,8 @@
 import org.opensearch.common.crypto.MasterKeyProvider;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.AbstractRefCounted;
+import org.opensearch.encryption.frame.AwsCrypto;
+import org.opensearch.encryption.frame.FrameCryptoHandler;
 import org.opensearch.encryption.keyprovider.CryptoMasterKey;
 
 import java.security.SecureRandom;
@@ -38,16 +40,12 @@ public CryptoManagerFactory(String algorithm, TimeValue keyRefreshInterval, int
         dataKeyCacheTTL = keyRefreshInterval.getMillis();
     }
 
-    private String validateAndGetAlgorithmId(String algorithm) {
+    protected String validateAndGetAlgorithmId(String algorithm) {
         // Supporting only 256 bit algorithm
-        switch (algorithm) {
-            case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY":
-                return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY.getDataKeyAlgo();
-            case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384":
-                return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384.getDataKeyAlgo();
-            default:
-                throw new IllegalArgumentException("Unsupported algorithm: " + algorithm);
+        if (algorithm.equals("ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256")) {
+            return CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256.getDataKeyAlgo();
         }
+        throw new IllegalArgumentException("Unsupported algorithm: " + algorithm);
     }
 
     public CryptoManager<?, ?> getOrCreateCryptoManager(
@@ -71,7 +69,22 @@ private String validateAndGetAlgorithmId(String algorithm) {
         CachingCryptoMaterialsManager materialsManager,
         MasterKeyProvider masterKeyProvider
     ) {
-        return new NoOpCryptoHandler();
+        // Supporting only 256 bit algorithm as of now. To provide support for other bit size algorithms, necessary
+        // changes in key providers are required. Following 2 constraints should be satisfied to add support for
+        // another algorithm :
+        // 1. It should be safe to cache. Unsafe cache algorithms can't be used at it would require generation of data
+        //    keys on every encrypt which is not a practical approach.
+        // 2. It shouldn't have any trailing metadata. This is needed to handle cases where full content is read
+        //    till the length of the decrypted bytes are reached. This skips reading trailing metadata and closes
+        //    remote streams. Remote store can throw an error for such reads saying that content wasn't fully read.
+        // With the above constraints ESDK, currently we can only add support for one algorithm.
+        if (algorithm.equals("ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256")) {
+            return new FrameCryptoHandler(
+                    new AwsCrypto(materialsManager, CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256),
+                    masterKeyProvider.getEncryptionContext()
+            );
+        }
+        throw new IllegalArgumentException("Unsupported algorithm: " + algorithm);
     }
 
     // Package private for tests

libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/AwsCrypto.java

@@ -0,0 +1,140 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.encryption.frame;
+
+import org.opensearch.common.io.InputStreamContainer;
+
+import java.io.InputStream;
+import java.util.Map;
+
+import com.amazonaws.encryptionsdk.CommitmentPolicy;
+import com.amazonaws.encryptionsdk.CryptoAlgorithm;
+import com.amazonaws.encryptionsdk.CryptoMaterialsManager;
+import com.amazonaws.encryptionsdk.ParsedCiphertext;
+import com.amazonaws.encryptionsdk.exception.AwsCryptoException;
+import com.amazonaws.encryptionsdk.internal.LazyMessageCryptoHandler;
+import com.amazonaws.encryptionsdk.internal.MessageCryptoHandler;
+import com.amazonaws.encryptionsdk.model.EncryptionMaterialsRequest;
+
+public class AwsCrypto {
+    private final CryptoMaterialsManager materialsManager;
+    private final CryptoAlgorithm cryptoAlgorithm;
+
+    public AwsCrypto(final CryptoMaterialsManager materialsManager, final CryptoAlgorithm cryptoAlgorithm) {
+        Utils.assertNonNull(materialsManager, "materialsManager");
+        this.materialsManager = materialsManager;
+        this.cryptoAlgorithm = cryptoAlgorithm;
+
+    }
+
+    public EncryptionMetadata createCryptoContext(final Map<String, String> encryptionContext, int frameSize) {
+        Utils.assertNonNull(encryptionContext, "encryptionContext");
+
+        EncryptionMaterialsRequest.Builder requestBuilder = EncryptionMaterialsRequest.newBuilder()
+            .setContext(encryptionContext)
+            .setRequestedAlgorithm(cryptoAlgorithm)
+            .setPlaintextSize(0) // To avoid skipping cache
+            .setCommitmentPolicy(CommitmentPolicy.ForbidEncryptAllowDecrypt);
+
+        return new EncryptionMetadata(frameSize, materialsManager.getMaterialsForEncrypt(requestBuilder.build()));
+    }
+
+    public InputStreamContainer createEncryptingStream(
+        final InputStreamContainer stream,
+        int streamIdx,
+        int totalStreams,
+        int frameNumber,
+        EncryptionMetadata encryptionMetadata
+    ) {
+
+        boolean isLastStream = streamIdx == totalStreams - 1;
+        boolean firstOperation = streamIdx == 0;
+        if (stream.getContentLength() % encryptionMetadata.getFrameSize() != 0 && !isLastStream) {
+            throw new AwsCryptoException(
+                "Length of each inputStream should be exactly divisible by frame size except "
+                    + "the last inputStream. Current frame size is "
+                    + encryptionMetadata.getFrameSize()
+                    + " and inputStream length is "
+                    + stream.getContentLength()
+            );
+        }
+        final MessageCryptoHandler cryptoHandler = getEncryptingStreamHandler(frameNumber, firstOperation, encryptionMetadata);
+        CryptoInputStream<?> cryptoInputStream = new CryptoInputStream<>(stream.getInputStream(), cryptoHandler, isLastStream);
+        cryptoInputStream.setMaxInputLength(stream.getContentLength());
+
+        long encryptedLength = 0;
+        if (streamIdx == 0) {
+            encryptedLength = encryptionMetadata.getCiphertextHeaderBytes().length;
+        }
+        if (streamIdx == (totalStreams - 1)) {
+            encryptedLength += estimateOutputSizeWithFooter(
+                encryptionMetadata.getFrameSize(),
+                encryptionMetadata.getNonceLen(),
+                encryptionMetadata.getCryptoAlgo().getTagLen(),
+                stream.getContentLength(),
+                encryptionMetadata.getCryptoAlgo()
+            );
+        } else {
+            encryptedLength += estimatePartialOutputSize(
+                encryptionMetadata.getFrameSize(),
+                encryptionMetadata.getNonceLen(),
+                encryptionMetadata.getCryptoAlgo().getTagLen(),
+                stream.getContentLength()
+            );
+        }
+        return new InputStreamContainer(cryptoInputStream, encryptedLength, -1);
+    }
+
+    public MessageCryptoHandler getEncryptingStreamHandler(
+        int frameStartNumber,
+        boolean firstOperation,
+        EncryptionMetadata encryptionMetadata
+    ) {
+        return new LazyMessageCryptoHandler(info -> new EncryptionHandler(encryptionMetadata, firstOperation, frameStartNumber));
+    }
+
+    public long estimatePartialOutputSize(int frameLen, int nonceLen, int tagLen, long contentLength) {
+        return FrameEncryptionHandler.estimatePartialSizeFromMetadata(contentLength, false, frameLen, nonceLen, tagLen);
+    }
+
+    public long estimateOutputSizeWithFooter(int frameLen, int nonceLen, int tagLen, long contentLength, CryptoAlgorithm cryptoAlgorithm) {
+        return FrameEncryptionHandler.estimatePartialSizeFromMetadata(contentLength, true, frameLen, nonceLen, tagLen)
+            + getTrailingSignatureSize(cryptoAlgorithm);
+    }
+
+    public long estimateDecryptedSize(int frameLen, int nonceLen, int tagLen, long contentLength, CryptoAlgorithm cryptoAlgorithm) {
+        long contentLenWithoutTrailingSig = contentLength - getTrailingSignatureSize(cryptoAlgorithm);
+        return FrameDecryptionHandler.estimateDecryptedSize(contentLenWithoutTrailingSig, frameLen, nonceLen, tagLen);
+    }
+
+    public int getTrailingSignatureSize(CryptoAlgorithm cryptoAlgorithm) {
+        return EncryptionHandler.getAlgoTrailingLength(cryptoAlgorithm);
+    }
+
+    public CryptoInputStream<?> createDecryptingStream(final InputStream inputStream) {
+
+        final MessageCryptoHandler cryptoHandler = DecryptionHandler.create(materialsManager);
+        return new CryptoInputStream<>(inputStream, cryptoHandler, true);
+    }
+
+    public CryptoInputStream<?> createDecryptingStream(
+        final InputStream inputStream,
+        final long size,
+        final ParsedCiphertext parsedCiphertext,
+        final int frameStartNum,
+        boolean isLastPart
+    ) {
+
+        final MessageCryptoHandler cryptoHandler = DecryptionHandler.create(materialsManager, parsedCiphertext, frameStartNum);
+        CryptoInputStream<?> cryptoInputStream = new CryptoInputStream<>(inputStream, cryptoHandler, isLastPart);
+        cryptoInputStream.setMaxInputLength(size);
+        return cryptoInputStream;
+    }
+
+}

libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CipherHandler.java

@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except
+ * in compliance with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package org.opensearch.encryption.frame;
+
+import javax.crypto.Cipher;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.GCMParameterSpec;
+
+import java.security.GeneralSecurityException;
+import java.security.spec.AlgorithmParameterSpec;
+
+import com.amazonaws.encryptionsdk.CryptoAlgorithm;
+import com.amazonaws.encryptionsdk.exception.AwsCryptoException;
+import com.amazonaws.encryptionsdk.exception.BadCiphertextException;
+
+/**
+ * This class provides a cryptographic cipher handler powered by an underlying block cipher. The
+ * block cipher performs authenticated encryption of the provided bytes using Additional
+ * Authenticated Data (AAD).
+ *
+ * <p>This class implements a method called cipherData() that encrypts or decrypts a byte array by
+ * calling methods on the underlying block cipher.
+ */
+public class CipherHandler {
+    private final int cipherMode_;
+    private final SecretKey key_;
+    private final CryptoAlgorithm cryptoAlgorithm_;
+    private final Cipher cipher_;
+
+    /**
+     * Process data through the cipher.
+     *
+     * <p>This method calls the <code>update</code> and <code>doFinal</code> methods on the underlying
+     * cipher to complete processing of the data.
+     *
+     * @param nonce the nonce to be used by the underlying cipher
+     * @param contentAad the optional additional authentication data to be used by the underlying
+     *     cipher
+     * @param content the content to be processed by the underlying cipher
+     * @param off the offset into content array to be processed
+     * @param len the number of bytes to process
+     * @return the bytes processed by the underlying cipher
+     * @throws AwsCryptoException if cipher initialization fails
+     * @throws BadCiphertextException if processing the data through the cipher fails
+     */
+    public byte[] cipherData(byte[] nonce, byte[] contentAad, final byte[] content, final int off, final int len) {
+        if (nonce.length != cryptoAlgorithm_.getNonceLen()) {
+            throw new IllegalArgumentException("Invalid nonce length");
+        }
+        final AlgorithmParameterSpec spec = new GCMParameterSpec(cryptoAlgorithm_.getTagLen() * 8, nonce, 0, nonce.length);
+
+        try {
+            cipher_.init(cipherMode_, key_, spec);
+            if (contentAad != null) {
+                cipher_.updateAAD(contentAad);
+            }
+        } catch (final GeneralSecurityException gsx) {
+            throw new AwsCryptoException(gsx);
+        }
+        try {
+            return cipher_.doFinal(content, off, len);
+        } catch (final GeneralSecurityException gsx) {
+            throw new BadCiphertextException(gsx);
+        }
+    }
+
+    /**
+     * Create a cipher handler for processing bytes using an underlying block cipher.
+     *
+     * @param key the key to use in encrypting or decrypting bytes
+     * @param cipherMode the mode for processing the bytes as defined in {@link Cipher#init(int,
+     *     java.security.Key)}
+     * @param cryptoAlgorithm the cryptography algorithm to be used by the underlying block cipher.
+     */
+    public CipherHandler(final SecretKey key, final int cipherMode, final CryptoAlgorithm cryptoAlgorithm) {
+        this.cipherMode_ = cipherMode;
+        this.key_ = key;
+        this.cryptoAlgorithm_ = cryptoAlgorithm;
+        this.cipher_ = buildCipherObject(cryptoAlgorithm);
+    }
+
+    private static Cipher buildCipherObject(final CryptoAlgorithm alg) {
+        try {
+            // Right now, just GCM is supported
+            return Cipher.getInstance("AES/GCM/NoPadding");
+        } catch (final GeneralSecurityException ex) {
+            throw new IllegalStateException("Java does not support the requested algorithm", ex);
+        }
+    }
+}