From cee25b3d762bbbf52a2e2ee9277a995548f2ad3e Mon Sep 17 00:00:00 2001 From: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> Date: Fri, 1 Sep 2023 18:57:21 +0530 Subject: [PATCH] ESDK v1.7 custom changes Signed-off-by: Vikas Bansal <43470111+vikasvb90@users.noreply.github.com> --- libs/encryption-sdk/build.gradle | 2 +- .../aws-encryption-sdk-java-1.7.0.jar.sha1 | 1 + .../aws-encryption-sdk-java-2.4.0.jar.sha1 | 1 - .../encryption/CryptoManagerFactory.java | 31 +- .../encryption/frame/AwsCrypto.java | 140 +++++ .../encryption/frame/CipherHandler.java | 109 ++++ .../encryption/frame/CryptoInputStream.java | 244 ++++++++ .../encryption/frame/DecryptionHandler.java | 532 ++++++++++++++++++ .../encryption/frame/EncryptionHandler.java | 368 ++++++++++++ .../encryption/frame/EncryptionMetadata.java | 252 +++++++++ .../encryption/frame/FrameCryptoHandler.java | 234 ++++++++ .../frame/FrameDecryptionHandler.java | 319 +++++++++++ .../frame/FrameEncryptionHandler.java | 376 +++++++++++++ .../opensearch/encryption/frame/Utils.java | 106 ++++ .../encryption/CryptoManagerFactoryTests.java | 15 +- .../encryption/MockKeyProvider.java | 1 + .../encryption/NoOpCryptoHandlerTests.java | 3 +- .../encryption/frame/CipherHandlerTests.java | 35 ++ .../encryption/frame/CryptoTests.java | 489 ++++++++++++++++ .../resources/raw_content_for_crypto_test | 25 - .../org/opensearch/crypto/kms/KmsService.java | 5 +- .../crypto/CryptoManagerRegistry.java | 2 +- 22 files changed, 3247 insertions(+), 43 deletions(-) create mode 100644 libs/encryption-sdk/licenses/aws-encryption-sdk-java-1.7.0.jar.sha1 delete mode 100644 libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/AwsCrypto.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CipherHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CryptoInputStream.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/DecryptionHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionMetadata.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameCryptoHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameDecryptionHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameEncryptionHandler.java create mode 100644 libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/Utils.java create mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CipherHandlerTests.java create mode 100644 libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CryptoTests.java delete mode 100644 libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test diff --git a/libs/encryption-sdk/build.gradle b/libs/encryption-sdk/build.gradle index d229d4edf0a83..0e7f71d34ccf7 100644 --- a/libs/encryption-sdk/build.gradle +++ b/libs/encryption-sdk/build.gradle @@ -24,7 +24,7 @@ dependencies { implementation 'commons-logging:commons-logging:1.2' // Encryption - implementation "com.amazonaws:aws-encryption-sdk-java:2.4.0" + implementation "com.amazonaws:aws-encryption-sdk-java:1.7.0" implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" implementation "org.apache.commons:commons-lang3:${versions.commonslang}" diff --git a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-1.7.0.jar.sha1 b/libs/encryption-sdk/licenses/aws-encryption-sdk-java-1.7.0.jar.sha1 new file mode 100644 index 0000000000000..e0bb769bbf849 --- /dev/null +++ b/libs/encryption-sdk/licenses/aws-encryption-sdk-java-1.7.0.jar.sha1 @@ -0,0 +1 @@ +51704a672e65456d37f444c5992c079feff31218 \ No newline at end of file diff --git a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 b/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 deleted file mode 100644 index 504b4a423a975..0000000000000 --- a/libs/encryption-sdk/licenses/aws-encryption-sdk-java-2.4.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -98943eda1dc05bb01f4f5405e115b08dc541afbf \ No newline at end of file diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java index e1dc9291ed1a6..2bfa08dc217c8 100644 --- a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/CryptoManagerFactory.java @@ -12,6 +12,8 @@ import org.opensearch.common.crypto.MasterKeyProvider; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.util.concurrent.AbstractRefCounted; +import org.opensearch.encryption.frame.AwsCrypto; +import org.opensearch.encryption.frame.FrameCryptoHandler; import org.opensearch.encryption.keyprovider.CryptoMasterKey; import java.security.SecureRandom; @@ -38,16 +40,12 @@ public CryptoManagerFactory(String algorithm, TimeValue keyRefreshInterval, int dataKeyCacheTTL = keyRefreshInterval.getMillis(); } - private String validateAndGetAlgorithmId(String algorithm) { + protected String validateAndGetAlgorithmId(String algorithm) { // Supporting only 256 bit algorithm - switch (algorithm) { - case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY": - return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY.getDataKeyAlgo(); - case "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384": - return CryptoAlgorithm.ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384.getDataKeyAlgo(); - default: - throw new IllegalArgumentException("Unsupported algorithm: " + algorithm); + if (algorithm.equals("ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256")) { + return CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256.getDataKeyAlgo(); } + throw new IllegalArgumentException("Unsupported algorithm: " + algorithm); } public CryptoManager getOrCreateCryptoManager( @@ -71,7 +69,22 @@ private String validateAndGetAlgorithmId(String algorithm) { CachingCryptoMaterialsManager materialsManager, MasterKeyProvider masterKeyProvider ) { - return new NoOpCryptoHandler(); + // Supporting only 256 bit algorithm as of now. To provide support for other bit size algorithms, necessary + // changes in key providers are required. Following 2 constraints should be satisfied to add support for + // another algorithm : + // 1. It should be safe to cache. Unsafe cache algorithms can't be used at it would require generation of data + // keys on every encrypt which is not a practical approach. + // 2. It shouldn't have any trailing metadata. This is needed to handle cases where full content is read + // till the length of the decrypted bytes are reached. This skips reading trailing metadata and closes + // remote streams. Remote store can throw an error for such reads saying that content wasn't fully read. + // With the above constraints ESDK, currently we can only add support for one algorithm. + if (algorithm.equals("ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256")) { + return new FrameCryptoHandler( + new AwsCrypto(materialsManager, CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256), + masterKeyProvider.getEncryptionContext() + ); + } + throw new IllegalArgumentException("Unsupported algorithm: " + algorithm); } // Package private for tests diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/AwsCrypto.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/AwsCrypto.java new file mode 100644 index 0000000000000..241b82db5273c --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/AwsCrypto.java @@ -0,0 +1,140 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.encryption.frame; + +import org.opensearch.common.io.InputStreamContainer; + +import java.io.InputStream; +import java.util.Map; + +import com.amazonaws.encryptionsdk.CommitmentPolicy; +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.CryptoMaterialsManager; +import com.amazonaws.encryptionsdk.ParsedCiphertext; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.internal.LazyMessageCryptoHandler; +import com.amazonaws.encryptionsdk.internal.MessageCryptoHandler; +import com.amazonaws.encryptionsdk.model.EncryptionMaterialsRequest; + +public class AwsCrypto { + private final CryptoMaterialsManager materialsManager; + private final CryptoAlgorithm cryptoAlgorithm; + + public AwsCrypto(final CryptoMaterialsManager materialsManager, final CryptoAlgorithm cryptoAlgorithm) { + Utils.assertNonNull(materialsManager, "materialsManager"); + this.materialsManager = materialsManager; + this.cryptoAlgorithm = cryptoAlgorithm; + + } + + public EncryptionMetadata createCryptoContext(final Map encryptionContext, int frameSize) { + Utils.assertNonNull(encryptionContext, "encryptionContext"); + + EncryptionMaterialsRequest.Builder requestBuilder = EncryptionMaterialsRequest.newBuilder() + .setContext(encryptionContext) + .setRequestedAlgorithm(cryptoAlgorithm) + .setPlaintextSize(0) // To avoid skipping cache + .setCommitmentPolicy(CommitmentPolicy.ForbidEncryptAllowDecrypt); + + return new EncryptionMetadata(frameSize, materialsManager.getMaterialsForEncrypt(requestBuilder.build())); + } + + public InputStreamContainer createEncryptingStream( + final InputStreamContainer stream, + int streamIdx, + int totalStreams, + int frameNumber, + EncryptionMetadata encryptionMetadata + ) { + + boolean isLastStream = streamIdx == totalStreams - 1; + boolean firstOperation = streamIdx == 0; + if (stream.getContentLength() % encryptionMetadata.getFrameSize() != 0 && !isLastStream) { + throw new AwsCryptoException( + "Length of each inputStream should be exactly divisible by frame size except " + + "the last inputStream. Current frame size is " + + encryptionMetadata.getFrameSize() + + " and inputStream length is " + + stream.getContentLength() + ); + } + final MessageCryptoHandler cryptoHandler = getEncryptingStreamHandler(frameNumber, firstOperation, encryptionMetadata); + CryptoInputStream cryptoInputStream = new CryptoInputStream<>(stream.getInputStream(), cryptoHandler, isLastStream); + cryptoInputStream.setMaxInputLength(stream.getContentLength()); + + long encryptedLength = 0; + if (streamIdx == 0) { + encryptedLength = encryptionMetadata.getCiphertextHeaderBytes().length; + } + if (streamIdx == (totalStreams - 1)) { + encryptedLength += estimateOutputSizeWithFooter( + encryptionMetadata.getFrameSize(), + encryptionMetadata.getNonceLen(), + encryptionMetadata.getCryptoAlgo().getTagLen(), + stream.getContentLength(), + encryptionMetadata.getCryptoAlgo() + ); + } else { + encryptedLength += estimatePartialOutputSize( + encryptionMetadata.getFrameSize(), + encryptionMetadata.getNonceLen(), + encryptionMetadata.getCryptoAlgo().getTagLen(), + stream.getContentLength() + ); + } + return new InputStreamContainer(cryptoInputStream, encryptedLength, -1); + } + + public MessageCryptoHandler getEncryptingStreamHandler( + int frameStartNumber, + boolean firstOperation, + EncryptionMetadata encryptionMetadata + ) { + return new LazyMessageCryptoHandler(info -> new EncryptionHandler(encryptionMetadata, firstOperation, frameStartNumber)); + } + + public long estimatePartialOutputSize(int frameLen, int nonceLen, int tagLen, long contentLength) { + return FrameEncryptionHandler.estimatePartialSizeFromMetadata(contentLength, false, frameLen, nonceLen, tagLen); + } + + public long estimateOutputSizeWithFooter(int frameLen, int nonceLen, int tagLen, long contentLength, CryptoAlgorithm cryptoAlgorithm) { + return FrameEncryptionHandler.estimatePartialSizeFromMetadata(contentLength, true, frameLen, nonceLen, tagLen) + + getTrailingSignatureSize(cryptoAlgorithm); + } + + public long estimateDecryptedSize(int frameLen, int nonceLen, int tagLen, long contentLength, CryptoAlgorithm cryptoAlgorithm) { + long contentLenWithoutTrailingSig = contentLength - getTrailingSignatureSize(cryptoAlgorithm); + return FrameDecryptionHandler.estimateDecryptedSize(contentLenWithoutTrailingSig, frameLen, nonceLen, tagLen); + } + + public int getTrailingSignatureSize(CryptoAlgorithm cryptoAlgorithm) { + return EncryptionHandler.getAlgoTrailingLength(cryptoAlgorithm); + } + + public CryptoInputStream createDecryptingStream(final InputStream inputStream) { + + final MessageCryptoHandler cryptoHandler = DecryptionHandler.create(materialsManager); + return new CryptoInputStream<>(inputStream, cryptoHandler, true); + } + + public CryptoInputStream createDecryptingStream( + final InputStream inputStream, + final long size, + final ParsedCiphertext parsedCiphertext, + final int frameStartNum, + boolean isLastPart + ) { + + final MessageCryptoHandler cryptoHandler = DecryptionHandler.create(materialsManager, parsedCiphertext, frameStartNum); + CryptoInputStream cryptoInputStream = new CryptoInputStream<>(inputStream, cryptoHandler, isLastPart); + cryptoInputStream.setMaxInputLength(size); + return cryptoInputStream; + } + +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CipherHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CipherHandler.java new file mode 100644 index 0000000000000..941062dea015d --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CipherHandler.java @@ -0,0 +1,109 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; + +import java.security.GeneralSecurityException; +import java.security.spec.AlgorithmParameterSpec; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; + +/** + * This class provides a cryptographic cipher handler powered by an underlying block cipher. The + * block cipher performs authenticated encryption of the provided bytes using Additional + * Authenticated Data (AAD). + * + *

This class implements a method called cipherData() that encrypts or decrypts a byte array by + * calling methods on the underlying block cipher. + */ +public class CipherHandler { + private final int cipherMode_; + private final SecretKey key_; + private final CryptoAlgorithm cryptoAlgorithm_; + private final Cipher cipher_; + + /** + * Process data through the cipher. + * + *

This method calls the update and doFinal methods on the underlying + * cipher to complete processing of the data. + * + * @param nonce the nonce to be used by the underlying cipher + * @param contentAad the optional additional authentication data to be used by the underlying + * cipher + * @param content the content to be processed by the underlying cipher + * @param off the offset into content array to be processed + * @param len the number of bytes to process + * @return the bytes processed by the underlying cipher + * @throws AwsCryptoException if cipher initialization fails + * @throws BadCiphertextException if processing the data through the cipher fails + */ + public byte[] cipherData(byte[] nonce, byte[] contentAad, final byte[] content, final int off, final int len) { + if (nonce.length != cryptoAlgorithm_.getNonceLen()) { + throw new IllegalArgumentException("Invalid nonce length"); + } + final AlgorithmParameterSpec spec = new GCMParameterSpec(cryptoAlgorithm_.getTagLen() * 8, nonce, 0, nonce.length); + + try { + cipher_.init(cipherMode_, key_, spec); + if (contentAad != null) { + cipher_.updateAAD(contentAad); + } + } catch (final GeneralSecurityException gsx) { + throw new AwsCryptoException(gsx); + } + try { + return cipher_.doFinal(content, off, len); + } catch (final GeneralSecurityException gsx) { + throw new BadCiphertextException(gsx); + } + } + + /** + * Create a cipher handler for processing bytes using an underlying block cipher. + * + * @param key the key to use in encrypting or decrypting bytes + * @param cipherMode the mode for processing the bytes as defined in {@link Cipher#init(int, + * java.security.Key)} + * @param cryptoAlgorithm the cryptography algorithm to be used by the underlying block cipher. + */ + public CipherHandler(final SecretKey key, final int cipherMode, final CryptoAlgorithm cryptoAlgorithm) { + this.cipherMode_ = cipherMode; + this.key_ = key; + this.cryptoAlgorithm_ = cryptoAlgorithm; + this.cipher_ = buildCipherObject(cryptoAlgorithm); + } + + private static Cipher buildCipherObject(final CryptoAlgorithm alg) { + try { + // Right now, just GCM is supported + return Cipher.getInstance("AES/GCM/NoPadding"); + } catch (final GeneralSecurityException ex) { + throw new IllegalStateException("Java does not support the requested algorithm", ex); + } + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CryptoInputStream.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CryptoInputStream.java new file mode 100644 index 0000000000000..e8d51fb2440d5 --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/CryptoInputStream.java @@ -0,0 +1,244 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import java.io.IOException; +import java.io.InputStream; + +import com.amazonaws.encryptionsdk.AwsCrypto; +import com.amazonaws.encryptionsdk.MasterKey; +import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import com.amazonaws.encryptionsdk.internal.MessageCryptoHandler; +import com.amazonaws.encryptionsdk.internal.Utils; + +import static com.amazonaws.encryptionsdk.internal.Utils.assertNonNull; + +/** + * A CryptoInputStream is a subclass of java.io.InputStream. It performs cryptographic + * transformation of the bytes passing through it. + * + *

+ * The CryptoInputStream wraps a provided InputStream object and performs cryptographic + * transformation of the bytes read from the wrapped InputStream. It uses the cryptography handler + * provided during construction to invoke methods that perform the cryptographic transformations. + * + *

+ * In short, reading from the CryptoInputStream returns bytes that are the cryptographic + * transformations of the bytes read from the wrapped InputStream. + * + *

+ * For example, if the cryptography handler provides methods for decryption, the CryptoInputStream + * will read ciphertext bytes from the wrapped InputStream, decrypt, and return them as plaintext + * bytes. + * + *

+ * This class adheres strictly to the semantics, especially the failure semantics, of its ancestor + * class java.io.InputStream. This class overrides all the methods specified in its ancestor class. + * + *

+ * To instantiate an instance of this class, please see {@link AwsCrypto}. + * + * @param + * The type of {@link MasterKey}s used to manipulate the data. + */ +public class CryptoInputStream> extends InputStream { + private static final int MAX_READ_LEN = 4096; + + private byte[] outBytes_ = new byte[0]; + private int outStart_; + private int outEnd_; + private final InputStream inputStream_; + private final MessageCryptoHandler cryptoHandler_; + private boolean hasFinalCalled_; + private boolean hasProcessBytesCalled_; + private final boolean isLastPart_; + + /** + * Constructs a CryptoInputStream that wraps the provided InputStream object. It performs + * cryptographic transformation of the bytes read from the wrapped InputStream using the methods + * provided in the provided CryptoHandler implementation. + * + * @param inputStream + * the inputStream object to be wrapped. + * @param cryptoHandler + * the cryptoHandler implementation that provides the methods to use in performing + * cryptographic transformation of the bytes read from the inputStream. + */ + CryptoInputStream(final InputStream inputStream, final MessageCryptoHandler cryptoHandler, boolean isLastPart) { + inputStream_ = Utils.assertNonNull(inputStream, "inputStream"); + cryptoHandler_ = Utils.assertNonNull(cryptoHandler, "cryptoHandler"); + isLastPart_ = isLastPart; + } + + /** + * Fill the output bytes by reading from the wrapped InputStream and processing it through the + * crypto handler. + * + * @return the number of bytes processed and returned by the crypto handler. + */ + private int fillOutBytes() throws IOException, BadCiphertextException { + final byte[] inputStreamBytes = new byte[MAX_READ_LEN]; + + final int readLen = inputStream_.read(inputStreamBytes); + + outStart_ = 0; + + int processedLen = -1; + if (readLen < 0 && isLastPart_) { + // Mark end of stream until doFinal returns something. + + if (!hasFinalCalled_) { + int outOffset = 0; + int outLen = 0; + + // Handle the case where processBytes() was never called before. + // This happens with an empty file where the end of stream is + // reached on the first read attempt. In this case, + // processBytes() must be called so the header bytes are written + // during encryption. + if (!hasProcessBytesCalled_) { + outBytes_ = new byte[cryptoHandler_.estimateOutputSize(0)]; + outLen += cryptoHandler_.processBytes(inputStreamBytes, 0, 0, outBytes_, outOffset).getBytesWritten(); + outOffset += outLen; + } else { + outBytes_ = new byte[cryptoHandler_.estimateFinalOutputSize()]; + } + + // Get final bytes. + outLen += cryptoHandler_.doFinal(outBytes_, outOffset); + processedLen = outLen; + hasFinalCalled_ = true; + } + } else if (readLen > 0) { + // process the read bytes. + outBytes_ = new byte[cryptoHandler_.estimatePartialOutputSize(readLen)]; + processedLen = cryptoHandler_.processBytes(inputStreamBytes, 0, readLen, outBytes_, outStart_).getBytesWritten(); + hasProcessBytesCalled_ = true; + } + + outEnd_ = processedLen; + return processedLen; + } + + /** + * {@inheritDoc} + * + * @throws BadCiphertextException + * This is thrown only during decryption if b contains invalid or corrupt + * ciphertext. + */ + @Override + public int read(final byte[] b, final int off, final int len) throws IllegalArgumentException, IOException, BadCiphertextException { + assertNonNull(b, "b"); + + if (len < 0 || off < 0) { + throw new IllegalArgumentException("Invalid values for offset: " + off + " and length: " + len); + } + + if (b.length == 0 || len == 0) { + return 0; + } + + // fill the output bytes if there aren't any left to return. + if ((outEnd_ - outStart_) <= 0) { + int newBytesLen = 0; + + // Block until a byte is read or end of stream in the underlying + // stream is reached. + while (newBytesLen == 0) { + newBytesLen = fillOutBytes(); + } + if (newBytesLen < 0) { + return -1; + } + } + + final int copyLen = Math.min((outEnd_ - outStart_), len); + System.arraycopy(outBytes_, outStart_, b, off, copyLen); + outStart_ += copyLen; + + return copyLen; + } + + /** + * {@inheritDoc} + * + * @throws BadCiphertextException + * This is thrown only during decryption if b contains invalid or corrupt + * ciphertext. + */ + @Override + public int read(final byte[] b) throws IllegalArgumentException, IOException, BadCiphertextException { + return read(b, 0, b.length); + } + + /** + * {@inheritDoc} + * + * @throws BadCiphertextException + * if b contains invalid or corrupt ciphertext. This is thrown only during + * decryption. + */ + @Override + public int read() throws IOException, BadCiphertextException { + final byte[] bArray = new byte[1]; + int result = 0; + + while (result == 0) { + result = read(bArray, 0, 1); + } + + if (result > 0) { + return (bArray[0] & 0xFF); + } else { + return result; + } + } + + @Override + public void close() throws IOException { + inputStream_.close(); + } + + /** + * Returns metadata associated with the performed cryptographic operation. + */ + @Override + public int available() throws IOException { + return (outBytes_.length + inputStream_.available()); + } + + /** + * Sets an upper bound on the size of the input data. This method should be called before reading any data from the + * stream. If this method is not called prior to reading any data, performance may be reduced (notably, it will not + * be possible to cache data keys when encrypting). + * Among other things, this size is used to enforce limits configured on the {@link CachingCryptoMaterialsManager}. + * If the input size set here is exceeded, an exception will be thrown, and the encyption or decryption will fail. + * + * @param size Maximum input size. + */ + public void setMaxInputLength(long size) { + cryptoHandler_.setMaxInputLength(size); + } + +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/DecryptionHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/DecryptionHandler.java new file mode 100644 index 0000000000000..572e456788f97 --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/DecryptionHandler.java @@ -0,0 +1,532 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; + +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.PublicKey; +import java.security.Signature; +import java.security.SignatureException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.CryptoMaterialsManager; +import com.amazonaws.encryptionsdk.DataKey; +import com.amazonaws.encryptionsdk.DefaultCryptoMaterialsManager; +import com.amazonaws.encryptionsdk.MasterKey; +import com.amazonaws.encryptionsdk.MasterKeyProvider; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import com.amazonaws.encryptionsdk.internal.CryptoHandler; +import com.amazonaws.encryptionsdk.internal.EncryptionHandler; +import com.amazonaws.encryptionsdk.internal.MessageCryptoHandler; +import com.amazonaws.encryptionsdk.internal.ProcessingSummary; +import com.amazonaws.encryptionsdk.internal.TrailingSignatureAlgorithm; +import com.amazonaws.encryptionsdk.model.CiphertextFooters; +import com.amazonaws.encryptionsdk.model.CiphertextHeaders; +import com.amazonaws.encryptionsdk.model.CiphertextType; +import com.amazonaws.encryptionsdk.model.ContentType; +import com.amazonaws.encryptionsdk.model.DecryptionMaterials; +import com.amazonaws.encryptionsdk.model.DecryptionMaterialsRequest; + +/** + * This class implements the CryptoHandler interface by providing methods for + * the decryption of ciphertext produced by the methods in + * {@link EncryptionHandler}. + * + *

+ * This class reads and parses the values in the ciphertext headers and + * delegates the decryption of the ciphertext to the + * content type parsed in the ciphertext headers. + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class DecryptionHandler> implements MessageCryptoHandler { + private final CryptoMaterialsManager materialsManager_; + + private final CiphertextHeaders ciphertextHeaders_; + private final CiphertextFooters ciphertextFooters_; + private boolean ciphertextHeadersParsed_; + + private CryptoHandler contentCryptoHandler_; + + private DataKey dataKey_; + private SecretKey decryptionKey_; + private CryptoAlgorithm cryptoAlgo_; + private Signature trailingSig_; + + private Map encryptionContext_ = null; + + private byte[] unparsedBytes_ = new byte[0]; + private boolean complete_ = false; + + private long ciphertextSizeBound_ = -1; + private long ciphertextBytesSupplied_ = 0; + + // These ctors are private to ensure type safety - we must ensure construction using a CMM results in a + // DecryptionHandler, not a DecryptionHandler, since the CryptoMaterialsManager is not itself + // genericized. + private DecryptionHandler(final CryptoMaterialsManager materialsManager) { + com.amazonaws.encryptionsdk.internal.Utils.assertNonNull(materialsManager, "materialsManager"); + + this.materialsManager_ = materialsManager; + ciphertextHeaders_ = new CiphertextHeaders(); + ciphertextFooters_ = new CiphertextFooters(); + } + + private DecryptionHandler(final CryptoMaterialsManager materialsManager, final CiphertextHeaders headers, final int frameStartNum) + throws AwsCryptoException { + com.amazonaws.encryptionsdk.internal.Utils.assertNonNull(materialsManager, "materialsManager"); + + materialsManager_ = materialsManager; + ciphertextHeaders_ = headers; + ciphertextFooters_ = new CiphertextFooters(); + readHeaderFields(headers, frameStartNum); + updateTrailingSignature(headers); + } + + /** + * Create a decryption handler using the provided master key. + * + *

+ * Note the methods in the provided master key are used in decrypting the + * encrypted data key parsed from the ciphertext headers. + * + * @param customerMasterKeyProvider + * the master key provider to use in picking a master key from + * the key blobs encoded in the provided ciphertext. + * @throws AwsCryptoException + * if the master key is null. + */ + @SuppressWarnings("unchecked") + public static > DecryptionHandler create(final MasterKeyProvider customerMasterKeyProvider) + throws AwsCryptoException { + Utils.assertNonNull(customerMasterKeyProvider, "customerMasterKeyProvider"); + + return (DecryptionHandler) create(new DefaultCryptoMaterialsManager(customerMasterKeyProvider)); + } + + /** + * Create a decryption handler using the provided materials manager. + * + *

+ * Note the methods in the provided materials manager are used in decrypting the encrypted data key + * parsed from the ciphertext headers. + * + * @param materialsManager + * the materials manager to use in decrypting the data key from the key blobs encoded + * in the provided ciphertext. + * @throws AwsCryptoException + * if the master key is null. + */ + public static DecryptionHandler create(final CryptoMaterialsManager materialsManager) throws AwsCryptoException { + return new DecryptionHandler(materialsManager); + } + + /** + * Create a decryption handler using the provided materials manager and already parsed {@code headers}. + * + *

+ * Note the methods in the provided materials manager are used in decrypting the encrypted data key + * parsed from the ciphertext headers. + * + * @param materialsManager the materials manager to use in decrypting the data key from the key + * blobs encoded in the provided ciphertext. + * @param headers already parsed headers which will not be passed into {@link + * #processBytes(byte[], int, int, byte[], int)} + * decryption; zero indicates no maximum + * @throws AwsCryptoException if the master key is null. + * @param frameStartNum Number from which assignment has to start for new frames + * @return instance of {@link DecryptionHandler} * + */ + public static DecryptionHandler create( + final CryptoMaterialsManager materialsManager, + final CiphertextHeaders headers, + final int frameStartNum + ) throws AwsCryptoException { + return new DecryptionHandler(materialsManager, headers, frameStartNum); + } + + /** + * Decrypt the ciphertext bytes provided in {@code in} and copy the plaintext bytes to + * {@code out}. + * + *

+ * This method consumes and parses the ciphertext headers. The decryption of the actual content + * is delegated to {@link FrameDecryptionHandler} based on the + * content type parsed in the ciphertext header. + * + * @param in + * the input byte array. + * @param off + * the offset into the in array where the data to be decrypted starts. + * @param len + * the number of bytes to be decrypted. + * @param out + * the output buffer the decrypted plaintext bytes go into. + * @param outOff + * the offset into the output byte array the decrypted data starts at. + * @return the number of bytes written to {@code out} and processed. + * + * @throws BadCiphertextException + * if the ciphertext header contains invalid entries or if the header integrity + * check fails. + * @throws AwsCryptoException + * if any of the offset or length arguments are negative or if the total bytes to + * decrypt exceeds the maximum allowed value. + */ + @Override + public ProcessingSummary processBytes(final byte[] in, final int off, final int len, final byte[] out, final int outOff) + throws BadCiphertextException, AwsCryptoException { + + if (len < 0 || off < 0) { + throw new AwsCryptoException("Invalid values for input offset: " + off + "and length:" + len); + } + + if (in.length == 0 || len == 0) { + return ProcessingSummary.ZERO; + } + + final long totalBytesToParse = unparsedBytes_.length + (long) len; + // check for integer overflow + if (totalBytesToParse > Integer.MAX_VALUE) { + throw new AwsCryptoException("Size of the total bytes to parse and decrypt exceeded allowed maximum:" + Integer.MAX_VALUE); + } + + checkSizeBound(len); + ciphertextBytesSupplied_ += len; + + final byte[] bytesToParse = new byte[(int) totalBytesToParse]; + final int leftoverBytes = unparsedBytes_.length; + // If there were previously unparsed bytes, add them as the first + // set of bytes to be parsed in this call. + System.arraycopy(unparsedBytes_, 0, bytesToParse, 0, unparsedBytes_.length); + System.arraycopy(in, off, bytesToParse, unparsedBytes_.length, len); + + int totalParsedBytes = 0; + if (ciphertextHeadersParsed_ == false) { + totalParsedBytes += ciphertextHeaders_.deserialize(bytesToParse, 0); + // When ciphertext headers are complete, we have the data + // key and cipher mode to initialize the underlying cipher + if (ciphertextHeaders_.isComplete() == true) { + readHeaderFields(ciphertextHeaders_, 1); + updateTrailingSignature(ciphertextHeaders_); + // reset unparsed bytes as parsing of ciphertext headers is + // complete. + unparsedBytes_ = new byte[0]; + } else { + // If there aren't enough bytes to parse ciphertext + // headers, we don't have anymore bytes to continue parsing. + // But first copy the leftover bytes to unparsed bytes. + unparsedBytes_ = Arrays.copyOfRange(bytesToParse, totalParsedBytes, bytesToParse.length); + return new ProcessingSummary(0, len); + } + } + + int actualOutLen = 0; + if (!contentCryptoHandler_.isComplete()) { + // if there are bytes to parse further, pass it off to underlying + // content cryptohandler. + if ((bytesToParse.length - totalParsedBytes) > 0) { + final ProcessingSummary contentResult = contentCryptoHandler_.processBytes( + bytesToParse, + totalParsedBytes, + bytesToParse.length - totalParsedBytes, + out, + outOff + ); + updateTrailingSignature(bytesToParse, totalParsedBytes, contentResult.getBytesProcessed()); + actualOutLen = contentResult.getBytesWritten(); + totalParsedBytes += contentResult.getBytesProcessed(); + + } + if (contentCryptoHandler_.isComplete()) { + actualOutLen += contentCryptoHandler_.doFinal(out, outOff + actualOutLen); + } + } + + if (contentCryptoHandler_.isComplete()) { + // If the crypto algorithm contains trailing signature, we will need to verify + // the footer of the message. + if (cryptoAlgo_.getTrailingSignatureLength() > 0) { + totalParsedBytes += ciphertextFooters_.deserialize(bytesToParse, totalParsedBytes); + if (ciphertextFooters_.isComplete() && trailingSig_ != null) { + try { + if (!trailingSig_.verify(ciphertextFooters_.getMAuth())) { + throw new BadCiphertextException("Bad trailing signature"); + } + } catch (final SignatureException ex) { + throw new BadCiphertextException("Bad trailing signature", ex); + } + complete_ = true; + } + } else { + complete_ = true; + } + } + return new ProcessingSummary(actualOutLen, totalParsedBytes - leftoverBytes); + } + + /** + * Finish processing of the bytes. + * + * @param out + * space for any resulting output data. + * @param outOff + * offset into {@code out} to start copying the data at. + * @return + * number of bytes written into {@code out}. + * @throws BadCiphertextException + * if the bytes do not decrypt correctly. + */ + @Override + public int doFinal(final byte[] out, final int outOff) throws BadCiphertextException { + // check if cryptohandler for content has been created. There are cases + // when it might not have been created such as when doFinal() is called + // before the ciphertext headers are fully received and parsed. + if (contentCryptoHandler_ == null) { + return 0; + } else { + + int result = contentCryptoHandler_.doFinal(out, outOff); + + if (!ciphertextHeaders_.isComplete() || !contentCryptoHandler_.isComplete()) { + throw new BadCiphertextException("Unable to process entire ciphertext."); + } + + return result; + } + } + + /** + * Return the size of the output buffer required for a + * processBytes plus a doFinal with an input of + * inLen bytes. + * + * @param inLen + * the length of the input. + * @return + * the space required to accommodate a call to processBytes and + * doFinal with input of size {@code inLen} bytes. + */ + @Override + public int estimateOutputSize(final int inLen) { + if (contentCryptoHandler_ != null) { + return contentCryptoHandler_.estimateOutputSize(inLen); + } else { + return Math.max(inLen, 0); + } + } + + @Override + public int estimatePartialOutputSize(int inLen) { + if (contentCryptoHandler_ != null) { + return contentCryptoHandler_.estimatePartialOutputSize(inLen); + } else { + return Math.max(inLen, 0); + } + } + + @Override + public int estimateFinalOutputSize() { + if (contentCryptoHandler_ != null) { + return contentCryptoHandler_.estimateFinalOutputSize(); + } else { + return 0; + } + } + + /** + * Return the encryption context. This value is parsed from the ciphertext. + * + * @return + * the key-value map containing the encryption client. + */ + @Override + public Map getEncryptionContext() { + return encryptionContext_; + } + + private void checkSizeBound(long additionalBytes) { + if (ciphertextSizeBound_ != -1 && ciphertextBytesSupplied_ + additionalBytes > ciphertextSizeBound_) { + throw new IllegalStateException("Ciphertext size exceeds size bound"); + } + } + + @Override + public void setMaxInputLength(long size) { + if (size < 0) { + throw Utils.cannotBeNegative("Max input length"); + } + + if (ciphertextSizeBound_ != -1 && ciphertextSizeBound_ < size) { + ciphertextSizeBound_ = size; + } + + // check that we haven't already exceeded the limit + checkSizeBound(0); + } + + /** + * Check integrity of the header bytes by processing the parsed MAC tag in + * the headers through the cipher. + * + * @param ciphertextHeaders + * the ciphertext headers object whose integrity needs to be + * checked. + */ + private void verifyHeaderIntegrity(final CiphertextHeaders ciphertextHeaders) throws BadCiphertextException { + final CipherHandler cipherHandler = new CipherHandler(decryptionKey_, Cipher.DECRYPT_MODE, cryptoAlgo_); + + try { + final byte[] headerTag = ciphertextHeaders.getHeaderTag(); + cipherHandler.cipherData( + ciphertextHeaders.getHeaderNonce(), + ciphertextHeaders.serializeAuthenticatedFields(), + headerTag, + 0, + headerTag.length + ); + } catch (BadCiphertextException e) { + throw new BadCiphertextException("Header integrity check failed.", e); + } + } + + /** + * Read the fields in the ciphertext headers to populate the corresponding + * instance variables used during decryption. + * + * @param ciphertextHeaders + * the ciphertext headers object to read. + */ + @SuppressWarnings("unchecked") + private void readHeaderFields(final CiphertextHeaders ciphertextHeaders, final int frameStartNum) { + cryptoAlgo_ = ciphertextHeaders.getCryptoAlgoId(); + + final CiphertextType ciphertextType = ciphertextHeaders.getType(); + if (ciphertextType != CiphertextType.CUSTOMER_AUTHENTICATED_ENCRYPTED_DATA) { + throw new BadCiphertextException("Invalid type in ciphertext."); + } + + final byte[] messageId = ciphertextHeaders.getMessageId(); + + encryptionContext_ = ciphertextHeaders.getEncryptionContextMap(); + + DecryptionMaterialsRequest request = DecryptionMaterialsRequest.newBuilder() + .setAlgorithm(cryptoAlgo_) + .setEncryptionContext(encryptionContext_) + .setEncryptedDataKeys(ciphertextHeaders.getEncryptedKeyBlobs()) + .build(); + + DecryptionMaterials result = materialsManager_.decryptMaterials(request); + + // noinspection unchecked + dataKey_ = (DataKey) result.getDataKey(); + PublicKey trailingPublicKey = result.getTrailingSignatureKey(); + + try { + decryptionKey_ = cryptoAlgo_.getEncryptionKeyFromDataKey(dataKey_.getKey(), ciphertextHeaders); + } catch (final InvalidKeyException ex) { + throw new AwsCryptoException(ex); + } + + if (cryptoAlgo_.getTrailingSignatureLength() > 0) { + Utils.assertNonNull(trailingPublicKey, "trailing public key"); + + TrailingSignatureAlgorithm trailingSignatureAlgorithm = TrailingSignatureAlgorithm.forCryptoAlgorithm(cryptoAlgo_); + + try { + trailingSig_ = Signature.getInstance(trailingSignatureAlgorithm.getHashAndSignAlgorithm()); + + trailingSig_.initVerify(trailingPublicKey); + } catch (GeneralSecurityException e) { + throw new AwsCryptoException(e); + } + } else { + if (trailingPublicKey != null) { + throw new AwsCryptoException("Unexpected trailing signature key in context"); + } + + trailingSig_ = null; + } + + final ContentType contentType = ciphertextHeaders.getContentType(); + + final short nonceLen = ciphertextHeaders.getNonceLength(); + final int frameLen = ciphertextHeaders.getFrameLength(); + + verifyHeaderIntegrity(ciphertextHeaders); + + // should never get here because an invalid content type is + // detected when parsing. + if (Objects.requireNonNull(contentType) == ContentType.FRAME) { + contentCryptoHandler_ = new FrameDecryptionHandler( + decryptionKey_, + (byte) nonceLen, + cryptoAlgo_, + messageId, + frameLen, + frameStartNum + ); + } + + ciphertextHeadersParsed_ = true; + } + + private void updateTrailingSignature(final CiphertextHeaders headers) { + if (trailingSig_ != null) { + final byte[] reserializedHeaders = headers.toByteArray(); + updateTrailingSignature(reserializedHeaders, 0, reserializedHeaders.length); + } + } + + private void updateTrailingSignature(byte[] input, int offset, int len) { + if (trailingSig_ != null) { + try { + trailingSig_.update(input, offset, len); + } catch (final SignatureException ex) { + throw new AwsCryptoException(ex); + } + } + } + + @Override + public CiphertextHeaders getHeaders() { + return ciphertextHeaders_; + } + + @Override + public List getMasterKeys() { + return Collections.singletonList(dataKey_.getMasterKey()); + } + + @Override + public boolean isComplete() { + return complete_; + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionHandler.java new file mode 100644 index 0000000000000..af4521cf4ee13 --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionHandler.java @@ -0,0 +1,368 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import org.bouncycastle.asn1.ASN1Encodable; +import org.bouncycastle.asn1.ASN1Integer; +import org.bouncycastle.asn1.ASN1Sequence; +import org.bouncycastle.asn1.DERSequence; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.PrivateKey; +import java.security.Signature; +import java.security.SignatureException; +import java.security.interfaces.ECPrivateKey; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.MasterKey; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import com.amazonaws.encryptionsdk.internal.CryptoHandler; +import com.amazonaws.encryptionsdk.internal.MessageCryptoHandler; +import com.amazonaws.encryptionsdk.internal.ProcessingSummary; +import com.amazonaws.encryptionsdk.model.CiphertextFooters; +import com.amazonaws.encryptionsdk.model.CiphertextHeaders; +import com.amazonaws.encryptionsdk.model.CiphertextType; +import com.amazonaws.encryptionsdk.model.ContentType; +import com.amazonaws.encryptionsdk.model.KeyBlob; + +/** + * This class implements the CryptoHandler interface by providing methods for the encryption of + * plaintext data. + * + *

+ * This class creates the ciphertext headers and delegates the encryption of the plaintext to the + */ +@SuppressWarnings({ "rawtypes", "unchecked" }) +public class EncryptionHandler implements MessageCryptoHandler { + + private final Map encryptionContext_; + private final CryptoAlgorithm cryptoAlgo_; + private final List masterKeys_; + private final List keyBlobs_; + private final byte version_; + private final CiphertextType type_; + private final byte nonceLen_; + private final PrivateKey trailingSignaturePrivateKey_; + private final MessageDigest trailingDigest_; + private final Signature trailingSig_; + + private final CiphertextHeaders ciphertextHeaders_; + private final byte[] ciphertextHeaderBytes_; + private final CryptoHandler contentCryptoHandler_; + + private boolean firstOperation_; + private boolean complete_ = false; + + private long plaintextBytes_ = 0; + private long plaintextByteLimit_ = -1; + + /** + * Create an encryption handler using the provided master key and encryption context. + * @param encryptionMetadata Context object created before encryption + * @param isFirstStream In case of first stream, file header is additionally created which consists of crypto + * materials. + * @param frameStartNumber Number from which assignment has to start for new frames + */ + public EncryptionHandler(EncryptionMetadata encryptionMetadata, boolean isFirstStream, int frameStartNumber) throws AwsCryptoException { + this.encryptionContext_ = encryptionMetadata.getEncryptionContext(); + this.cryptoAlgo_ = encryptionMetadata.getCryptoAlgo(); + this.masterKeys_ = encryptionMetadata.getMasterKeys(); + this.keyBlobs_ = encryptionMetadata.getKeyBlobs(); + this.trailingSignaturePrivateKey_ = encryptionMetadata.getTrailingSignaturePrivateKey(); + + if (keyBlobs_.isEmpty()) { + throw new IllegalArgumentException("No encrypted data keys in materials result"); + } + + if (trailingSignaturePrivateKey_ != null) { + trailingDigest_ = encryptionMetadata.getTrailingDigest(); + trailingSig_ = encryptionMetadata.getTrailingSig(); + } else { + trailingDigest_ = null; + trailingSig_ = null; + } + + version_ = encryptionMetadata.getVersion(); + type_ = encryptionMetadata.getType(); + nonceLen_ = encryptionMetadata.getNonceLen(); + ciphertextHeaders_ = encryptionMetadata.getCiphertextHeaders(); + ciphertextHeaderBytes_ = encryptionMetadata.getCiphertextHeaderBytes(); + firstOperation_ = isFirstStream; + + byte[] messageId_ = encryptionMetadata.getMessageId(); + + if (encryptionMetadata.getContentType() == ContentType.FRAME) { + contentCryptoHandler_ = new FrameEncryptionHandler( + encryptionMetadata.getEncryptionKey(), + nonceLen_, + cryptoAlgo_, + messageId_, + encryptionMetadata.getFrameSize(), + frameStartNumber + ); + } else {// should never get here because a valid content type is always + // set above based on the frame size. + throw new AwsCryptoException("Unknown content type."); + } + } + + /** + * Encrypt a block of bytes from {@code in} putting the plaintext result into {@code out}. + * + *

+ * It encrypts by performing the following operations: + *

    + *
  1. if this is the first call to encrypt, write the ciphertext headers to the output being + * returned.
    2. + *
  2. else, pass off the input data to underlying content cryptohandler.
    3. + *
+ * + * @param in + * the input byte array. + * @param off + * the offset into the in array where the data to be encrypted starts. + * @param len + * the number of bytes to be encrypted. + * @param out + * the output buffer the encrypted bytes go into. + * @param outOff + * the offset into the output byte array the encrypted data starts at. + * @return the number of bytes written to out and processed + * @throws AwsCryptoException + * if len or offset values are negative. + * @throws BadCiphertextException + * thrown by the underlying cipher handler. + */ + @Override + public ProcessingSummary processBytes(final byte[] in, final int off, final int len, final byte[] out, final int outOff) + throws AwsCryptoException, BadCiphertextException { + if (len < 0 || off < 0) { + throw new AwsCryptoException( + String.format(Locale.getDefault(), "Invalid values for input offset: %d and length: %d", off, len) + ); + } + + checkPlaintextSizeLimit(len); + + int actualOutLen = 0; + + if (firstOperation_ == true) { + System.arraycopy(ciphertextHeaderBytes_, 0, out, outOff, ciphertextHeaderBytes_.length); + actualOutLen += ciphertextHeaderBytes_.length; + + firstOperation_ = false; + } + + ProcessingSummary contentOut = contentCryptoHandler_.processBytes(in, off, len, out, outOff + actualOutLen); + actualOutLen += contentOut.getBytesWritten(); + updateTrailingSignature(out, outOff, actualOutLen); + plaintextBytes_ += contentOut.getBytesProcessed(); + return new ProcessingSummary(actualOutLen, contentOut.getBytesProcessed()); + } + + /** + * Finish encryption of the plaintext bytes. + * + * @param out + * space for any resulting output data. + * @param outOff + * offset into out to start copying the data at. + * @return number of bytes written into out. + * @throws BadCiphertextException + * thrown by the underlying cipher handler. + */ + @Override + public int doFinal(final byte[] out, final int outOff) throws BadCiphertextException { + if (complete_) { + throw new IllegalStateException("Attempted to call doFinal twice"); + } + + complete_ = true; + + checkPlaintextSizeLimit(0); + + int written = contentCryptoHandler_.doFinal(out, outOff); + updateTrailingSignature(out, outOff, written); + if (cryptoAlgo_.getTrailingSignatureLength() > 0) { + try { + CiphertextFooters footer = new CiphertextFooters(signContent()); + byte[] fBytes = footer.toByteArray(); + System.arraycopy(fBytes, 0, out, outOff + written, fBytes.length); + return written + fBytes.length; + } catch (final SignatureException ex) { + throw new AwsCryptoException(ex); + } + } else { + return written; + } + } + + private byte[] signContent() throws SignatureException { + if (trailingDigest_ != null) { + if (!trailingSig_.getAlgorithm().contains("ECDSA")) { + throw new UnsupportedOperationException("Signatures calculated in pieces is only supported for ECDSA."); + } + final byte[] digest = trailingDigest_.digest(); + return generateEcdsaFixedLengthSignature(digest); + } + return trailingSig_.sign(); + } + + private byte[] generateEcdsaFixedLengthSignature(final byte[] digest) throws SignatureException { + byte[] signature; + // Unfortunately, we need deterministic lengths some signatures are non-deterministic in length. + // So, retry until we get the right length :-( + do { + trailingSig_.update(digest); + signature = trailingSig_.sign(); + if (signature.length != cryptoAlgo_.getTrailingSignatureLength()) { + // Most of the time, a signature of the wrong length can be fixed + // be negating s in the signature relative to the group order. + ASN1Sequence seq = ASN1Sequence.getInstance(signature); + ASN1Integer r = (ASN1Integer) seq.getObjectAt(0); + ASN1Integer s = (ASN1Integer) seq.getObjectAt(1); + ECPrivateKey ecKey = (ECPrivateKey) trailingSignaturePrivateKey_; + s = new ASN1Integer(ecKey.getParams().getOrder().subtract(s.getPositiveValue())); + seq = new DERSequence(new ASN1Encodable[] { r, s }); + try { + signature = seq.getEncoded(); + } catch (IOException ex) { + throw new SignatureException(ex); + } + } + } while (signature.length != cryptoAlgo_.getTrailingSignatureLength()); + return signature; + } + + /** + * Return the size of the output buffer required for a {@code processBytes} plus a + * {@code doFinal} with an input of inLen bytes. + * + * @param inLen + * the length of the input. + * @return the space required to accommodate a call to processBytes and doFinal with len bytes + * of input. + */ + @Override + public int estimateOutputSize(final int inLen) { + int outSize = 0; + if (firstOperation_ == true) { + outSize += ciphertextHeaderBytes_.length; + } + outSize += contentCryptoHandler_.estimateOutputSize(inLen); + + outSize += getAlgoTrailingLength(cryptoAlgo_); + + return outSize; + } + + public static int getAlgoTrailingLength(CryptoAlgorithm cryptoAlgo) { + int outSize = 0; + if (cryptoAlgo.getTrailingSignatureLength() > 0) { + outSize += 2; // Length field in footer + outSize += cryptoAlgo.getTrailingSignatureLength(); + } + + return outSize; + } + + @Override + public int estimatePartialOutputSize(int inLen) { + int outSize = 0; + if (firstOperation_ == true) { + outSize += ciphertextHeaderBytes_.length; + } + outSize += contentCryptoHandler_.estimatePartialOutputSize(inLen); + + return outSize; + } + + @Override + public int estimateFinalOutputSize() { + return estimateOutputSize(0); + } + + /** + * Return the encryption context. + * + * @return the key-value map containing encryption context. + */ + @Override + public Map getEncryptionContext() { + return encryptionContext_; + } + + @Override + public CiphertextHeaders getHeaders() { + return ciphertextHeaders_; + } + + @Override + public void setMaxInputLength(long size) { + if (size < 0) { + throw Utils.cannotBeNegative("Max input length"); + } + + if (plaintextByteLimit_ == -1 || plaintextByteLimit_ > size) { + plaintextByteLimit_ = size; + } + + // check that we haven't already exceeded the limit + checkPlaintextSizeLimit(0); + } + + private void checkPlaintextSizeLimit(long additionalBytes) { + if (plaintextByteLimit_ != -1 && plaintextBytes_ + additionalBytes > plaintextByteLimit_) { + throw new IllegalStateException("Plaintext size exceeds max input size limit"); + } + } + + @Override + @SuppressWarnings("unchecked") + public List> getMasterKeys() { + // noinspection unchecked + return (List) masterKeys_; // This is unmodifiable + } + + private void updateTrailingSignature(byte[] input, int offset, int len) { + if (trailingDigest_ != null) { + trailingDigest_.update(input, offset, len); + } else if (trailingSig_ != null) { + try { + trailingSig_.update(input, offset, len); + } catch (final SignatureException ex) { + throw new AwsCryptoException(ex); + } + } + + } + + @Override + public boolean isComplete() { + return complete_; + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionMetadata.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionMetadata.java new file mode 100644 index 0000000000000..3663b25c2a8ae --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/EncryptionMetadata.java @@ -0,0 +1,252 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; + +import java.security.GeneralSecurityException; +import java.security.InvalidKeyException; +import java.security.MessageDigest; +import java.security.PrivateKey; +import java.security.Signature; +import java.util.List; +import java.util.Map; + +import com.amazonaws.encryptionsdk.CommitmentPolicy; +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.MasterKey; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.internal.EncryptionContextSerializer; +import com.amazonaws.encryptionsdk.internal.TrailingSignatureAlgorithm; +import com.amazonaws.encryptionsdk.model.CiphertextHeaders; +import com.amazonaws.encryptionsdk.model.CiphertextType; +import com.amazonaws.encryptionsdk.model.ContentType; +import com.amazonaws.encryptionsdk.model.EncryptionMaterials; +import com.amazonaws.encryptionsdk.model.KeyBlob; + +@SuppressWarnings({ "rawtypes" }) +public class EncryptionMetadata { + private static final CiphertextType CIPHERTEXT_TYPE = CiphertextType.CUSTOMER_AUTHENTICATED_ENCRYPTED_DATA; + + private final Map encryptionContext_; + private final CryptoAlgorithm cryptoAlgo; + private final List masterKeys; + private final List keyBlobs; + private final SecretKey encryptionKey; + private final byte version; + private final CiphertextType type; + private final byte nonceLen; + + private final CiphertextHeaders ciphertextHeaders; + private final byte[] ciphertextHeaderBytes; + private final byte[] messageId; + private final int frameSize; + private final PrivateKey trailingSignaturePrivateKey; + private final MessageDigest trailingDigest; + private final Signature trailingSig; + private final ContentType contentType; + + public EncryptionMetadata(int frameSize, EncryptionMaterials result) throws AwsCryptoException { + Utils.assertNonNull(result, "result"); + + this.encryptionContext_ = result.getEncryptionContext(); + + this.cryptoAlgo = result.getAlgorithm(); + this.masterKeys = result.getMasterKeys(); + this.keyBlobs = result.getEncryptedDataKeys(); + this.trailingSignaturePrivateKey = result.getTrailingSignatureKey(); + + if (keyBlobs.isEmpty()) { + throw new IllegalArgumentException("No encrypted data keys in materials result"); + } + + if (trailingSignaturePrivateKey != null) { + try { + TrailingSignatureAlgorithm algorithm = TrailingSignatureAlgorithm.forCryptoAlgorithm(cryptoAlgo); + trailingDigest = MessageDigest.getInstance(algorithm.getMessageDigestAlgorithm()); + trailingSig = Signature.getInstance(algorithm.getRawSignatureAlgorithm()); + + trailingSig.initSign(trailingSignaturePrivateKey, com.amazonaws.encryptionsdk.internal.Utils.getSecureRandom()); + } catch (final GeneralSecurityException ex) { + throw new AwsCryptoException(ex); + } + } else { + trailingDigest = null; + trailingSig = null; + } + + // set default values + version = cryptoAlgo.getMessageFormatVersion(); + + // only allow to encrypt with version 1 crypto algorithms + if (version != 1) { + throw new AwsCryptoException( + "Configuration conflict. Cannot encrypt due to CommitmentPolicy " + + CommitmentPolicy.ForbidEncryptAllowDecrypt + + " requiring only non-committed messages. Algorithm ID was " + + cryptoAlgo + + ". See: https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/troubleshooting-migration.html" + ); + } + + type = CIPHERTEXT_TYPE; + nonceLen = cryptoAlgo.getNonceLen(); + + if (frameSize > 0) { + contentType = ContentType.FRAME; + } else if (frameSize == 0) { + contentType = ContentType.SINGLEBLOCK; + } else { + throw Utils.cannotBeNegative("Frame size"); + } + + final CiphertextHeaders unsignedHeaders = createCiphertextHeaders(contentType, frameSize); + try { + encryptionKey = cryptoAlgo.getEncryptionKeyFromDataKey(result.getCleartextDataKey(), unsignedHeaders); + } catch (final InvalidKeyException ex) { + throw new AwsCryptoException(ex); + } + ciphertextHeaders = signCiphertextHeaders(unsignedHeaders); + ciphertextHeaderBytes = ciphertextHeaders.toByteArray(); + messageId = ciphertextHeaders.getMessageId(); + this.frameSize = frameSize; + } + + public ContentType getContentType() { + return contentType; + } + + /** + * Create ciphertext headers using the instance variables, and the provided content type and + * frame size. + * + * @param contentType + * the content type to set in the ciphertext headers. + * @param frameSize + * the frame size to set in the ciphertext headers. + * @return the bytes containing the ciphertext headers. + */ + private CiphertextHeaders createCiphertextHeaders(final ContentType contentType, final int frameSize) { + // create the ciphertext headers + final byte[] headerNonce = new byte[nonceLen]; + // We use a deterministic IV of zero for the header authentication. + + final byte[] encryptionContextBytes = EncryptionContextSerializer.serialize(encryptionContext_); + final CiphertextHeaders ciphertextHeaders = new CiphertextHeaders( + type, + cryptoAlgo, + encryptionContextBytes, + keyBlobs, + contentType, + frameSize + ); + ciphertextHeaders.setHeaderNonce(headerNonce); + + return ciphertextHeaders; + } + + private CiphertextHeaders signCiphertextHeaders(final CiphertextHeaders unsignedHeaders) { + final byte[] headerFields = unsignedHeaders.serializeAuthenticatedFields(); + final byte[] headerTag = computeHeaderTag(unsignedHeaders.getHeaderNonce(), headerFields); + + unsignedHeaders.setHeaderTag(headerTag); + + return unsignedHeaders; + } + + /** + * Compute the MAC tag of the header bytes using the provided key, nonce, AAD, and crypto + * algorithm identifier. + * + * @param nonce + * the nonce to use in computing the MAC tag. + * @param aad + * the AAD to use in computing the MAC tag. + * @return the bytes containing the computed MAC tag. + */ + private byte[] computeHeaderTag(final byte[] nonce, final byte[] aad) { + final CipherHandler cipherHandler = new CipherHandler(encryptionKey, Cipher.ENCRYPT_MODE, cryptoAlgo); + + return cipherHandler.cipherData(nonce, aad, new byte[0], 0, 0); + } + + public Map getEncryptionContext() { + return encryptionContext_; + } + + public CryptoAlgorithm getCryptoAlgo() { + return cryptoAlgo; + } + + public List getMasterKeys() { + return masterKeys; + } + + public List getKeyBlobs() { + return keyBlobs; + } + + public SecretKey getEncryptionKey() { + return encryptionKey; + } + + public byte getVersion() { + return version; + } + + public CiphertextType getType() { + return type; + } + + public byte getNonceLen() { + return nonceLen; + } + + public CiphertextHeaders getCiphertextHeaders() { + return ciphertextHeaders; + } + + public byte[] getCiphertextHeaderBytes() { + return ciphertextHeaderBytes; + } + + public byte[] getMessageId() { + return messageId; + } + + public int getFrameSize() { + return frameSize; + } + + public PrivateKey getTrailingSignaturePrivateKey() { + return trailingSignaturePrivateKey; + } + + public MessageDigest getTrailingDigest() { + return trailingDigest; + } + + public Signature getTrailingSig() { + return trailingSig; + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameCryptoHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameCryptoHandler.java new file mode 100644 index 0000000000000..8b1028bad1110 --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameCryptoHandler.java @@ -0,0 +1,234 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.encryption.frame; + +import org.opensearch.common.crypto.CryptoHandler; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; +import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.encryption.TrimmingStream; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import com.amazonaws.encryptionsdk.ParsedCiphertext; + +public class FrameCryptoHandler implements CryptoHandler { + private final AwsCrypto awsCrypto; + private final Map encryptionContext; + + // package private for tests + private final int FRAME_SIZE = 8 * 1024; + + public FrameCryptoHandler(AwsCrypto awsCrypto, Map encryptionContext) { + this.awsCrypto = awsCrypto; + this.encryptionContext = encryptionContext; + } + + public int getFrameSize() { + return FRAME_SIZE; + } + + /** + * Initialises metadata store used in encryption. + * @return crypto metadata object constructed with encryption metadata like data key pair, encryption algorithm, etc. + */ + public EncryptionMetadata initEncryptionMetadata() { + return awsCrypto.createCryptoContext(encryptionContext, getFrameSize()); + } + + /** + * Context: This SDK uses Frame encryption which means that encrypted content is composed of frames i.e., a frame + * is the smallest unit of encryption or decryption. + * Due to this in cases where more than one stream is used to produce content, each stream content except the + * last should line up along the frame boundary i.e. there can't be any partial frame. + * Hence, size of each stream except the last, should be exactly divisible by the frame size and therefore, this + * method should be called before committing on the stream size. + * This is not required if number of streams for a content is only 1. + * + * @param encryptionMetadata stateful object for a request consisting of materials required in encryption. + * @param streamSize Size of the stream to be adjusted. + * @return Adjusted size of the stream. + */ + public long adjustContentSizeForPartialEncryption(EncryptionMetadata encryptionMetadata, long streamSize) { + return (streamSize - (streamSize % encryptionMetadata.getFrameSize())) + encryptionMetadata.getFrameSize(); + } + + /** + * Estimate length of the encrypted stream. + * + * @param encryptionMetadata crypto metadata instance + * @param contentLength Size of the raw content + * @return Calculated size of the encrypted stream for the provided raw stream. + */ + public long estimateEncryptedLengthOfEntireContent(EncryptionMetadata encryptionMetadata, long contentLength) { + return encryptionMetadata.getCiphertextHeaderBytes().length + awsCrypto.estimateOutputSizeWithFooter( + encryptionMetadata.getFrameSize(), + encryptionMetadata.getNonceLen(), + encryptionMetadata.getCryptoAlgo().getTagLen(), + contentLength, + encryptionMetadata.getCryptoAlgo() + ); + } + + /** + * Estimate length of the decrypted stream. + * + * @param parsedCiphertext crypto metadata instance + * @param contentLength Size of the encrypted content + * @return Calculated size of the encrypted stream for the provided raw stream. + */ + public long estimateDecryptedLength(ParsedCiphertext parsedCiphertext, long contentLength) { + return awsCrypto.estimateDecryptedSize( + parsedCiphertext.getFrameLength(), + parsedCiphertext.getNonceLength(), + parsedCiphertext.getCryptoAlgoId().getTagLen(), + contentLength - parsedCiphertext.getOffset(), + parsedCiphertext.getCryptoAlgoId() + ); + } + + /** + * Wraps a raw InputStream with encrypting stream + * @param encryptionMetadata consists encryption metadata. + * @param stream Raw InputStream to encrypt + * @return encrypting stream wrapped around raw InputStream. + */ + public InputStreamContainer createEncryptingStream(EncryptionMetadata encryptionMetadata, InputStreamContainer stream) { + return createEncryptingStreamOfPart(encryptionMetadata, stream, 1, 0); + } + + /** + * Provides encrypted stream for a raw stream emitted for a part of content. This method doesn't require streams of + * the content to be provided in sequence and is thread safe. + * Note: This method assumes that all streams except the last stream are of same size. Also, length of the stream + * except the last index must exactly align with frame length. + * + * @param encryptionMetadata stateful object for a request consisting of materials required in encryption. + * @param stream raw stream for which encrypted stream has to be created. + * @param totalStreams Number of streams being used for the entire content. + * @param streamIdx Index of the current stream. + * @return Encrypted stream for the provided raw stream. + */ + public InputStreamContainer createEncryptingStreamOfPart( + EncryptionMetadata encryptionMetadata, + InputStreamContainer stream, + int totalStreams, + int streamIdx + ) { + int frameStartNumber = (int) (stream.getOffset() / getFrameSize()) + 1; + + return awsCrypto.createEncryptingStream(stream, streamIdx, totalStreams, frameStartNumber, encryptionMetadata); + } + + /** + * + * @param encryptedHeaderContentSupplier Supplier used to fetch bytes from source for header creation + * @return parsed encryption metadata object + * @throws IOException if content fetch for header creation fails + */ + public ParsedCiphertext loadEncryptionMetadata(EncryptedHeaderContentSupplier encryptedHeaderContentSupplier) throws IOException { + byte[] encryptedHeader = encryptedHeaderContentSupplier.supply(0, 4095); + return new ParsedCiphertext(encryptedHeader); + } + + /** + * This method accepts an encrypted stream and provides a decrypting wrapper. + * + * @param encryptedStream to be decrypted. + * @return Decrypting wrapper stream + */ + public InputStream createDecryptingStream(InputStream encryptedStream) { + return awsCrypto.createDecryptingStream(encryptedStream); + } + + /** + * Provides trailing signature length if any based on the crypto algorithm used. + * @param encryptionMetadata Context object needed to calculate trailing length. + * @return Trailing signature length + */ + public int getTrailingSignatureLength(EncryptionMetadata encryptionMetadata) { + return awsCrypto.getTrailingSignatureSize(encryptionMetadata.getCryptoAlgo()); + } + + private InputStream createBlockDecryptionStream( + ParsedCiphertext parsedCiphertext, + InputStream inputStream, + long startPosOfRawContent, + long endPosOfRawContent, + long[] encryptedRange + ) { + if (startPosOfRawContent % parsedCiphertext.getFrameLength() != 0 + || (endPosOfRawContent + 1) % parsedCiphertext.getFrameLength() != 0) { + throw new IllegalArgumentException("Start and end positions of the raw content must be aligned with frame length"); + } + int frameStartNumber = (int) (startPosOfRawContent / parsedCiphertext.getFrameLength()) + 1; + long encryptedSize = encryptedRange[1] - encryptedRange[0] + 1; + return awsCrypto.createDecryptingStream(inputStream, encryptedSize, parsedCiphertext, frameStartNumber, false); + } + + /** + * For partial reads of encrypted content, frame based encryption requires the range of content to be adjusted for + * successful decryption. Adjusted range may or may not be same as the provided range. If range is adjusted then + * starting offset of resultant range can be lesser than the starting offset of provided range and end + * offset can be greater than the ending offset of the provided range. + * It provides supplier for creating decrypted stream out of the provided encrypted stream. Decrypted content is + * trimmed down to the desired range with the help of bounded stream. This method assumes that provided encrypted + * stream supplies content for the adjusted range. + * + * @param encryptionMetadata crypto metadata instance consisting of encryption metadata used in encryption. + * @param startPosOfRawContent starting position in the raw/decrypted content + * @param endPosOfRawContent ending position in the raw/decrypted content + * @return stream provider for decrypted stream for the specified range of content including adjusted range + */ + public DecryptedRangedStreamProvider createDecryptingStreamOfRange( + ParsedCiphertext encryptionMetadata, + long startPosOfRawContent, + long endPosOfRawContent + ) { + + long adjustedStartPos = startPosOfRawContent - (startPosOfRawContent % encryptionMetadata.getFrameLength()); + long endPosOverhead = (endPosOfRawContent + 1) % encryptionMetadata.getFrameLength(); + long adjustedEndPos = endPosOverhead == 0 + ? endPosOfRawContent + : (endPosOfRawContent - endPosOverhead + encryptionMetadata.getFrameLength()); + long[] encryptedRange = transformToEncryptedRange(encryptionMetadata, adjustedStartPos, adjustedEndPos); + return new DecryptedRangedStreamProvider(encryptedRange, (encryptedStream) -> { + InputStream decryptedStream = createBlockDecryptionStream( + encryptionMetadata, + encryptedStream, + adjustedStartPos, + adjustedEndPos, + encryptedRange + ); + return new TrimmingStream(adjustedStartPos, adjustedEndPos, startPosOfRawContent, endPosOfRawContent, decryptedStream); + }); + } + + private long[] transformToEncryptedRange(ParsedCiphertext parsedCiphertext, long startPosOfRawContent, long endPosOfRawContent) { + + long startPos = awsCrypto.estimatePartialOutputSize( + parsedCiphertext.getFrameLength(), + parsedCiphertext.getCryptoAlgoId().getNonceLen(), + parsedCiphertext.getCryptoAlgoId().getTagLen(), + startPosOfRawContent + ) + parsedCiphertext.getOffset(); + + long endPos = awsCrypto.estimatePartialOutputSize( + parsedCiphertext.getFrameLength(), + parsedCiphertext.getCryptoAlgoId().getNonceLen(), + parsedCiphertext.getCryptoAlgoId().getTagLen(), + endPosOfRawContent + ) + parsedCiphertext.getOffset(); + + return new long[] { startPos, endPos }; + } + +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameDecryptionHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameDecryptionHandler.java new file mode 100644 index 0000000000000..2612e7608774d --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameDecryptionHandler.java @@ -0,0 +1,319 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; + +import java.util.Arrays; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import com.amazonaws.encryptionsdk.internal.Constants; +import com.amazonaws.encryptionsdk.internal.CryptoHandler; +import com.amazonaws.encryptionsdk.internal.ProcessingSummary; +import com.amazonaws.encryptionsdk.model.CipherFrameHeaders; + +/** + * The frame decryption handler is a subclass of the decryption handler and + * thereby provides an implementation of the Cryptography handler. + * + *

+ * It implements methods for decrypting content that was encrypted and stored in + * frames. + */ +class FrameDecryptionHandler implements CryptoHandler { + private final SecretKey decryptionKey_; + private final CryptoAlgorithm cryptoAlgo_; + private final CipherHandler cipherHandler_; + private final byte[] messageId_; + + private final short nonceLen_; + + private CipherFrameHeaders currentFrameHeaders_; + private final int frameSize_; + private long frameNumber_; + + boolean complete_ = false; + private byte[] unparsedBytes_ = new byte[0]; + + /** + * Construct a decryption handler for decrypting bytes stored in frames. + * + */ + FrameDecryptionHandler( + final SecretKey decryptionKey, + final short nonceLen, + final CryptoAlgorithm cryptoAlgo, + final byte[] messageId, + final int frameLen, + final int frameStartNumber + ) { + decryptionKey_ = decryptionKey; + nonceLen_ = nonceLen; + cryptoAlgo_ = cryptoAlgo; + messageId_ = messageId; + frameSize_ = frameLen; + cipherHandler_ = new CipherHandler(decryptionKey_, Cipher.DECRYPT_MODE, cryptoAlgo_); + frameNumber_ = frameStartNumber; + } + + /** + * Decrypt the ciphertext bytes containing content encrypted using frames and put the plaintext + * bytes into out. + * + *

+ * It decrypts by performing the following operations: + *

    + *
  1. parse the ciphertext headers
    2. + *
  2. parse the ciphertext until encrypted content in a frame is available
    3. + *
  3. decrypt the encrypted content
    4. + *
  4. return decrypted bytes as output
    5. + *
+ * + * @param in + * the input byte array. + * @param out + * the output buffer the decrypted plaintext bytes go into. + * @param outOff + * the offset into the output byte array the decrypted data starts at. + * @return the number of bytes written to out and processed + * @throws AwsCryptoException + * if the content type found in the headers is not of frame type. + */ + @Override + public ProcessingSummary processBytes(final byte[] in, final int off, final int len, final byte[] out, final int outOff) + throws AwsCryptoException { + + if (complete_) { + throw new AwsCryptoException("Ciphertext has already been processed."); + } + + final long totalBytesToParse = unparsedBytes_.length + (long) len; + if (totalBytesToParse > Integer.MAX_VALUE) { + throw new AwsCryptoException("Integer overflow of the total bytes to parse and decrypt occured."); + } + + final byte[] bytesToParse = new byte[(int) totalBytesToParse]; + // If there were previously unparsed bytes, add them as the first + // set of bytes to be parsed in this call. + System.arraycopy(unparsedBytes_, 0, bytesToParse, 0, unparsedBytes_.length); + System.arraycopy(in, off, bytesToParse, unparsedBytes_.length, len); + + int actualOutLen = 0; + int totalParsedBytes = 0; + + // Parse available bytes. Stop parsing when there aren't enough + // bytes to complete parsing: + // - the ciphertext headers + // - the cipher frame + while (!complete_ && totalParsedBytes < bytesToParse.length) { + if (currentFrameHeaders_ == null) { + currentFrameHeaders_ = new CipherFrameHeaders(); + currentFrameHeaders_.setNonceLength(nonceLen_); + if (frameSize_ == 0) { + // if frame size in ciphertext headers is 0, the frame size + // will need to be parsed in individual frame headers. + currentFrameHeaders_.includeFrameSize(true); + } + } + + totalParsedBytes += currentFrameHeaders_.deserialize(bytesToParse, totalParsedBytes); + + // if we have all frame fields, process the encrypted content. + if (currentFrameHeaders_.isComplete() == true) { + int protectedContentLen = -1; + if (currentFrameHeaders_.isFinalFrame()) { + protectedContentLen = currentFrameHeaders_.getFrameContentLength(); + + // The final frame should not be able to exceed the frameLength + if (frameSize_ > 0 && protectedContentLen > frameSize_) { + throw new BadCiphertextException("Final frame length exceeds frame length."); + } + } else { + protectedContentLen = frameSize_; + } + + // include the tag which is added by the underlying cipher. + protectedContentLen += cryptoAlgo_.getTagLen(); + + if ((bytesToParse.length - totalParsedBytes) < protectedContentLen) { + // if we don't have all of the encrypted bytes, break + // until they become available. + break; + } + + final byte[] bytesToDecrypt_ = Arrays.copyOfRange(bytesToParse, totalParsedBytes, totalParsedBytes + protectedContentLen); + totalParsedBytes += protectedContentLen; + + if (frameNumber_ == com.amazonaws.encryptionsdk.internal.Constants.MAX_FRAME_NUMBER) { + throw new BadCiphertextException("Frame number exceeds the maximum allowed value."); + } + + final byte[] decryptedBytes = decryptContent(bytesToDecrypt_, 0, bytesToDecrypt_.length); + + System.arraycopy(decryptedBytes, 0, out, (outOff + actualOutLen), decryptedBytes.length); + actualOutLen += decryptedBytes.length; + frameNumber_++; + + complete_ = currentFrameHeaders_.isFinalFrame(); + // reset frame headers as we are done processing current frame. + currentFrameHeaders_ = null; + } else { + // if there aren't enough bytes to parse cipher frame, + // we can't continue parsing. + break; + } + } + + if (!complete_) { + // buffer remaining bytes for parsing in the next round. + unparsedBytes_ = Arrays.copyOfRange(bytesToParse, totalParsedBytes, bytesToParse.length); + return new ProcessingSummary(actualOutLen, len); + } else { + final ProcessingSummary result = new ProcessingSummary(actualOutLen, totalParsedBytes - unparsedBytes_.length); + unparsedBytes_ = new byte[0]; + return result; + } + } + + /** + * Finish processing of the bytes. This function does nothing since the + * final frame will be processed and decrypted in processBytes(). + * + * @param out + * space for any resulting output data. + * @param outOff + * offset into out to start copying the data at. + * @return + * 0 + */ + @Override + public int doFinal(final byte[] out, final int outOff) { + if (!complete_) { + throw new BadCiphertextException("Unable to process entire ciphertext."); + } + + return 0; + } + + /** + * Return the size of the output buffer required for a processBytes plus a + * doFinal with an input of inLen bytes. + * + * @param inLen + * the length of the input. + * @return + * the space required to accommodate a call to processBytes and + * doFinal with len bytes of input. + */ + @Override + public int estimateOutputSize(final int inLen) { + int outSize = 0; + + final int totalBytesToDecrypt = unparsedBytes_.length + inLen; + if (totalBytesToDecrypt > 0) { + int frames = totalBytesToDecrypt / frameSize_; + frames += 1; // add one for final frame which might be < frame size. + outSize += (frameSize_ * frames); + } + + return outSize; + } + + public static long estimateDecryptedSize(long encryptedSize, int frameSize, int nonceLen, int tagLenBytes) { + // Calculate the size of sequence number for the last frame + long lastFrameSeqNumberSize = (Integer.SIZE / Byte.SIZE); + + // Calculate the size of the final frame size + long finalFrameSizeSize = (Integer.SIZE / Byte.SIZE); + + // Calculate the total size of header overhead for the last frame + long lastFrameHeaderOverhead = lastFrameSeqNumberSize + finalFrameSizeSize; + + // Calculate the number of frames + long frames = (encryptedSize - lastFrameHeaderOverhead) / (frameSize + nonceLen + tagLenBytes + (Integer.SIZE / Byte.SIZE)) + 1; + + // Calculate the size of the actual content in frames + long contentSizeWithoutLastFrame = (frames - 1) * frameSize; + + // Calculate the sequence number size for all frames + long seqNumberSize = frames * (Integer.SIZE / Byte.SIZE); + + // Calculate the total size of header overhead for all frames + long headerOverhead = (nonceLen + tagLenBytes) * frames + seqNumberSize; + + // Calculate the size of the last frame content + long lastFrameSize = encryptedSize - contentSizeWithoutLastFrame - headerOverhead - lastFrameHeaderOverhead; + + return contentSizeWithoutLastFrame + lastFrameSize; + } + + @Override + public int estimatePartialOutputSize(int inLen) { + return estimateOutputSize(inLen); + } + + @Override + public int estimateFinalOutputSize() { + return 0; + } + + /** + * Returns the plaintext bytes of the encrypted content. + * + * @param input + * the input bytes containing the content + * @param off + * the offset into the input array where the data to be decrypted + * starts. + * @param len + * the number of bytes to be decrypted. + * @return + * the plaintext bytes of the encrypted content. + * @throws BadCiphertextException + * if the bytes do not decrypt correctly. + */ + private byte[] decryptContent(final byte[] input, final int off, final int len) throws BadCiphertextException { + final byte[] nonce = currentFrameHeaders_.getNonce(); + + byte[] contentAad = null; + if (currentFrameHeaders_.isFinalFrame() == true) { + contentAad = Utils.generateContentAad( + messageId_, + Constants.FINAL_FRAME_STRING_ID, + (int) frameNumber_, + currentFrameHeaders_.getFrameContentLength() + ); + } else { + contentAad = Utils.generateContentAad(messageId_, Constants.FRAME_STRING_ID, (int) frameNumber_, frameSize_); + } + + return cipherHandler_.cipherData(nonce, contentAad, input, off, len); + } + + @Override + public boolean isComplete() { + return complete_; + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameEncryptionHandler.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameEncryptionHandler.java new file mode 100644 index 0000000000000..e5091f8e5caf8 --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/FrameEncryptionHandler.java @@ -0,0 +1,376 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import com.amazonaws.encryptionsdk.internal.Constants; +import com.amazonaws.encryptionsdk.internal.CryptoHandler; +import com.amazonaws.encryptionsdk.internal.ProcessingSummary; +import com.amazonaws.encryptionsdk.model.CipherFrameHeaders; + +/** + * The frame encryption handler is a subclass of the encryption handler and + * thereby provides an implementation of the Cryptography handler. + * + *

+ * It implements methods for encrypting content and storing the encrypted bytes + * in frames. + */ +class FrameEncryptionHandler implements CryptoHandler { + private final SecretKey encryptionKey_; + private final CryptoAlgorithm cryptoAlgo_; + private final CipherHandler cipherHandler_; + private final int nonceLen_; + private final byte[] messageId_; + private final int frameSize_; + private final int tagLenBytes_; + + private long frameNumber_; + private boolean isFinalFrame_; + + private final byte[] bytesToFrame_; + private int bytesToFrameLen_; + private boolean complete_ = false; + + /** + * Construct an encryption handler for encrypting bytes and storing them in + * frames. + */ + public FrameEncryptionHandler( + final SecretKey encryptionKey, + final int nonceLen, + final CryptoAlgorithm cryptoAlgo, + final byte[] messageId, + final int frameSize, + final int frameStartNumber + ) { + encryptionKey_ = encryptionKey; + cryptoAlgo_ = cryptoAlgo; + nonceLen_ = nonceLen; + messageId_ = messageId.clone(); + frameSize_ = frameSize; + tagLenBytes_ = cryptoAlgo_.getTagLen(); + bytesToFrame_ = new byte[frameSize_]; + bytesToFrameLen_ = 0; + cipherHandler_ = new CipherHandler(encryptionKey_, Cipher.ENCRYPT_MODE, cryptoAlgo_); + frameNumber_ = frameStartNumber; + } + + /** + * Encrypt a block of bytes from in putting the plaintext result into out. + * + *

+ * It encrypts by performing the following operations: + *

    + *
  1. determine the size of encrypted content that can fit into current frame
    2. + *
  2. call processBytes() of the underlying cipher to do corresponding cryptographic encryption + * of plaintext
    3. + *
  3. check if current frame is fully filled using the processed bytes, write current frame to + * the output being returned.
    4. + *
+ * + * @param in + * the input byte array. + * @param out + * the output buffer the encrypted bytes go into. + * @param outOff + * the offset into the output byte array the encrypted data starts at. + * @return the number of bytes written to out and processed + */ + @Override + public ProcessingSummary processBytes(final byte[] in, final int off, final int len, final byte[] out, final int outOff) + throws BadCiphertextException { + int actualOutLen = 0; + + int size = len; + int offset = off; + while (size > 0) { + final int currentFrameCapacity = frameSize_ - bytesToFrameLen_; + // bind size to the capacity of the current frame + size = Math.min(currentFrameCapacity, size); + + System.arraycopy(in, offset, bytesToFrame_, bytesToFrameLen_, size); + bytesToFrameLen_ += size; + + // check if there is enough bytes to create a frame + if (bytesToFrameLen_ == frameSize_) { + actualOutLen += writeEncryptedFrame(bytesToFrame_, 0, bytesToFrameLen_, out, outOff + actualOutLen); + + // reset buffer len as a new frame is created in next iteration + bytesToFrameLen_ = 0; + } + + // update offset by the size of bytes being encrypted. + offset += size; + // update size to the remaining bytes starting at offset. + size = len - offset; + } + + return new ProcessingSummary(actualOutLen, len); + } + + /** + * Finish processing of the bytes by writing out the ciphertext or final + * frame if framing. + * + * @param out + * space for any resulting output data. + * @param outOff + * offset into out to start copying the data at. + * @return + * number of bytes written into out. + */ + @Override + public int doFinal(final byte[] out, final int outOff) throws BadCiphertextException { + isFinalFrame_ = true; + complete_ = true; + return writeEncryptedFrame(bytesToFrame_, 0, bytesToFrameLen_, out, outOff); + } + + /** + * Return the size of the output buffer required for a processBytes plus a + * doFinal with an input of inLen bytes. + * + * @param inLen + * the length of the input. + * @return + * the space required to accommodate a call to processBytes and + * doFinal with len bytes of input. + */ + @Override + public int estimateOutputSize(final int inLen) { + // include any bytes held for inclusion in a subsequent frame + int totalContent = bytesToFrameLen_ + inLen; + return (int) estimatePartialSizeFromMetadata(totalContent, true, frameSize_, nonceLen_, tagLenBytes_); + } + + public static long estimatePartialSizeFromMetadata( + long totalContent, + boolean includeLastFrame, + int frameSize, + int nonceLen, + int tagLenBytes + ) { + // compute the size of the frames that will be constructed + long frames = totalContent / frameSize; + long outSize = (frameSize * frames); + + // account for remaining data that will need a new frame. + final long leftover = totalContent % frameSize; + outSize += leftover; + // even if leftover is 0, there will be a final frame. + if (includeLastFrame || leftover > 0) { + frames += 1; + } + + /* + * Calculate overhead of frame headers. + */ + // nonce and MAC tag. + outSize += frames * (nonceLen + tagLenBytes); + + // sequence number for all frames + outSize += frames * (Integer.SIZE / Byte.SIZE); + + if (includeLastFrame) { + // sequence number end for final frame + outSize += Integer.SIZE / Byte.SIZE; + + // integer for storing final frame size + outSize += Integer.SIZE / Byte.SIZE; + } + + return outSize; + } + + @Override + public int estimatePartialOutputSize(int inLen) { + int outSize = 0; + int frames = 0; + + // include any bytes held for inclusion in a subsequent frame + int totalContent = bytesToFrameLen_; + if (inLen >= 0) { + totalContent += inLen; + } + + // compute the size of the frames that will be constructed + frames = totalContent / frameSize_; + outSize += (frameSize_ * frames); + + /* + * Calculate overhead of frame headers. + */ + // nonce and MAC tag. + outSize += frames * (nonceLen_ + tagLenBytes_); + + // sequence number for all frames + outSize += frames * (Integer.SIZE / Byte.SIZE); + + return outSize; + } + + @Override + public int estimateFinalOutputSize() { + int outSize = 0; + int frames = 0; + + // include any bytes held for inclusion in a subsequent frame + int totalContent = bytesToFrameLen_; + + // compute the size of the frames that will be constructed + frames = totalContent / frameSize_; + outSize += (frameSize_ * frames); + + // account for remaining data that will need a new frame. + final int leftover = totalContent % frameSize_; + outSize += leftover; + // even if leftover is 0, there will be a final frame. + frames += 1; + + /* + * Calculate overhead of frame headers. + */ + // nonce and MAC tag. + outSize += frames * (nonceLen_ + tagLenBytes_); + + // sequence number for all frames + outSize += frames * (Integer.SIZE / Byte.SIZE); + + // sequence number end for final frame + outSize += Integer.SIZE / Byte.SIZE; + + // integer for storing final frame size + outSize += Integer.SIZE / Byte.SIZE; + + return outSize; + } + + /** + * We encrypt the bytes, create the headers for the block, and assemble the + * frame containing the headers and the encrypted bytes. + * @param out + * the output buffer the encrypted bytes go into. + * @param outOff + * the offset into the output byte array the encrypted data + * starts at. + * @return + * the number of bytes written to out. + * @throws BadCiphertextException + * thrown by the underlying cipher handler. + * @throws AwsCryptoException + * if frame number exceeds the maximum allowed value. + */ + private int writeEncryptedFrame(final byte[] input, final int off, final int len, final byte[] out, final int outOff) + throws BadCiphertextException, AwsCryptoException { + if (frameNumber_ > com.amazonaws.encryptionsdk.internal.Constants.MAX_FRAME_NUMBER + // Make sure we have the appropriate flag set for the final frame; we don't want to accept + // non-final-frame data when there won't be a subsequent frame for it to go into. + || (frameNumber_ == com.amazonaws.encryptionsdk.internal.Constants.MAX_FRAME_NUMBER && !isFinalFrame_)) { + throw new AwsCryptoException("Frame number exceeded the maximum allowed value."); + } + + if (out.length == 0) { + return 0; + } + + int outLen = 0; + + byte[] contentAad; + if (isFinalFrame_ == true) { + contentAad = Utils.generateContentAad( + messageId_, + com.amazonaws.encryptionsdk.internal.Constants.FINAL_FRAME_STRING_ID, + (int) frameNumber_, + len + ); + } else { + contentAad = Utils.generateContentAad( + messageId_, + com.amazonaws.encryptionsdk.internal.Constants.FRAME_STRING_ID, + (int) frameNumber_, + frameSize_ + ); + } + + final byte[] nonce = getNonce(); + + final byte[] encryptedBytes = cipherHandler_.cipherData(nonce, contentAad, input, off, len); + + // create the cipherblock headers now for the encrypted data + final int encryptedContentLen = encryptedBytes.length - tagLenBytes_; + final CipherFrameHeaders cipherFrameHeaders = new CipherFrameHeaders((int) frameNumber_, nonce, encryptedContentLen, isFinalFrame_); + final byte[] cipherFrameHeaderBytes = cipherFrameHeaders.toByteArray(); + + // assemble the headers and the encrypted bytes into a single block + System.arraycopy(cipherFrameHeaderBytes, 0, out, outOff + outLen, cipherFrameHeaderBytes.length); + outLen += cipherFrameHeaderBytes.length; + System.arraycopy(encryptedBytes, 0, out, outOff + outLen, encryptedBytes.length); + outLen += encryptedBytes.length; + + frameNumber_++; + + return outLen; + } + + private byte[] getNonce() { + /* + * To mitigate the risk of IVs colliding within the same message, we use deterministic IV generation within a + * message. + */ + + if (frameNumber_ < 1) { + // This should never happen - however, since we use a "frame number zero" IV elsewhere (for header auth), + // we must be sure that we don't reuse it here. + throw new IllegalStateException("Illegal frame number"); + } + + if ((int) frameNumber_ == Constants.ENDFRAME_SEQUENCE_NUMBER && !isFinalFrame_) { + throw new IllegalStateException("Too many frames"); + } + + final byte[] nonce = new byte[nonceLen_]; + + ByteBuffer buf = ByteBuffer.wrap(nonce); + buf.order(ByteOrder.BIG_ENDIAN); + // We technically only allocate the low 32 bits for the frame number, and the other bits are defined to be + // zero. However, since MAX_FRAME_NUMBER is 2^32-1, the high-order four bytes of the long will be zero, so the + // big-endian representation will also have zeros in that position. + Utils.position(buf, buf.limit() - Long.BYTES); + buf.putLong(frameNumber_); + + return nonce; + } + + @Override + public boolean isComplete() { + return complete_; + } +} diff --git a/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/Utils.java b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/Utils.java new file mode 100644 index 0000000000000..1878089f8abfd --- /dev/null +++ b/libs/encryption-sdk/src/main/java/org/opensearch/encryption/frame/Utils.java @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except + * in compliance with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.opensearch.encryption.frame; + +import java.nio.Buffer; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; + +/** Internal utility methods. */ +public final class Utils { + // SecureRandom objects can both be expensive to initialize and incur synchronization costs. + // This allows us to minimize both initializations and keep SecureRandom usage thread local + // to avoid lock contention. + private static final ThreadLocal LOCAL_RANDOM = new ThreadLocal() { + @Override + protected SecureRandom initialValue() { + final SecureRandom rnd = new SecureRandom(); + rnd.nextBoolean(); // Force seeding + return rnd; + } + }; + + private Utils() { + // Prevent instantiation + } + + /** + * Throws {@link NullPointerException} with message {@code paramName} if {@code object} is null. + * + * @param object value to be null-checked + * @param paramName message for the potential {@link NullPointerException} + * @return {@code object} + * @throws NullPointerException if {@code object} is null + * @param Type of object on which null check is to be performed + */ + public static T assertNonNull(final T object, final String paramName) throws NullPointerException { + if (object == null) { + throw new NullPointerException(paramName + " must not be null"); + } + return object; + } + + public static SecureRandom getSecureRandom() { + return LOCAL_RANDOM.get(); + } + + /** + * Generate the AAD bytes to use when encrypting/decrypting content. The generated AAD is a block + * of bytes containing the provided message identifier, the string identifier, the sequence + * number, and the length of the content. + * + * @param messageId the unique message identifier for the ciphertext. + * @param idString the string describing the type of content processed. + * @param seqNum the sequence number. + * @param len the length of the content. + * @return the bytes containing the generated AAD. + */ + static byte[] generateContentAad(final byte[] messageId, final String idString, final int seqNum, final long len) { + final byte[] idBytes = idString.getBytes(StandardCharsets.UTF_8); + final int aadLen = messageId.length + idBytes.length + Integer.SIZE / Byte.SIZE + Long.SIZE / Byte.SIZE; + final ByteBuffer aad = ByteBuffer.allocate(aadLen); + + aad.put(messageId); + aad.put(idBytes); + aad.putInt(seqNum); + aad.putLong(len); + + return aad.array(); + } + + public static IllegalArgumentException cannotBeNegative(String field) { + return new IllegalArgumentException(field + " cannot be negative"); + } + + /** + * Equivalent to calling {@link ByteBuffer#position(int)} but in a manner which is safe when + * compiled on Java 9 or newer but used on Java 8 or older. + * @param buff on which position needs to be set. + * @param newPosition New position to be set + * @return {@link ByteBuffer} object with new position set. + */ + public static ByteBuffer position(final ByteBuffer buff, final int newPosition) { + ((Buffer) buff).position(newPosition); + return buff; + } +} diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java index fb5c477232bc4..25c0c16059891 100644 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java +++ b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/CryptoManagerFactoryTests.java @@ -28,7 +28,7 @@ public class CryptoManagerFactoryTests extends OpenSearchTestCase { @Before public void setup() { cryptoManagerFactory = new CryptoManagerFactory( - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256", TimeValue.timeValueDays(2), 10 ); @@ -54,7 +54,7 @@ public void testCreateCryptoProvider() { when(mockKeyProvider.getEncryptionContext()).thenReturn(Collections.emptyMap()); CryptoHandler cryptoHandler = cryptoManagerFactory.createCryptoProvider( - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384", + "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256", mockMaterialsManager, mockKeyProvider ); @@ -69,7 +69,7 @@ public void testCreateMaterialsManager() { CachingCryptoMaterialsManager materialsManager = cryptoManagerFactory.createMaterialsManager( mockKeyProvider, "keyProviderName", - "ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY_ECDSA_P384" + "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256" ); assertNotNull(materialsManager); @@ -88,5 +88,14 @@ public void testCreateCryptoManager() { public void testUnsupportedAlgorithm() { expectThrows(IllegalArgumentException.class, () -> new CryptoManagerFactory("Unsupported_algo", TimeValue.timeValueDays(2), 10)); + + expectThrows( + IllegalArgumentException.class, + () -> cryptoManagerFactory.createCryptoProvider( + "ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384", + mock(CachingCryptoMaterialsManager.class), + mock(MasterKeyProvider.class) + ) + ); } } diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java index a5e74534ef32b..04fc196e1a062 100644 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java +++ b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/MockKeyProvider.java @@ -14,6 +14,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Collection; import java.util.Map; diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java index 5e3836fd10988..65617a8479eac 100644 --- a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java +++ b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/NoOpCryptoHandlerTests.java @@ -16,6 +16,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; public class NoOpCryptoHandlerTests extends OpenSearchTestCase { @@ -61,7 +62,7 @@ public void testCreateEncryptingStreamOfPart() { } private InputStreamContainer randomStream() { - byte[] bytes = randomAlphaOfLength(10).getBytes(); + byte[] bytes = randomAlphaOfLength(10).getBytes(StandardCharsets.UTF_8); ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(bytes); int offset = randomIntBetween(0, bytes.length - 1); return new InputStreamContainer(byteArrayInputStream, bytes.length, offset); diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CipherHandlerTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CipherHandlerTests.java new file mode 100644 index 0000000000000..43a0331ebbb5a --- /dev/null +++ b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CipherHandlerTests.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.encryption.frame; + +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; + +import java.nio.charset.StandardCharsets; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.exception.AwsCryptoException; + +public class CipherHandlerTests extends OpenSearchTestCase { + + public void testInvalidNonce() { + CipherHandler cipherHandler = new CipherHandler(null, 1, CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256); + byte[] nonce = "random".getBytes(StandardCharsets.UTF_8); + byte[] content = "content".getBytes(StandardCharsets.UTF_8); + Assert.assertThrows(IllegalArgumentException.class, () -> cipherHandler.cipherData(nonce, null, content, 0, content.length)); + } + + public void testInvalidSecretKey() { + CryptoAlgorithm cryptoAlgorithm = CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256; + CipherHandler cipherHandler = new CipherHandler(null, 1, cryptoAlgorithm); + byte[] nonce = new byte[cryptoAlgorithm.getNonceLen()]; + byte[] content = "content".getBytes(StandardCharsets.UTF_8); + Assert.assertThrows(AwsCryptoException.class, () -> cipherHandler.cipherData(nonce, null, content, 0, content.length)); + } +} diff --git a/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CryptoTests.java b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CryptoTests.java new file mode 100644 index 0000000000000..def861ef396e8 --- /dev/null +++ b/libs/encryption-sdk/src/test/java/org/opensearch/encryption/frame/CryptoTests.java @@ -0,0 +1,489 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.encryption.frame; + +import org.opensearch.common.blobstore.transfer.stream.OffsetRangeFileInputStream; +import org.opensearch.common.crypto.DecryptedRangedStreamProvider; +import org.opensearch.common.crypto.EncryptedHeaderContentSupplier; +import org.opensearch.common.io.InputStreamContainer; +import org.opensearch.encryption.MockKeyProvider; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Assert; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; + +import com.amazonaws.encryptionsdk.CryptoAlgorithm; +import com.amazonaws.encryptionsdk.CryptoMaterialsManager; +import com.amazonaws.encryptionsdk.ParsedCiphertext; +import com.amazonaws.encryptionsdk.caching.CachingCryptoMaterialsManager; +import com.amazonaws.encryptionsdk.caching.LocalCryptoMaterialsCache; +import com.amazonaws.encryptionsdk.exception.BadCiphertextException; +import org.mockito.Mockito; + +public class CryptoTests extends OpenSearchTestCase { + + private static FrameCryptoHandler frameCryptoHandler; + + private static FrameCryptoHandler frameCryptoHandlerTrailingAlgo; + + static class CustomFrameCryptoHandlerTest extends FrameCryptoHandler { + private final int frameSize; + + CustomFrameCryptoHandlerTest(AwsCrypto awsCrypto, HashMap config, int frameSize) { + super(awsCrypto, config); + this.frameSize = frameSize; + } + + @Override + public int getFrameSize() { + return frameSize; + } + } + + @Before + public void setupResources() { + frameCryptoHandler = new CustomFrameCryptoHandlerTest( + createAwsCrypto(CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256), + new HashMap<>(), + 100 + ); + frameCryptoHandlerTrailingAlgo = new CustomFrameCryptoHandlerTest( + createAwsCrypto(CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384), + new HashMap<>(), + 100 + ); + } + + private AwsCrypto createAwsCrypto(CryptoAlgorithm cryptoAlgorithm) { + MockKeyProvider keyProvider = new MockKeyProvider(); + CachingCryptoMaterialsManager cachingMaterialsManager = CachingCryptoMaterialsManager.newBuilder() + .withMasterKeyProvider(keyProvider) + .withCache(new LocalCryptoMaterialsCache(1000)) + .withMaxAge(10, TimeUnit.MINUTES) + .build(); + + return new AwsCrypto(cachingMaterialsManager, cryptoAlgorithm); + } + + static class EncryptedStoreTest { + byte[] encryptedContent; + int encryptedLength; + long rawLength; + byte[] rawContent; + } + + private EncryptedStoreTest verifyAndGetEncryptedContent() throws IOException, URISyntaxException { + return verifyAndGetEncryptedContent(false, frameCryptoHandler); + } + + private EncryptedStoreTest verifyAndGetEncryptedContent(boolean truncateRemainderPart, FrameCryptoHandler frameCryptoHandler) + throws IOException, URISyntaxException { + + long maxLength = 50 * 1024; + long rawContentLength = truncateRemainderPart + ? maxLength / frameCryptoHandler.getFrameSize() * frameCryptoHandler.getFrameSize() + : maxLength; + byte[] rawContent = randomAlphaOfLength((int) rawContentLength).getBytes(StandardCharsets.UTF_8); + + EncryptionMetadata cryptoContext = frameCryptoHandler.initEncryptionMetadata(); + + int encLength = 0; + byte[] encryptedContent = new byte[100 * 1024]; + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(rawContent, 0, rawContent.length)) { + + InputStreamContainer stream = new InputStreamContainer(inputStream, rawContent.length, 0); + InputStreamContainer encInputStream = frameCryptoHandler.createEncryptingStream(cryptoContext, stream); + assertNotNull(encInputStream); + + int readBytes; + while ((readBytes = encInputStream.getInputStream().read(encryptedContent, encLength, 1024)) != -1) { + encLength += readBytes; + } + } + + long calculatedEncryptedLength = frameCryptoHandler.estimateEncryptedLengthOfEntireContent(cryptoContext, rawContentLength); + assertEquals(encLength, calculatedEncryptedLength); + + EncryptedStoreTest encryptedStoreTest = new EncryptedStoreTest(); + encryptedStoreTest.encryptedLength = encLength; + encryptedStoreTest.encryptedContent = encryptedContent; + encryptedStoreTest.rawLength = rawContentLength; + encryptedStoreTest.rawContent = rawContent; + return encryptedStoreTest; + } + + public void testEncryptedDecryptedLengthEstimations() { + // Testing for 100 iterations + for (int i = 0; i < 100; i++) { + // Raw content size cannot be max value as encrypted size will overflow for the same. + long n = randomLongBetween(0, Integer.MAX_VALUE / 2); + FrameCryptoHandler frameCryptoHandler = new CustomFrameCryptoHandlerTest( + createAwsCrypto(CryptoAlgorithm.ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA384_ECDSA_P384), + new HashMap<>(), + randomIntBetween(10, 10240) + ); + EncryptionMetadata cryptoContext = frameCryptoHandler.initEncryptionMetadata(); + long encryptedLength = frameCryptoHandler.estimateEncryptedLengthOfEntireContent(cryptoContext, n); + ParsedCiphertext parsedCiphertext = new ParsedCiphertext(cryptoContext.getCiphertextHeaderBytes()); + long decryptedLength = frameCryptoHandler.estimateDecryptedLength(parsedCiphertext, encryptedLength); + assertEquals(n, decryptedLength); + } + } + + public void testSingleStreamEncryption() throws IOException, URISyntaxException { + + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( + encryptedStoreTest.encryptedContent, + 0, + encryptedStoreTest.encryptedLength + ); + long decryptedRawBytes = decryptAndVerify(byteArrayInputStream, encryptedStoreTest.encryptedLength, encryptedStoreTest.rawContent); + assertEquals(encryptedStoreTest.rawLength, decryptedRawBytes); + } + + public void testSingleStreamEncryptionTrailingSignatureAlgo() throws IOException, URISyntaxException { + + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(false, frameCryptoHandlerTrailingAlgo); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( + encryptedStoreTest.encryptedContent, + 0, + encryptedStoreTest.encryptedLength + ); + long decryptedRawBytes = decryptAndVerify(byteArrayInputStream, encryptedStoreTest.encryptedLength, encryptedStoreTest.rawContent); + assertEquals(encryptedStoreTest.rawLength, decryptedRawBytes); + } + + public void testDecryptionOfCorruptedContent() throws IOException, URISyntaxException { + + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + encryptedStoreTest.encryptedContent = "Corrupted content".getBytes(StandardCharsets.UTF_8); + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( + encryptedStoreTest.encryptedContent, + 0, + encryptedStoreTest.encryptedLength + ); + + Assert.assertThrows( + BadCiphertextException.class, + () -> decryptAndVerify(byteArrayInputStream, encryptedStoreTest.encryptedLength, encryptedStoreTest.rawContent) + ); + + } + + private long decryptAndVerify(InputStream encryptedStream, long encSize, byte[] rawContent) throws IOException { + long totalRawBytes = 0; + try ( + ByteArrayInputStream fis = new ByteArrayInputStream(rawContent); + InputStream decryptingStream = frameCryptoHandler.createDecryptingStream(encryptedStream) + ) { + byte[] decryptedBuffer = new byte[1024]; + byte[] actualBuffer = new byte[1024]; + int readActualBytes; + int readBytes; + while ((readBytes = decryptingStream.read(decryptedBuffer, 0, decryptedBuffer.length)) != -1) { + readActualBytes = fis.read(actualBuffer, 0, readBytes); + assertEquals(readActualBytes, readBytes); + assertArrayEquals(actualBuffer, decryptedBuffer); + totalRawBytes += readActualBytes; + } + assertEquals(rawContent.length, totalRawBytes); + } + return totalRawBytes; + } + + public void testMultiPartStreamsEncryption() throws IOException, URISyntaxException { + EncryptionMetadata encryptionMetadata = frameCryptoHandler.initEncryptionMetadata(); + + long rawContentLength = 50 * 1024; + byte[] rawContent = randomAlphaOfLength((int) rawContentLength).getBytes(StandardCharsets.UTF_8); + assertEquals(rawContentLength, rawContent.length); + + Path path = createTempFile(); + File file = path.toFile(); + Files.write(path, rawContent); + assertEquals(rawContentLength, Files.size(path)); + + byte[] encryptedContent = new byte[1024 * 100]; + int parts; + long partSize, lastPartSize; + partSize = getPartSize(rawContentLength, frameCryptoHandler.getFrameSize()); + parts = numberOfParts(rawContentLength, partSize); + lastPartSize = rawContentLength - (partSize * (parts - 1)); + + int encLength = 0; + for (int partNo = 0; partNo < parts; partNo++) { + long size = partNo == parts - 1 ? lastPartSize : partSize; + long pos = partNo * partSize; + try (InputStream inputStream = getMultiPartStreamSupplier(file).apply(size, pos)) { + InputStreamContainer rawStream = new InputStreamContainer(inputStream, size, pos); + InputStreamContainer encStream = frameCryptoHandler.createEncryptingStreamOfPart( + encryptionMetadata, + rawStream, + parts, + partNo + ); + int readBytes; + int curEncryptedBytes = 0; + while ((readBytes = encStream.getInputStream().read(encryptedContent, encLength, 1024)) != -1) { + encLength += readBytes; + curEncryptedBytes += readBytes; + } + assertEquals(encStream.getContentLength(), curEncryptedBytes); + } + } + encLength += frameCryptoHandler.getTrailingSignatureLength(encryptionMetadata); + + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(encryptedContent, 0, encLength); + decryptAndVerify(byteArrayInputStream, encLength, rawContent); + + } + + private long getPartSize(long contentLength, int frameSize) { + + double optimalPartSizeDecimal = (double) contentLength / randomIntBetween(5, 10); + // round up so we don't push the upload over the maximum number of parts + long optimalPartSize = (long) Math.ceil(optimalPartSizeDecimal); + if (optimalPartSize < frameSize) { + optimalPartSize = frameSize; + } + + if (optimalPartSize >= contentLength) { + return contentLength; + } + + if (optimalPartSize % frameSize > 0) { + // When using encryption, parts must line up correctly along cipher block boundaries + optimalPartSize = optimalPartSize - (optimalPartSize % frameSize) + frameSize; + } + return optimalPartSize; + } + + private int numberOfParts(final long totalSize, final long partSize) { + if (totalSize % partSize == 0) { + return (int) (totalSize / partSize); + } + return (int) (totalSize / partSize) + 1; + } + + private BiFunction getMultiPartStreamSupplier(File localFile) { + return (size, position) -> { + OffsetRangeFileInputStream offsetRangeInputStream; + try { + offsetRangeInputStream = new OffsetRangeFileInputStream(localFile.toPath(), size, position); + } catch (IOException e) { + return null; + } + return new CheckedInputStream(offsetRangeInputStream, new CRC32()); + }; + } + + public void testBlockBasedDecryptionForEntireFile() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + validateBlockDownload(encryptedStoreTest, 0, (int) encryptedStoreTest.rawLength - 1); + } + + public void testBlockBasedDecryptionForEntireFileWithLinedUpFrameAlongFileBoundary() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(true, frameCryptoHandler); + assertEquals( + "This test is meant for file size exactly divisible by frame size", + 0, + (encryptedStoreTest.rawLength % frameCryptoHandler.getFrameSize()) + ); + validateBlockDownload(encryptedStoreTest, 0, (int) encryptedStoreTest.rawLength - 1); + } + + public void testCorruptedTrailingSignature() throws IOException, URISyntaxException { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(false, frameCryptoHandlerTrailingAlgo); + byte[] trailingData = "corrupted".getBytes(StandardCharsets.UTF_8); + byte[] corruptedTrailingContent = Arrays.copyOf( + encryptedStoreTest.encryptedContent, + encryptedStoreTest.encryptedContent.length + trailingData.length + ); + System.arraycopy(trailingData, 0, corruptedTrailingContent, encryptedStoreTest.encryptedContent.length, trailingData.length); + encryptedStoreTest.encryptedContent = corruptedTrailingContent; + encryptedStoreTest.encryptedLength = corruptedTrailingContent.length; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( + encryptedStoreTest.encryptedContent, + 0, + encryptedStoreTest.encryptedLength + ); + BadCiphertextException ex = Assert.assertThrows( + BadCiphertextException.class, + () -> decryptAndVerify(byteArrayInputStream, encryptedStoreTest.encryptedLength, encryptedStoreTest.rawContent) + ); + Assert.assertEquals("Bad trailing signature", ex.getMessage()); + } + + public void testNoTrailingSignatureForTrailingAlgo() throws IOException, URISyntaxException { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(false, frameCryptoHandlerTrailingAlgo); + EncryptionMetadata cryptoContext = frameCryptoHandlerTrailingAlgo.initEncryptionMetadata(); + int trailingLength = frameCryptoHandler.getTrailingSignatureLength(cryptoContext); + byte[] removedTrailingContent = Arrays.copyOf( + encryptedStoreTest.encryptedContent, + encryptedStoreTest.encryptedContent.length - trailingLength + ); + encryptedStoreTest.encryptedContent = removedTrailingContent; + encryptedStoreTest.encryptedLength = removedTrailingContent.length; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream( + encryptedStoreTest.encryptedContent, + 0, + encryptedStoreTest.encryptedLength + ); + BadCiphertextException ex = Assert.assertThrows( + BadCiphertextException.class, + () -> decryptAndVerify(byteArrayInputStream, encryptedStoreTest.encryptedLength, encryptedStoreTest.rawContent) + ); + Assert.assertEquals("Bad trailing signature", ex.getMessage()); + } + + public void testOutputSizeEstimateWhenHandlerIsNull() { + CryptoMaterialsManager cryptoMaterialsManager = Mockito.mock(CryptoMaterialsManager.class); + DecryptionHandler decryptionHandler = DecryptionHandler.create(cryptoMaterialsManager); + int inputLen = 50; + int len = decryptionHandler.estimateOutputSize(inputLen); + assertEquals(inputLen, len); + } + + private EncryptedHeaderContentSupplier createEncryptedHeaderContentSupplier(byte[] encryptedContent) { + return (start, end) -> { + int len = (int) (end - start + 1); + byte[] bytes = new byte[len]; + System.arraycopy(encryptedContent, (int) start, bytes, (int) start, len); + return bytes; + }; + } + + public void testBlockBasedDecryptionForMiddleBlock() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + int maxBlockNum = (int) encryptedStoreTest.rawLength / frameCryptoHandler.getFrameSize(); + assert maxBlockNum > 5; + validateBlockDownload( + encryptedStoreTest, + randomIntBetween(5, maxBlockNum / 2) * frameCryptoHandler.getFrameSize(), + randomIntBetween(maxBlockNum / 2 + 1, maxBlockNum) * frameCryptoHandler.getFrameSize() - 1 + ); + } + + public void testRandomRangeDecryption() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + // Testing for 100 iterations + for (int testIteration = 0; testIteration < 100; testIteration++) { + int startPos = randomIntBetween(0, (int) encryptedStoreTest.rawLength - 1); + int endPos = randomIntBetween(startPos, (int) encryptedStoreTest.rawLength - 1); + validateBlockDownload(encryptedStoreTest, startPos, endPos); + } + } + + public void testDecryptionWithSameStartEndPos() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + int pos = randomIntBetween(0, (int) encryptedStoreTest.rawLength - 1); + for (int testIteration = 0; testIteration < frameCryptoHandler.getFrameSize(); testIteration++) { + validateBlockDownload(encryptedStoreTest, pos, pos); + } + } + + public void testBlockBasedDecryptionForLastBlock() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + int maxBlockNum = (int) encryptedStoreTest.rawLength / frameCryptoHandler.getFrameSize(); + assert maxBlockNum > 5; + validateBlockDownload( + encryptedStoreTest, + randomIntBetween(1, maxBlockNum - 1) * frameCryptoHandler.getFrameSize(), + (int) encryptedStoreTest.rawLength - 1 + ); + } + + private void validateBlockDownload(EncryptedStoreTest encryptedStoreTest, int startPos, int endPos) throws Exception { + + EncryptedHeaderContentSupplier encryptedHeaderContentSupplier = createEncryptedHeaderContentSupplier( + encryptedStoreTest.encryptedContent + ); + ParsedCiphertext cryptoContext = frameCryptoHandler.loadEncryptionMetadata(encryptedHeaderContentSupplier); + DecryptedRangedStreamProvider decryptedStreamProvider = frameCryptoHandler.createDecryptingStreamOfRange( + cryptoContext, + startPos, + endPos + ); + + long[] transformedRange = decryptedStreamProvider.getAdjustedRange(); + int encryptedBlockSize = (int) (transformedRange[1] - transformedRange[0] + 1); + byte[] encryptedBlockBytes = new byte[encryptedBlockSize]; + System.arraycopy(encryptedStoreTest.encryptedContent, (int) transformedRange[0], encryptedBlockBytes, 0, encryptedBlockSize); + ByteArrayInputStream encryptedStream = new ByteArrayInputStream(encryptedBlockBytes, 0, encryptedBlockSize); + InputStream decryptingStream = decryptedStreamProvider.getDecryptedStreamProvider().apply(encryptedStream); + + try (ByteArrayInputStream rawStream = new ByteArrayInputStream(encryptedStoreTest.rawContent, startPos, endPos - startPos + 1)) { + decryptAndVerifyBlock(decryptingStream, rawStream, startPos, endPos); + } + } + + public void testBlockBasedDecryptionForFirstBlock() throws Exception { + EncryptedStoreTest encryptedStoreTest = verifyAndGetEncryptedContent(); + // All block requests should properly line up with frames otherwise decryption will fail due to partial frames. + int blockEnd = randomIntBetween(5, (int) encryptedStoreTest.rawLength / frameCryptoHandler.getFrameSize()) * frameCryptoHandler + .getFrameSize() - 1; + validateBlockDownload(encryptedStoreTest, 0, blockEnd); + } + + private long decryptAndVerifyBlock( + InputStream decryptedStream, + ByteArrayInputStream rawStream, + int rawContentStartPos, + int rawContentEndPos + ) throws IOException { + long totalRawBytes = 0; + + byte[] decryptedBuffer = new byte[100]; + byte[] actualBuffer = new byte[100]; + + int readActualBytes; + int readBytes; + while ((readBytes = decryptedStream.read(decryptedBuffer, 0, decryptedBuffer.length)) != -1) { + readActualBytes = rawStream.read(actualBuffer, 0, Math.min(actualBuffer.length, readBytes)); + assertEquals(readActualBytes, readBytes); + assertArrayEquals(actualBuffer, decryptedBuffer); + totalRawBytes += readActualBytes; + } + assertEquals(rawContentEndPos - rawContentStartPos + 1, totalRawBytes); + return totalRawBytes; + } + + public void testEmptyContentCrypto() throws IOException { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(new byte[] {}); + EncryptionMetadata cryptoContext = frameCryptoHandler.initEncryptionMetadata(); + InputStreamContainer stream = new InputStreamContainer(byteArrayInputStream, 0, 0); + InputStreamContainer encryptingStream = frameCryptoHandler.createEncryptingStream(cryptoContext, stream); + InputStream decryptingStream = frameCryptoHandler.createDecryptingStream(encryptingStream.getInputStream()); + decryptingStream.readAllBytes(); + } + + public void testEmptyContentCryptoTrailingSignatureAlgo() throws IOException { + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(new byte[] {}); + EncryptionMetadata cryptoContext = frameCryptoHandlerTrailingAlgo.initEncryptionMetadata(); + InputStreamContainer stream = new InputStreamContainer(byteArrayInputStream, 0, 0); + InputStreamContainer encryptingStream = frameCryptoHandlerTrailingAlgo.createEncryptingStream(cryptoContext, stream); + InputStream decryptingStream = frameCryptoHandlerTrailingAlgo.createDecryptingStream(encryptingStream.getInputStream()); + decryptingStream.readAllBytes(); + } + +} diff --git a/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test b/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test deleted file mode 100644 index c93b6161ac8d6..0000000000000 --- a/libs/encryption-sdk/src/test/resources/raw_content_for_crypto_test +++ /dev/null @@ -1,25 +0,0 @@ -ewogICJmaWxlSW5mb3MiOiBbCiAgICB7CiAgICAgICJuYW1lIjogIl80LmZubSIsCiAgICAgICJyZW1vdGVfc -GF0aCI6ICIyYzYwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYz -AyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL2luZGljZXMvMSIsCiAgICAgICJzaXplIjogOTQz -CiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJfMl9MdWNlbmU4MF8wLmR2ZCIsCiAgICAgICJyZW1vdGVfcGF -0aCI6ICIyYzYwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjg -zMy0xMGE0YTA5M2VkNTcvMC8wL2luZGljZXMvMSIsCiAgICAgICJzaXplIjogMzU1CiAgICB9CiAgXQp9 -ewogICJja3BfZmlsZSI6IHsKICAgICJuYW1lIjogInRyYW5zbG9nLTguY2twIiwKICAgICJyZW1vdGVfcGF0aCI6ICIyYz -YwMzNmNmZlZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0 -xMGE0YTA5M2VkNTcvMC8wL3RyYW5zbG9nLzEiLAogICAgInNpemUiOiAwCiAgfSwKICAidGxvZ192ZXJzaW9uIjogewogICAgIjg -iOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMT -BhNGEwOTNlZDU3LzAvMC90cmFuc2xvZy8xIgogIH0KfQ== -ewogICJmaWxlSW5mb3MiOiBbCiAgICB7CiAgICAgICJuYW1lIjogIl80LmZubSIsCiAgICAgICJyZW1vdGVfcGF0aCI6ICIyYzYwMzNmNmZl -ZTY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL2luZG -ljZXMvMSIsCiAgICAgICJzaXplIjogOTQzCiAgICB9LAogICAgewogICAgICAibmFtZSI6ICJfNC5mZHQiLAogICAgICAicmVtb3RlX3BhdGgiOiAi -MmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMTBhNGEwOTNlZDU3 -LzAvMC9pbmRpY2VzLzEiLAogICAgICAic2l6ZSI6IDQ1MTMKICAgIH0sCiAgICB7CiAgICAgICJuYW1lIjogInNlZ21lbnRzX2MiLAogICAgICAicmVtb3R -lX3BhdGgiOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZmU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzM -tMTBhNGEwOTNlZDU3LzAvMC9pbmRpY2VzLzEiLAogICAgICAic2l6ZSI6IDM1NQogICAgfQogIF0KfQ== -ewogICJja3BfZmlsZSI6IHsKICAgICJuYW1lIjogInRyYW5zbG9nLTcuY2twIiwKICAgICJyZW1vdGVfcGF0aCI6ICIyYzYwMzNmNmZlZ -TY0NTY1YTU3YzQzZWVmZThmY2QzMS9kdW1teS1jb2xsZWN0aW9uMi9kMDRmYzAyZi0wMDQ0LTRhYmYtYjgzMy0xMGE0YTA5M2VkNTcvMC8wL3RyY -W5zbG9nLzEiLAogICAgInNpemUiOiAwCiAgfSwKICAidGxvZ192ZXJzaW9uIjogewogICAgIjYiOiAiMmM2MDMzZjZmZWU2NDU2NWE1N2M0M2VlZ -mU4ZmNkMzEvZHVtbXktY29sbGVjdGlvbjIvZDA0ZmMwMmYtMDA0NC00YWJmLWI4MzMtMTBhNGEwOTNlZDU3LzAvMC90cmFuc2xvZy8xIiwKICAgICI3Ijo -gIjJjNjAzM2Y2ZmVlNjQ1NjVhNTdjNDNlZWZlOGZjZDMxL2R1bW15LWNvbGxlY3Rpb24yL2QwNGZjMDJmLTAwNDQtNGFiZi1iODMzLTEwYTRhMDkzZW -Q1Ny8wLzAvdHJhbnNsb2cvMSIKICB9Cn0= - diff --git a/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java b/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java index 87f6cfbb254c6..108c88bd3bf80 100644 --- a/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java +++ b/plugins/crypto-kms/src/main/java/org/opensearch/crypto/kms/KmsService.java @@ -242,12 +242,13 @@ static void setDefaultAwsProfilePath() { } public MasterKeyProvider createMasterKeyProvider(CryptoMetadata cryptoMetadata) { - String keyArn = KEY_ARN_SETTING.get(cryptoMetadata.settings()); + Settings cryptoSettings = Settings.builder().put(cryptoMetadata.settings()).normalizePrefix("kms.").build(); + String keyArn = KEY_ARN_SETTING.get(cryptoSettings); if (!Strings.hasText(keyArn)) { throw new IllegalArgumentException("Missing key_arn setting"); } - String kmsEncCtx = ENC_CTX_SETTING.get(cryptoMetadata.settings()); + String kmsEncCtx = ENC_CTX_SETTING.get(cryptoSettings); Map encCtx; if (Strings.hasText(kmsEncCtx)) { try { diff --git a/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java b/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java index c6a8b56a8d8c8..d2bd6ce230260 100644 --- a/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java +++ b/server/src/main/java/org/opensearch/crypto/CryptoManagerRegistry.java @@ -49,7 +49,7 @@ public class CryptoManagerRegistry { * @param settings Crypto settings. */ protected CryptoManagerRegistry(List cryptoPlugins, Settings settings) { - cryptoManagerFactory.set(new CryptoManagerFactory("ALG_AES_256_GCM_HKDF_SHA512_COMMIT_KEY", TimeValue.timeValueDays(2), 500)); + cryptoManagerFactory.set(new CryptoManagerFactory("ALG_AES_256_GCM_IV12_TAG16_HKDF_SHA256", TimeValue.timeValueDays(2), 500)); registry.set(loadCryptoFactories(cryptoPlugins)); }