Merge branch 'master' into users/ndeshpan/hybridSearch

Directory.Build.props

@@ -1,9 +1,9 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
 	<PropertyGroup>
-		<ClientOfficialVersion>3.43.1</ClientOfficialVersion>
-		<ClientPreviewVersion>3.44.0</ClientPreviewVersion>
-		<ClientPreviewSuffixVersion>preview.1</ClientPreviewSuffixVersion>
-		<DirectVersion>3.36.0</DirectVersion>
+		<ClientOfficialVersion>3.44.0</ClientOfficialVersion>
+		<ClientPreviewVersion>3.45.0</ClientPreviewVersion>
+		<ClientPreviewSuffixVersion>preview.0</ClientPreviewSuffixVersion>
+		<DirectVersion>3.36.1</DirectVersion>
 		<EncryptionOfficialVersion>2.0.4</EncryptionOfficialVersion>
 		<EncryptionPreviewVersion>2.1.0</EncryptionPreviewVersion>
 		<EncryptionPreviewSuffixVersion>preview4</EncryptionPreviewSuffixVersion>

Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs

@@ -0,0 +1,122 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Encryption.Custom
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
+    using System.IO;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using Newtonsoft.Json;
+    using Newtonsoft.Json.Linq;
+
+#pragma warning disable IDE0057 // Use range operator
+#pragma warning disable VSTHRD103 // Call async methods when in an async method
+    internal static class AeAesEncryptionProcessor
+    {
+        public static async Task<Stream> EncryptAsync(
+            Stream input,
+            Encryptor encryptor,
+            EncryptionOptions encryptionOptions,
+            CancellationToken cancellationToken)
+        {
+            JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream<JObject>(input);
+            List<string> pathsEncrypted = new ();
+            EncryptionProperties encryptionProperties = null;
+            byte[] plainText = null;
+            byte[] cipherText = null;
+
+            JObject toEncryptJObj = new ();
+
+            foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt)
+            {
+                string propertyName = pathToEncrypt.Substring(1);
+                if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue))
+                {
+                    continue;
+                }
+
+                toEncryptJObj.Add(propertyName, propertyValue.Value<JToken>());
+                itemJObj.Remove(propertyName);
+            }
+
+            MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream<JObject>(toEncryptJObj);
+            Debug.Assert(memoryStream != null);
+            Debug.Assert(memoryStream.TryGetBuffer(out _));
+            plainText = memoryStream.ToArray();
+
+            cipherText = await encryptor.EncryptAsync(
+                plainText,
+                encryptionOptions.DataEncryptionKeyId,
+                encryptionOptions.EncryptionAlgorithm,
+                cancellationToken);
+
+            if (cipherText == null)
+            {
+                throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}.");
+            }
+
+            encryptionProperties = new EncryptionProperties(
+                    encryptionFormatVersion: 2,
+                    encryptionOptions.EncryptionAlgorithm,
+                    encryptionOptions.DataEncryptionKeyId,
+                    encryptedData: cipherText,
+                    encryptionOptions.PathsToEncrypt);
+
+            itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties));
+
+            input.Dispose();
+            return EncryptionProcessor.BaseSerializer.ToStream(itemJObj);
+        }
+
+        internal static async Task<DecryptionContext> DecryptContentAsync(
+            JObject document,
+            EncryptionProperties encryptionProperties,
+            Encryptor encryptor,
+            CosmosDiagnosticsContext diagnosticsContext,
+            CancellationToken cancellationToken)
+        {
+            _ = diagnosticsContext;
+
+            if (encryptionProperties.EncryptionFormatVersion != 2)
+            {
+                throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version.");
+            }
+
+            byte[] plainText = await encryptor.DecryptAsync(
+                encryptionProperties.EncryptedData,
+                encryptionProperties.DataEncryptionKeyId,
+                encryptionProperties.EncryptionAlgorithm,
+                cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(EncryptionProcessor.DecryptAsync)}.");
+            JObject plainTextJObj;
+            using (MemoryStream memoryStream = new (plainText))
+            using (StreamReader streamReader = new (memoryStream))
+            using (JsonTextReader jsonTextReader = new (streamReader))
+            {
+                jsonTextReader.ArrayPool = JsonArrayPool.Instance;
+                plainTextJObj = JObject.Load(jsonTextReader);
+            }
+
+            List<string> pathsDecrypted = new ();
+            foreach (JProperty property in plainTextJObj.Properties())
+            {
+                document.Add(property.Name, property.Value);
+                pathsDecrypted.Add("/" + property.Name);
+            }
+
+            DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext(
+                pathsDecrypted,
+                encryptionProperties.DataEncryptionKeyId);
+
+            document.Remove(Constants.EncryptedInfo);
+
+            return decryptionContext;
+        }
+    }
+
+#pragma warning restore IDE0057 // Use range operator
+#pragma warning restore VSTHRD103 // Call async methods when in an async method
+}

Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs → Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs

@@ -10,6 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom
     using System.IO;
     using System.Security.Cryptography;
 
+#pragma warning disable SYSLIB0021 // Type or member is obsolete
+
     /// <summary>
     /// This class implements authenticated encryption algorithm with associated data as described in
     /// http://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05 - specifically this implements
@@ -26,11 +28,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey
         /// </summary>
         private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8;
 
+        /// <summary>
+        /// Authentication tag size in bytes
+        /// </summary>
+        private const int AuthenticationTagSizeInBytes = KeySizeInBytes;
+
         /// <summary>
         /// Block size in bytes. AES uses 16 byte blocks.
         /// </summary>
         private const int BlockSizeInBytes = 16;
 
+        /// <summary>
+        /// Size of Initialization Vector in bytes.
+        /// </summary>
+        private const int IvSizeInBytes = 16;
+
         /// <summary>
         /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text)
         /// </summary>
@@ -89,7 +101,9 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey
 
         public override byte[] RawKey => this.dataEncryptionKey.RootKey;
 
+#pragma warning disable CS0618 // Type or member is obsolete
         public override string EncryptionAlgorithm => CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized;
+#pragma warning restore CS0618 // Type or member is obsolete
 
         /// <summary>
         /// Initializes a new instance of AeadAes256CbcHmac256Algorithm algorithm with a given key and encryption type
@@ -137,6 +151,27 @@ public override byte[] EncryptData(byte[] plainText)
             return this.EncryptData(plainText, hasAuthenticationTag: true);
         }
 
+        /// <summary>
+        /// Encryption Algorithm
+        /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits
+        /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding.
+        /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length)
+        /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext
+        /// </summary>
+        /// <returns>Returns the ciphertext corresponding to the plaintext.</returns>
+        public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset)
+        {
+            byte[] buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray());
+
+            if (buffer.Length > output.Length - outputOffset)
+            {
+                throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes.");
+            }
+
+            buffer.CopyTo(output, outputOffset);
+            return buffer.Length;
+        }
+
         /// <summary>
         /// Encryption Algorithm
         /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits
@@ -169,23 +204,20 @@ protected byte[] EncryptData(byte[] plainText, bool hasAuthenticationTag)
 
             // Final blob we return = version + HMAC + iv + cipherText
             const int hmacStartIndex = 1;
-            int authenticationTagLen = hasAuthenticationTag ? KeySizeInBytes : 0;
+            int authenticationTagLen = hasAuthenticationTag ? AuthenticationTagSizeInBytes : 0;
             int ivStartIndex = hmacStartIndex + authenticationTagLen;
             int cipherStartIndex = ivStartIndex + BlockSizeInBytes; // this is where hmac starts.
 
-            // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks.
-            int outputBufSize = sizeof(byte) + authenticationTagLen + iv.Length + (numBlocks * BlockSizeInBytes);
+            int outputBufSize = this.GetEncryptByteCount(plainText.Length) - (hasAuthenticationTag ? 0 : authenticationTagLen);
             byte[] outBuffer = new byte[outputBufSize];
 
             // Store the version and IV rightaway
             outBuffer[0] = this.algorithmVersion;
             Buffer.BlockCopy(iv, 0, outBuffer, ivStartIndex, iv.Length);
 
-            AesCryptoServiceProvider aesAlg;
-
             // Try to get a provider from the pool.
             // If no provider is available, create a new one.
-            if (!this.cryptoProviderPool.TryDequeue(out aesAlg))
+            if (!this.cryptoProviderPool.TryDequeue(out AesCryptoServiceProvider aesAlg))
             {
                 aesAlg = new AesCryptoServiceProvider();
 
@@ -228,7 +260,7 @@ protected byte[] EncryptData(byte[] plainText, bool hasAuthenticationTag)
 
                 if (hasAuthenticationTag)
                 {
-                    using (HMACSHA256 hmac = new HMACSHA256(this.dataEncryptionKey.MACKey))
+                    using (HMACSHA256 hmac = new (this.dataEncryptionKey.MACKey))
                     {
                         Debug.Assert(hmac.CanTransformMultipleBlocks, "HMAC can't transform multiple blocks");
                         hmac.TransformBlock(version, 0, version.Length, version, 0);
@@ -332,11 +364,10 @@ private byte[] DecryptData(byte[] iv, byte[] cipherText, int offset, int count)
             Debug.Assert((count + offset) <= cipherText.Length);
 
             byte[] plainText;
-            AesCryptoServiceProvider aesAlg;
 
             // Try to get a provider from the pool.
             // If no provider is available, create a new one.
-            if (!this.cryptoProviderPool.TryDequeue(out aesAlg))
+            if (!this.cryptoProviderPool.TryDequeue(out AesCryptoServiceProvider aesAlg))
             {
                 aesAlg = new AesCryptoServiceProvider();
 
@@ -361,12 +392,12 @@ private byte[] DecryptData(byte[] iv, byte[] cipherText, int offset, int count)
                 aesAlg.IV = iv;
 
                 // Create the streams used for decryption.
-                using (MemoryStream msDecrypt = new MemoryStream())
+                using (MemoryStream msDecrypt = new ())
                 {
                     // Create an encryptor to perform the stream transform.
                     using (ICryptoTransform decryptor = aesAlg.CreateDecryptor())
                     {
-                        using (CryptoStream csDecrypt = new CryptoStream(msDecrypt, decryptor, CryptoStreamMode.Write))
+                        using (CryptoStream csDecrypt = new (msDecrypt, decryptor, CryptoStreamMode.Write))
                         {
                             // Decrypt the secret message and get the plain text data
                             csDecrypt.Write(cipherText, offset, count);
@@ -401,7 +432,7 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset
             //              1 block for IV (16 bytes)
             //              cipherText.Length
             //              1 byte for version byte length
-            using (HMACSHA256 hmac = new HMACSHA256(this.dataEncryptionKey.MACKey))
+            using (HMACSHA256 hmac = new (this.dataEncryptionKey.MACKey))
             {
                 int retVal = 0;
                 retVal = hmac.TransformBlock(version, 0, version.Length, version, 0);
@@ -418,5 +449,42 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset
             Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length);
             return authenticationTag;
         }
+
+        public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset)
+        {
+            byte[] buffer = this.DecryptData(cipherText.AsSpan(cipherTextOffset, cipherTextLength).ToArray(), true);
+
+            if (buffer.Length > output.Length - outputOffset)
+            {
+                throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes");
+            }
+
+            buffer.CopyTo(output, outputOffset);
+            return buffer.Length;
+        }
+
+        public override int GetEncryptByteCount(int plainTextLength)
+        {
+            // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks.
+            return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength);
+        }
+
+        public override int GetDecryptByteCount(int cipherTextLength)
+        {
+            int value = cipherTextLength - (sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes);
+            if (value < BlockSizeInBytes)
+            {
+                throw new ArgumentOutOfRangeException(nameof(cipherTextLength), $"Cipher text length is too short.");
+            }
+
+            return value;
+        }
+
+        private static int GetCipherTextLength(int inputSize)
+        {
+            return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes;
+        }
     }
+
+#pragma warning restore SYSLIB0021 // Type or member is obsolete
 }

Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs → Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs

@@ -101,25 +101,16 @@ internal AeadAes256CbcHmac256EncryptionKey(byte[] rootKey, string algorithmName)
         /// <summary>
         /// Gets Encryption key that should be used for encryption and decryption.
         /// </summary>
-        internal byte[] EncryptionKey
-        {
-            get { return this.encryptionKey.RootKey; }
-        }
+        internal byte[] EncryptionKey => this.encryptionKey.RootKey;
 
         /// <summary>
         /// Gets MAC key should be used to compute and validate HMAC.
         /// </summary>
-        internal byte[] MACKey
-        {
-            get { return this.macKey.RootKey; }
-        }
+        internal byte[] MACKey => this.macKey.RootKey;
 
         /// <summary>
         /// Gets IV key should be used to compute synthetic IV from a given plain text.
         /// </summary>
-        internal byte[] IVKey
-        {
-            get { return this.ivKey.RootKey; }
-        }
+        internal byte[] IVKey => this.ivKey.RootKey;
     }
 }

Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs → Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs

@@ -36,13 +36,7 @@ internal SymmetricKey(byte[] rootKey)
         /// Gets a copy of the plain text key
         /// This is needed for actual encryption/decryption.
         /// </summary>
-        internal virtual byte[] RootKey
-        {
-            get
-            {
-                return this.rootKey;
-            }
-        }
+        internal virtual byte[] RootKey => this.rootKey;
 
         /// <summary>
         /// Computes SHA256 value of the plain text key bytes