Enabling serialization customization through CosmosSerializerOptions (#…

…650)

* Made JsonSerializerSettings internal. Exposed a new constructor with popular flags to make it easier for users that only require basic flags. Refactored all serializer classes into a single folder.

* Fixed contract UT and updated changelog

* Made CosmosJsonDotNetSerializer internal. Added CosmosSerializerOptions to CosmosClientOptions. Added new UT.

* Fixed naming

* Updating file name and changelog

* Update contract test

* Fixed spacing

* Adding additional info to ValidateAsync test to understand transient failure

* Fixing contract enforcement test

* Updated naming

* Additional contract checks

* Updating contract from VS2017

* Fixed contract test for VS2019 to skip is IsReadOnlyAttribute

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Renamed CosmosSerializerOptions to CosmosSerializationOptions

* Added more comments

* Reverting DocumentClientWithUriParameters changes

* Revert "Reverting DocumentClientWithUriParameters changes"

This reverts commit febd523.

* Reverting accidental changes

* Adding comment examples, Converted options to be nullable

* Updated contract test

* Fixed Unit test

* Converted to a class with default values and updated changelog

* Updated contract api

Microsoft.Azure.Cosmos/src/CosmosClient.cs

@@ -150,7 +150,7 @@ protected CosmosClient()
         /// <seealso href="https://docs.microsoft.com/azure/cosmos-db/troubleshoot-dot-net-sdk"/>
         /// </remarks>
         public CosmosClient(
-            string connectionString, 
+            string connectionString,
             CosmosClientOptions clientOptions = null)
             : this(
                   CosmosClientOptions.GetAccountEndpoint(connectionString),
@@ -612,18 +612,19 @@ internal void Init(
 
             this.RequestHandler = clientPipelineBuilder.Build();
 
+            CosmosSerializer userSerializer = this.ClientOptions.GetCosmosSerializerWithWrapperOrDefault();
             this.ResponseFactory = new CosmosResponseFactory(
                 defaultJsonSerializer: this.ClientOptions.PropertiesSerializer,
-                userJsonSerializer: this.ClientOptions.CosmosSerializerWithWrapperOrDefault);
+                userJsonSerializer: userSerializer);
 
             CosmosSerializer sqlQuerySpecSerializer = CosmosSqlQuerySpecJsonConverter.CreateSqlQuerySpecSerializer(
-                this.ClientOptions.CosmosSerializerWithWrapperOrDefault,
-                this.ClientOptions.PropertiesSerializer);
+                cosmosSerializer: userSerializer,
+                propertiesSerializer: this.ClientOptions.PropertiesSerializer);
 
             this.ClientContext = new ClientContextCore(
                 client: this,
                 clientOptions: this.ClientOptions,
-                userJsonSerializer: this.ClientOptions.CosmosSerializerWithWrapperOrDefault,
+                userJsonSerializer: userSerializer,
                 defaultJsonSerializer: this.ClientOptions.PropertiesSerializer,
                 sqlQuerySpecSerializer: sqlQuerySpecSerializer,
                 cosmosResponseFactory: this.ResponseFactory,
@@ -632,7 +633,7 @@ internal void Init(
                 documentQueryClient: new DocumentQueryClient(this.DocumentClient));
         }
 
-        internal async virtual Task<ConsistencyLevel> GetAccountConsistencyLevelAsync()
+        internal virtual async Task<ConsistencyLevel> GetAccountConsistencyLevelAsync()
         {
             if (!this.accountConsistencyLevel.HasValue)
             {

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos
     using System.Collections.ObjectModel;
     using System.Data.Common;
     using System.Linq;
-    using System.Runtime.ConstrainedExecution;
     using Microsoft.Azure.Cosmos.Fluent;
     using Microsoft.Azure.Documents;
     using Microsoft.Azure.Documents.Client;
@@ -17,6 +16,18 @@ namespace Microsoft.Azure.Cosmos
     /// <summary>
     /// Defines all the configurable options that the CosmosClient requires.
     /// </summary>
+    /// <example>
+    /// An example on how to configure the serialization option to ignore null values
+    /// CosmosClientOptions clientOptions = new CosmosClientOptions()
+    /// {
+    ///     SerializerOptions = new CosmosSerializationOptions(){
+    ///         IgnoreNullValues = true
+    ///     },
+    ///     ConnectionMode = ConnectionMode.Gateway,
+    /// };
+    /// 
+    /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
+    /// </example>
     public class CosmosClientOptions
     {
         /// <summary>
@@ -39,11 +50,12 @@ public class CosmosClientOptions
         /// </summary>
         private static readonly CosmosSerializer propertiesSerializer = new CosmosJsonSerializerWrapper(new CosmosJsonDotNetSerializer());
 
-        private readonly Collection<RequestHandler> customHandlers;
         private readonly string currentEnvironmentInformation;
 
         private int gatewayModeMaxConnectionLimit;
         private string applicationName;
+        private CosmosSerializationOptions serializerOptions;
+        private CosmosSerializer serializer;
 
         /// <summary>
         /// Creates a new CosmosClientOptions
@@ -59,7 +71,7 @@ public CosmosClientOptions()
             this.ConnectionMode = CosmosClientOptions.DefaultConnectionMode;
             this.ConnectionProtocol = CosmosClientOptions.DefaultProtocol;
             this.ApiType = CosmosClientOptions.DefaultApiType;
-            this.customHandlers = new Collection<RequestHandler>();
+            this.CustomHandlers = new Collection<RequestHandler>();
         }
 
         /// <summary>
@@ -132,10 +144,7 @@ public int GatewayModeMaxConnectionLimit
         /// </summary>
         /// <seealso cref="CosmosClientBuilder.AddCustomHandlers(RequestHandler[])"/>
         [JsonConverter(typeof(ClientOptionJsonConverter))]
-        public Collection<RequestHandler> CustomHandlers
-        {
-            get => this.customHandlers;
-        }
+        public Collection<RequestHandler> CustomHandlers { get; }
 
         /// <summary>
         /// Get or set the connection mode used by the client when connecting to the Azure Cosmos DB service.
@@ -157,7 +166,7 @@ public Collection<RequestHandler> CustomHandlers
         public ConsistencyLevel? ConsistencyLevel { get; set; }
 
         /// <summary>
-        /// Get ot set the number of times client should retry on rate throttled requests.
+        /// Get or set the number of times client should retry on rate throttled requests.
         /// </summary>
         /// <seealso cref="CosmosClientBuilder.WithThrottlingRetryOptions(TimeSpan, int)"/>
         public int? MaxRetryAttemptsOnRateLimitedRequests { get; set; }
@@ -171,6 +180,35 @@ public Collection<RequestHandler> CustomHandlers
         /// <seealso cref="CosmosClientBuilder.WithThrottlingRetryOptions(TimeSpan, int)"/>
         public TimeSpan? MaxRetryWaitTimeOnRateLimitedRequests { get; set; }
 
+        /// <summary>
+        /// Get to set optional serializer options. 
+        /// </summary>
+        /// <example>
+        /// An example on how to configure the serialization option to ignore null values
+        /// CosmosClientOptions clientOptions = new CosmosClientOptions()
+        /// {
+        ///     SerializerOptions = new CosmosSerializationOptions(){
+        ///         IgnoreNullValues = true
+        ///     }
+        /// };
+        /// 
+        /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
+        /// </example>
+        public CosmosSerializationOptions SerializerOptions
+        {
+            get => this.serializerOptions;
+            set
+            {
+                if (this.Serializer != null)
+                {
+                    throw new ArgumentException(
+                        $"{nameof(this.SerializerOptions)} is not compatible with {nameof(this.Serializer)}. Only one can be set.  ");
+                }
+
+                this.serializerOptions = value;
+            }
+        }
+
         /// <summary>
         /// Get to set an optional JSON serializer. The client will use it to serialize or de-serialize user's cosmos request/responses.
         /// SDK owned types such as DatabaseProperties and ContainerProperties will always use the SDK default serializer.
@@ -182,10 +220,7 @@ public Collection<RequestHandler> CustomHandlers
         /// <example>
         /// // An example on how to configure the serializer to ignore null values
         /// CosmosSerializer ignoreNullSerializer = new CosmosJsonDotNetSerializer(
-        ///         new JsonSerializerSettings()
-        ///         {
-        ///             NullValueHandling = NullValueHandling.Ignore
-        ///         });
+        ///             NullValueHandling = NullValueHandling.Ignore);
         ///         
         /// CosmosClientOptions clientOptions = new CosmosClientOptions()
         /// {
@@ -195,7 +230,20 @@ public Collection<RequestHandler> CustomHandlers
         /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
         /// </example>
         [JsonConverter(typeof(ClientOptionJsonConverter))]
-        public CosmosSerializer Serializer { get; set; }
+        public CosmosSerializer Serializer
+        {
+            get => this.serializer;
+            set
+            {
+                if (this.SerializerOptions != null)
+                {
+                    throw new ArgumentException(
+                        $"{nameof(this.Serializer)} is not compatible with {nameof(this.SerializerOptions)}. Only one can be set.  ");
+                }
+
+                this.serializer = value;
+            }
+        }
 
         /// <summary>
         /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses.
@@ -205,12 +253,6 @@ public Collection<RequestHandler> CustomHandlers
         [JsonConverter(typeof(ClientOptionJsonConverter))]
         internal CosmosSerializer PropertiesSerializer => CosmosClientOptions.propertiesSerializer;
 
-        /// <summary>
-        /// Gets the user json serializer with the CosmosJsonSerializerWrapper or the default
-        /// </summary>
-        [JsonIgnore]
-        internal CosmosSerializer CosmosSerializerWithWrapperOrDefault => this.Serializer == null ? this.PropertiesSerializer : new CosmosJsonSerializerWrapper(this.Serializer);
-
         /// <summary>
         /// Gets or sets the connection protocol when connecting to the Azure Cosmos service.
         /// </summary>
@@ -309,6 +351,22 @@ public Collection<RequestHandler> CustomHandlers
         /// </summary>
         internal bool? EnableCpuMonitor { get; set; }
 
+        /// <summary>
+        /// Gets the user json serializer with the CosmosJsonSerializerWrapper or the default
+        /// </summary>
+        internal CosmosSerializer GetCosmosSerializerWithWrapperOrDefault()
+        {
+            if (this.SerializerOptions != null)
+            {
+                CosmosJsonDotNetSerializer cosmosJsonDotNetSerializer = new CosmosJsonDotNetSerializer(this.SerializerOptions);
+                return new CosmosJsonSerializerWrapper(cosmosJsonDotNetSerializer);
+            }
+            else
+            {
+                return this.Serializer == null ? this.PropertiesSerializer : new CosmosJsonSerializerWrapper(this.Serializer);
+            }
+        }
+
         internal CosmosClientOptions Clone()
         {
             CosmosClientOptions cloneConfiguration = (CosmosClientOptions)this.MemberwiseClone();

Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElementSerializer.cs

@@ -29,7 +29,7 @@ static class CosmosElementSerializer
         internal static CosmosArray ToCosmosElements(
             MemoryStream memoryStream,
             ResourceType resourceType,
-            CosmosSerializationOptions cosmosSerializationOptions = null)
+            CosmosSerializationFormatOptions cosmosSerializationOptions = null)
         {
             if (!memoryStream.CanRead)
             {
@@ -103,7 +103,7 @@ internal static Stream ToStream(
             string containerRid,
             IEnumerable<CosmosElement> cosmosElements,
             ResourceType resourceType,
-            CosmosSerializationOptions cosmosSerializationOptions = null)
+            CosmosSerializationFormatOptions cosmosSerializationOptions = null)
         {
             IJsonWriter jsonWriter;
             if (cosmosSerializationOptions != null)
@@ -172,7 +172,7 @@ internal static IEnumerable<T> Deserialize<T>(
             IEnumerable<CosmosElement> cosmosElements,
             ResourceType resourceType,
             CosmosSerializer jsonSerializer,
-            CosmosSerializationOptions cosmosSerializationOptions = null)
+            CosmosSerializationFormatOptions cosmosSerializationOptions = null)
         {
             if (!cosmosElements.Any())
             {

Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs

@@ -255,15 +255,27 @@ public CosmosClientBuilder WithThrottlingRetryOptions(TimeSpan maxRetryWaitTimeO
             return this;
         }
 
+        /// <summary>
+        /// Set a custom serializer option. 
+        /// </summary>
+        /// <param name="cosmosSerializerOptions">The custom class that implements <see cref="CosmosSerializer"/> </param>
+        /// <returns>The <see cref="CosmosClientBuilder"/> object</returns>
+        /// <seealso cref="CosmosSerializer"/>
+        /// <seealso cref="CosmosClientOptions.SerializerOptions"/>
+        public CosmosClientBuilder WithSerializerOptions(CosmosSerializationOptions cosmosSerializerOptions)
+        {
+            this.clientOptions.SerializerOptions = cosmosSerializerOptions;
+            return this;
+        }
+
         /// <summary>
         /// Set a custom JSON serializer. 
         /// </summary>
         /// <param name="cosmosJsonSerializer">The custom class that implements <see cref="CosmosSerializer"/> </param>
         /// <returns>The <see cref="CosmosClientBuilder"/> object</returns>
         /// <seealso cref="CosmosSerializer"/>
         /// <seealso cref="CosmosClientOptions.Serializer"/>
-        public CosmosClientBuilder WithCustomSerializer(
-            CosmosSerializer cosmosJsonSerializer)
+        public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSerializer)
         {
             this.clientOptions.Serializer = cosmosJsonSerializer;
             return this;

Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs

@@ -140,7 +140,7 @@ public ConsistencyLevel? ConsistencyLevel
         /// </remarks>
         internal string SessionToken { get; set; }
 
-        internal CosmosSerializationOptions CosmosSerializationOptions { get; set; }
+        internal CosmosSerializationFormatOptions CosmosSerializationOptions { get; set; }
 
         /// <summary>
         /// Gets or sets the flag that enables skip take across partitions.

Microsoft.Azure.Cosmos/src/Resource/QueryResponses/QueryResponse.cs

@@ -72,7 +72,7 @@ private QueryResponse(
         /// </remarks>
         internal ClientSideRequestStatistics RequestStatistics { get; }
 
-        internal virtual CosmosSerializationOptions CosmosSerializationOptions { get; set; }
+        internal virtual CosmosSerializationFormatOptions CosmosSerializationOptions { get; set; }
 
         internal bool GetHasMoreResults()
         {
@@ -137,15 +137,15 @@ internal class QueryResponse<T> : FeedResponse<T>
     {
         private readonly IEnumerable<CosmosElement> cosmosElements;
         private readonly CosmosSerializer jsonSerializer;
-        private readonly CosmosSerializationOptions serializationOptions;
+        private readonly CosmosSerializationFormatOptions serializationOptions;
         private IEnumerable<T> resources;
 
         private QueryResponse(
             HttpStatusCode httpStatusCode,
             IEnumerable<CosmosElement> cosmosElements,
             CosmosQueryResponseMessageHeaders responseMessageHeaders,
             CosmosSerializer jsonSerializer,
-            CosmosSerializationOptions serializationOptions)
+            CosmosSerializationFormatOptions serializationOptions)
         {
             this.cosmosElements = cosmosElements;
             this.QueryHeaders = responseMessageHeaders;

Microsoft.Azure.Cosmos/src/CosmosJsonDotNetSerializer.cs → Microsoft.Azure.Cosmos/src/Serializer/CosmosJsonDotNetSerializer.cs

@@ -7,32 +7,61 @@ namespace Microsoft.Azure.Cosmos
     using System.IO;
     using System.Text;
     using Newtonsoft.Json;
+    using Newtonsoft.Json.Serialization;
 
     /// <summary>
-    /// The default Cosmos JSON.NET serializer
+    /// The default Cosmos JSON.NET serializer.
     /// </summary>
-    public sealed class CosmosJsonDotNetSerializer : CosmosSerializer
+    internal sealed class CosmosJsonDotNetSerializer : CosmosSerializer
     {
         private static readonly Encoding DefaultEncoding = new UTF8Encoding(false, true);
         private readonly JsonSerializer Serializer;
 
         /// <summary>
         /// Create a serializer that uses the JSON.net serializer
         /// </summary>
-        /// <param name="jsonSerializerSettings">Optional serializer settings</param>
-        public CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSettings = null)
+        /// <remarks>
+        /// This is internal to reduce exposure of JSON.net types so
+        /// it is easier to convert to System.Text.Json
+        /// </remarks>
+        internal CosmosJsonDotNetSerializer()
         {
-            if (jsonSerializerSettings == null)
+            this.Serializer = JsonSerializer.Create();
+        }
+
+        /// <summary>
+        /// Create a serializer that uses the JSON.net serializer
+        /// </summary>
+        /// <remarks>
+        /// This is internal to reduce exposure of JSON.net types so
+        /// it is easier to convert to System.Text.Json
+        /// </remarks>
+        internal CosmosJsonDotNetSerializer(CosmosSerializationOptions cosmosSerializerOptions)
+        {
+            JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings()
             {
-                jsonSerializerSettings = new JsonSerializerSettings()
-                {
-                    NullValueHandling = NullValueHandling.Include
-                };
-            }
+                NullValueHandling = cosmosSerializerOptions.IgnoreNullValues ? NullValueHandling.Ignore : NullValueHandling.Include,
+                Formatting = cosmosSerializerOptions.Indented ? Formatting.Indented : Formatting.None,
+                ContractResolver = cosmosSerializerOptions.PropertyNamingPolicy == CosmosPropertyNamingPolicy.CamelCase
+                    ? new CamelCasePropertyNamesContractResolver()
+                    : null
+            };
 
             this.Serializer = JsonSerializer.Create(jsonSerializerSettings);
         }
 
+        /// <summary>
+        /// Create a serializer that uses the JSON.net serializer
+        /// </summary>
+        /// <remarks>
+        /// This is internal to reduce exposure of JSON.net types so
+        /// it is easier to convert to System.Text.Json
+        /// </remarks>
+        internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSettings)
+        {
+            this.Serializer = JsonSerializer.Create(jsonSerializerSettings);
+        }
+
         /// <summary>
         /// Convert a Stream to the passed in type.
         /// </summary>
@@ -45,7 +74,7 @@ public override T FromStream<T>(Stream stream)
             {
                 if (typeof(Stream).IsAssignableFrom(typeof(T)))
                 {
-                    return (T)(object)(stream);
+                    return (T)(object)stream;
                 }
 
                 using (StreamReader sr = new StreamReader(stream))