Enabling serialization customization through CosmosSerializerOptions (#…

…650)

* Made JsonSerializerSettings internal. Exposed a new constructor with popular flags to make it easier for users that only require basic flags. Refactored all serializer classes into a single folder.

* Fixed contract UT and updated changelog

* Made CosmosJsonDotNetSerializer internal. Added CosmosSerializerOptions to CosmosClientOptions. Added new UT.

* Fixed naming

* Updating file name and changelog

* Update contract test

* Fixed spacing

* Adding additional info to ValidateAsync test to understand transient failure

* Fixing contract enforcement test

* Updated naming

* Additional contract checks

* Updating contract from VS2017

* Fixed contract test for VS2019 to skip is IsReadOnlyAttribute

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Update Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializerOptions.cs

Co-Authored-By: Matias Quaranta <ealsur@users.noreply.github.com>

* Renamed CosmosSerializerOptions to CosmosSerializationOptions

* Added more comments

* Reverting DocumentClientWithUriParameters changes

* Revert "Reverting DocumentClientWithUriParameters changes"

This reverts commit febd523.

* Reverting accidental changes

* Adding comment examples, Converted options to be nullable

* Updated contract test

* Fixed Unit test

* Converted to a class with default values and updated changelog

* Updated contract api

Microsoft.Azure.Cosmos/src/CosmosClient.cs

@@ -150,7 +150,7 @@ protected CosmosClient()
  /// <seealso href="https://docs.microsoft.com/azure/cosmos-db/troubleshoot-dot-net-sdk"/>
  /// </remarks>
  public CosmosClient(
- string connectionString, 
+ string connectionString,
  CosmosClientOptions clientOptions = null)
  : this(
  CosmosClientOptions.GetAccountEndpoint(connectionString),
@@ -612,18 +612,19 @@ internal void Init(
 
  this.RequestHandler = clientPipelineBuilder.Build();
 
+ CosmosSerializer userSerializer = this.ClientOptions.GetCosmosSerializerWithWrapperOrDefault();
  this.ResponseFactory = new CosmosResponseFactory(
  defaultJsonSerializer: this.ClientOptions.PropertiesSerializer,
- userJsonSerializer: this.ClientOptions.CosmosSerializerWithWrapperOrDefault);
+ userJsonSerializer: userSerializer);
 
  CosmosSerializer sqlQuerySpecSerializer = CosmosSqlQuerySpecJsonConverter.CreateSqlQuerySpecSerializer(
- this.ClientOptions.CosmosSerializerWithWrapperOrDefault,
- this.ClientOptions.PropertiesSerializer);
+ cosmosSerializer: userSerializer,
+ propertiesSerializer: this.ClientOptions.PropertiesSerializer);
 
  this.ClientContext = new ClientContextCore(
  client: this,
  clientOptions: this.ClientOptions,
- userJsonSerializer: this.ClientOptions.CosmosSerializerWithWrapperOrDefault,
+ userJsonSerializer: userSerializer,
  defaultJsonSerializer: this.ClientOptions.PropertiesSerializer,
  sqlQuerySpecSerializer: sqlQuerySpecSerializer,
  cosmosResponseFactory: this.ResponseFactory,
@@ -632,7 +633,7 @@ internal void Init(
  documentQueryClient: new DocumentQueryClient(this.DocumentClient));
  }
 
- internal async virtual Task<ConsistencyLevel> GetAccountConsistencyLevelAsync()
+ internal virtual async Task<ConsistencyLevel> GetAccountConsistencyLevelAsync()
  {
  if (!this.accountConsistencyLevel.HasValue)
  {

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos
  using System.Collections.ObjectModel;
  using System.Data.Common;
  using System.Linq;
- using System.Runtime.ConstrainedExecution;
  using Microsoft.Azure.Cosmos.Fluent;
  using Microsoft.Azure.Documents;
  using Microsoft.Azure.Documents.Client;
@@ -17,6 +16,18 @@ namespace Microsoft.Azure.Cosmos
  /// <summary>
  /// Defines all the configurable options that the CosmosClient requires.
  /// </summary>
+ /// <example>
+ /// An example on how to configure the serialization option to ignore null values
+ /// CosmosClientOptions clientOptions = new CosmosClientOptions()
+ /// {
+ /// SerializerOptions = new CosmosSerializationOptions(){
+ /// IgnoreNullValues = true
+ /// },
+ /// ConnectionMode = ConnectionMode.Gateway,
+ /// };
+ /// 
+ /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
+ /// </example>
  public class CosmosClientOptions
  {
  /// <summary>
@@ -39,11 +50,12 @@ public class CosmosClientOptions
  /// </summary>
  private static readonly CosmosSerializer propertiesSerializer = new CosmosJsonSerializerWrapper(new CosmosJsonDotNetSerializer());
 
- private readonly Collection<RequestHandler> customHandlers;
  private readonly string currentEnvironmentInformation;
 
  private int gatewayModeMaxConnectionLimit;
  private string applicationName;
+ private CosmosSerializationOptions serializerOptions;
+ private CosmosSerializer serializer;
 
  /// <summary>
  /// Creates a new CosmosClientOptions
@@ -59,7 +71,7 @@ public CosmosClientOptions()
  this.ConnectionMode = CosmosClientOptions.DefaultConnectionMode;
  this.ConnectionProtocol = CosmosClientOptions.DefaultProtocol;
  this.ApiType = CosmosClientOptions.DefaultApiType;
- this.customHandlers = new Collection<RequestHandler>();
+ this.CustomHandlers = new Collection<RequestHandler>();
  }
 
  /// <summary>
@@ -132,10 +144,7 @@ public int GatewayModeMaxConnectionLimit
  /// </summary>
  /// <seealso cref="CosmosClientBuilder.AddCustomHandlers(RequestHandler[])"/>
  [JsonConverter(typeof(ClientOptionJsonConverter))]
- public Collection<RequestHandler> CustomHandlers
- {
- get => this.customHandlers;
- }
+ public Collection<RequestHandler> CustomHandlers { get; }
 
  /// <summary>
  /// Get or set the connection mode used by the client when connecting to the Azure Cosmos DB service.
@@ -157,7 +166,7 @@ public Collection<RequestHandler> CustomHandlers
  public ConsistencyLevel? ConsistencyLevel { get; set; }
 
  /// <summary>
- /// Get ot set the number of times client should retry on rate throttled requests.
+ /// Get or set the number of times client should retry on rate throttled requests.
  /// </summary>
  /// <seealso cref="CosmosClientBuilder.WithThrottlingRetryOptions(TimeSpan, int)"/>
  public int? MaxRetryAttemptsOnRateLimitedRequests { get; set; }
@@ -171,6 +180,35 @@ public Collection<RequestHandler> CustomHandlers
  /// <seealso cref="CosmosClientBuilder.WithThrottlingRetryOptions(TimeSpan, int)"/>
  public TimeSpan? MaxRetryWaitTimeOnRateLimitedRequests { get; set; }
 
+ /// <summary>
+ /// Get to set optional serializer options. 
+ /// </summary>
+ /// <example>
+ /// An example on how to configure the serialization option to ignore null values
+ /// CosmosClientOptions clientOptions = new CosmosClientOptions()
+ /// {
+ /// SerializerOptions = new CosmosSerializationOptions(){
+ /// IgnoreNullValues = true
+ /// }
+ /// };
+ /// 
+ /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
+ /// </example>
+ public CosmosSerializationOptions SerializerOptions
+ {
+ get => this.serializerOptions;
+ set
+ {
+ if (this.Serializer != null)
+ {
+ throw new ArgumentException(
+ $"{nameof(this.SerializerOptions)} is not compatible with {nameof(this.Serializer)}. Only one can be set. ");
+ }
+
+ this.serializerOptions = value;
+ }
+ }
+
  /// <summary>
  /// Get to set an optional JSON serializer. The client will use it to serialize or de-serialize user's cosmos request/responses.
  /// SDK owned types such as DatabaseProperties and ContainerProperties will always use the SDK default serializer.
@@ -182,10 +220,7 @@ public Collection<RequestHandler> CustomHandlers
  /// <example>
  /// // An example on how to configure the serializer to ignore null values
  /// CosmosSerializer ignoreNullSerializer = new CosmosJsonDotNetSerializer(
- /// new JsonSerializerSettings()
- /// {
- /// NullValueHandling = NullValueHandling.Ignore
- /// });
+ /// NullValueHandling = NullValueHandling.Ignore);
  /// 
  /// CosmosClientOptions clientOptions = new CosmosClientOptions()
  /// {
@@ -195,7 +230,20 @@ public Collection<RequestHandler> CustomHandlers
  /// CosmosClient client = new CosmosClient("endpoint", "key", clientOptions);
  /// </example>
  [JsonConverter(typeof(ClientOptionJsonConverter))]
- public CosmosSerializer Serializer { get; set; }
+ public CosmosSerializer Serializer
+ {
+ get => this.serializer;
+ set
+ {
+ if (this.SerializerOptions != null)
+ {
+ throw new ArgumentException(
+ $"{nameof(this.Serializer)} is not compatible with {nameof(this.SerializerOptions)}. Only one can be set. ");
+ }
+
+ this.serializer = value;
+ }
+ }
 
  /// <summary>
  /// A JSON serializer used by the CosmosClient to serialize or de-serialize cosmos request/responses.
@@ -205,12 +253,6 @@ public Collection<RequestHandler> CustomHandlers
  [JsonConverter(typeof(ClientOptionJsonConverter))]
  internal CosmosSerializer PropertiesSerializer => CosmosClientOptions.propertiesSerializer;
 
- /// <summary>
- /// Gets the user json serializer with the CosmosJsonSerializerWrapper or the default
- /// </summary>
- [JsonIgnore]
- internal CosmosSerializer CosmosSerializerWithWrapperOrDefault => this.Serializer == null ? this.PropertiesSerializer : new CosmosJsonSerializerWrapper(this.Serializer);
-
  /// <summary>
  /// Gets or sets the connection protocol when connecting to the Azure Cosmos service.
  /// </summary>
@@ -309,6 +351,22 @@ public Collection<RequestHandler> CustomHandlers
  /// </summary>
  internal bool? EnableCpuMonitor { get; set; }
 
+ /// <summary>
+ /// Gets the user json serializer with the CosmosJsonSerializerWrapper or the default
+ /// </summary>
+ internal CosmosSerializer GetCosmosSerializerWithWrapperOrDefault()
+ {
+ if (this.SerializerOptions != null)
+ {
+ CosmosJsonDotNetSerializer cosmosJsonDotNetSerializer = new CosmosJsonDotNetSerializer(this.SerializerOptions);
+ return new CosmosJsonSerializerWrapper(cosmosJsonDotNetSerializer);
+ }
+ else
+ {
+ return this.Serializer == null ? this.PropertiesSerializer : new CosmosJsonSerializerWrapper(this.Serializer);
+ }
+ }
+
  internal CosmosClientOptions Clone()
  {
  CosmosClientOptions cloneConfiguration = (CosmosClientOptions)this.MemberwiseClone();

Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElementSerializer.cs

@@ -29,7 +29,7 @@ static class CosmosElementSerializer
  internal static CosmosArray ToCosmosElements(
  MemoryStream memoryStream,
  ResourceType resourceType,
- CosmosSerializationOptions cosmosSerializationOptions = null)
+ CosmosSerializationFormatOptions cosmosSerializationOptions = null)
  {
  if (!memoryStream.CanRead)
  {
@@ -103,7 +103,7 @@ internal static Stream ToStream(
  string containerRid,
  IEnumerable<CosmosElement> cosmosElements,
  ResourceType resourceType,
- CosmosSerializationOptions cosmosSerializationOptions = null)
+ CosmosSerializationFormatOptions cosmosSerializationOptions = null)
  {
  IJsonWriter jsonWriter;
  if (cosmosSerializationOptions != null)
@@ -172,7 +172,7 @@ internal static IEnumerable<T> Deserialize<T>(
  IEnumerable<CosmosElement> cosmosElements,
  ResourceType resourceType,
  CosmosSerializer jsonSerializer,
- CosmosSerializationOptions cosmosSerializationOptions = null)
+ CosmosSerializationFormatOptions cosmosSerializationOptions = null)
  {
  if (!cosmosElements.Any())
  {

Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs

@@ -255,15 +255,27 @@ public CosmosClientBuilder WithThrottlingRetryOptions(TimeSpan maxRetryWaitTimeO
  return this;
  }
 
+ /// <summary>
+ /// Set a custom serializer option. 
+ /// </summary>
+ /// <param name="cosmosSerializerOptions">The custom class that implements <see cref="CosmosSerializer"/> </param>
+ /// <returns>The <see cref="CosmosClientBuilder"/> object</returns>
+ /// <seealso cref="CosmosSerializer"/>
+ /// <seealso cref="CosmosClientOptions.SerializerOptions"/>
+ public CosmosClientBuilder WithSerializerOptions(CosmosSerializationOptions cosmosSerializerOptions)
+ {
+ this.clientOptions.SerializerOptions = cosmosSerializerOptions;
+ return this;
+ }
+
  /// <summary>
  /// Set a custom JSON serializer. 
  /// </summary>
  /// <param name="cosmosJsonSerializer">The custom class that implements <see cref="CosmosSerializer"/> </param>
  /// <returns>The <see cref="CosmosClientBuilder"/> object</returns>
  /// <seealso cref="CosmosSerializer"/>
  /// <seealso cref="CosmosClientOptions.Serializer"/>
- public CosmosClientBuilder WithCustomSerializer(
- CosmosSerializer cosmosJsonSerializer)
+ public CosmosClientBuilder WithCustomSerializer(CosmosSerializer cosmosJsonSerializer)
  {
  this.clientOptions.Serializer = cosmosJsonSerializer;
  return this;

Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs

@@ -140,7 +140,7 @@ public ConsistencyLevel? ConsistencyLevel
  /// </remarks>
  internal string SessionToken { get; set; }
 
- internal CosmosSerializationOptions CosmosSerializationOptions { get; set; }
+ internal CosmosSerializationFormatOptions CosmosSerializationOptions { get; set; }
 
  /// <summary>
  /// Gets or sets the flag that enables skip take across partitions.

Microsoft.Azure.Cosmos/src/Resource/QueryResponses/QueryResponse.cs

@@ -72,7 +72,7 @@ private QueryResponse(
  /// </remarks>
  internal ClientSideRequestStatistics RequestStatistics { get; }
 
- internal virtual CosmosSerializationOptions CosmosSerializationOptions { get; set; }
+ internal virtual CosmosSerializationFormatOptions CosmosSerializationOptions { get; set; }
 
  internal bool GetHasMoreResults()
  {
@@ -137,15 +137,15 @@ internal class QueryResponse<T> : FeedResponse<T>
  {
  private readonly IEnumerable<CosmosElement> cosmosElements;
  private readonly CosmosSerializer jsonSerializer;
- private readonly CosmosSerializationOptions serializationOptions;
+ private readonly CosmosSerializationFormatOptions serializationOptions;
  private IEnumerable<T> resources;
 
  private QueryResponse(
  HttpStatusCode httpStatusCode,
  IEnumerable<CosmosElement> cosmosElements,
  CosmosQueryResponseMessageHeaders responseMessageHeaders,
  CosmosSerializer jsonSerializer,
- CosmosSerializationOptions serializationOptions)
+ CosmosSerializationFormatOptions serializationOptions)
  {
  this.cosmosElements = cosmosElements;
  this.QueryHeaders = responseMessageHeaders;

Microsoft.Azure.Cosmos/src/CosmosJsonDotNetSerializer.cs → Microsoft.Azure.Cosmos/src/Serializer/CosmosJsonDotNetSerializer.cs

@@ -7,32 +7,61 @@ namespace Microsoft.Azure.Cosmos
  using System.IO;
  using System.Text;
  using Newtonsoft.Json;
+ using Newtonsoft.Json.Serialization;
 
  /// <summary>
- /// The default Cosmos JSON.NET serializer
+ /// The default Cosmos JSON.NET serializer.
  /// </summary>
- public sealed class CosmosJsonDotNetSerializer : CosmosSerializer
+ internal sealed class CosmosJsonDotNetSerializer : CosmosSerializer
  {
  private static readonly Encoding DefaultEncoding = new UTF8Encoding(false, true);
  private readonly JsonSerializer Serializer;
 
  /// <summary>
  /// Create a serializer that uses the JSON.net serializer
  /// </summary>
- /// <param name="jsonSerializerSettings">Optional serializer settings</param>
- public CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSettings = null)
+ /// <remarks>
+ /// This is internal to reduce exposure of JSON.net types so
+ /// it is easier to convert to System.Text.Json
+ /// </remarks>
+ internal CosmosJsonDotNetSerializer()
  {
- if (jsonSerializerSettings == null)
+ this.Serializer = JsonSerializer.Create();
+ }
+
+ /// <summary>
+ /// Create a serializer that uses the JSON.net serializer
+ /// </summary>
+ /// <remarks>
+ /// This is internal to reduce exposure of JSON.net types so
+ /// it is easier to convert to System.Text.Json
+ /// </remarks>
+ internal CosmosJsonDotNetSerializer(CosmosSerializationOptions cosmosSerializerOptions)
+ {
+ JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings()
  {
- jsonSerializerSettings = new JsonSerializerSettings()
- {
- NullValueHandling = NullValueHandling.Include
- };
- }
+ NullValueHandling = cosmosSerializerOptions.IgnoreNullValues ? NullValueHandling.Ignore : NullValueHandling.Include,
+ Formatting = cosmosSerializerOptions.Indented ? Formatting.Indented : Formatting.None,
+ ContractResolver = cosmosSerializerOptions.PropertyNamingPolicy == CosmosPropertyNamingPolicy.CamelCase
+ ? new CamelCasePropertyNamesContractResolver()
+ : null
+ };
 
  this.Serializer = JsonSerializer.Create(jsonSerializerSettings);
  }
 
+ /// <summary>
+ /// Create a serializer that uses the JSON.net serializer
+ /// </summary>
+ /// <remarks>
+ /// This is internal to reduce exposure of JSON.net types so
+ /// it is easier to convert to System.Text.Json
+ /// </remarks>
+ internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSettings)
+ {
+ this.Serializer = JsonSerializer.Create(jsonSerializerSettings);
+ }
+
  /// <summary>
  /// Convert a Stream to the passed in type.
  /// </summary>
@@ -45,7 +74,7 @@ public override T FromStream<T>(Stream stream)
  {
  if (typeof(Stream).IsAssignableFrom(typeof(T)))
  {
- return (T)(object)(stream);
+ return (T)(object)stream;
  }
 
  using (StreamReader sr = new StreamReader(stream))