diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index e53a5deada..ceeefbc491 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -88,10 +88,10 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private const string RntbdReceiveHangDetectionTimeConfig = "CosmosDbTcpReceiveHangDetectionTimeSeconds"; private const string RntbdSendHangDetectionTimeConfig = "CosmosDbTcpSendHangDetectionTimeSeconds"; private const string EnableCpuMonitorConfig = "CosmosDbEnableCpuMonitor"; - - ////The MAC signature found in the HTTP request is not the same as the computed signature.Server used following string to sign - ////The input authorization token can't serve the request. Please check that the expected payload is built as per the protocol, and check the key being used. Server used the following payload to sign - private const string MacSignatureString = "to sign"; + + ////The MAC signature found in the HTTP request is not the same as the computed signature.Server used following string to sign + ////The input authorization token can't serve the request. Please check that the expected payload is built as per the protocol, and check the key being used. Server used the following payload to sign + private const string MacSignatureString = "to sign"; private const int MaxConcurrentConnectionOpenRequestsPerProcessor = 25; private const int DefaultMaxRequestsPerRntbdChannel = 30; @@ -103,7 +103,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private const bool DefaultEnableCpuMonitor = true; private readonly IDictionary> resourceTokens; - private ConnectionPolicy connectionPolicy; private RetryPolicy retryPolicy; private bool allowOverrideStrongerConsistency = false; private int maxConcurrentConnectionOpenRequests = Environment.ProcessorCount * MaxConcurrentConnectionOpenRequestsPerProcessor; @@ -147,11 +146,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider // This flag is used to allow shared store client factory survive disposition of a document client while other clients continue using it. private bool isStoreClientFactoryCreatedInternally; - //Based on connectivity mode we will either have ServerStoreModel / GatewayStoreModel here. - private IStoreModel storeModel; - //We will always have GatewayStoreModel for certain Master operations(viz: Collection CRUD) - private IStoreModel gatewayStoreModel; - //Id counter. private static int idCounter; //Trace Id. @@ -164,9 +158,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private readonly string authKeyResourceToken = string.Empty; private DocumentClientEventSource eventSource; - private GlobalEndpointManager globalEndpointManager; - private bool useMultipleWriteLocations; - internal Task initializeTask; private JsonSerializerSettings serializerSettings; @@ -174,10 +165,6 @@ internal partial class DocumentClient : IDisposable, IAuthorizationTokenProvider private event EventHandler receivedResponse; private Func transportClientHandlerFactory; - //Callback for on execution of scalar LINQ queries event. - //This callback is meant for tests only. - private Action onExecuteScalarQueryCallback; - /// /// Initializes a new instance of the class using the /// specified Azure Cosmos DB service endpoint, key, and connection policy for the Azure Cosmos DB service. @@ -450,8 +437,8 @@ internal DocumentClient(Uri serviceEndpoint, serviceEndpoint: serviceEndpoint, connectionPolicy: connectionPolicy, desiredConsistencyLevel: desiredConsistencyLevel, - handler: handler, - sessionContainer: sessionContainer, + handler: handler, + sessionContainer: sessionContainer, enableCpuMonitor: enableCpuMonitor, storeClientFactory: storeClientFactory); } @@ -727,10 +714,7 @@ internal event EventHandler SendingRequest } } - internal GlobalEndpointManager GlobalEndpointManager - { - get { return this.globalEndpointManager; } - } + internal GlobalEndpointManager GlobalEndpointManager { get; private set; } /// /// Open the connection to validate that the client initialization is successful in the Azure Cosmos DB service. @@ -755,7 +739,7 @@ internal GlobalEndpointManager GlobalEndpointManager /// public Task OpenAsync(CancellationToken cancellationToken = default(CancellationToken)) { - return TaskHelper.InlineIfPossibleAsync(() => OpenPrivateInlineAsync(cancellationToken), null, cancellationToken); + return TaskHelper.InlineIfPossibleAsync(() => this.OpenPrivateInlineAsync(cancellationToken), null, cancellationToken); } private async Task OpenPrivateInlineAsync(CancellationToken cancellationToken) @@ -792,8 +776,8 @@ private async Task OpenPrivateAsync(CancellationToken cancellationToken) catch (DocumentClientException ex) { // Clear the caches to ensure that we don't have partial results - this.collectionCache = new ClientCollectionCache(this.sessionContainer, this.gatewayStoreModel, this, this.retryPolicy); - this.partitionKeyRangeCache = new PartitionKeyRangeCache(this, this.gatewayStoreModel, this.collectionCache); + this.collectionCache = new ClientCollectionCache(this.sessionContainer, this.GatewayStoreModel, this, this.retryPolicy); + this.partitionKeyRangeCache = new PartitionKeyRangeCache(this, this.GatewayStoreModel, this.collectionCache); DefaultTrace.TraceWarning("{0} occurred while OpenAsync. Exception Message: {1}", ex.ToString(), ex.Message); } @@ -812,157 +796,157 @@ internal virtual void Initialize(Uri serviceEndpoint, throw new ArgumentNullException("serviceEndpoint"); } - DefaultTrace.InitEventListener(); - + DefaultTrace.InitEventListener(); + #if !(NETSTANDARD15 || NETSTANDARD16) #if NETSTANDARD20 // GetEntryAssembly returns null when loaded from native netstandard2.0 - if (System.Reflection.Assembly.GetEntryAssembly() != null) - { -#endif - // For tests we want to allow stronger consistency during construction or per call - string allowOverrideStrongerConsistencyConfig = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.AllowOverrideStrongerConsistency]; - if (!string.IsNullOrEmpty(allowOverrideStrongerConsistencyConfig)) - { - if (!bool.TryParse(allowOverrideStrongerConsistencyConfig, out this.allowOverrideStrongerConsistency)) - { - this.allowOverrideStrongerConsistency = false; - } - } - - // We might want to override the defaults sometime - string maxConcurrentConnectionOpenRequestsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxConcurrentConnectionOpenConfig]; - if (!string.IsNullOrEmpty(maxConcurrentConnectionOpenRequestsOverrideString)) - { - int maxConcurrentConnectionOpenRequestOverrideInt = 0; - if (Int32.TryParse(maxConcurrentConnectionOpenRequestsOverrideString, out maxConcurrentConnectionOpenRequestOverrideInt)) - { - this.maxConcurrentConnectionOpenRequests = maxConcurrentConnectionOpenRequestOverrideInt; - } - } - - string openConnectionTimeoutInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.OpenConnectionTimeoutInSecondsConfig]; - if (!string.IsNullOrEmpty(openConnectionTimeoutInSecondsOverrideString)) - { - int openConnectionTimeoutInSecondsOverrideInt = 0; - if (Int32.TryParse(openConnectionTimeoutInSecondsOverrideString, out openConnectionTimeoutInSecondsOverrideInt)) - { - this.openConnectionTimeoutInSeconds = openConnectionTimeoutInSecondsOverrideInt; - } - } - - string idleConnectionTimeoutInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.IdleConnectionTimeoutInSecondsConfig]; - if (!string.IsNullOrEmpty(idleConnectionTimeoutInSecondsOverrideString)) - { - int idleConnectionTimeoutInSecondsOverrideInt = 0; - if (Int32.TryParse(idleConnectionTimeoutInSecondsOverrideString, out idleConnectionTimeoutInSecondsOverrideInt)) - { - this.idleConnectionTimeoutInSeconds = idleConnectionTimeoutInSecondsOverrideInt; - } - } - - string transportTimerPoolGranularityInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.TransportTimerPoolGranularityInSecondsConfig]; - if (!string.IsNullOrEmpty(transportTimerPoolGranularityInSecondsOverrideString)) - { - int timerPoolGranularityInSecondsOverrideInt = 0; - if (Int32.TryParse(transportTimerPoolGranularityInSecondsOverrideString, out timerPoolGranularityInSecondsOverrideInt)) - { - // timeoutgranularity specified should be greater than min(5 seconds) - if (timerPoolGranularityInSecondsOverrideInt > this.timerPoolGranularityInSeconds) - { - this.timerPoolGranularityInSeconds = timerPoolGranularityInSecondsOverrideInt; - } - } - } - - string enableRntbdChannelOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.EnableTcpChannelConfig]; - if (!string.IsNullOrEmpty(enableRntbdChannelOverrideString)) - { - bool enableRntbdChannel = false; - if (bool.TryParse(enableRntbdChannelOverrideString, out enableRntbdChannel)) - { - this.enableRntbdChannel = enableRntbdChannel; - } - } - - string maxRequestsPerRntbdChannelOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxRequestsPerChannelConfig]; - if (!string.IsNullOrEmpty(maxRequestsPerRntbdChannelOverrideString)) - { - int maxRequestsPerChannel = DocumentClient.DefaultMaxRequestsPerRntbdChannel; - if (int.TryParse(maxRequestsPerRntbdChannelOverrideString, out maxRequestsPerChannel)) - { - this.maxRequestsPerRntbdChannel = maxRequestsPerChannel; - } - } - - string rntbdPartitionCountOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.TcpPartitionCount]; - if (!string.IsNullOrEmpty(rntbdPartitionCountOverrideString)) - { - int rntbdPartitionCount = DocumentClient.DefaultRntbdPartitionCount; - if (int.TryParse(rntbdPartitionCountOverrideString, out rntbdPartitionCount)) - { - this.rntbdPartitionCount = rntbdPartitionCount; - } - } - - string maxRntbdChannelsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxChannelsPerHostConfig]; - if (!string.IsNullOrEmpty(maxRntbdChannelsOverrideString)) - { - int maxRntbdChannels = DefaultMaxRntbdChannelsPerHost; - if (int.TryParse(maxRntbdChannelsOverrideString, out maxRntbdChannels)) - { - this.maxRntbdChannels = maxRntbdChannels; - } - } - - string rntbdPortReuseModeOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdPortReuseMode]; - if (!string.IsNullOrEmpty(rntbdPortReuseModeOverrideString)) - { - PortReuseMode portReuseMode = DefaultRntbdPortReuseMode; - if (Enum.TryParse(rntbdPortReuseModeOverrideString, out portReuseMode)) - { - this.rntbdPortReuseMode = portReuseMode; - } - } - - string rntbdReceiveHangDetectionTimeSecondsString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdReceiveHangDetectionTimeConfig]; - if (!string.IsNullOrEmpty(rntbdReceiveHangDetectionTimeSecondsString)) - { - int rntbdReceiveHangDetectionTimeSeconds = DefaultRntbdReceiveHangDetectionTimeSeconds; - if (int.TryParse(rntbdReceiveHangDetectionTimeSecondsString, out rntbdReceiveHangDetectionTimeSeconds)) - { - this.rntbdReceiveHangDetectionTimeSeconds = rntbdReceiveHangDetectionTimeSeconds; - } - } - - string rntbdSendHangDetectionTimeSecondsString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdSendHangDetectionTimeConfig]; - if (!string.IsNullOrEmpty(rntbdSendHangDetectionTimeSecondsString)) - { - int rntbdSendHangDetectionTimeSeconds = DefaultRntbdSendHangDetectionTimeSeconds; - if (int.TryParse(rntbdSendHangDetectionTimeSecondsString, out rntbdSendHangDetectionTimeSeconds)) - { - this.rntbdSendHangDetectionTimeSeconds = rntbdSendHangDetectionTimeSeconds; - } - } - - if (enableCpuMonitor.HasValue) - { - this.enableCpuMonitor = enableCpuMonitor.Value; - } - else - { - string enableCpuMonitorString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.EnableCpuMonitorConfig]; - if (!string.IsNullOrEmpty(enableCpuMonitorString)) - { - bool enableCpuMonitorFlag = DefaultEnableCpuMonitor; - if (bool.TryParse(enableCpuMonitorString, out enableCpuMonitorFlag)) - { - this.enableCpuMonitor = enableCpuMonitorFlag; - } - } - } -#if NETSTANDARD20 - } + if (System.Reflection.Assembly.GetEntryAssembly() != null) + { +#endif + // For tests we want to allow stronger consistency during construction or per call + string allowOverrideStrongerConsistencyConfig = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.AllowOverrideStrongerConsistency]; + if (!string.IsNullOrEmpty(allowOverrideStrongerConsistencyConfig)) + { + if (!bool.TryParse(allowOverrideStrongerConsistencyConfig, out this.allowOverrideStrongerConsistency)) + { + this.allowOverrideStrongerConsistency = false; + } + } + + // We might want to override the defaults sometime + string maxConcurrentConnectionOpenRequestsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxConcurrentConnectionOpenConfig]; + if (!string.IsNullOrEmpty(maxConcurrentConnectionOpenRequestsOverrideString)) + { + int maxConcurrentConnectionOpenRequestOverrideInt = 0; + if (Int32.TryParse(maxConcurrentConnectionOpenRequestsOverrideString, out maxConcurrentConnectionOpenRequestOverrideInt)) + { + this.maxConcurrentConnectionOpenRequests = maxConcurrentConnectionOpenRequestOverrideInt; + } + } + + string openConnectionTimeoutInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.OpenConnectionTimeoutInSecondsConfig]; + if (!string.IsNullOrEmpty(openConnectionTimeoutInSecondsOverrideString)) + { + int openConnectionTimeoutInSecondsOverrideInt = 0; + if (Int32.TryParse(openConnectionTimeoutInSecondsOverrideString, out openConnectionTimeoutInSecondsOverrideInt)) + { + this.openConnectionTimeoutInSeconds = openConnectionTimeoutInSecondsOverrideInt; + } + } + + string idleConnectionTimeoutInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.IdleConnectionTimeoutInSecondsConfig]; + if (!string.IsNullOrEmpty(idleConnectionTimeoutInSecondsOverrideString)) + { + int idleConnectionTimeoutInSecondsOverrideInt = 0; + if (Int32.TryParse(idleConnectionTimeoutInSecondsOverrideString, out idleConnectionTimeoutInSecondsOverrideInt)) + { + this.idleConnectionTimeoutInSeconds = idleConnectionTimeoutInSecondsOverrideInt; + } + } + + string transportTimerPoolGranularityInSecondsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.TransportTimerPoolGranularityInSecondsConfig]; + if (!string.IsNullOrEmpty(transportTimerPoolGranularityInSecondsOverrideString)) + { + int timerPoolGranularityInSecondsOverrideInt = 0; + if (Int32.TryParse(transportTimerPoolGranularityInSecondsOverrideString, out timerPoolGranularityInSecondsOverrideInt)) + { + // timeoutgranularity specified should be greater than min(5 seconds) + if (timerPoolGranularityInSecondsOverrideInt > this.timerPoolGranularityInSeconds) + { + this.timerPoolGranularityInSeconds = timerPoolGranularityInSecondsOverrideInt; + } + } + } + + string enableRntbdChannelOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.EnableTcpChannelConfig]; + if (!string.IsNullOrEmpty(enableRntbdChannelOverrideString)) + { + bool enableRntbdChannel = false; + if (bool.TryParse(enableRntbdChannelOverrideString, out enableRntbdChannel)) + { + this.enableRntbdChannel = enableRntbdChannel; + } + } + + string maxRequestsPerRntbdChannelOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxRequestsPerChannelConfig]; + if (!string.IsNullOrEmpty(maxRequestsPerRntbdChannelOverrideString)) + { + int maxRequestsPerChannel = DocumentClient.DefaultMaxRequestsPerRntbdChannel; + if (int.TryParse(maxRequestsPerRntbdChannelOverrideString, out maxRequestsPerChannel)) + { + this.maxRequestsPerRntbdChannel = maxRequestsPerChannel; + } + } + + string rntbdPartitionCountOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.TcpPartitionCount]; + if (!string.IsNullOrEmpty(rntbdPartitionCountOverrideString)) + { + int rntbdPartitionCount = DocumentClient.DefaultRntbdPartitionCount; + if (int.TryParse(rntbdPartitionCountOverrideString, out rntbdPartitionCount)) + { + this.rntbdPartitionCount = rntbdPartitionCount; + } + } + + string maxRntbdChannelsOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.MaxChannelsPerHostConfig]; + if (!string.IsNullOrEmpty(maxRntbdChannelsOverrideString)) + { + int maxRntbdChannels = DefaultMaxRntbdChannelsPerHost; + if (int.TryParse(maxRntbdChannelsOverrideString, out maxRntbdChannels)) + { + this.maxRntbdChannels = maxRntbdChannels; + } + } + + string rntbdPortReuseModeOverrideString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdPortReuseMode]; + if (!string.IsNullOrEmpty(rntbdPortReuseModeOverrideString)) + { + PortReuseMode portReuseMode = DefaultRntbdPortReuseMode; + if (Enum.TryParse(rntbdPortReuseModeOverrideString, out portReuseMode)) + { + this.rntbdPortReuseMode = portReuseMode; + } + } + + string rntbdReceiveHangDetectionTimeSecondsString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdReceiveHangDetectionTimeConfig]; + if (!string.IsNullOrEmpty(rntbdReceiveHangDetectionTimeSecondsString)) + { + int rntbdReceiveHangDetectionTimeSeconds = DefaultRntbdReceiveHangDetectionTimeSeconds; + if (int.TryParse(rntbdReceiveHangDetectionTimeSecondsString, out rntbdReceiveHangDetectionTimeSeconds)) + { + this.rntbdReceiveHangDetectionTimeSeconds = rntbdReceiveHangDetectionTimeSeconds; + } + } + + string rntbdSendHangDetectionTimeSecondsString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.RntbdSendHangDetectionTimeConfig]; + if (!string.IsNullOrEmpty(rntbdSendHangDetectionTimeSecondsString)) + { + int rntbdSendHangDetectionTimeSeconds = DefaultRntbdSendHangDetectionTimeSeconds; + if (int.TryParse(rntbdSendHangDetectionTimeSecondsString, out rntbdSendHangDetectionTimeSeconds)) + { + this.rntbdSendHangDetectionTimeSeconds = rntbdSendHangDetectionTimeSeconds; + } + } + + if (enableCpuMonitor.HasValue) + { + this.enableCpuMonitor = enableCpuMonitor.Value; + } + else + { + string enableCpuMonitorString = System.Configuration.ConfigurationManager.AppSettings[DocumentClient.EnableCpuMonitorConfig]; + if (!string.IsNullOrEmpty(enableCpuMonitorString)) + { + bool enableCpuMonitorFlag = DefaultEnableCpuMonitor; + if (bool.TryParse(enableCpuMonitorString, out enableCpuMonitorFlag)) + { + this.enableCpuMonitor = enableCpuMonitorFlag; + } + } + } +#if NETSTANDARD20 + } #endif #endif @@ -994,29 +978,29 @@ internal virtual void Initialize(Uri serviceEndpoint, this.maxRntbdChannels = this.ConnectionPolicy.MaxTcpConnectionsPerEndpoint.Value; } - if (this.ConnectionPolicy.PortReuseMode.HasValue) - { - this.rntbdPortReuseMode = this.ConnectionPolicy.PortReuseMode.Value; + if (this.ConnectionPolicy.PortReuseMode.HasValue) + { + this.rntbdPortReuseMode = this.ConnectionPolicy.PortReuseMode.Value; } } this.ServiceEndpoint = serviceEndpoint.OriginalString.EndsWith("/", StringComparison.Ordinal) ? serviceEndpoint : new Uri(serviceEndpoint.OriginalString + "/"); - this.connectionPolicy = connectionPolicy ?? ConnectionPolicy.Default; + this.ConnectionPolicy = connectionPolicy ?? ConnectionPolicy.Default; #if !NETSTANDARD16 ServicePoint servicePoint = ServicePointManager.FindServicePoint(this.ServiceEndpoint); - servicePoint.ConnectionLimit = this.connectionPolicy.MaxConnectionLimit; + servicePoint.ConnectionLimit = this.ConnectionPolicy.MaxConnectionLimit; #endif - this.globalEndpointManager = new GlobalEndpointManager(this, this.connectionPolicy); + this.GlobalEndpointManager = new GlobalEndpointManager(this, this.ConnectionPolicy); this.httpMessageHandler = new HttpRequestMessageHandler(this.sendingRequest, this.receivedResponse, handler); this.mediaClient = new HttpClient(this.httpMessageHandler); this.mediaClient.DefaultRequestHeaders.CacheControl = new CacheControlHeaderValue { NoCache = true }; - this.mediaClient.AddUserAgentHeader(this.connectionPolicy.UserAgentContainer); + this.mediaClient.AddUserAgentHeader(this.ConnectionPolicy.UserAgentContainer); this.mediaClient.AddApiTypeHeader(this.ApiType); @@ -1037,10 +1021,10 @@ internal virtual void Initialize(Uri serviceEndpoint, this.sessionContainer = new SessionContainer(this.ServiceEndpoint.Host); } - this.retryPolicy = new RetryPolicy(this.globalEndpointManager, this.connectionPolicy); + this.retryPolicy = new RetryPolicy(this.GlobalEndpointManager, this.ConnectionPolicy); this.ResetSessionTokenRetryPolicy = this.retryPolicy; - this.mediaClient.Timeout = this.connectionPolicy.MediaRequestTimeout; + this.mediaClient.Timeout = this.ConnectionPolicy.MediaRequestTimeout; this.desiredConsistencyLevel = desiredConsistencyLevel; // Setup the proxy to be used based on connection mode. @@ -1053,8 +1037,8 @@ internal virtual void Initialize(Uri serviceEndpoint, this.initializeTask = TaskHelper.InlineIfPossibleAsync( () => this.GetInitializationTaskAsync(storeClientFactory: storeClientFactory), new ResourceThrottleRetryPolicy( - this.connectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests, - this.connectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds)); + this.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests, + this.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds)); // ContinueWith on the initialization task is needed for handling the UnobservedTaskException // if this task throws for some reason. Awaiting inside a constructor is not supported and @@ -1075,8 +1059,8 @@ internal virtual void Initialize(Uri serviceEndpoint, "DocumentClient with id {0} initialized at endpoint: {1} with ConnectionMode: {2}, connection Protocol: {3}, and consistency level: {4}", this.traceId, serviceEndpoint.ToString(), - this.connectionPolicy.ConnectionMode.ToString(), - this.connectionPolicy.ConnectionProtocol.ToString(), + this.ConnectionPolicy.ConnectionMode.ToString(), + this.ConnectionPolicy.ConnectionProtocol.ToString(), desiredConsistencyLevel != null ? desiredConsistencyLevel.ToString() : "null")); this.QueryCompatibilityMode = QueryCompatibilityMode.Default; @@ -1093,25 +1077,25 @@ private async Task GetInitializationTaskAsync(IStoreClientFactory storeClientFac } GatewayStoreModel gatewayStoreModel = new GatewayStoreModel( - this.globalEndpointManager, + this.GlobalEndpointManager, this.sessionContainer, - this.connectionPolicy.RequestTimeout, + this.ConnectionPolicy.RequestTimeout, (Cosmos.ConsistencyLevel)this.accountServiceConfiguration.DefaultConsistencyLevel, this.eventSource, this.serializerSettings, - this.connectionPolicy.UserAgentContainer, + this.ConnectionPolicy.UserAgentContainer, this.ApiType, this.httpMessageHandler); - this.gatewayStoreModel = gatewayStoreModel; + this.GatewayStoreModel = gatewayStoreModel; - this.collectionCache = new ClientCollectionCache(this.sessionContainer, this.gatewayStoreModel, this, this.retryPolicy); - this.partitionKeyRangeCache = new PartitionKeyRangeCache(this, this.gatewayStoreModel, this.collectionCache); + this.collectionCache = new ClientCollectionCache(this.sessionContainer, this.GatewayStoreModel, this, this.retryPolicy); + this.partitionKeyRangeCache = new PartitionKeyRangeCache(this, this.GatewayStoreModel, this.collectionCache); this.ResetSessionTokenRetryPolicy = new ResetSessionTokenRetryPolicyFactory(this.sessionContainer, this.collectionCache, this.retryPolicy); - if (this.connectionPolicy.ConnectionMode == ConnectionMode.Gateway) + if (this.ConnectionPolicy.ConnectionMode == ConnectionMode.Gateway) { - this.storeModel = this.gatewayStoreModel; + this.StoreModel = this.GatewayStoreModel; } else { @@ -1229,7 +1213,7 @@ internal ApiType ApiType get; private set; } - internal bool UseMultipleWriteLocations => this.useMultipleWriteLocations; + internal bool UseMultipleWriteLocations { get; private set; } /// /// Gets the endpoint Uri for the service endpoint from the Azure Cosmos DB service. @@ -1251,7 +1235,7 @@ public Uri WriteEndpoint { get { - return this.globalEndpointManager.WriteEndpoints.FirstOrDefault(); + return this.GlobalEndpointManager.WriteEndpoints.FirstOrDefault(); } } @@ -1262,7 +1246,7 @@ public Uri ReadEndpoint { get { - return this.globalEndpointManager.ReadEndpoints.FirstOrDefault(); + return this.GlobalEndpointManager.ReadEndpoints.FirstOrDefault(); } } @@ -1273,13 +1257,7 @@ public Uri ReadEndpoint /// The Connection policy used by the client. /// /// - public ConnectionPolicy ConnectionPolicy - { - get - { - return this.connectionPolicy; - } - } + public ConnectionPolicy ConnectionPolicy { get; private set; } /// /// Gets a dictionary of resource tokens used by the client from the Azure Cosmos DB service. @@ -1358,16 +1336,16 @@ public void Dispose() return; } - if (this.storeModel != null) + if (this.StoreModel != null) { - this.storeModel.Dispose(); - this.storeModel = null; + this.StoreModel.Dispose(); + this.StoreModel = null; } if (this.storeClientFactory != null) { // Dispose only if this store client factory was created and is owned by this instance of document client, otherwise just release the reference - if (isStoreClientFactoryCreatedInternally) + if (this.isStoreClientFactoryCreatedInternally) { this.storeClientFactory.Dispose(); } @@ -1393,10 +1371,10 @@ public void Dispose() this.authKeyHashFunction = null; } - if (this.globalEndpointManager != null) + if (this.GlobalEndpointManager != null) { - this.globalEndpointManager.Dispose(); - this.globalEndpointManager = null; + this.GlobalEndpointManager.Dispose(); + this.GlobalEndpointManager = null; } DefaultTrace.TraceInformation("DocumentClient with id {0} disposed.", this.traceId); @@ -1426,11 +1404,7 @@ public void Dispose() /// /// Test hook to enable unit test of DocumentClient. /// - internal IStoreModel StoreModel - { - get { return this.storeModel; } - set { this.storeModel = value; } - } + internal IStoreModel StoreModel { get; set; } /// /// Gets and sets the gateway IStoreModel object. @@ -1438,11 +1412,7 @@ internal IStoreModel StoreModel /// /// Test hook to enable unit test of DocumentClient. /// - internal IStoreModel GatewayStoreModel - { - get { return this.gatewayStoreModel; } - set { this.gatewayStoreModel = value; } - } + internal IStoreModel GatewayStoreModel { get; set; } /// /// Gets and sets on execute scalar query callback @@ -1450,11 +1420,7 @@ internal IStoreModel GatewayStoreModel /// /// Test hook to enable unit test for scalar queries /// - internal Action OnExecuteScalarQueryCallback - { - get { return this.onExecuteScalarQueryCallback; } - set { this.onExecuteScalarQueryCallback = value; } - } + internal Action OnExecuteScalarQueryCallback { get; set; } internal virtual async Task> GetQueryEngineConfigurationAsync() { @@ -1473,62 +1439,62 @@ internal virtual async Task GetDefaultConsistencyLevelAsync() return Task.FromResult(this.desiredConsistencyLevel); } - internal async Task ProcessRequestAsync( - string verb, - DocumentServiceRequest request, - IDocumentClientRetryPolicy retryPolicyInstance, - CancellationToken cancellationToken, - string testAuthorization = null) // Only for unit-tests - { - if (request == null) - { - throw new ArgumentNullException(nameof(request)); - } - - if (verb == null) - { - throw new ArgumentNullException(nameof(verb)); - } - - string payload; - string authorization = ((IAuthorizationTokenProvider)this).GetUserAuthorizationToken( - request.ResourceAddress, - PathsHelper.GetResourcePath(request.ResourceType), - verb, - request.Headers, - AuthorizationTokenType.PrimaryMasterKey, - out payload); - - // Unit-test hook - if (testAuthorization != null) - { - payload = testAuthorization; - authorization = testAuthorization; - } - request.Headers[HttpConstants.HttpHeaders.Authorization] = authorization; - - try - { - return await this.ProcessRequestAsync(request, retryPolicyInstance, cancellationToken); - } - catch (DocumentClientException dce) - { - if (payload != null - && dce.Message != null - && dce.StatusCode.HasValue - && dce.StatusCode.Value == HttpStatusCode.Unauthorized - && dce.Message.Contains(DocumentClient.MacSignatureString) - && !dce.Message.Contains(payload)) - { - DefaultTrace.TraceError("Un-expected authorization payload mis-match. Actual {0} service expected {1}", payload, dce.Message); - } - - throw; - } - } - - internal async Task ProcessRequestAsync( - DocumentServiceRequest request, + internal async Task ProcessRequestAsync( + string verb, + DocumentServiceRequest request, + IDocumentClientRetryPolicy retryPolicyInstance, + CancellationToken cancellationToken, + string testAuthorization = null) // Only for unit-tests + { + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (verb == null) + { + throw new ArgumentNullException(nameof(verb)); + } + + string payload; + string authorization = ((IAuthorizationTokenProvider)this).GetUserAuthorizationToken( + request.ResourceAddress, + PathsHelper.GetResourcePath(request.ResourceType), + verb, + request.Headers, + AuthorizationTokenType.PrimaryMasterKey, + out payload); + + // Unit-test hook + if (testAuthorization != null) + { + payload = testAuthorization; + authorization = testAuthorization; + } + request.Headers[HttpConstants.HttpHeaders.Authorization] = authorization; + + try + { + return await this.ProcessRequestAsync(request, retryPolicyInstance, cancellationToken); + } + catch (DocumentClientException dce) + { + if (payload != null + && dce.Message != null + && dce.StatusCode.HasValue + && dce.StatusCode.Value == HttpStatusCode.Unauthorized + && dce.Message.Contains(DocumentClient.MacSignatureString) + && !dce.Message.Contains(payload)) + { + DefaultTrace.TraceError("Un-expected authorization payload mis-match. Actual {0} service expected {1}", payload, dce.Message); + } + + throw; + } + } + + internal async Task ProcessRequestAsync( + DocumentServiceRequest request, IDocumentClientRetryPolicy retryPolicyInstance, CancellationToken cancellationToken) { @@ -1721,7 +1687,7 @@ internal virtual async Task EnsureValidClientAsync() /// public Task> CreateDatabaseIfNotExistsAsync(Documents.Database database, Documents.Client.RequestOptions options = null) { - return TaskHelper.InlineIfPossible(() => CreateDatabaseIfNotExistsPrivateAsync(database, options), null); + return TaskHelper.InlineIfPossible(() => this.CreateDatabaseIfNotExistsPrivateAsync(database, options), null); } private async Task> CreateDatabaseIfNotExistsPrivateAsync(Documents.Database database, @@ -1865,7 +1831,7 @@ public Task> CreateDocumentAsync(string documentsFeed CancellationToken cancellationToken = default(CancellationToken)) { // This call is to just run CreateDocumentInlineAsync in a SynchronizationContext aware environment - return TaskHelper.InlineIfPossible(() => CreateDocumentInlineAsync(documentsFeedOrDatabaseLink, document, options, disableAutomaticIdGeneration, cancellationToken), null, cancellationToken); + return TaskHelper.InlineIfPossible(() => this.CreateDocumentInlineAsync(documentsFeedOrDatabaseLink, document, options, disableAutomaticIdGeneration, cancellationToken), null, cancellationToken); } private async Task> CreateDocumentInlineAsync(string documentsFeedOrDatabaseLink, object document, Documents.Client.RequestOptions options, bool disableAutomaticIdGeneration, CancellationToken cancellationToken) @@ -2063,7 +2029,7 @@ private async Task> CreateDocumentCollectio /// public Task> CreateDocumentCollectionIfNotExistsAsync(string databaseLink, DocumentCollection documentCollection, Documents.Client.RequestOptions options = null) { - return TaskHelper.InlineIfPossible(() => CreateDocumentCollectionIfNotExistsPrivateAsync(databaseLink, documentCollection, options), null); + return TaskHelper.InlineIfPossible(() => this.CreateDocumentCollectionIfNotExistsPrivateAsync(databaseLink, documentCollection, options), null); } private async Task> CreateDocumentCollectionIfNotExistsPrivateAsync( @@ -3094,7 +3060,7 @@ private async Task> ReplaceDocumentCollecti public Task> ReplaceDocumentAsync(string documentLink, object document, Documents.Client.RequestOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { // This call is to just run ReplaceDocumentInlineAsync in a SynchronizationContext aware environment - return TaskHelper.InlineIfPossible(() => ReplaceDocumentInlineAsync(documentLink, document, options, cancellationToken), null, cancellationToken); + return TaskHelper.InlineIfPossible(() => this.ReplaceDocumentInlineAsync(documentLink, document, options, cancellationToken), null, cancellationToken); } private async Task> ReplaceDocumentInlineAsync(string documentLink, object document, Documents.Client.RequestOptions options, CancellationToken cancellationToken) @@ -3170,12 +3136,12 @@ private Task> ReplaceDocumentPrivateAsync(string docu { IDocumentClientRetryPolicy retryPolicyInstance = this.ResetSessionTokenRetryPolicy.GetRequestPolicy(); return TaskHelper.InlineIfPossible(() => this.ReplaceDocumentPrivateAsync( - this.GetLinkForRouting(document), - document, - options, - retryPolicyInstance, - cancellationToken), - retryPolicyInstance, + this.GetLinkForRouting(document), + document, + options, + retryPolicyInstance, + cancellationToken), + retryPolicyInstance, cancellationToken); } @@ -3486,7 +3452,7 @@ private async Task> ReplaceOfferPrivateAsync(Offer offer AuthorizationTokenType.PrimaryMasterKey)) { return new ResourceResponse( - await this.UpdateAsync(request, retryPolicyInstance), + await this.UpdateAsync(request, retryPolicyInstance), OfferTypeResolver.ResponseOfferTypeResolver); } } @@ -4179,8 +4145,8 @@ public Task> ReadConflictAsync(string conflictLink, D private async Task> ReadConflictPrivateAsync(string conflictLink, Documents.Client.RequestOptions options, IDocumentClientRetryPolicy retryPolicyInstance) { - await this.EnsureValidClientAsync(); - + await this.EnsureValidClientAsync(); + if (string.IsNullOrEmpty(conflictLink)) { throw new ArgumentNullException("conflictLink"); @@ -4895,7 +4861,7 @@ private async Task> ReadUserDefinedFun /// public Task> ReadDocumentFeedAsync(string documentsLink, FeedOptions options = null, CancellationToken cancellationToken = default(CancellationToken)) { - return TaskHelper.InlineIfPossible(() => ReadDocumentFeedInlineAsync(documentsLink, options, cancellationToken), null, cancellationToken); + return TaskHelper.InlineIfPossible(() => this.ReadDocumentFeedInlineAsync(documentsLink, options, cancellationToken), null, cancellationToken); } private async Task> ReadDocumentFeedInlineAsync(string documentsLink, FeedOptions options, CancellationToken cancellationToken) @@ -4909,12 +4875,12 @@ private async Task> ReadDocumentFeedInlineAsync(st DocumentFeedResponse response = await this.CreateDocumentFeedReader(documentsLink, options).ExecuteNextAsync(cancellationToken); return new DocumentFeedResponse( - response.Cast(), - response.Count, - response.Headers, - response.UseETagAsContinuation, - response.QueryMetrics, - response.RequestStatistics, + response.Cast(), + response.Count, + response.Headers, + response.UseETagAsContinuation, + response.QueryMetrics, + response.RequestStatistics, responseLengthBytes: response.ResponseLengthBytes); } @@ -4970,7 +4936,7 @@ private async Task> ReadDocumentFeedInlineAsync(st /// public Task> ReadConflictFeedAsync(string conflictsLink, FeedOptions options = null) { - return TaskHelper.InlineIfPossible(() => ReadConflictFeedInlineAsync(conflictsLink, options), null); + return TaskHelper.InlineIfPossible(() => this.ReadConflictFeedInlineAsync(conflictsLink, options), null); } private async Task> ReadConflictFeedInlineAsync(string conflictsLink, FeedOptions options) @@ -5246,10 +5212,10 @@ public Task> ExecuteStoredProcedureAsync IDocumentClientRetryPolicy retryPolicyInstance = this.ResetSessionTokenRetryPolicy.GetRequestPolicy(); return TaskHelper.InlineIfPossible( () => this.ExecuteStoredProcedurePrivateAsync( - storedProcedureLink, - options, + storedProcedureLink, + options, retryPolicyInstance, - default(CancellationToken), + default(CancellationToken), procedureParams), retryPolicyInstance); } @@ -5288,20 +5254,20 @@ public Task> ExecuteStoredProcedureAsync IDocumentClientRetryPolicy retryPolicyInstance = this.ResetSessionTokenRetryPolicy.GetRequestPolicy(); return TaskHelper.InlineIfPossible( () => this.ExecuteStoredProcedurePrivateAsync( - storedProcedureLink, - options, - retryPolicyInstance, - cancellationToken, + storedProcedureLink, + options, + retryPolicyInstance, + cancellationToken, procedureParams), - retryPolicyInstance, + retryPolicyInstance, cancellationToken); } private async Task> ExecuteStoredProcedurePrivateAsync( string storedProcedureLink, Documents.Client.RequestOptions options, - IDocumentClientRetryPolicy retryPolicyInstance, - CancellationToken cancellationToken, + IDocumentClientRetryPolicy retryPolicyInstance, + CancellationToken cancellationToken, params dynamic[] procedureParams) { await this.EnsureValidClientAsync(); @@ -5347,9 +5313,9 @@ await this.AddPartitionKeyInformationAsync( request.SerializerSettings = this.GetSerializerSettingsForRequest(options); return new StoredProcedureResponse(await this.ExecuteProcedureAsync( - request, - retryPolicyInstance, - cancellationToken), + request, + retryPolicyInstance, + cancellationToken), this.GetSerializerSettingsForRequest(options)); } } @@ -5530,7 +5496,7 @@ await this.AddPartitionKeyInformationAsync( public Task> UpsertDocumentAsync(string documentsFeedOrDatabaseLink, object document, Documents.Client.RequestOptions options = null, bool disableAutomaticIdGeneration = false, CancellationToken cancellationToken = default(CancellationToken)) { // This call is to just run UpsertDocumentInlineAsync in a SynchronizationContext aware environment - return TaskHelper.InlineIfPossible(() => UpsertDocumentInlineAsync(documentsFeedOrDatabaseLink, document, options, disableAutomaticIdGeneration, cancellationToken), null, cancellationToken); + return TaskHelper.InlineIfPossible(() => this.UpsertDocumentInlineAsync(documentsFeedOrDatabaseLink, document, options, disableAutomaticIdGeneration, cancellationToken), null, cancellationToken); } private async Task> UpsertDocumentInlineAsync(string documentsFeedOrDatabaseLink, object document, Documents.Client.RequestOptions options, bool disableAutomaticIdGeneration, CancellationToken cancellationToken) @@ -6157,32 +6123,32 @@ string IAuthorizationTokenProvider.GetUserAuthorizationToken( } } } - - Task IAuthorizationTokenProvider.AddSystemAuthorizationHeaderAsync( - DocumentServiceRequest request, - string federationId, - string verb, - string resourceId) - { + + Task IAuthorizationTokenProvider.AddSystemAuthorizationHeaderAsync( + DocumentServiceRequest request, + string federationId, + string verb, + string resourceId) + { request.Headers[HttpConstants.HttpHeaders.XDate] = DateTime.UtcNow.ToString("r", CultureInfo.InvariantCulture); - + request.Headers[HttpConstants.HttpHeaders.Authorization] = ((IAuthorizationTokenProvider)this).GetUserAuthorizationToken( resourceId ?? request.ResourceAddress, PathsHelper.GetResourcePath(request.ResourceType), verb, request.Headers, request.RequestAuthorizationTokenType, - payload: out _); - - return Task.FromResult(0); + payload: out _); + + return Task.FromResult(0); } #endregion #region Core Implementation internal Task CreateAsync( - DocumentServiceRequest request, - IDocumentClientRetryPolicy retryPolicy, + DocumentServiceRequest request, + IDocumentClientRetryPolicy retryPolicy, CancellationToken cancellationToken = default(CancellationToken)) { if (request == null) @@ -6190,7 +6156,7 @@ internal Task CreateAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); } internal Task UpdateAsync( @@ -6203,7 +6169,7 @@ internal Task UpdateAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Put, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Put, request, retryPolicy, cancellationToken); } internal Task ReadAsync( @@ -6216,7 +6182,7 @@ internal Task ReadAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Get, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Get, request, retryPolicy, cancellationToken); } internal Task ReadFeedAsync( @@ -6229,7 +6195,7 @@ internal Task ReadFeedAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Get, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Get, request, retryPolicy, cancellationToken); } internal Task DeleteAsync( @@ -6242,7 +6208,7 @@ internal Task DeleteAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Delete, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Delete, request, retryPolicy, cancellationToken); } internal Task ExecuteProcedureAsync( @@ -6255,7 +6221,7 @@ internal Task ExecuteProcedureAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); } internal Task ExecuteQueryAsync( @@ -6268,7 +6234,7 @@ internal Task ExecuteQueryAsync( throw new ArgumentNullException("request"); } - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); } internal Task UpsertAsync( @@ -6282,7 +6248,7 @@ internal Task UpsertAsync( } request.Headers[HttpConstants.HttpHeaders.IsUpsert] = bool.TrueString; - return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); + return this.ProcessRequestAsync(HttpConstants.HttpMethods.Post, request, retryPolicy, cancellationToken); } #endregion @@ -6314,7 +6280,7 @@ Task IDocumentClientInternal.GetDatabaseAccountInternalAsync( private async Task GetDatabaseAccountPrivateAsync(Uri serviceEndpoint, CancellationToken cancellationToken = default(CancellationToken)) { await this.EnsureValidClientAsync(); - GatewayStoreModel gatewayModel = this.gatewayStoreModel as GatewayStoreModel; + GatewayStoreModel gatewayModel = this.GatewayStoreModel as GatewayStoreModel; if (gatewayModel != null) { using (HttpRequestMessage request = new HttpRequestMessage()) @@ -6347,7 +6313,7 @@ Task IDocumentClientInternal.GetDatabaseAccountInternalAsync( AccountProperties databaseAccount = await gatewayModel.GetDatabaseAccountAsync(request); - this.useMultipleWriteLocations = this.connectionPolicy.UseMultipleWriteLocations && databaseAccount.EnableMultipleWriteLocations; + this.UseMultipleWriteLocations = this.ConnectionPolicy.UseMultipleWriteLocations && databaseAccount.EnableMultipleWriteLocations; return databaseAccount; } @@ -6370,7 +6336,7 @@ internal IStoreModel GetStoreProxy(DocumentServiceRequest request) // we return the Gateway store model if (request.UseGatewayMode) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } ResourceType resourceType = request.ResourceType; @@ -6380,7 +6346,7 @@ internal IStoreModel GetStoreProxy(DocumentServiceRequest request) (resourceType.IsScript() && operationType != OperationType.ExecuteJavaScript) || resourceType == ResourceType.PartitionKeyRange) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } if (operationType == OperationType.Create @@ -6391,11 +6357,11 @@ internal IStoreModel GetStoreProxy(DocumentServiceRequest request) resourceType == ResourceType.Collection || resourceType == ResourceType.Permission) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } else { - return this.storeModel; + return this.StoreModel; } } else if (operationType == OperationType.Delete) @@ -6404,45 +6370,45 @@ internal IStoreModel GetStoreProxy(DocumentServiceRequest request) resourceType == ResourceType.User || resourceType == ResourceType.Collection) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } else { - return this.storeModel; + return this.StoreModel; } } else if (operationType == OperationType.Replace) { if (resourceType == ResourceType.Collection) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } else { - return this.storeModel; + return this.StoreModel; } } else if (operationType == OperationType.Read) { if (resourceType == ResourceType.Collection) { - return this.gatewayStoreModel; + return this.GatewayStoreModel; } else { - return this.storeModel; + return this.StoreModel; } } else { - return this.storeModel; + return this.StoreModel; } - } - + } + /// /// The preferred link used in replace operation in SDK. /// - private string GetLinkForRouting(Resource resource) + private string GetLinkForRouting(Documents.Resource resource) { // we currently prefer the selflink return resource.SelfLink ?? resource.AltLink; @@ -6483,10 +6449,10 @@ private void InitializeDirectConnectivity(IStoreClientFactory storeClientFactory else { StoreClientFactory newClientFactory = new StoreClientFactory( - this.connectionPolicy.ConnectionProtocol, - (int)this.connectionPolicy.RequestTimeout.TotalSeconds, + this.ConnectionPolicy.ConnectionProtocol, + (int)this.ConnectionPolicy.RequestTimeout.TotalSeconds, this.maxConcurrentConnectionOpenRequests, - this.connectionPolicy.UserAgentContainer, + this.ConnectionPolicy.UserAgentContainer, this.eventSource, null, this.openConnectionTimeoutInSeconds, @@ -6498,7 +6464,7 @@ private void InitializeDirectConnectivity(IStoreClientFactory storeClientFactory receiveHangDetectionTimeSeconds: this.rntbdReceiveHangDetectionTimeSeconds, sendHangDetectionTimeSeconds: this.rntbdSendHangDetectionTimeSeconds, enableCpuMonitor: this.enableCpuMonitor, - retryWithConfiguration: this.connectionPolicy.RetryOptions?.GetRetryWithConfiguration(), + retryWithConfiguration: this.ConnectionPolicy.RetryOptions?.GetRetryWithConfiguration(), rntbdPortReuseMode: (Documents.PortReuseMode)this.rntbdPortReuseMode); if (this.transportClientHandlerFactory != null) @@ -6511,15 +6477,15 @@ private void InitializeDirectConnectivity(IStoreClientFactory storeClientFactory } this.AddressResolver = new GlobalAddressResolver( - this.globalEndpointManager, - this.connectionPolicy.ConnectionProtocol, + this.GlobalEndpointManager, + this.ConnectionPolicy.ConnectionProtocol, this, this.collectionCache, this.partitionKeyRangeCache, - this.connectionPolicy.UserAgentContainer, + this.ConnectionPolicy.UserAgentContainer, this.accountServiceConfiguration, this.httpMessageHandler, - this.connectionPolicy, + this.ConnectionPolicy, this.ApiType); this.CreateStoreModel(subscribeRntbdStatus: true); @@ -6536,9 +6502,9 @@ private void CreateStoreModel(bool subscribeRntbdStatus) this.accountServiceConfiguration, this, true, - this.connectionPolicy.EnableReadRequestsFallback ?? (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.BoundedStaleness), + this.ConnectionPolicy.EnableReadRequestsFallback ?? (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.BoundedStaleness), !this.enableRntbdChannel, - this.useMultipleWriteLocations && (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.Strong), + this.UseMultipleWriteLocations && (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.Strong), true); if (subscribeRntbdStatus) @@ -6548,7 +6514,7 @@ private void CreateStoreModel(bool subscribeRntbdStatus) storeClient.SerializerSettings = this.serializerSettings; - this.storeModel = new ServerStoreModel(storeClient, this.sendingRequest, this.receivedResponse); + this.StoreModel = new ServerStoreModel(storeClient, this.sendingRequest, this.receivedResponse); } private void DisableRntbdChannel() @@ -6565,7 +6531,7 @@ private async Task InitializeGatewayConfigurationReaderAsync() this.authKeyHashFunction, this.hasAuthKeyResourceToken, this.authKeyResourceToken, - this.connectionPolicy, + this.ConnectionPolicy, this.ApiType, this.httpMessageHandler); @@ -6573,9 +6539,9 @@ private async Task InitializeGatewayConfigurationReaderAsync() await this.accountServiceConfiguration.InitializeAsync(); AccountProperties accountProperties = this.accountServiceConfiguration.AccountProperties; - this.useMultipleWriteLocations = this.connectionPolicy.UseMultipleWriteLocations && accountProperties.EnableMultipleWriteLocations; + this.UseMultipleWriteLocations = this.ConnectionPolicy.UseMultipleWriteLocations && accountProperties.EnableMultipleWriteLocations; - await this.globalEndpointManager.RefreshLocationAsync(accountProperties); + await this.GlobalEndpointManager.RefreshLocationAsync(accountProperties); } internal void CaptureSessionToken(DocumentServiceRequest request, DocumentServiceResponse response) @@ -6609,13 +6575,13 @@ internal DocumentServiceRequest CreateDocumentServiceRequest( } } - internal void ValidateResource(Resource resource) + internal void ValidateResource(Documents.Resource resource) { this.ValidateResource(resource.Id); } internal void ValidateResource(string resourceId) - { + { if (!string.IsNullOrEmpty(resourceId)) { int match = resourceId.IndexOfAny(new char[] { '/', '\\', '?', '#' }); @@ -6702,7 +6668,7 @@ private INameValueCollection GetRequestHeaders(Documents.Client.RequestOptions o INameValueCollection headers = new DictionaryNameValueCollection(); - if (this.useMultipleWriteLocations) + if (this.UseMultipleWriteLocations) { headers.Set(HttpConstants.HttpHeaders.AllowTentativeWrites, bool.TrueString); } @@ -6885,7 +6851,7 @@ public ResetSessionTokenRetryPolicyFactory(ISessionContainer sessionContainer, C public IDocumentClientRetryPolicy GetRequestPolicy() { - return new RenameCollectionAwareClientRetryPolicy(this.sessionContainer, this.collectionCache, retryPolicy.GetRequestPolicy()); + return new RenameCollectionAwareClientRetryPolicy(this.sessionContainer, this.collectionCache, this.retryPolicy.GetRequestPolicy()); } } @@ -6899,7 +6865,7 @@ public HttpRequestMessageHandler(EventHandler sendingRe this.sendingRequest = sendingRequest; this.receivedResponse = receivedResponse; - InnerHandler = innerHandler ?? new HttpClientHandler(); + this.InnerHandler = innerHandler ?? new HttpClientHandler(); } protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) diff --git a/Microsoft.Azure.Cosmos/src/FeedResponseBinder.cs b/Microsoft.Azure.Cosmos/src/FeedResponseBinder.cs index c3c86a5698..d6aff24b33 100644 --- a/Microsoft.Azure.Cosmos/src/FeedResponseBinder.cs +++ b/Microsoft.Azure.Cosmos/src/FeedResponseBinder.cs @@ -6,8 +6,6 @@ namespace Microsoft.Azure.Cosmos using System; using System.Collections.Generic; using System.Linq; - using System.Runtime.InteropServices; - using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Documents; @@ -84,7 +82,7 @@ public static DocumentFeedResponse ConvertCosmosElementFeed( // If the resource type is an offer and the requested type is either a Offer or OfferV2 or dynamic // create a OfferV2 object and cast it to T. This is a temporary fix until offers is moved to v3 API. if (resourceType == ResourceType.Offer && - (typeof(T).IsSubclassOf(typeof(Resource)) || typeof(T) == typeof(object))) + (typeof(T).IsSubclassOf(typeof(Documents.Resource)) || typeof(T) == typeof(object))) { typedResults = JsonConvert.DeserializeObject>(jsonText, settings).Cast(); } diff --git a/Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs b/Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs index 1aea6c6ae1..be955ce9aa 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs @@ -166,7 +166,7 @@ internal static async Task CreateDocumentClientExceptio if (string.Equals(responseMessage.Content?.Headers?.ContentType?.MediaType, "application/json", StringComparison.OrdinalIgnoreCase)) { Stream readStream = await responseMessage.Content.ReadAsStreamAsync(); - Error error = Resource.LoadFrom(readStream); + Error error = Documents.Resource.LoadFrom(readStream); return new DocumentClientException( error, responseMessage.Headers, @@ -324,19 +324,21 @@ private async Task InvokeClientAsync( DateTime sendTimeUtc = DateTime.UtcNow; Guid localGuid = Guid.NewGuid(); // For correlating HttpRequest and HttpResponse Traces + Guid requestedActivityId = Trace.CorrelationManager.ActivityId; this.eventSource.Request( - Guid.Empty, + requestedActivityId, localGuid, requestMessage.RequestUri.ToString(), resourceType.ToResourceTypeString(), requestMessage.Headers); + TimeSpan durationTimeSpan; try { HttpResponseMessage responseMessage = await this.httpClient.SendAsync(requestMessage, cancellationToken); DateTime receivedTimeUtc = DateTime.UtcNow; - double durationInMilliSeconds = (receivedTimeUtc - sendTimeUtc).TotalMilliseconds; + durationTimeSpan = receivedTimeUtc - sendTimeUtc; IEnumerable headerValues; Guid activityId = Guid.Empty; @@ -350,7 +352,7 @@ private async Task InvokeClientAsync( activityId, localGuid, (short)responseMessage.StatusCode, - durationInMilliSeconds, + durationTimeSpan.TotalMilliseconds, responseMessage.Headers); return responseMessage; @@ -360,7 +362,23 @@ private async Task InvokeClientAsync( if (!cancellationToken.IsCancellationRequested) { // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out) - throw new RequestTimeoutException(ex, requestMessage.RequestUri); + durationTimeSpan = DateTime.UtcNow - sendTimeUtc; + string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};"; + throw new RequestTimeoutException(message, ex, requestMessage.RequestUri); + } + else + { + throw; + } + } + catch (OperationCanceledException ex) + { + if (!cancellationToken.IsCancellationRequested) + { + // throw timeout if the cancellationToken is not canceled (i.e. httpClient timed out) + durationTimeSpan = DateTime.UtcNow - sendTimeUtc; + string message = $"GatewayStoreClient Request Timeout. Start Time:{sendTimeUtc}; Total Duration:{durationTimeSpan}; Http Client Timeout:{this.httpClient.Timeout}; Activity id: {requestedActivityId}; Inner Message: {ex.Message};"; + throw new RequestTimeoutException(message, ex, requestMessage.RequestUri); } else { diff --git a/Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs b/Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs index 4895059f87..3e68293741 100644 --- a/Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs +++ b/Microsoft.Azure.Cosmos/src/Handler/ResponseMessage.cs @@ -220,7 +220,7 @@ private void EnsureErrorMessage() { try { - Error error = Resource.LoadFrom(this.content); + Error error = Documents.Resource.LoadFrom(this.content); if (error != null) { // Error format is not consistent across modes diff --git a/Microsoft.Azure.Cosmos/src/IDocumentClient.cs b/Microsoft.Azure.Cosmos/src/IDocumentClient.cs index ae75b79da4..be433b13dc 100644 --- a/Microsoft.Azure.Cosmos/src/IDocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/IDocumentClient.cs @@ -1269,7 +1269,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Database if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Database if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}" only /// the values within the {} change depending on which method you wish to use to address the resource. @@ -1364,7 +1364,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the DocumentCollection if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the DocumentCollection if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}/colls/{coll identifier}" only /// the values within the {} change depending on which method you wish to use to address the resource. @@ -1462,7 +1462,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Document if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Document if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "dbs/{db identifier}/colls/{coll identifier}/docs/{doc identifier}" only /// the values within the {} change depending on which method you wish to use to address the resource. @@ -1562,7 +1562,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Document if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Document if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "dbs/{db identifier}/colls/{coll identifier}/docs/{doc identifier}" only /// the values within the {} change depending on which method you wish to use to address the resource. @@ -1661,7 +1661,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Stored Procedure if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Stored Procedure if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}/colls/{coll identifier}/sprocs/{sproc identifier}" /// only the values within the {...} change depending on which method you wish to use to address the resource. @@ -1759,7 +1759,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Trigger if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Trigger if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}/colls/{coll identifier}/triggers/{trigger identifier}" /// only the values within the {...} change depending on which method you wish to use to address the resource. @@ -1857,7 +1857,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the User Defined Function if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the User Defined Function if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}/colls/{coll identifier}/udfs/{udf identifier}" /// only the values within the {...} change depending on which method you wish to use to address the resource. @@ -1955,7 +1955,7 @@ Task> ReplaceUserDefinedFunctionAsync( /// /// /// The example shown uses ID-based links, where the link is composed of the ID properties used when the resources were created. - /// You can still use the property of the Conflict if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). + /// You can still use the property of the Conflict if you prefer. A self-link is a URI for a resource that is made up of Resource Identifiers (or the _rid properties). /// ID-based links and SelfLink will both work. /// The format for is always "/dbs/{db identifier}/colls/{collectioon identifier}/conflicts/{conflict identifier}" /// only the values within the {...} change depending on which method you wish to use to address the resource. diff --git a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj index 66dd2e1930..b6f5211226 100644 --- a/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj +++ b/Microsoft.Azure.Cosmos/src/Microsoft.Azure.Cosmos.csproj @@ -109,7 +109,7 @@ $(DefineConstants);PREVIEW $(DefineConstants);SignAssembly - DelaySignKeys + $(DefineConstants);DelaySignKeys $(DefineConstants);DOCDBCLIENT;COSMOSCLIENT;NETSTANDARD20 7.3 diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/AverageAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/AverageAggregator.cs index 1363ee9734..0f3d3eae01 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/AverageAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/AverageAggregator.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Query.Aggregation { using System; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; /// @@ -58,7 +59,8 @@ public static TryCatch TryCreate(string continuationToken) { if (!AverageInfo.TryParse(continuationToken, out averageInfo)) { - return TryCatch.FromException(new ArgumentException($"Invalid continuation token: {continuationToken}")); + return TryCatch.FromException( + new MalformedContinuationTokenException($"Invalid continuation token: {continuationToken}")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/CountAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/CountAggregator.cs index c35271187e..ffc9689436 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/CountAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/CountAggregator.cs @@ -5,8 +5,8 @@ namespace Microsoft.Azure.Cosmos.Query.Aggregation { using System; using System.Globalization; - using System.Net; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; /// @@ -74,7 +74,7 @@ public static TryCatch TryCreate(string continuationToken) if (!long.TryParse(continuationToken, out partialCount)) { return TryCatch.FromException( - new Exception($@"Invalid count continuation token: ""{continuationToken}"".")); + new MalformedContinuationTokenException($@"Invalid count continuation token: ""{continuationToken}"".")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/MinMaxAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/MinMaxAggregator.cs index 1e846fa8db..a04b4abdca 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/MinMaxAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/MinMaxAggregator.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Query.Aggregation { using System; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; /// @@ -194,7 +195,7 @@ private static TryCatch TryCreate(bool isMinAggregation, string con if (!CosmosElement.TryParse(continuationToken, out globalMinMax)) { return TryCatch.FromException( - new Exception($"Malformed continuation token: {continuationToken}")); + new MalformedContinuationTokenException($"Malformed continuation token: {continuationToken}")); } } } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SingleGroupAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SingleGroupAggregator.cs index fe8359d00c..6080da3cc8 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SingleGroupAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SingleGroupAggregator.cs @@ -6,10 +6,10 @@ namespace Microsoft.Azure.Cosmos.Query using System; using System.Collections.Generic; using System.Linq; - using System.Text; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Query.Aggregation; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; /// @@ -188,7 +188,8 @@ public static TryCatch TryCreate( if (!CosmosElement.TryParse(continuationToken, out aliasToContinuationToken)) { return TryCatch.FromException( - new Exception($"{nameof(SelectListAggregateValues)} continuation token is malformed: {continuationToken}.")); + new MalformedContinuationTokenException( + $"{nameof(SelectListAggregateValues)} continuation token is malformed: {continuationToken}.")); } } else @@ -207,7 +208,8 @@ public static TryCatch TryCreate( if (!(aliasToContinuationToken[alias] is CosmosString parsedAliasContinuationToken)) { return TryCatch.FromException( - new Exception($"{nameof(SelectListAggregateValues)} continuation token is malformed: {continuationToken}.")); + new MalformedContinuationTokenException( + $"{nameof(SelectListAggregateValues)} continuation token is malformed: {continuationToken}.")); } aliasContinuationToken = parsedAliasContinuationToken.Value; @@ -406,7 +408,7 @@ public static TryCatch TryCreate(string continuationToken) out CosmosObject rawContinuationToken)) { return TryCatch.FromException( - new ArgumentException($"Invalid {nameof(ScalarAggregateValue)}: {continuationToken}")); + new MalformedContinuationTokenException($"Invalid {nameof(ScalarAggregateValue)}: {continuationToken}")); } if (!rawContinuationToken.TryGetValue( @@ -414,7 +416,7 @@ public static TryCatch TryCreate(string continuationToken) out CosmosBoolean rawInitialized)) { return TryCatch.FromException( - new ArgumentException($"Invalid {nameof(ScalarAggregateValue)}: {continuationToken}")); + new MalformedContinuationTokenException($"Invalid {nameof(ScalarAggregateValue)}: {continuationToken}")); } if (!rawContinuationToken.TryGetValue(nameof(ScalarAggregateValue.value), out value)) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SumAggregator.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SumAggregator.cs index 661a8d0ab4..60ce68af00 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SumAggregator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Aggregation/SumAggregator.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Query.Aggregation using System; using System.Globalization; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; /// @@ -81,7 +82,7 @@ public static TryCatch TryCreate(string continuationToken) if (!double.TryParse(continuationToken, out partialSum)) { return TryCatch.FromException( - new Exception($"Malformed {nameof(SumAggregator)} continuation token: {continuationToken}")); + new MalformedContinuationTokenException($"Malformed {nameof(SumAggregator)} continuation token: {continuationToken}")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/AsyncLazy.cs b/Microsoft.Azure.Cosmos/src/Query/Core/AsyncLazy.cs new file mode 100644 index 0000000000..c8214fb7d7 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/AsyncLazy.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + internal sealed class AsyncLazy + { + private readonly Func> valueFactory; + private T value; + + public AsyncLazy(Func> valueFactory) + { + if (valueFactory == null) + { + throw new ArgumentNullException(nameof(valueFactory)); + } + + this.valueFactory = valueFactory; + } + + public bool ValueInitialized { get; private set; } + + public async Task GetValueAsync(CancellationToken cancellationToken) + { + // Note that this class is not thread safe. + // if the valueFactory has side effects than this will have issues. + cancellationToken.ThrowIfCancellationRequested(); + if (!this.ValueInitialized) + { + this.value = await this.valueFactory(cancellationToken); + this.ValueInitialized = true; + } + + return this.value; + } + + public T Result + { + get + { + if (!this.ValueInitialized) + { + throw new InvalidOperationException("Can not retrieve value before initialization."); + } + + return this.value; + } + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Collections/PriorityQueue.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Collections/PriorityQueue.cs index c4809b602e..ff10e272bf 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Collections/PriorityQueue.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Collections/PriorityQueue.cs @@ -15,9 +15,7 @@ namespace Microsoft.Azure.Cosmos.Collections.Generic internal sealed class PriorityQueue : IProducerConsumerCollection { private const int DefaultInitialCapacity = 17; - private readonly bool isSynchronized; private readonly List queue; - private readonly IComparer comparer; public PriorityQueue(bool isSynchronized = false) : this(DefaultInitialCapacity, isSynchronized) @@ -62,9 +60,9 @@ private PriorityQueue(List queue, IComparer comparer, bool isSynchronized) throw new ArgumentNullException("comparer"); } - this.isSynchronized = isSynchronized; + this.IsSynchronized = isSynchronized; this.queue = queue; - this.comparer = comparer; + this.Comparer = comparer; } public int Count @@ -75,18 +73,9 @@ public int Count } } - public IComparer Comparer - { - get - { - return this.comparer; - } - } + public IComparer Comparer { get; } - public bool IsSynchronized - { - get { return this.isSynchronized; } - } + public bool IsSynchronized { get; } public object SyncRoot { @@ -95,7 +84,7 @@ public object SyncRoot public void CopyTo(T[] array, int index) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -115,7 +104,7 @@ public bool TryAdd(T item) public bool TryTake(out T item) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -128,7 +117,7 @@ public bool TryTake(out T item) public bool TryPeek(out T item) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -146,7 +135,7 @@ public void CopyTo(Array array, int index) public void Clear() { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -160,7 +149,7 @@ public void Clear() public bool Contains(T item) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -173,7 +162,7 @@ public bool Contains(T item) public T Dequeue() { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -186,7 +175,7 @@ public T Dequeue() public void Enqueue(T item) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -200,7 +189,7 @@ public void Enqueue(T item) public void EnqueueRange(IEnumerable items) { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -214,7 +203,7 @@ public void EnqueueRange(IEnumerable items) public T Peek() { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -227,7 +216,7 @@ public T Peek() public T[] ToArray() { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -240,7 +229,7 @@ public T[] ToArray() public IEnumerator GetEnumerator() { - if (this.isSynchronized) + if (this.IsSynchronized) { lock (this.SyncRoot) { @@ -377,7 +366,7 @@ private void UpHeap(int itemIndex) T item = this.queue[itemIndex]; - if (this.comparer.Compare(item, parent) >= 0) + if (this.Comparer.Compare(item, parent) >= 0) { break; } @@ -395,13 +384,13 @@ private int GetSmallestChildIndex(int parentIndex) int smallestChildIndex = parentIndex; if (leftChildIndex < this.queue.Count - && this.comparer.Compare(this.queue[smallestChildIndex], this.queue[leftChildIndex]) > 0) + && this.Comparer.Compare(this.queue[smallestChildIndex], this.queue[leftChildIndex]) > 0) { smallestChildIndex = leftChildIndex; } if (rightChildIndex < this.queue.Count - && this.comparer.Compare(this.queue[smallestChildIndex], this.queue[rightChildIndex]) > 0) + && this.Comparer.Compare(this.queue[smallestChildIndex], this.queue[rightChildIndex]) > 0) { smallestChildIndex = rightChildIndex; } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.OrderedDistinctMap.cs b/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.OrderedDistinctMap.cs index ba21de9c92..1d5b49d72e 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.OrderedDistinctMap.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.OrderedDistinctMap.cs @@ -79,7 +79,8 @@ public static TryCatch TryCreate(string continuationToken) if (!UInt128.TryParse(continuationToken, out lastHash)) { return TryCatch.FromException( - new Exception($"Malformed {nameof(OrderedDistinctMap)} continuation token: {continuationToken}.")); + new MalformedContinuationTokenException( + $"Malformed {nameof(OrderedDistinctMap)} continuation token: {continuationToken}.")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.UnorderedDistinctMap.cs b/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.UnorderedDistinctMap.cs index a21e61d7b3..97be3cff19 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.UnorderedDistinctMap.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/DistinctMap.UnorderedDistinctMap.cs @@ -450,14 +450,16 @@ public static TryCatch TryCreate(string continuationToken) if (!(cosmosElement is CosmosObject hashDictionary)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } // Numbers if (!hashDictionary.TryGetValue(UnorderdDistinctMap.NumbersName, out CosmosArray numbersArray)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } foreach (CosmosElement rawNumber in numbersArray) @@ -465,7 +467,8 @@ public static TryCatch TryCreate(string continuationToken) if (!(rawNumber is CosmosNumber64 number)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } numbers.Add(number.GetValue()); @@ -475,7 +478,8 @@ public static TryCatch TryCreate(string continuationToken) if (!hashDictionary.TryGetValue(UnorderdDistinctMap.StringsLength4Name, out CosmosArray stringsLength4Array)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } foreach (CosmosElement rawStringLength4 in stringsLength4Array) @@ -483,7 +487,8 @@ public static TryCatch TryCreate(string continuationToken) if (!(rawStringLength4 is CosmosUInt32 stringlength4)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } stringsLength4.Add(stringlength4.GetValue()); @@ -493,7 +498,8 @@ public static TryCatch TryCreate(string continuationToken) if (!hashDictionary.TryGetValue(UnorderdDistinctMap.StringsLength8Name, out CosmosArray stringsLength8Array)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } foreach (CosmosElement rawStringLength8 in stringsLength8Array) @@ -501,7 +507,8 @@ public static TryCatch TryCreate(string continuationToken) if (!(rawStringLength8 is CosmosInt64 stringlength8)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } stringsLength8.Add((ulong)stringlength8.GetValue()); @@ -524,13 +531,15 @@ public static TryCatch TryCreate(string continuationToken) if (!(rawSimpleValues is CosmosString simpleValuesString)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } if (!Enum.TryParse(simpleValuesString.Value, out simpleValues)) { return TryCatch.FromException( - new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); + new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed.")); } } @@ -550,14 +559,16 @@ private static HashSet Parse128BitHashes(CosmosObject hashDictionary, s HashSet hashSet = new HashSet(); if (!hashDictionary.TryGetValue(propertyName, out CosmosArray array)) { - throw new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed."); + throw new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed."); } foreach (CosmosElement item in array) { if (!(item is CosmosBinary binary)) { - throw new ArgumentException($"{nameof(UnorderdDistinctMap)} continuation token was malformed."); + throw new MalformedContinuationTokenException( + $"{nameof(UnorderdDistinctMap)} continuation token was malformed."); } // Todo have this method work with span instead to avoid the allocation. diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/MalformedContinuationTokenException.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/MalformedContinuationTokenException.cs new file mode 100644 index 0000000000..a05b1eb41d --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/MalformedContinuationTokenException.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + using System; + + internal sealed class MalformedContinuationTokenException : QueryException + { + public MalformedContinuationTokenException() + : base() + { + } + + public MalformedContinuationTokenException(string message) + : base(message) + { + } + + public MalformedContinuationTokenException(string message, Exception innerException) + : base(message, innerException) + { + } + + public override TResult Accept(QueryExceptionVisitor visitor) + { + return visitor.Visit(this); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryException.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryException.cs new file mode 100644 index 0000000000..306d622d27 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryException.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + using System; + + internal abstract class QueryException : Exception + { + protected QueryException() + : base() + { + } + + protected QueryException(string message) + : base(message) + { + } + + protected QueryException(string message, Exception innerException) + : base(message, innerException) + { + } + + public abstract TResult Accept(QueryExceptionVisitor visitor); + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryExceptionVisitor.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryExceptionVisitor.cs new file mode 100644 index 0000000000..ce0435f7f8 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryExceptionVisitor.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + using Microsoft.Azure.Cosmos.Query.Core.Exceptions; + + internal abstract class QueryExceptionVisitor + { + public abstract TResult Visit(MalformedContinuationTokenException malformedContinuationTokenException); + public abstract TResult Visit(UnexpectedQueryPartitionProviderException unexpectedQueryPartitionProviderException); + public abstract TResult Visit(ExpectedQueryPartitionProviderException expectedQueryPartitionProviderException); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryPartitionProviderException.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryPartitionProviderException.cs new file mode 100644 index 0000000000..f0c43612d3 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Exceptions/QueryPartitionProviderException.cs @@ -0,0 +1,72 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.Exceptions +{ + using System; + + internal abstract class QueryPartitionProviderException : QueryException + { + protected QueryPartitionProviderException() + : base() + { + } + + protected QueryPartitionProviderException(string message) + : base(message) + { + } + + protected QueryPartitionProviderException(string message, Exception innerException) + : base(message, innerException) + { + } + } + + internal sealed class UnexpectedQueryPartitionProviderException : QueryPartitionProviderException + { + public UnexpectedQueryPartitionProviderException() + : base() + { + } + + public UnexpectedQueryPartitionProviderException(string message) + : base(message) + { + } + + public UnexpectedQueryPartitionProviderException(string message, Exception innerException) + : base(message, innerException) + { + } + + public override TResult Accept(QueryExceptionVisitor visitor) + { + return visitor.Visit(this); + } + } + + internal sealed class ExpectedQueryPartitionProviderException : QueryPartitionProviderException + { + public ExpectedQueryPartitionProviderException() + : base() + { + } + + public ExpectedQueryPartitionProviderException(string message) + : base(message) + { + } + + public ExpectedQueryPartitionProviderException(string message, Exception innerException) + : base(message, innerException) + { + } + + public override TResult Accept(QueryExceptionVisitor visitor) + { + return visitor.Visit(this); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/AggregateDocumentQueryExecutionComponent.Compute.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/AggregateDocumentQueryExecutionComponent.Compute.cs index 3384cb5504..1f9aad752c 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/AggregateDocumentQueryExecutionComponent.Compute.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/AggregateDocumentQueryExecutionComponent.Compute.cs @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionComponent using System.Threading.Tasks; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query.Core.Monads; + using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; internal abstract partial class AggregateDocumentQueryExecutionComponent : DocumentQueryExecutionComponentBase { @@ -41,7 +42,7 @@ public static async Task> TryCreateAs if (!AggregateContinuationToken.TryParse(requestContinuation, out AggregateContinuationToken aggregateContinuationToken)) { return TryCatch.FromException( - new Exception($"Malfomed {nameof(AggregateContinuationToken)}: '{requestContinuation}'")); + new MalformedContinuationTokenException($"Malfomed {nameof(AggregateContinuationToken)}: '{requestContinuation}'")); } sourceContinuationToken = aggregateContinuationToken.SourceContinuationToken; diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Client.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Client.cs index ea62b8fb15..52601a5a0a 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Client.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Client.cs @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionComponent using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query.Core.Monads; + using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; internal abstract partial class DistinctDocumentQueryExecutionComponent : DocumentQueryExecutionComponentBase { @@ -48,7 +49,7 @@ public static async Task> TryCreateAs if (!DistinctContinuationToken.TryParse(requestContinuation, out distinctContinuationToken)) { return TryCatch.FromException( - new Exception($"Invalid {nameof(DistinctContinuationToken)}: {requestContinuation}")); + new MalformedContinuationTokenException($"Invalid {nameof(DistinctContinuationToken)}: {requestContinuation}")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Compute.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Compute.cs index 48b6fa9679..5003b94da0 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Compute.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/DistinctDocumentQueryExecutionComponent.Compute.cs @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionComponent using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query.Core.Monads; + using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; internal abstract partial class DistinctDocumentQueryExecutionComponent : DocumentQueryExecutionComponentBase { @@ -40,7 +41,7 @@ public static async Task> TryCreateAs if (!DistinctContinuationToken.TryParse(requestContinuation, out distinctContinuationToken)) { return TryCatch.FromException( - new Exception($"Invalid {nameof(DistinctContinuationToken)}: {requestContinuation}")); + new MalformedContinuationTokenException($"Invalid {nameof(DistinctContinuationToken)}: {requestContinuation}")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.Compute.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.Compute.cs index e23ae95447..f58c35cd11 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.Compute.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.Compute.cs @@ -42,7 +42,7 @@ public static async Task> TryCreateAs if (!GroupByContinuationToken.TryParse(requestContinuation, out groupByContinuationToken)) { return TryCatch.FromException( - new Exception($"Invalid {nameof(GroupByContinuationToken)}: '{requestContinuation}'")); + new MalformedContinuationTokenException($"Invalid {nameof(GroupByContinuationToken)}: '{requestContinuation}'")); } } else diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.cs index d6cf3489ad..05b92d3ab4 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/GroupByDocumentQueryExecutionComponent.cs @@ -284,7 +284,8 @@ public static TryCatch TryCreateFromContinuationToken( groupingTableContinuationToken, out CosmosObject parsedGroupingTableContinuations)) { - return TryCatch.FromException(new Exception($"Invalid GroupingTableContinuationToken")); + return TryCatch.FromException( + new MalformedContinuationTokenException($"Invalid GroupingTableContinuationToken")); } foreach (KeyValuePair kvp in parsedGroupingTableContinuations) @@ -294,12 +295,14 @@ public static TryCatch TryCreateFromContinuationToken( if (!UInt128.TryParse(key, out UInt128 groupByKey)) { - return TryCatch.FromException(new Exception($"Invalid GroupingTableContinuationToken")); + return TryCatch.FromException( + new MalformedContinuationTokenException($"Invalid GroupingTableContinuationToken")); } if (!(value is CosmosString singleGroupAggregatorContinuationToken)) { - return TryCatch.FromException(new Exception($"Invalid GroupingTableContinuationToken")); + return TryCatch.FromException( + new MalformedContinuationTokenException($"Invalid GroupingTableContinuationToken")); } TryCatch tryCreateSingleGroupAggregator = SingleGroupAggregator.TryCreate( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/SkipDocumentQueryExecutionComponent.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/SkipDocumentQueryExecutionComponent.cs index 10dfa5fd53..618ab7cec2 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/SkipDocumentQueryExecutionComponent.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/SkipDocumentQueryExecutionComponent.cs @@ -40,7 +40,7 @@ public static async Task> TryCreateAs if (!OffsetContinuationToken.TryParse(continuationToken, out offsetContinuationToken)) { return TryCatch.FromException( - new Exception($"Invalid {nameof(SkipDocumentQueryExecutionComponent)}: {continuationToken}.")); + new MalformedContinuationTokenException($"Invalid {nameof(SkipDocumentQueryExecutionComponent)}: {continuationToken}.")); } } else @@ -51,7 +51,7 @@ public static async Task> TryCreateAs if (offsetContinuationToken.Offset > offsetCount) { return TryCatch.FromException( - new Exception("offset count in continuation token can not be greater than the offsetcount in the query.")); + new MalformedContinuationTokenException("offset count in continuation token can not be greater than the offsetcount in the query.")); } return (await tryCreateSourceAsync(offsetContinuationToken.SourceToken)) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs index 6fdedd31b4..adc633f9b3 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionComponent/TakeDocumentQueryExecutionComponent.cs @@ -45,7 +45,7 @@ public static async Task> TryCreateLi if (!LimitContinuationToken.TryParse(continuationToken, out limitContinuationToken)) { return TryCatch.FromException( - new ArgumentException($"Malformed {nameof(LimitContinuationToken)}: {continuationToken}.")); + new MalformedContinuationTokenException($"Malformed {nameof(LimitContinuationToken)}: {continuationToken}.")); } } else @@ -56,13 +56,13 @@ public static async Task> TryCreateLi if (limitContinuationToken.Limit > limitCount) { return TryCatch.FromException( - new ArgumentOutOfRangeException($"{nameof(LimitContinuationToken.Limit)} in {nameof(LimitContinuationToken)}: {continuationToken}: {limitContinuationToken.Limit} can not be greater than the limit count in the query: {limitCount}.")); + new MalformedContinuationTokenException($"{nameof(LimitContinuationToken.Limit)} in {nameof(LimitContinuationToken)}: {continuationToken}: {limitContinuationToken.Limit} can not be greater than the limit count in the query: {limitCount}.")); } if (limitCount < 0) { return TryCatch.FromException( - new ArgumentException($"{nameof(limitCount)}: {limitCount} must be a non negative number.")); + new MalformedContinuationTokenException($"{nameof(limitCount)}: {limitCount} must be a non negative number.")); } return (await tryCreateSourceAsync(limitContinuationToken.SourceToken)) @@ -88,7 +88,7 @@ public static async Task> TryCreateTo if (!TopContinuationToken.TryParse(continuationToken, out topContinuationToken)) { return TryCatch.FromException( - new ArgumentException($"Malformed {nameof(LimitContinuationToken)}: {continuationToken}.")); + new MalformedContinuationTokenException($"Malformed {nameof(LimitContinuationToken)}: {continuationToken}.")); } } else @@ -99,13 +99,13 @@ public static async Task> TryCreateTo if (topContinuationToken.Top > topCount) { return TryCatch.FromException( - new ArgumentOutOfRangeException($"{nameof(TopContinuationToken.Top)} in {nameof(TopContinuationToken)}: {continuationToken}: {topContinuationToken.Top} can not be greater than the top count in the query: {topCount}.")); + new MalformedContinuationTokenException($"{nameof(TopContinuationToken.Top)} in {nameof(TopContinuationToken)}: {continuationToken}: {topContinuationToken.Top} can not be greater than the top count in the query: {topCount}.")); } if (topCount < 0) { return TryCatch.FromException( - new ArgumentException($"{nameof(topCount)}: {topCount} must be a non negative number.")); + new MalformedContinuationTokenException($"{nameof(topCount)}: {topCount} must be a non negative number.")); } return (await tryCreateSourceAsync(topContinuationToken.SourceToken)) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CatchAllCosmosQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CatchAllCosmosQueryExecutionContext.cs new file mode 100644 index 0000000000..3e5ddf1592 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CatchAllCosmosQueryExecutionContext.cs @@ -0,0 +1,67 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionContext +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + internal sealed class CatchAllCosmosQueryExecutionContext : CosmosQueryExecutionContext + { + private readonly CosmosQueryExecutionContext cosmosQueryExecutionContext; + private bool hitException; + + public CatchAllCosmosQueryExecutionContext( + CosmosQueryExecutionContext cosmosQueryExecutionContext) + { + if (cosmosQueryExecutionContext == null) + { + throw new ArgumentNullException(nameof(cosmosQueryExecutionContext)); + } + + this.cosmosQueryExecutionContext = cosmosQueryExecutionContext; + } + + public override bool IsDone => this.hitException || this.cosmosQueryExecutionContext.IsDone; + + public override void Dispose() + { + this.cosmosQueryExecutionContext.Dispose(); + } + + public override async Task ExecuteNextAsync(CancellationToken cancellationToken) + { + if (this.IsDone) + { + throw new InvalidOperationException( + $"Can not {nameof(ExecuteNextAsync)} from a {nameof(CosmosQueryExecutionContext)} where {nameof(this.IsDone)}."); + } + + cancellationToken.ThrowIfCancellationRequested(); + + QueryResponseCore queryResponseCore; + try + { + queryResponseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); + } + catch (Exception ex) + { + queryResponseCore = QueryResponseFactory.CreateFromException(ex); + } + + if (!queryResponseCore.IsSuccess) + { + this.hitException = true; + } + + return queryResponseCore; + } + + public override bool TryGetContinuationToken(out string continuationToken) + { + return this.cosmosQueryExecutionContext.TryGetContinuationToken(out continuationToken); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosCrossPartitionQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosCrossPartitionQueryExecutionContext.cs index e6acfa2fd4..1533ea00ee 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosCrossPartitionQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosCrossPartitionQueryExecutionContext.cs @@ -6,14 +6,13 @@ namespace Microsoft.Azure.Cosmos.Query using System; using System.Collections.Concurrent; using System.Collections.Generic; - using System.Globalization; using System.Linq; - using System.Net; using System.Threading; using System.Threading.Tasks; using Collections.Generic; using Core.ExecutionComponent; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; using ParallelQuery; using PartitionKeyRange = Documents.PartitionKeyRange; @@ -35,6 +34,11 @@ internal abstract class CosmosCrossPartitionQueryExecutionContext : CosmosQueryE /// private const double DynamicPageSizeAdjustmentFactor = 1.6; + /// + /// Request Charge Tracker used to atomically add request charges (doubles). + /// + protected readonly RequestChargeTracker requestChargeTracker; + /// /// Priority Queue of ItemProducerTrees that make a forest that can be iterated on. /// @@ -54,12 +58,6 @@ internal abstract class CosmosCrossPartitionQueryExecutionContext : CosmosQueryE /// The equality comparer used to determine whether a document producer needs it's continuation token to be part of the composite continuation token. /// private readonly IEqualityComparer equalityComparer; - - /// - /// Request Charge Tracker used to atomically add request charges (doubles). - /// - private readonly RequestChargeTracker requestChargeTracker; - /// /// The actual max page size after all the optimizations have been made it in the create document query execution context layer. /// @@ -70,6 +68,12 @@ internal abstract class CosmosCrossPartitionQueryExecutionContext : CosmosQueryE /// private readonly long actualMaxBufferedItemCount; + /// + /// Injections used to reproduce special failure cases. + /// Convert this to a mock in the future. + /// + private readonly TestInjections testSettings; + private CosmosQueryContext queryContext; protected CosmosQueryClient queryClient; @@ -110,6 +114,7 @@ internal abstract class CosmosCrossPartitionQueryExecutionContext : CosmosQueryE /// Comparer used to figure out that document producer tree to serve documents from next. /// The priority function to determine which partition to fetch documents from next. /// Used to determine whether we need to return the continuation token for a partition. + /// Test settings. protected CosmosCrossPartitionQueryExecutionContext( CosmosQueryContext queryContext, int? maxConcurrency, @@ -117,7 +122,8 @@ protected CosmosCrossPartitionQueryExecutionContext( int? maxBufferedItemCount, IComparer moveNextComparer, Func fetchPrioirtyFunction, - IEqualityComparer equalityComparer) + IEqualityComparer equalityComparer, + TestInjections testSettings) { if (moveNextComparer == null) { @@ -140,6 +146,7 @@ protected CosmosCrossPartitionQueryExecutionContext( this.fetchPrioirtyFunction = fetchPrioirtyFunction; this.comparableTaskScheduler = new ComparableTaskScheduler(maxConcurrency.GetValueOrDefault(0)); this.equalityComparer = equalityComparer; + this.testSettings = testSettings; this.requestChargeTracker = new RequestChargeTracker(); this.diagnosticsPages = new ConcurrentBag(); this.actualMaxPageSize = maxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalMaxItemCount); @@ -181,16 +188,6 @@ protected CosmosCrossPartitionQueryExecutionContext( /// public override bool IsDone => !this.HasMoreResults; - /// - /// If a failure is hit store it and return it on the next drain call. - /// This allows returning the results computed before the failure. - /// - public QueryResponseCore? FailureResponse - { - get; - protected set; - } - protected int ActualMaxBufferedItemCount => (int)this.actualMaxBufferedItemCount; protected int ActualMaxPageSize => (int)this.actualMaxPageSize; @@ -212,7 +209,7 @@ protected abstract string ContinuationToken /// /// Gets a value indicating whether the context still has more results. /// - private bool HasMoreResults => this.FailureResponse != null || (this.itemProducerForest.Count != 0 && this.CurrentItemProducerTree().HasMoreResults); + private bool HasMoreResults => (this.itemProducerForest.Count != 0) && this.CurrentItemProducerTree().HasMoreResults; /// /// Gets the number of documents we can still buffer. @@ -245,22 +242,19 @@ protected abstract string ContinuationToken /// public IEnumerable GetActiveItemProducers() { - lock (this.itemProducerForest) + ItemProducerTree current = this.itemProducerForest.Peek().CurrentItemProducerTree; + if (current.HasMoreResults && !current.IsActive) { - ItemProducerTree current = this.itemProducerForest.Peek().CurrentItemProducerTree; - if (current.HasMoreResults && !current.IsActive) - { - // If the current document producer tree has more results, but isn't active. - // then we still want to emit it, since it won't get picked up in the below for loop. - yield return current.Root; - } + // If the current document producer tree has more results, but isn't active. + // then we still want to emit it, since it won't get picked up in the below for loop. + yield return current.Root; + } - foreach (ItemProducerTree itemProducerTree in this.itemProducerForest) + foreach (ItemProducerTree itemProducerTree in this.itemProducerForest) + { + foreach (ItemProducer itemProducer in itemProducerTree.GetActiveItemProducers()) { - foreach (ItemProducer itemProducer in itemProducerTree.GetActiveItemProducers()) - { - yield return itemProducer; - } + yield return itemProducer; } } } @@ -279,6 +273,7 @@ public ItemProducerTree CurrentItemProducerTree() /// public void PushCurrentItemProducerTree(ItemProducerTree itemProducerTree) { + itemProducerTree.UpdatePriority(); this.itemProducerForest.Enqueue(itemProducerTree); } @@ -307,80 +302,6 @@ public override void Stop() this.comparableTaskScheduler.Stop(); } - /// - /// A helper to move next and set the failure response if one is received - /// - /// The item producer tree - /// The cancellation token - /// True if it move next failed. It can fail from an error or hitting the end of the tree - protected async Task MoveNextHelperAsync(ItemProducerTree itemProducerTree, CancellationToken cancellationToken) - { - (bool successfullyMovedNext, QueryResponseCore? failureResponse) moveNextResponse = await itemProducerTree.MoveNextAsync(cancellationToken); - if (moveNextResponse.failureResponse != null) - { - this.FailureResponse = moveNextResponse.failureResponse; - } - - return moveNextResponse.successfullyMovedNext; - } - - /// - /// Drains documents from this component. This has the common drain logic for each implementation. - /// - /// The maximum number of documents to drain. - /// The cancellation token to cancel tasks. - /// A task that when awaited on returns a feed response. - public override async Task DrainAsync(int maxElements, CancellationToken cancellationToken) - { - cancellationToken.ThrowIfCancellationRequested(); - - // The initialization or previous Drain Async failed. Just return the failure. - if (this.FailureResponse != null) - { - this.Stop(); - - QueryResponseCore failure = this.FailureResponse.Value; - this.FailureResponse = null; - return failure; - } - - // Drain the results. If there is no results and a failure then return the failure. - IReadOnlyList results = await this.InternalDrainAsync(maxElements, cancellationToken); - if ((results == null || results.Count == 0) && this.FailureResponse != null) - { - this.Stop(); - QueryResponseCore failure = this.FailureResponse.Value; - this.FailureResponse = null; - return failure; - - } - - string continuation = this.ContinuationToken; - if (continuation == "[]") - { - throw new InvalidOperationException("Somehow a document query execution context returned an empty array of continuations."); - } - - IReadOnlyCollection diagnostics = this.GetAndResetDiagnostics(); - - return QueryResponseCore.CreateSuccess( - result: results, - requestCharge: this.requestChargeTracker.GetAndResetCharge(), - activityId: null, - diagnostics: diagnostics, - disallowContinuationTokenMessage: null, - continuationToken: continuation, - responseLengthBytes: this.GetAndResetResponseLengthBytes()); - } - - /// - /// The drain async logic for the different implementation - /// - /// The maximum number of documents to drain. - /// The cancellation token to cancel tasks. - /// A task that when awaited on returns a feed response. - public abstract Task> InternalDrainAsync(int maxElements, CancellationToken token); - /// /// Initializes cross partition query execution context by initializing the necessary document producers. /// @@ -430,6 +351,7 @@ protected async Task> TryInitializeAsync( this.OnItemProducerTreeCompleteFetching, this.itemProducerForest.Comparer as IComparer, this.equalityComparer, + this.testSettings, deferFirstPage, collectionRid, initialPageSize, @@ -452,31 +374,48 @@ protected async Task> TryInitializeAsync( { if (!deferFirstPage) { - (bool successfullyMovedNext, QueryResponseCore? failureResponse) = await itemProducerTree.MoveNextIfNotSplitAsync(cancellationToken); - if (failureResponse != null) + while (true) { - // Set the failure so on drain it can be returned. - this.FailureResponse = failureResponse; - - // No reason to enqueue the rest of the itemProducerTrees since there is a failure. - break; + (bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationToken); + + if (failureResponse.HasValue) + { + return TryCatch.FromException( + new CosmosException( + statusCode: failureResponse.Value.StatusCode, + subStatusCode: (int)failureResponse.Value.SubStatusCode.GetValueOrDefault(0), + message: failureResponse.Value.ErrorMessage, + activityId: failureResponse.Value.ActivityId, + requestCharge: failureResponse.Value.RequestCharge)); + } + + if (!movedToNextPage) + { + break; + } + + if (itemProducerTree.IsAtBeginningOfPage) + { + break; + } + + if (itemProducerTree.TryMoveNextDocumentWithinPage()) + { + break; + } } } if (tryFilterAsync != null) { TryCatch tryFilter = await tryFilterAsync(itemProducerTree); - if (!tryFilter.Succeeded) { return tryFilter; } } - if (itemProducerTree.HasMoreResults) - { - this.itemProducerForest.Enqueue(itemProducerTree); - } + this.itemProducerForest.Enqueue(itemProducerTree); } return TryCatch.FromResult(true); @@ -559,7 +498,7 @@ public static TryCatch> TryFindTargetRangeAndExtrac if (minIndex < 0) { return TryCatch>.FromException( - new ArgumentException( + new MalformedContinuationTokenException( $"{RMResources.InvalidContinuationToken} - Could not find continuation token: {firstContinuationToken}")); } @@ -579,7 +518,7 @@ public static TryCatch> TryFindTargetRangeAndExtrac if (replacementRanges.Count() == 0) { return TryCatch>.FromException( - new ArgumentException( + new MalformedContinuationTokenException( $"{RMResources.InvalidContinuationToken} - Could not find continuation token: {continuationToken}")); } @@ -598,7 +537,7 @@ public static TryCatch> TryFindTargetRangeAndExtrac child1Min == parentMin)) { return TryCatch>.FromException( - new ArgumentException( + new MalformedContinuationTokenException( $"{RMResources.InvalidContinuationToken} - PMax = C2Max > C2Min > C1Max > C1Min = PMin: {continuationToken}")); } @@ -628,7 +567,7 @@ protected virtual long IncrementResponseLengthBytes(long incrementValue) /// Since query metrics are being aggregated asynchronously to the feed responses as explained in the member documentation, /// this function allows us to take a snapshot of the query metrics. /// - private IReadOnlyCollection GetAndResetDiagnostics() + protected IReadOnlyCollection GetAndResetDiagnostics() { // Safely swap the current ConcurrentBag for a new instance. ConcurrentBag queryPageDiagnostics = Interlocked.Exchange( @@ -740,6 +679,7 @@ public struct CrossPartitionInitParams /// The max concurrency /// The max buffered item count /// Max item count + /// Test settings. public CrossPartitionInitParams( SqlQuerySpec sqlQuerySpec, string collectionRid, @@ -748,7 +688,8 @@ public CrossPartitionInitParams( int initialPageSize, int? maxConcurrency, int? maxItemCount, - int? maxBufferedItemCount) + int? maxBufferedItemCount, + TestInjections testSettings) { if (string.IsNullOrWhiteSpace(collectionRid)) { @@ -792,6 +733,7 @@ public CrossPartitionInitParams( this.MaxBufferedItemCount = maxBufferedItemCount; this.MaxConcurrency = maxConcurrency; this.MaxItemCount = maxItemCount; + this.TestSettings = testSettings; } /// @@ -833,6 +775,8 @@ public CrossPartitionInitParams( /// Gets the max buffered item count /// public int? MaxBufferedItemCount { get; } + + public TestInjections TestSettings { get; } } #region ItemProducerTreeComparableTask diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosOrderByItemQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosOrderByItemQueryExecutionContext.cs index 87689160ca..37d005dfe8 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosOrderByItemQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosOrderByItemQueryExecutionContext.cs @@ -12,8 +12,10 @@ namespace Microsoft.Azure.Cosmos.Query using System.Threading.Tasks; using Collections.Generic; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; using Newtonsoft.Json; + using Newtonsoft.Json.Linq; using ParallelQuery; using PartitionKeyRange = Documents.PartitionKeyRange; using ResourceId = Documents.ResourceId; @@ -46,6 +48,11 @@ internal sealed class CosmosOrderByItemQueryExecutionContext : CosmosCrossPartit /// private static readonly Func FetchPriorityFunction = itemProducerTree => itemProducerTree.BufferedItemCount; + private static readonly JsonSerializerSettings NoAsciiCharactersSerializerSettings = new JsonSerializerSettings() + { + StringEscapeHandling = StringEscapeHandling.EscapeNonAscii, + }; + /// /// Skip count used for JOIN queries. /// You can read up more about this in the documentation for the continuation token. @@ -57,6 +64,8 @@ internal sealed class CosmosOrderByItemQueryExecutionContext : CosmosCrossPartit /// private string previousRid; + private IList previousOrderByItems; + /// /// Initializes a new instance of the CosmosOrderByItemQueryExecutionContext class. /// @@ -73,12 +82,14 @@ internal sealed class CosmosOrderByItemQueryExecutionContext : CosmosCrossPartit /// The max buffered item count /// Max item count /// Comparer used to internally compare documents from different sorted partitions. + /// Test settings. private CosmosOrderByItemQueryExecutionContext( CosmosQueryContext initPararms, int? maxConcurrency, int? maxItemCount, int? maxBufferedItemCount, - OrderByConsumeComparer consumeComparer) + OrderByConsumeComparer consumeComparer, + TestInjections testSettings) : base( queryContext: initPararms, maxConcurrency: maxConcurrency, @@ -86,7 +97,8 @@ private CosmosOrderByItemQueryExecutionContext( maxBufferedItemCount: maxBufferedItemCount, moveNextComparer: consumeComparer, fetchPrioirtyFunction: CosmosOrderByItemQueryExecutionContext.FetchPriorityFunction, - equalityComparer: new OrderByEqualityComparer(consumeComparer)) + equalityComparer: new OrderByEqualityComparer(consumeComparer), + testSettings: testSettings) { } @@ -107,34 +119,40 @@ protected override string ContinuationToken // With this information we have captured the progress for all partitions in a single continuation token. get { - if (this.IsDone) - { - return null; - } - IEnumerable activeItemProducers = this.GetActiveItemProducers(); - return activeItemProducers.Count() > 0 ? JsonConvert.SerializeObject( - activeItemProducers.Select( - (itemProducer) => + string continuationToken; + if (activeItemProducers.Any()) { - OrderByQueryResult orderByQueryResult = new OrderByQueryResult(itemProducer.Current); - string filter = itemProducer.Filter; - return new OrderByContinuationToken( - this.queryClient, - new CompositeContinuationToken - { - Token = itemProducer.PreviousContinuationToken, - Range = itemProducer.PartitionKeyRange.ToRange(), - }, - orderByQueryResult.OrderByItems, - orderByQueryResult.Rid, - this.ShouldIncrementSkipCount(itemProducer) ? this.skipCount + 1 : 0, - filter); - }), - new JsonSerializerSettings() + IEnumerable orderByContinuationTokens = activeItemProducers.Select((itemProducer) => { - StringEscapeHandling = StringEscapeHandling.EscapeNonAscii, - }) : null; + OrderByQueryResult orderByQueryResult = new OrderByQueryResult(itemProducer.Current); + string filter = itemProducer.Filter; + OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( + new CompositeContinuationToken + { + Token = itemProducer.PreviousContinuationToken, + Range = itemProducer.PartitionKeyRange.ToRange(), + }, + orderByQueryResult.OrderByItems, + orderByQueryResult.Rid, + this.ShouldIncrementSkipCount(itemProducer) ? this.skipCount + 1 : 0, + filter); + + return orderByContinuationToken; + }); + + continuationToken = JsonConvert.SerializeObject(orderByContinuationTokens, NoAsciiCharactersSerializerSettings); + + // Newtonsoft has a bug where non ascii characters aren't escaped for custom POGOs + // so the workaround is to double serialize + continuationToken = JsonConvert.SerializeObject(JToken.Parse(continuationToken), NoAsciiCharactersSerializerSettings); + } + else + { + continuationToken = null; + } + + return continuationToken; } } @@ -160,7 +178,8 @@ public static async Task> TryCr maxConcurrency: initParams.MaxConcurrency, maxItemCount: initParams.MaxItemCount, maxBufferedItemCount: initParams.MaxBufferedItemCount, - consumeComparer: new OrderByConsumeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy)); + consumeComparer: new OrderByConsumeComparer(initParams.PartitionedQueryExecutionInfo.QueryInfo.OrderBy), + testSettings: initParams.TestSettings); return (await context.TryInitializeAsync( sqlQuerySpec: initParams.SqlQuerySpec, @@ -180,8 +199,10 @@ public static async Task> TryCr /// The maximum number of elements. /// The cancellation token. /// A task that when awaited on return a page of documents. - public override async Task> InternalDrainAsync(int maxElements, CancellationToken cancellationToken) + public override async Task DrainAsync(int maxElements, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + //// In order to maintain the continuation token for the user we must drain with a few constraints //// 1) We always drain from the partition, which has the highest priority item first //// 2) If multiple partitions have the same priority item then we drain from the left most first @@ -208,38 +229,81 @@ public override async Task> InternalDrainAsync(int //// 2) always come before where j < k List results = new List(); - bool isSuccessToMoveNext = true; - while (!this.IsDone && results.Count < maxElements && isSuccessToMoveNext) + while (results.Count < maxElements) { // Only drain from the highest priority document producer // We need to pop and push back the document producer tree, since the priority changes according to the sort order. ItemProducerTree currentItemProducerTree = this.PopCurrentItemProducerTree(); + try + { + if (!currentItemProducerTree.HasMoreResults) + { + // This means there are no more items to drain + break; + } - OrderByQueryResult orderByQueryResult = new OrderByQueryResult(currentItemProducerTree.Current); + OrderByQueryResult orderByQueryResult = new OrderByQueryResult(currentItemProducerTree.Current); - // Only add the payload, since other stuff is garbage from the caller's perspective. - results.Add(orderByQueryResult.Payload); + // Only add the payload, since other stuff is garbage from the caller's perspective. + results.Add(orderByQueryResult.Payload); - // If we are at the beginning of the page and seeing an rid from the previous page we should increment the skip count - // due to the fact that JOINs can make a document appear multiple times and across continuations, so we don't want to - // surface this more than needed. More information can be found in the continuation token docs. - if (this.ShouldIncrementSkipCount(currentItemProducerTree.CurrentItemProducerTree.Root)) - { - ++this.skipCount; - } - else - { - this.skipCount = 0; - } + // If we are at the beginning of the page and seeing an rid from the previous page we should increment the skip count + // due to the fact that JOINs can make a document appear multiple times and across continuations, so we don't want to + // surface this more than needed. More information can be found in the continuation token docs. + if (this.ShouldIncrementSkipCount(currentItemProducerTree.CurrentItemProducerTree.Root)) + { + ++this.skipCount; + } + else + { + this.skipCount = 0; + } + + this.previousRid = orderByQueryResult.Rid; + this.previousOrderByItems = orderByQueryResult.OrderByItems; + + if (!currentItemProducerTree.TryMoveNextDocumentWithinPage()) + { + while (true) + { + (bool movedToNextPage, QueryResponseCore? failureResponse) = await currentItemProducerTree.TryMoveNextPageAsync(cancellationToken); + if (!movedToNextPage) + { + if (failureResponse.HasValue) + { + // TODO: We can buffer this failure so that the user can still get the pages we already got. + return failureResponse.Value; + } - this.previousRid = orderByQueryResult.Rid; + break; + } - isSuccessToMoveNext = await this.MoveNextHelperAsync(currentItemProducerTree, cancellationToken); + if (currentItemProducerTree.IsAtBeginningOfPage) + { + break; + } - this.PushCurrentItemProducerTree(currentItemProducerTree); + if (currentItemProducerTree.TryMoveNextDocumentWithinPage()) + { + break; + } + } + } + } + finally + { + this.PushCurrentItemProducerTree(currentItemProducerTree); + } } - return results; + return QueryResponseCore.CreateSuccess( + result: results, + requestCharge: this.requestChargeTracker.GetAndResetCharge(), + activityId: null, + responseLengthBytes: this.GetAndResetResponseLengthBytes(), + disallowContinuationTokenMessage: null, + continuationToken: this.ContinuationToken, + diagnostics: this.GetAndResetDiagnostics()); } /// @@ -310,7 +374,6 @@ private async Task> TryInitializeAsync( deferFirstPage: false, filter: null, tryFilterAsync: null); - if (!tryInitialize.Succeeded) { return tryInitialize; @@ -364,7 +427,7 @@ private async Task> TryInitializeAsync( sqlQuerySpec.QueryText.Replace(FormatPlaceHolder, info.Filter), sqlQuerySpec.Parameters); - await base.TryInitializeAsync( + TryCatch tryInitialize = await base.TryInitializeAsync( collectionRid, partialRanges, initialPageSize, @@ -395,6 +458,10 @@ await base.TryInitializeAsync( return TryCatch.FromResult(true); }, cancellationToken); + if (!tryInitialize.Succeeded) + { + return tryInitialize; + } } } @@ -428,7 +495,7 @@ private static TryCatch TryExtractContinuationTokens if (suppliedOrderByContinuationTokens.Length == 0) { return TryCatch.FromException( - new Exception($"Order by continuation token cannot be empty: {requestContinuation}.")); + new MalformedContinuationTokenException($"Order by continuation token cannot be empty: {requestContinuation}.")); } foreach (OrderByContinuationToken suppliedOrderByContinuationToken in suppliedOrderByContinuationTokens) @@ -436,7 +503,7 @@ private static TryCatch TryExtractContinuationTokens if (suppliedOrderByContinuationToken.OrderByItems.Count != sortOrders.Length) { return TryCatch.FromException( - new Exception($"Invalid order-by items in continuation token {requestContinuation} for OrderBy~Context.")); + new MalformedContinuationTokenException($"Invalid order-by items in continuation token {requestContinuation} for OrderBy~Context.")); } } @@ -445,7 +512,7 @@ private static TryCatch TryExtractContinuationTokens catch (JsonException ex) { return TryCatch.FromException( - new Exception($"Invalid JSON in continuation token {requestContinuation} for OrderBy~Context: {ex.Message}")); + new MalformedContinuationTokenException($"Invalid JSON in continuation token {requestContinuation} for OrderBy~Context: {ex.Message}")); } } @@ -476,7 +543,7 @@ private async Task> TryFilterAsync( if (!ResourceId.TryParse(continuationToken.Rid, out ResourceId continuationRid)) { return TryCatch.FromException( - new Exception($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context.")); + new MalformedContinuationTokenException($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context.")); } Dictionary resourceIds = new Dictionary(); @@ -485,6 +552,12 @@ private async Task> TryFilterAsync( while (true) { + if (tree.Current == null) + { + // This document producer doesn't have anymore items. + break; + } + OrderByQueryResult orderByResult = new OrderByQueryResult(tree.Current); // Throw away documents until it matches the item from the continuation token. int cmp = 0; @@ -514,7 +587,7 @@ private async Task> TryFilterAsync( if (!ResourceId.TryParse(orderByResult.Rid, out rid)) { return TryCatch.FromException( - new Exception($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context~TryParse.")); + new MalformedContinuationTokenException($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context~TryParse.")); } @@ -526,7 +599,7 @@ private async Task> TryFilterAsync( if (continuationRid.Database != rid.Database || continuationRid.DocumentCollection != rid.DocumentCollection) { return TryCatch.FromException( - new Exception($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context.")); + new MalformedContinuationTokenException($"Invalid Rid in the continuation token {continuationToken.CompositeContinuationToken.Token} for OrderBy~Context.")); } continuationRidVerified = true; @@ -551,15 +624,37 @@ private async Task> TryFilterAsync( } } - (bool successfullyMovedNext, QueryResponseCore? failureResponse) moveNextResponse = await tree.MoveNextAsync(cancellationToken); - if (!moveNextResponse.successfullyMovedNext) + if (!tree.TryMoveNextDocumentWithinPage()) { - if (moveNextResponse.failureResponse != null) + while (true) { - this.FailureResponse = moveNextResponse.failureResponse; - } + (bool successfullyMovedNext, QueryResponseCore? failureResponse) = await tree.TryMoveNextPageAsync(cancellationToken); + if (!successfullyMovedNext) + { + if (failureResponse.HasValue) + { + return TryCatch.FromException( + new CosmosException( + statusCode: failureResponse.Value.StatusCode, + subStatusCode: (int)failureResponse.Value.SubStatusCode.GetValueOrDefault(0), + message: failureResponse.Value.ErrorMessage, + activityId: failureResponse.Value.ActivityId, + requestCharge: 0)); + } - break; + break; + } + + if (tree.IsAtBeginningOfPage) + { + break; + } + + if (tree.TryMoveNextDocumentWithinPage()) + { + break; + } + } } } } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosParallelItemQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosParallelItemQueryExecutionContext.cs index fd4aa8f79a..b1f064e392 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosParallelItemQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosParallelItemQueryExecutionContext.cs @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.Query using System.Threading.Tasks; using Collections.Generic; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.Monads; using Newtonsoft.Json; using PartitionKeyRange = Documents.PartitionKeyRange; @@ -47,11 +48,13 @@ internal sealed class CosmosParallelItemQueryExecutionContext : CosmosCrossParti /// The max concurrency /// The max buffered item count /// Max item count + /// Test settings. private CosmosParallelItemQueryExecutionContext( CosmosQueryContext queryContext, int? maxConcurrency, int? maxItemCount, - int? maxBufferedItemCount) + int? maxBufferedItemCount, + TestInjections testSettings) : base( queryContext: queryContext, maxConcurrency: maxConcurrency, @@ -59,7 +62,8 @@ private CosmosParallelItemQueryExecutionContext( maxBufferedItemCount: maxBufferedItemCount, moveNextComparer: CosmosParallelItemQueryExecutionContext.MoveNextComparer, fetchPrioirtyFunction: CosmosParallelItemQueryExecutionContext.FetchPriorityFunction, - equalityComparer: CosmosParallelItemQueryExecutionContext.EqualityComparer) + equalityComparer: CosmosParallelItemQueryExecutionContext.EqualityComparer, + testSettings: testSettings) { } @@ -76,19 +80,23 @@ protected override string ContinuationToken { get { - if (this.IsDone) - { - return null; - } - IEnumerable activeItemProducers = this.GetActiveItemProducers(); - return activeItemProducers.Count() > 0 ? JsonConvert.SerializeObject( - activeItemProducers.Select((documentProducer) => new CompositeContinuationToken + string continuationToken; + if (activeItemProducers.Any()) + { + IEnumerable compositeContinuationTokens = activeItemProducers.Select((documentProducer) => new CompositeContinuationToken { - Token = documentProducer.PreviousContinuationToken, + Token = documentProducer.CurrentContinuationToken, Range = documentProducer.PartitionKeyRange.ToRange() - }), - DefaultJsonSerializationSettings.Value) : null; + }); + continuationToken = JsonConvert.SerializeObject(compositeContinuationTokens, DefaultJsonSerializationSettings.Value); + } + else + { + continuationToken = null; + } + + return continuationToken; } } @@ -108,7 +116,8 @@ public static async Task> TryC queryContext: queryContext, maxConcurrency: initParams.MaxConcurrency, maxItemCount: initParams.MaxItemCount, - maxBufferedItemCount: initParams.MaxBufferedItemCount); + maxBufferedItemCount: initParams.MaxBufferedItemCount, + testSettings: initParams.TestSettings); return await context.TryInitializeAsync( sqlQuerySpec: initParams.SqlQuerySpec, @@ -119,15 +128,7 @@ public static async Task> TryC cancellationToken: cancellationToken); } - /// - /// Drains documents from this execution context. - /// - /// The maximum number of documents to drains. - /// The cancellation token. - /// A task that when awaited on returns a DoucmentFeedResponse of results. - public override async Task> InternalDrainAsync( - int maxElements, - CancellationToken cancellationToken) + public override async Task DrainAsync(int maxElements, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -138,30 +139,42 @@ public override async Task> InternalDrainAsync( // Only drain from the leftmost (current) document producer tree ItemProducerTree currentItemProducerTree = this.PopCurrentItemProducerTree(); - - // This might be the first time we have seen this document producer tree so we need to buffer documents - if (currentItemProducerTree.Current == null) - { - await this.MoveNextHelperAsync(currentItemProducerTree, cancellationToken); - } - - int itemsLeftInCurrentPage = currentItemProducerTree.ItemsLeftInCurrentPage; - - // Only drain full pages or less if this is a top query. List results = new List(); - for (int i = 0; i < Math.Min(itemsLeftInCurrentPage, maxElements); i++) + try { - results.Add(currentItemProducerTree.Current); - if (!await this.MoveNextHelperAsync(currentItemProducerTree, cancellationToken)) + (bool gotNextPage, QueryResponseCore? failureResponse) = await currentItemProducerTree.TryMoveNextPageAsync(cancellationToken); + if (failureResponse != null) { - break; + return failureResponse.Value; } - } - this.PushCurrentItemProducerTree(currentItemProducerTree); + if (gotNextPage) + { + int itemsLeftInCurrentPage = currentItemProducerTree.ItemsLeftInCurrentPage; + + // Only drain full pages or less if this is a top query. + currentItemProducerTree.TryMoveNextDocumentWithinPage(); + int numberOfItemsToDrain = Math.Min(itemsLeftInCurrentPage, maxElements); + for (int i = 0; i < numberOfItemsToDrain; i++) + { + results.Add(currentItemProducerTree.Current); + currentItemProducerTree.TryMoveNextDocumentWithinPage(); + } + } + } + finally + { + this.PushCurrentItemProducerTree(currentItemProducerTree); + } - // At this point the document producer tree should have internally called MoveNextPage, since we fully drained a page. - return results; + return QueryResponseCore.CreateSuccess( + result: results, + requestCharge: this.requestChargeTracker.GetAndResetCharge(), + activityId: null, + responseLengthBytes: this.GetAndResetResponseLengthBytes(), + disallowContinuationTokenMessage: null, + continuationToken: this.ContinuationToken, + diagnostics: this.GetAndResetDiagnostics()); } /// @@ -237,7 +250,7 @@ private static TryCatch TryGetInitializationInfoFromContinuati if (!TryParseContinuationToken(continuationToken, out CompositeContinuationToken[] tokens)) { return TryCatch.FromException( - new Exception($"Invalid format for continuation token {continuationToken} for {nameof(CosmosParallelItemQueryExecutionContext)}")); + new MalformedContinuationTokenException($"Invalid format for continuation token {continuationToken} for {nameof(CosmosParallelItemQueryExecutionContext)}")); } return CosmosCrossPartitionQueryExecutionContext.TryFindTargetRangeAndExtractContinuationTokens( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContext.cs index 5985155a0e..56a82a74ba 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContext.cs @@ -26,10 +26,10 @@ public abstract bool IsDone /// /// Executes the context to feed the next page of results. /// - /// The cancellation token. + /// The cancellation token. /// A task to await on, which in return provides a DoucmentFeedResponse of documents. - public abstract Task ExecuteNextAsync(CancellationToken token); + public abstract Task ExecuteNextAsync(CancellationToken cancellationToken); - public abstract bool TryGetContinuationToken(out string state); + public abstract bool TryGetContinuationToken(out string continuationToken); } } diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextFactory.cs index 515e129069..f6f329619d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextFactory.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextFactory.cs @@ -1,49 +1,24 @@ //------------------------------------------------------------ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Query +namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionContext { using System; using System.Collections.Generic; using System.Diagnostics; - using System.Globalization; - using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.ContinuationTokens; - using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext; using Microsoft.Azure.Cosmos.Query.Core.Monads; - using Microsoft.Azure.Cosmos.Query.ParallelQuery; - /// - /// Factory class for creating the appropriate DocumentQueryExecutionContext for the provided type of query. - /// - internal sealed class CosmosQueryExecutionContextFactory : CosmosQueryExecutionContext + internal static class CosmosQueryExecutionContextFactory { - internal const string InternalPartitionKeyDefinitionProperty = "x-ms-query-partitionkey-definition"; + private const string InternalPartitionKeyDefinitionProperty = "x-ms-query-partitionkey-definition"; private const int PageSizeFactorForTop = 5; - internal readonly CosmosQueryContext CosmosQueryContext; - private InputParameters inputParameters; - private CosmosQueryExecutionContext innerExecutionContext; - - /// - /// Store the failed response - /// - private QueryResponseCore? responseMessageException; - - /// - /// Store any exception thrown - /// - private Exception exception; - - /// - /// The query plan that will be embeded in the continuation token. - /// - private PartitionedQueryExecutionInfo partitionedQueryExecutionInfo; - - public CosmosQueryExecutionContextFactory( + public static CosmosQueryExecutionContext Create( CosmosQueryContext cosmosQueryContext, InputParameters inputParameters) { @@ -52,174 +27,56 @@ public CosmosQueryExecutionContextFactory( throw new ArgumentNullException(nameof(cosmosQueryContext)); } - this.CosmosQueryContext = cosmosQueryContext; - this.inputParameters = inputParameters; - - // Swapping out negative values in feedOptions for int.MaxValue - if (this.inputParameters.MaxBufferedItemCount.HasValue && this.inputParameters.MaxBufferedItemCount < 0) - { - this.inputParameters.MaxBufferedItemCount = int.MaxValue; - } - - if (this.inputParameters.MaxConcurrency.HasValue && this.inputParameters.MaxConcurrency < 0) - { - this.inputParameters.MaxConcurrency = int.MaxValue; - } - - if (this.inputParameters.MaxItemCount.HasValue && this.inputParameters.MaxItemCount < 0) + if (inputParameters == null) { - this.inputParameters.MaxItemCount = int.MaxValue; + throw new ArgumentNullException(nameof(inputParameters)); } - } - public override bool IsDone - { - get - { - // No more results if an exception is hit - if (this.responseMessageException != null || this.exception != null) + CosmosQueryExecutionContextWithNameCacheStaleRetry cosmosQueryExecutionContextWithNameCacheStaleRetry = new CosmosQueryExecutionContextWithNameCacheStaleRetry( + cosmosQueryContext: cosmosQueryContext, + cosmosQueryExecutionContextFactory: () => { - return true; - } - - return this.innerExecutionContext != null ? this.innerExecutionContext.IsDone : false; - } - } - - public override async Task ExecuteNextAsync(CancellationToken cancellationToken) - { - if (this.responseMessageException != null) - { - return this.responseMessageException.Value; - } - - if (this.exception != null) - { - throw this.exception; - } - - try - { - bool isFirstExecute = false; - QueryResponseCore response; - while (true) - { - // The retry policy handles the scenario when the name cache is stale. If the cache is stale the entire - // execute context has incorrect values and should be recreated. This should only be done for the first - // execution. If results have already been pulled an error should be returned to the user since it's - // not possible to combine query results from multiple containers. - if (this.innerExecutionContext == null) + // Query Iterator requires that the creation of the query context is defered until the user calls ReadNextAsync + AsyncLazy> lazyTryCreateCosmosQueryExecutionContext = new AsyncLazy>(valueFactory: (innerCancellationToken) => { - TryCatch tryCreateItemQueryExecutionContext = await this.TryCreateItemQueryExecutionContextAsync(cancellationToken); - if (!tryCreateItemQueryExecutionContext.Succeeded) - { - // Failed to create pipeline (due to a bad request). - return QueryResponseCore.CreateFailure( - HttpStatusCode.BadRequest, - subStatusCodes: null, - errorMessage: tryCreateItemQueryExecutionContext.Exception.ToString(), - requestCharge: 0, - activityId: this.CosmosQueryContext.CorrelatedActivityId.ToString(), - diagnostics: QueryResponseCore.EmptyDiagnostics); - } - - this.innerExecutionContext = tryCreateItemQueryExecutionContext.Result; - isFirstExecute = true; - } - - response = await this.innerExecutionContext.ExecuteNextAsync(cancellationToken); - - if (response.IsSuccess) - { - break; - } - - if (isFirstExecute - && (response.StatusCode == HttpStatusCode.Gone) - && (response.SubStatusCode == Documents.SubStatusCodes.NameCacheIsStale)) - { - await this.CosmosQueryContext.QueryClient.ForceRefreshCollectionCacheAsync( - this.CosmosQueryContext.ResourceLink.OriginalString, - cancellationToken); - TryCatch tryCreateItemQueryExecutionContext = await this.TryCreateItemQueryExecutionContextAsync(cancellationToken); - if (!tryCreateItemQueryExecutionContext.Succeeded) - { - // Failed to create pipeline (due to a bad request). - return QueryResponseCore.CreateFailure( - HttpStatusCode.BadRequest, - subStatusCodes: null, - errorMessage: tryCreateItemQueryExecutionContext.Exception.ToString(), - requestCharge: 0, - activityId: this.CosmosQueryContext.CorrelatedActivityId.ToString(), - diagnostics: QueryResponseCore.EmptyDiagnostics); - } - - this.innerExecutionContext = tryCreateItemQueryExecutionContext.Result; - isFirstExecute = false; - } - else - { - break; - } - } - - if (!response.IsSuccess) - { - this.responseMessageException = response; - } - - return response; - } - catch (Exception e) - { - this.exception = e; - this.Dispose(); - throw; - } + innerCancellationToken.ThrowIfCancellationRequested(); + return CosmosQueryExecutionContextFactory.TryCreateCoreContextAsync( + cosmosQueryContext, + inputParameters, + innerCancellationToken); + }); + LazyCosmosQueryExecutionContext lazyCosmosQueryExecutionContext = new LazyCosmosQueryExecutionContext(lazyTryCreateCosmosQueryExecutionContext); + return lazyCosmosQueryExecutionContext; + }); + + CatchAllCosmosQueryExecutionContext catchAllCosmosQueryExecutionContext = new CatchAllCosmosQueryExecutionContext(cosmosQueryExecutionContextWithNameCacheStaleRetry); + + return catchAllCosmosQueryExecutionContext; } - public override bool TryGetContinuationToken(out string continuationToken) - { - if (this.innerExecutionContext.IsDone) - { - continuationToken = null; - return true; - } - - if (!this.innerExecutionContext.TryGetContinuationToken(out string innerContinuationToken)) - { - continuationToken = null; - return false; - } - - PipelineContinuationTokenV1_1 pipelineContinuationToken = new PipelineContinuationTokenV1_1( - this.partitionedQueryExecutionInfo, - innerContinuationToken); - - continuationToken = pipelineContinuationToken.ToString(this.inputParameters.ResponseContinuationTokenLimitInKb.GetValueOrDefault(12) * 1024); - return true; - } - - private async Task> TryCreateItemQueryExecutionContextAsync( + private static async Task> TryCreateCoreContextAsync( + CosmosQueryContext cosmosQueryContext, + InputParameters inputParameters, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); - - string continuationToken = this.inputParameters.InitialUserContinuationToken; - if (this.inputParameters.InitialUserContinuationToken != null) + // Try to parse the continuation token. + string continuationToken = inputParameters.InitialUserContinuationToken; + PartitionedQueryExecutionInfo queryPlanFromContinuationToken = inputParameters.PartitionedQueryExecutionInfo; + if (continuationToken != null) { if (!PipelineContinuationToken.TryParse( continuationToken, out PipelineContinuationToken pipelineContinuationToken)) { return TryCatch.FromException( - new Exception($"Malformed {nameof(PipelineContinuationToken)}: {continuationToken}.")); + new MalformedContinuationTokenException( + $"Malformed {nameof(PipelineContinuationToken)}: {continuationToken}.")); } if (PipelineContinuationToken.IsTokenFromTheFuture(pipelineContinuationToken)) { return TryCatch.FromException( - new Exception( + new MalformedContinuationTokenException( $"{nameof(PipelineContinuationToken)} Continuation token is from a newer version of the SDK. " + $"Upgrade the SDK to avoid this issue." + $"{continuationToken}.")); @@ -230,40 +87,39 @@ private async Task> TryCreateItemQueryExec out PipelineContinuationTokenV1_1 latestVersionPipelineContinuationToken)) { return TryCatch.FromException( - new Exception($"{nameof(PipelineContinuationToken)}: '{continuationToken}' is no longer supported.")); + new MalformedContinuationTokenException( + $"{nameof(PipelineContinuationToken)}: '{continuationToken}' is no longer supported.")); } continuationToken = latestVersionPipelineContinuationToken.SourceContinuationToken; - - this.inputParameters.InitialUserContinuationToken = continuationToken; if (latestVersionPipelineContinuationToken.QueryPlan != null) { - this.inputParameters.PartitionedQueryExecutionInfo = latestVersionPipelineContinuationToken.QueryPlan; + queryPlanFromContinuationToken = latestVersionPipelineContinuationToken.QueryPlan; } } - CosmosQueryClient cosmosQueryClient = this.CosmosQueryContext.QueryClient; + CosmosQueryClient cosmosQueryClient = cosmosQueryContext.QueryClient; ContainerQueryProperties containerQueryProperties = await cosmosQueryClient.GetCachedContainerQueryPropertiesAsync( - this.CosmosQueryContext.ResourceLink, - this.inputParameters.PartitionKey, + cosmosQueryContext.ResourceLink, + inputParameters.PartitionKey, cancellationToken); - this.CosmosQueryContext.ContainerResourceId = containerQueryProperties.ResourceId; + cosmosQueryContext.ContainerResourceId = containerQueryProperties.ResourceId; PartitionedQueryExecutionInfo partitionedQueryExecutionInfo; - if (this.inputParameters.PartitionedQueryExecutionInfo != null) + if (queryPlanFromContinuationToken != null) { - partitionedQueryExecutionInfo = this.inputParameters.PartitionedQueryExecutionInfo; + partitionedQueryExecutionInfo = queryPlanFromContinuationToken; } else { - if (this.CosmosQueryContext.QueryClient.ByPassQueryParsing()) + if (cosmosQueryContext.QueryClient.ByPassQueryParsing()) { // For non-Windows platforms(like Linux and OSX) in .NET Core SDK, we cannot use ServiceInterop, so need to bypass in that case. // We are also now bypassing this for 32 bit host process running even on Windows as there are many 32 bit apps that will not work without this partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanThroughGatewayAsync( - this.CosmosQueryContext.QueryClient, - this.inputParameters.SqlQuerySpec, - this.CosmosQueryContext.ResourceLink, + cosmosQueryContext.QueryClient, + inputParameters.SqlQuerySpec, + cosmosQueryContext.ResourceLink, cancellationToken); } else @@ -272,9 +128,8 @@ private async Task> TryCreateItemQueryExec //if collection is deleted/created with same name. //need to make it not rely on information from collection cache. Documents.PartitionKeyDefinition partitionKeyDefinition; - object partitionKeyDefinitionObject; - if (this.inputParameters.Properties != null - && this.inputParameters.Properties.TryGetValue(InternalPartitionKeyDefinitionProperty, out partitionKeyDefinitionObject)) + if ((inputParameters.Properties != null) + && inputParameters.Properties.TryGetValue(InternalPartitionKeyDefinitionProperty, out object partitionKeyDefinitionObject)) { if (partitionKeyDefinitionObject is Documents.PartitionKeyDefinition definition) { @@ -293,56 +148,70 @@ private async Task> TryCreateItemQueryExec } partitionedQueryExecutionInfo = await QueryPlanRetriever.GetQueryPlanWithServiceInteropAsync( - this.CosmosQueryContext.QueryClient, - this.inputParameters.SqlQuerySpec, + cosmosQueryContext.QueryClient, + inputParameters.SqlQuerySpec, partitionKeyDefinition, - this.inputParameters.PartitionKey != null, + inputParameters.PartitionKey != null, cancellationToken); } } - this.partitionedQueryExecutionInfo = partitionedQueryExecutionInfo; - - return await this.TryCreateFromPartitionedQuerExecutionInfoAsync( + return await TryCreateFromPartitionedQuerExecutionInfoAsync( partitionedQueryExecutionInfo, containerQueryProperties, + cosmosQueryContext, + inputParameters, cancellationToken); } - public async Task> TryCreateFromPartitionedQuerExecutionInfoAsync( - PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, - ContainerQueryProperties containerQueryProperties, - CancellationToken cancellationToken = default(CancellationToken)) + public static async Task> TryCreateFromPartitionedQuerExecutionInfoAsync( + PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, + ContainerQueryProperties containerQueryProperties, + CosmosQueryContext cosmosQueryContext, + InputParameters inputParameters, + CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); List targetRanges = await CosmosQueryExecutionContextFactory.GetTargetPartitionKeyRangesAsync( - this.CosmosQueryContext.QueryClient, - this.CosmosQueryContext.ResourceLink.OriginalString, + cosmosQueryContext.QueryClient, + cosmosQueryContext.ResourceLink.OriginalString, partitionedQueryExecutionInfo, containerQueryProperties, - this.inputParameters.Properties); + inputParameters.Properties); if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery)) { // We need pass down the rewritten query. - this.inputParameters.SqlQuerySpec = new SqlQuerySpec() + SqlQuerySpec rewrittenQuerySpec = new SqlQuerySpec() { QueryText = partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery, - Parameters = this.inputParameters.SqlQuerySpec.Parameters + Parameters = inputParameters.SqlQuerySpec.Parameters }; + + inputParameters = new InputParameters( + rewrittenQuerySpec, + inputParameters.InitialUserContinuationToken, + inputParameters.MaxConcurrency, + inputParameters.MaxItemCount, + inputParameters.MaxBufferedItemCount, + inputParameters.PartitionKey, + inputParameters.Properties, + inputParameters.PartitionedQueryExecutionInfo, + inputParameters.ExecutionEnvironment, + inputParameters.TestInjections); } return await CosmosQueryExecutionContextFactory.TryCreateSpecializedDocumentQueryExecutionContextAsync( - this.CosmosQueryContext, - this.inputParameters, + cosmosQueryContext, + inputParameters, partitionedQueryExecutionInfo, targetRanges, containerQueryProperties.ResourceId, cancellationToken); } - public static async Task> TryCreateSpecializedDocumentQueryExecutionContextAsync( + private static async Task> TryCreateSpecializedDocumentQueryExecutionContextAsync( CosmosQueryContext cosmosQueryContext, InputParameters inputParameters, PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, @@ -350,27 +219,12 @@ public static async Task> TryCreateSpecial string collectionRid, CancellationToken cancellationToken) { - if (!string.IsNullOrEmpty(partitionedQueryExecutionInfo.QueryInfo?.RewrittenQuery)) - { - inputParameters.SqlQuerySpec = new SqlQuerySpec( - partitionedQueryExecutionInfo.QueryInfo.RewrittenQuery, - inputParameters.SqlQuerySpec.Parameters); - } - - // Figure out the optimal page size. - long initialPageSize = inputParameters.MaxItemCount.GetValueOrDefault(ParallelQueryConfig.GetConfig().ClientInternalPageSize); - - if (initialPageSize < -1 || initialPageSize == 0) - { - return TryCatch.FromException( - new Exception($"Invalid MaxItemCount {initialPageSize}")); - } - QueryInfo queryInfo = partitionedQueryExecutionInfo.QueryInfo; bool getLazyFeedResponse = queryInfo.HasTop; // We need to compute the optimal initial page size for order-by queries + long optimalPageSize = inputParameters.MaxItemCount; if (queryInfo.HasOrderBy) { int top; @@ -378,51 +232,33 @@ public static async Task> TryCreateSpecial { // All partitions should initially fetch about 1/nth of the top value. long pageSizeWithTop = (long)Math.Min( - Math.Ceiling(top / (double)targetRanges.Count) * PageSizeFactorForTop, + Math.Ceiling(top / (double)targetRanges.Count) * CosmosQueryExecutionContextFactory.PageSizeFactorForTop, top); - if (initialPageSize > 0) - { - initialPageSize = Math.Min(pageSizeWithTop, initialPageSize); - } - else - { - initialPageSize = pageSizeWithTop; - } + optimalPageSize = Math.Min(pageSizeWithTop, optimalPageSize); } else if (cosmosQueryContext.IsContinuationExpected) { - if (initialPageSize < 0) - { - if (inputParameters.MaxBufferedItemCount.HasValue) - { - // Max of what the user is willing to buffer and the default (note this is broken if MaxBufferedItemCount = -1) - initialPageSize = Math.Max(inputParameters.MaxBufferedItemCount.Value, ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize); - } - else - { - initialPageSize = ParallelQueryConfig.GetConfig().DefaultMaximumBufferSize; - } - } - - initialPageSize = (long)Math.Min( - Math.Ceiling(initialPageSize / (double)targetRanges.Count) * PageSizeFactorForTop, - initialPageSize); + optimalPageSize = (long)Math.Min( + Math.Ceiling(optimalPageSize / (double)targetRanges.Count) * CosmosQueryExecutionContextFactory.PageSizeFactorForTop, + optimalPageSize); } } - Debug.Assert(initialPageSize > 0 && initialPageSize <= int.MaxValue, - string.Format(CultureInfo.InvariantCulture, "Invalid MaxItemCount {0}", initialPageSize)); + Debug.Assert( + (optimalPageSize > 0) && (optimalPageSize <= int.MaxValue), + $"Invalid MaxItemCount {optimalPageSize}"); CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams initParams = new CosmosCrossPartitionQueryExecutionContext.CrossPartitionInitParams( sqlQuerySpec: inputParameters.SqlQuerySpec, collectionRid: collectionRid, partitionedQueryExecutionInfo: partitionedQueryExecutionInfo, partitionKeyRanges: targetRanges, - initialPageSize: (int)initialPageSize, + initialPageSize: (int)optimalPageSize, maxConcurrency: inputParameters.MaxConcurrency, maxItemCount: inputParameters.MaxItemCount, - maxBufferedItemCount: inputParameters.MaxBufferedItemCount); + maxBufferedItemCount: inputParameters.MaxBufferedItemCount, + testSettings: inputParameters.TestInjections); return await PipelinedDocumentQueryExecutionContext.TryCreateAsync( inputParameters.ExecutionEnvironment, @@ -444,7 +280,7 @@ public static async Task> TryCreateSpecial string resourceLink, PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, ContainerQueryProperties containerQueryProperties, - IDictionary properties) + IReadOnlyDictionary properties) { List targetRanges; if (containerQueryProperties.EffectivePartitionKeyString != null) @@ -473,7 +309,7 @@ public static async Task> TryCreateSpecial } private static bool TryGetEpkProperty( - IDictionary properties, + IReadOnlyDictionary properties, out string effectivePartitionKeyString) { if (properties != null @@ -494,26 +330,73 @@ private static bool TryGetEpkProperty( return false; } - public override void Dispose() + public sealed class InputParameters { - if (this.innerExecutionContext != null && !this.innerExecutionContext.IsDone) + private const int DefaultMaxConcurrency = 0; + private const int DefaultMaxItemCount = 1000; + private const int DefaultMaxBufferedItemCount = 1000; + private const ExecutionEnvironment DefaultExecutionEnvironment = ExecutionEnvironment.Client; + + public InputParameters( + SqlQuerySpec sqlQuerySpec, + string initialUserContinuationToken, + int? maxConcurrency, + int? maxItemCount, + int? maxBufferedItemCount, + PartitionKey? partitionKey, + IReadOnlyDictionary properties, + PartitionedQueryExecutionInfo partitionedQueryExecutionInfo, + ExecutionEnvironment? executionEnvironment, + TestInjections testInjections) { - this.innerExecutionContext.Dispose(); + if (sqlQuerySpec == null) + { + throw new ArgumentNullException(nameof(sqlQuerySpec)); + } + + this.SqlQuerySpec = sqlQuerySpec; + this.InitialUserContinuationToken = initialUserContinuationToken; + + int resolvedMaxConcurrency = maxConcurrency.GetValueOrDefault(InputParameters.DefaultMaxConcurrency); + if (resolvedMaxConcurrency < 0) + { + resolvedMaxConcurrency = int.MaxValue; + } + this.MaxConcurrency = resolvedMaxConcurrency; + + int resolvedMaxItemCount = maxItemCount.GetValueOrDefault(InputParameters.DefaultMaxItemCount); + if (resolvedMaxItemCount < 0) + { + resolvedMaxItemCount = int.MaxValue; + } + this.MaxItemCount = resolvedMaxItemCount; + + int resolvedMaxBufferedItemCount = maxBufferedItemCount.GetValueOrDefault(InputParameters.DefaultMaxBufferedItemCount); + if (resolvedMaxBufferedItemCount < 0) + { + resolvedMaxBufferedItemCount = int.MaxValue; + } + this.MaxBufferedItemCount = resolvedMaxBufferedItemCount; + + this.PartitionKey = partitionKey; + this.Properties = properties; + this.PartitionedQueryExecutionInfo = partitionedQueryExecutionInfo; + + ExecutionEnvironment resolvedExecutionEnvironment = executionEnvironment.GetValueOrDefault(InputParameters.DefaultExecutionEnvironment); + this.ExecutionEnvironment = resolvedExecutionEnvironment; + this.TestInjections = testInjections; } - } - public struct InputParameters - { - internal SqlQuerySpec SqlQuerySpec { get; set; } - internal string InitialUserContinuationToken { get; set; } - internal int? MaxConcurrency { get; set; } - internal int? MaxItemCount { get; set; } - internal int? MaxBufferedItemCount { get; set; } - internal PartitionKey? PartitionKey { get; set; } - internal int? ResponseContinuationTokenLimitInKb { get; set; } - internal IDictionary Properties { get; set; } - internal PartitionedQueryExecutionInfo PartitionedQueryExecutionInfo { get; set; } - internal ExecutionEnvironment ExecutionEnvironment { get; set; } + public SqlQuerySpec SqlQuerySpec { get; } + public string InitialUserContinuationToken { get; } + public int MaxConcurrency { get; } + public int MaxItemCount { get; } + public int MaxBufferedItemCount { get; } + public PartitionKey? PartitionKey { get; } + public IReadOnlyDictionary Properties { get; } + public PartitionedQueryExecutionInfo PartitionedQueryExecutionInfo { get; } + public ExecutionEnvironment ExecutionEnvironment { get; } + public TestInjections TestInjections { get; } } } -} \ No newline at end of file +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextWithNameCacheStaleRetry.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextWithNameCacheStaleRetry.cs new file mode 100644 index 0000000000..ac219c5c8d --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/CosmosQueryExecutionContextWithNameCacheStaleRetry.cs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionContext +{ + using System; + using System.Threading; + using System.Threading.Tasks; + + internal sealed class CosmosQueryExecutionContextWithNameCacheStaleRetry : CosmosQueryExecutionContext + { + private readonly CosmosQueryContext cosmosQueryContext; + private readonly Func cosmosQueryExecutionContextFactory; + private CosmosQueryExecutionContext currentCosmosQueryExecutionContext; + private bool alreadyRetried; + + public CosmosQueryExecutionContextWithNameCacheStaleRetry( + CosmosQueryContext cosmosQueryContext, + Func cosmosQueryExecutionContextFactory) + { + if (cosmosQueryContext == null) + { + throw new ArgumentNullException(nameof(cosmosQueryContext)); + } + + if (cosmosQueryExecutionContextFactory == null) + { + throw new ArgumentNullException(nameof(cosmosQueryExecutionContextFactory)); + } + + this.cosmosQueryContext = cosmosQueryContext; + this.cosmosQueryExecutionContextFactory = cosmosQueryExecutionContextFactory; + this.currentCosmosQueryExecutionContext = cosmosQueryExecutionContextFactory(); + } + + public override bool IsDone => this.currentCosmosQueryExecutionContext.IsDone; + + public override void Dispose() + { + this.currentCosmosQueryExecutionContext.Dispose(); + } + + public override async Task ExecuteNextAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + // If the cache is stale the entire execute context has incorrect values and should be recreated. + // This should only be done for the first execution. + // If results have already been pulled, + // then an error should be returned to the user, + // since it's not possible to combine query results from multiple containers. + QueryResponseCore queryResponse = await this.currentCosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); + if ( + (queryResponse.StatusCode == System.Net.HttpStatusCode.Gone) && + (queryResponse.SubStatusCode == Documents.SubStatusCodes.NameCacheIsStale) && + !this.alreadyRetried) + { + await this.cosmosQueryContext.QueryClient.ForceRefreshCollectionCacheAsync( + this.cosmosQueryContext.ResourceLink.OriginalString, + cancellationToken); + this.alreadyRetried = true; + this.currentCosmosQueryExecutionContext.Dispose(); + this.currentCosmosQueryExecutionContext = this.cosmosQueryExecutionContextFactory(); + return await this.ExecuteNextAsync(cancellationToken); + } + + return queryResponse; + } + + public override bool TryGetContinuationToken(out string continuationToken) + { + return this.currentCosmosQueryExecutionContext.TryGetContinuationToken(out continuationToken); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/LazyCosmosQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/LazyCosmosQueryExecutionContext.cs new file mode 100644 index 0000000000..ab2b2320a1 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/LazyCosmosQueryExecutionContext.cs @@ -0,0 +1,100 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core.ExecutionContext +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Query.Core.Monads; + + /// + /// Implementation of that composes another context and defers it's initialization until the first read. + /// + internal sealed class LazyCosmosQueryExecutionContext : CosmosQueryExecutionContext + { + private readonly AsyncLazy> lazyTryCreateCosmosQueryExecutionContext; + + public LazyCosmosQueryExecutionContext(AsyncLazy> lazyTryCreateCosmosQueryExecutionContext) + { + if (lazyTryCreateCosmosQueryExecutionContext == null) + { + throw new ArgumentNullException(nameof(lazyTryCreateCosmosQueryExecutionContext)); + } + + this.lazyTryCreateCosmosQueryExecutionContext = lazyTryCreateCosmosQueryExecutionContext; + } + + public override bool IsDone + { + get + { + bool isDone; + if (this.lazyTryCreateCosmosQueryExecutionContext.ValueInitialized) + { + TryCatch tryCreateCosmosQueryExecutionContext = this.lazyTryCreateCosmosQueryExecutionContext.Result; + if (tryCreateCosmosQueryExecutionContext.Succeeded) + { + isDone = tryCreateCosmosQueryExecutionContext.Result.IsDone; + } + else + { + isDone = true; + } + } + else + { + isDone = false; + } + + return isDone; + } + } + + public override void Dispose() + { + if (this.lazyTryCreateCosmosQueryExecutionContext.ValueInitialized) + { + TryCatch tryCreateCosmosQueryExecutionContext = this.lazyTryCreateCosmosQueryExecutionContext.Result; + if (tryCreateCosmosQueryExecutionContext.Succeeded) + { + tryCreateCosmosQueryExecutionContext.Result.Dispose(); + } + } + } + + public override async Task ExecuteNextAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + TryCatch tryCreateCosmosQueryExecutionContext = await this.lazyTryCreateCosmosQueryExecutionContext.GetValueAsync(cancellationToken); + if (!tryCreateCosmosQueryExecutionContext.Succeeded) + { + return QueryResponseFactory.CreateFromException(tryCreateCosmosQueryExecutionContext.Exception); + } + + CosmosQueryExecutionContext cosmosQueryExecutionContext = tryCreateCosmosQueryExecutionContext.Result; + QueryResponseCore queryResponseCore = await cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); + return queryResponseCore; + } + + public override bool TryGetContinuationToken(out string state) + { + if (!this.lazyTryCreateCosmosQueryExecutionContext.ValueInitialized) + { + state = null; + return false; + } + + TryCatch tryCreateCosmosQueryExecutionContext = this.lazyTryCreateCosmosQueryExecutionContext.Result; + if (!tryCreateCosmosQueryExecutionContext.Succeeded) + { + state = null; + return false; + } + + return tryCreateCosmosQueryExecutionContext.Result.TryGetContinuationToken(out state); + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/PipelinedDocumentQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/PipelinedDocumentQueryExecutionContext.cs index ef2c3d8c50..6a9d35066f 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/PipelinedDocumentQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ExecutionContext/PipelinedDocumentQueryExecutionContext.cs @@ -153,7 +153,8 @@ public static async Task> TryCreateAsync( initialPageSize: int.MaxValue, maxConcurrency: initParams.MaxConcurrency, maxItemCount: int.MaxValue, - maxBufferedItemCount: initParams.MaxBufferedItemCount); + maxBufferedItemCount: initParams.MaxBufferedItemCount, + testSettings: initParams.TestSettings); } async Task> tryCreateOrderByComponentAsync(string continuationToken) diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducer.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducer.cs index 9d83d636e1..7d0b8ef72a 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducer.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducer.cs @@ -10,6 +10,7 @@ namespace Microsoft.Azure.Cosmos.Query using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Collections.Generic; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; using PartitionKeyRange = Documents.PartitionKeyRange; using PartitionKeyRangeIdentity = Documents.PartitionKeyRangeIdentity; @@ -56,17 +57,13 @@ internal sealed class ItemProducer private readonly SqlQuerySpec querySpecForInit; + private readonly TestInjections testFlags; + /// /// Over the duration of the life time of a document producer the page size will change, since we have an adaptive page size. /// private long pageSize; - /// - /// The current continuation token that the user has read from the document producer tree. - /// This is used for determining whether there are more results. - /// - private string currentContinuationToken; - /// /// The current page that is being enumerated. /// @@ -92,6 +89,8 @@ internal sealed class ItemProducer /// private bool hitException; + private bool enumeratorPrimed; + /// /// Initializes a new instance of the ItemProducer class. /// @@ -100,6 +99,7 @@ internal sealed class ItemProducer /// The partition key range. /// The callback to call once you are done fetching. /// The comparer to use to determine whether the producer has seen a new document. + /// Flags used to help faciliate testing. /// The initial page size. /// The initial continuation token. public ItemProducer( @@ -108,6 +108,7 @@ public ItemProducer( PartitionKeyRange partitionKeyRange, ProduceAsyncCompleteDelegate produceAsyncCompleteCallback, IEqualityComparer equalityComparer, + TestInjections testFlags, long initialPageSize = 50, string initialContinuationToken = null) { @@ -137,7 +138,7 @@ public ItemProducer( this.produceAsyncCompleteCallback = produceAsyncCompleteCallback; this.equalityComparer = equalityComparer; this.pageSize = initialPageSize; - this.currentContinuationToken = initialContinuationToken; + this.CurrentContinuationToken = initialContinuationToken; this.BackendContinuationToken = initialContinuationToken; this.PreviousContinuationToken = initialContinuationToken; if (!string.IsNullOrEmpty(initialContinuationToken)) @@ -149,6 +150,8 @@ public ItemProducer( this.fetchSchedulingMetrics = new SchedulingStopwatch(); this.fetchSchedulingMetrics.Ready(); + this.testFlags = testFlags; + this.HasMoreResults = true; } @@ -177,6 +180,12 @@ public PartitionKeyRange PartitionKeyRange /// public string PreviousContinuationToken { get; private set; } + /// + /// The current continuation token that the user has read from the document producer tree. + /// This is used for determining whether there are more results. + /// + public string CurrentContinuationToken { get; private set; } + /// /// Gets the backend continuation token. /// @@ -226,7 +235,7 @@ public PartitionKeyRange PartitionKeyRange /// /// Gets the current document in this producer. /// - public CosmosElement Current { get; private set; } + public CosmosElement Current => this.CurrentPage?.Current; /// /// A static object representing that the move next operation succeeded, and was able to load the next page @@ -238,26 +247,6 @@ public PartitionKeyRange PartitionKeyRange /// internal static readonly (bool successfullyMovedNext, QueryResponseCore? failureResponse) IsDoneResponse = (false, null); - /// - /// Moves to the next document in the producer. - /// - /// The cancellation token. - /// Whether or not we successfully moved to the next document. - public async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> MoveNextAsync(CancellationToken token) - { - token.ThrowIfCancellationRequested(); - - CosmosElement originalCurrent = this.Current; - (bool successfullyMovedNext, QueryResponseCore? failureResponse) movedNext = await this.MoveNextAsyncImplementationAsync(token); - - if (!movedNext.successfullyMovedNext || (originalCurrent != null && !this.equalityComparer.Equals(originalCurrent, this.Current))) - { - this.IsActive = false; - } - - return movedNext; - } - /// /// Buffers more documents if the producer is empty. /// @@ -304,6 +293,38 @@ public async Task BufferMoreDocumentsAsync(CancellationToken token) schedulingStopwatch: this.fetchSchedulingMetrics, cancellationToken: token); + if ((this.testFlags != null) && this.testFlags.SimulateThrottles) + { + Random random = new Random(); + if (random.Next() % 2 == 0) + { + feedResponse = QueryResponseCore.CreateFailure( + statusCode: (System.Net.HttpStatusCode)429, + subStatusCodes: null, + errorMessage: "Request Rate Too Large", + requestCharge: 0, + activityId: QueryResponseCore.EmptyGuidString, + diagnostics: QueryResponseCore.EmptyDiagnostics); + } + } + + // Can not simulate an empty page on the first page, since we would return a null continuation token, which will end the query early. + if ((this.testFlags != null) && this.testFlags.SimulateEmptyPages && (this.BackendContinuationToken != null)) + { + Random random = new Random(); + if (random.Next() % 2 == 0) + { + feedResponse = QueryResponseCore.CreateSuccess( + result: new List(), + requestCharge: 0, + activityId: QueryResponseCore.EmptyGuidString, + responseLengthBytes: 0, + disallowContinuationTokenMessage: null, + continuationToken: this.BackendContinuationToken, + diagnostics: QueryResponseCore.EmptyDiagnostics); + } + } + this.hasStartedFetching = true; this.ActivityId = Guid.Parse(feedResponse.ActivityId); await this.bufferedPages.AddAsync(feedResponse); @@ -340,127 +361,75 @@ public void Shutdown() this.HasMoreResults = false; } - /// - /// Implementation of move next async. - /// After this function is called the wrapper function determines if a distinct document has been read and updates the 'IsActive' flag. - /// - /// The cancellation token. - /// Whether or not we successfully moved to the next document in the producer. - private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> MoveNextAsyncImplementationAsync(CancellationToken token) + public async Task<(bool movedToNextPage, QueryResponseCore? failureReponse)> TryMoveNextPageAsync(CancellationToken cancellationToken) { - token.ThrowIfCancellationRequested(); - - if (!this.HasMoreResults) + cancellationToken.ThrowIfCancellationRequested(); + if (this.itemsLeftInCurrentPage != 0) { - return ItemProducer.IsDoneResponse; + throw new InvalidOperationException("Tried to move onto the next page before finishing the first page."); } - // Always try reading from current page first - if (this.MoveNextDocumentWithinCurrentPage()) - { - return ItemProducer.IsSuccessResponse; - } - else + // We need to buffer pages if empty, since thats how we know there are no more pages left. + await this.BufferMoreIfEmptyAsync(cancellationToken); + if (this.bufferedPages.Count == 0) { - // We might be at a continuation boundary so we need to move to the next page - (bool successfullyMovedNext, QueryResponseCore? failureResponse) response = await this.TryMoveNextPageAsync(token); - if (!response.successfullyMovedNext) - { - this.HasMoreResults = false; - } - - return response; + this.HasMoreResults = false; + return ItemProducer.IsDoneResponse; } - } - private bool MoveToFirstDocumentInPage() - { - if (this.CurrentPage == null || !this.CurrentPage.MoveNext()) + // Pull a FeedResponse using TryCatch (we could have buffered an exception). + QueryResponseCore queryResponse = await this.bufferedPages.TakeAsync(cancellationToken); + if (!queryResponse.IsSuccess) { - return false; + this.HasMoreResults = false; + return (false, queryResponse); } - this.Current = this.CurrentPage.Current; - this.IsAtBeginningOfPage = true; + // Update the state. + this.PreviousContinuationToken = this.CurrentContinuationToken; + this.CurrentContinuationToken = queryResponse.ContinuationToken; + this.CurrentPage = queryResponse.CosmosElements.GetEnumerator(); + this.itemsLeftInCurrentPage = queryResponse.CosmosElements.Count; + this.enumeratorPrimed = false; + this.IsAtBeginningOfPage = false; - return true; + return ItemProducer.IsSuccessResponse; } - /// - /// Tries to moved to the next document within the current page that we are reading from. - /// - /// Whether the operation was successful. - private bool MoveNextDocumentWithinCurrentPage() + public bool TryMoveNextDocumentWithinPage() { if (this.CurrentPage == null) { return false; } + CosmosElement originalCurrent = this.Current; bool movedNext = this.CurrentPage.MoveNext(); - this.Current = this.CurrentPage.Current; - this.IsAtBeginningOfPage = false; - - Interlocked.Decrement(ref this.bufferedItemCount); - Interlocked.Decrement(ref this.itemsLeftInCurrentPage); - - return movedNext; - } - - /// - /// Tries to the move to the next page in the document producer. - /// - /// The cancellation token. - /// Whether the operation was successful. - private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> TryMoveNextPageAsync(CancellationToken token) - { - token.ThrowIfCancellationRequested(); - if (this.itemsLeftInCurrentPage != 0) + if (!movedNext || ((originalCurrent != null) && !this.equalityComparer.Equals(originalCurrent, this.Current))) { - throw new InvalidOperationException("Tried to move onto the next page before finishing the first page."); + this.IsActive = false; } - // We need to buffer pages if empty, since that how we know there are no more pages left. - await this.BufferMoreIfEmptyAsync(token); - - if (this.bufferedPages.Count == 0) + if (!this.enumeratorPrimed) { - return ItemProducer.IsDoneResponse; + this.IsAtBeginningOfPage = true; + this.enumeratorPrimed = true; } - - // Pull a FeedResponse using TryCatch (we could have buffered an exception). - QueryResponseCore queryResponse = await this.bufferedPages.TakeAsync(token); - - // Update the state. - this.PreviousContinuationToken = this.currentContinuationToken; - this.currentContinuationToken = queryResponse.ContinuationToken; - this.CurrentPage = queryResponse.CosmosElements.GetEnumerator(); - this.IsAtBeginningOfPage = true; - this.itemsLeftInCurrentPage = queryResponse.CosmosElements.Count; - - if (!queryResponse.IsSuccess) + else { - return (false, queryResponse); + Interlocked.Decrement(ref this.bufferedItemCount); + Interlocked.Decrement(ref this.itemsLeftInCurrentPage); + + this.IsAtBeginningOfPage = false; } - // Prime the enumerator, - // so that current is pointing to the first document instead of one before. - if (this.MoveToFirstDocumentInPage()) + if (!movedNext && (this.CurrentContinuationToken == null)) { - this.IsAtBeginningOfPage = true; - return ItemProducer.IsSuccessResponse; + this.HasMoreResults = false; } - else - { - // We got an empty page - if (this.currentContinuationToken != null) - { - return await this.TryMoveNextPageAsync(token); - } - return ItemProducer.IsDoneResponse; - } + return movedNext; } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducerTree.cs b/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducerTree.cs index 97755366c5..ed1a24ef97 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducerTree.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/ItemProducerTree/ItemProducerTree.cs @@ -13,6 +13,7 @@ namespace Microsoft.Azure.Cosmos.Query using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Collections.Generic; using Microsoft.Azure.Cosmos.CosmosElements; + using Microsoft.Azure.Cosmos.Query.Core; /// /// This class is responsible for fetching documents from a partition and all it's descendants, which is modeled as a tree of document producers. @@ -69,6 +70,7 @@ internal sealed class ItemProducerTree : IEnumerable /// Callback to invoke once a fetch finishes. /// Comparer to determine, which tree to produce from. /// Comparer to see if we need to return the continuation token for a partition. + /// Test flags. /// Whether or not to defer fetching the first page. /// The collection to drain from. /// The initial page size. @@ -80,6 +82,7 @@ public ItemProducerTree( ProduceAsyncCompleteDelegate produceAsyncCompleteCallback, IComparer itemProducerTreeComparer, IEqualityComparer equalityComparer, + TestInjections testSettings, bool deferFirstPage, string collectionRid, long initialPageSize = 50, @@ -121,6 +124,7 @@ public ItemProducerTree( partitionKeyRange, (itemsBuffered, resourceUnitUsage, diagnostics, requestLength, token) => produceAsyncCompleteCallback(this, itemsBuffered, resourceUnitUsage, diagnostics, requestLength, token), equalityComparer, + testSettings, initialPageSize, initialContinuationToken); @@ -134,6 +138,7 @@ public ItemProducerTree( produceAsyncCompleteCallback, itemProducerTreeComparer, equalityComparer, + testSettings, deferFirstPage, collectionRid, initialPageSize); @@ -366,47 +371,6 @@ public CosmosElement Current /// private bool HasSplit => this.children.Count != 0; - /// - /// Moves to the next item in the document producer tree. - /// - /// The cancellation token. - /// A task to await on that returns whether we successfully moved next. - /// This function is split proofed. - public async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> MoveNextAsync(CancellationToken token) - { - return await this.ExecuteWithSplitProofingAsync( - function: this.TryMoveNextAsyncImplementationAsync, - functionNeedsBeReexecuted: false, - cancellationToken: token); - } - - /// - /// Moves next only if the producer has not split. - /// This is used to avoid calling move next twice during splits. - /// - /// The cancellation token. - /// A task to await on which in turn returns whether or not we moved next. - public async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> MoveNextIfNotSplitAsync(CancellationToken token) - { - return await this.ExecuteWithSplitProofingAsync( - function: this.TryMoveNextIfNotSplitAsyncImplementationAsync, - functionNeedsBeReexecuted: false, - cancellationToken: token); - } - - /// - /// Buffers more documents in a split proof manner. - /// - /// The cancellation token. - /// A task to await on. - public Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> BufferMoreDocumentsAsync(CancellationToken token) - { - return this.ExecuteWithSplitProofingAsync( - function: this.BufferMoreDocumentsImplementationAsync, - functionNeedsBeReexecuted: true, - cancellationToken: token); - } - /// /// Gets the document producers that need their continuation token return to the user. /// @@ -478,6 +442,7 @@ IEnumerator IEnumerable.GetEnumerator() /// Callback to invoke once a fetch finishes. /// Comparer to determine, which tree to produce from. /// Comparer to see if we need to return the continuation token for a partition. + /// Test flags. /// Whether or not to defer fetching the first page. /// The collection to drain from. /// The initial page size. @@ -488,6 +453,7 @@ IEnumerator IEnumerable.GetEnumerator() ProduceAsyncCompleteDelegate produceAsyncCompleteCallback, IComparer itemProducerTreeComparer, IEqualityComparer equalityComparer, + TestInjections testFlags, bool deferFirstPage, string collectionRid, long initialPageSize = 50) @@ -501,6 +467,7 @@ IEnumerator IEnumerable.GetEnumerator() produceAsyncCompleteCallback, itemProducerTreeComparer, equalityComparer, + testFlags, deferFirstPage, collectionRid, initialPageSize, @@ -518,66 +485,82 @@ private static bool IsSplitException(QueryResponseCore ex) return ex.StatusCode == HttpStatusCode.Gone && ex.SubStatusCode == Documents.SubStatusCodes.PartitionKeyRangeGone; } - /// - /// Implementation for moving to the next item in the document producer tree. - /// - /// The cancellation token. - /// A task with whether or not move next succeeded. - private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> TryMoveNextAsyncImplementationAsync(CancellationToken token) + public void UpdatePriority() { - if (!this.HasMoreResults) + if (this.children.Count != 0) { - return ItemProducer.IsDoneResponse; + this.children.Enqueue(this.children.Dequeue()); } + } + + public async Task<(bool movedToNextPage, QueryResponseCore? failureResponse)> TryMoveNextPageAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return await this.ExecuteWithSplitProofingAsync( + function: this.TryMoveNextPageImplementationAsync, + functionNeedsBeReexecuted: false, + cancellationToken: cancellationToken); + } + + public async Task<(bool movedToNextPage, QueryResponseCore? failureResponse)> TryMoveNextPageIfNotSplitAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return await this.ExecuteWithSplitProofingAsync( + function: this.TryMoveNextPageIfNotSplitAsyncImplementationAsync, + functionNeedsBeReexecuted: false, + cancellationToken: cancellationToken); + } + + public async Task<(bool bufferedMoreDocuments, QueryResponseCore? failureResponse)> BufferMoreDocumentsAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return await this.ExecuteWithSplitProofingAsync( + function: this.BufferMoreDocumentsImplementationAsync, + functionNeedsBeReexecuted: true, + cancellationToken: cancellationToken); + } + public bool TryMoveNextDocumentWithinPage() + { if (this.CurrentItemProducerTree == this) { - return await this.Root.MoveNextAsync(token); + return this.Root.TryMoveNextDocumentWithinPage(); } else { - // Keep track of the current tree - ItemProducerTree itemProducerTree = this.CurrentItemProducerTree; - (bool successfullyMovedNext, QueryResponseCore? failureResponse) response = await itemProducerTree.MoveNextAsync(token); - - // Update the priority queue for the new values - this.children.Enqueue(this.children.Dequeue()); - - // If the current tree is done, but other trees still have a result - // then return true. - if (!response.successfullyMovedNext && - response.failureResponse == null && - this.HasMoreResults) - { - return ItemProducer.IsSuccessResponse; - } + return this.CurrentItemProducerTree.TryMoveNextDocumentWithinPage(); + } + } - return response; + private async Task<(bool succeeded, QueryResponseCore? failureResponse)> TryMoveNextPageImplementationAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (this.CurrentItemProducerTree == this) + { + return await this.Root.TryMoveNextPageAsync(cancellationToken); + } + else + { + return await this.CurrentItemProducerTree.TryMoveNextPageAsync(cancellationToken); } } - /// - /// Implementation for moving next if the tree has not split. - /// - /// The cancellation token. - /// A task to await on which in turn return whether we successfully moved next. - private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> TryMoveNextIfNotSplitAsyncImplementationAsync(CancellationToken token) + private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> TryMoveNextPageIfNotSplitAsyncImplementationAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (this.HasSplit) { return ItemProducer.IsDoneResponse; } - return await this.TryMoveNextAsyncImplementationAsync(token); + return await this.TryMoveNextPageImplementationAsync(cancellationToken); } - /// - /// Implementation for buffering more documents. - /// - /// The cancellation token. - /// A task to await on. - private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> BufferMoreDocumentsImplementationAsync(CancellationToken token) + private async Task<(bool successfullyMovedNext, QueryResponseCore? failureResponse)> BufferMoreDocumentsImplementationAsync(CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + if (this.CurrentItemProducerTree == this) { if (!this.HasMoreBackendResults || this.HasSplit) @@ -586,11 +569,11 @@ private static bool IsSplitException(QueryResponseCore ex) return ItemProducer.IsSuccessResponse; } - await this.Root.BufferMoreDocumentsAsync(token); + await this.Root.BufferMoreDocumentsAsync(cancellationToken); } else { - await this.CurrentItemProducerTree.BufferMoreDocumentsAsync(token); + await this.CurrentItemProducerTree.BufferMoreDocumentsAsync(cancellationToken); } return ItemProducer.IsSuccessResponse; @@ -656,7 +639,25 @@ private static bool IsSplitException(QueryResponseCore ex) if (!this.deferFirstPage) { - await replacementItemProducerTree.MoveNextAsync(cancellationToken); + while (true) + { + (bool movedToNextPage, QueryResponseCore? failureResponse) = await replacementItemProducerTree.TryMoveNextPageAsync(cancellationToken); + if (!movedToNextPage) + { + if (failureResponse.HasValue) + { + return (movedToNextPage, failureResponse); + } + + break; + } + + // Order By requires at least one item in the tree so that the comparator will work. + if (replacementItemProducerTree.TryMoveNextDocumentWithinPage()) + { + break; + } + } } replacementItemProducerTree.Filter = splitItemProducerTree.Root.Filter; @@ -669,10 +670,21 @@ private static bool IsSplitException(QueryResponseCore ex) } } - if (!functionNeedsBeReexecuted) + if (!this.deferFirstPage) { - // We don't want to call move next async again, since we already did when creating the document producers - return ItemProducer.IsSuccessResponse; + (bool, QueryResponseCore?) fakeResponse; + if (splitItemProducerTree.children.Count == 0) + { + // None of the children had results + fakeResponse = ItemProducer.IsDoneResponse; + } + else + { + // We don't want to call move next async again, since we already did when creating the document producers + fakeResponse = ItemProducer.IsSuccessResponse; + } + + return fakeResponse; } } finally diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/Monads/TryCatch.cs b/Microsoft.Azure.Cosmos/src/Query/Core/Monads/TryCatch.cs index e34f056465..d163f7a202 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/Monads/TryCatch.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/Monads/TryCatch.cs @@ -78,14 +78,7 @@ public TryCatch Try( TryCatch matchResult; if (this.Succeeded) { - try - { - matchResult = TryCatch.FromResult(onSuccess(this.either.FromRight(default))); - } - catch (Exception ex) - { - matchResult = TryCatch.FromException(ex); - } + matchResult = TryCatch.FromResult(onSuccess(this.either.FromRight(default))); } else { @@ -101,14 +94,7 @@ public async Task> TryAsync( TryCatch matchResult; if (this.Succeeded) { - try - { - matchResult = TryCatch.FromResult(await onSuccess(this.either.FromRight(default))); - } - catch (Exception ex) - { - matchResult = TryCatch.FromException(ex); - } + matchResult = TryCatch.FromResult(await onSuccess(this.either.FromRight(default))); } else { diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByContinuationToken.cs b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByContinuationToken.cs index 7018949775..a181e021b6 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByContinuationToken.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/OrderByQuery/OrderByContinuationToken.cs @@ -55,14 +55,12 @@ internal sealed class OrderByContinuationToken /// /// Initializes a new instance of the OrderByContinuationToken struct. /// - /// The query client /// The composite continuation token (refer to property documentation). /// The order by items (refer to property documentation). /// The rid (refer to property documentation). /// The skip count (refer to property documentation). /// The filter (refer to property documentation). public OrderByContinuationToken( - CosmosQueryClient queryClient, CompositeContinuationToken compositeContinuationToken, IList orderByItems, string rid, diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/QueryResponseCore.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/QueryResponseCore.cs index c3edc05e11..2e97bc6b61 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/QueryResponseCore.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryClient/QueryResponseCore.cs @@ -5,15 +5,14 @@ namespace Microsoft.Azure.Cosmos.Query { using System; using System.Collections.Generic; - using System.Linq; using System.Net; using Microsoft.Azure.Cosmos.CosmosElements; - using IClientSideRequestStatistics = Documents.IClientSideRequestStatistics; using SubStatusCodes = Documents.SubStatusCodes; internal struct QueryResponseCore { private static readonly IReadOnlyList EmptyList = new List().AsReadOnly(); + internal static readonly string EmptyGuidString = Guid.Empty.ToString(); internal static readonly IReadOnlyCollection EmptyDiagnostics = new List(); private QueryResponseCore( diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPartitionProvider.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPartitionProvider.cs index 40b7f41f5f..a5289fc335 100644 --- a/Microsoft.Azure.Cosmos/src/Query/Core/QueryPartitionProvider.cs +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryPartitionProvider.cs @@ -11,6 +11,7 @@ namespace Microsoft.Azure.Cosmos.Query using System.Runtime.InteropServices; using System.Text; using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Query.Core.Exceptions; using Microsoft.Azure.Cosmos.Query.Core.Monads; using Microsoft.Azure.Cosmos.Routing; using Newtonsoft.Json; @@ -233,18 +234,22 @@ internal TryCatch TryGetPartitionedQueryE Exception exception = Marshal.GetExceptionForHR((int)errorCode); if (exception != null) { - DefaultTrace.TraceInformation("QueryEngineConfiguration: " + this.queryengineConfiguration); - string errorMessage; + QueryPartitionProviderException queryPartitionProviderException; if (string.IsNullOrEmpty(serializedQueryExecutionInfo)) { - errorMessage = $"Message: Query service interop parsing hit an unexpected exception: {exception.ToString()}"; + queryPartitionProviderException = new UnexpectedQueryPartitionProviderException( + "Query service interop parsing hit an unexpected exception", + exception); } else { - errorMessage = "Message: " + serializedQueryExecutionInfo; + queryPartitionProviderException = new ExpectedQueryPartitionProviderException( + serializedQueryExecutionInfo, + exception); } - return TryCatch.FromException(new Exception(errorMessage)); + return TryCatch.FromException( + queryPartitionProviderException); } PartitionedQueryExecutionInfoInternal queryInfoInternal = diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/QueryResponseFactory.cs b/Microsoft.Azure.Cosmos/src/Query/Core/QueryResponseFactory.cs new file mode 100644 index 0000000000..0766f0f852 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/QueryResponseFactory.cs @@ -0,0 +1,105 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + using System; + using Microsoft.Azure.Cosmos.Query.Core.Exceptions; + using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; + + internal static class QueryResponseFactory + { + private static readonly string EmptyGuidString = Guid.Empty.ToString(); + + public static QueryResponseCore CreateFromException(Exception exception) + { + // Get the inner most exception + while (exception.InnerException != null) exception = exception.InnerException; + + QueryResponseCore queryResponseCore; + if (exception is CosmosException cosmosException) + { + queryResponseCore = CreateFromCosmosException(cosmosException); + } + else if (exception is Microsoft.Azure.Documents.DocumentClientException documentClientException) + { + queryResponseCore = CreateFromDocumentClientException(documentClientException); + } + else if (exception is QueryException queryException) + { + CosmosException convertedException = queryException.Accept(QueryExceptionConverter.Singleton); + queryResponseCore = CreateFromCosmosException(convertedException); + } + else + { + // Unknown exception type should become a 500 + queryResponseCore = QueryResponseCore.CreateFailure( + statusCode: System.Net.HttpStatusCode.InternalServerError, + subStatusCodes: null, + errorMessage: exception.ToString(), + requestCharge: 0, + activityId: QueryResponseCore.EmptyGuidString, + diagnostics: QueryResponseCore.EmptyDiagnostics); + } + + return queryResponseCore; + } + + private static QueryResponseCore CreateFromCosmosException(CosmosException cosmosException) + { + QueryResponseCore queryResponseCore = QueryResponseCore.CreateFailure( + statusCode: cosmosException.StatusCode, + subStatusCodes: (Microsoft.Azure.Documents.SubStatusCodes)cosmosException.SubStatusCode, + errorMessage: cosmosException.Message, + requestCharge: 0, + activityId: cosmosException.ActivityId, + diagnostics: QueryResponseCore.EmptyDiagnostics); + + return queryResponseCore; + } + + private static QueryResponseCore CreateFromDocumentClientException(Microsoft.Azure.Documents.DocumentClientException documentClientException) + { + QueryResponseCore queryResponseCore = QueryResponseCore.CreateFailure( + statusCode: documentClientException.StatusCode.GetValueOrDefault(System.Net.HttpStatusCode.InternalServerError), + subStatusCodes: null, + errorMessage: documentClientException.Message, + requestCharge: 0, + activityId: documentClientException.ActivityId, + diagnostics: QueryResponseCore.EmptyDiagnostics); + + return queryResponseCore; + } + + private sealed class QueryExceptionConverter : QueryExceptionVisitor + { + public static readonly QueryExceptionConverter Singleton = new QueryExceptionConverter(); + + private QueryExceptionConverter() + { + } + + public override CosmosException Visit(MalformedContinuationTokenException malformedContinuationTokenException) + { + return new BadRequestException( + $"{nameof(BadRequestException)} due to {nameof(MalformedContinuationTokenException)}", + malformedContinuationTokenException); + } + + public override CosmosException Visit(UnexpectedQueryPartitionProviderException unexpectedQueryPartitionProviderException) + { + return new InternalServerErrorException( + $"{nameof(InternalServerErrorException)} due to {nameof(UnexpectedQueryPartitionProviderException)}", + unexpectedQueryPartitionProviderException); + } + + public override CosmosException Visit(ExpectedQueryPartitionProviderException expectedQueryPartitionProviderException) + { + return new BadRequestException( + $"{nameof(BadRequestException)} due to {nameof(ExpectedQueryPartitionProviderException)}", + expectedQueryPartitionProviderException); + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Query/Core/TestInjections.cs b/Microsoft.Azure.Cosmos/src/Query/Core/TestInjections.cs new file mode 100644 index 0000000000..edc77db439 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Query/Core/TestInjections.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Query.Core +{ + internal sealed class TestInjections + { + public TestInjections(bool simulate429s, bool simulateEmptyPages) + { + this.SimulateThrottles = simulate429s; + this.SimulateEmptyPages = simulateEmptyPages; + } + + public bool SimulateThrottles { get; } + + public bool SimulateEmptyPages { get; } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Query/v2Query/DefaultDocumentQueryExecutionContext.cs b/Microsoft.Azure.Cosmos/src/Query/v2Query/DefaultDocumentQueryExecutionContext.cs index 591a782445..5707497c3d 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v2Query/DefaultDocumentQueryExecutionContext.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v2Query/DefaultDocumentQueryExecutionContext.cs @@ -22,6 +22,8 @@ namespace Microsoft.Azure.Cosmos.Query /// internal sealed class DefaultDocumentQueryExecutionContext : DocumentQueryExecutionContextBase { + private const string InternalPartitionKeyDefinitionProperty = "x-ms-query-partitionkey-definition"; + /// /// Whether or not a continuation is expected. /// @@ -280,8 +282,9 @@ private static bool ServiceInteropAvailable() { FeedOptions feedOptions = this.GetFeedOptions(null); PartitionKeyDefinition partitionKeyDefinition; - object partitionKeyDefinitionObject; - if (feedOptions.Properties != null && feedOptions.Properties.TryGetValue(CosmosQueryExecutionContextFactory.InternalPartitionKeyDefinitionProperty, out partitionKeyDefinitionObject)) + if ((feedOptions.Properties != null) && feedOptions.Properties.TryGetValue( + DefaultDocumentQueryExecutionContext.InternalPartitionKeyDefinitionProperty, + out object partitionKeyDefinitionObject)) { if (partitionKeyDefinitionObject is PartitionKeyDefinition definition) { @@ -299,7 +302,6 @@ private static bool ServiceInteropAvailable() partitionKeyDefinition = collection.PartitionKey; } - QueryInfo queryInfo; providedRanges = PartitionRoutingHelper.GetProvidedPartitionKeyRanges( (errorMessage) => new BadRequestException(errorMessage), this.QuerySpec, @@ -310,7 +312,7 @@ private static bool ServiceInteropAvailable() partitionKeyDefinition, queryPartitionProvider, version, - out queryInfo); + out QueryInfo queryInfo); } else if (request.Properties != null && request.Properties.TryGetValue( WFConstants.BackendHeaders.EffectivePartitionKeyString, @@ -367,7 +369,7 @@ private async Task CreateRequestAsync() INameValueCollection requestHeaders = await this.CreateCommonHeadersAsync( this.GetFeedOptions(this.ContinuationToken)); - requestHeaders[HttpConstants.HttpHeaders.IsContinuationExpected] = isContinuationExpected.ToString(); + requestHeaders[HttpConstants.HttpHeaders.IsContinuationExpected] = this.isContinuationExpected.ToString(); return this.CreateDocumentServiceRequest( requestHeaders, diff --git a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryIterator.cs b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryIterator.cs index 22d82645b2..cba957c25f 100644 --- a/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryIterator.cs +++ b/Microsoft.Azure.Cosmos/src/Query/v3Query/QueryIterator.cs @@ -8,21 +8,32 @@ namespace Microsoft.Azure.Cosmos.Query using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Handlers; + using Microsoft.Azure.Cosmos.Query.Core; + using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext; + using Microsoft.Azure.Cosmos.Query.Core.Monads; internal class QueryIterator : FeedIterator { - private readonly CosmosQueryExecutionContextFactory cosmosQueryExecutionContext; + private readonly CosmosQueryContext cosmosQueryContext; + private readonly CosmosQueryExecutionContext cosmosQueryExecutionContext; private readonly CosmosSerializationFormatOptions cosmosSerializationFormatOptions; private QueryIterator( - CosmosQueryExecutionContextFactory cosmosQueryExecutionContext, + CosmosQueryContext cosmosQueryContext, + CosmosQueryExecutionContext cosmosQueryExecutionContext, CosmosSerializationFormatOptions cosmosSerializationFormatOptions) { + if (cosmosQueryContext == null) + { + throw new ArgumentNullException(nameof(cosmosQueryContext)); + } + if (cosmosQueryExecutionContext == null) { throw new ArgumentNullException(nameof(cosmosQueryExecutionContext)); } + this.cosmosQueryContext = cosmosQueryContext; this.cosmosQueryExecutionContext = cosmosQueryExecutionContext; this.cosmosSerializationFormatOptions = cosmosSerializationFormatOptions; } @@ -42,7 +53,7 @@ public static QueryIterator Create( queryRequestOptions = new QueryRequestOptions(); } - CosmosQueryContext context = new CosmosQueryContextCore( + CosmosQueryContext cosmosQueryContext = new CosmosQueryContextCore( client: client, queryRequestOptions: queryRequestOptions, resourceTypeEnum: Documents.ResourceType.Document, @@ -53,24 +64,21 @@ public static QueryIterator Create( allowNonValueAggregateQuery: allowNonValueAggregateQuery, correlatedActivityId: Guid.NewGuid()); - CosmosQueryExecutionContextFactory.InputParameters inputParams = new CosmosQueryExecutionContextFactory.InputParameters() - { - SqlQuerySpec = sqlQuerySpec, - InitialUserContinuationToken = continuationToken, - MaxBufferedItemCount = queryRequestOptions.MaxBufferedItemCount, - MaxConcurrency = queryRequestOptions.MaxConcurrency, - MaxItemCount = queryRequestOptions.MaxItemCount, - PartitionKey = queryRequestOptions.PartitionKey, - Properties = queryRequestOptions.Properties, - PartitionedQueryExecutionInfo = partitionedQueryExecutionInfo, - ExecutionEnvironment = queryRequestOptions.ExecutionEnvironment.GetValueOrDefault(Core.ExecutionContext.ExecutionEnvironment.Client), - ResponseContinuationTokenLimitInKb = queryRequestOptions.ResponseContinuationTokenLimitInKb, - }; + CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters( + sqlQuerySpec: sqlQuerySpec, + initialUserContinuationToken: continuationToken, + maxConcurrency: queryRequestOptions.MaxConcurrency, + maxItemCount: queryRequestOptions.MaxItemCount, + maxBufferedItemCount: queryRequestOptions.MaxBufferedItemCount, + partitionKey: queryRequestOptions.PartitionKey, + properties: queryRequestOptions.Properties, + partitionedQueryExecutionInfo: partitionedQueryExecutionInfo, + executionEnvironment: queryRequestOptions.ExecutionEnvironment, + testInjections: queryRequestOptions.TestSettings); return new QueryIterator( - new CosmosQueryExecutionContextFactory( - cosmosQueryContext: context, - inputParameters: inputParams), + cosmosQueryContext, + CosmosQueryExecutionContextFactory.Create(cosmosQueryContext, inputParameters), queryRequestOptions.CosmosSerializationFormatOptions); } @@ -79,73 +87,51 @@ public static QueryIterator Create( public override async Task ReadNextAsync(CancellationToken cancellationToken = default(CancellationToken)) { // This catches exception thrown by the pipeline and converts it to QueryResponse - ResponseMessage response; - try + QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); + CosmosQueryContext cosmosQueryContext = this.cosmosQueryContext; + QueryAggregateDiagnostics diagnostics = new QueryAggregateDiagnostics(responseCore.Diagnostics); + QueryResponse queryResponse; + if (responseCore.IsSuccess) { - QueryResponseCore responseCore = await this.cosmosQueryExecutionContext.ExecuteNextAsync(cancellationToken); - CosmosQueryContext cosmosQueryContext = this.cosmosQueryExecutionContext.CosmosQueryContext; - QueryAggregateDiagnostics diagnostics = new QueryAggregateDiagnostics(responseCore.Diagnostics); - QueryResponse queryResponse; - if (responseCore.IsSuccess) - { - queryResponse = QueryResponse.CreateSuccess( - result: responseCore.CosmosElements, - count: responseCore.CosmosElements.Count, - responseLengthBytes: responseCore.ResponseLengthBytes, - diagnostics: diagnostics, - responseHeaders: new CosmosQueryResponseMessageHeaders( - responseCore.ContinuationToken, - responseCore.DisallowContinuationTokenMessage, - cosmosQueryContext.ResourceTypeEnum, - cosmosQueryContext.ContainerResourceId) - { - RequestCharge = responseCore.RequestCharge, - ActivityId = responseCore.ActivityId, - SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown - }); - } - else - { - queryResponse = QueryResponse.CreateFailure( - statusCode: responseCore.StatusCode, - error: null, - errorMessage: responseCore.ErrorMessage, - requestMessage: null, - diagnostics: diagnostics, - responseHeaders: new CosmosQueryResponseMessageHeaders( - responseCore.ContinuationToken, - responseCore.DisallowContinuationTokenMessage, - cosmosQueryContext.ResourceTypeEnum, - cosmosQueryContext.ContainerResourceId) - { - RequestCharge = responseCore.RequestCharge, - ActivityId = responseCore.ActivityId, - SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown - }); - } - - queryResponse.CosmosSerializationOptions = this.cosmosSerializationFormatOptions; - - response = queryResponse; + queryResponse = QueryResponse.CreateSuccess( + result: responseCore.CosmosElements, + count: responseCore.CosmosElements.Count, + responseLengthBytes: responseCore.ResponseLengthBytes, + diagnostics: diagnostics, + responseHeaders: new CosmosQueryResponseMessageHeaders( + responseCore.ContinuationToken, + responseCore.DisallowContinuationTokenMessage, + cosmosQueryContext.ResourceTypeEnum, + cosmosQueryContext.ContainerResourceId) + { + RequestCharge = responseCore.RequestCharge, + ActivityId = responseCore.ActivityId, + SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown + }); } - catch (Documents.DocumentClientException exception) + else { - response = exception.ToCosmosResponseMessage(requestMessage: null); - } - catch (CosmosException exception) - { - response = exception.ToCosmosResponseMessage(request: null); - } - catch (AggregateException ae) - { - response = TransportHandler.AggregateExceptionConverter(ae, null); - if (response == null) - { - throw; - } + queryResponse = QueryResponse.CreateFailure( + statusCode: responseCore.StatusCode, + error: null, + errorMessage: responseCore.ErrorMessage, + requestMessage: null, + diagnostics: diagnostics, + responseHeaders: new CosmosQueryResponseMessageHeaders( + responseCore.ContinuationToken, + responseCore.DisallowContinuationTokenMessage, + cosmosQueryContext.ResourceTypeEnum, + cosmosQueryContext.ContainerResourceId) + { + RequestCharge = responseCore.RequestCharge, + ActivityId = responseCore.ActivityId, + SubStatusCode = responseCore.SubStatusCode ?? Documents.SubStatusCodes.Unknown + }); } - return response; + queryResponse.CosmosSerializationOptions = this.cosmosSerializationFormatOptions; + + return queryResponse; } internal bool TryGetContinuationToken(out string state) diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs index 8487899b4d..951d68bdf7 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/QueryRequestOptions.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos { using System; using System.Globalization; + using Microsoft.Azure.Cosmos.Query.Core; using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext; using Microsoft.Azure.Documents; @@ -144,6 +145,8 @@ public ConsistencyLevel? ConsistencyLevel internal ExecutionEnvironment? ExecutionEnvironment { get; set; } + internal TestInjections TestSettings { get; set; } + /// /// Fill the CosmosRequestMessage headers with the set properties /// diff --git a/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs b/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs index 58f911cb88..10d5d4c03f 100644 --- a/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs +++ b/Microsoft.Azure.Cosmos/src/RequestOptions/RequestOptions.cs @@ -13,7 +13,7 @@ namespace Microsoft.Azure.Cosmos /// public class RequestOptions { - internal IDictionary Properties { get; set; } + internal Dictionary Properties { get; set; } /// /// Gets or sets the If-Match (ETag) associated with the request in the Azure Cosmos DB service. diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs index 83a79c7281..05240ed67c 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosException.cs @@ -17,8 +17,9 @@ public class CosmosException : Exception internal CosmosException( HttpStatusCode statusCode, string message, - Error error = null) - : base(message) + Error error = null, + Exception inner = null) + : base(message, inner) { this.StatusCode = statusCode; this.Error = error; diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/BadRequestException.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/BadRequestException.cs new file mode 100644 index 0000000000..c7646d369b --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/BadRequestException.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Resource.CosmosExceptions +{ + using System; + using System.Net; + + internal sealed class BadRequestException : CosmosHttpException + { + public BadRequestException() + : base(statusCode: HttpStatusCode.BadRequest, message: null) + { + } + + public BadRequestException(string message) + : base(statusCode: HttpStatusCode.BadRequest, message: message) + { + } + + public BadRequestException(string message, Exception innerException) + : base(statusCode: HttpStatusCode.BadRequest, message: message, innerException: innerException) + { + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosHttpException.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosHttpException.cs new file mode 100644 index 0000000000..1b1a76b486 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/CosmosHttpException.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Resource.CosmosExceptions +{ + using System; + using System.Net; + + internal abstract class CosmosHttpException : CosmosException + { + protected CosmosHttpException(HttpStatusCode statusCode) + : this(statusCode, message: null, innerException: null) + { + } + + protected CosmosHttpException(HttpStatusCode statusCode, string message) + : this(statusCode, message: message, innerException: null) + { + } + + protected CosmosHttpException(HttpStatusCode statusCode, string message, Exception innerException) + : base(statusCode: statusCode, message: message, inner: innerException) + { + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/InternalServerErrorException.cs b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/InternalServerErrorException.cs new file mode 100644 index 0000000000..4d4ceb9122 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/Resource/CosmosExceptions/InternalServerErrorException.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Resource.CosmosExceptions +{ + using System; + using System.Net; + + internal sealed class InternalServerErrorException : CosmosHttpException + { + public InternalServerErrorException() + : base(statusCode: HttpStatusCode.InternalServerError, message: null) + { + } + + public InternalServerErrorException(string message) + : base(statusCode: HttpStatusCode.InternalServerError, message: message) + { + } + + public InternalServerErrorException(string message, Exception innerException) + : base(statusCode: HttpStatusCode.InternalServerError, message: message, innerException: innerException) + { + } + } +} diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/AccountProperties.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/AccountProperties.cs index b2e424af56..534ad9221a 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/AccountProperties.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/AccountProperties.cs @@ -49,7 +49,7 @@ internal AccountProperties() /// /// /// Every resource within an Azure Cosmos DB database account needs to have a unique identifier. - /// Unlike , which is set internally, this Id is settable by the user and is not immutable. + /// Unlike , which is set internally, this Id is settable by the user and is not immutable. /// /// /// When working with document resources, they too have this settable Id property. diff --git a/Microsoft.Azure.Cosmos/src/Resource/Settings/ContainerProperties.cs b/Microsoft.Azure.Cosmos/src/Resource/Settings/ContainerProperties.cs index 0bfb634916..6a82222298 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Settings/ContainerProperties.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Settings/ContainerProperties.cs @@ -142,7 +142,7 @@ public ConflictResolutionPolicy ConflictResolutionPolicy /// /// /// Every resource within an Azure Cosmos DB database account needs to have a unique identifier. - /// Unlike , which is set internally, this Id is settable by the user and is not immutable. + /// Unlike , which is set internally, this Id is settable by the user and is not immutable. /// /// /// When working with document resources, they too have this settable Id property. diff --git a/Microsoft.Azure.Cosmos/src/StoredProcedureResponse.cs b/Microsoft.Azure.Cosmos/src/StoredProcedureResponse.cs index be67e8e9fa..776ee179a1 100644 --- a/Microsoft.Azure.Cosmos/src/StoredProcedureResponse.cs +++ b/Microsoft.Azure.Cosmos/src/StoredProcedureResponse.cs @@ -47,11 +47,11 @@ internal StoredProcedureResponse(DocumentServiceResponse response, JsonSerialize // load resource if (typeof(TValue) == typeof(Document) || typeof(Document).IsAssignableFrom(typeof(TValue))) { - this.responseBody = Resource.LoadFromWithConstructor(response.ResponseBody, () => (TValue)(object)new Document(), this.serializerSettings); + this.responseBody = Documents.Resource.LoadFromWithConstructor(response.ResponseBody, () => (TValue)(object)new Document(), this.serializerSettings); } else if (typeof(TValue) == typeof(Attachment) || typeof(Attachment).IsAssignableFrom(typeof(TValue))) { - this.responseBody = Resource.LoadFromWithConstructor(response.ResponseBody, () => (TValue)(object)new Attachment(), this.serializerSettings); + this.responseBody = Documents.Resource.LoadFromWithConstructor(response.ResponseBody, () => (TValue)(object)new Attachment(), this.serializerSettings); } else { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateAvg.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateAvg.xml index 7f7dc7638b..e3eadfca37 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateAvg.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateAvg.xml @@ -111,7 +111,7 @@ FROM ( OFFSET 90 LIMIT 2147483647 ) AS r0 ]]> - + @@ -128,7 +128,7 @@ FROM ( OFFSET 90 LIMIT 5 ) AS r0 ]]> - + @@ -145,7 +145,7 @@ FROM ( OFFSET 5 LIMIT 5 ) AS r0 ]]> - + @@ -167,7 +167,7 @@ FROM ( OFFSET 10 LIMIT 20 ) AS r1 ]]> - + @@ -253,7 +253,7 @@ FROM ( ) AS r8 ) AS r9 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateCount.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateCount.xml index c3bafa35f0..f793c36fbf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateCount.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateCount.xml @@ -108,7 +108,7 @@ FROM ( OFFSET 90 LIMIT 2147483647 ) AS r0 ]]> - + @@ -125,7 +125,7 @@ FROM ( OFFSET 90 LIMIT 5 ) AS r0 ]]> - + @@ -142,7 +142,7 @@ FROM ( OFFSET 5 LIMIT 5 ) AS r0 ]]> - + @@ -164,7 +164,7 @@ FROM ( OFFSET 10 LIMIT 20 ) AS r1 ]]> - + @@ -215,7 +215,7 @@ FROM ( OFFSET 1 LIMIT 10 ) AS r5 ]]> - + @@ -291,7 +291,7 @@ FROM ( ) AS r8 WHERE ((r8["v0"] + r8["v1"]) > (r8["v2"] + r8["v3"])) ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMax.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMax.xml index fc6747b17b..f9d1840e63 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMax.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMax.xml @@ -105,7 +105,7 @@ FROM ( OFFSET 90 LIMIT 2147483647 ) AS r0 ]]> - + @@ -122,7 +122,7 @@ FROM ( OFFSET 90 LIMIT 5 ) AS r0 ]]> - + @@ -139,7 +139,7 @@ FROM ( OFFSET 5 LIMIT 5 ) AS r0 ]]> - + @@ -161,7 +161,7 @@ FROM ( OFFSET 10 LIMIT 20 ) AS r1 ]]> - + @@ -243,7 +243,7 @@ FROM ( OFFSET 1 LIMIT 10 ) AS r8 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMin.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMin.xml index 2cdb9a47cd..f68c73bfdc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMin.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateMin.xml @@ -105,7 +105,7 @@ FROM ( OFFSET 90 LIMIT 2147483647 ) AS r0 ]]> - + @@ -122,7 +122,7 @@ FROM ( OFFSET 90 LIMIT 5 ) AS r0 ]]> - + @@ -139,7 +139,7 @@ FROM ( OFFSET 5 LIMIT 5 ) AS r0 ]]> - + @@ -161,7 +161,7 @@ FROM ( OFFSET 10 LIMIT 20 ) AS r1 ]]> - + @@ -243,7 +243,7 @@ FROM ( OFFSET 1 LIMIT 10 ) AS r8 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateSum.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateSum.xml index 9d01fa1218..c679dd6d03 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateSum.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAggregateSum.xml @@ -104,7 +104,7 @@ FROM ( OFFSET 90 LIMIT 2147483647 ) AS r0 ]]> - + @@ -121,7 +121,7 @@ FROM ( OFFSET 90 LIMIT 5 ) AS r0 ]]> - + @@ -138,7 +138,7 @@ FROM ( OFFSET 5 LIMIT 5 ) AS r0 ]]> - + @@ -160,7 +160,7 @@ FROM ( OFFSET 10 LIMIT 20 ) AS r1 ]]> - + @@ -232,7 +232,7 @@ FROM ( OFFSET 1 LIMIT 10 ) AS r8 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAny.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAny.xml index a6963db2c4..9b7f90aaa8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAny.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqAggregateFunctionBaselineTests.TestAny.xml @@ -12,7 +12,7 @@ FROM ( FROM root ) AS v0 ]]> - + @@ -29,7 +29,7 @@ FROM ( WHERE root["Flag"] ) AS v0 ]]> - + @@ -46,7 +46,7 @@ FROM ( WHERE (NOT root["Flag"]) ) AS v0 ]]> - + @@ -62,7 +62,7 @@ FROM ( FROM root ) AS v0 ]]> - + @@ -80,7 +80,7 @@ FROM ( WHERE ((m0 % 3) = 0) ) AS v0 ]]> - + @@ -97,7 +97,7 @@ FROM ( WHERE root["Flag"] ) AS v0 ]]> - + @@ -114,7 +114,7 @@ FROM ( WHERE (root["Number"] < -7) ) AS v0 ]]> - + @@ -131,7 +131,7 @@ FROM ( WHERE (root["Number"] < -13) ) AS v0 ]]> - + @@ -156,7 +156,7 @@ FROM ( WHERE (v1 > 5) ) AS v2 ]]> - + @@ -189,7 +189,7 @@ FROM ( WHERE (v1 > 150) ) AS v2 ]]> - + @@ -206,7 +206,7 @@ FROM ( OFFSET 20 LIMIT 1 ) AS v2 ]]> - + @@ -224,7 +224,7 @@ FROM ( OFFSET 4 LIMIT 2147483647 ) AS v2 ]]> - + @@ -311,7 +311,7 @@ FROM ( OFFSET 1 LIMIT 10 ) AS v26 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestDistinctTranslation.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestDistinctTranslation.xml index f0b0377995..e4b9d8c1f7 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestDistinctTranslation.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestDistinctTranslation.xml @@ -148,7 +148,7 @@ FROM ( FROM root ) AS r0 ]]> - + @@ -189,7 +189,7 @@ FROM ( ) AS r1 ORDER BY r1 ASC ]]> - + @@ -206,7 +206,7 @@ FROM ( ORDER BY root["Int"] ASC ) AS r1 ]]> - + @@ -414,7 +414,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -434,7 +434,7 @@ JOIN ( WHERE ((LENGTH(v2["FamilyName"]) > 10) AND (LENGTH(v2["FamilyName"]) < 20)) ORDER BY v2 ASC ]]> - + @@ -454,7 +454,7 @@ JOIN ( WHERE ((LENGTH(v2["FamilyName"]) > 10) AND (LENGTH(v2["FamilyName"]) < 20)) ORDER BY v2 ASC ]]> - + @@ -477,7 +477,7 @@ FROM ( ORDER BY v2 ASC ) AS r0 ]]> - + @@ -497,7 +497,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -520,7 +520,7 @@ FROM ( ORDER BY v2 ASC ) AS r0 ]]> - + @@ -560,7 +560,7 @@ FROM ( WHERE (LENGTH(v2["FamilyName"]) > 10) ) AS r0 ]]> - + @@ -580,7 +580,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -600,7 +600,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -620,7 +620,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2["GivenName"]["Length"] ASC ]]> - + @@ -640,7 +640,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2["GivenName"]["Length"] ASC ]]> - + @@ -660,7 +660,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -680,7 +680,7 @@ JOIN ( WHERE (LENGTH(v2["FamilyName"]) > 10) ORDER BY v2 ASC ]]> - + @@ -717,7 +717,7 @@ JOIN ( ) AS v2 ORDER BY v2["FamilyName"] ASC ]]> - + @@ -736,7 +736,7 @@ JOIN ( ) AS v2 ORDER BY v2["FamilyName"] ASC ]]> - + @@ -806,7 +806,7 @@ JOIN ( ) AS v1 ORDER BY v1 ASC ]]> - + @@ -826,7 +826,7 @@ JOIN ( WHERE (LENGTH(v1) > 10) ORDER BY v1 ASC ]]> - + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSelectMany.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSelectMany.xml index 4f3622150b..a37660fcd8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSelectMany.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSelectMany.xml @@ -236,7 +236,7 @@ JOIN ( WHERE (LENGTH(r0["FamilyName"]) > 10) ) AS v2 ]]> - + @@ -259,7 +259,7 @@ JOIN ( WHERE (LENGTH(r0["FamilyName"]) > 10) ) AS v1 ]]> - + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSimpleSubquery.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSimpleSubquery.xml index 448e7b317c..a98ec2f823 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSimpleSubquery.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSimpleSubquery.xml @@ -37,7 +37,7 @@ FROM ( ) AS r1 ORDER BY LENGTH(r1) ASC ]]> - + @@ -55,7 +55,7 @@ FROM ( ) AS r1 ORDER BY ARRAY_LENGTH(r1["Parents"]) ASC ]]> - + @@ -73,7 +73,7 @@ FROM ( ) AS r0 ORDER BY ARRAY_LENGTH(r0["Parents"]) ASC ]]> - + @@ -90,7 +90,7 @@ FROM ( ) AS r0 ORDER BY r0["FamilyId"] ASC ]]> - + @@ -115,7 +115,7 @@ FROM ( ) AS r2 WHERE (LENGTH(r2["FamilyId"]) > 10) ]]> - + @@ -139,7 +139,7 @@ FROM ( ) AS r2 WHERE (LENGTH(r2["f"]["FamilyId"]) > 10) ]]> - + @@ -163,7 +163,7 @@ FROM ( ) AS r2 WHERE (ARRAY_LENGTH(r2["f"]["Parents"]) > 0) ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSubquery.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSubquery.xml index da9dd839b7..3c7a1b8afc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSubquery.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestSubquery.xml @@ -36,7 +36,7 @@ JOIN ( ) ) AS v0 ]]> - + @@ -56,7 +56,7 @@ JOIN ( ) ) AS v2 ]]> - + @@ -188,7 +188,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -207,7 +207,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -227,7 +227,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -285,7 +285,7 @@ JOIN ( ) ) AS v1 ]]> - + @@ -345,7 +345,7 @@ JOIN ( ) ) AS v1 ]]> - + @@ -366,7 +366,7 @@ JOIN ( ) ) AS v1 ]]> - + @@ -387,7 +387,7 @@ JOIN ( ) ) AS v1 ]]> - + @@ -428,7 +428,7 @@ JOIN ( ) ) AS v1 ]]> - + @@ -452,7 +452,7 @@ JOIN ( ) ) AS v2 ]]> - + @@ -533,7 +533,7 @@ JOIN ( ) AS v1 WHERE (v1 > 0) ]]> - + @@ -555,7 +555,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -575,7 +575,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -594,7 +594,7 @@ JOIN ( ) AS v1 ORDER BY v1 ASC ]]> - + @@ -617,7 +617,7 @@ JOIN ( ) AS v1 ORDER BY v1 ASC ]]> - + @@ -1023,7 +1023,7 @@ JOIN ( ) AS v1 WHERE (v0 >= 2) ]]> - + @@ -1096,7 +1096,7 @@ JOIN ( ) AS v0 ORDER BY LOG(v0) ASC ]]> - + @@ -1145,7 +1145,7 @@ FROM ( WHERE ((r0["FamilyId"] > "ABC") AND (ARRAY_LENGTH(r0["SmartChildren"]) > 0)) ORDER BY r0["ChildrenCount"] ASC ]]> - + @@ -1198,7 +1198,7 @@ JOIN ( ) AS v6 WHERE (r0["FirstChild"] != null) ]]> - + @@ -1359,7 +1359,7 @@ JOIN ( WHERE (v0 AND root["IsRegistered"]) ORDER BY v1 ASC ]]> - + @@ -1465,7 +1465,7 @@ JOIN ( ) AS v3 WHERE (ARRAY_LENGTH(root["Children"]) > 0) ]]> - + @@ -1972,7 +1972,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC ]]> - + @@ -1990,7 +1990,7 @@ JOIN ( JOIN v0 IN root["Children"] ) AS v2 ]]> - + @@ -2009,7 +2009,7 @@ JOIN ( ORDER BY c0["Grade"] ASC ) AS v0 ]]> - + @@ -2028,7 +2028,7 @@ JOIN ( WHERE (LENGTH(c0["FamilyName"]) > 10) ) AS v1 ]]> - + @@ -2047,7 +2047,7 @@ JOIN ( WHERE (LENGTH(c0["FamilyName"]) > 10) ) AS v1 ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestThenByTranslation.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestThenByTranslation.xml index aa7e2abb6d..1705c5db5b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestThenByTranslation.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqGeneralBaselineTests.TestThenByTranslation.xml @@ -64,7 +64,7 @@ JOIN ( ) AS v0 ORDER BY v0 ASC, root["Int"] DESC ]]> - + @@ -84,7 +84,7 @@ JOIN ( ) AS v0 ORDER BY root["FamilyId"] ASC, v0 DESC ]]> - + @@ -110,7 +110,7 @@ JOIN ( ) AS v1 ORDER BY v0 DESC, v1 ASC ]]> - + @@ -123,7 +123,7 @@ ORDER BY v0 DESC, v1 ASC SELECT VALUE root FROM root ORDER BY root["FamilyId"] ASC, root["FamilyId"] ASC ]]> - + @@ -136,7 +136,7 @@ ORDER BY root["FamilyId"] ASC, root["FamilyId"] ASC ]]> SELECT VALUE root FROM root ORDER BY root["FamilyId"] ASC, root["FamilyId"] DESC ]]> - + @@ -162,7 +162,7 @@ JOIN ( ) AS v1 ORDER BY v0 DESC, v1 ASC ]]> - + @@ -175,7 +175,7 @@ ORDER BY v0 DESC, v1 ASC SELECT VALUE root FROM root ORDER BY (root["Int"] * 2) ASC, SUBSTRING(root["FamilyId"], 2, 3) ASC ]]> - + @@ -188,7 +188,7 @@ ORDER BY (root["Int"] * 2) ASC, SUBSTRING(root["FamilyId"], 2, 3) ASC ]]> - + @@ -209,7 +209,7 @@ JOIN ( ) AS v0 ORDER BY (root["IsRegistered"] ? root["FamilyId"] : root["Int"]) DESC, (v0[0] % 1000) ASC ]]> - + @@ -259,7 +259,7 @@ JOIN ( ) AS v1 ORDER BY v0 ASC, v1 ASC ]]> - + @@ -285,7 +285,7 @@ JOIN ( ) AS v1 ORDER BY root["FamilyId"] ASC, v0 DESC, v1 ASC ]]> - + @@ -307,7 +307,7 @@ JOIN ( ) AS v0 ORDER BY root["FamilyId"] ASC, v0 DESC ]]> - + @@ -337,7 +337,7 @@ JOIN ( ) AS v1 ORDER BY v0 ASC, v1 DESC ]]> - + @@ -359,7 +359,7 @@ JOIN ( ) AS v1 ORDER BY root["FamilyId"] ASC, v1 DESC ]]> - + @@ -393,7 +393,7 @@ JOIN ( ) AS v2 ORDER BY v1 ASC, v2 DESC ]]> - + @@ -415,7 +415,7 @@ FROM ( ) AS r1 ORDER BY r1["FamilyId"] ASC, r1["FamilyNumber"] ASC ]]> - + @@ -443,7 +443,7 @@ JOIN ( ORDER BY r0["FamilyId"] ASC, r0["FamilyNumber"] ASC ) AS v1 ]]> - + @@ -467,7 +467,7 @@ FROM ( ) AS r0 ORDER BY r0["FamilyId"] ASC, r0["FamilyNumber"] ASC ]]> - + @@ -497,7 +497,7 @@ FROM ( ) AS r0 ORDER BY ARRAY_LENGTH(r0["ChildrenWithPets"]) DESC, r0["TotalExpenses"] DESC ]]> - + @@ -536,7 +536,7 @@ FROM ( ) AS r2 ORDER BY r2["GoodChildrenCount"] DESC, r2["UniquePetsNameCount"] ASC ]]> - + @@ -564,7 +564,7 @@ FROM root JOIN f0 IN root["Children"] WHERE (ARRAY_LENGTH(root["Children"]) > 0) ORDER BY f0["Grade"] ASC, ARRAY_LENGTH(f0["Pets"]) DESC ]]> - + @@ -594,7 +594,7 @@ FROM ( WHERE (v2 > 0) ) AS r1 ]]> - + @@ -618,7 +618,7 @@ FROM ( ) AS r2 ORDER BY r2["Length"] ASC ]]> - + @@ -633,7 +633,7 @@ FROM root JOIN f0 IN root["Records"]["Transactions"] WHERE (ARRAY_LENGTH(root["Children"]) > 0) ORDER BY f0["Type"] ASC, f0["Amount"] ASC ]]> - + @@ -650,7 +650,7 @@ FROM ( ) AS r0 ORDER BY r0["IsRegistered"] ASC, r0["Int"] DESC ]]> - + @@ -668,7 +668,7 @@ FROM ( ORDER BY r0["IsRegistered"] ASC, r0["Int"] DESC OFFSET 5 LIMIT 2147483647 ]]> - + @@ -705,7 +705,7 @@ FROM ( ) AS r0 ORDER BY r0["Name"] DESC, r0["PetWithLongNames"] ASC ]]> - + @@ -735,7 +735,7 @@ JOIN ( ) AS v1 ORDER BY v0 ASC, v1 DESC ]]> - + @@ -775,7 +775,7 @@ JOIN ( ) AS v3 ORDER BY v0[0] ASC, v1[0] DESC, v2 ASC, v3[0] DESC ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml index 6e6736418c..1442223b30 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqSQLTranslationBaselineTest.ValidateSQLTranslation.xml @@ -169,7 +169,7 @@ FROM root ]]> SELECT VALUE [1, 2, 3][root["x"]] FROM root WHERE ((root["x"] >= 0) AND (root["x"] < 3)) ]]> - + @@ -226,7 +226,7 @@ FROM root ]]> SELECT VALUE [1, 2, 3][root["x"]] FROM root WHERE ((root["x"] >= 0) AND (root["x"] < 3)) ]]> - + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectManyWithFilters.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectManyWithFilters.xml index 065095bf41..1fa6ec3921 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectManyWithFilters.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectManyWithFilters.xml @@ -107,7 +107,7 @@ FROM ( JOIN number0 IN r0["EnumerableField"] WHERE (number0 > 10) ]]> - + \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectTop.xml b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectTop.xml index 9de33b7ad8..15f6b92930 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectTop.xml +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestSelectTop.xml @@ -84,7 +84,7 @@ FROM ( ) AS r0 WHERE (r0["NumericField"] > 100) ]]> - + @@ -101,7 +101,7 @@ FROM ( ) AS r0 WHERE (r0["NumericField"] > 100) ]]> - + @@ -118,7 +118,7 @@ FROM ( ) AS r0 WHERE (r0 > 100) ]]> - + @@ -135,7 +135,7 @@ FROM ( ) AS r0 WHERE (r0 > 100) ]]> - + @@ -153,7 +153,7 @@ FROM ( WHERE (r0["NumericField"] > 100) ORDER BY r0["NumericField"] DESC ]]> - + @@ -203,7 +203,7 @@ FROM ( ) AS r0 WHERE (r0["NumericField"] > 100) ]]> - + diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosGatewayTimeoutTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosGatewayTimeoutTests.cs new file mode 100644 index 0000000000..d34607cc34 --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosGatewayTimeoutTests.cs @@ -0,0 +1,72 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests +{ + + using System; + using System.Collections.Generic; + using System.Net.Http; + using System.Net.Http.Headers; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Fluent; + using Microsoft.Azure.Documents; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class CosmosGatewayTimeoutTests + { + [TestMethod] + public async Task GatewayStoreClientTimeout() + { + using (CosmosClient client = TestCommon.CreateCosmosClient(useGateway: true)) + { + // Creates the store clients in the document client + await client.DocumentClient.EnsureValidClientAsync(); + + // Get the GatewayStoreModel + GatewayStoreModel gatewayStore; + using (DocumentServiceRequest serviceRequest = new DocumentServiceRequest( + operationType: OperationType.Read, + resourceIdOrFullName: null, + resourceType: ResourceType.Database, + body: null, + headers: null, + isNameBased: false, + authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey)) + { + serviceRequest.UseGatewayMode = true; + gatewayStore = (GatewayStoreModel)client.DocumentClient.GetStoreProxy(serviceRequest); + } + + // Get the GatewayStoreClient + FieldInfo gatewayStoreClientProperty = gatewayStore.GetType().GetField("gatewayStoreClient", BindingFlags.NonPublic | BindingFlags.Instance); + GatewayStoreClient storeClient = (GatewayStoreClient)gatewayStoreClientProperty.GetValue(gatewayStore); + + // Set the http request timeout to 10 ms to cause a timeout exception + FieldInfo httpClientProperty = storeClient.GetType().GetField("httpClient", BindingFlags.NonPublic | BindingFlags.Instance); + HttpClient gatewayStoreHttpClient = (HttpClient)httpClientProperty.GetValue(storeClient); + gatewayStoreHttpClient.Timeout = TimeSpan.FromMilliseconds(1); + + // Verify the failure has the required info + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + try + { + await client.CreateDatabaseAsync("TestGatewayTimeoutDb" + Guid.NewGuid().ToString()); + Assert.Fail("Operation should have timed out:" + gatewayStoreHttpClient.Timeout); + } + catch (CosmosException rte) + { + string message = rte.ToString(); + Assert.IsTrue(message.Contains("Start Time"), "Start Time:" + message); + Assert.IsTrue(message.Contains("Total Duration"), "Total Duration:" + message); + Assert.IsTrue(message.Contains("Http Client Timeout"), "Http Client Timeout:" + message); + Assert.IsTrue(message.Contains("Activity id"), "Activity id:" + message); + } + } + } + } +} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs index 9e212e9fed..b470c75941 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemTests.cs @@ -17,6 +17,7 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Json; using Microsoft.Azure.Cosmos.Query; + using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -667,8 +668,6 @@ public async Task ItemStreamContractVerifier() requestOptions: options); FeedIterators.Add(queryIterator); - string previousResult = null; - foreach (FeedIterator iterator in FeedIterators) { int count = 0; @@ -702,15 +701,6 @@ public async Task ItemStreamContractVerifier() Assert.IsNotNull(item["_attachments"]); Assert.IsNotNull(item["_ts"]); } - - if (previousResult != null) - { - Assert.AreEqual(previousResult, jObject.ToString()); - } - else - { - previousResult = jObject.ToString(); ; - } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosNotFoundTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosNotFoundTests.cs index 2b1ae5bf1a..1874d0d0d8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosNotFoundTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosNotFoundTests.cs @@ -52,27 +52,46 @@ public async Task ValidateQueryNotFoundResponse() await container.DeleteContainerAsync(); - FeedIterator crossPartitionQueryIterator = container.GetItemQueryStreamIterator( - "select * from t where true", - requestOptions: new QueryRequestOptions() { MaxConcurrency = 2 }); + { + // Querying after delete should be a gone exception even after the retry. + FeedIterator crossPartitionQueryIterator = container.GetItemQueryStreamIterator( + "select * from t where true", + requestOptions: new QueryRequestOptions() { MaxConcurrency = 2 }); + + await this.VerifyQueryNotFoundResponse(crossPartitionQueryIterator); + } - await this.VerifyQueryNotFoundResponse(crossPartitionQueryIterator); + { + // Also try with partition key. + FeedIterator queryIterator = container.GetItemQueryStreamIterator( + "select * from t where true", + requestOptions: new QueryRequestOptions() + { + MaxConcurrency = 1, + PartitionKey = new Cosmos.PartitionKey("testpk"), + }); - FeedIterator queryIterator = container.GetItemQueryStreamIterator( - "select * from t where true", - requestOptions: new QueryRequestOptions() - { - MaxConcurrency = 1, - PartitionKey = new Cosmos.PartitionKey("testpk"), - }); + await this.VerifyQueryNotFoundResponse(queryIterator); + } - await this.VerifyQueryNotFoundResponse(queryIterator); + { + // Recreate the collection with the same name on a different client. + CosmosClient newClient = TestCommon.CreateCosmosClient(); + Database db2 = newClient.GetDatabase(db.Id); + Container container2 = await db2.CreateContainerAsync( + id: container.Id, + partitionKeyPath: "/pk", + throughput: 500); + await container2.CreateItemAsync(randomItem); - FeedIterator crossPartitionQueryIterator2 = container.GetItemQueryStreamIterator( - "select * from t where true", - requestOptions: new QueryRequestOptions() { MaxConcurrency = 2 }); + FeedIterator queryIterator = container.GetItemQueryStreamIterator( + "select * from t where true", + requestOptions: new QueryRequestOptions() { MaxConcurrency = 2 }); - await this.VerifyQueryNotFoundResponse(crossPartitionQueryIterator2); + ResponseMessage response = await queryIterator.ReadNextAsync(); + Assert.IsNotNull(response); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + } await db.DeleteAsync(); } @@ -144,14 +163,10 @@ private async Task ItemOperations(Container container, bool containerNotExist) private async Task VerifyQueryNotFoundResponse(FeedIterator iterator) { - // Verify that even if the user ignores the HasMoreResults it still returns the exception - for(int i = 0; i < 3; i++) - { - ResponseMessage response = await iterator.ReadNextAsync(); - Assert.IsNotNull(response); - Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); - Assert.IsFalse(iterator.HasMoreResults); - } + ResponseMessage response = await iterator.ReadNextAsync(); + Assert.IsNotNull(response); + Assert.AreEqual(HttpStatusCode.NotFound, response.StatusCode); + Assert.IsFalse(iterator.HasMoreResults); } private void VerifyNotFoundResponse(ResponseMessage response) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs index 4b0fda6834..cd11beea5b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CrossPartitionQueryTests.cs @@ -20,7 +20,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using System.Xml; using Microsoft.Azure.Cosmos.CosmosElements; using Microsoft.Azure.Cosmos.Query.Core; - using Microsoft.Azure.Cosmos.Query.Core.ExecutionComponent; using Microsoft.Azure.Cosmos.Routing; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Routing; @@ -30,7 +29,6 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Query; - using Query.ParallelQuery; using UInt128 = Query.Core.UInt128; /// @@ -609,7 +607,7 @@ private static async Task> QueryWithTryGetContinuationTokens( } List resultsFromTryGetContinuationToken = new List(); - string state = null; + string continuationToken = null; do { QueryRequestOptions computeRequestOptions = queryRequestOptions.Clone(); @@ -618,21 +616,34 @@ private static async Task> QueryWithTryGetContinuationTokens( FeedIteratorCore itemQuery = (FeedIteratorCore)container.GetItemQueryIterator( queryText: query, requestOptions: computeRequestOptions, - continuationToken: state); - - FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); - if (queryRequestOptions.MaxItemCount.HasValue) + continuationToken: continuationToken); + try { + FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); + if (queryRequestOptions.MaxItemCount.HasValue) + { + Assert.IsTrue( + cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, + "Max Item Count is not being honored"); + } + + resultsFromTryGetContinuationToken.AddRange(cosmosQueryResponse); Assert.IsTrue( - cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); + itemQuery.TryGetContinuationToken(out continuationToken), + "Failed to get state for query"); + if (continuationToken != null) + { + Assert.IsTrue(continuationToken.All((character) => character < 128)); + } } - - resultsFromTryGetContinuationToken.AddRange(cosmosQueryResponse); - Assert.IsTrue( - itemQuery.TryGetContinuationToken(out state), - "Failed to get state for query"); - } while (state != null); + catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) + { + itemQuery = (FeedIteratorCore)container.GetItemQueryIterator( + queryText: query, + requestOptions: queryRequestOptions, + continuationToken: continuationToken); + } + } while (continuationToken != null); return resultsFromTryGetContinuationToken; } @@ -656,16 +667,34 @@ private static async Task> QueryWithContinuationTokens( requestOptions: queryRequestOptions, continuationToken: continuationToken); - FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); - if (queryRequestOptions.MaxItemCount.HasValue) + while (true) { - Assert.IsTrue( - cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); - } + try + { + FeedResponse cosmosQueryResponse = await itemQuery.ReadNextAsync(); + if (queryRequestOptions.MaxItemCount.HasValue) + { + Assert.IsTrue( + cosmosQueryResponse.Count <= queryRequestOptions.MaxItemCount.Value, + "Max Item Count is not being honored"); + } - resultsFromContinuationToken.AddRange(cosmosQueryResponse); - continuationToken = cosmosQueryResponse.ContinuationToken; + resultsFromContinuationToken.AddRange(cosmosQueryResponse); + continuationToken = cosmosQueryResponse.ContinuationToken; + if (continuationToken != null) + { + Assert.IsTrue(continuationToken.All((character) => character < 128)); + } + break; + } + catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) + { + itemQuery = container.GetItemQueryIterator( + queryText: query, + requestOptions: queryRequestOptions, + continuationToken: continuationToken); + } + } } while (continuationToken != null); return resultsFromContinuationToken; @@ -686,16 +715,47 @@ private static async Task> QueryWithoutContinuationTokens( queryText: query, requestOptions: queryRequestOptions); + string continuationTokenForRetries = null; while (itemQuery.HasMoreResults) { - FeedResponse page = await itemQuery.ReadNextAsync(); - results.AddRange(page); + try + { + FeedResponse page = await itemQuery.ReadNextAsync(); + results.AddRange(page); - if (queryRequestOptions.MaxItemCount.HasValue) + if (queryRequestOptions.MaxItemCount.HasValue) + { + Assert.IsTrue( + page.Count <= queryRequestOptions.MaxItemCount.Value, + "Max Item Count is not being honored"); + } + + try + { + continuationTokenForRetries = page.ContinuationToken; + } + catch (Exception) + { + // Grabbing a continuation token is not supported on all queries. + } + + if (continuationTokenForRetries != null) + { + Assert.IsTrue(continuationTokenForRetries.All((character) => character < 128)); + } + } + catch (CosmosException cosmosException) when (cosmosException.StatusCode == (HttpStatusCode)429) { - Assert.IsTrue( - page.Count <= queryRequestOptions.MaxItemCount.Value, - "Max Item Count is not being honored"); + itemQuery = container.GetItemQueryIterator( + queryText: query, + requestOptions: queryRequestOptions, + continuationToken: continuationTokenForRetries); + + if (continuationTokenForRetries == null) + { + // The query failed and we don't have a save point, so just restart the whole thing. + results = new List(); + } } } @@ -737,7 +797,6 @@ public void TestContinuationTokenSerialization() byte[] bytes = Encoding.UTF8.GetBytes(orderByItemSerialized); OrderByItem orderByItem = new OrderByItem(CosmosElement.CreateFromBuffer(bytes)); OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( - new Mock().Object, compositeContinuationToken, new List { orderByItem }, "asdf", @@ -781,7 +840,7 @@ private async Task TestBadQueriesOverMultiplePartitionsHelper(Container containe } catch (CosmosException exception) when (exception.StatusCode == HttpStatusCode.BadRequest) { - Assert.IsTrue(exception.Message.StartsWith("Response status code does not indicate success: 400 Substatus: 0 Reason: (Message: {\"errors\":[{\"severity\":\"Error\",\"location\":{\"start\":27,\"end\":28},\"code\":\"SC2001\",\"message\":\"Identifier 'a' could not be resolved.\"}]}"), + Assert.IsTrue(exception.Message.StartsWith(@"Response status code does not indicate success: 400 Substatus: 0 Reason: ({""errors"":[{""severity"":""Error"",""location"":{""start"":27,""end"":28},""code"":""SC2001"",""message"":""Identifier 'a' could not be resolved.""}]})."), exception.Message); } } @@ -1069,7 +1128,7 @@ private async Task TestQueryWithSpecialPartitionKeysHelper(Container container, SpecialPropertyDocument specialPropertyDocument = new SpecialPropertyDocument { - id = Guid.NewGuid().ToString() + Id = Guid.NewGuid().ToString() }; specialPropertyDocument.GetType().GetProperty(args.Name).SetValue(specialPropertyDocument, args.Value); @@ -1080,7 +1139,7 @@ private async Task TestQueryWithSpecialPartitionKeysHelper(Container container, Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); PartitionKey key = new PartitionKey(args.ValueToPartitionKey(args.Value)); - response = await container.ReadItemAsync(response.Resource.id, new Cosmos.PartitionKey(key)); + response = await container.ReadItemAsync(response.Resource.Id, new Cosmos.PartitionKey(key)); returnedDoc = response.Resource; Assert.AreEqual(args.Value, getPropertyValueFunction((SpecialPropertyDocument)returnedDoc)); @@ -1129,7 +1188,7 @@ private async Task TestQueryWithSpecialPartitionKeysHelper(Container container, private sealed class SpecialPropertyDocument { - public string id + public string Id { get; set; @@ -1323,7 +1382,7 @@ private async Task TestBasicCrossPartitionQueryHelper( { foreach (int maxItemCount in new int[] { 10, 100 }) { - foreach (string query in new string[] { "SELECT * FROM c", "SELECT * FROM c ORDER BY c._ts" }) + foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) { QueryRequestOptions feedOptions = new QueryRequestOptions { @@ -1346,6 +1405,94 @@ private async Task TestBasicCrossPartitionQueryHelper( } } + [TestMethod] + public async Task TestExceptionlessFailures() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, + documents, + this.TestExceptionlessFailuresHelper); + } + + private async Task TestExceptionlessFailuresHelper( + Container container, + IEnumerable documents) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) + { + QueryRequestOptions feedOptions = new QueryRequestOptions + { + MaxBufferedItemCount = 7000, + MaxConcurrency = 2, + MaxItemCount = maxItemCount, + TestSettings = new TestInjections(simulate429s: true, simulateEmptyPages: false) + }; + + List queryResults = await CrossPartitionQueryTests.RunQuery( + container, + query, + feedOptions); + + Assert.AreEqual( + documents.Count(), + queryResults.Count, + $"query: {query} failed with {nameof(maxItemCount)}: {maxItemCount}"); + } + } + } + + [TestMethod] + public async Task TestEmptyPages() + { + int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; + uint numberOfDocuments = 100; + QueryOracle.QueryOracleUtil util = new QueryOracle.QueryOracle2(seed); + IEnumerable documents = util.GetDocuments(numberOfDocuments); + + await this.CreateIngestQueryDelete( + ConnectionModes.Direct, + CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, + documents, + this.TestEmptyPagesHelper); + } + + private async Task TestEmptyPagesHelper( + Container container, + IEnumerable documents) + { + foreach (int maxItemCount in new int[] { 10, 100 }) + { + foreach (string query in new string[] { "SELECT c.id FROM c", "SELECT c._ts, c.id FROM c ORDER BY c._ts" }) + { + QueryRequestOptions feedOptions = new QueryRequestOptions + { + MaxBufferedItemCount = 7000, + MaxConcurrency = 2, + MaxItemCount = maxItemCount, + TestSettings = new TestInjections(simulate429s: false, simulateEmptyPages: true) + }; + + List queryResults = await CrossPartitionQueryTests.RunQuery( + container, + query, + feedOptions); + + Assert.AreEqual( + documents.Count(), + queryResults.Count, + $"query: {query} failed with {nameof(maxItemCount)}: {maxItemCount}"); + } + } + } + [TestMethod] public async Task TestQueryPlanGatewayAndServiceInterop() { @@ -3223,7 +3370,7 @@ await this.CreateIngestQueryDelete( public async Task TestQueryCrossPartitionOffsetLimit() { int seed = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds; - uint numberOfDocuments = 100; + uint numberOfDocuments = 10; Random rand = new Random(seed); List people = new List(); @@ -3248,7 +3395,7 @@ public async Task TestQueryCrossPartitionOffsetLimit() } await this.CreateIngestQueryDelete( - ConnectionModes.Direct | ConnectionModes.Gateway, + ConnectionModes.Direct, CollectionTypes.SinglePartition | CollectionTypes.MultiPartition, documents, this.TestQueryCrossPartitionOffsetLimit, @@ -3259,11 +3406,11 @@ private async Task TestQueryCrossPartitionOffsetLimit( Container container, IEnumerable documents) { - foreach (int offsetCount in new int[] { 0, 1, 10, 100, documents.Count() }) + foreach (int offsetCount in new int[] { 0, 1, 10, documents.Count() }) { - foreach (int limitCount in new int[] { 0, 1, 10, 100, documents.Count() }) + foreach (int limitCount in new int[] { 0, 1, 10, documents.Count() }) { - foreach (int pageSize in new int[] { 1, 10, 100, documents.Count() }) + foreach (int pageSize in new int[] { 1, 10, documents.Count() }) { string query = $@" SELECT VALUE c.guid @@ -4602,9 +4749,9 @@ private static async Task> RunQueryCombinations( Assert.IsTrue( queryDrainingModeAsJTokens1.SequenceEqual(queryDrainingModeAsJTokens2, JsonTokenEqualityComparer.Value), - $"{query} returned different results." + - $"{queryDrainingMode1}: {JsonConvert.SerializeObject(queryDrainingModeAsJTokens1)}" + - $"{queryDrainingMode2}: {JsonConvert.SerializeObject(queryDrainingModeAsJTokens2)}"); + $"{query} returned different results.\n" + + $"{queryDrainingMode1}: {JsonConvert.SerializeObject(queryDrainingModeAsJTokens1)}\n" + + $"{queryDrainingMode2}: {JsonConvert.SerializeObject(queryDrainingModeAsJTokens2)}\n"); } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTestsCommon.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTestsCommon.cs index f53dce323c..38f6a94949 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTestsCommon.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/LinqTestsCommon.cs @@ -45,7 +45,7 @@ private static bool NestedListsSequenceEqual(IEnumerable queryResults, IEnumerab { IEnumerator queryIter, dataIter; for (queryIter = queryResults.GetEnumerator(), dataIter = dataResults.GetEnumerator(); - queryIter.MoveNext() && dataIter.MoveNext(); ) + queryIter.MoveNext() && dataIter.MoveNext();) { IEnumerable queryEnumerable = queryIter.Current as IEnumerable; IEnumerable dataEnumerable = dataIter.Current as IEnumerable; @@ -58,7 +58,7 @@ private static bool NestedListsSequenceEqual(IEnumerable queryResults, IEnumerab else if (queryEnumerable == null || dataEnumerable == null) { return false; - } + } else { @@ -166,7 +166,7 @@ public static void ValidateResults(IQueryable queryResults, IQueryable dataResul // execution validation IEnumerator queryEnumerator = queryResults.GetEnumerator(); List queryResultsList = new List(); - while (queryEnumerator.MoveNext()) + while (queryEnumerator.MoveNext()) { queryResultsList.Add(queryEnumerator.Current); } @@ -296,7 +296,7 @@ public static Func> GenerateTestCosmosData(Func query = container.GetItemLinqQueryable(allowSynchronousQueryExecution : true); + IOrderedQueryable query = container.GetItemLinqQueryable(allowSynchronousQueryExecution: true); // To cover both query against backend and queries on the original data using LINQ nicely, // the LINQ expression should be written once and they should be compiled and executed against the two sources. @@ -483,7 +483,7 @@ Cosmos.Database cosmosDatabase } FeedOptions feedOptions = new FeedOptions() { EnableScanInQuery = true, EnableCrossPartitionQuery = true }; - IOrderedQueryable query = container.GetItemLinqQueryable(allowSynchronousQueryExecution : true); + IOrderedQueryable query = container.GetItemLinqQueryable(allowSynchronousQueryExecution: true); // To cover both query against backend and queries on the original data using LINQ nicely, // the LINQ expression should be written once and they should be compiled and executed against the two sources. @@ -515,16 +515,16 @@ public static LinqTestOutput ExecuteTest(LinqTestInput input) } catch (Exception e) { - while (!(e is DocumentClientException) && e.InnerException != null) + while (e.InnerException != null) e = e.InnerException; + + string message; + if (e is CosmosException cosmosException) { - e = e.InnerException; + message = $"Status Code: {cosmosException.StatusCode}"; } - - string message = null; - DocumentClientException dce = e as DocumentClientException; - if (dce != null) + else if(e is DocumentClientException documentClientException) { - message = dce.RawErrorMessage; + message = documentClientException.RawErrorMessage; } else { @@ -555,7 +555,7 @@ public override string ToString() public override bool Equals(object obj) { - if (!(obj is LinqTestObject && + if (!(obj is LinqTestObject && obj.GetType().IsAssignableFrom(this.GetType()) && this.GetType().IsAssignableFrom(obj.GetType()))) return false; if (obj == null) return false; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosQueryUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosQueryUnitTests.cs index 9a116550a9..5f6269c060 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosQueryUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosQueryUnitTests.cs @@ -7,11 +7,13 @@ namespace Microsoft.Azure.Cosmos.Tests using System; using System.Collections.Generic; using System.IO; + using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Query; using Microsoft.Azure.Cosmos.Query.Core.ExecutionComponent; + using Microsoft.Azure.Cosmos.Query.Core.ExecutionContext; using Microsoft.Azure.Cosmos.Query.Core.Monads; using Microsoft.Azure.Documents; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -90,7 +92,6 @@ public async Task TestCosmosQueryExecutionComponentCancellation() } [TestMethod] - [ExpectedException(typeof(CosmosException))] public async Task TestCosmosQueryPartitionKeyDefinition() { PartitionKeyDefinition partitionKeyDefinition = new PartitionKeyDefinition(); @@ -109,6 +110,7 @@ public async Task TestCosmosQueryPartitionKeyDefinition() CancellationToken cancellationtoken = cancellationTokenSource.Token; Mock client = new Mock(); + string exceptionMessage = "Verified that the PartitionKeyDefinition was correctly set. Cancel the rest of the query"; client .Setup(x => x.GetCachedContainerQueryPropertiesAsync(It.IsAny(), It.IsAny(), cancellationtoken)) .ReturnsAsync(new ContainerQueryProperties("mockContainer", null, partitionKeyDefinition)); @@ -126,18 +128,19 @@ public async Task TestCosmosQueryPartitionKeyDefinition() It.IsAny())) .ReturnsAsync(TryCatch.FromException( new InvalidOperationException( - "Verified that the PartitionKeyDefinition was correctly set. Cancel the rest of the query"))); - - CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters() - { - SqlQuerySpec = sqlQuerySpec, - InitialUserContinuationToken = null, - MaxBufferedItemCount = queryRequestOptions?.MaxBufferedItemCount, - MaxConcurrency = queryRequestOptions?.MaxConcurrency, - MaxItemCount = queryRequestOptions?.MaxItemCount, - PartitionKey = queryRequestOptions?.PartitionKey, - Properties = queryRequestOptions?.Properties - }; + exceptionMessage))); + + CosmosQueryExecutionContextFactory.InputParameters inputParameters = new CosmosQueryExecutionContextFactory.InputParameters( + sqlQuerySpec: sqlQuerySpec, + initialUserContinuationToken: null, + maxConcurrency: queryRequestOptions?.MaxConcurrency, + maxItemCount: queryRequestOptions?.MaxItemCount, + maxBufferedItemCount: queryRequestOptions?.MaxBufferedItemCount, + partitionKey: queryRequestOptions?.PartitionKey, + properties: queryRequestOptions?.Properties, + partitionedQueryExecutionInfo: null, + executionEnvironment: queryRequestOptions?.ExecutionEnvironment, + testInjections: queryRequestOptions?.TestSettings); CosmosQueryContext cosmosQueryContext = new CosmosQueryContextCore( client: client.Object, @@ -150,11 +153,13 @@ public async Task TestCosmosQueryPartitionKeyDefinition() allowNonValueAggregateQuery: allowNonValueAggregateQuery, correlatedActivityId: new Guid("221FC86C-1825-4284-B10E-A6029652CCA6")); - CosmosQueryExecutionContextFactory factory = new CosmosQueryExecutionContextFactory( - cosmosQueryContext: cosmosQueryContext, - inputParameters: inputParameters); + CosmosQueryExecutionContext context = CosmosQueryExecutionContextFactory.Create( + cosmosQueryContext, + inputParameters); - await factory.ExecuteNextAsync(cancellationtoken); + QueryResponseCore queryResponse = await context.ExecuteNextAsync(cancellationtoken); + Assert.AreEqual(HttpStatusCode.BadRequest, queryResponse.StatusCode); + Assert.IsTrue(queryResponse.ErrorMessage.Contains(exceptionMessage)); } private async Task<(IList components, QueryResponseCore response)> GetAllExecutionComponents() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTests.cs index 1ca8dc0fbf..de568ad822 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTests.cs @@ -53,12 +53,15 @@ public async Task TestMoveNextAsync(int pageSize, int maxPageSize) Assert.IsTrue(itemProducer.HasMoreResults); - while ((await itemProducer.MoveNextAsync(this.cancellationToken)).successfullyMovedNext) + while ((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage) { - Assert.IsTrue(itemProducer.HasMoreResults); - string jsonValue = itemProducer.Current.ToString(); - ToDoItem item = JsonConvert.DeserializeObject(jsonValue); - itemsRead.Add(item); + while (itemProducer.TryMoveNextDocumentWithinPage()) + { + Assert.IsTrue(itemProducer.HasMoreResults); + string jsonValue = itemProducer.Current.ToString(); + ToDoItem item = JsonConvert.DeserializeObject(jsonValue); + itemsRead.Add(item); + } } Assert.IsFalse(itemProducer.HasMoreResults); @@ -159,13 +162,13 @@ public async Task ConcurrentMoveNextAndBufferMore() // BufferMore // Fire and Forget this task. #pragma warning disable 4014 - Task bufferTask = Task.Run(()=> itemProducer.BufferMoreDocumentsAsync(this.cancellationToken)); + Task bufferTask = Task.Run(() => itemProducer.BufferMoreDocumentsAsync(this.cancellationToken)); // Verify the task started int waitCount = 0; - while(callBackCount == 0) + while (callBackCount == 0) { - if(waitCount++ > 100) + if (waitCount++ > 100) { Assert.Fail("The task never started to buffer the items. The callback was never called"); } @@ -211,7 +214,14 @@ public async Task ConcurrentMoveNextAndBufferMore() itemsToRead = pageSizes[2] + pageSizes[3]; #pragma warning disable 4014 - Task moveNext = Task.Run(() => itemProducer.MoveNextAsync(this.cancellationToken)); + Task moveNext = Task.Run(async () => + { + if (!itemProducer.TryMoveNextDocumentWithinPage()) + { + Assert.IsTrue((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage); + Assert.IsTrue(itemProducer.TryMoveNextDocumentWithinPage()); + } + }); #pragma warning restore 4014 while (callBackCount == 2) { @@ -240,8 +250,12 @@ private async Task> ReadItemProducer(ItemProducer itemProducer, i List itemsRead = new List(); for (int i = 0; i < numItems; i++) { - (bool successfullyMovedNext, QueryResponseCore? failureResponse) movedNext = await itemProducer.MoveNextAsync(this.cancellationToken); - Assert.IsTrue(movedNext.successfullyMovedNext); + if (!itemProducer.TryMoveNextDocumentWithinPage()) + { + Assert.IsTrue((await itemProducer.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage); + Assert.IsTrue(itemProducer.TryMoveNextDocumentWithinPage()); + } + Assert.IsTrue(itemProducer.HasMoreResults); itemsRead.Add(this.ConvertCosmosElement(itemProducer.Current)); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTreeUnitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTreeUnitTests.cs index d690a6f739..a766843d45 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTreeUnitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/ItemProducerTreeUnitTests.cs @@ -54,6 +54,7 @@ public async Task TestMoveNextWithEmptyPagesAsync(string initialContinuationToke produceAsyncCompleteCallback: MockItemProducerFactory.DefaultTreeProduceAsyncCompleteDelegate, itemProducerTreeComparer: new ParallelItemProducerTreeComparer(), equalityComparer: CosmosElementEqualityComparer.Value, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false), deferFirstPage: true, collectionRid: MockQueryFactory.DefaultCollectionRid, initialPageSize: maxPageSize, @@ -62,12 +63,15 @@ public async Task TestMoveNextWithEmptyPagesAsync(string initialContinuationToke Assert.IsTrue(itemProducerTree.HasMoreResults); List itemsRead = new List(); - while ((await itemProducerTree.MoveNextAsync(this.cancellationToken)).successfullyMovedNext) + while ((await itemProducerTree.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage) { - Assert.IsTrue(itemProducerTree.HasMoreResults); - string jsonValue = itemProducerTree.Current.ToString(); - ToDoItem item = JsonConvert.DeserializeObject(jsonValue); - itemsRead.Add(item); + while (itemProducerTree.TryMoveNextDocumentWithinPage()) + { + Assert.IsTrue(itemProducerTree.HasMoreResults); + string jsonValue = itemProducerTree.Current.ToString(); + ToDoItem item = JsonConvert.DeserializeObject(jsonValue); + itemsRead.Add(item); + } } Assert.IsFalse(itemProducerTree.HasMoreResults); @@ -109,6 +113,7 @@ public async Task TestMoveNextWithEmptyPagesAndSplitAsync(string initialContinua MockItemProducerFactory.DefaultTreeProduceAsyncCompleteDelegate, new ParallelItemProducerTreeComparer(), CosmosElementEqualityComparer.Value, + new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false), true, MockQueryFactory.DefaultCollectionRid, maxPageSize, @@ -117,15 +122,20 @@ public async Task TestMoveNextWithEmptyPagesAndSplitAsync(string initialContinua Assert.IsTrue(itemProducerTree.HasMoreResults); List itemsRead = new List(); - while ((await itemProducerTree.MoveNextAsync(this.cancellationToken)).successfullyMovedNext) + while ((await itemProducerTree.TryMoveNextPageAsync(this.cancellationToken)).movedToNextPage) { - Assert.IsTrue(itemProducerTree.HasMoreResults); - if (itemProducerTree.Current != null) + while (itemProducerTree.TryMoveNextDocumentWithinPage()) { - string jsonValue = itemProducerTree.Current.ToString(); - ToDoItem item = JsonConvert.DeserializeObject(jsonValue); - itemsRead.Add(item); + Assert.IsTrue(itemProducerTree.HasMoreResults); + if (itemProducerTree.Current != null) + { + string jsonValue = itemProducerTree.Current.ToString(); + ToDoItem item = JsonConvert.DeserializeObject(jsonValue); + itemsRead.Add(item); + } } + + itemProducerTree.UpdatePriority(); } Assert.IsFalse(itemProducerTree.HasMoreResults); @@ -207,6 +217,7 @@ public async Task TestItemProducerTreeWithFailure() produceAsyncCompleteCallback: produceAsyncCompleteCallback, itemProducerTreeComparer: comparer.Object, equalityComparer: cosmosElementComparer.Object, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false), deferFirstPage: false, collectionRid: "collectionRid", initialContinuationToken: null, @@ -254,22 +265,32 @@ public async Task TestItemProducerTreeWithFailure() await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); // First item should be a success - (bool successfullyMovedNext, QueryResponseCore? failureResponse) result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); - Assert.IsTrue(result.successfullyMovedNext); - Assert.IsNull(result.failureResponse); - Assert.IsTrue(itemProducerTree.HasMoreResults); + { + (bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token); + Assert.IsTrue(movedToNextPage); + Assert.IsNull(failureResponse); + Assert.IsTrue(itemProducerTree.TryMoveNextDocumentWithinPage()); + Assert.IsFalse(itemProducerTree.TryMoveNextDocumentWithinPage()); + Assert.IsTrue(itemProducerTree.HasMoreResults); + } // Second item should be a success - result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); - Assert.IsTrue(result.successfullyMovedNext); - Assert.IsNull(result.failureResponse); - Assert.IsTrue(itemProducerTree.HasMoreResults); + { + (bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token); + Assert.IsTrue(movedToNextPage); + Assert.IsNull(failureResponse); + Assert.IsTrue(itemProducerTree.TryMoveNextDocumentWithinPage()); + Assert.IsFalse(itemProducerTree.TryMoveNextDocumentWithinPage()); + Assert.IsTrue(itemProducerTree.HasMoreResults); + } // Third item should be a failure - result = await itemProducerTree.MoveNextAsync(cancellationTokenSource.Token); - Assert.IsFalse(result.successfullyMovedNext); - Assert.IsNotNull(result.failureResponse); - Assert.IsFalse(itemProducerTree.HasMoreResults); + { + (bool movedToNextPage, QueryResponseCore? failureResponse) = await itemProducerTree.TryMoveNextPageAsync(cancellationTokenSource.Token); + Assert.IsFalse(movedToNextPage); + Assert.IsNotNull(failureResponse); + Assert.IsFalse(itemProducerTree.HasMoreResults); + } // Try to buffer after failure. It should return the previous cached failure and not try to buffer again. mockQueryContext.Setup(x => x.ExecuteQueryAsync( @@ -283,8 +304,6 @@ public async Task TestItemProducerTreeWithFailure() Throws(new Exception("Previous buffer failed. Operation should return original failure and not try again")); await itemProducerTree.BufferMoreDocumentsAsync(cancellationTokenSource.Token); - Assert.IsFalse(result.successfullyMovedNext); - Assert.IsNotNull(result.failureResponse); Assert.IsFalse(itemProducerTree.HasMoreResults); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/MockItemProducerFactory.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/MockItemProducerFactory.cs index 65ac8918a6..4aeea7979a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/MockItemProducerFactory.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/MockItemProducerFactory.cs @@ -90,6 +90,7 @@ public static (ItemProducer itemProducer, ReadOnlyCollection allItems) partitionKeyRange, completeDelegate, CosmosElementEqualityComparer.Value, + new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false), maxPageSize, initialContinuationToken: continuationToken); @@ -171,6 +172,7 @@ public static (ItemProducerTree itemProducerTree, ReadOnlyCollection a completeDelegate, itemProducerTreeComparer, CosmosElementEqualityComparer.Value, + new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false), deferFirstPage, collectionRid, maxPageSize, diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPipelineMockTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPipelineMockTests.cs index 6015c2a6c5..2341b04782 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPipelineMockTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Query/QueryPipelineMockTests.cs @@ -76,7 +76,8 @@ public async Task TestCosmosCrossPartitionQueryExecutionContextWithEmptyPagesAnd initialPageSize: maxPageSize, maxConcurrency: null, maxItemCount: maxPageSize, - maxBufferedItemCount: null); + maxBufferedItemCount: null, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false)); CosmosParallelItemQueryExecutionContext executionContext = (await CosmosParallelItemQueryExecutionContext.TryCreateAsync( context, @@ -159,7 +160,8 @@ public async Task TestCosmosCrossPartitionQueryExecutionContextWithFailuresAsync initialPageSize: maxPageSize, maxConcurrency: null, maxItemCount: maxPageSize, - maxBufferedItemCount: null); + maxBufferedItemCount: null, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false)); CosmosParallelItemQueryExecutionContext executionContext = (await CosmosParallelItemQueryExecutionContext.TryCreateAsync( context, @@ -242,7 +244,6 @@ public async Task TestCosmosOrderByQueryExecutionContextWithEmptyPagesAndSplitAs }; OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( - queryClient: mockQueryClient.Object, compositeContinuationToken: compositeContinuation, orderByItems: orderByItems, rid: itemToRepresentPreviousQuery._rid, @@ -287,7 +288,8 @@ public async Task TestCosmosOrderByQueryExecutionContextWithEmptyPagesAndSplitAs initialPageSize: maxPageSize, maxConcurrency: null, maxItemCount: maxPageSize, - maxBufferedItemCount: null); + maxBufferedItemCount: null, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false)); CosmosOrderByItemQueryExecutionContext executionContext = (await CosmosOrderByItemQueryExecutionContext.TryCreateAsync( context, @@ -365,7 +367,6 @@ public async Task TestCosmosOrderByQueryExecutionContextWithFailurePageAsync(boo }; OrderByContinuationToken orderByContinuationToken = new OrderByContinuationToken( - queryClient: mockQueryClient.Object, compositeContinuationToken: compositeContinuation, orderByItems: orderByItems, rid: itemToRepresentPreviousQuery._rid, @@ -410,48 +411,60 @@ public async Task TestCosmosOrderByQueryExecutionContextWithFailurePageAsync(boo initialPageSize: maxPageSize, maxConcurrency: null, maxItemCount: maxPageSize, - maxBufferedItemCount: null); + maxBufferedItemCount: null, + testSettings: new Query.Core.TestInjections(simulate429s: false, simulateEmptyPages: false)); - CosmosOrderByItemQueryExecutionContext executionContext = (await CosmosOrderByItemQueryExecutionContext.TryCreateAsync( + TryCatch tryCreate = await CosmosOrderByItemQueryExecutionContext.TryCreateAsync( context, initParams, fullConitnuationToken, - this.cancellationToken)).Result; + this.cancellationToken); - Assert.IsTrue(!executionContext.IsDone); - - // Read all the pages from both splits - List itemsRead = new List(); - QueryResponseCore? failure = null; - while (!executionContext.IsDone) + if (tryCreate.Succeeded) { - QueryResponseCore queryResponse = await executionContext.DrainAsync( - maxPageSize, - this.cancellationToken); - if (queryResponse.IsSuccess) + CosmosOrderByItemQueryExecutionContext executionContext = tryCreate.Result; + + Assert.IsTrue(!executionContext.IsDone); + + // Read all the pages from both splits + List itemsRead = new List(); + QueryResponseCore? failure = null; + while (!executionContext.IsDone) { - string responseContinuationToken = queryResponse.ContinuationToken; - foreach (CosmosElement element in queryResponse.CosmosElements) + QueryResponseCore queryResponse = await executionContext.DrainAsync( + maxPageSize, + this.cancellationToken); + if (queryResponse.IsSuccess) { - string jsonValue = element.ToString(); - ToDoItem item = JsonConvert.DeserializeObject(jsonValue); - itemsRead.Add(item); + string responseContinuationToken = queryResponse.ContinuationToken; + foreach (CosmosElement element in queryResponse.CosmosElements) + { + string jsonValue = element.ToString(); + ToDoItem item = JsonConvert.DeserializeObject(jsonValue); + itemsRead.Add(item); + } + } + else + { + Assert.IsNull(failure, "There should only be one error"); + failure = queryResponse; } } - else - { - Assert.IsNull(failure, "There should only be one error"); - failure = queryResponse; - } - } - Assert.IsNotNull(failure); - Assert.AreEqual((HttpStatusCode)429, failure.Value.StatusCode); - Assert.IsNull(failure.Value.ErrorMessage); + Assert.IsNotNull(failure); + Assert.AreEqual((HttpStatusCode)429, failure.Value.StatusCode); + Assert.IsNull(failure.Value.ErrorMessage); - Assert.AreEqual(allItems.Count, itemsRead.Count); + Assert.AreEqual(0 /*We don't get any items, since we don't buffer the failure anymore*/, itemsRead.Count); - CollectionAssert.AreEqual(allItems.ToList(), itemsRead, new ToDoItemComparer()); + //CollectionAssert.AreEqual(allItems.ToList(), itemsRead, new ToDoItemComparer()); + } + else + { + CosmosException cosmosException = tryCreate.Exception as CosmosException; + Assert.IsNotNull(cosmosException); + Assert.AreEqual((HttpStatusCode)429, cosmosException.StatusCode); + } } } diff --git a/changelog.md b/changelog.md index f807e71538..6ff5852b60 100644 --- a/changelog.md +++ b/changelog.md @@ -15,12 +15,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [#944](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/944) Change Feed Processor won't use user serializer for internal operations +- [#988](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/988) Fixed query mutating due to retry of gone / name cache is stale. +- [#999](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/999) Fixed grabbing extra page and updated continuation token on exception path. +- [#1013](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/1013) Gateway OperationCanceledException are now returned as request timeouts ## [3.4.1](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.4.1) - 2019-11-06 ### Fixed -- [#978](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/978) Fix mocking for FeedIterator and Response classes +- [#978](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/978) Fixed mocking for FeedIterator and Response classes ## [3.4.0](https://www.nuget.org/packages/Microsoft.Azure.Cosmos/3.4.0) - 2019-11-04 @@ -35,7 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#965](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/965) Batch API is now public ### Fixed -- [#901](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/901) Fix a bug causing query response to create a new stream for each content call +- [#901](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/901) Fixed a bug causing query response to create a new stream for each content call - [#918](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/918) Fixed serializer being used for Scripts, Permissions, and Conflict related iterators - [#936](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/936) Fixed bulk requests with large resources to have natural exception @@ -84,7 +87,7 @@ flow. - [#100](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/100) Configurable Tcp settings to CosmosClientOptions - [#615](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/615), [#775](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/775) Added request diagnostics to Response's -- [#622](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/622) Added CRUD and query operations for Users and Permissions which enables [ResourceToken](https://docs.microsoft.com/en-us/azure/cosmos-db/secure-access-to-data#resource-tokens) support +- [#622](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/622) Added CRUD and query operations for Users and Permissions which enables [ResourceToken](https://docs.microsoft.com/azure/cosmos-db/secure-access-to-data#resource-tokens) support - [#716](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/716) Added camel case serialization on LINQ query generation - [#729](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/729), [#776](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/776) Added aggregate(CountAsync/SumAsync etc.) extensions for LINQ query - [#743](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/743) Added WebProxy to CosmosClientOptions diff --git a/templates/static-tools.yml b/templates/static-tools.yml index cbcde5f5f3..4aa641cf9a 100644 --- a/templates/static-tools.yml +++ b/templates/static-tools.yml @@ -2,19 +2,19 @@ parameters: BuildConfiguration: '' - VmImage: '' + VmImage: '' -jobs: -- job: - displayName: Static Analysis +jobs: +- job: + displayName: Static Analysis pool: - vmImage: '${{ parameters.VmImage }}' - + vmImage: '${{ parameters.VmImage }}' + steps: - checkout: self # self represents the repo where the initial Pipelines YAML file was found clean: true # if true, execute `execute git clean -ffdx && git reset --hard HEAD` before fetching - - #Analyze source code for type of content and target types to help determine which tools to run + + #Analyze source code for type of content and target types to help determine which tools to run - task: securedevelopmentteam.vss-secure-development-tools.build-task-autoapplicability.AutoApplicability@1 displayName: 'AutoApplicability' inputs: @@ -22,47 +22,47 @@ jobs: ExternalRelease: true InternalRelease: true IsService: true - IsSoftware: true - + IsSoftware: true + # Analyze source and build output text files for credentials - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 displayName: 'CredScan' inputs: scanFolder: $(Build.SourcesDirectory) - suppressionsFile: CredScanSuppressions.json + suppressionsFile: CredScanSuppressions.json debugMode: true - + # Scan text elements including code, code comments, and content/web pages, for sensitive terms based on legal, cultural, or geopolitical reasons - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 displayName: 'PoliCheck' inputs: targetType: F - + # AntiMalware scan - - task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 - displayName: 'AntiMalware' - continueOnError: true # signature refresh failing resulting in tasks failures - inputs: - EnableServices: true - + #- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + # displayName: 'AntiMalware' + # continueOnError: true # signature refresh failing resulting in tasks failures + # inputs: + # EnableServices: true + # Run checks for recently discovered vulnerabilities which are not yet incorporated to another tool - task: securedevelopmentteam.vss-secure-development-tools.build-task-vulnerabilityassessment.VulnerabilityAssessment@0 displayName: 'Vulnerability Assessment' - + - task: DotNetCoreCLI@2 displayName: Build Microsoft.Azure.Cosmos.sln inputs: - command: build + command: build projects: 'Microsoft.Azure.Cosmos.sln' configuration: '${{ parameters.BuildConfiguration }}' - publishTestResults: true + publishTestResults: true - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 displayName: 'Component Governance Detection' #https://docs.opensource.microsoft.com/tools/cg.html inputs: alertWarningLevel: Medium - failOnAlert: true - + failOnAlert: true + # - task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 # displayName: 'BinSkim' # inputs: