From c1dc7ffd4e3848a772c0009139ea5b4bfb34f4ee Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 28 Jun 2021 17:06:32 -0700 Subject: [PATCH 01/22] Initial types and implementation --- .../Monitoring/ChangeFeedProcessorEvent.cs | 42 ++++++++++++ .../ChangeFeedProcessorHealthMonitor.cs | 67 +++++++++++++++++++ .../Monitoring/HealthMonitor.cs | 19 ------ ...hMonitoringPartitionControllerDecorator.cs | 10 ++- .../Monitoring/MonitoredOperation.cs | 17 ----- .../Monitoring/TraceHealthMonitor.cs | 31 ++++++--- 6 files changed, 134 insertions(+), 52 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs create mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitor.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/MonitoredOperation.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs new file mode 100644 index 0000000000..1b09d9e930 --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs @@ -0,0 +1,42 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + /// + /// The event type during the Change Feed Processor lifecycle + /// + /// + /// Lifecyle is: + /// 1. AcquireLease. Lease is acquire and starts processing. + /// 2. ReadingChangeFeedLease. New changes were read from Change Feed for the lease. + /// 3. ProcessingLease. New changes were delivered to the delegate for the lease. + /// 4. CheckpointLease. Lease was updated after a successful processing. + /// 4. ReadingChangeFeedLease, ProcessingLease, and CheckpointLease keep happening until: + /// 5. ReleaseLease. Lease is being released by the host. + /// + public enum ChangeFeedProcessorEvent + { + /// + /// The host acquires the lease and starts processing + /// + AcquireLease, + /// + /// The host is reading the Change Feed for the lease + /// + ReadingChangeFeedLease, + /// + /// The host sent the new changes to the delegate for processing + /// + ProcessingLease, + /// + /// The host updates the lease after a successful processing + /// + CheckpointLease, + /// + /// The host releases the lease due to shutdown, rebalancing, error during processing + /// + ReleaseLease, + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs new file mode 100644 index 0000000000..ff77953c8e --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs @@ -0,0 +1,67 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos +{ + using System; + using System.Threading.Tasks; + + /// + /// Health monitor to capture lifecycle events of the Change Feed Processor. + /// + /// + /// Lifecyle is: + /// 1. AcquireLease. Lease is acquire and starts processing. + /// 2. ReadingChangeFeedLease. New changes were read from Change Feed for the lease. + /// 3. ProcessingLease. New changes were delivered to the delegate for the lease. + /// 4. CheckpointLease. Lease was updated after a successful processing. + /// 4. ReadingChangeFeedLease, ProcessingLease, and CheckpointLease keep happening until: + /// 5. ReleaseLease. Lease is being released by the host. + /// + public abstract class ChangeFeedProcessorHealthMonitor + { + /// + /// For normal informational events happening on the context of a lease + /// + /// The type of event. + /// A unique identifier for the lease. + /// An asynchronous operation representing the logging operation. + public virtual Task NotifyInformationAsync( + ChangeFeedProcessorEvent changeFeedProcessorEvent, + string leaseToken) + { + return Task.CompletedTask; + } + + /// + /// For transient errors that the Change Feed Processor attemps retrying on. + /// + /// The type of event. + /// A unique identifier for the lease. + /// The exception that happened. + /// An asynchronous operation representing the logging operation. + public virtual Task NotifyErrorAsync( + ChangeFeedProcessorEvent changeFeedProcessorEvent, + string leaseToken, + Exception exception) + { + return Task.CompletedTask; + } + + /// + /// For critical errors that the Change Feed Processor that might not be recoverable. + /// + /// The type of event. + /// A unique identifier for the lease. + /// The exception that happened. + /// An asynchronous operation representing the logging operation. + public virtual Task NotifyCriticalAsync( + ChangeFeedProcessorEvent changeFeedProcessorEvent, + string leaseToken, + Exception exception) + { + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitor.cs deleted file mode 100644 index b9ba2a8fab..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitor.cs +++ /dev/null @@ -1,19 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring -{ - using System.Threading.Tasks; - - /// - /// A strategy for handling the situation when the change feed processor is not able to acquire lease due to unknown reasons. - /// - internal abstract class HealthMonitor - { - /// - /// A logic to handle that exceptional situation. - /// - public abstract Task InspectAsync(HealthMonitoringRecord record); - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs index 1fa92cb409..50dcc543e9 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs @@ -8,15 +8,14 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed using System.Threading.Tasks; using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; - using Microsoft.Azure.Cosmos.ChangeFeed.Monitoring; using Microsoft.Azure.Documents; internal class HealthMonitoringPartitionControllerDecorator : PartitionController { private readonly PartitionController inner; - private readonly HealthMonitor monitor; + private readonly ChangeFeedProcessorHealthMonitor monitor; - public HealthMonitoringPartitionControllerDecorator(PartitionController inner, HealthMonitor monitor) + public HealthMonitoringPartitionControllerDecorator(PartitionController inner, ChangeFeedProcessorHealthMonitor monitor) { this.inner = inner ?? throw new ArgumentNullException(nameof(inner)); this.monitor = monitor ?? throw new ArgumentNullException(nameof(monitor)); @@ -27,7 +26,7 @@ public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease) try { await this.inner.AddOrUpdateLeaseAsync(lease); - await this.monitor.InspectAsync(new HealthMonitoringRecord(HealthSeverity.Informational, MonitoredOperation.AcquireLease, lease, null)); + await this.monitor.NotifyInformationAsync(ChangeFeedProcessorEvent.AcquireLease, lease.CurrentLeaseToken); } catch (DocumentClientException) { @@ -35,8 +34,7 @@ public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease) } catch (Exception exception) { - await this.monitor.InspectAsync(new HealthMonitoringRecord(HealthSeverity.Error, MonitoredOperation.AcquireLease, lease, exception)); - + await this.monitor.NotifyErrorAsync(ChangeFeedProcessorEvent.AcquireLease, lease.CurrentLeaseToken, exception); throw; } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/MonitoredOperation.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/MonitoredOperation.cs deleted file mode 100644 index 479bbb6a29..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/MonitoredOperation.cs +++ /dev/null @@ -1,17 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring -{ - /// - /// The health monitoring phase - /// - internal enum MonitoredOperation - { - /// - /// A phase when the instance tries to acquire the lease - /// - AcquireLease, - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs index c5f32d07dc..89d0fde99e 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs @@ -4,24 +4,35 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring { + using System; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Core.Trace; /// - /// A monitor which logs the errors only. + /// A monitor which logs the errors only as traces /// - internal sealed class TraceHealthMonitor : HealthMonitor + internal sealed class TraceHealthMonitor : ChangeFeedProcessorHealthMonitor { - /// - public override Task InspectAsync(HealthMonitoringRecord record) + public override Task NotifyErrorAsync( + ChangeFeedProcessorEvent changeFeedProcessorEvent, + string leaseToken, + Exception exception) { - if (record.Severity == HealthSeverity.Error) - { - Extensions.TraceException(record.Exception); - DefaultTrace.TraceError($"Unhealthiness detected in the operation {record.Operation} for {record.Lease}. "); - } + Extensions.TraceException(exception); + DefaultTrace.TraceError($"Unhealthiness detected in the operation {changeFeedProcessorEvent} for {leaseToken}. "); - return Task.FromResult(true); + return Task.CompletedTask; + } + + public override Task NotifyCriticalAsync( + ChangeFeedProcessorEvent changeFeedProcessorEvent, + string leaseToken, + Exception exception) + { + Extensions.TraceException(exception); + DefaultTrace.TraceCritical($"Unhealthiness detected in the operation {changeFeedProcessorEvent} for {leaseToken}. "); + + return Task.CompletedTask; } } } \ No newline at end of file From d424b70df783e5ba1c9a06456e391c37af9e3549 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Mon, 28 Jun 2021 17:06:38 -0700 Subject: [PATCH 02/22] Wiring through known places --- .../ChangeFeedProcessorBuilder.cs | 23 ++++++++++--------- .../ChangeFeedProcessorCore.cs | 2 +- .../ChangeFeedProcessorOptions.cs | 3 +++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs index 39ef6a3f75..a89945864b 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs @@ -25,7 +25,7 @@ public class ChangeFeedProcessorBuilder ChangeFeedProcessorOptions, ContainerInternal> applyBuilderConfiguration; - private ChangeFeedProcessorOptions changeFeedProcessorOptions; + private readonly ChangeFeedProcessorOptions changeFeedProcessorOptions = new ChangeFeedProcessorOptions(); private ContainerInternal leaseContainer; private string InstanceName; @@ -96,7 +96,6 @@ public ChangeFeedProcessorBuilder WithPollInterval(TimeSpan pollInterval) throw new ArgumentNullException(nameof(pollInterval)); } - this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.FeedPollDelay = pollInterval; return this; } @@ -114,7 +113,6 @@ public ChangeFeedProcessorBuilder WithPollInterval(TimeSpan pollInterval) /// The instance of to use. internal virtual ChangeFeedProcessorBuilder WithStartFromBeginning() { - this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.StartFromBeginning = true; return this; } @@ -137,7 +135,6 @@ public ChangeFeedProcessorBuilder WithStartTime(DateTime startTime) throw new ArgumentNullException(nameof(startTime)); } - this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.StartTime = startTime; return this; } @@ -154,7 +151,6 @@ public ChangeFeedProcessorBuilder WithMaxItems(int maxItemCount) throw new ArgumentOutOfRangeException(nameof(maxItemCount)); } - this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); this.changeFeedProcessorOptions.MaxItemCount = maxItemCount; return this; } @@ -213,6 +209,17 @@ internal virtual ChangeFeedProcessorBuilder WithInMemoryLeaseContainer() return this; } + /// + /// Sets a health monitor to log events happening during the lifecycle of the processor. + /// + /// An implementation of to log lifecycle events. + /// The instance of to use. + public ChangeFeedProcessorBuilder WithHealthMonitor(ChangeFeedProcessorHealthMonitor healthMonitor) + { + this.changeFeedProcessorOptions.HealthMonitor = healthMonitor; + return this; + } + /// /// Builds a new instance of the with the specified configuration. /// @@ -239,16 +246,10 @@ public ChangeFeedProcessor Build() throw new InvalidOperationException("Processor name not specified during creation."); } - this.InitializeDefaultOptions(); this.applyBuilderConfiguration(this.LeaseStoreManager, this.leaseContainer, this.InstanceName, this.changeFeedLeaseOptions, this.changeFeedProcessorOptions, this.monitoredContainer); this.isBuilt = true; return this.changeFeedProcessor; } - - private void InitializeDefaultOptions() - { - this.changeFeedProcessorOptions = this.changeFeedProcessorOptions ?? new ChangeFeedProcessorOptions(); - } } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs index e2325e5ad7..d312c20ffb 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs @@ -119,7 +119,7 @@ private PartitionManager BuildPartitionManager( PartitionController partitionController = new PartitionControllerCore(this.documentServiceLeaseStoreManager.LeaseContainer, this.documentServiceLeaseStoreManager.LeaseManager, partitionSuperviserFactory, synchronizer); - partitionController = new HealthMonitoringPartitionControllerDecorator(partitionController, new TraceHealthMonitor()); + partitionController = new HealthMonitoringPartitionControllerDecorator(partitionController, this.changeFeedProcessorOptions.HealthMonitor); PartitionLoadBalancerCore partitionLoadBalancer = new PartitionLoadBalancerCore( partitionController, this.documentServiceLeaseStoreManager.LeaseContainer, diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs index f102f72be5..22afb060ca 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Configuration { using System; using Microsoft.Azure.Cosmos; + using Microsoft.Azure.Cosmos.ChangeFeed.Monitoring; /// /// Options to control various aspects of partition distribution happening within instance. @@ -81,5 +82,7 @@ public DateTime? StartTime /// /// public bool StartFromBeginning { get; set; } + + public ChangeFeedProcessorHealthMonitor HealthMonitor { get; set; } = new TraceHealthMonitor(); } } From 08f4d95e69a4e3efc0f9120b9d012578c2ef3f84 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 09:18:57 -0700 Subject: [PATCH 03/22] Refactor monitoring --- .../ChangeFeedProcessorCore.cs | 9 ++-- .../FeedManagement/PartitionControllerCore.cs | 28 ++++++---- .../FeedProcessing/FeedProcessorCore.cs | 2 - .../Monitoring/ChangeFeedProcessorEvent.cs | 42 --------------- .../ChangeFeedProcessorHealthMonitor.cs | 25 ++------- ...hMonitoringPartitionControllerDecorator.cs | 52 ------------------- .../Monitoring/HealthMonitoringRecord.cs | 51 ------------------ .../Monitoring/HealthSeverity.cs | 26 ---------- .../Monitoring/TraceHealthMonitor.cs | 14 +---- 9 files changed, 28 insertions(+), 221 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringRecord.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthSeverity.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs index d312c20ffb..94c3e09dac 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorCore.cs @@ -11,7 +11,6 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; using Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing; using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; - using Microsoft.Azure.Cosmos.ChangeFeed.Monitoring; using Microsoft.Azure.Cosmos.ChangeFeed.Utils; using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Cosmos.Tracing; @@ -117,9 +116,13 @@ private PartitionManager BuildPartitionManager( EqualPartitionsBalancingStrategy.DefaultMaxLeaseCount, this.changeFeedLeaseOptions.LeaseExpirationInterval); - PartitionController partitionController = new PartitionControllerCore(this.documentServiceLeaseStoreManager.LeaseContainer, this.documentServiceLeaseStoreManager.LeaseManager, partitionSuperviserFactory, synchronizer); + PartitionController partitionController = new PartitionControllerCore( + this.documentServiceLeaseStoreManager.LeaseContainer, + this.documentServiceLeaseStoreManager.LeaseManager, + partitionSuperviserFactory, + synchronizer, + this.changeFeedProcessorOptions.HealthMonitor); - partitionController = new HealthMonitoringPartitionControllerDecorator(partitionController, this.changeFeedProcessorOptions.HealthMonitor); PartitionLoadBalancerCore partitionLoadBalancer = new PartitionLoadBalancerCore( partitionController, this.documentServiceLeaseStoreManager.LeaseContainer, diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs index 712617c369..5426db6079 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs @@ -23,18 +23,21 @@ internal sealed class PartitionControllerCore : PartitionController private readonly DocumentServiceLeaseManager leaseManager; private readonly PartitionSupervisorFactory partitionSupervisorFactory; private readonly PartitionSynchronizer synchronizer; + private readonly ChangeFeedProcessorHealthMonitor monitor; private CancellationTokenSource shutdownCts; public PartitionControllerCore( DocumentServiceLeaseContainer leaseContainer, DocumentServiceLeaseManager leaseManager, PartitionSupervisorFactory partitionSupervisorFactory, - PartitionSynchronizer synchronizer) + PartitionSynchronizer synchronizer, + ChangeFeedProcessorHealthMonitor monitor) { this.leaseContainer = leaseContainer; this.leaseManager = leaseManager; this.partitionSupervisorFactory = partitionSupervisorFactory; this.synchronizer = synchronizer; + this.monitor = monitor; } public override async Task InitializeAsync() @@ -63,10 +66,12 @@ public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease) } DefaultTrace.TraceInformation("Lease with token {0}: acquired", lease.CurrentLeaseToken); + await this.monitor.NotifyLeaseAcquireAsync(lease.CurrentLeaseToken); } - catch (Exception) + catch (Exception ex) { await this.RemoveLeaseAsync(lease).ConfigureAwait(false); + await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex); throw; } @@ -101,15 +106,16 @@ private async Task RemoveLeaseAsync(DocumentServiceLease lease) return; } - DefaultTrace.TraceInformation("Lease with token {0}: released", lease.CurrentLeaseToken); - try { await this.leaseManager.ReleaseAsync(lease).ConfigureAwait(false); + + DefaultTrace.TraceInformation("Lease with token {0}: released", lease.CurrentLeaseToken); + await this.monitor.NotifyLeaseReleaseAsync(lease.CurrentLeaseToken); } - catch (Exception e) + catch (Exception ex) { - Extensions.TraceException(e); + await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex); DefaultTrace.TraceWarning("Lease with token {0}: failed to remove lease", lease.CurrentLeaseToken); } finally @@ -132,9 +138,9 @@ private async Task ProcessPartitionAsync(PartitionSupervisor partitionSupervisor { DefaultTrace.TraceVerbose("Lease with token {0}: processing canceled", lease.CurrentLeaseToken); } - catch (Exception e) + catch (Exception ex) { - Extensions.TraceException(e); + await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex); DefaultTrace.TraceWarning("Lease with token {0}: processing failed", lease.CurrentLeaseToken); } @@ -160,10 +166,10 @@ private async Task HandlePartitionGoneAsync(DocumentServiceLease lease, string l await Task.WhenAll(addLeaseTasks).ConfigureAwait(false); } - catch (Exception e) + catch (Exception ex) { - Extensions.TraceException(e); - DefaultTrace.TraceWarning("Lease with token {0}: failed to split", e, lease.CurrentLeaseToken); + await this.monitor.NotifyErrorAsync(lease.CurrentLeaseToken, ex); + DefaultTrace.TraceWarning("Lease with token {0}: failed to handle gone", ex, lease.CurrentLeaseToken); } } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs index 3182d33369..382dee0502 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs @@ -5,9 +5,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing { using System; - using System.Collections.Generic; using System.Diagnostics; - using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs deleted file mode 100644 index 1b09d9e930..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorEvent.cs +++ /dev/null @@ -1,42 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos -{ - /// - /// The event type during the Change Feed Processor lifecycle - /// - /// - /// Lifecyle is: - /// 1. AcquireLease. Lease is acquire and starts processing. - /// 2. ReadingChangeFeedLease. New changes were read from Change Feed for the lease. - /// 3. ProcessingLease. New changes were delivered to the delegate for the lease. - /// 4. CheckpointLease. Lease was updated after a successful processing. - /// 4. ReadingChangeFeedLease, ProcessingLease, and CheckpointLease keep happening until: - /// 5. ReleaseLease. Lease is being released by the host. - /// - public enum ChangeFeedProcessorEvent - { - /// - /// The host acquires the lease and starts processing - /// - AcquireLease, - /// - /// The host is reading the Change Feed for the lease - /// - ReadingChangeFeedLease, - /// - /// The host sent the new changes to the delegate for processing - /// - ProcessingLease, - /// - /// The host updates the lease after a successful processing - /// - CheckpointLease, - /// - /// The host releases the lease due to shutdown, rebalancing, error during processing - /// - ReleaseLease, - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs index ff77953c8e..e9095b1384 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs @@ -10,25 +10,14 @@ namespace Microsoft.Azure.Cosmos /// /// Health monitor to capture lifecycle events of the Change Feed Processor. /// - /// - /// Lifecyle is: - /// 1. AcquireLease. Lease is acquire and starts processing. - /// 2. ReadingChangeFeedLease. New changes were read from Change Feed for the lease. - /// 3. ProcessingLease. New changes were delivered to the delegate for the lease. - /// 4. CheckpointLease. Lease was updated after a successful processing. - /// 4. ReadingChangeFeedLease, ProcessingLease, and CheckpointLease keep happening until: - /// 5. ReleaseLease. Lease is being released by the host. - /// public abstract class ChangeFeedProcessorHealthMonitor { /// /// For normal informational events happening on the context of a lease /// - /// The type of event. /// A unique identifier for the lease. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyInformationAsync( - ChangeFeedProcessorEvent changeFeedProcessorEvent, + public virtual Task NotifyLeaseAcquireAsync( string leaseToken) { return Task.CompletedTask; @@ -37,14 +26,10 @@ public virtual Task NotifyInformationAsync( /// /// For transient errors that the Change Feed Processor attemps retrying on. /// - /// The type of event. /// A unique identifier for the lease. - /// The exception that happened. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyErrorAsync( - ChangeFeedProcessorEvent changeFeedProcessorEvent, - string leaseToken, - Exception exception) + public virtual Task NotifyLeaseReleaseAsync( + string leaseToken) { return Task.CompletedTask; } @@ -52,12 +37,10 @@ public virtual Task NotifyErrorAsync( /// /// For critical errors that the Change Feed Processor that might not be recoverable. /// - /// The type of event. /// A unique identifier for the lease. /// The exception that happened. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyCriticalAsync( - ChangeFeedProcessorEvent changeFeedProcessorEvent, + public virtual Task NotifyErrorAsync( string leaseToken, Exception exception) { diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs deleted file mode 100644 index 50dcc543e9..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringPartitionControllerDecorator.cs +++ /dev/null @@ -1,52 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed -{ - using System; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; - using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; - using Microsoft.Azure.Documents; - - internal class HealthMonitoringPartitionControllerDecorator : PartitionController - { - private readonly PartitionController inner; - private readonly ChangeFeedProcessorHealthMonitor monitor; - - public HealthMonitoringPartitionControllerDecorator(PartitionController inner, ChangeFeedProcessorHealthMonitor monitor) - { - this.inner = inner ?? throw new ArgumentNullException(nameof(inner)); - this.monitor = monitor ?? throw new ArgumentNullException(nameof(monitor)); - } - - public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease) - { - try - { - await this.inner.AddOrUpdateLeaseAsync(lease); - await this.monitor.NotifyInformationAsync(ChangeFeedProcessorEvent.AcquireLease, lease.CurrentLeaseToken); - } - catch (DocumentClientException) - { - throw; - } - catch (Exception exception) - { - await this.monitor.NotifyErrorAsync(ChangeFeedProcessorEvent.AcquireLease, lease.CurrentLeaseToken, exception); - throw; - } - } - - public override Task InitializeAsync() - { - return this.inner.InitializeAsync(); - } - - public override Task ShutdownAsync() - { - return this.inner.ShutdownAsync(); - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringRecord.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringRecord.cs deleted file mode 100644 index 25ca720325..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthMonitoringRecord.cs +++ /dev/null @@ -1,51 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring -{ - using System; - using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; - - /// - /// A record used in the health monitoring. - /// - internal class HealthMonitoringRecord - { - /// - /// Initializes a new instance of the class. - /// - /// The health severity level. - /// The operation. - /// The lease. - /// The exception. - public HealthMonitoringRecord(HealthSeverity severity, MonitoredOperation operation, DocumentServiceLease lease, Exception exception) - { - if (lease == null) throw new ArgumentNullException(nameof(lease)); - this.Severity = severity; - this.Operation = operation; - this.Lease = lease; - this.Exception = exception; - } - - /// - /// Gets the health severity. - /// - public HealthSeverity Severity { get; } - - /// - /// Gets the monitored operation. - /// - public MonitoredOperation Operation { get; } - - /// - /// Gets the lease which triggered the operation. - /// - public DocumentServiceLease Lease { get; } - - /// - /// Gets the exception details in case of failure. - /// - public Exception Exception { get; } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthSeverity.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthSeverity.cs deleted file mode 100644 index 7077516557..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/HealthSeverity.cs +++ /dev/null @@ -1,26 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring -{ - /// - /// The health severity level - /// - internal enum HealthSeverity - { - /// - /// Critical level. - /// - Critical = 10, - - /// - /// Error level. - /// - Error = 20, - - /// - /// Information level. - /// - Informational = 30, - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs index 89d0fde99e..75e9742362 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs @@ -14,23 +14,11 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring internal sealed class TraceHealthMonitor : ChangeFeedProcessorHealthMonitor { public override Task NotifyErrorAsync( - ChangeFeedProcessorEvent changeFeedProcessorEvent, string leaseToken, Exception exception) { Extensions.TraceException(exception); - DefaultTrace.TraceError($"Unhealthiness detected in the operation {changeFeedProcessorEvent} for {leaseToken}. "); - - return Task.CompletedTask; - } - - public override Task NotifyCriticalAsync( - ChangeFeedProcessorEvent changeFeedProcessorEvent, - string leaseToken, - Exception exception) - { - Extensions.TraceException(exception); - DefaultTrace.TraceCritical($"Unhealthiness detected in the operation {changeFeedProcessorEvent} for {leaseToken}. "); + DefaultTrace.TraceError($"Error detected for lease {leaseToken}. "); return Task.CompletedTask; } From 212f88c0e76b87286833da9221ff6bec06119b1d Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 11:35:43 -0700 Subject: [PATCH 04/22] New tests --- .../FeedManagement/PartitionControllerCore.cs | 2 +- .../ChangeFeed/HealthinessMonitorTests.cs | 48 ---------- .../PartitionControllerSplitTests.cs | 28 ++++-- .../ChangeFeed/PartitionControllerTests.cs | 91 ++++++++++++++++--- 4 files changed, 96 insertions(+), 73 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/HealthinessMonitorTests.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs index 5426db6079..797a0a543a 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs @@ -134,7 +134,7 @@ private async Task ProcessPartitionAsync(PartitionSupervisor partitionSupervisor { await this.HandlePartitionGoneAsync(lease, ex.LastContinuation).ConfigureAwait(false); } - catch (TaskCanceledException) + catch (TaskCanceledException) when (this.shutdownCts.IsCancellationRequested) { DefaultTrace.TraceVerbose("Lease with token {0}: processing canceled", lease.CurrentLeaseToken); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/HealthinessMonitorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/HealthinessMonitorTests.cs deleted file mode 100644 index 301abc45fb..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/HealthinessMonitorTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests -{ - using System; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; - using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; - using Microsoft.Azure.Cosmos.ChangeFeed.Monitoring; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Moq; - - [TestClass] - [TestCategory("ChangeFeed")] - public class HealthinessMonitorTests - { - [TestMethod] - public async Task AcquireLease_ShouldReportHealthy_IfNoIssues() - { - Mock monitor = new Mock(); - HealthMonitoringPartitionControllerDecorator sut = new HealthMonitoringPartitionControllerDecorator(Mock.Of(), monitor.Object); - DocumentServiceLease lease = Mock.Of(); - await sut.AddOrUpdateLeaseAsync(lease); - - monitor.Verify(m => m.InspectAsync(It.Is(r => r.Severity == HealthSeverity.Informational && r.Lease == lease && r.Operation == MonitoredOperation.AcquireLease && r.Exception == null))); - } - - [TestMethod] - public async Task AcquireLease_ShouldReportFailure_IfSystemIssue() - { - DocumentServiceLease lease = Mock.Of(); - Mock monitor = new Mock(); - Mock controller = new Mock(); - - Exception exception = new InvalidOperationException(); - controller - .Setup(c => c.AddOrUpdateLeaseAsync(lease)) - .Returns(Task.FromException(exception)); - - HealthMonitoringPartitionControllerDecorator sut = new HealthMonitoringPartitionControllerDecorator(controller.Object, monitor.Object); - await Assert.ThrowsExceptionAsync(() => sut.AddOrUpdateLeaseAsync(lease)); - - monitor.Verify(m => m.InspectAsync(It.Is(r => r.Severity == HealthSeverity.Error && r.Lease == lease && r.Operation == MonitoredOperation.AcquireLease && r.Exception == exception))); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerSplitTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerSplitTests.cs index 2755364705..25704e05c8 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerSplitTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerSplitTests.cs @@ -37,7 +37,7 @@ public async Task Controller_ShouldSignalSynchronizerSplitPartition_IfPartitionS DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -65,7 +65,7 @@ public async Task Controller_ShouldPassLastKnownContinuationTokenToSynchronizer_ DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -96,7 +96,7 @@ public async Task Controller_ShouldCopyParentLeaseProperties_IfPartitionSplitHap DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -121,7 +121,7 @@ public async Task Controller_ShouldKeepParentLease_IfSplitThrows() DocumentServiceLeaseManager leaseManager = Mock.Of(); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -157,7 +157,9 @@ public async Task Controller_ShouldRunProcessingOnChildPartitions_IfHappyPath() DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + Mock monitor = new Mock(); + + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, monitor.Object); await sut.InitializeAsync().ConfigureAwait(false); @@ -175,6 +177,9 @@ public async Task Controller_ShouldRunProcessingOnChildPartitions_IfHappyPath() Mock.Get(partitionSupervisor1).Verify(p => p.RunAsync(It.IsAny()), Times.Once); Mock.Get(partitionSupervisor2).Verify(p => p.RunAsync(It.IsAny()), Times.Once); + + monitor.Verify(m => m.NotifyLeaseAcquireAsync(leaseChild1.CurrentLeaseToken), Times.Once); + monitor.Verify(m => m.NotifyLeaseReleaseAsync(leaseChild2.CurrentLeaseToken), Times.Once); } [TestMethod] @@ -194,7 +199,7 @@ public async Task Controller_ShouldRunNotDeleteLease() DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -230,7 +235,7 @@ public async Task Controller_ShouldIgnoreProcessingChildPartition_IfPartitionAlr DocumentServiceLeaseManager leaseManager = Mock.Of(manager => manager.AcquireAsync(lease) == Task.FromResult(lease)); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -272,7 +277,7 @@ public async Task Controller_ShouldDeleteParentLease_IfChildLeasesCreatedByAnoth ); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, Mock.Of()); await sut.InitializeAsync().ConfigureAwait(false); @@ -304,7 +309,9 @@ public async Task Controller_ShouldDeleteParentLease_IfChildLeaseAcquireThrows() ); DocumentServiceLeaseContainer leaseContainer = Mock.Of(); - PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer); + Mock monitor = new Mock(); + + PartitionControllerCore sut = new PartitionControllerCore(leaseContainer, leaseManager, partitionSupervisorFactory, synchronizer, monitor.Object); await sut.InitializeAsync().ConfigureAwait(false); @@ -315,6 +322,9 @@ public async Task Controller_ShouldDeleteParentLease_IfChildLeaseAcquireThrows() await sut.ShutdownAsync().ConfigureAwait(false); Mock.Get(leaseManager).Verify(manager => manager.DeleteAsync(lease), Times.Once); + + monitor.Verify(m => m.NotifyLeaseAcquireAsync(lease.CurrentLeaseToken), Times.Once); + monitor.Verify(m => m.NotifyLeaseReleaseAsync(lease.CurrentLeaseToken), Times.Once); } private DocumentServiceLease CreateMockLease(string partitionId = null) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs index 4c34da49b6..8da8e9692e 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionControllerTests.cs @@ -25,6 +25,7 @@ public class PartitionControllerTests private readonly PartitionSynchronizer synchronizer; private readonly PartitionController sut; private readonly PartitionSupervisorFactory partitionSupervisorFactory; + private readonly Mock healthMonitor; public PartitionControllerTests() { @@ -49,7 +50,8 @@ public PartitionControllerTests() DocumentServiceLeaseContainer leaseContainer = Mock.Of(); this.synchronizer = Mock.Of(); - this.sut = new PartitionControllerCore(leaseContainer, this.leaseManager, this.partitionSupervisorFactory, this.synchronizer); + this.healthMonitor = new Mock(); + this.sut = new PartitionControllerCore(leaseContainer, this.leaseManager, this.partitionSupervisorFactory, this.synchronizer, this.healthMonitor.Object); } [TestInitialize] @@ -82,6 +84,15 @@ public async Task AddLease_ShouldAcquireLease_WhenCalled() .Verify(manager => manager.AcquireAsync(this.lease), Times.Once); } + [TestMethod] + public async Task AddLease_ShouldNotify_Monitor() + { + await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false); + + this.healthMonitor + .Verify(m => m.NotifyLeaseAcquireAsync(this.lease.CurrentLeaseToken), Times.Once); + } + [TestMethod] public async Task AddLease_ShouldRunObserver_WhenCalled() { @@ -96,8 +107,8 @@ public async Task AddLease_ShouldntReleaseLease_WhenCalled() { await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false); - Mock.Get(this.partitionProcessor) - .Verify(p => p.RunAsync(It.IsAny()), Times.Once); + Mock.Get(this.leaseManager) + .Verify(manager => manager.ReleaseAsync(this.lease), Times.Never); } [TestMethod] @@ -224,23 +235,39 @@ public async Task Controller_ShouldReleasesLease_IfObserverExits() } [TestMethod] - public async Task AddLease_ShouldFail_IfLeaseAcquireThrows() + public async Task Controller_ShouldNotify_IfProcessingFails() { Mock.Get(this.partitionProcessor) .Reset(); - Mock.Get(this.leaseManager) - .Reset(); + Mock supervisor = new Mock(); - Mock.Get(this.leaseManager) - .Setup(manager => manager.AcquireAsync(this.lease)) - .Throws(new NullReferenceException()); + Exception exception = new NotImplementedException(); - Mock.Get(this.leaseManager) - .Setup(manager => manager.ReleaseAsync(this.lease)) - .Returns(Task.CompletedTask); + ManualResetEvent manualResetEvent = new ManualResetEvent(false); - await Assert.ThrowsExceptionAsync(() => this.sut.AddOrUpdateLeaseAsync(this.lease)).ConfigureAwait(false); + supervisor + .Setup(s => s.RunAsync(It.IsAny())) + .Callback((CancellationToken ct) => + { + manualResetEvent.Set(); + throw exception; + }); + + Mock.Get(this.partitionSupervisorFactory) + .Setup(f => f.Create(this.lease)) + .Returns(supervisor.Object); + + await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false); + + bool timeout = manualResetEvent.WaitOne(100); + Assert.IsTrue(timeout, "Partition supervisor not started"); + + this.healthMonitor + .Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, exception), Times.Once); + + Mock.Get(this.leaseManager) + .Verify(manager => manager.ReleaseAsync(this.lease), Times.Once); } [TestMethod] @@ -262,8 +289,11 @@ public async Task AddLease_ShouldReleaseLease_IfLeaseAcquireThrows() await Assert.ThrowsExceptionAsync(() => this.sut.AddOrUpdateLeaseAsync(this.lease)).ConfigureAwait(false); - Mock.Get(this.leaseManager) - .Verify(manager => manager.ReleaseAsync(It.IsAny()), Times.Once); + this.healthMonitor + .Verify(m => m.NotifyLeaseAcquireAsync(this.lease.CurrentLeaseToken), Times.Never); + + this.healthMonitor + .Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once); } [TestMethod] @@ -287,6 +317,37 @@ public async Task AddLease_ShouldntRunObserver_IfLeaseAcquireThrows() Mock.Get(this.partitionProcessor) .Verify(processor => processor.RunAsync(It.IsAny()), Times.Never); + + this.healthMonitor + .Verify(m => m.NotifyErrorAsync(this.lease.CurrentLeaseToken, It.Is(ex => ex is NullReferenceException)), Times.Once); + + this.healthMonitor + .Verify(m => m.NotifyLeaseAcquireAsync(this.lease.CurrentLeaseToken), Times.Never); + + this.healthMonitor + .Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once); + } + + [TestMethod] + public async Task Shutdown_ShouldNotify_Monitor() + { + Mock.Get(this.leaseManager) + .Reset(); + + Mock.Get(this.partitionProcessor) + .Reset(); + + await this.sut.AddOrUpdateLeaseAsync(this.lease).ConfigureAwait(false); + + await Task.Delay(100); + + await this.sut.ShutdownAsync().ConfigureAwait(false); + + Mock.Get(this.leaseManager) + .Verify(manager => manager.ReleaseAsync(this.lease), Times.Once); + + this.healthMonitor + .Verify(m => m.NotifyLeaseReleaseAsync(this.lease.CurrentLeaseToken), Times.Once); } public Task InitializeAsync() From e92da8b3e4b70a7e95cbb75afb5e1391ae8e8fa7 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 11:46:31 -0700 Subject: [PATCH 05/22] public exception --- ...cs => ChangeFeedProcessorUserException.cs} | 14 ++++----- .../FeedManagement/PartitionSupervisorCore.cs | 2 +- .../ChangeFeedObserverFactoryCore.cs | 2 +- ...tionWrappingChangeFeedObserverDecorator.cs | 31 +++---------------- 4 files changed, 14 insertions(+), 35 deletions(-) rename Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/{ObserverException.cs => ChangeFeedProcessorUserException.cs} (72%) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ObserverException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs similarity index 72% rename from Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ObserverException.cs rename to Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs index b7161fee25..e8cc1e2c0a 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ObserverException.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs @@ -11,25 +11,25 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Exceptions /// Exception occurred when an operation in an IChangeFeedObserver is running and throws by user code /// [Serializable] - internal class ObserverException : Exception + public class ChangeFeedProcessorUserException : Exception { - private static readonly string DefaultMessage = "Exception has been thrown by the Observer."; + private static readonly string DefaultMessage = "Exception has been thrown by the change feed processor delegate."; /// - /// Initializes a new instance of the class using the specified internal exception. + /// Initializes a new instance of the class using the specified internal exception. /// /// thrown by the user code. - public ObserverException(Exception originalException) - : base(ObserverException.DefaultMessage, originalException) + public ChangeFeedProcessorUserException(Exception originalException) + : base(ChangeFeedProcessorUserException.DefaultMessage, originalException) { } /// - /// Initializes a new instance of the class using default values. + /// Initializes a new instance of the for serialization purposes. /// /// The SerializationInfo object that holds serialized object data for the exception being thrown. /// The StreamingContext that contains contextual information about the source or destination. - protected ObserverException(SerializationInfo info, StreamingContext context) + protected ChangeFeedProcessorUserException(SerializationInfo info, StreamingContext context) : this((Exception)info.GetValue("InnerException", typeof(Exception))) { } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs index f2ca81a9f4..93191ffb69 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs @@ -73,7 +73,7 @@ public override async Task RunAsync(CancellationToken shutdownToken) { closeReason = ChangeFeedObserverCloseReason.Shutdown; } - catch (ObserverException) + catch (ChangeFeedProcessorUserException) { closeReason = ChangeFeedObserverCloseReason.ObserverError; throw; diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs index ffbf338fc7..4d1bfa8e04 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs @@ -122,7 +122,7 @@ private IReadOnlyCollection AsIReadOnlyCollection(Stream stream) catch (Exception serializationException) { // Error using custom serializer to parse stream - throw new ObserverException(serializationException); + throw new ChangeFeedProcessorUserException(serializationException); } } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs index d0a7e2e4ba..cf66148599 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs @@ -9,7 +9,6 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; - using Microsoft.Azure.Cosmos.Core.Trace; internal sealed class ObserverExceptionWrappingChangeFeedObserverDecorator : ChangeFeedObserver { @@ -20,32 +19,14 @@ public ObserverExceptionWrappingChangeFeedObserverDecorator(ChangeFeedObserver c this.changeFeedObserver = changeFeedObserver ?? throw new ArgumentNullException(nameof(changeFeedObserver)); } - public override async Task CloseAsync(string leaseToken, ChangeFeedObserverCloseReason reason) + public override Task CloseAsync(string leaseToken, ChangeFeedObserverCloseReason reason) { - try - { - await this.changeFeedObserver.CloseAsync(leaseToken, reason).ConfigureAwait(false); - } - catch (Exception userException) - { - Extensions.TraceException(userException); - DefaultTrace.TraceWarning("Exception happened on Observer.CloseAsync"); - throw new ObserverException(userException); - } + return this.changeFeedObserver.CloseAsync(leaseToken, reason); } - public override async Task OpenAsync(string leaseToken) + public override Task OpenAsync(string leaseToken) { - try - { - await this.changeFeedObserver.OpenAsync(leaseToken).ConfigureAwait(false); - } - catch (Exception userException) - { - Extensions.TraceException(userException); - DefaultTrace.TraceWarning("Exception happened on Observer.OpenAsync"); - throw new ObserverException(userException); - } + return this.OpenAsync(leaseToken); } public override async Task ProcessChangesAsync(ChangeFeedObserverContextCore context, Stream stream, CancellationToken cancellationToken) @@ -56,9 +37,7 @@ public override async Task ProcessChangesAsync(ChangeFeedObserverContextCore con } catch (Exception userException) { - Extensions.TraceException(userException); - DefaultTrace.TraceWarning("Exception happened on Observer.ProcessChangesAsync"); - throw new ObserverException(userException); + throw new ChangeFeedProcessorUserException(userException); } } } From 0107f65bf815dd49438d0db199203c7e666413f3 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 11:48:37 -0700 Subject: [PATCH 06/22] tests --- .../ChangeFeed/Exceptions/ObserverExceptionTests.cs | 10 +++++----- .../ChangeFeed/FeedProcessorCoreTests.cs | 2 +- ...xceptionWrappingChangeFeedObserverDecoratorTests.cs | 4 ++-- .../ChangeFeed/PartitionSupervisorTests.cs | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs index f6cd67a71f..0e3ed5cd5a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs @@ -18,7 +18,7 @@ public class ObserverExceptionTests public void ValidateConstructor() { Exception exception = new Exception("randomMessage"); - ObserverException ex = new ObserverException(exception); + ChangeFeedProcessorUserException ex = new ChangeFeedProcessorUserException(exception); Assert.AreEqual(exception.Message, ex.InnerException.Message); Assert.AreEqual(exception, ex.InnerException); } @@ -28,14 +28,14 @@ public void ValidateConstructor() public void ValidateSerialization_AllFields() { Exception exception = new Exception("randomMessage"); - ObserverException originalException = new ObserverException(exception); + ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(exception); byte[] buffer = new byte[4096]; BinaryFormatter formatter = new BinaryFormatter(); MemoryStream stream1 = new MemoryStream(buffer); MemoryStream stream2 = new MemoryStream(buffer); formatter.Serialize(stream1, originalException); - ObserverException deserializedException = (ObserverException)formatter.Deserialize(stream2); + ChangeFeedProcessorUserException deserializedException = (ChangeFeedProcessorUserException)formatter.Deserialize(stream2); Assert.AreEqual(originalException.Message, deserializedException.Message); Assert.AreEqual(originalException.InnerException.Message, deserializedException.InnerException.Message); @@ -45,14 +45,14 @@ public void ValidateSerialization_AllFields() [TestMethod] public void ValidateSerialization_NullFields() { - ObserverException originalException = new ObserverException(null); + ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(null); byte[] buffer = new byte[4096]; BinaryFormatter formatter = new BinaryFormatter(); MemoryStream stream1 = new MemoryStream(buffer); MemoryStream stream2 = new MemoryStream(buffer); formatter.Serialize(stream1, originalException); - ObserverException deserializedException = (ObserverException)formatter.Deserialize(stream2); + ChangeFeedProcessorUserException deserializedException = (ChangeFeedProcessorUserException)formatter.Deserialize(stream2); Assert.AreEqual(originalException.Message, deserializedException.Message); Assert.IsNull(deserializedException.InnerException); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs index 5d362155ef..b1a8c22d84 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs @@ -73,7 +73,7 @@ public async Task ThrowsOnFailedCustomSerializer() ChangeFeedObserverFactoryCore factory = new ChangeFeedObserverFactoryCore(handler, new CosmosSerializerCore(serializer)); FeedProcessorCore processor = new FeedProcessorCore(factory.CreateObserver(), mockIterator.Object, FeedProcessorCoreTests.DefaultSettings, mockCheckpointer.Object); - ObserverException caughtException = await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); + ChangeFeedProcessorUserException caughtException = await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); Assert.IsInstanceOfType(caughtException.InnerException, typeof(CustomException)); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs index 0f960f1e49..1aa56e449c 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs @@ -90,7 +90,7 @@ public async Task ProcessChangesAsync_ShouldThrow_IfObserverThrows() await this.observerWrapper.ProcessChangesAsync(this.changeFeedObserverContext, stream, this.cancellationTokenSource.Token); Assert.Fail("Should had thrown"); } - catch (ObserverException ex) + catch (ChangeFeedProcessorUserException ex) { Assert.IsInstanceOfType(ex.InnerException, typeof(Exception)); } @@ -118,7 +118,7 @@ public async Task ProcessChangesAsync_ShouldThrow_IfObserverThrowsDocumentClient await this.observerWrapper.ProcessChangesAsync(this.changeFeedObserverContext, stream, this.cancellationTokenSource.Token); Assert.Fail("Should had thrown"); } - catch (ObserverException ex) + catch (ChangeFeedProcessorUserException ex) { Assert.IsInstanceOfType(ex.InnerException, typeof(Documents.DocumentClientException)); } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs index 18e617d7b0..bd6b8c77f5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs @@ -120,9 +120,9 @@ public async Task RunObserver_ShouldCloseWithObserverError_IfObserverFailed() { Mock.Get(this.partitionProcessor) .Setup(processor => processor.RunAsync(It.IsAny())) - .ThrowsAsync(new ObserverException(new Exception())); + .ThrowsAsync(new ChangeFeedProcessorUserException(new Exception())); - await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); + await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); Mock.Get(this.observer) .Verify(feedObserver => feedObserver From 66bc67feaffbece2aca4c69dcbf05dfbcb4bf6a3 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 12:41:47 -0700 Subject: [PATCH 07/22] Refactoring error messaging so users get diagnostics --- .../DocDBErrors/DocDbError.cs | 4 +- .../DocDBErrors/ExceptionClassifier.cs | 5 -- .../Exceptions/FeedNotFoundException.cs | 47 ------------- .../FeedReadSessionNotAvailableException.cs | 47 ------------- .../FeedManagement/PartitionSupervisorCore.cs | 9 +-- .../FeedProcessing/FeedProcessorCore.cs | 16 ++--- .../ChangeFeedObserverCloseReason.cs | 5 ++ ...tionWrappingChangeFeedObserverDecorator.cs | 2 +- .../Exceptions/FeedNotFoundExceptionTests.cs | 69 ------------------- ...edReadSessionNotAvailableExceptionTests.cs | 69 ------------------- .../ChangeFeed/FeedProcessorCoreTests.cs | 8 ++- ...rappingChangeFeedObserverDecoratorTests.cs | 4 +- .../ChangeFeed/PartitionSupervisorTests.cs | 24 ++----- 13 files changed, 27 insertions(+), 282 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedNotFoundException.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedReadSessionNotAvailableException.cs delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedNotFoundExceptionTests.cs delete mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedReadSessionNotAvailableExceptionTests.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/DocDbError.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/DocDbError.cs index 16a15973ec..61edfb7c7a 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/DocDbError.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/DocDbError.cs @@ -7,8 +7,6 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.DocDBErrors internal enum DocDbError { Undefined, - PartitionSplit, - PartitionNotFound, - ReadSessionNotAvailable + PartitionSplit } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/ExceptionClassifier.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/ExceptionClassifier.cs index 07e4271fa3..1aa272091d 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/ExceptionClassifier.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/DocDBErrors/ExceptionClassifier.cs @@ -18,11 +18,6 @@ public static DocDbError ClassifyStatusCodes( return DocDbError.PartitionSplit; } - if (statusCode == HttpStatusCode.NotFound) - { - return subStatusCode == (int)SubStatusCodes.ReadSessionNotAvailable ? DocDbError.ReadSessionNotAvailable : DocDbError.PartitionNotFound; - } - return DocDbError.Undefined; } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedNotFoundException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedNotFoundException.cs deleted file mode 100644 index 361c61109a..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedNotFoundException.cs +++ /dev/null @@ -1,47 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Exceptions -{ - using System; - using System.Runtime.Serialization; - - /// - /// Exception occurred when partition wasn't found. - /// - [Serializable] - internal class FeedNotFoundException : FeedException - { - /// - /// Initializes a new instance of the class using error message and last continuation token. - /// - /// The exception error message. - /// Request continuation token. - public FeedNotFoundException(string message, string lastContinuation) - : base(message, lastContinuation) - { - } - - /// - /// Initializes a new instance of the class using error message and inner exception. - /// - /// The exception error message. - /// The last known continuation token - /// The inner exception. - public FeedNotFoundException(string message, string lastContinuation, Exception innerException) - : base(message, lastContinuation, innerException) - { - } - - /// - /// Initializes a new instance of the class using default values. - /// - /// The SerializationInfo object that holds serialized object data for the exception being thrown. - /// The StreamingContext that contains contextual information about the source or destination. - protected FeedNotFoundException(SerializationInfo info, StreamingContext context) - : base(info, context) - { - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedReadSessionNotAvailableException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedReadSessionNotAvailableException.cs deleted file mode 100644 index 7aa6502451..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/FeedReadSessionNotAvailableException.cs +++ /dev/null @@ -1,47 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Exceptions -{ - using System; - using System.Runtime.Serialization; - - /// - /// Exception occurred when all retries on StatusCode.NotFound/SubStatusCode.ReadSessionNotAvaialable are over. - /// - [Serializable] - internal class FeedReadSessionNotAvailableException : FeedException - { - /// - /// Initializes a new instance of the class using error message and last continuation token. - /// - /// The exception error message. - /// Request continuation token. - public FeedReadSessionNotAvailableException(string message, string lastContinuation) - : base(message, lastContinuation) - { - } - - /// - /// Initializes a new instance of the class using error message and inner exception. - /// - /// The exception error message. - /// The last known continuation token - /// The inner exception. - public FeedReadSessionNotAvailableException(string message, string lastContinuation, Exception innerException) - : base(message, lastContinuation, innerException) - { - } - - /// - /// Initializes a new instance of the class using default values. - /// - /// The SerializationInfo object that holds serialized object data for the exception being thrown. - /// The StreamingContext that contains contextual information about the source or destination. - protected FeedReadSessionNotAvailableException(SerializationInfo info, StreamingContext context) - : base(info, context) - { - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs index 93191ffb69..0ce72c6642 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionSupervisorCore.cs @@ -59,14 +59,9 @@ public override async Task RunAsync(CancellationToken shutdownToken) closeReason = ChangeFeedObserverCloseReason.LeaseGone; throw; } - catch (FeedNotFoundException) + catch (CosmosException) { - closeReason = ChangeFeedObserverCloseReason.ResourceGone; - throw; - } - catch (FeedReadSessionNotAvailableException) - { - closeReason = ChangeFeedObserverCloseReason.ReadSessionNotAvailable; + closeReason = ChangeFeedObserverCloseReason.CosmosException; throw; } catch (OperationCanceledException) when (shutdownToken.IsCancellationRequested) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs index 382dee0502..11c59f9045 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedProcessorCore.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; using Microsoft.Azure.Cosmos.Core.Trace; + using Microsoft.Azure.Cosmos.Resource.CosmosExceptions; internal sealed class FeedProcessorCore : FeedProcessor { @@ -50,7 +51,7 @@ public override async Task RunAsync(CancellationToken cancellationToken) if (response.StatusCode != HttpStatusCode.NotModified && !response.IsSuccessStatusCode) { DefaultTrace.TraceWarning("unsuccessful feed read: lease token '{0}' status code {1}. substatuscode {2}", this.options.LeaseToken, response.StatusCode, response.Headers.SubStatusCode); - this.HandleFailedRequest(response.StatusCode, (int)response.Headers.SubStatusCode, lastContinuation); + this.HandleFailedRequest(response, lastContinuation); if (response.Headers.RetryAfter.HasValue) { @@ -87,25 +88,20 @@ public override async Task RunAsync(CancellationToken cancellationToken) } private void HandleFailedRequest( - HttpStatusCode statusCode, - int subStatusCode, + ResponseMessage responseMessage, string lastContinuation) { - DocDbError docDbError = ExceptionClassifier.ClassifyStatusCodes(statusCode, subStatusCode); + DocDbError docDbError = ExceptionClassifier.ClassifyStatusCodes(responseMessage.StatusCode, (int)responseMessage.Headers.SubStatusCode); switch (docDbError) { case DocDbError.PartitionSplit: throw new FeedRangeGoneException("Partition split.", lastContinuation); - case DocDbError.PartitionNotFound: - throw new FeedNotFoundException("Partition not found.", lastContinuation); - case DocDbError.ReadSessionNotAvailable: - throw new FeedReadSessionNotAvailableException("Read session not availalbe.", lastContinuation); case DocDbError.Undefined: - throw new InvalidOperationException($"Undefined DocDbError for status code {statusCode} and substatus code {subStatusCode}"); + throw CosmosExceptionFactory.Create(responseMessage); default: DefaultTrace.TraceCritical($"Unrecognized DocDbError enum value {docDbError}"); Debug.Fail($"Unrecognized DocDbError enum value {docDbError}"); - throw new InvalidOperationException($"Unrecognized DocDbError enum value {docDbError} for status code {statusCode} and substatus code {subStatusCode}"); + throw new InvalidOperationException($"Unrecognized DocDbError enum value {docDbError} for status code {responseMessage.StatusCode} and substatus code {responseMessage.Headers.SubStatusCode}"); } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverCloseReason.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverCloseReason.cs index 931334a679..a0966dc85e 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverCloseReason.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverCloseReason.cs @@ -44,5 +44,10 @@ internal enum ChangeFeedObserverCloseReason /// Note: SDK retries on this error. /// ReadSessionNotAvailable, + + /// + /// Related to a server response + /// + CosmosException, } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs index cf66148599..94e96a0ca6 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs @@ -26,7 +26,7 @@ public override Task CloseAsync(string leaseToken, ChangeFeedObserverCloseReason public override Task OpenAsync(string leaseToken) { - return this.OpenAsync(leaseToken); + return this.changeFeedObserver.OpenAsync(leaseToken); } public override async Task ProcessChangesAsync(ChangeFeedObserverContextCore context, Stream stream, CancellationToken cancellationToken) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedNotFoundExceptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedNotFoundExceptionTests.cs deleted file mode 100644 index ebf8c4a9b4..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedNotFoundExceptionTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests -{ - using System; - using System.IO; - using System.Runtime.Serialization.Formatters.Binary; - using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - [TestCategory("ChangeFeed")] - public class FeedNotFoundExceptionTests - { - [TestMethod] - public void ValidateRecommendedConstructors() - { - string message = "message"; - string lastContinuation = "lastContinuation"; - FeedNotFoundException ex = new FeedNotFoundException(message, lastContinuation); - Assert.AreEqual(message, ex.Message); - Assert.AreEqual(lastContinuation, ex.LastContinuation); - - Exception innerException = new Exception(); - ex = new FeedNotFoundException(message, lastContinuation, innerException); - Assert.AreEqual(message, ex.Message); - Assert.AreEqual(innerException, ex.InnerException); - Assert.AreEqual(lastContinuation, ex.LastContinuation); - } - - // Tests the GetObjectData method and the serialization ctor. - [TestMethod] - public void ValidateSerialization_AllFields() - { - FeedNotFoundException originalException = new FeedNotFoundException("message", "continuation", new Exception("foo")); - byte[] buffer = new byte[4096]; - BinaryFormatter formatter = new BinaryFormatter(); - MemoryStream stream1 = new MemoryStream(buffer); - MemoryStream stream2 = new MemoryStream(buffer); - - formatter.Serialize(stream1, originalException); - FeedNotFoundException deserializedException = (FeedNotFoundException)formatter.Deserialize(stream2); - - Assert.AreEqual(originalException.Message, deserializedException.Message); - Assert.AreEqual(originalException.InnerException.Message, deserializedException.InnerException.Message); - Assert.AreEqual(originalException.LastContinuation, deserializedException.LastContinuation); - } - - // Make sure that when some fields are not set, serialization is not broken. - [TestMethod] - public void ValidateSerialization_NullFields() - { - FeedNotFoundException originalException = new FeedNotFoundException("message", null); - byte[] buffer = new byte[4096]; - BinaryFormatter formatter = new BinaryFormatter(); - MemoryStream stream1 = new MemoryStream(buffer); - MemoryStream stream2 = new MemoryStream(buffer); - - formatter.Serialize(stream1, originalException); - FeedNotFoundException deserializedException = (FeedNotFoundException)formatter.Deserialize(stream2); - - Assert.AreEqual(originalException.Message, deserializedException.Message); - Assert.IsNull(deserializedException.InnerException); - Assert.IsNull(deserializedException.LastContinuation); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedReadSessionNotAvailableExceptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedReadSessionNotAvailableExceptionTests.cs deleted file mode 100644 index ea621ee00a..0000000000 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/FeedReadSessionNotAvailableExceptionTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests -{ - using System; - using System.IO; - using System.Runtime.Serialization.Formatters.Binary; - using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - [TestCategory("ChangeFeed")] - public class FeedReadSessionNotAvailableExceptionTests - { - [TestMethod] - public void ValidateRecommendedConstructors() - { - string message = "message"; - string lastContinuation = "lastContinuation"; - FeedReadSessionNotAvailableException ex = new FeedReadSessionNotAvailableException(message, lastContinuation); - Assert.AreEqual(message, ex.Message); - Assert.AreEqual(lastContinuation, ex.LastContinuation); - - Exception innerException = new Exception(); - ex = new FeedReadSessionNotAvailableException(message, lastContinuation, innerException); - Assert.AreEqual(message, ex.Message); - Assert.AreEqual(innerException, ex.InnerException); - Assert.AreEqual(lastContinuation, ex.LastContinuation); - } - - // Tests the GetObjectData method and the serialization ctor. - [TestMethod] - public void ValidateSerialization_AllFields() - { - FeedReadSessionNotAvailableException originalException = new FeedReadSessionNotAvailableException("message", "continuation", new Exception("foo")); - byte[] buffer = new byte[4096]; - BinaryFormatter formatter = new BinaryFormatter(); - MemoryStream stream1 = new MemoryStream(buffer); - MemoryStream stream2 = new MemoryStream(buffer); - - formatter.Serialize(stream1, originalException); - FeedReadSessionNotAvailableException deserializedException = (FeedReadSessionNotAvailableException)formatter.Deserialize(stream2); - - Assert.AreEqual(originalException.Message, deserializedException.Message); - Assert.AreEqual(originalException.InnerException.Message, deserializedException.InnerException.Message); - Assert.AreEqual(originalException.LastContinuation, deserializedException.LastContinuation); - } - - // Make sure that when some fields are not set, serialization is not broken. - [TestMethod] - public void ValidateSerialization_NullFields() - { - FeedReadSessionNotAvailableException originalException = new FeedReadSessionNotAvailableException("message", null); - byte[] buffer = new byte[4096]; - BinaryFormatter formatter = new BinaryFormatter(); - MemoryStream stream1 = new MemoryStream(buffer); - MemoryStream stream2 = new MemoryStream(buffer); - - formatter.Serialize(stream1, originalException); - FeedReadSessionNotAvailableException deserializedException = (FeedReadSessionNotAvailableException)formatter.Deserialize(stream2); - - Assert.AreEqual(originalException.Message, deserializedException.Message); - Assert.IsNull(deserializedException.InnerException); - Assert.IsNull(deserializedException.LastContinuation); - } - } -} diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs index b1a8c22d84..06aceab12e 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedProcessorCoreTests.cs @@ -111,7 +111,9 @@ public async Task ThrowOnPartitionGone(HttpStatusCode statusCode, int subStatusC FeedProcessorCore processor = new FeedProcessorCore(mockObserver.Object, mockIterator.Object, FeedProcessorCoreTests.DefaultSettings, mockCheckpointer.Object); - await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); + CosmosException cosmosException = await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); + Assert.AreEqual(statusCode, cosmosException.StatusCode); + Assert.AreEqual(subStatusCode, (int)cosmosException.Headers.SubStatusCode); } [DataRow(HttpStatusCode.NotFound, (int)Documents.SubStatusCodes.ReadSessionNotAvailable)] @@ -133,7 +135,9 @@ public async Task ThrowOnReadSessionNotAvailable(HttpStatusCode statusCode, int FeedProcessorCoreTests.DefaultSettings, mockCheckpointer.Object); - await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); + CosmosException cosmosException = await Assert.ThrowsExceptionAsync(() => processor.RunAsync(cancellationTokenSource.Token)); + Assert.AreEqual(statusCode, cosmosException.StatusCode); + Assert.AreEqual(subStatusCode, (int)cosmosException.Headers.SubStatusCode); } private static ResponseMessage GetResponse(HttpStatusCode statusCode, bool includeItem, int subStatusCode = 0) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs index 1aa56e449c..4aa21cd8a5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ObserverExceptionWrappingChangeFeedObserverDecoratorTests.cs @@ -111,7 +111,7 @@ public async Task ProcessChangesAsync_ShouldThrow_IfObserverThrowsDocumentClient Mock.Get(this.observer.Object) .SetupSequence(feedObserver => feedObserver .ProcessChangesAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .Throws(new Documents.DocumentClientException("Some message", (HttpStatusCode) 429, Documents.SubStatusCodes.Unknown)); + .Throws(new CosmosException("Some message", (HttpStatusCode) 429, (int)Documents.SubStatusCodes.Unknown, Guid.NewGuid().ToString(), 0)); try { @@ -120,7 +120,7 @@ public async Task ProcessChangesAsync_ShouldThrow_IfObserverThrowsDocumentClient } catch (ChangeFeedProcessorUserException ex) { - Assert.IsInstanceOfType(ex.InnerException, typeof(Documents.DocumentClientException)); + Assert.IsInstanceOfType(ex.InnerException, typeof(CosmosException)); } Mock.Get(this.observer.Object) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs index bd6b8c77f5..4c604ca1ab 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs @@ -144,35 +144,19 @@ public async Task RunObserver_ShouldPassPartitionToObserver_WhenExecuted() } [TestMethod] - public async Task RunObserver_ResourceGoneCloseReason_IfProcessorFailedWithPartitionNotFoundException() + public async Task RunObserver_CosmosException_IfProcessorFailedWithCosmosException() { Mock.Get(this.partitionProcessor) .Setup(processor => processor.RunAsync(It.IsAny())) - .ThrowsAsync(new FeedNotFoundException("processorException", "12345")); + .ThrowsAsync(new CosmosException("processorException", System.Net.HttpStatusCode.NotFound, 0, Guid.NewGuid().ToString(), 0)); - Exception exception = await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); + Exception exception = await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); Assert.AreEqual("processorException", exception.Message); Mock.Get(this.observer) .Verify(feedObserver => feedObserver .CloseAsync(It.Is(lt => lt == this.lease.CurrentLeaseToken), - ChangeFeedObserverCloseReason.ResourceGone)); - } - - [TestMethod] - public async Task RunObserver_ReadSessionNotAvailableCloseReason_IfProcessorFailedWithReadSessionNotAvailableException() - { - Mock.Get(this.partitionProcessor) - .Setup(processor => processor.RunAsync(It.IsAny())) - .ThrowsAsync(new FeedReadSessionNotAvailableException("processorException", "12345")); - - Exception exception = await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); - Assert.AreEqual("processorException", exception.Message); - - Mock.Get(this.observer) - .Verify(feedObserver => feedObserver - .CloseAsync(It.Is(lt => lt == this.lease.CurrentLeaseToken), - ChangeFeedObserverCloseReason.ReadSessionNotAvailable)); + ChangeFeedObserverCloseReason.CosmosException)); } [TestMethod] From 0bbcfcabff4873f9b707b366209831819f96de74 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 13:19:46 -0700 Subject: [PATCH 08/22] Wiring through estimator --- .../src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs | 7 +++++-- .../FeedProcessing/FeedEstimatorRunner.cs | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs index e266c7ad7e..d1eaf930a2 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs @@ -22,6 +22,7 @@ internal sealed class ChangeFeedEstimatorRunner : ChangeFeedProcessor private const string EstimatorDefaultHostName = "Estimator"; private readonly ChangesEstimationHandler initialEstimateDelegate; private readonly TimeSpan? estimatorPeriod; + private ChangeFeedProcessorHealthMonitor healthMonitor; private CancellationTokenSource shutdownCts; private ContainerInternal leaseContainer; private ContainerInternal monitoredContainer; @@ -54,7 +55,8 @@ internal ChangeFeedEstimatorRunner( this.remainingWorkEstimator = remainingWorkEstimator; } - private ChangeFeedEstimatorRunner(TimeSpan? estimatorPeriod) + private ChangeFeedEstimatorRunner( + TimeSpan? estimatorPeriod) { if (estimatorPeriod.HasValue && estimatorPeriod.Value <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(estimatorPeriod)); @@ -75,6 +77,7 @@ public void ApplyBuildConfiguration( this.monitoredContainer = monitoredContainer ?? throw new ArgumentNullException(nameof(monitoredContainer)); this.changeFeedLeaseOptions = changeFeedLeaseOptions; this.documentServiceLeaseContainer = customDocumentServiceLeaseStoreManager?.LeaseContainer; + this.healthMonitor = changeFeedProcessorOptions.HealthMonitor; } public override async Task StartAsync() @@ -128,7 +131,7 @@ private FeedEstimatorRunner BuildFeedEstimatorRunner() this.documentServiceLeaseContainer); } - return new FeedEstimatorRunner(this.initialEstimateDelegate, this.remainingWorkEstimator, this.estimatorPeriod); + return new FeedEstimatorRunner(this.initialEstimateDelegate, this.remainingWorkEstimator, this.healthMonitor, this.estimatorPeriod); } private async Task InitializeLeaseStoreAsync() diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedEstimatorRunner.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedEstimatorRunner.cs index 8038ff2735..27d04e3738 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedEstimatorRunner.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedProcessing/FeedEstimatorRunner.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing using System.Linq; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Core.Trace; using static Microsoft.Azure.Cosmos.Container; /// @@ -16,18 +15,22 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.FeedProcessing /// internal sealed class FeedEstimatorRunner { + private static readonly string EstimationLeaseIdentifier = "Change Feed Estimator"; private static TimeSpan defaultMonitoringDelay = TimeSpan.FromSeconds(5); private readonly ChangeFeedEstimator remainingWorkEstimator; private readonly TimeSpan monitoringDelay; private readonly ChangesEstimationHandler dispatchEstimation; + private readonly ChangeFeedProcessorHealthMonitor healthMonitor; public FeedEstimatorRunner( ChangesEstimationHandler dispatchEstimation, ChangeFeedEstimator remainingWorkEstimator, + ChangeFeedProcessorHealthMonitor healthMonitor, TimeSpan? estimationPeriod = null) { this.dispatchEstimation = dispatchEstimation; this.remainingWorkEstimator = remainingWorkEstimator; + this.healthMonitor = healthMonitor; this.monitoringDelay = estimationPeriod ?? FeedEstimatorRunner.defaultMonitoringDelay; } @@ -64,8 +67,7 @@ private async Task EstimateAsync(CancellationToken cancellationToken) } catch (Exception userException) { - Extensions.TraceException(userException); - DefaultTrace.TraceWarning("Exception happened on ChangeFeedEstimatorDispatcher.DispatchEstimation"); + await this.healthMonitor.NotifyErrorAsync(FeedEstimatorRunner.EstimationLeaseIdentifier, userException); } } From 05ba82ac0d485e18aeee8105740e8dfc9d44c425 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 13:22:18 -0700 Subject: [PATCH 09/22] tests --- .../ChangeFeed/FeedEstimatorRunnerTests.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedEstimatorRunnerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedEstimatorRunnerTests.cs index 7e219810ed..216d67ba4a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedEstimatorRunnerTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/FeedEstimatorRunnerTests.cs @@ -40,7 +40,7 @@ Task estimatorDispatcher(long detectedEstimation, CancellationToken token) Mock mockedEstimator = new Mock(); mockedEstimator.Setup(e => e.GetCurrentStateIterator(It.IsAny())).Returns(mockedIterator.Object); - FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, TimeSpan.FromMilliseconds(10)); + FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, Mock.Of(), TimeSpan.FromMilliseconds(10)); try { @@ -71,15 +71,18 @@ Task estimatorDispatcher(long detectedEstimation, CancellationToken token) mockedResponse.Setup(r => r.Count).Returns(1); mockedResponse.Setup(r => r.GetEnumerator()).Returns(new List() { new ChangeFeedProcessorState(string.Empty, estimation, string.Empty) }.GetEnumerator()); + CosmosException exception = CosmosExceptionFactory.CreateThrottledException("throttled", new Headers()); Mock> mockedIterator = new Mock>(); mockedIterator.SetupSequence(i => i.ReadNextAsync(It.IsAny())) - .ThrowsAsync(CosmosExceptionFactory.CreateThrottledException("throttled", new Headers())) + .ThrowsAsync(exception) .ReturnsAsync(mockedResponse.Object); Mock mockedEstimator = new Mock(); mockedEstimator.Setup(e => e.GetCurrentStateIterator(It.IsAny())).Returns(mockedIterator.Object); - FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, TimeSpan.FromMilliseconds(10)); + Mock healthMonitor = new Mock(); + + FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, healthMonitor.Object, TimeSpan.FromMilliseconds(10)); try { @@ -92,6 +95,9 @@ Task estimatorDispatcher(long detectedEstimation, CancellationToken token) Assert.IsTrue(detectedEstimationCorrectly); mockedIterator.Verify(i => i.ReadNextAsync(It.IsAny()), Times.Exactly(2)); + + healthMonitor + .Verify(m => m.NotifyErrorAsync(It.IsAny(), exception), Times.Once); } [TestMethod] @@ -116,7 +122,7 @@ Task estimatorDispatcher(long detectedEstimation, CancellationToken token) Mock mockedEstimator = new Mock(); mockedEstimator.Setup(e => e.GetCurrentStateIterator(It.IsAny())).Returns(mockedIterator.Object); - FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, TimeSpan.FromMilliseconds(10)); + FeedEstimatorRunner estimatorCore = new FeedEstimatorRunner(estimatorDispatcher, mockedEstimator.Object, Mock.Of(), TimeSpan.FromMilliseconds(10)); try { From fd3760f8a4728c19bff4d311d7d35571ee7bcc72 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 9 Jul 2021 13:31:08 -0700 Subject: [PATCH 10/22] undoing and cleaning up --- .../ChangeFeedEstimatorRunner.cs | 3 +-- .../FeedManagement/PartitionControllerCore.cs | 2 -- .../Monitoring/TraceHealthMonitor.cs | 16 +++++++++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs index d1eaf930a2..8f8a8b9d81 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedEstimatorRunner.cs @@ -55,8 +55,7 @@ internal ChangeFeedEstimatorRunner( this.remainingWorkEstimator = remainingWorkEstimator; } - private ChangeFeedEstimatorRunner( - TimeSpan? estimatorPeriod) + private ChangeFeedEstimatorRunner(TimeSpan? estimatorPeriod) { if (estimatorPeriod.HasValue && estimatorPeriod.Value <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(estimatorPeriod)); diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs index 797a0a543a..c6a30cd65f 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/FeedManagement/PartitionControllerCore.cs @@ -65,7 +65,6 @@ public override async Task AddOrUpdateLeaseAsync(DocumentServiceLease lease) lease = updatedLease; } - DefaultTrace.TraceInformation("Lease with token {0}: acquired", lease.CurrentLeaseToken); await this.monitor.NotifyLeaseAcquireAsync(lease.CurrentLeaseToken); } catch (Exception ex) @@ -110,7 +109,6 @@ private async Task RemoveLeaseAsync(DocumentServiceLease lease) { await this.leaseManager.ReleaseAsync(lease).ConfigureAwait(false); - DefaultTrace.TraceInformation("Lease with token {0}: released", lease.CurrentLeaseToken); await this.monitor.NotifyLeaseReleaseAsync(lease.CurrentLeaseToken); } catch (Exception ex) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs index 75e9742362..a7222d23e9 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs @@ -9,10 +9,24 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring using Microsoft.Azure.Cosmos.Core.Trace; /// - /// A monitor which logs the errors only as traces + /// A monitor which uses the default trace /// internal sealed class TraceHealthMonitor : ChangeFeedProcessorHealthMonitor { + public override Task NotifyLeaseAcquireAsync(string leaseToken) + { + DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); + + return Task.CompletedTask; + } + + public override Task NotifyLeaseReleaseAsync(string leaseToken) + { + DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); + + return Task.CompletedTask; + } + public override Task NotifyErrorAsync( string leaseToken, Exception exception) From a30b96900d8b0f1a28764778fd8202279cd896cc Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 16:03:20 -0700 Subject: [PATCH 11/22] Rename of base implementation and refactor of public API --- .../ChangeFeedProcessorBuilder.cs | 65 ++++++++++++++- .../ChangeFeedProcessorOptions.cs | 2 +- .../ChangeFeedProcessorUserException.cs | 8 +- .../ChangeFeedProcessorHealthMonitor.cs | 21 +---- .../ChangeFeedProcessorHealthMonitorCore.cs | 79 +++++++++++++++++++ .../Monitoring/TraceHealthMonitor.cs | 40 ---------- .../src/Resource/Container/Container.cs | 24 ++++++ .../Resource/Container/ContainerInternal.cs | 8 ++ 8 files changed, 184 insertions(+), 63 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs delete mode 100644 Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs index a89945864b..7bbe5794b5 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs @@ -7,6 +7,11 @@ namespace Microsoft.Azure.Cosmos using System; using Microsoft.Azure.Cosmos.ChangeFeed.Configuration; using Microsoft.Azure.Cosmos.ChangeFeed.LeaseManagement; +#if PREVIEW + using static Microsoft.Azure.Cosmos.Container; +#else + using static Microsoft.Azure.Cosmos.ContainerInternal; +#endif /// /// Provides a flexible way to create an instance of with custom set of parameters. @@ -210,13 +215,65 @@ internal virtual ChangeFeedProcessorBuilder WithInMemoryLeaseContainer() } /// - /// Sets a health monitor to log events happening during the lifecycle of the processor. + /// Defines a delegate to receive notifications on errors that occur during change feed processor execution. /// - /// An implementation of to log lifecycle events. + /// A delegate to receive notifications for change feed processor related errors. /// The instance of to use. - public ChangeFeedProcessorBuilder WithHealthMonitor(ChangeFeedProcessorHealthMonitor healthMonitor) +#if PREVIEW + public +#else + internal +#endif + ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate errorDelegate) { - this.changeFeedProcessorOptions.HealthMonitor = healthMonitor; + if (errorDelegate == null) + { + throw new ArgumentNullException(nameof(errorDelegate)); + } + + this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(errorDelegate); + return this; + } + + /// + /// Defines a delegate to receive notifications on lease acquires that occur during change feed processor execution. + /// + /// A delegate to receive notifications when a change feed processor acquires a lease. + /// The instance of to use. +#if PREVIEW + public +#else + internal +#endif + ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAcquireDelegate acquireDelegate) + { + if (acquireDelegate == null) + { + throw new ArgumentNullException(nameof(acquireDelegate)); + } + + this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(acquireDelegate); + return this; + } + + /// + /// Defines a delegate to receive notifications on lease releases that occur during change feed processor execution. + /// + /// A delegate to receive notifications when a change feed processor releases a lease. + /// The instance of to use. +#if PREVIEW + public +#else + internal +#endif + ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseReleaseDelegate releaseDelegate) + { + if (releaseDelegate == null) + { + throw new ArgumentNullException(nameof(releaseDelegate)); + } + + this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(releaseDelegate); return this; } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs index 22afb060ca..81e38dde16 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Configuration/ChangeFeedProcessorOptions.cs @@ -83,6 +83,6 @@ public DateTime? StartTime /// public bool StartFromBeginning { get; set; } - public ChangeFeedProcessorHealthMonitor HealthMonitor { get; set; } = new TraceHealthMonitor(); + public ChangeFeedProcessorHealthMonitorCore HealthMonitor { get; set; } = new ChangeFeedProcessorHealthMonitorCore(); } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs index e8cc1e2c0a..9d29759379 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs @@ -11,7 +11,13 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Exceptions /// Exception occurred when an operation in an IChangeFeedObserver is running and throws by user code /// [Serializable] - public class ChangeFeedProcessorUserException : Exception + +#if PREVIEW + public +#else + internal +#endif + class ChangeFeedProcessorUserException : Exception { private static readonly string DefaultMessage = "Exception has been thrown by the change feed processor delegate."; diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs index e9095b1384..91685c3caf 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitor.cs @@ -10,29 +10,21 @@ namespace Microsoft.Azure.Cosmos /// /// Health monitor to capture lifecycle events of the Change Feed Processor. /// - public abstract class ChangeFeedProcessorHealthMonitor + internal abstract class ChangeFeedProcessorHealthMonitor { /// /// For normal informational events happening on the context of a lease /// /// A unique identifier for the lease. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyLeaseAcquireAsync( - string leaseToken) - { - return Task.CompletedTask; - } + public abstract Task NotifyLeaseAcquireAsync(string leaseToken); /// /// For transient errors that the Change Feed Processor attemps retrying on. /// /// A unique identifier for the lease. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyLeaseReleaseAsync( - string leaseToken) - { - return Task.CompletedTask; - } + public abstract Task NotifyLeaseReleaseAsync(string leaseToken); /// /// For critical errors that the Change Feed Processor that might not be recoverable. @@ -40,11 +32,6 @@ public virtual Task NotifyLeaseReleaseAsync( /// A unique identifier for the lease. /// The exception that happened. /// An asynchronous operation representing the logging operation. - public virtual Task NotifyErrorAsync( - string leaseToken, - Exception exception) - { - return Task.CompletedTask; - } + public abstract Task NotifyErrorAsync(string leaseToken, Exception exception); } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs new file mode 100644 index 0000000000..b206e322ce --- /dev/null +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs @@ -0,0 +1,79 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring +{ + using System; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Core.Trace; +#if PREVIEW + using static Microsoft.Azure.Cosmos.Container; +#else + using static Microsoft.Azure.Cosmos.ContainerInternal; +#endif + + /// + /// A monitor which uses the default trace + /// + internal sealed class ChangeFeedProcessorHealthMonitorCore : ChangeFeedProcessorHealthMonitor + { + private ChangeFeedMonitorErrorDelegate errorDelegate; + private ChangeFeedMonitorLeaseAcquireDelegate acquireDelegate; + private ChangeFeedMonitorLeaseReleaseDelegate releaseDelegate; + + public void SetDelegate(ChangeFeedMonitorErrorDelegate delegateCallback) + { + this.errorDelegate = delegateCallback; + } + + public void SetDelegate(ChangeFeedMonitorLeaseAcquireDelegate delegateCallback) + { + this.acquireDelegate = delegateCallback; + } + + public void SetDelegate(ChangeFeedMonitorLeaseReleaseDelegate delegateCallback) + { + this.releaseDelegate = delegateCallback; + } + + public override Task NotifyLeaseAcquireAsync(string leaseToken) + { + if (this.acquireDelegate != null) + { + return this.acquireDelegate(leaseToken); + } + + DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); + + return Task.CompletedTask; + } + + public override Task NotifyLeaseReleaseAsync(string leaseToken) + { + if (this.releaseDelegate != null) + { + return this.releaseDelegate(leaseToken); + } + + DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); + + return Task.CompletedTask; + } + + public override Task NotifyErrorAsync( + string leaseToken, + Exception exception) + { + if (this.errorDelegate != null) + { + return this.errorDelegate(leaseToken, exception); + } + + Extensions.TraceException(exception); + DefaultTrace.TraceError($"Error detected for lease {leaseToken}. "); + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs deleted file mode 100644 index a7222d23e9..0000000000 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/TraceHealthMonitor.cs +++ /dev/null @@ -1,40 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.ChangeFeed.Monitoring -{ - using System; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Core.Trace; - - /// - /// A monitor which uses the default trace - /// - internal sealed class TraceHealthMonitor : ChangeFeedProcessorHealthMonitor - { - public override Task NotifyLeaseAcquireAsync(string leaseToken) - { - DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); - - return Task.CompletedTask; - } - - public override Task NotifyLeaseReleaseAsync(string leaseToken) - { - DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); - - return Task.CompletedTask; - } - - public override Task NotifyErrorAsync( - string leaseToken, - Exception exception) - { - Extensions.TraceException(exception); - DefaultTrace.TraceError($"Error detected for lease {leaseToken}. "); - - return Task.CompletedTask; - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs index be357fd6b0..d60e123ed0 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs @@ -1588,6 +1588,30 @@ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( string processorName, ChangeFeedStreamHandlerWithManualCheckpoint onChangesDelegate); + + /// + /// Delegate to notify errors during change feed operations. + /// + /// A unique identifier for the lease. + /// The exception that happened. + /// A representing the asynchronous operation that is going to be done with the notification. + public delegate Task ChangeFeedMonitorErrorDelegate( + string leaseToken, + Exception exception); + + /// + /// Delegate to notify events of leases being acquired by a change feed processor. + /// + /// A unique identifier for the lease. + /// A representing the asynchronous operation that is going to be done with the notification. + public delegate Task ChangeFeedMonitorLeaseAcquireDelegate(string leaseToken); + + /// + /// Delegate to notify events of leases being releases by a change feed processor. + /// + /// A unique identifier for the lease. + /// A representing the asynchronous operation that is going to be done with the notification. + public delegate Task ChangeFeedMonitorLeaseReleaseDelegate(string leaseToken); #endif } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInternal.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInternal.cs index 2427c322fd..14474d0a3c 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInternal.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/ContainerInternal.cs @@ -200,6 +200,14 @@ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( string processorName, ChangeFeedStreamHandlerWithManualCheckpoint onChangesDelegate); + + public delegate Task ChangeFeedMonitorErrorDelegate( + string leaseToken, + Exception exception); + + public delegate Task ChangeFeedMonitorLeaseAcquireDelegate(string leaseToken); + + public delegate Task ChangeFeedMonitorLeaseReleaseDelegate(string leaseToken); #endif public abstract class TryExecuteQueryResult From 7bc0107e8c10f5fb99eb68069ade276845752d6b Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 16:20:35 -0700 Subject: [PATCH 12/22] more unit tests --- .../ChangeFeedProcessorBuilder.cs | 6 +- .../ChangeFeedProcessorHealthMonitorCore.cs | 6 +- ...angeFeedProcessorHealthMonitorCoreTests.cs | 72 +++++++++++++++++++ 3 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs index 7bbe5794b5..bb01205e16 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorBuilder.cs @@ -231,7 +231,7 @@ ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate throw new ArgumentNullException(nameof(errorDelegate)); } - this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(errorDelegate); + this.changeFeedProcessorOptions.HealthMonitor.SetErrorDelegate(errorDelegate); return this; } @@ -252,7 +252,7 @@ ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAc throw new ArgumentNullException(nameof(acquireDelegate)); } - this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(acquireDelegate); + this.changeFeedProcessorOptions.HealthMonitor.SetLeaseAcquireDelegate(acquireDelegate); return this; } @@ -273,7 +273,7 @@ ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseRe throw new ArgumentNullException(nameof(releaseDelegate)); } - this.changeFeedProcessorOptions.HealthMonitor.SetDelegate(releaseDelegate); + this.changeFeedProcessorOptions.HealthMonitor.SetLeaseReleaseDelegate(releaseDelegate); return this; } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs index b206e322ce..d4581d088c 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs @@ -22,17 +22,17 @@ internal sealed class ChangeFeedProcessorHealthMonitorCore : ChangeFeedProcessor private ChangeFeedMonitorLeaseAcquireDelegate acquireDelegate; private ChangeFeedMonitorLeaseReleaseDelegate releaseDelegate; - public void SetDelegate(ChangeFeedMonitorErrorDelegate delegateCallback) + public void SetErrorDelegate(ChangeFeedMonitorErrorDelegate delegateCallback) { this.errorDelegate = delegateCallback; } - public void SetDelegate(ChangeFeedMonitorLeaseAcquireDelegate delegateCallback) + public void SetLeaseAcquireDelegate(ChangeFeedMonitorLeaseAcquireDelegate delegateCallback) { this.acquireDelegate = delegateCallback; } - public void SetDelegate(ChangeFeedMonitorLeaseReleaseDelegate delegateCallback) + public void SetLeaseReleaseDelegate(ChangeFeedMonitorLeaseReleaseDelegate delegateCallback) { this.releaseDelegate = delegateCallback; } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs new file mode 100644 index 0000000000..4adab1756d --- /dev/null +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs @@ -0,0 +1,72 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests +{ + using System; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.ChangeFeed.Monitoring; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + [TestCategory("ChangeFeed")] + public class ChangeFeedProcessorHealthMonitorCoreTests + { + [TestMethod] + public async Task Delegates_CallsAcquire() + { + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetLeaseAcquireDelegate((string leaseToken) => + { + called = true; + Assert.AreEqual(token, leaseToken); + return Task.CompletedTask; + }); + + await monitor.NotifyLeaseAcquireAsync(token); + + Assert.IsTrue(called); + } + + [TestMethod] + public async Task Delegates_CallsRelease() + { + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetLeaseReleaseDelegate((string leaseToken) => + { + called = true; + Assert.AreEqual(token, leaseToken); + return Task.CompletedTask; + }); + + await monitor.NotifyLeaseReleaseAsync(token); + + Assert.IsTrue(called); + } + + [TestMethod] + public async Task Delegates_CallsError() + { + Exception ex = new Exception(); + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetErrorDelegate((string leaseToken, Exception exception) => + { + called = true; + Assert.AreEqual(token, leaseToken); + Assert.ReferenceEquals(ex, exception); + return Task.CompletedTask; + }); + + await monitor.NotifyErrorAsync(token, ex); + + Assert.IsTrue(called); + } + } +} From bc023ef5eb409d2da96bb73028307eb6df95796a Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 16:35:17 -0700 Subject: [PATCH 13/22] emulator tests --- .../ChangeFeedProcessorUserException.cs | 2 +- .../ChangeFeed/DynamicTests.cs | 28 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs index 9d29759379..d51ff3984a 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.ChangeFeed.Exceptions +namespace Microsoft.Azure.Cosmos { using System; using System.Runtime.Serialization; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs index cc93082c3d..6f25fdb1a6 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs @@ -83,6 +83,10 @@ public async Task TestWithRunningProcessor() [TestMethod] public async Task TestWithRunningProcessor_WithManualCheckpoint() { + int leaseAcquireCount = 0; + int leaseReleaseCount = 0; + int errorCount = 0; + Exception exceptionToPropagate = new Exception("Stop here"); int partitionKey = 0; ManualResetEvent allDocsProcessed = new ManualResetEvent(false); @@ -101,7 +105,7 @@ public async Task TestWithRunningProcessor_WithManualCheckpoint() if (processedDocCount == 3) { // Throwing on the 3rd document, since we checkpointed only on the 1st, we would repeat 2nd and 3rd - throw new Exception("Stop here"); + throw exceptionToPropagate; } if (processedDocCount == 1) { @@ -117,6 +121,23 @@ public async Task TestWithRunningProcessor_WithManualCheckpoint() }) .WithInstanceName("random") .WithMaxItems(1) + .WithLeaseAcquireNotification((string leaseToken) => + { + leaseAcquireCount++; + return Task.CompletedTask; + }) + .WithLeaseReleaseNotification((string leaseToken) => + { + leaseReleaseCount++; + return Task.CompletedTask; + }) + .WithErrorNotification((string leaseToken, Exception exception) => + { + errorCount++; + Assert.IsTrue(exception is ChangeFeedProcessorUserException cfpException); + Assert.ReferenceEquals(exceptionToPropagate, exception.InnerException); + return Task.CompletedTask; + }) .WithLeaseContainer(this.LeaseContainer).Build(); // Start the processor, insert 1 document to generate a checkpoint @@ -131,6 +152,11 @@ public async Task TestWithRunningProcessor_WithManualCheckpoint() await processor.StopAsync(); Assert.IsTrue(isStartOk, "Timed out waiting for docs to process"); Assert.AreEqual("0.1.2.1.2.3.4.5.6.7.8.9.", accumulator); + + // Make sure the notification APIs got all the events + Assert.IsTrue(leaseAcquireCount > 0); + Assert.IsTrue(leaseReleaseCount > 0); + Assert.AreEqual(1, errorCount); } [TestMethod] From bece93f6b438bf8f1666366fd70f2d0f61e8ce08 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 16:41:40 -0700 Subject: [PATCH 14/22] docs --- .../src/Resource/Container/Container.cs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs index d60e123ed0..675288ea6b 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs @@ -1595,6 +1595,27 @@ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu /// A unique identifier for the lease. /// The exception that happened. /// A representing the asynchronous operation that is going to be done with the notification. + /// + /// + /// + /// { + /// if (exception is ChangeFeedProcessorUserException userException) + /// { + /// Console.WriteLine($"Current instance's delegate had an unhandled when processing lease {leaseToken}."); + /// Console.WriteLine(userException.ToString()); + /// } + /// else + /// { + /// Console.WriteLine($"Current instance faced an exception when processing lease {leaseToken}."); + /// Console.WriteLine(exception.ToString()); + /// } + /// + /// return Task.CompletedTask; + /// } + /// ]]> + /// + /// public delegate Task ChangeFeedMonitorErrorDelegate( string leaseToken, Exception exception); @@ -1604,6 +1625,18 @@ public delegate Task ChangeFeedMonitorErrorDelegate( /// /// A unique identifier for the lease. /// A representing the asynchronous operation that is going to be done with the notification. + /// + /// + /// + /// { + /// Console.WriteLine($"Current instance released lease {leaseToken} and stopped processing it."); + /// + /// return Task.CompletedTask; + /// } + /// ]]> + /// + /// public delegate Task ChangeFeedMonitorLeaseAcquireDelegate(string leaseToken); /// @@ -1611,6 +1644,18 @@ public delegate Task ChangeFeedMonitorErrorDelegate( /// /// A unique identifier for the lease. /// A representing the asynchronous operation that is going to be done with the notification. + /// + /// + /// + /// { + /// Console.WriteLine($"Current instance acquired lease {leaseToken} and will start processing it."); + /// + /// return Task.CompletedTask; + /// } + /// ]]> + /// + /// public delegate Task ChangeFeedMonitorLeaseReleaseDelegate(string leaseToken); #endif } From b3893bb393d7f5facb9a8c4272ec9f798578a8f1 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 17:06:38 -0700 Subject: [PATCH 15/22] adding context to exception --- .../ChangeFeedProcessorContext.cs | 3 ++- .../ChangeFeedProcessorUserException.cs | 13 +++++++++++-- .../Observers/ChangeFeedObserverFactoryCore.cs | 8 +++++--- .../Observers/ChangeFeedProcessorContextCore.cs | 1 + ...xceptionWrappingChangeFeedObserverDecorator.cs | 2 +- .../src/Resource/Container/Container.cs | 2 ++ .../ChangeFeed/DynamicTests.cs | 5 ++++- .../Exceptions/ObserverExceptionTests.cs | 15 ++++++++++++--- .../ChangeFeed/PartitionSupervisorTests.cs | 2 +- 9 files changed, 39 insertions(+), 12 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs index 92a71ef4fe..92224c193f 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs @@ -4,10 +4,11 @@ namespace Microsoft.Azure.Cosmos { + using System; + /// /// Context that is related to the set of delivered changes. /// - #if PREVIEW public #else diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs index d51ff3984a..9b48b206d8 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs @@ -25,9 +25,13 @@ class ChangeFeedProcessorUserException : Exception /// Initializes a new instance of the class using the specified internal exception. /// /// thrown by the user code. - public ChangeFeedProcessorUserException(Exception originalException) + /// Context under which the exception occurred. + public ChangeFeedProcessorUserException( + Exception originalException, + ChangeFeedProcessorContext context) : base(ChangeFeedProcessorUserException.DefaultMessage, originalException) { + this.ExceptionContext = context; } /// @@ -36,10 +40,15 @@ public ChangeFeedProcessorUserException(Exception originalException) /// The SerializationInfo object that holds serialized object data for the exception being thrown. /// The StreamingContext that contains contextual information about the source or destination. protected ChangeFeedProcessorUserException(SerializationInfo info, StreamingContext context) - : this((Exception)info.GetValue("InnerException", typeof(Exception))) + : this((Exception)info.GetValue("InnerException", typeof(Exception)), null) { } + /// + /// Contextual information that identifies which was the payload that was delivered to the delegate when this error occurred. + /// + public ChangeFeedProcessorContext ExceptionContext { get; private set; } + /// /// Sets the System.Runtime.Serialization.SerializationInfo with information about the exception. /// diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs index 4d1bfa8e04..9386032f4a 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedObserverFactoryCore.cs @@ -92,7 +92,7 @@ private Task ChangesStreamHandlerAsync( Stream stream, CancellationToken cancellationToken) { - IReadOnlyCollection changes = this.AsIReadOnlyCollection(stream); + IReadOnlyCollection changes = this.AsIReadOnlyCollection(stream, context); if (changes.Count == 0) { return Task.CompletedTask; @@ -111,7 +111,9 @@ private Task ChangesStreamHandlerAsync( return this.onChangesWithManualCheckpoint(context, changes, context.CheckpointAsync, cancellationToken); } - private IReadOnlyCollection AsIReadOnlyCollection(Stream stream) + private IReadOnlyCollection AsIReadOnlyCollection( + Stream stream, + ChangeFeedObserverContextCore context) { try { @@ -122,7 +124,7 @@ private IReadOnlyCollection AsIReadOnlyCollection(Stream stream) catch (Exception serializationException) { // Error using custom serializer to parse stream - throw new ChangeFeedProcessorUserException(serializationException); + throw new ChangeFeedProcessorUserException(serializationException, context); } } } diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedProcessorContextCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedProcessorContextCore.cs index e488f9437e..cb4ceb9d2e 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedProcessorContextCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ChangeFeedProcessorContextCore.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed { + using System; using Microsoft.Azure.Cosmos; internal sealed class ChangeFeedProcessorContextCore : ChangeFeedProcessorContext diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs index 94e96a0ca6..917fd416ee 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Observers/ObserverExceptionWrappingChangeFeedObserverDecorator.cs @@ -37,7 +37,7 @@ public override async Task ProcessChangesAsync(ChangeFeedObserverContextCore con } catch (Exception userException) { - throw new ChangeFeedProcessorUserException(userException); + throw new ChangeFeedProcessorUserException(userException, context); } } } diff --git a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs index 675288ea6b..4eec19b5d6 100644 --- a/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs +++ b/Microsoft.Azure.Cosmos/src/Resource/Container/Container.cs @@ -1603,6 +1603,8 @@ public abstract ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu /// if (exception is ChangeFeedProcessorUserException userException) /// { /// Console.WriteLine($"Current instance's delegate had an unhandled when processing lease {leaseToken}."); + /// Console.WriteLine($"Diagnostics {userException.ExceptionContext.Diagnostics}"); + /// Console.WriteLine($"Headers {userException.ExceptionContext.Headers}"); /// Console.WriteLine(userException.ToString()); /// } /// else diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs index 6f25fdb1a6..9de084daf5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs @@ -134,8 +134,11 @@ public async Task TestWithRunningProcessor_WithManualCheckpoint() .WithErrorNotification((string leaseToken, Exception exception) => { errorCount++; - Assert.IsTrue(exception is ChangeFeedProcessorUserException cfpException); + ChangeFeedProcessorUserException cfpException = exception as ChangeFeedProcessorUserException; + Assert.IsNotNull(cfpException); Assert.ReferenceEquals(exceptionToPropagate, exception.InnerException); + Assert.IsNotNull(cfpException.ExceptionContext.Diagnostics); + Assert.IsNotNull(cfpException.ExceptionContext.Headers); return Task.CompletedTask; }) .WithLeaseContainer(this.LeaseContainer).Build(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs index 0e3ed5cd5a..992f855098 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs @@ -8,7 +8,9 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests using System.IO; using System.Runtime.Serialization.Formatters.Binary; using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; + using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Moq; [TestClass] [TestCategory("ChangeFeed")] @@ -17,18 +19,25 @@ public class ObserverExceptionTests [TestMethod] public void ValidateConstructor() { + ResponseMessage responseMessage = new ResponseMessage(); + ChangeFeedObserverContextCore observerContext = new ChangeFeedObserverContextCore(Guid.NewGuid().ToString(), feedResponse: responseMessage, Mock.Of()); + ChangeFeedProcessorContextCore changeFeedProcessorContext = new ChangeFeedProcessorContextCore(observerContext); Exception exception = new Exception("randomMessage"); - ChangeFeedProcessorUserException ex = new ChangeFeedProcessorUserException(exception); + ChangeFeedProcessorUserException ex = new ChangeFeedProcessorUserException(exception, changeFeedProcessorContext); Assert.AreEqual(exception.Message, ex.InnerException.Message); Assert.AreEqual(exception, ex.InnerException); + Assert.ReferenceEquals(changeFeedProcessorContext, ex.ExceptionContext); } // Tests the GetObjectData method and the serialization ctor. [TestMethod] public void ValidateSerialization_AllFields() { + ResponseMessage responseMessage = new ResponseMessage(); + ChangeFeedObserverContextCore observerContext = new ChangeFeedObserverContextCore(Guid.NewGuid().ToString(), feedResponse: responseMessage, Mock.Of()); + ChangeFeedProcessorContextCore changeFeedProcessorContext = new ChangeFeedProcessorContextCore(observerContext); Exception exception = new Exception("randomMessage"); - ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(exception); + ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(exception, changeFeedProcessorContext); byte[] buffer = new byte[4096]; BinaryFormatter formatter = new BinaryFormatter(); MemoryStream stream1 = new MemoryStream(buffer); @@ -45,7 +54,7 @@ public void ValidateSerialization_AllFields() [TestMethod] public void ValidateSerialization_NullFields() { - ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(null); + ChangeFeedProcessorUserException originalException = new ChangeFeedProcessorUserException(null, null); byte[] buffer = new byte[4096]; BinaryFormatter formatter = new BinaryFormatter(); MemoryStream stream1 = new MemoryStream(buffer); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs index 4c604ca1ab..6140b1c6ba 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/PartitionSupervisorTests.cs @@ -120,7 +120,7 @@ public async Task RunObserver_ShouldCloseWithObserverError_IfObserverFailed() { Mock.Get(this.partitionProcessor) .Setup(processor => processor.RunAsync(It.IsAny())) - .ThrowsAsync(new ChangeFeedProcessorUserException(new Exception())); + .ThrowsAsync(new ChangeFeedProcessorUserException(new Exception(), Mock.Of())); await Assert.ThrowsExceptionAsync(() => this.sut.RunAsync(this.shutdownToken.Token)).ConfigureAwait(false); From 0b88d949e070e6badf7ada94213719b1a0f63e54 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 17:13:20 -0700 Subject: [PATCH 16/22] contract --- .../Contracts/DotNetPreviewSDKAPI.json | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index c25fdbd948..f6792888b9 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -54,6 +54,27 @@ }, "NestedTypes": {} }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { + "Subclasses": {}, + "Members": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAcquireDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAcquireDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseReleaseDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseReleaseDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + } + }, + "NestedTypes": {} + }, "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext;System.Object;IsAbstract:True;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { @@ -90,6 +111,34 @@ }, "NestedTypes": {} }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorUserException;System.Exception;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ExceptionContext": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ExceptionContext;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext), Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext)]" + }, + "Void GetObjectData(System.Runtime.Serialization.SerializationInfo, System.Runtime.Serialization.StreamingContext)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Void GetObjectData(System.Runtime.Serialization.SerializationInfo, System.Runtime.Serialization.StreamingContext);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + } + }, + "NestedTypes": {} + }, "Microsoft.Azure.Cosmos.ClientEncryptionIncludedPath;System.Object;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { @@ -524,6 +573,21 @@ "Attributes": [], "MethodInfo": null }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, "Microsoft.Azure.Cosmos.Container+ChangeFeedStreamHandler": { "Type": "NestedType", "Attributes": [], @@ -613,6 +677,84 @@ }, "NestedTypes": {} }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String, System.Exception)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String, System.Exception);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, "Microsoft.Azure.Cosmos.Container+ChangeFeedStreamHandler;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { "Subclasses": {}, "Members": { @@ -719,6 +861,84 @@ }, "NestedTypes": {} }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String, System.Exception)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String, System.Exception);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, "Microsoft.Azure.Cosmos.Container+ChangeFeedStreamHandler;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { "Subclasses": {}, "Members": { From 919ba04358e518a4b5168b9250ffb1915cc2d802 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 17:18:40 -0700 Subject: [PATCH 17/22] undo file --- .../src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs index 92224c193f..42d50b7a9d 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs @@ -4,8 +4,6 @@ namespace Microsoft.Azure.Cosmos { - using System; - /// /// Context that is related to the set of delivered changes. /// From 1e487b5d1ad2f04eac30544b879040dc67290aac Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 13 Jul 2021 17:19:19 -0700 Subject: [PATCH 18/22] undo more --- .../src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs index 42d50b7a9d..92a71ef4fe 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/ChangeFeedProcessorContext.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos /// /// Context that is related to the set of delivered changes. /// + #if PREVIEW public #else From 2260bc1cfcef5f6b5a61482ef0b12f6a87cb95bf Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 6 Aug 2021 11:40:20 -0700 Subject: [PATCH 19/22] logging always --- .../ChangeFeedProcessorHealthMonitorCore.cs | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs index d4581d088c..fe0aef7dd5 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs @@ -39,41 +39,47 @@ public void SetLeaseReleaseDelegate(ChangeFeedMonitorLeaseReleaseDelegate delega public override Task NotifyLeaseAcquireAsync(string leaseToken) { + DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); + if (this.acquireDelegate != null) { return this.acquireDelegate(leaseToken); } - - DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); - - return Task.CompletedTask; + else + { + return Task.CompletedTask; + } } public override Task NotifyLeaseReleaseAsync(string leaseToken) { + DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); + if (this.releaseDelegate != null) { return this.releaseDelegate(leaseToken); } - - DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); - - return Task.CompletedTask; + else + { + return Task.CompletedTask; + } } public override Task NotifyErrorAsync( string leaseToken, Exception exception) { + Extensions.TraceException(exception); + DefaultTrace.TraceError($"Error detected for lease {leaseToken}. "); + if (this.errorDelegate != null) { return this.errorDelegate(leaseToken, exception); } - - Extensions.TraceException(exception); - DefaultTrace.TraceError($"Error detected for lease {leaseToken}. "); - - return Task.CompletedTask; + else + { + return Task.CompletedTask; + } } } } \ No newline at end of file From 81d9f9cdd38347dbc135f53c2f5cdbdd7a26a9bc Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 10 Aug 2021 13:58:49 -0700 Subject: [PATCH 20/22] Addressing comments --- .../ChangeFeedProcessorUserException.cs | 4 +- .../ChangeFeedProcessorHealthMonitorCore.cs | 48 ++++++++++------ .../ChangeFeed/DynamicTests.cs | 4 +- ...angeFeedProcessorHealthMonitorCoreTests.cs | 56 +++++++++++++++++++ .../Exceptions/ObserverExceptionTests.cs | 3 +- 5 files changed, 91 insertions(+), 24 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs index 9b48b206d8..cda12b9326 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Exceptions/ChangeFeedProcessorUserException.cs @@ -31,7 +31,7 @@ public ChangeFeedProcessorUserException( ChangeFeedProcessorContext context) : base(ChangeFeedProcessorUserException.DefaultMessage, originalException) { - this.ExceptionContext = context; + this.ChangeFeedProcessorContext = context; } /// @@ -47,7 +47,7 @@ protected ChangeFeedProcessorUserException(SerializationInfo info, StreamingCont /// /// Contextual information that identifies which was the payload that was delivered to the delegate when this error occurred. /// - public ChangeFeedProcessorContext ExceptionContext { get; private set; } + public ChangeFeedProcessorContext ChangeFeedProcessorContext { get; private set; } /// /// Sets the System.Runtime.Serialization.SerializationInfo with information about the exception. diff --git a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs index fe0aef7dd5..193abe18a1 100644 --- a/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs +++ b/Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/Monitoring/ChangeFeedProcessorHealthMonitorCore.cs @@ -37,35 +37,43 @@ public void SetLeaseReleaseDelegate(ChangeFeedMonitorLeaseReleaseDelegate delega this.releaseDelegate = delegateCallback; } - public override Task NotifyLeaseAcquireAsync(string leaseToken) + public override async Task NotifyLeaseAcquireAsync(string leaseToken) { DefaultTrace.TraceInformation("Lease with token {0}: acquired", leaseToken); if (this.acquireDelegate != null) { - return this.acquireDelegate(leaseToken); - } - else - { - return Task.CompletedTask; + try + { + await this.acquireDelegate(leaseToken); + } + catch (Exception ex) + { + Extensions.TraceException(ex); + DefaultTrace.TraceError($"Lease acquire notification failed for {leaseToken}. "); + } } } - public override Task NotifyLeaseReleaseAsync(string leaseToken) + public override async Task NotifyLeaseReleaseAsync(string leaseToken) { DefaultTrace.TraceInformation("Lease with token {0}: released", leaseToken); if (this.releaseDelegate != null) { - return this.releaseDelegate(leaseToken); - } - else - { - return Task.CompletedTask; + try + { + await this.releaseDelegate(leaseToken); + } + catch (Exception ex) + { + Extensions.TraceException(ex); + DefaultTrace.TraceError($"Lease release notification failed for {leaseToken}. "); + } } } - public override Task NotifyErrorAsync( + public override async Task NotifyErrorAsync( string leaseToken, Exception exception) { @@ -74,11 +82,15 @@ public override Task NotifyErrorAsync( if (this.errorDelegate != null) { - return this.errorDelegate(leaseToken, exception); - } - else - { - return Task.CompletedTask; + try + { + await this.errorDelegate(leaseToken, exception); + } + catch (Exception ex) + { + Extensions.TraceException(ex); + DefaultTrace.TraceError($"Error notification failed for {leaseToken}. "); + } } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs index 9de084daf5..80bcb5f6d5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/ChangeFeed/DynamicTests.cs @@ -137,8 +137,8 @@ public async Task TestWithRunningProcessor_WithManualCheckpoint() ChangeFeedProcessorUserException cfpException = exception as ChangeFeedProcessorUserException; Assert.IsNotNull(cfpException); Assert.ReferenceEquals(exceptionToPropagate, exception.InnerException); - Assert.IsNotNull(cfpException.ExceptionContext.Diagnostics); - Assert.IsNotNull(cfpException.ExceptionContext.Headers); + Assert.IsNotNull(cfpException.ChangeFeedProcessorContext.Diagnostics); + Assert.IsNotNull(cfpException.ChangeFeedProcessorContext.Headers); return Task.CompletedTask; }) .WithLeaseContainer(this.LeaseContainer).Build(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs index 4adab1756d..f2d970d97f 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/ChangeFeedProcessorHealthMonitorCoreTests.cs @@ -31,6 +31,24 @@ public async Task Delegates_CallsAcquire() Assert.IsTrue(called); } + [TestMethod] + public async Task Delegates_CallsAcquire_OnFailure() + { + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetLeaseAcquireDelegate((string leaseToken) => + { + called = true; + Assert.AreEqual(token, leaseToken); + throw new Exception("Should not fail process"); + }); + + await monitor.NotifyLeaseAcquireAsync(token); + + Assert.IsTrue(called); + } + [TestMethod] public async Task Delegates_CallsRelease() { @@ -49,6 +67,24 @@ public async Task Delegates_CallsRelease() Assert.IsTrue(called); } + [TestMethod] + public async Task Delegates_CallsRelease_OnFailure() + { + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetLeaseReleaseDelegate((string leaseToken) => + { + called = true; + Assert.AreEqual(token, leaseToken); + throw new Exception("Should not fail process"); + }); + + await monitor.NotifyLeaseReleaseAsync(token); + + Assert.IsTrue(called); + } + [TestMethod] public async Task Delegates_CallsError() { @@ -68,5 +104,25 @@ public async Task Delegates_CallsError() Assert.IsTrue(called); } + + [TestMethod] + public async Task Delegates_CallsError_OnFailure() + { + Exception ex = new Exception(); + string token = Guid.NewGuid().ToString(); + bool called = false; + ChangeFeedProcessorHealthMonitorCore monitor = new ChangeFeedProcessorHealthMonitorCore(); + monitor.SetErrorDelegate((string leaseToken, Exception exception) => + { + called = true; + Assert.AreEqual(token, leaseToken); + Assert.ReferenceEquals(ex, exception); + throw new Exception("should not fail process"); + }); + + await monitor.NotifyErrorAsync(token, ex); + + Assert.IsTrue(called); + } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs index 992f855098..7b2e751381 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ChangeFeed/Exceptions/ObserverExceptionTests.cs @@ -7,7 +7,6 @@ namespace Microsoft.Azure.Cosmos.ChangeFeed.Tests using System; using System.IO; using System.Runtime.Serialization.Formatters.Binary; - using Microsoft.Azure.Cosmos.ChangeFeed.Exceptions; using Microsoft.Azure.Cosmos.ChangeFeed.FeedManagement; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; @@ -26,7 +25,7 @@ public void ValidateConstructor() ChangeFeedProcessorUserException ex = new ChangeFeedProcessorUserException(exception, changeFeedProcessorContext); Assert.AreEqual(exception.Message, ex.InnerException.Message); Assert.AreEqual(exception, ex.InnerException); - Assert.ReferenceEquals(changeFeedProcessorContext, ex.ExceptionContext); + Assert.ReferenceEquals(changeFeedProcessorContext, ex.ChangeFeedProcessorContext); } // Tests the GetObjectData method and the serialization ctor. From 7adb42559c494ff27017ba7edcb22a322095be70 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Tue, 10 Aug 2021 14:02:16 -0700 Subject: [PATCH 21/22] contract --- .../Contracts/DotNetPreviewSDKAPI.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index f6792888b9..fcf73e1c2a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -114,17 +114,17 @@ "Microsoft.Azure.Cosmos.ChangeFeedProcessorUserException;System.Exception;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:True": { "Subclasses": {}, "Members": { - "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ExceptionContext": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ChangeFeedProcessorContext": { "Type": "Property", "Attributes": [], - "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ExceptionContext;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ChangeFeedProcessorContext;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { "Type": "Method", "Attributes": [ "CompilerGeneratedAttribute" ], - "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ExceptionContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, "Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext)": { "Type": "Constructor", From 95ef92f35d1eb7b0949a9d1d14542c03bd3ed967 Mon Sep 17 00:00:00 2001 From: Matias Quaranta Date: Fri, 20 Aug 2021 14:15:23 -0700 Subject: [PATCH 22/22] preview update after merge --- .../Contracts/DotNetPreviewSDKAPI.json | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json index 148f277444..399c0171f5 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetPreviewSDKAPI.json @@ -54,6 +54,55 @@ }, "NestedTypes": {} }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { + "Subclasses": {}, + "Members": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithErrorNotification(ChangeFeedMonitorErrorDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAcquireDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseAcquireNotification(ChangeFeedMonitorLeaseAcquireDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseReleaseDelegate)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorBuilder WithLeaseReleaseNotification(ChangeFeedMonitorLeaseReleaseDelegate);IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorUserException;System.Exception;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ChangeFeedProcessorContext": { + "Type": "Property", + "Attributes": [], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext ChangeFeedProcessorContext;CanRead:True;CanWrite:True;Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext()[System.Runtime.CompilerServices.CompilerGeneratedAttribute()]": { + "Type": "Method", + "Attributes": [ + "CompilerGeneratedAttribute" + ], + "MethodInfo": "Microsoft.Azure.Cosmos.ChangeFeedProcessorContext get_ChangeFeedProcessorContext();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext), Void .ctor(System.Exception, Microsoft.Azure.Cosmos.ChangeFeedProcessorContext)]" + }, + "Void GetObjectData(System.Runtime.Serialization.SerializationInfo, System.Runtime.Serialization.StreamingContext)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Void GetObjectData(System.Runtime.Serialization.SerializationInfo, System.Runtime.Serialization.StreamingContext);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + } + }, + "NestedTypes": {} + }, "Microsoft.Azure.Cosmos.ClientEncryptionIncludedPath;System.Object;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { @@ -458,6 +507,21 @@ "Microsoft.Azure.Cosmos.Container;System.Object;IsAbstract:True;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": { "Subclasses": {}, "Members": { + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate": { + "Type": "NestedType", + "Attributes": [], + "MethodInfo": null + }, "Microsoft.Azure.Cosmos.FeedIterator GetItemQueryStreamIterator(Microsoft.Azure.Cosmos.FeedRange, Microsoft.Azure.Cosmos.QueryDefinition, System.String, Microsoft.Azure.Cosmos.QueryRequestOptions)": { "Type": "Method", "Attributes": [], @@ -484,6 +548,163 @@ "MethodInfo": "System.Threading.Tasks.Task`1[System.Collections.Generic.IEnumerable`1[System.String]] GetPartitionKeyRangesAsync(Microsoft.Azure.Cosmos.FeedRange, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, + "NestedTypes": { + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String, System.Exception)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String, System.Exception);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + } + } + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorErrorDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.Exception, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String, System.Exception)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String, System.Exception);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseAcquireDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, + "NestedTypes": {} + }, + "Microsoft.Azure.Cosmos.Container+ChangeFeedMonitorLeaseReleaseDelegate;System.MulticastDelegate;IsAbstract:False;IsSealed:True;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:True;IsGenericType:False;IsSerializable:True": { + "Subclasses": {}, + "Members": { + "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.IAsyncResult BeginInvoke(System.String, System.AsyncCallback, System.Object);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task EndInvoke(System.IAsyncResult);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task Invoke(System.String)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task Invoke(System.String);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Void .ctor(System.Object, IntPtr)": { + "Type": "Constructor", + "Attributes": [], + "MethodInfo": "[Void .ctor(System.Object, IntPtr), Void .ctor(System.Object, IntPtr)]" + } + }, "NestedTypes": {} }, "Microsoft.Azure.Cosmos.ContainerProperties;System.Object;IsAbstract:False;IsSealed:False;IsInterface:False;IsEnum:False;IsClass:True;IsValueType:False;IsNested:False;IsGenericType:False;IsSerializable:False": {