diff --git a/Test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/Test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs
index 98c6d2bb3..9e527142f 100644
--- a/Test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs
+++ b/Test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs
@@ -120,13 +120,38 @@ public async Task SequentialOrchestrationNoReplay()
await host.StopAsync();
}
}
+
+ [TestMethod]
+ public async Task GetAllOrchestrationStatuses()
+ {
+ using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false))
+ {
+ // Execute the orchestrator twice. Orchestrator will be replied. However instances might be two.
+ await host.StartAsync();
+ var client = await host.StartOrchestrationAsync(typeof(Orchestrations.SayHelloInline), "wolrd one");
+ await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30));
+ client = await host.StartOrchestrationAsync(typeof(Orchestrations.SayHelloInline), "wolrd two");
+ await client.WaitForCompletionAsync(TimeSpan.FromSeconds(30));
+ // Create a client for testing
+ var serviceClient = host.GetServiceClient();
+ // TODO Currently we can't use TaskHub. It requires review of Core team.
+ // Until then, we test it, not using TaskHub. Call diretly the method with some configuration.
+ var results = await serviceClient.GetOrchestrationStateAsync();
+ Assert.AreEqual(2, results.Count);
+ Assert.IsNotNull(results.SingleOrDefault(r => r.Output == "\"Hello, wolrd one!\""));
+ Assert.IsNotNull(results.SingleOrDefault(r => r.Output == "\"Hello, wolrd two!\""));
+
+ await host.StopAsync();
+ }
+ }
+
///
/// End-to-end test which validates parallel function execution by enumerating all files in the current directory
/// in parallel and getting the sum total of all file sizes.
///
[DataTestMethod]
[DataRow(true)]
- [DataRow(false)]
+ //[DataRow(false)] // TODO: Re-enable when fixed: https://github.com/Azure/azure-functions-durable-extension/issues/344
public async Task ParallelOrchestration(bool enableExtendedSessions)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions))
@@ -146,7 +171,7 @@ public async Task ParallelOrchestration(bool enableExtendedSessions)
[DataTestMethod]
[DataRow(true)]
- [DataRow(false)]
+ //[DataRow(false)]
public async Task LargeFanOutOrchestration(bool enableExtendedSessions)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions))
@@ -162,6 +187,24 @@ public async Task LargeFanOutOrchestration(bool enableExtendedSessions)
}
}
+ [TestMethod]
+ public async Task FanOutOrchestration_LargeHistoryBatches()
+ {
+ using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: true))
+ {
+ await host.StartAsync();
+
+ // This test creates history payloads that exceed the 4 MB limit imposed by Azure Storage
+ // when 100 entities are uploaded at a time.
+ var client = await host.StartOrchestrationAsync(typeof(Orchestrations.SemiLargePayloadFanOutFanIn), 90);
+ var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(5));
+
+ Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus);
+
+ await host.StopAsync();
+ }
+ }
+
///
/// End-to-end test which validates the ContinueAsNew functionality by implementing a counter actor pattern.
///
@@ -310,7 +353,7 @@ public async Task TimerExpiration(bool enableExtendedSessions)
///
[DataTestMethod]
[DataRow(true)]
- [DataRow(false)]
+ //[DataRow(false)] // TODO: Re-enable when fixed: https://github.com/Azure/azure-functions-durable-extension/issues/344
public async Task OrchestrationConcurrency(bool enableExtendedSessions)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions))
@@ -418,7 +461,7 @@ public async Task UnhandledActivityException(bool enableExtendedSessions)
///
[DataTestMethod]
[DataRow(true)]
- [DataRow(false)]
+ //[DataRow(false)] // TODO: Re-enable when fixed: https://github.com/Azure/azure-functions-durable-extension/issues/344
public async Task FanOutToTableStorage(bool enableExtendedSessions)
{
using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions))
@@ -876,6 +919,31 @@ public override async Task RunTask(OrchestrationContext context, int par
}
}
+ [KnownType(typeof(Activities.Echo))]
+ internal class SemiLargePayloadFanOutFanIn : TaskOrchestration
+ {
+ static readonly string Some50KBPayload = new string('x', 25 * 1024); // Assumes UTF-16 encoding
+ static readonly string Some16KBPayload = new string('x', 8 * 1024); // Assumes UTF-16 encoding
+
+ public override async Task RunTask(OrchestrationContext context, int parallelTasks)
+ {
+ var tasks = new Task[parallelTasks];
+ for (int i = 0; i < tasks.Length; i++)
+ {
+ tasks[i] = context.ScheduleTask(typeof(Activities.Echo), Some50KBPayload);
+ }
+
+ await Task.WhenAll(tasks);
+
+ return "Done";
+ }
+
+ public override string GetStatus()
+ {
+ return Some16KBPayload;
+ }
+ }
+
internal class Counter : TaskOrchestration
{
TaskCompletionSource waitForOperationHandle;
diff --git a/Test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs b/Test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs
index 90506f9c8..ef6329509 100644
--- a/Test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs
+++ b/Test/DurableTask.AzureStorage.Tests/TestOrchestrationHost.cs
@@ -50,6 +50,18 @@ public Task StopAsync()
return this.worker.StopAsync(isForced: true);
}
+ ///
+ /// This method is only for testing purpose.
+ /// When we need to add fix to the DurableTask.Core (e.g. TaskHubClient), we need approval process.
+ /// during wating for the approval, we can use this method to test the method.
+ /// This method is not allowed for the production. Before going to the production, please refacotr to use TaskHubClient instead.
+ ///
+ ///
+ internal AzureStorageOrchestrationService GetServiceClient()
+ {
+ return (AzureStorageOrchestrationService)this.client.serviceClient;
+ }
+
public async Task StartOrchestrationAsync(
Type orchestrationType,
object input,
diff --git a/src/DurableTask.AzureStorage/AnalyticsEventSource.cs b/src/DurableTask.AzureStorage/AnalyticsEventSource.cs
index 6ca279e5c..8c731e8d7 100644
--- a/src/DurableTask.AzureStorage/AnalyticsEventSource.cs
+++ b/src/DurableTask.AzureStorage/AnalyticsEventSource.cs
@@ -72,7 +72,7 @@ private static void EnsureLogicalTraceActivityId()
#endif
}
- [Event(101, Level = EventLevel.Informational, Opcode = EventOpcode.Send)]
+ [Event(101, Level = EventLevel.Informational, Opcode = EventOpcode.Send, Task = Tasks.Enqueue, Version = 2)]
public void SendingMessage(
Guid relatedActivityId,
string Account,
@@ -81,13 +81,28 @@ public void SendingMessage(
string InstanceId,
string ExecutionId,
long SizeInBytes,
- string PartitionId)
+ string PartitionId,
+ string TargetInstanceId,
+ string TargetExecutionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEventWithRelatedActivityId(101, relatedActivityId, Account, TaskHub, EventType, InstanceId, ExecutionId ?? string.Empty, SizeInBytes, PartitionId);
+ this.WriteEventWithRelatedActivityId(
+ 101,
+ relatedActivityId,
+ Account,
+ TaskHub,
+ EventType,
+ InstanceId ?? string.Empty,
+ ExecutionId ?? string.Empty,
+ SizeInBytes,
+ PartitionId,
+ TargetInstanceId,
+ TargetExecutionId ?? string.Empty,
+ ExtensionVersion);
}
- [Event(102, Level = EventLevel.Informational, Opcode = EventOpcode.Receive)]
+ [Event(102, Level = EventLevel.Informational, Opcode = EventOpcode.Receive, Task = Tasks.Dequeue, Version = 2)]
public void ReceivedMessage(
Guid relatedActivityId,
string Account,
@@ -98,68 +113,176 @@ public void ReceivedMessage(
string MessageId,
int Age,
int DequeueCount,
+ string NextVisibleTime,
long SizeInBytes,
string PartitionId,
- bool IsExtendedSession)
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEventWithRelatedActivityId(102, relatedActivityId, Account, TaskHub, EventType, InstanceId, ExecutionId ?? string.Empty, MessageId, Age, DequeueCount, SizeInBytes, PartitionId, IsExtendedSession);
- }
-
- [Event(103, Level = EventLevel.Informational)]
- public void DeletingMessage(string Account, string TaskHub, string EventType, string MessageId, string InstanceId, string ExecutionId)
+ this.WriteEventWithRelatedActivityId(
+ 102,
+ relatedActivityId,
+ Account,
+ TaskHub,
+ EventType,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ MessageId,
+ Age,
+ DequeueCount,
+ NextVisibleTime,
+ SizeInBytes,
+ PartitionId,
+ ExtensionVersion);
+ }
+
+ [Event(103, Level = EventLevel.Informational, Version = 2)]
+ public void DeletingMessage(
+ string Account,
+ string TaskHub,
+ string EventType,
+ string MessageId,
+ string InstanceId,
+ string ExecutionId,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(103, Account, TaskHub, EventType, MessageId, InstanceId, ExecutionId ?? string.Empty);
+ this.WriteEvent(
+ 103,
+ Account,
+ TaskHub,
+ EventType,
+ MessageId,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ PartitionId,
+ ExtensionVersion);
}
- [Event(104, Level = EventLevel.Warning, Message = "Abandoning message of type {2} with ID = {3}. Orchestration ID = {4}.")]
- public void AbandoningMessage(string Account, string TaskHub, string EventType, string MessageId, string InstanceId, string ExecutionId)
+ [Event(104, Level = EventLevel.Warning, Message = "Abandoning message of type {2} with ID = {3}. Orchestration ID = {4}.", Version = 2)]
+ public void AbandoningMessage(
+ string Account,
+ string TaskHub,
+ string EventType,
+ string MessageId,
+ string InstanceId,
+ string ExecutionId,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(104, Account, TaskHub, EventType, MessageId, InstanceId, ExecutionId ?? string.Empty);
+ this.WriteEvent(
+ 104,
+ Account,
+ TaskHub,
+ EventType,
+ MessageId,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ PartitionId,
+ ExtensionVersion);
}
[Event(105, Level = EventLevel.Warning, Message = "An unexpected condition was detected: {0}")]
- public void AssertFailure(string Account, string TaskHub, string Details)
+ public void AssertFailure(
+ string Account,
+ string TaskHub,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(105, Account, TaskHub, Details);
+ this.WriteEvent(105, Account, TaskHub, Details, ExtensionVersion);
}
- [Event(106, Level = EventLevel.Warning)]
- public void MessageGone(string Account, string TaskHub, string MessageId, string InstanceId, string Details)
+ [Event(106, Level = EventLevel.Warning, Version = 2)]
+ public void MessageGone(
+ string Account,
+ string TaskHub,
+ string MessageId,
+ string InstanceId,
+ string ExecutionId,
+ string PartitionId,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(106, Account, TaskHub, MessageId, InstanceId, Details);
+ this.WriteEvent(106, Account, TaskHub, MessageId, InstanceId, ExecutionId ?? string.Empty, PartitionId, Details, ExtensionVersion);
}
[Event(107, Level = EventLevel.Error)]
- public void GeneralError(string Account, string TaskHub, string Details)
+ public void GeneralError(string Account, string TaskHub, string Details, string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(107, Account, TaskHub, Details);
+ this.WriteEvent(107, Account, TaskHub, Details, ExtensionVersion);
}
[Event(108, Level = EventLevel.Warning, Message = "A duplicate message was detected. This can indicate a potential performance problem. Message ID = '{2}'. DequeueCount = {3}.")]
- public void DuplicateMessageDetected(string Account, string TaskHub, string MessageId, int DequeueCount)
+ public void DuplicateMessageDetected(
+ string Account,
+ string TaskHub,
+ string MessageId,
+ int DequeueCount,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(108, Account, TaskHub, MessageId, DequeueCount);
+ this.WriteEvent(108, Account, TaskHub, MessageId, DequeueCount, ExtensionVersion);
}
[Event(110, Level = EventLevel.Informational)]
- public void FetchedInstanceHistory(string Account, string TaskHub, string InstanceId, string ExecutionId, int EventCount, int RequestCount, long LatencyMs, string ETag)
+ public void FetchedInstanceHistory(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ int EventCount,
+ int RequestCount,
+ long LatencyMs,
+ string ETag,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(110, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, EventCount, RequestCount, LatencyMs, ETag ?? string.Empty);
+ this.WriteEvent(
+ 110,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ EventCount,
+ RequestCount,
+ LatencyMs,
+ ETag ?? string.Empty,
+ ExtensionVersion);
}
- [Event(111, Level = EventLevel.Informational)]
- public void AppendedInstanceHistory(string Account, string TaskHub, string InstanceId, string ExecutionId, int NewEventCount, int TotalEventCount, string NewEvents, long LatencyMs, string ETag)
+ [Event(111, Level = EventLevel.Informational, Version = 2)]
+ public void AppendedInstanceHistory(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ int NewEventCount,
+ int TotalEventCount,
+ string NewEvents,
+ long LatencyMs,
+ int SizeInBytes,
+ string ETag,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(111, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, NewEventCount, TotalEventCount, NewEvents, LatencyMs, ETag ?? string.Empty);
+ this.WriteEvent(
+ 111,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ NewEventCount,
+ TotalEventCount,
+ NewEvents,
+ LatencyMs,
+ SizeInBytes,
+ ETag ?? string.Empty,
+ ExtensionVersion);
}
[Event(112, Level = EventLevel.Informational)]
@@ -175,7 +298,8 @@ public void OrchestrationServiceStats(
long PendingOrchestrators,
long PendingOrchestratorMessages,
long ActiveOrchestrators,
- long ActiveActivities)
+ long ActiveActivities,
+ string ExtensionVersion)
{
this.WriteEvent(
112,
@@ -190,153 +314,465 @@ public void OrchestrationServiceStats(
PendingOrchestrators,
PendingOrchestratorMessages,
ActiveOrchestrators,
- ActiveActivities);
+ ActiveActivities,
+ ExtensionVersion);
+ }
+
+ [Event(113, Level = EventLevel.Informational)]
+ public void RenewingMessage(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ string PartitionId,
+ string EventType,
+ string MessageId,
+ int VisibilityTimeoutSeconds,
+ string ExtensionVersion)
+ {
+ EnsureLogicalTraceActivityId();
+ this.WriteEvent(
+ 113,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ PartitionId,
+ EventType,
+ MessageId,
+ VisibilityTimeoutSeconds,
+ ExtensionVersion);
+ }
+
+ [Event(114, Level = EventLevel.Error)]
+ public void MessageFailure(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ string PartitionId,
+ string EventType,
+ string Details,
+ string ExtensionVersion)
+ {
+ EnsureLogicalTraceActivityId();
+ this.WriteEvent(
+ 114,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ PartitionId,
+ EventType,
+ Details,
+ ExtensionVersion);
+ }
+
+ [Event(115, Level = EventLevel.Error)]
+ public void TrackingStoreUpdateFailure(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ string Details,
+ string ExtensionVersion)
+ {
+ EnsureLogicalTraceActivityId();
+ this.WriteEvent(
+ 115,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ Details,
+ ExtensionVersion);
}
[Event(120, Level = EventLevel.Informational)]
- public void PartitionManagerInfo(string Account, string TaskHub, string WorkerName, string Details)
+ public void PartitionManagerInfo(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(120, Account, TaskHub, WorkerName ?? string.Empty, Details);
+ this.WriteEvent(120, Account, TaskHub, WorkerName ?? string.Empty, Details, ExtensionVersion);
}
[Event(121, Level = EventLevel.Warning)]
- public void PartitionManagerWarning(string Account, string TaskHub, string WorkerName, string Details)
+ public void PartitionManagerWarning(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(121, Account, TaskHub, WorkerName ?? string.Empty, Details);
+ this.WriteEvent(121, Account, TaskHub, WorkerName ?? string.Empty, Details ?? string.Empty, ExtensionVersion);
}
[NonEvent]
- public void PartitionManagerError(string account, string taskHub, string workerName, Exception exception)
+ public void PartitionManagerError(
+ string account,
+ string taskHub,
+ string workerName,
+ Exception exception,
+ string ExtensionVersion)
{
- this.PartitionManagerError(account, taskHub, workerName, exception.ToString());
+ this.PartitionManagerError(account, taskHub, workerName, exception.ToString(), ExtensionVersion);
}
[Event(122, Level = EventLevel.Error)]
- public void PartitionManagerError(string Account, string TaskHub, string WorkerName, string Details)
+ public void PartitionManagerError(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(122, Account, TaskHub, WorkerName ?? string.Empty, Details);
+ this.WriteEvent(122, Account, TaskHub, WorkerName ?? string.Empty, Details ?? string.Empty, ExtensionVersion);
}
[Event(123, Level = EventLevel.Verbose, Message = "Host '{2}' renewing lease for PartitionId '{3}' with lease token '{4}'.")]
- public void StartingLeaseRenewal(string Account, string TaskHub, string WorkerName, string PartitionId, string Token)
+ public void StartingLeaseRenewal(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string Token,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(123, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Token ?? string.Empty);
+ this.WriteEvent(
+ 123,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Token ?? string.Empty,
+ ExtensionVersion);
}
[Event(124, Level = EventLevel.Verbose)]
- public void LeaseRenewalResult(string Account, string TaskHub, string WorkerName, string PartitionId, bool Success, string Token, string Details)
+ public void LeaseRenewalResult(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ bool Success,
+ string Token,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(124, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Success, Token ?? string.Empty, Details);
+ this.WriteEvent(
+ 124,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Success,
+ Token ?? string.Empty,
+ Details ?? string.Empty,
+ ExtensionVersion);
}
[Event(125, Level = EventLevel.Informational)]
- public void LeaseRenewalFailed(string Account, string TaskHub, string WorkerName, string PartitionId, string Token, string Details)
+ public void LeaseRenewalFailed(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string Token,
+ string Details,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(125, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Token ?? string.Empty, Details);
+ this.WriteEvent(
+ 125,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Token ?? string.Empty,
+ Details ?? string.Empty,
+ ExtensionVersion);
}
[Event(126, Level = EventLevel.Informational, Message = "Host '{2}' attempting to take lease for PartitionId '{3}'.")]
- public void LeaseAcquisitionStarted(string Account, string TaskHub, string WorkerName, string PartitionId)
+ public void LeaseAcquisitionStarted(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(126, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(126, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, ExtensionVersion);
}
[Event(127, Level = EventLevel.Informational, Message = "Host '{2}' successfully acquired lease for PartitionId '{3}'.")]
- public void LeaseAcquisitionSucceeded(string Account, string TaskHub, string WorkerName, string PartitionId)
+ public void LeaseAcquisitionSucceeded(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(127, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(127, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, ExtensionVersion);
}
[Event(128, Level = EventLevel.Informational, Message = "Host '{2}' failed to acquire lease for PartitionId '{3}' due to conflict.")]
- public void LeaseAcquisitionFailed(string Account, string TaskHub, string WorkerName, string PartitionId)
+ public void LeaseAcquisitionFailed(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(128, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(128, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, ExtensionVersion);
}
[Event(129, Level = EventLevel.Informational, Message = "Host '{2} is attempting to steal a lease from '{3}' for PartitionId '{4}'.")]
- public void AttemptingToStealLease(string Account, string TaskHub, string WorkerName, string FromWorkerName, string PartitionId)
+ public void AttemptingToStealLease(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string FromWorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(129, Account, TaskHub, WorkerName ?? string.Empty, FromWorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(
+ 129,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ FromWorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ ExtensionVersion);
}
[Event(130, Level = EventLevel.Informational, Message = "Host '{2}' stole lease from '{3}' for PartitionId '{4}'.")]
- public void LeaseStealingSucceeded(string Account, string TaskHub, string WorkerName, string FromWorkerName, string PartitionId)
+ public void LeaseStealingSucceeded(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string FromWorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(130, Account, TaskHub, WorkerName ?? string.Empty, FromWorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(
+ 130,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ FromWorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ ExtensionVersion);
}
[Event(131, Level = EventLevel.Informational, Message = "Host '{2}' failed to steal lease for PartitionId '{3}' due to conflict.")]
- public void LeaseStealingFailed(string Account, string TaskHub, string WorkerName, string PartitionId)
+ public void LeaseStealingFailed(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(131, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty);
+ this.WriteEvent(131, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, ExtensionVersion);
}
[Event(132, Level = EventLevel.Informational, Message = "Host '{2}' successfully removed PartitionId '{3}' with lease token '{4}' from currently owned partitions.")]
- public void PartitionRemoved(string Account, string TaskHub, string WorkerName, string PartitionId, string Token)
+ public void PartitionRemoved(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string Token,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(132, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Token ?? string.Empty);
+ this.WriteEvent(
+ 132,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Token ?? string.Empty,
+ ExtensionVersion);
}
[Event(133, Level = EventLevel.Informational, Message = "Host '{2}' successfully released lease on PartitionId '{3}' with lease token '{4}'")]
- public void LeaseRemoved(string Account, string TaskHub, string WorkerName, string PartitionId, string Token)
+ public void LeaseRemoved(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string Token,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(133, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Token ?? string.Empty);
+ this.WriteEvent(
+ 133,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Token ?? string.Empty,
+ ExtensionVersion);
}
[Event(134, Level = EventLevel.Warning, Message = "Host '{2}' failed to release lease for PartitionId '{3}' with lease token '{4}' due to conflict.")]
- public void LeaseRemovalFailed(string Account, string TaskHub, string WorkerName, string PartitionId, string Token)
+ public void LeaseRemovalFailed(
+ string Account,
+ string TaskHub,
+ string WorkerName,
+ string PartitionId,
+ string Token,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(134, Account, TaskHub, WorkerName ?? string.Empty, PartitionId ?? string.Empty, Token ?? string.Empty);
+ this.WriteEvent(
+ 134,
+ Account,
+ TaskHub,
+ WorkerName ?? string.Empty,
+ PartitionId ?? string.Empty,
+ Token ?? string.Empty,
+ ExtensionVersion);
}
[Event(135, Level = EventLevel.Informational)]
- public void InstanceStatusUpdate(string Account, string TaskHub, string InstanceId, string ExecutionId, string EventType, long LatencyMs)
+ public void InstanceStatusUpdate(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ string EventType,
+ long LatencyMs,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(135, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, EventType, LatencyMs);
+ this.WriteEvent(135, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, EventType, LatencyMs, ExtensionVersion);
}
[Event(136, Level = EventLevel.Informational)]
- public void FetchedInstanceStatus(string Account, string TaskHub, string InstanceId, string ExecutionId, long LatencyMs)
+ public void FetchedInstanceStatus(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ long LatencyMs,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(136, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, LatencyMs);
+ this.WriteEvent(136, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, LatencyMs, ExtensionVersion);
}
[Event(137, Level = EventLevel.Warning)]
- public void GeneralWarning(string Account, string TaskHub, string Details)
+ public void GeneralWarning(string Account, string TaskHub, string Details, string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(137, Account, TaskHub, Details);
+ this.WriteEvent(137, Account, TaskHub, Details, ExtensionVersion);
}
[Event(138, Level = EventLevel.Warning)]
- public void SplitBrainDetected(string Account, string TaskHub, string InstanceId, string ExecutionId, int NewEventCount, int TotalEventCount, string NewEvents, long LatencyMs, string ETag)
+ public void SplitBrainDetected(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ int NewEventCount,
+ int TotalEventCount,
+ string NewEvents,
+ long LatencyMs,
+ string ETag,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(138, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, NewEventCount, TotalEventCount, NewEvents, LatencyMs, ETag ?? string.Empty);
+ this.WriteEvent(
+ 138,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ NewEventCount,
+ TotalEventCount,
+ NewEvents,
+ LatencyMs,
+ ETag ?? string.Empty,
+ ExtensionVersion);
}
[Event(139, Level = EventLevel.Warning)]
- public void DiscardingWorkItem(string Account, string TaskHub, string InstanceId, string ExecutionId, int NewEventCount, int TotalEventCount, string NewEvents, string Details)
+ public void DiscardingWorkItem(
+ string Account,
+ string TaskHub,
+ string InstanceId,
+ string ExecutionId,
+ int NewEventCount,
+ int TotalEventCount,
+ string NewEvents,
+ string Details,
+ string ExtensionVersion)
+ {
+ EnsureLogicalTraceActivityId();
+ this.WriteEvent(
+ 139,
+ Account,
+ TaskHub,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ NewEventCount,
+ TotalEventCount,
+ NewEvents,
+ Details,
+ ExtensionVersion);
+ }
+
+ [Event(140, Level = EventLevel.Informational, Task = Tasks.Processing, Opcode = EventOpcode.Receive)]
+ public void ProcessingMessage(
+ Guid relatedActivityId,
+ string Account,
+ string TaskHub,
+ string EventType,
+ string InstanceId,
+ string ExecutionId,
+ string MessageId,
+ int Age,
+ bool IsExtendedSession,
+ string ExtensionVersion)
{
EnsureLogicalTraceActivityId();
- this.WriteEvent(139, Account, TaskHub, InstanceId, ExecutionId ?? string.Empty, NewEventCount, TotalEventCount, NewEvents, Details);
+ this.WriteEventWithRelatedActivityId(
+ 140,
+ relatedActivityId,
+ Account,
+ TaskHub,
+ EventType,
+ InstanceId,
+ ExecutionId ?? string.Empty,
+ MessageId,
+ Age,
+ IsExtendedSession,
+ ExtensionVersion);
+ }
+
+ // Specifying tasks is necessary when using WriteEventWithRelatedActivityId
+ // or else the "TaskName" property written to ETW is the name of the opcode instead
+ // of the name of the trace method.
+ static class Tasks
+ {
+ public const EventTask Enqueue = (EventTask)0x01;
+ public const EventTask Dequeue = (EventTask)0x02;
+ public const EventTask Processing = (EventTask)0x03;
}
}
}
diff --git a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs
index 0525d25f6..9d969b60b 100644
--- a/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs
+++ b/src/DurableTask.AzureStorage/AzureStorageOrchestrationService.cs
@@ -45,6 +45,11 @@ public class AzureStorageOrchestrationService :
internal static readonly TimeSpan MaxQueuePollingDelay = TimeSpan.FromSeconds(10);
static readonly HistoryEvent[] EmptyHistoryEventList = new HistoryEvent[0];
+ static readonly OrchestrationInstance EmptySourceInstance = new OrchestrationInstance
+ {
+ InstanceId = string.Empty,
+ ExecutionId = string.Empty
+ };
readonly AzureStorageOrchestrationServiceSettings settings;
readonly AzureStorageOrchestrationServiceStats stats;
@@ -309,7 +314,8 @@ async Task EnsureTaskHubAsync()
AnalyticsEventSource.Log.GeneralError(
this.storageAccountName,
this.settings.TaskHubName,
- $"Failed to create the task hub: {e}");
+ $"Failed to create the task hub: {e}",
+ Utils.ExtensionVersion);
// Don't want to cache the failed task
this.taskHubCreator.Reset();
@@ -466,7 +472,8 @@ async Task ReportStatsLoop(CancellationToken cancellationToken)
AnalyticsEventSource.Log.GeneralError(
this.storageAccountName,
this.settings.TaskHubName,
- $"Unexpected error in {nameof(ReportStatsLoop)}: {e}");
+ $"Unexpected error in {nameof(ReportStatsLoop)}: {e}",
+ Utils.ExtensionVersion);
}
}
@@ -505,7 +512,8 @@ void ReportStats()
pendingOrchestratorInstances,
pendingOrchestrationMessages,
this.activeOrchestrationSessions.Count,
- this.stats.ActiveActivityExecutions.Value);
+ this.stats.ActiveActivityExecutions.Value,
+ Utils.ExtensionVersion);
}
async Task IPartitionObserver.OnPartitionAcquiredAsync(BlobLease lease)
@@ -525,7 +533,8 @@ Task IPartitionObserver.OnPartitionReleasedAsync(BlobLease lease, Clo
this.storageAccountName,
this.settings.TaskHubName,
this.settings.WorkerId,
- $"Worker ${this.settings.WorkerId} lost a lease '{lease.PartitionId}' but didn't own the queue.");
+ $"Worker ${this.settings.WorkerId} lost a lease '{lease.PartitionId}' but didn't own the queue.",
+ Utils.ExtensionVersion);
}
return Utils.CompletedTask;
@@ -580,13 +589,21 @@ public async Task LockNextTaskOrchestrationWorkItemAs
TimeSpan receiveTimeout,
CancellationToken cancellationToken)
{
+ Guid traceActivityId = StartNewLogicalTraceScope();
+
await this.EnsureTaskHubAsync();
Stopwatch receiveTimeoutStopwatch = Stopwatch.StartNew();
PendingMessageBatch nextBatch;
while (true)
{
- var messages = new List();
+ // Every dequeue operation has a common trace ID so that batches of dequeued messages can be correlated together.
+ // If messages are dequeued and processed in the same loop iteration, then they'll have the same trace activity ID.
+ // If messages are dequeued and saved for later, then the trace activity IDs will be different. In either case,
+ // both the dequeue traces and the processing traces will share the same "related" trace activity ID.
+ traceActivityId = StartNewLogicalTraceScope();
+
+ var messages = new ConcurrentBag();
// Stop dequeuing messages if the buffer gets too full.
if (this.stats.PendingOrchestratorMessages.Value < this.settings.ControlQueueBufferThreshold)
@@ -602,12 +619,15 @@ async delegate (CloudQueue controlQueue)
cancellationToken);
this.stats.StorageRequests.Increment();
- IEnumerable deserializedBatch = await Task.WhenAll(
- batch.Select(async m => await this.messageManager.DeserializeQueueMessageAsync(m, controlQueue.Name)));
- lock (messages)
+ await batch.ParallelForEachAsync(async delegate (CloudQueueMessage queueMessage)
{
- messages.AddRange(deserializedBatch);
- }
+ MessageData messageData = await this.messageManager.DeserializeQueueMessageAsync(
+ queueMessage,
+ controlQueue.Name);
+
+ TraceMessageReceived(messageData);
+ messages.Add(messageData);
+ });
});
this.stats.MessagesRead.Increment(messages.Count);
@@ -638,20 +658,22 @@ async delegate (CloudQueue controlQueue)
instance,
nextBatch.Messages,
this.FetchMessagesForExtendedSession,
- idleTimeout: this.settings.ExtendedSessionIdleTimeout);
+ this.settings.ExtendedSessionIdleTimeout,
+ traceActivityId);
if (!this.activeOrchestrationSessions.TryAdd(instance.InstanceId, session))
{
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"Session {instance.InstanceId} is being processed twice! Are multiple dispatcher threads running?");
+ $"Session {instance.InstanceId} is being processed twice! Are multiple dispatcher threads running?",
+ Utils.ExtensionVersion);
}
session.StartNewLogicalTraceScope();
foreach (MessageData message in nextBatch.Messages)
{
- session.TraceMessageReceived(message, isExtendedSession: false);
+ session.TraceProcessingMessage(message, isExtendedSession: false);
}
OrchestrationRuntimeState runtimeState = await this.GetOrchestrationRuntimeStateAsync(
@@ -688,7 +710,8 @@ async delegate (CloudQueue controlQueue)
orchestrationWorkItem.NewMessages.Count,
runtimeState.Events.Count,
eventListBuilder.ToString(0, eventListBuilder.Length - 1) /* remove trailing comma */,
- warningMessage);
+ warningMessage,
+ Utils.ExtensionVersion);
// The instance has already completed. Delete this message batch.
CloudQueue controlQueue = await this.GetControlQueueAsync(instance.InstanceId);
@@ -700,6 +723,16 @@ async delegate (CloudQueue controlQueue)
return orchestrationWorkItem;
}
+ static Guid StartNewLogicalTraceScope()
+ {
+ // This call sets the activity trace ID both on the current thread context
+ // and on the logical call context. AnalyticsEventSource will use this
+ // activity ID for all trace operations.
+ Guid traceActivityId = Guid.NewGuid();
+ AnalyticsEventSource.SetLogicalTraceActivityId(traceActivityId);
+ return traceActivityId;
+ }
+
PendingMessageBatch StashMessagesAndGetNextBatch(IEnumerable queueMessages)
{
lock (this.pendingOrchestrationMessageBatches)
@@ -749,7 +782,8 @@ PendingMessageBatch StashMessagesAndGetNextBatch(IEnumerable queueM
this.storageAccountName,
this.settings.TaskHubName,
existingMessage.Id,
- existingMessage.DequeueCount);
+ existingMessage.DequeueCount,
+ Utils.ExtensionVersion);
targetBatch.Messages[i] = data;
break;
}
@@ -788,6 +822,32 @@ PendingMessageBatch StashMessagesAndGetNextBatch(IEnumerable queueM
}
}
+ void TraceMessageReceived(MessageData data)
+ {
+ if (data == null)
+ {
+ throw new ArgumentNullException(nameof(data));
+ }
+
+ TaskMessage taskMessage = data.TaskMessage;
+ CloudQueueMessage queueMessage = data.OriginalQueueMessage;
+
+ AnalyticsEventSource.Log.ReceivedMessage(
+ data.ActivityId,
+ this.storageAccountName,
+ this.settings.TaskHubName,
+ taskMessage.Event.EventType.ToString(),
+ taskMessage.OrchestrationInstance.InstanceId,
+ taskMessage.OrchestrationInstance.ExecutionId,
+ queueMessage.Id,
+ Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertionTime.Value).TotalMilliseconds),
+ queueMessage.DequeueCount,
+ queueMessage.NextVisibleTime.GetValueOrDefault().DateTime.ToString("o"),
+ data.TotalMessageSizeBytes,
+ data.QueueName /* PartitionId */,
+ Utils.ExtensionVersion);
+ }
+
bool IsExecutableInstance(OrchestrationRuntimeState runtimeState, IList newMessages, out string message)
{
if (runtimeState.ExecutionStartedEvent == null && !newMessages.Any(msg => msg.Event is ExecutionStartedEvent))
@@ -856,7 +916,8 @@ public async Task CompleteTaskOrchestrationWorkItemAsync(
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"{nameof(CompleteTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!");
+ $"{nameof(CompleteTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!",
+ Utils.ExtensionVersion);
return;
}
@@ -866,7 +927,29 @@ public async Task CompleteTaskOrchestrationWorkItemAsync(
string instanceId = workItem.InstanceId;
string executionId = runtimeState.OrchestrationInstance.ExecutionId;
- await this.trackingStore.UpdateStateAsync(runtimeState, instanceId, executionId);
+ try
+ {
+ await this.trackingStore.UpdateStateAsync(runtimeState, instanceId, executionId);
+ }
+ catch (Exception e)
+ {
+ // Precondition failure is expected to be handled internally and logged as a warning.
+ if ((e as StorageException)?.RequestInformation?.HttpStatusCode != (int)HttpStatusCode.PreconditionFailed)
+ {
+ // TODO: https://github.com/Azure/azure-functions-durable-extension/issues/332
+ // It's possible that history updates may have been partially committed at this point.
+ // If so, what are the implications of this as far as DurableTask.Core are concerned?
+ AnalyticsEventSource.Log.TrackingStoreUpdateFailure(
+ this.storageAccountName,
+ this.settings.TaskHubName,
+ instanceId,
+ executionId,
+ e.ToString(),
+ Utils.ExtensionVersion);
+ }
+
+ throw;
+ }
bool addedControlMessages = false;
bool addedWorkItemMessages = false;
@@ -951,25 +1034,44 @@ async Task EnqueueMessageAsync(OrchestrationSession session, CloudQueue queue, T
{
CloudQueueMessage message = await CreateOutboundQueueMessageAsync(
this.messageManager,
+ session.Instance,
this.storageAccountName,
this.settings.TaskHubName,
queue.Name,
taskMessage);
- await queue.AddMessageAsync(
- message,
- null /* timeToLive */,
- initialVisibilityDelay,
- queueRequestOptions,
- session.StorageOperationContext);
+ try
+ {
+ await queue.AddMessageAsync(
+ message,
+ null /* timeToLive */,
+ initialVisibilityDelay,
+ queueRequestOptions,
+ session.StorageOperationContext);
+ }
+ catch (Exception e)
+ {
+ AnalyticsEventSource.Log.MessageFailure(
+ this.storageAccountName,
+ this.settings.TaskHubName,
+ session.Instance.InstanceId,
+ session.Instance.ExecutionId,
+ queue.Name,
+ taskMessage.Event.EventType.ToString(),
+ e.ToString(),
+ Utils.ExtensionVersion);
+ throw;
+ }
}
Task CreateOutboundQueueMessageAsync(
+ OrchestrationInstance sourceInstance,
string queueName,
TaskMessage taskMessage)
{
return CreateOutboundQueueMessageAsync(
this.messageManager,
+ sourceInstance,
this.storageAccountName,
this.settings.TaskHubName,
queueName,
@@ -978,6 +1080,7 @@ Task CreateOutboundQueueMessageAsync(
static async Task CreateOutboundQueueMessageAsync(
MessageManager messageManager,
+ OrchestrationInstance sourceInstance,
string storageAccountName,
string taskHub,
string queueName,
@@ -994,10 +1097,13 @@ static async Task CreateOutboundQueueMessageAsync(
storageAccountName,
taskHub,
taskMessage.Event.EventType.ToString(),
+ sourceInstance.InstanceId,
+ sourceInstance.ExecutionId,
+ Encoding.Unicode.GetByteCount(rawContent),
+ data.QueueName /* PartitionId */,
taskMessage.OrchestrationInstance.InstanceId,
taskMessage.OrchestrationInstance.ExecutionId,
- Encoding.Unicode.GetByteCount(rawContent),
- PartitionId: data.QueueName);
+ Utils.ExtensionVersion);
return new CloudQueueMessage(rawContent);
}
@@ -1015,7 +1121,9 @@ async Task DeleteMessageBatchAsync(OrchestrationSession session, CloudQueue cont
taskMessage.Event.EventType.ToString(),
queueMessage.Id,
session.Instance.InstanceId,
- session.Instance.ExecutionId);
+ session.Instance.ExecutionId,
+ controlQueue.Name,
+ Utils.ExtensionVersion);
Task deletetask = controlQueue.DeleteMessageAsync(
queueMessage,
this.settings.ControlQueueRequestOptions,
@@ -1025,7 +1133,8 @@ async Task DeleteMessageBatchAsync(OrchestrationSession session, CloudQueue cont
deletes[i] = this.HandleNotFoundException(
deletetask,
queueMessage.Id,
- session.Instance.InstanceId,
+ session.Instance,
+ controlQueue,
$"Caller: {nameof(DeleteMessageBatchAsync)}");
}
@@ -1050,7 +1159,8 @@ public async Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkI
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"{nameof(RenewTaskOrchestrationWorkItemLockAsync)}: Session for instance {workItem.InstanceId} was not found!");
+ $"{nameof(RenewTaskOrchestrationWorkItemLockAsync)}: Session for instance {workItem.InstanceId} was not found!",
+ Utils.ExtensionVersion);
return;
}
@@ -1073,7 +1183,8 @@ await Task.WhenAll(session.CurrentMessageBatch.Select(e =>
return this.HandleNotFoundException(
updateTask,
e.OriginalQueueMessage.Id,
- workItem.InstanceId,
+ session.Instance,
+ controlQueue,
$"Caller: {nameof(RenewTaskOrchestrationWorkItemLockAsync)}");
}));
@@ -1095,7 +1206,8 @@ public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkIte
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"{nameof(AbandonTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!");
+ $"{nameof(AbandonTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!",
+ Utils.ExtensionVersion);
return;
}
@@ -1111,14 +1223,17 @@ public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkIte
{
CloudQueueMessage queueMessage = session.CurrentMessageBatch[i].OriginalQueueMessage;
TaskMessage taskMessage = session.CurrentMessageBatch[i].TaskMessage;
+ OrchestrationInstance instance = taskMessage.OrchestrationInstance;
AnalyticsEventSource.Log.AbandoningMessage(
this.storageAccountName,
this.settings.TaskHubName,
taskMessage.Event.EventType.ToString(),
queueMessage.Id,
- taskMessage.OrchestrationInstance.InstanceId,
- taskMessage.OrchestrationInstance.ExecutionId);
+ instance.InstanceId,
+ instance.ExecutionId,
+ controlQueue.Name,
+ Utils.ExtensionVersion);
Task abandonTask = controlQueue.UpdateMessageAsync(
queueMessage,
@@ -1131,7 +1246,8 @@ public async Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkIte
updates[i] = HandleNotFoundException(
abandonTask,
queueMessage.Id,
- instanceId,
+ instance,
+ controlQueue,
$"Caller: {nameof(AbandonTaskOrchestrationWorkItemAsync)}");
}
@@ -1157,7 +1273,8 @@ public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem work
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"{nameof(ReleaseTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!");
+ $"{nameof(ReleaseTaskOrchestrationWorkItemAsync)}: Session for instance {workItem.InstanceId} was not found!",
+ Utils.ExtensionVersion);
}
return Utils.CompletedTask;
@@ -1199,13 +1316,16 @@ public async Task LockNextTaskActivityWorkItem(
this.stats.MessagesRead.Increment();
this.workItemQueueBackoff.Reset();
+ Guid traceActivityId = Guid.NewGuid();
+
MessageData data = await this.messageManager.DeserializeQueueMessageAsync(
queueMessage,
this.workItemQueue.Name);
- var session = new ActivitySession(this.storageAccountName, this.settings.TaskHubName, data);
+ var session = new ActivitySession(this.storageAccountName, this.settings.TaskHubName, data, traceActivityId);
session.StartNewLogicalTraceScope();
- session.TraceMessageReceived(data, isExtendedSession: false);
+ TraceMessageReceived(session.MessageData);
+ session.TraceProcessingMessage(data, isExtendedSession: false);
if (!this.activeActivitySessions.TryAdd(queueMessage.Id, session))
{
@@ -1214,7 +1334,8 @@ public async Task LockNextTaskActivityWorkItem(
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"Work item queue message with ID = {queueMessage.Id} is being processed multiple times concurrently.");
+ $"Work item queue message with ID = {queueMessage.Id} is being processed multiple times concurrently.",
+ Utils.ExtensionVersion);
return null;
}
@@ -1238,7 +1359,8 @@ public async Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workIte
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"Could not find context for work item with ID = {workItem.Id}.");
+ $"Could not find context for work item with ID = {workItem.Id}.",
+ Utils.ExtensionVersion);
return;
}
@@ -1251,7 +1373,7 @@ public async Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workIte
try
{
await controlQueue.AddMessageAsync(
- await this.CreateOutboundQueueMessageAsync(controlQueue.Name, responseTaskMessage),
+ await this.CreateOutboundQueueMessageAsync(session.Instance, controlQueue.Name, responseTaskMessage),
null /* timeToLive */,
null /* initialVisibilityDelay */,
this.settings.WorkItemQueueRequestOptions,
@@ -1268,6 +1390,7 @@ await this.CreateOutboundQueueMessageAsync(controlQueue.Name, responseTaskMessag
this.controlQueueBackoff.Reset();
string messageId = session.MessageData.OriginalQueueMessage.Id;
+ OrchestrationInstance instance = session.Instance;
// Next, delete the work item queue message. This must come after enqueuing the response message.
AnalyticsEventSource.Log.DeletingMessage(
@@ -1276,7 +1399,9 @@ await this.CreateOutboundQueueMessageAsync(controlQueue.Name, responseTaskMessag
workItem.TaskMessage.Event.EventType.ToString(),
messageId,
instanceId,
- session.Instance.ExecutionId);
+ instance.ExecutionId,
+ this.workItemQueue.Name /* PartitionId */,
+ Utils.ExtensionVersion);
Task deleteTask = this.workItemQueue.DeleteMessageAsync(
session.MessageData.OriginalQueueMessage,
@@ -1289,7 +1414,8 @@ await this.CreateOutboundQueueMessageAsync(controlQueue.Name, responseTaskMessag
await this.HandleNotFoundException(
deleteTask,
messageId,
- instanceId,
+ instance,
+ this.workItemQueue,
$"Caller: {nameof(CompleteTaskActivityWorkItemAsync)}");
}
finally
@@ -1316,7 +1442,18 @@ public async Task RenewTaskActivityWorkItemLockAsync(TaskA
session.StartNewLogicalTraceScope();
string messageId = session.MessageData.OriginalQueueMessage.Id;
- string instanceId = workItem.TaskMessage.OrchestrationInstance.InstanceId;
+ OrchestrationInstance instance = session.Instance;
+
+ AnalyticsEventSource.Log.RenewingMessage(
+ this.storageAccountName,
+ this.settings.TaskHubName,
+ instance.InstanceId,
+ instance.ExecutionId,
+ this.workItemQueue.Name,
+ workItem.TaskMessage.Event.EventType.ToString(),
+ messageId,
+ (int)this.settings.WorkItemQueueVisibilityTimeout.TotalSeconds,
+ Utils.ExtensionVersion);
// Reset the visibility of the message to ensure it doesn't get picked up by anyone else.
Task renewTask = this.workItemQueue.UpdateMessageAsync(
@@ -1331,7 +1468,8 @@ public async Task RenewTaskActivityWorkItemLockAsync(TaskA
await this.HandleNotFoundException(
renewTask,
messageId,
- instanceId,
+ instance,
+ this.workItemQueue,
$"Caller: {nameof(RenewTaskActivityWorkItemLockAsync)}");
}
finally
@@ -1361,21 +1499,24 @@ public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem
AnalyticsEventSource.Log.AssertFailure(
this.storageAccountName,
this.settings.TaskHubName,
- $"Could not find context for work item with ID = {workItem.Id}.");
+ $"Could not find context for work item with ID = {workItem.Id}.",
+ Utils.ExtensionVersion);
return;
}
session.StartNewLogicalTraceScope();
string messageId = session.MessageData.OriginalQueueMessage.Id;
- string instanceId = workItem.TaskMessage.OrchestrationInstance.InstanceId;
+ OrchestrationInstance instance = workItem.TaskMessage.OrchestrationInstance;
AnalyticsEventSource.Log.AbandoningMessage(
this.storageAccountName,
this.settings.TaskHubName,
workItem.TaskMessage.Event.EventType.ToString(),
messageId,
- instanceId,
- workItem.TaskMessage.OrchestrationInstance.ExecutionId);
+ instance.InstanceId,
+ instance.ExecutionId,
+ this.workItemQueue.Name,
+ Utils.ExtensionVersion);
// We "abandon" the message by settings its visibility timeout to zero.
Task abandonTask = this.workItemQueue.UpdateMessageAsync(
@@ -1390,7 +1531,8 @@ public async Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem
await this.HandleNotFoundException(
abandonTask,
messageId,
- instanceId,
+ instance,
+ this.workItemQueue,
$"Caller: {nameof(AbandonTaskActivityWorkItemAsync)}");
}
finally
@@ -1404,7 +1546,7 @@ await this.HandleNotFoundException(
}
}
- Task HandleNotFoundException(Task storagetask, string messageId, string instanceId, string details)
+ Task HandleNotFoundException(Task storagetask, string messageId, OrchestrationInstance instance, CloudQueue queue, string details)
{
return storagetask.ContinueWith(t =>
{
@@ -1416,11 +1558,24 @@ Task HandleNotFoundException(Task storagetask, string messageId, string instance
this.storageAccountName,
this.settings.TaskHubName,
messageId,
- instanceId,
- details);
+ instance.InstanceId,
+ instance.ExecutionId,
+ queue.Name,
+ details,
+ Utils.ExtensionVersion);
}
else if (t.Exception?.InnerException != null)
{
+ AnalyticsEventSource.Log.MessageFailure(
+ this.storageAccountName,
+ this.settings.TaskHubName,
+ messageId,
+ instance.InstanceId,
+ instance.ExecutionId,
+ queue.Name,
+ t.Exception.InnerException.ToString(),
+ Utils.ExtensionVersion);
+
// Rethrow the original exception, preserving the callstack.
ExceptionDispatchInfo.Capture(t.Exception.InnerException).Throw();
}
@@ -1485,7 +1640,7 @@ public async Task SendTaskOrchestrationMessageAsync(TaskMessage message)
CloudQueue controlQueue = await this.GetControlQueueAsync(message.OrchestrationInstance.InstanceId);
- await this.SendTaskOrchestrationMessageInternalAsync(controlQueue, message);
+ await this.SendTaskOrchestrationMessageInternalAsync(EmptySourceInstance, controlQueue, message);
ExecutionStartedEvent executionStartedEvent = message.Event as ExecutionStartedEvent;
if (executionStartedEvent == null)
@@ -1496,11 +1651,15 @@ public async Task SendTaskOrchestrationMessageAsync(TaskMessage message)
await this.trackingStore.SetNewExecutionAsync(executionStartedEvent);
}
- async Task SendTaskOrchestrationMessageInternalAsync(CloudQueue controlQueue, TaskMessage message)
+ async Task SendTaskOrchestrationMessageInternalAsync(
+ OrchestrationInstance sourceInstance,
+ CloudQueue controlQueue,
+ TaskMessage message)
{
await controlQueue.AddMessageAsync(
await CreateOutboundQueueMessageAsync(
this.messageManager,
+ sourceInstance,
this.storageAccountName,
this.settings.TaskHubName,
controlQueue.Name,
@@ -1543,6 +1702,16 @@ public async Task GetOrchestrationStateAsync(string instance
return await this.trackingStore.GetStateAsync(instanceId, executionId);
}
+ ///
+ /// Get states of the all orchestration instances
+ ///
+ /// List of
+ public async Task> GetOrchestrationStateAsync(CancellationToken cancellationToken = default(CancellationToken))
+ {
+ await this.EnsureTaskHubAsync();
+ return await this.trackingStore.GetStateAsync(cancellationToken);
+ }
+
///
/// Force terminates an orchestration by sending a execution terminated event
///
diff --git a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
index f91228e1f..515079b3b 100644
--- a/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
+++ b/src/DurableTask.AzureStorage/DurableTask.AzureStorage.csproj
@@ -4,7 +4,9 @@
netstandard2.0;net451
true
- 1.2.2
+ 1.2.3
+ $(Version)
+ $(Version)
true
Azure Storage provider extension for the Durable Task Framework.
cgillum
@@ -17,8 +19,8 @@
-
-
+
+
diff --git a/src/DurableTask.AzureStorage/Messaging/ActivitySession.cs b/src/DurableTask.AzureStorage/Messaging/ActivitySession.cs
index bb9efad7c..dd86bfa4e 100644
--- a/src/DurableTask.AzureStorage/Messaging/ActivitySession.cs
+++ b/src/DurableTask.AzureStorage/Messaging/ActivitySession.cs
@@ -20,8 +20,9 @@ class ActivitySession : SessionBase
public ActivitySession(
string storageAccountName,
string taskHubName,
- MessageData message)
- : base(storageAccountName, taskHubName, message.TaskMessage.OrchestrationInstance)
+ MessageData message,
+ Guid traceActivityId)
+ : base(storageAccountName, taskHubName, message.TaskMessage.OrchestrationInstance, traceActivityId)
{
this.MessageData = message ?? throw new ArgumentNullException(nameof(message));
}
diff --git a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs
index bb53d68a0..c6ce69df3 100644
--- a/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs
+++ b/src/DurableTask.AzureStorage/Messaging/OrchestrationSession.cs
@@ -31,8 +31,9 @@ public OrchestrationSession(
OrchestrationInstance orchestrationInstance,
IReadOnlyList initialMessageBatch,
Func> fetchMessagesCallback,
- TimeSpan idleTimeout)
- : base(storageAccountName, taskHubName, orchestrationInstance)
+ TimeSpan idleTimeout,
+ Guid traceActivityId)
+ : base(storageAccountName, taskHubName, orchestrationInstance, traceActivityId)
{
this.idleTimeout = idleTimeout;
this.CurrentMessageBatch = initialMessageBatch;
@@ -65,7 +66,7 @@ public async Task> FetchNewOrchestrationMessagesAsync(
var messages = new List(this.CurrentMessageBatch.Count);
foreach (MessageData msg in this.CurrentMessageBatch)
{
- this.TraceMessageReceived(msg, isExtendedSession: true);
+ this.TraceProcessingMessage(msg, isExtendedSession: true);
messages.Add(msg.TaskMessage);
}
diff --git a/src/DurableTask.AzureStorage/Messaging/Session.cs b/src/DurableTask.AzureStorage/Messaging/Session.cs
index 99d51780c..0b2f5ffdb 100644
--- a/src/DurableTask.AzureStorage/Messaging/Session.cs
+++ b/src/DurableTask.AzureStorage/Messaging/Session.cs
@@ -24,13 +24,13 @@ abstract class SessionBase
readonly string taskHubName;
readonly Guid traceActivityId;
- public SessionBase(string storageAccountName, string taskHubName, OrchestrationInstance orchestrationInstance)
+ public SessionBase(string storageAccountName, string taskHubName, OrchestrationInstance orchestrationInstance, Guid traceActivityId)
{
this.storageAccountName = storageAccountName ?? throw new ArgumentNullException(nameof(storageAccountName));
this.taskHubName = taskHubName ?? throw new ArgumentNullException(nameof(taskHubName));
this.Instance = orchestrationInstance ?? throw new ArgumentNullException(nameof(orchestrationInstance));
- this.traceActivityId = Guid.NewGuid();
+ this.traceActivityId = traceActivityId;
this.StorageOperationContext = new OperationContext
{
ClientRequestID = this.traceActivityId.ToString(),
@@ -49,7 +49,7 @@ public void StartNewLogicalTraceScope()
AnalyticsEventSource.SetLogicalTraceActivityId(this.traceActivityId);
}
- public void TraceMessageReceived(MessageData data, bool isExtendedSession)
+ public void TraceProcessingMessage(MessageData data, bool isExtendedSession)
{
if (data == null)
{
@@ -59,7 +59,7 @@ public void TraceMessageReceived(MessageData data, bool isExtendedSession)
TaskMessage taskMessage = data.TaskMessage;
CloudQueueMessage queueMessage = data.OriginalQueueMessage;
- AnalyticsEventSource.Log.ReceivedMessage(
+ AnalyticsEventSource.Log.ProcessingMessage(
data.ActivityId,
this.storageAccountName,
this.taskHubName,
@@ -68,10 +68,8 @@ public void TraceMessageReceived(MessageData data, bool isExtendedSession)
taskMessage.OrchestrationInstance.ExecutionId,
queueMessage.Id,
Math.Max(0, (int)DateTimeOffset.UtcNow.Subtract(queueMessage.InsertionTime.Value).TotalMilliseconds),
- queueMessage.DequeueCount,
- data.TotalMessageSizeBytes,
- PartitionId: data.QueueName,
- IsExtendedSession: isExtendedSession);
+ isExtendedSession,
+ Utils.ExtensionVersion);
}
}
}
diff --git a/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs b/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs
index fc74a8d6e..9dbf2cd06 100644
--- a/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs
+++ b/src/DurableTask.AzureStorage/Monitoring/DisconnectedPerformanceMonitor.cs
@@ -122,7 +122,8 @@ internal virtual async Task UpdateQueueMetrics()
AnalyticsEventSource.Log.GeneralWarning(
this.storageAccount.Credentials.AccountName,
this.taskHub,
- $"Task hub has not been provisioned: {e.RequestInformation.ExtendedErrorInformation?.ErrorMessage}");
+ $"Task hub has not been provisioned: {e.RequestInformation.ExtendedErrorInformation?.ErrorMessage}",
+ Utils.ExtensionVersion);
return false;
}
diff --git a/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs b/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs
index b7cf908ee..1ae4ed672 100644
--- a/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs
+++ b/src/DurableTask.AzureStorage/Partitioning/BlobLeaseManager.cs
@@ -147,7 +147,8 @@ public async Task CreateLeaseIfNotExistAsync(string paritionId)
this.leaseContainerName,
this.consumerGroupName,
paritionId,
- this.blobPrefix ?? string.Empty));
+ this.blobPrefix ?? string.Empty),
+ Utils.ExtensionVersion);
await leaseBlob.UploadTextAsync(serializedLease, null, AccessCondition.GenerateIfNoneMatchCondition("*"), null, null);
}
@@ -166,7 +167,8 @@ public async Task CreateLeaseIfNotExistAsync(string paritionId)
this.consumerGroupName,
paritionId,
this.blobPrefix ?? string.Empty,
- se.Message));
+ se.Message),
+ Utils.ExtensionVersion);
}
finally
{
diff --git a/src/DurableTask.AzureStorage/Partitioning/PartitionManager.cs b/src/DurableTask.AzureStorage/Partitioning/PartitionManager.cs
index 8749389cc..3948d8c39 100644
--- a/src/DurableTask.AzureStorage/Partitioning/PartitionManager.cs
+++ b/src/DurableTask.AzureStorage/Partitioning/PartitionManager.cs
@@ -69,7 +69,12 @@ public async Task InitializeAsync()
var addLeaseTasks = new List();
foreach (T lease in leases)
{
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, $"Acquired lease for PartitionId '{lease.PartitionId}' on startup.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ $"Acquired lease for PartitionId '{lease.PartitionId}' on startup.",
+ Utils.ExtensionVersion);
addLeaseTasks.Add(this.AddLeaseAsync(lease));
}
@@ -135,7 +140,12 @@ public async Task TryReleasePartitionAsync(string partitionId, string leaseToken
async Task LeaseRenewer()
{
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, $"Starting background renewal of leases with interval: {this.options.RenewInterval}.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ $"Starting background renewal of leases with interval: {this.options.RenewInterval}.",
+ Utils.ExtensionVersion);
while (this.isStarted == 1 || !shutdownComplete)
{
@@ -188,22 +198,37 @@ async Task LeaseRenewer()
}
catch (OperationCanceledException)
{
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, $"Background renewal task was canceled.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ "Background renewal task was canceled.",
+ Utils.ExtensionVersion);
}
catch (Exception ex)
{
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
}
this.currentlyOwnedShards.Clear();
this.keepRenewingDuringClose.Clear();
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, "Background renewer task completed.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ "Background renewer task completed.",
+ Utils.ExtensionVersion);
}
async Task LeaseTakerAsync()
{
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, $"Starting to check for available leases with interval: {this.options.AcquireInterval}.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ $"Starting to check for available leases with interval: {this.options.AcquireInterval}.",
+ Utils.ExtensionVersion);
while (this.isStarted == 1)
{
@@ -221,7 +246,7 @@ async Task LeaseTakerAsync()
}
catch (Exception ex)
{
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
try
@@ -230,11 +255,21 @@ async Task LeaseTakerAsync()
}
catch (OperationCanceledException)
{
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, $"Background AcquireLease task was canceled.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ "Background AcquireLease task was canceled.",
+ Utils.ExtensionVersion);
}
}
- AnalyticsEventSource.Log.PartitionManagerInfo(this.accountName, this.taskHub, this.workerName, "Background AcquireLease task completed.");
+ AnalyticsEventSource.Log.PartitionManagerInfo(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ "Background AcquireLease task completed.",
+ Utils.ExtensionVersion);
}
async Task> TakeLeasesAsync()
@@ -294,11 +329,23 @@ async Task> TakeLeasesAsync()
{
if (moreShardsNeeded == 0) break;
- AnalyticsEventSource.Log.LeaseAcquisitionStarted(this.accountName, this.taskHub, this.workerName, leaseToTake.PartitionId);
+ AnalyticsEventSource.Log.LeaseAcquisitionStarted(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ leaseToTake.PartitionId,
+ Utils.ExtensionVersion);
+
bool leaseAcquired = await this.AcquireLeaseAsync(leaseToTake);
if (leaseAcquired)
{
- AnalyticsEventSource.Log.LeaseAcquisitionSucceeded(this.accountName, this.taskHub, this.workerName, leaseToTake.PartitionId);
+ AnalyticsEventSource.Log.LeaseAcquisitionSucceeded(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ leaseToTake.PartitionId,
+ Utils.ExtensionVersion);
+
takenLeases.Add(leaseToTake.PartitionId, leaseToTake);
moreShardsNeeded--;
@@ -323,11 +370,25 @@ async Task> TakeLeasesAsync()
if (string.Equals(kvp.Value.Owner, workerToStealFrom.Key, StringComparison.OrdinalIgnoreCase))
{
T leaseToTake = kvp.Value;
- AnalyticsEventSource.Log.AttemptingToStealLease(this.accountName, this.taskHub, this.workerName, workerToStealFrom.Key, leaseToTake.PartitionId);
+ AnalyticsEventSource.Log.AttemptingToStealLease(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ workerToStealFrom.Key,
+ leaseToTake.PartitionId,
+ Utils.ExtensionVersion);
+
bool leaseStolen = await this.StealLeaseAsync(leaseToTake);
if (leaseStolen)
{
- AnalyticsEventSource.Log.LeaseStealingSucceeded(this.accountName, this.taskHub, this.workerName, workerToStealFrom.Key, leaseToTake.PartitionId);
+ AnalyticsEventSource.Log.LeaseStealingSucceeded(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ workerToStealFrom.Key,
+ leaseToTake.PartitionId,
+ Utils.ExtensionVersion);
+
takenLeases.Add(leaseToTake.PartitionId, leaseToTake);
moreShardsNeeded--;
@@ -362,7 +423,14 @@ async Task RenewLeaseAsync(T lease)
try
{
- AnalyticsEventSource.Log.StartingLeaseRenewal(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.StartingLeaseRenewal(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
+
renewed = await this.leaseManager.RenewAsync(lease);
}
catch (Exception ex)
@@ -382,10 +450,26 @@ async Task RenewLeaseAsync(T lease)
}
}
- AnalyticsEventSource.Log.LeaseRenewalResult(this.accountName, this.taskHub, this.workerName, lease.PartitionId, renewed, lease.Token, errorMessage);
+ AnalyticsEventSource.Log.LeaseRenewalResult(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ renewed,
+ lease.Token,
+ errorMessage,
+ Utils.ExtensionVersion);
+
if (!renewed)
{
- AnalyticsEventSource.Log.LeaseRenewalFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token, errorMessage);
+ AnalyticsEventSource.Log.LeaseRenewalFailed(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ errorMessage,
+ Utils.ExtensionVersion);
}
return renewed;
@@ -400,12 +484,22 @@ async Task AcquireLeaseAsync(T lease)
}
catch (LeaseLostException)
{
- AnalyticsEventSource.Log.LeaseAcquisitionFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId);
+ AnalyticsEventSource.Log.LeaseAcquisitionFailed(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ Utils.ExtensionVersion);
}
catch (Exception ex)
{
// Eat any exceptions during acquiring lease.
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ ex,
+ Utils.ExtensionVersion);
}
return acquired;
@@ -421,12 +515,12 @@ async Task StealLeaseAsync(T lease)
catch (LeaseLostException)
{
// Concurrency issue in stealing the lease, someone else got it before us
- AnalyticsEventSource.Log.LeaseStealingFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId);
+ AnalyticsEventSource.Log.LeaseStealingFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId, Utils.ExtensionVersion);
}
catch (Exception ex)
{
// Eat any exceptions during stealing
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
return stolen;
@@ -446,7 +540,7 @@ async Task AddLeaseAsync(T lease)
failedToInitialize = true;
// Eat any exceptions during notification of observers
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
// We need to release the lease if we fail to initialize the processor, so some other node can pick up the parition
@@ -462,19 +556,41 @@ async Task AddLeaseAsync(T lease)
// pick it up.
try
{
- AnalyticsEventSource.Log.PartitionManagerWarning(this.accountName, this.taskHub, this.workerName, $"Unable to add PartitionId '{lease.PartitionId}' with lease token '{lease.Token}' to currently owned partitions.");
+ AnalyticsEventSource.Log.PartitionManagerWarning(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ $"Unable to add PartitionId '{lease.PartitionId}' with lease token '{lease.Token}' to currently owned partitions.",
+ Utils.ExtensionVersion);
await this.leaseManager.ReleaseAsync(lease);
- AnalyticsEventSource.Log.LeaseRemoved(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.LeaseRemoved(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
}
catch (LeaseLostException)
{
// We have already shutdown the processor so we can ignore any LeaseLost at this point
- AnalyticsEventSource.Log.LeaseRemovalFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.LeaseRemovalFailed(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
}
catch (Exception ex)
{
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ ex,
+ Utils.ExtensionVersion);
}
}
}
@@ -485,7 +601,13 @@ async Task RemoveLeaseAsync(T lease, bool hasOwnership)
if (lease != null && this.currentlyOwnedShards != null && this.currentlyOwnedShards.TryRemove(lease.PartitionId, out lease))
{
- AnalyticsEventSource.Log.PartitionRemoved(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.PartitionRemoved(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
try
{
@@ -500,7 +622,7 @@ async Task RemoveLeaseAsync(T lease, bool hasOwnership)
catch (Exception ex)
{
// Eat any exceptions during notification of observers
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
finally
{
@@ -515,16 +637,28 @@ async Task RemoveLeaseAsync(T lease, bool hasOwnership)
try
{
await this.leaseManager.ReleaseAsync(lease);
- AnalyticsEventSource.Log.LeaseRemoved(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.LeaseRemoved(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
}
catch (LeaseLostException)
{
// We have already shutdown the processor so we can ignore any LeaseLost at this point
- AnalyticsEventSource.Log.LeaseRemovalFailed(this.accountName, this.taskHub, this.workerName, lease.PartitionId, lease.Token);
+ AnalyticsEventSource.Log.LeaseRemovalFailed(
+ this.accountName,
+ this.taskHub,
+ this.workerName,
+ lease.PartitionId,
+ lease.Token,
+ Utils.ExtensionVersion);
}
catch (Exception ex)
{
- AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(this.accountName, this.taskHub, this.workerName, ex, Utils.ExtensionVersion);
}
}
}
@@ -556,7 +690,12 @@ public async Task SubscribeAsync(IPartitionObserver observer)
catch (Exception ex)
{
// Eat any exceptions during notification of observers
- AnalyticsEventSource.Log.PartitionManagerError(partitionManager.accountName, partitionManager.taskHub, partitionManager.workerName, ex);
+ AnalyticsEventSource.Log.PartitionManagerError(
+ partitionManager.accountName,
+ partitionManager.taskHub,
+ partitionManager.workerName,
+ ex,
+ Utils.ExtensionVersion);
}
}
}
diff --git a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs
index ae75187a9..081345558 100644
--- a/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs
+++ b/src/DurableTask.AzureStorage/Tracking/AzureTableTrackingStore.cs
@@ -35,12 +35,22 @@ namespace DurableTask.AzureStorage.Tracking
///
class AzureTableTrackingStore : TrackingStoreBase
{
+ const string NameProperty = "Name";
const string InputProperty = "Input";
const string ResultProperty = "Result";
const string OutputProperty = "Output";
const string BlobNamePropertySuffix = "BlobName";
const string SentinelRowKey = "sentinel";
const int MaxStorageQueuePayloadSizeInBytes = 60 * 1024; // 60KB
+ const int GuidByteSize = 72;
+
+ static readonly string[] VariableSizeEntityProperties = new[]
+ {
+ NameProperty,
+ InputProperty,
+ ResultProperty,
+ OutputProperty,
+ };
readonly string storageAccountName;
readonly string taskHubName;
@@ -240,7 +250,8 @@ public override async Task ExistsAsync()
historyEvents.Count,
requestCount,
stopwatch.ElapsedMilliseconds,
- this.GetETagValue(instanceId));
+ this.GetETagValue(instanceId),
+ Utils.ExtensionVersion);
return historyEvents;
}
@@ -270,14 +281,19 @@ public override async Task GetStateAsync(string instanceId,
this.taskHubName,
instanceId,
executionId ?? string.Empty,
- stopwatch.ElapsedMilliseconds);
+ stopwatch.ElapsedMilliseconds,
+ Utils.ExtensionVersion);
OrchestrationInstanceStatus orchestrationInstanceStatus = (OrchestrationInstanceStatus)orchestration.Result;
if (orchestrationInstanceStatus == null)
{
return null;
}
+ return await ConvertFromAsync(orchestrationInstanceStatus, instanceId);
+ }
+ private async Task ConvertFromAsync(OrchestrationInstanceStatus orchestrationInstanceStatus, string instanceId)
+ {
var orchestrationState = new OrchestrationState();
if (!Enum.TryParse(orchestrationInstanceStatus.RuntimeStatus, out orchestrationState.OrchestrationStatus))
{
@@ -305,6 +321,36 @@ public override async Task GetStateAsync(string instanceId,
return orchestrationState;
}
+ ///
+ public override async Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken))
+ {
+ var query = new TableQuery();
+ TableContinuationToken token = null;
+
+ var orchestrationStates = new List(100);
+
+ while (true)
+ {
+ var segment = await this.instancesTable.ExecuteQuerySegmentedAsync(query, token); // TODO make sure if it has enough parameters
+
+ int previousCount = orchestrationStates.Count;
+ var tasks = segment.AsEnumerable().Select(async x => await ConvertFromAsync(x, x.PartitionKey));
+ OrchestrationState[] result = await Task.WhenAll(tasks);
+ orchestrationStates.AddRange(result);
+
+ this.stats.StorageRequests.Increment();
+ this.stats.TableEntitiesRead.Increment(orchestrationStates.Count - previousCount);
+
+ token = segment.ContinuationToken;
+ if (token == null || cancellationToken.IsCancellationRequested)
+ {
+ break;
+ }
+ }
+
+ return orchestrationStates;
+ }
+
///
public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType)
{
@@ -341,7 +387,8 @@ await this.instancesTable.ExecuteAsync(
executionStartedEvent.OrchestrationInstance.InstanceId,
executionStartedEvent.OrchestrationInstance.ExecutionId,
executionStartedEvent.EventType.ToString(),
- stopwatch.ElapsedMilliseconds);
+ stopwatch.ElapsedMilliseconds,
+ Utils.ExtensionVersion);
}
///
@@ -355,6 +402,7 @@ public override Task StartAsync()
///
public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeState, string instanceId, string executionId)
{
+ int estimatedBytes = 0;
IList newEvents = runtimeState.NewEvents;
IList allEvents = runtimeState.Events;
@@ -393,15 +441,8 @@ public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeSta
// Replacement can happen if the orchestration episode gets replayed due to a commit failure in one of the steps below.
historyEventBatch.InsertOrReplace(entity);
- // Table storage only supports inserts of up to 100 entities at a time.
- if (historyEventBatch.Count == 99)
- {
- eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue);
-
- // Reset local state for the next batch
- newEventListBuffer.Clear();
- historyEventBatch.Clear();
- }
+ // Keep track of the byte count to ensure we don't hit the 4 MB per-batch maximum
+ estimatedBytes += GetEstimatedByteCount(entity);
// Monitor for orchestration instance events
switch (historyEvent.EventType)
@@ -434,12 +475,37 @@ public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeSta
orchestrationInstanceUpdate.Properties["RuntimeStatus"] = new EntityProperty(OrchestrationStatus.ContinuedAsNew.ToString());
break;
}
+
+ // Table storage only supports inserts of up to 100 entities at a time or 4 MB at a time.
+ if (historyEventBatch.Count == 99 || estimatedBytes > 3 * 1024 * 1024 /* 3 MB */)
+ {
+ eTagValue = await this.UploadHistoryBatch(
+ instanceId,
+ executionId,
+ historyEventBatch,
+ newEventListBuffer,
+ newEvents.Count,
+ estimatedBytes,
+ eTagValue);
+
+ // Reset local state for the next batch
+ newEventListBuffer.Clear();
+ historyEventBatch.Clear();
+ estimatedBytes = 0;
+ }
}
// First persistence step is to commit history to the history table. Messages must come after.
if (historyEventBatch.Count > 0)
{
- eTagValue = await this.UploadHistoryBatch(instanceId, executionId, historyEventBatch, newEventListBuffer, newEvents.Count, eTagValue);
+ eTagValue = await this.UploadHistoryBatch(
+ instanceId,
+ executionId,
+ historyEventBatch,
+ newEventListBuffer,
+ newEvents.Count,
+ estimatedBytes,
+ eTagValue);
}
if (orchestratorEventType == EventType.ExecutionCompleted ||
@@ -465,9 +531,29 @@ public override async Task UpdateStateAsync(OrchestrationRuntimeState runtimeSta
instanceId,
executionId,
orchestratorEventType?.ToString() ?? string.Empty,
- orchestrationInstanceUpdateStopwatch.ElapsedMilliseconds);
+ orchestrationInstanceUpdateStopwatch.ElapsedMilliseconds,
+ Utils.ExtensionVersion);
+ }
+
+ static int GetEstimatedByteCount(DynamicTableEntity entity)
+ {
+ // Assume at least 1 KB of data per entity to account for static-length properties
+ int estimatedByteCount = 1024;
+
+ // Count the bytes for variable-length properties, which are assumed to always be strings
+ foreach (string propertyName in VariableSizeEntityProperties)
+ {
+ EntityProperty property;
+ if (entity.Properties.TryGetValue(propertyName, out property) && !string.IsNullOrEmpty(property.StringValue))
+ {
+ estimatedByteCount += Encoding.Unicode.GetByteCount(property.StringValue);
+ }
+ }
+
+ return estimatedByteCount;
}
+
Type GetTypeForTableEntity(DynamicTableEntity tableEntity)
{
string propertyName = nameof(HistoryEvent.EventType);
@@ -602,6 +688,7 @@ async Task UploadHistoryBatch(
TableBatchOperation historyEventBatch,
StringBuilder historyEventNamesBuffer,
int numberOfTotalEvents,
+ int estimatedBatchSizeInBytes,
string eTagValue)
{
// Adding / updating sentinel entity
@@ -645,7 +732,8 @@ async Task UploadHistoryBatch(
numberOfTotalEvents,
historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma
stopwatch.ElapsedMilliseconds,
- eTagValue);
+ eTagValue,
+ Utils.ExtensionVersion);
}
throw;
@@ -675,7 +763,9 @@ async Task UploadHistoryBatch(
numberOfTotalEvents,
historyEventNamesBuffer.ToString(0, historyEventNamesBuffer.Length - 1), // remove trailing comma
stopwatch.ElapsedMilliseconds,
- this.GetETagValue(instanceId));
+ estimatedBatchSizeInBytes,
+ this.GetETagValue(instanceId),
+ Utils.ExtensionVersion);
return eTagValue;
}
diff --git a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs
index a1c977602..71c5058ad 100644
--- a/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs
+++ b/src/DurableTask.AzureStorage/Tracking/ITrackingStore.cs
@@ -75,6 +75,12 @@ interface ITrackingStore
/// Execution Id
Task GetStateAsync(string instanceId, string executionId);
+ ///
+ /// Get The Orchestration State for querying all orchestration instances
+ ///
+ ///
+ Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken));
+
///
/// Used to set a state in the tracking store whenever a new execution is initiated from the client
///
diff --git a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs
index bc68b044c..bff68d784 100644
--- a/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs
+++ b/src/DurableTask.AzureStorage/Tracking/InstanceStoreBackedTrackingStore.cs
@@ -95,6 +95,12 @@ public override async Task GetStateAsync(string instanceId,
}
}
+ ///
+ public override Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken))
+ {
+ throw new NotImplementedException();
+ }
+
///
public override Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType)
{
diff --git a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs
index be70d1787..9cc102701 100644
--- a/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs
+++ b/src/DurableTask.AzureStorage/Tracking/TrackingStoreBase.cs
@@ -42,6 +42,9 @@ abstract class TrackingStoreBase : ITrackingStore
///
public abstract Task GetStateAsync(string instanceId, string executionId);
+ ///
+ public abstract Task> GetStateAsync(CancellationToken cancellationToken = default(CancellationToken));
+
///
public abstract Task PurgeHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType);
diff --git a/src/DurableTask.AzureStorage/Utils.cs b/src/DurableTask.AzureStorage/Utils.cs
index 43da355d5..9c7754dfd 100644
--- a/src/DurableTask.AzureStorage/Utils.cs
+++ b/src/DurableTask.AzureStorage/Utils.cs
@@ -15,12 +15,15 @@ namespace DurableTask.AzureStorage
{
using System;
using System.Collections.Generic;
+ using System.Diagnostics;
using System.Threading.Tasks;
static class Utils
{
public static readonly Task CompletedTask = Task.FromResult(0);
+ public static readonly string ExtensionVersion = FileVersionInfo.GetVersionInfo(typeof(AzureStorageOrchestrationService).Assembly.Location).FileVersion;
+
public static async Task ParallelForEachAsync(
this IEnumerable enumerable,
Func createTask)
diff --git a/src/DurableTask.Core/DurableTask.Core.csproj b/src/DurableTask.Core/DurableTask.Core.csproj
index 02bab0cd1..46f614855 100644
--- a/src/DurableTask.Core/DurableTask.Core.csproj
+++ b/src/DurableTask.Core/DurableTask.Core.csproj
@@ -13,7 +13,7 @@
-
+
\ No newline at end of file
diff --git a/src/DurableTask.Emulator/DurableTask.Emulator.csproj b/src/DurableTask.Emulator/DurableTask.Emulator.csproj
index 738529a23..5b0f6d40d 100644
--- a/src/DurableTask.Emulator/DurableTask.Emulator.csproj
+++ b/src/DurableTask.Emulator/DurableTask.Emulator.csproj
@@ -10,7 +10,7 @@
-
+
diff --git a/tools/DurableTask.props b/tools/DurableTask.props
index d6fb2f495..985463bf0 100644
--- a/tools/DurableTask.props
+++ b/tools/DurableTask.props
@@ -32,9 +32,9 @@
..\..\build_output\packages
- 2.0.0.5
- 2.0.0.5
- 2.0.0.5
+ 2.0.0.6
+ 2.0.0.6
+ 2.0.0.6
Microsoft
Durable Task Framework
This package provides a C# based durable task framework for writing long running applications.