Skip to content

Fix orphaned spans in WCF traces #7842

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Nov 21, 2025
Merged

Fix orphaned spans in WCF traces #7842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
09c559b
Add spans to Wcf service endpoints
andrewlock Nov 18, 2025
34f5a55
nit: mark exceptions as nullable to satisfy nullable annotations
andrewlock Nov 18, 2025
d420d35
Ensure we restore the ambient span before executing service body
andrewlock Nov 19, 2025
8917c4c
Update snapshots and tests
andrewlock Nov 19, 2025
f735779
Create separate traces to make more stable snapshots
andrewlock Nov 19, 2025
02ef8fd
nit: delay capturing the context unless we need it
andrewlock Nov 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 5 additions & 12 deletions ...r/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/AfterReceiveRequestIntegration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TMessageRpc>(TTarget inst
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exception exception, in CallTargetState state)
internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exception? exception, in CallTargetState state)
{
// Add an exception and close the span only if there was an uncaught exception,
// which should only happen if one of the IClientMessageInspector's has thrown an exception.
Expand All @@ -80,17 +80,10 @@ internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exceptio
state.Scope.DisposeWithException(exception);
}

if (state.Scope != null)
{
// OnMethodBegin started an active span that can be accessed by IDispatchMessageInspector's and the actual WCF endpoint
// Before returning, we must reset the scope to the previous active scope, so that callers of this method do not see this scope
// Don't worry, this will be accessed and closed by the BeforeSendReplyIntegration
if (Tracer.Instance.ScopeManager is IScopeRawAccess rawAccess)
{
rawAccess.Active = state.PreviousScope;
DistributedTracer.Instance.SetSpanContext(state.PreviousDistributedSpanContext);
}
}
// OnMethodBegin started an active span that can be accessed by IDispatchMessageInspector's and the actual WCF endpoint
// Before returning, we must reset the scope to the previous active scope, so that callers of this method do not see this scope
// Don't worry, this will be accessed and closed by the BeforeSendReplyIntegration
WcfCommon.RestorePreviousScope(in state);

return CallTargetReturn.GetDefault();
}
Expand Down
31 changes: 15 additions & 16 deletions ...g.Trace/ClrProfiler/AutoInstrumentation/Wcf/AsyncMethodInvoker_InvokeBegin_Integration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System;
using System.ComponentModel;
using Datadog.Trace.ClrProfiler.CallTarget;
using Datadog.Trace.DuckTyping;

namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
{
Expand All @@ -29,27 +28,27 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
[EditorBrowsable(EditorBrowsableState.Never)]
public class AsyncMethodInvoker_InvokeBegin_Integration
{
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception exception, in CallTargetState state)
internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object? instanceArg, object[]? inputs, ref AsyncCallback? callback, object? state)
{
if (exception is not null)
var tracer = Tracer.Instance;
if (!tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !tracer.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
{
var operationContext = WcfCommon.GetCurrentOperationContext?.Invoke();
return CallTargetState.GetDefault();
}

if (operationContext != null && operationContext.TryDuckCast<IOperationContextStruct>(out var operationContextProxy))
{
var requestContext = operationContextProxy.RequestContext;
return WcfCommon.ActivateScopeFromContext();
}

// Retrieve the scope that we saved during InvokeBegin
if (((IDuckType?)requestContext)?.Instance is object requestContextInstance
&& WcfCommon.Scopes.TryGetValue(requestContextInstance, out var scope))
{
// Add the exception but do not dispose the span.
// BeforeSendReplyIntegration is responsible for closing the span.
scope.Span?.SetException(exception);
}
}
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception? exception, in CallTargetState state)
{
if (state.Scope is not null && exception is not null)
{
// Add the exception but do not dispose the scope.
// BeforeSendReplyIntegration is responsible for closing the span.
state.Scope.Span.SetException(exception);
}

WcfCommon.RestorePreviousScope(in state);
return new CallTargetReturn<TReturn>(returnValue);
}
}
Expand Down
35 changes: 17 additions & 18 deletions ...dog.Trace/ClrProfiler/AutoInstrumentation/Wcf/AsyncMethodInvoker_InvokeEnd_Integration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System;
using System.ComponentModel;
using Datadog.Trace.ClrProfiler.CallTarget;
using Datadog.Trace.DuckTyping;

namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
{
Expand All @@ -29,6 +28,17 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
[EditorBrowsable(EditorBrowsableState.Never)]
public class AsyncMethodInvoker_InvokeEnd_Integration
{
internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object? instanceArg, object[]? outputs, IAsyncResult? result)
{
var tracer = Tracer.Instance;
if (!tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !tracer.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
{
return CallTargetState.GetDefault();
}

return WcfCommon.ActivateScopeFromContext();
}

/// <summary>
/// OnMethodEnd callback
/// </summary>
Expand All @@ -39,27 +49,16 @@ public class AsyncMethodInvoker_InvokeEnd_Integration
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception exception, in CallTargetState state)
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception? exception, in CallTargetState state)
{
if (exception is not null)
if (state.Scope is not null && exception is not null)
{
var operationContext = WcfCommon.GetCurrentOperationContext?.Invoke();

if (operationContext != null && operationContext.TryDuckCast<IOperationContextStruct>(out var operationContextProxy))
{
var requestContext = operationContextProxy.RequestContext;

// Retrieve the scope that we saved during InvokeBegin
if (((IDuckType?)requestContext)?.Instance is object requestContextInstance
&& WcfCommon.Scopes.TryGetValue(requestContextInstance, out var scope))
{
// Add the exception but do not dispose the span.
// BeforeSendReplyIntegration is responsible for closing the span.
scope.Span?.SetException(exception);
}
}
// Add the exception but do not dispose the scope.
// BeforeSendReplyIntegration is responsible for closing the span.
state.Scope.Span.SetException(exception);
}

WcfCommon.RestorePreviousScope(in state);
return new CallTargetReturn<TReturn>(returnValue);
}
}
Expand Down
4 changes: 2 additions & 2 deletions tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/BeforeSendReplyIntegration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TMessageRpc>(TTarget inst
}

var rpcProxy = rpc.DuckCast<MessageRpcStruct>();
if (((IDuckType?)rpcProxy.OperationContext.RequestContext)?.Instance is object requestContextInstance
if (rpcProxy.OperationContext.RequestContext?.Instance is { } requestContextInstance
&& WcfCommon.Scopes.TryGetValue(requestContextInstance, out var scope))
{
return new CallTargetState(scope);
Expand All @@ -65,7 +65,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TMessageRpc>(TTarget inst
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exception exception, in CallTargetState state)
internal static CallTargetReturn OnMethodEnd<TTarget>(TTarget instance, Exception? exception, in CallTargetState state)
{
state.Scope.DisposeWithException(exception);
return CallTargetReturn.GetDefault();
Expand Down
2 changes: 1 addition & 1 deletion tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/ChannelHandlerIntegration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ internal static CallTargetState OnMethodBegin<TTarget, TRequestContext, TOperati
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception exception, in CallTargetState state)
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception? exception, in CallTargetState state)
{
state.Scope.DisposeWithException(exception);
return new CallTargetReturn<TReturn>(returnValue);
Expand Down
4 changes: 3 additions & 1 deletion tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/IRequestContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@

#nullable enable

using Datadog.Trace.DuckTyping;

namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
{
/// <summary>
/// System.ServiceModel.Channels.RequestContext interface for duck-typing
/// </summary>
internal interface IRequestContext
internal interface IRequestContext : IDuckType
{
/// <summary>
/// Gets the request message
Expand Down
11 changes: 6 additions & 5 deletions tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/SyncMethodInvokerIntegration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
using System;
using System.ComponentModel;
using Datadog.Trace.ClrProfiler.CallTarget;
using Datadog.Trace.DuckTyping;

namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
{
Expand Down Expand Up @@ -40,12 +39,13 @@ public class SyncMethodInvokerIntegration
/// <returns>Calltarget state value</returns>
internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object instanceArg, object[] inputs, ref object[] outputs)
{
if (!Tracer.Instance.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !Tracer.Instance.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
var tracer = Tracer.Instance;
if (!tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !tracer.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
{
return CallTargetState.GetDefault();
}

return new CallTargetState(Tracer.Instance.ActiveScope as Scope);
return WcfCommon.ActivateScopeFromContext();
}

/// <summary>
Expand All @@ -58,15 +58,16 @@ internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception exception, in CallTargetState state)
internal static CallTargetReturn<TReturn> OnMethodEnd<TTarget, TReturn>(TTarget instance, TReturn returnValue, Exception? exception, in CallTargetState state)
{
if (state.Scope is not null && exception is not null)
{
// Add the exception but do not dispose the scope.
// BeforeSendReplyIntegration is responsible for closing the span.
state.Scope.Span?.SetException(exception);
state.Scope.Span.SetException(exception);
}

WcfCommon.RestorePreviousScope(in state);
return new CallTargetReturn<TReturn>(returnValue);
}
}
Expand Down
12 changes: 6 additions & 6 deletions tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/TaskMethodInvokerIntegration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
using System;
using System.ComponentModel;
using Datadog.Trace.ClrProfiler.CallTarget;
using Datadog.Trace.Configuration;
using Datadog.Trace.DuckTyping;

namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Wcf
{
Expand Down Expand Up @@ -40,12 +38,13 @@ public class TaskMethodInvokerIntegration
/// <returns>Calltarget state value</returns>
internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object instanceArg, object[] inputs)
{
if (!Tracer.Instance.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !Tracer.Instance.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
var tracer = Tracer.Instance;
if (!tracer.CurrentTraceSettings.Settings.IsIntegrationEnabled(WcfCommon.IntegrationId) || !tracer.Settings.DelayWcfInstrumentationEnabled || WcfCommon.GetCurrentOperationContext is null)
{
return CallTargetState.GetDefault();
}

return new CallTargetState(Tracer.Instance.ActiveScope as Scope);
return WcfCommon.ActivateScopeFromContext();
}

/// <summary>
Expand All @@ -58,15 +57,16 @@ internal static CallTargetState OnMethodBegin<TTarget>(TTarget instance, object
/// <param name="exception">Exception instance in case the original code threw an exception.</param>
/// <param name="state">Calltarget state value</param>
/// <returns>A response value, in an async scenario will be T of Task of T</returns>
internal static TResponse OnAsyncMethodEnd<TTarget, TResponse>(TTarget instance, TResponse returnValue, Exception exception, in CallTargetState state)
internal static TResponse OnAsyncMethodEnd<TTarget, TResponse>(TTarget instance, TResponse returnValue, Exception? exception, in CallTargetState state)
{
if (state.Scope is not null && exception is not null)
{
// Add the exception but do not dispose the scope.
// BeforeSendReplyIntegration is responsible for closing the span.
state.Scope.Span?.SetException(exception);
state.Scope.Span.SetException(exception);
}

WcfCommon.RestorePreviousScope(in state);
return returnValue;
}
}
Expand Down
50 changes: 49 additions & 1 deletion tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Wcf/WcfCommon.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using System.Net;
using System.Reflection;
using System.Runtime.CompilerServices;
using Datadog.Trace.ClrProfiler.CallTarget;
using Datadog.Trace.Configuration;
using Datadog.Trace.DuckTyping;
using Datadog.Trace.ExtensionMethods;
Expand Down Expand Up @@ -77,7 +78,11 @@ internal class WcfCommon
httpMethod = httpRequestPropertyProxy.Method?.ToUpperInvariant();

// try to extract propagated context values from http headers
if (tracer.ActiveScope == null)
if (tracer.ActiveScope is { } activeScope)
{
Log.Warning("Skipped extracting headers due to existing scope: {ActiveScope}", activeScope.Span);
}
else
{
try
{
Expand Down Expand Up @@ -216,6 +221,49 @@ internal class WcfCommon

return null;
}

public static CallTargetState ActivateScopeFromContext()
Copy link
Contributor

@zacharycmontoya zacharycmontoya Nov 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

super small nit: It seems like this method has slightly too many responsibilities in that it's calling both Tracer.Instance.ActivateSpan(Span) and returning a CallTargetState. The only refactoring that comes to mind is to make this a TryGetScopeFromContext and return the fields used to populate the CallTargetState. And then from the caller you can activate the span and create the CallTargetState, but then you're making each callsite have to repeat multiple lines of code. I'm not sure if that's better than what you've done here, so no need to take my suggestion

Copy link
Member Author

@andrewlock andrewlock Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I agree with you, but then also it's easy to screw up and forget to capture one or more parts that you need.. plus we can delay capturing the distributed context unless we need it (though we should always hit the "activate" path in normal behaviour, so maybe not a big deal)

As you say, given every callsite would need to copy the exact same code to use the API, I'm not sure it's worth splitting out. We could still capture the previous and then return a different carrier object instead of CallTargetState from the method, and then convert it into a CallTargetState at the callsite, but that seems like more hassle for little gain? 🤷‍♂️ meh 😅

Copy link
Contributor

@zacharycmontoya zacharycmontoya Nov 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed: meh 😆

{
// First, capture the active scope
var tracer = Tracer.Instance;
var activeScope = tracer.InternalActiveScope;

var operationContext = WcfCommon.GetCurrentOperationContext?.Invoke();
if (operationContext != null && operationContext.TryDuckCast<IOperationContextStruct>(out var operationContextProxy))
{
var requestContext = operationContextProxy.RequestContext;

// Retrieve the scope that we saved during InvokeBegin
if (requestContext?.Instance is { } requestContextInstance
&& Scopes.TryGetValue(requestContextInstance, out var scope)
&& scope is not null
&& scope.Span.SpanId != activeScope?.Span.SpanId)
{
// capture the raw context for later activation
var spanContextRaw = DistributedTracer.Instance.GetSpanContextRaw() ?? activeScope?.Span.Context;
Log.Debug("Activating scope from operation context {ActivatedSpan}", scope.Span);

tracer.ActivateSpan(scope.Span);
// Add the exception but do not dispose the span.
// BeforeSendReplyIntegration is responsible for closing the span.
return new CallTargetState(scope, activeScope, spanContextRaw);
}
}

return CallTargetState.GetDefault();
}

public static void RestorePreviousScope(in CallTargetState state)
{
// Reset the scope to the previous active scope, so that callers of this method do not see this scope
// Don't worry, this will be accessed and closed by the BeforeSendReplyIntegration
if (state.Scope is not null && Tracer.Instance.ScopeManager is IScopeRawAccess rawAccess)
{
Log.Debug("Restoring previous scope from {Previous} to {Restored}", state.Scope.Span, state.PreviousScope?.Span);
rawAccess.Active = state.PreviousScope;
DistributedTracer.Instance.SetSpanContext(state.PreviousDistributedSpanContext);
}
}
}
}
#endif
Loading
Loading