From d26940d90ed21e97a63580be4e2cab9c4dc3e693 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 27 May 2025 11:21:46 +0200 Subject: [PATCH 01/23] feat(trace-exporter): introduce native trace exporter - Added `TelemetryClientConfiguration` struct to manage telemetry settings. - Implemented `TraceExporter` class for sending traces to the Datadog agent. - Created `TraceExporterConfiguration` class to configure trace exporting options. - Defined `TraceExporterError` and `TraceExporterException` for error handling. - Introduced enums `TraceExporterInputFormat` and `TraceExporterOutputFormat` for trace format management. - Updated `TracerManagerFactory` to utilize the new trace exporter and telemetry configurations. - Enhanced integration tests to validate data pipeline functionality across various transports. - Added support for telemetry metrics in existing tests and configurations. - Updated configuration keys to include `TraceDataPipelineEnabled` for better control over data pipeline behavior. --- tracer/build/_build/Build.Steps.cs | 59 ++++++- tracer/build/_build/Build.cs | 2 + tracer/build/_build/Projects.cs | 9 + tracer/src/Datadog.Trace/Agent/AgentWriter.cs | 5 + .../Configuration/ConfigurationKeys.cs | 6 + .../Configuration/TracerSettings.cs | 18 ++ .../src/Datadog.Trace/LibDatadog/ByteSlice.cs | 28 +++ .../src/Datadog.Trace/LibDatadog/CharSlice.cs | 50 ++++++ .../src/Datadog.Trace/LibDatadog/ErrorCode.cs | 114 +++++++++++++ .../Datadog.Trace/LibDatadog/ErrorHandle.cs | 46 +++++ .../Datadog.Trace/LibDatadog/NativeInterop.cs | 61 ++++--- .../TelemetryClientConfiguration.cs | 40 +++++ .../Datadog.Trace/LibDatadog/TraceExporter.cs | 78 +++++++++ .../LibDatadog/TraceExporterConfiguration.cs | 149 ++++++++++++++++ .../LibDatadog/TraceExporterError.cs | 29 ++++ .../LibDatadog/TraceExporterException.cs | 25 +++ .../LibDatadog/TraceExporterInputFormat.cs | 24 +++ .../LibDatadog/TraceExporterOutputFormat.cs | 24 +++ .../src/Datadog.Trace/TracerManagerFactory.cs | 64 ++++++- .../AWS/AwsLambdaTests.cs | 14 +- .../AgentMalfunctionTests.cs | 12 +- .../TelemetryTests.cs | 3 + .../TransportTests.cs | 46 +++-- .../DefaultTransportLargePayloadTests.cs | 7 +- .../LargePayload/LargePayloadTestBase.cs | 11 +- .../UnixDomainSocketLargePayloadTests.cs | 9 +- .../WindowsNamedPipeLargePayloadTests.cs | 9 +- .../AspNetVersionConflictTests.cs | 2 +- .../ContainerTaggingTests.cs | 10 +- .../LibDatadog/TraceExporterTests.cs | 161 ++++++++++++++++++ .../StatsTests.cs | 3 + .../Datadog.Trace.TestHelpers/TracerHelper.cs | 16 +- .../Datadog.Trace.Tests/DogStatsDTests.cs | 1 + .../Telemetry/config_norm_rules.json | 1 + 34 files changed, 1072 insertions(+), 64 deletions(-) create mode 100644 tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs create mode 100644 tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs create mode 100644 tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index b16f2bd72f57..db2ca24dedb5 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -84,7 +84,7 @@ partial class Build AbsolutePath TempDirectory => (AbsolutePath)(IsWin ? Path.GetTempPath() : "/tmp/"); - readonly string[] WafWindowsArchitectureFolders = { "win-x86", "win-x64" }; + readonly string[] WindowsArchitectureFolders = { "win-x86", "win-x64" }; Project NativeTracerProject => Solution.GetProject(Projects.ClrProfilerNative); Project NativeTracerTestsProject => Solution.GetProject(Projects.NativeTracerNativeTests); Project NativeLoaderProject => Solution.GetProject(Projects.NativeLoader); @@ -546,7 +546,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { if (IsWin) { - foreach (var architecture in WafWindowsArchitectureFolders) + foreach (var architecture in WindowsArchitectureFolders) { var source = LibDdwafDirectory() / "runtimes" / architecture / "native" / "ddwaf.dll"; var dest = MonitoringHomeDirectory / architecture; @@ -642,6 +642,42 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo } }); + Target CopyNativeFilesForTests => _ => _ + .Unlisted() + .After(Clean) + .After(BuildTracerHome) + .Executes(() => + { + foreach(var projectName in Projects.NativeFilesDependentTests) + { + var project = Solution.GetProject(projectName); + var testDir = project.Directory; + var frameworks = project.GetTargetFrameworks(); + var testBinFolder = testDir / "bin" / BuildConfiguration; + + if (IsWin) + { + foreach (var fmk in frameworks) + { + var (arch, ext) = GetWinArchitectureAndExtension(fmk); + var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; + var dest = testBinFolder / fmk / "LibDatadog.dll"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); + } + } + else + { + var (arch, ext) = GetUnixArchitectureAndExtension(); + var source = MonitoringHomeDirectory / arch / $"libdatadog_profiling.{ext}"; + foreach (var fmk in frameworks) + { + var dest = testBinFolder / fmk / $"LibDatadog.{ext}"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); + } + } + } + }); + Target CopyNativeFilesForAppSecUnitTests => _ => _ .Unlisted() .After(Clean) @@ -662,7 +698,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { var oldVersionTempPath = TempDirectory / $"libddwaf.{olderLibDdwafVersion}"; await DownloadWafVersion(olderLibDdwafVersion, oldVersionTempPath); - foreach (var arch in WafWindowsArchitectureFolders) + foreach (var arch in WindowsArchitectureFolders) { var oldVersionPath = oldVersionTempPath / "runtimes" / arch / "native" / "ddwaf.dll"; var source = MonitoringHomeDirectory / arch; @@ -938,7 +974,8 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo .Executes(() => { // extract debug info from everything in monitoring home and copy it to the linux symbols directory - var files = MonitoringHomeDirectory.GlobFiles("linux-*/*.so"); + var files = MonitoringHomeDirectory.GlobFiles("linux-*/*.so") + .Where(file => !Path.GetFileName(file).Contains("libdatadog_profiling.so")); foreach (var file in files) { @@ -1263,6 +1300,7 @@ void PrepareMonitoringHomeLinuxForPackaging(AbsolutePath assetsDirectory, string .After(CompileManagedSrc) .After(BuildRunnerTool) .DependsOn(CopyNativeFilesForAppSecUnitTests) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CompileManagedTestHelpers) .Executes(() => { @@ -2707,6 +2745,17 @@ static LogLevel ParseNativeLogLevel(string value) (false, true) => ($"linux-musl-{UnixArchitectureIdentifier}", "so"), }; + private (string Arch, string Ext) GetWinArchitectureAndExtension(string fmk) + { + return RuntimeInformation.ProcessArchitecture switch + { + Architecture.X86 => ("win-x86", "dll"), + Architecture.X64 => ("win-x64", "dll"), + Architecture.Arm64 => ("win-arm64", "dll"), + _ => ("win-x86", "dll") // Default to x86 for unknown architectures + }; + } + // the integration tests need their own copy of the profiler, this achieved through build.props on Windows, but doesn't seem to work under Linux private void IntegrationTestLinuxOrOsxProfilerDirFudge(string project) { @@ -2841,7 +2890,7 @@ private void DotnetBuild( .CombineWith(projPaths, (settings, projPath) => settings.SetProjectFile(projPath))); } - + private async Task GetVcpkg() { var vcpkgFilePath = string.Empty; diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index c00abf508e07..80601a17cc8d 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -286,6 +286,7 @@ void DeleteReparsePoints(string path) .Description("Builds the integration tests for Windows") .DependsOn(CompileManagedTestHelpers) .DependsOn(CompileIntegrationTests) + .DependsOn(CopyNativeFilesForTests) .DependsOn(BuildRunnerTool); Target BuildAspNetIntegrationTests => _ => _ @@ -334,6 +335,7 @@ void DeleteReparsePoints(string path) .DependsOn(CompileLinuxOrOsxIntegrationTests) .DependsOn(CompileLinuxDdDotnetIntegrationTests) .DependsOn(BuildRunnerTool) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CopyServerlessArtifacts); Target BuildAndRunLinuxIntegrationTests => _ => _ diff --git a/tracer/build/_build/Projects.cs b/tracer/build/_build/Projects.cs index 4a2f984ea53e..58c586e96232 100644 --- a/tracer/build/_build/Projects.cs +++ b/tracer/build/_build/Projects.cs @@ -18,10 +18,12 @@ public static class Projects public const string BenchmarksOpenTelemetryApi = "Benchmarks.OpenTelemetry.Api"; public const string BenchmarksOpenTelemetryInstrumentedApi = "Benchmarks.OpenTelemetry.InstrumentedApi"; + public const string TraceTests = "Datadog.Trace.Tests"; public const string TraceIntegrationTests = "Datadog.Trace.IntegrationTests"; public const string AppSecUnitTests = "Datadog.Trace.Security.Unit.Tests"; public const string AppSecIntegrationTests = "Datadog.Trace.Security.IntegrationTests"; public const string ClrProfilerIntegrationTests = "Datadog.Trace.ClrProfiler.IntegrationTests"; + public const string ClrProfilerManagedTests = "Datadog.Trace.ClrProfiler.Managed.Tests"; public const string DdTraceIntegrationTests = "Datadog.Trace.Tools.Runner.IntegrationTests"; public const string DdTraceArtifactsTests = "Datadog.Trace.Tools.Runner.ArtifactTests"; public const string DdDotnetIntegrationTests = "Datadog.Trace.Tools.dd_dotnet.IntegrationTests"; @@ -39,6 +41,13 @@ public static class Projects public const string DebuggerUnreferencedExternal = "Samples.Probes.Unreferenced.External"; public const string RazorPages = "Samples.AspNetCoreRazorPages"; + + public static readonly string[] NativeFilesDependentTests = { + AppSecUnitTests, + ClrProfilerManagedTests, + TraceIntegrationTests, + TraceTests + }; } public static class FileNames diff --git a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs index 24030a6fb96f..f654d833d513 100644 --- a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs +++ b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs @@ -186,6 +186,11 @@ await Task.WhenAny(_flushTask, Task.Delay(TimeSpan.FromSeconds(20))) await _statsAggregator.DisposeAsync().ConfigureAwait(false); } + if (_api is IDisposable disposableApi) + { + disposableApi.Dispose(); + } + if (!success) { Log.Warning("Could not flush all traces before process exit"); diff --git a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs index 668d03805ee7..2a0db80607f6 100644 --- a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs +++ b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs @@ -293,6 +293,12 @@ internal static partial class ConfigurationKeys /// public const string RuntimeMetricsEnabled = "DD_RUNTIME_METRICS_ENABLED"; + /// + /// Use libdatadog data pipeline to send traces. + /// Default value is false (disabled). + /// + public const string TraceDataPipelineEnabled = "DD_TRACE_DATA_PIPELINE_ENABLED"; + /// /// Configuration key for when a standalone instance of the Trace Agent needs to be started. /// diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index a4dbc3ab3d3f..ff56ee9c3c7d 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -413,6 +413,18 @@ _ when x.ToBoolean() is { } boolean => boolean, .AsBoolResult() .OverrideWith(in otelRuntimeMetricsEnabled, ErrorLog, defaultValue: false); + DataPipelineEnabled = config + .WithKeys(ConfigurationKeys.TraceDataPipelineEnabled) + .AsBool(defaultValue: true); + + // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter + // as it may lead to different stats results than the managed one. + if (DataPipelineEnabled && StatsComputationEnabled) + { + DataPipelineEnabled = false; + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + // We should also be writing telemetry for OTEL_LOGS_EXPORTER similar to OTEL_METRICS_EXPORTER, but we don't have a corresponding Datadog config // When we do, we can insert that here @@ -1061,6 +1073,12 @@ public bool DiagnosticSourceEnabled /// internal bool RuntimeMetricsEnabled => DynamicSettings.RuntimeMetricsEnabled ?? _runtimeMetricsEnabled; + /// + /// Gets a value indicating whether libdatadog data pipeline + /// is enabled. + /// + internal bool DataPipelineEnabled { get; } + /// /// Gets the comma separated list of url patterns to skip tracing. /// diff --git a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs new file mode 100644 index 000000000000..ab47011133a5 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs @@ -0,0 +1,28 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a slice of a byte array in memory. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct ByteSlice +{ + /// + /// Pointer to the start of the slice. + /// + internal IntPtr Ptr; + + /// + /// Length of the slice. + /// + internal UIntPtr Len; +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs new file mode 100644 index 000000000000..7ac2a79c46cc --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs @@ -0,0 +1,50 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a slice of a UTF-8 encoded string in memory. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct CharSlice +{ + /// + /// Pointer to the start of the slice. + /// + internal IntPtr Ptr; + + /// + /// Length of the slice. + /// + internal UIntPtr Len; + + /// + /// Initializes a new instance of the struct. + /// This can be further optimized if we can avoid copying the string to unmanaged memory. + /// + /// The string to copy into memory. + internal CharSlice(string? str) + { + if (str == null) + { + Ptr = IntPtr.Zero; + Len = UIntPtr.Zero; + } + else + { + // copy over str to unmanaged memory + var bytes = System.Text.Encoding.UTF8.GetBytes(str); + Ptr = Marshal.AllocHGlobal(bytes.Length); + Marshal.Copy(bytes, 0, Ptr, bytes.Length); + Len = (UIntPtr)bytes.Length; + } + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs b/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs new file mode 100644 index 000000000000..3669da701268 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs @@ -0,0 +1,114 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents error codes that can occur when exporting traces. +/// +internal enum ErrorCode +{ + /// + /// Address already in use + /// + AddressInUse = 0, + + /// + /// Connection aborted + /// + ConnectionAborted = 1, + + /// + /// Connection refused + /// + ConnectionRefused = 2, + + /// + /// Connection reset by peer + /// + ConnectionReset = 3, + + /// + /// Error parsing HTTP body + /// + HttpBodyFormat = 4, + + /// + /// HTTP body too long + /// + HttpBodyTooLong = 5, + + /// + /// HTTP error originated by client + /// + HttpClient = 6, + + /// + /// HTTP empty body + /// + HttpEmptyBody = 7, + + /// + /// Error while parsing HTTP message + /// + HttpParse = 8, + + /// + /// HTTP error originated by server + /// + HttpServer = 9, + + /// + /// HTTP unknown error + /// + HttpUnknown = 10, + + /// + /// HTTP wrong status number + /// + HttpWrongStatus = 11, + + /// + /// Invalid argument provided + /// + InvalidArgument = 12, + + /// + /// Invalid data payload + /// + InvalidData = 13, + + /// + /// Invalid input + /// + InvalidInput = 14, + + /// + /// Invalid URL + /// + InvalidUrl = 15, + + /// + /// Input/Output error + /// + IoError = 16, + + /// + /// Unknown network error + /// + NetworkUnknown = 17, + + /// + /// Serialization/Deserialization error + /// + Serde = 18, + + /// + /// Operation timed out + /// + TimedOut = 19, +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs b/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs new file mode 100644 index 000000000000..3a1a2f5dfd13 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs @@ -0,0 +1,46 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +internal class ErrorHandle : SafeHandle +{ + public ErrorHandle() + : base(IntPtr.Zero, true) + { + } + + public ErrorHandle(IntPtr handle) + : base(handle, true) + { + SetHandle(handle); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + NativeInterop.Exporter.FreeError(handle); + return true; + } + + public TraceExporterException ToException() + { + return new TraceExporterException(Marshal.PtrToStructure(handle)); + } + + public void ThrowIfError() + { + if (!IsInvalid) + { + throw ToException(); + } + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs b/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs index d7aafa9a1cab..608e27067757 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs @@ -16,52 +16,61 @@ internal class NativeInterop internal static class Exporter { - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")] - // internal static extern ErrorHandle Create(out IntPtr outHandle, SafeHandle config); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")] + internal static extern ErrorHandle New(out IntPtr outHandle, SafeHandle config); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_error_free")] - internal static extern void ReleaseError(IntPtr error); + internal static extern void FreeError(IntPtr error); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_free")] - internal static extern void Release(IntPtr handle); + internal static extern void Free(IntPtr handle); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")] - // internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")] + internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response); } internal static class Config { [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_new")] - internal static extern void Create(out IntPtr outHandle); + internal static extern void New(out IntPtr outHandle); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_free")] - internal static extern void Release(IntPtr handle); + internal static extern void Free(IntPtr handle); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")] - // internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")] + internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")] - // internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")] + internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")] - // internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")] + internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")] - // internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")] + internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")] - // internal static extern ErrorHandle SetInterperter(SafeHandle config, CharSlice interpreter); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")] + internal static extern ErrorHandle SetInterpreter(SafeHandle config, CharSlice interpreter); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")] - // internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")] + internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")] - // internal static extern ErrorHandle SetEnvironment(SafeHandle config, CharSlice env); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")] + internal static extern ErrorHandle SetEnv(SafeHandle config, CharSlice env); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")] - // internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")] + internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")] - // internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")] + internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_compute_stats")] + internal static extern ErrorHandle SetComputeStats(SafeHandle config, bool isEnabled); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_enable_telemetry")] + internal static extern ErrorHandle EnableTelemetry(SafeHandle config, IntPtr telemetryConfig); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_client_computed_stats")] + internal static extern ErrorHandle SetClientComputedStats(SafeHandle config, bool clientComputedStats); } } diff --git a/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs new file mode 100644 index 000000000000..e9521ea3bcb5 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs @@ -0,0 +1,40 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a configuration for the telemetry client. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct TelemetryClientConfiguration +{ + /// + /// The interval at which telemetry should be sent, in milliseconds. + /// + public ulong Interval; + + /// + /// A V4 UUID that represents a tracer session. This ID should: + /// - Be generated when the tracer starts + /// - Be identical within the context of a host (i.e. multiple threads/processes + /// that belong to a single instrumented app should share the same runtime_id) + /// - Be associated with traces to allow correlation between traces and telemetry data + /// + public CharSlice RuntimeId; + + /* + /// TODO: enable with v18 + /// + /// Whether to enable debug mode for telemetry. When enabled, sets the dd-telemetry-debug-enabled header to true. + /// Defaults to false. + /// + /// public bool DebugEnabled; + */ +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs new file mode 100644 index 000000000000..cd9947a20391 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs @@ -0,0 +1,78 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.LibDatadog; + +internal class TraceExporter : SafeHandle, IApi +{ + private readonly TraceExporterConfiguration _configuration; + private readonly IDatadogLogger _log; + + public TraceExporter( + TraceExporterConfiguration configuration, + IDatadogLogger? log = null) + : base(IntPtr.Zero, true) + { + _log = log ?? DatadogLogging.GetLoggerFor(); + _configuration = configuration; + + _log.Debug("Creating new TraceExporter"); + using var errPtr = NativeInterop.Exporter.New(out var ptr, _configuration); + errPtr.ThrowIfError(); + SetHandle(ptr); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + public Task SendTracesAsync(ArraySegment traces, int numberOfTraces, bool statsComputationEnabled, long numberOfDroppedP0Traces, long numberOfDroppedP0Spans, bool appsecStandaloneEnabled) + { + _log.Debug("Sending {Count} traces to the Datadog Agent.", numberOfTraces); + + unsafe + { + fixed (byte* ptr = traces.Array) + { + var tracesSlice = new ByteSlice + { + Ptr = (IntPtr)ptr, + Len = (UIntPtr)traces.Count + }; + + var responsePtr = IntPtr.Zero; + using var error = NativeInterop.Exporter.Send(this, tracesSlice, (UIntPtr)numberOfTraces, ref responsePtr); + if (!error.IsInvalid) + { + var ex = error.ToException(); + _log.Error(ex, "An error occurred while sending data to the agent. Error Code: {ErrorCode}, message: {Message}", ex.ErrorCode, ex.Message); + throw ex; + } + } + } + + _log.Debug("Successfully sent {Count} traces to the Datadog Agent.", numberOfTraces); + + return Task.FromResult(true); + } + + public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) + { + _log.Debug("No-op: stats computation happens in the data pipeline."); + return Task.FromResult(true); + } + + protected override bool ReleaseHandle() + { + NativeInterop.Exporter.Free(handle); + return true; + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs new file mode 100644 index 000000000000..028bf1689e45 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs @@ -0,0 +1,149 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a configuration for the trace exporter. +/// +internal class TraceExporterConfiguration : SafeHandle +{ + private readonly IntPtr _telemetryClientConfigurationHandle; + + public TraceExporterConfiguration() + : base(IntPtr.Zero, true) + { + NativeInterop.Config.New(out var ptr); + SetHandle(ptr); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + public string Url + { + init + { + using var error = NativeInterop.Config.SetUrl(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string TraceVersion + { + init + { + using var error = NativeInterop.Config.SetTracerVersion(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string Language + { + init + { + using var error = NativeInterop.Config.SetLanguage(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string LanguageVersion + { + init + { + using var error = NativeInterop.Config.SetLanguageVersion(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string LanguageInterpreter + { + init + { + using var error = NativeInterop.Config.SetInterpreter(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string? Hostname + { + init + { + using var error = NativeInterop.Config.SetHostname(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string? Env + { + init + { + using var error = NativeInterop.Config.SetEnv(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string? Version + { + init + { + using var error = NativeInterop.Config.SetVersion(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public string? Service + { + init + { + using var error = NativeInterop.Config.SetService(this, new CharSlice(value)); + error.ThrowIfError(); + } + } + + public TelemetryClientConfiguration? TelemetryClientConfiguration + { + init + { + if (value.HasValue) + { + _telemetryClientConfigurationHandle = Marshal.AllocHGlobal(Marshal.SizeOf(value.Value)); + Marshal.StructureToPtr(value.Value, _telemetryClientConfigurationHandle, true); + + using var error = NativeInterop.Config.EnableTelemetry(this, _telemetryClientConfigurationHandle); + error.ThrowIfError(); + } + } + } + + public bool ComputeStats + { + init + { + using var error = NativeInterop.Config.SetComputeStats(this, value); + error.ThrowIfError(); + } + } + + public bool ClientComputedStats + { + init + { + using var error = NativeInterop.Config.SetClientComputedStats(this, value); + error.ThrowIfError(); + } + } + + protected override bool ReleaseHandle() + { + NativeInterop.Config.Free(handle); + Marshal.FreeHGlobal(_telemetryClientConfigurationHandle); + return true; + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs new file mode 100644 index 000000000000..62553fabf501 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs @@ -0,0 +1,29 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents errors that can occur when exporting traces. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct TraceExporterError +{ + /// + /// The error code representing the domain of the error. + /// Consumers can use this to determine how to handle the error. + /// + internal ErrorCode Code; + + /// + /// Human-readable error message describing the error. + /// + internal IntPtr Msg; +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs new file mode 100644 index 000000000000..a45d911f1f00 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs @@ -0,0 +1,25 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents an exception thrown by the libdatadog library. +/// +internal class TraceExporterException : Exception +{ + public TraceExporterException(TraceExporterError exporterError) + : base(Marshal.PtrToStringAnsi(exporterError.Msg)) + { + ErrorCode = exporterError.Code; + } + + public ErrorCode ErrorCode { get; } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs new file mode 100644 index 000000000000..e8c4b237cfc6 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents the format of the input traces, as expected by the trace exporter. +/// +internal enum TraceExporterInputFormat +{ + /// + /// Used when the traces are sent to the agent without processing. The whole payload is sent as is to the agent. + /// + Proxy = 0, + + /// + /// Version 0.4 of the trace exporter format. + /// + V04 = 1, +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs new file mode 100644 index 000000000000..1c55972a8af5 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents the format of the output traces, as expected by the trace exporter. +/// +internal enum TraceExporterOutputFormat +{ + /// + /// Version 0.4 of the trace exporter format. + /// + V04 = 0, + + /// + /// Version 0.7 of the trace exporter format. + /// + V07 = 1, +} diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 56b64e29e25d..8696bf79b770 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -15,9 +15,11 @@ using Datadog.Trace.DataStreamsMonitoring; using Datadog.Trace.DogStatsd; using Datadog.Trace.Iast; +using Datadog.Trace.LibDatadog; using Datadog.Trace.Logging; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.Logging.TracerFlare; +using Datadog.Trace.PlatformHelpers; using Datadog.Trace.Processors; using Datadog.Trace.Propagators; using Datadog.Trace.RemoteConfigurationManagement; @@ -30,6 +32,7 @@ using Datadog.Trace.Vendors.StatsdClient; using ConfigurationKeys = Datadog.Trace.Configuration.ConfigurationKeys; using MetricsTransportType = Datadog.Trace.Vendors.StatsdClient.Transport.TransportType; +using NativeInterop = Datadog.Trace.ContinuousProfiler.NativeInterop; using Stopwatch = System.Diagnostics.Stopwatch; namespace Datadog.Trace @@ -349,13 +352,72 @@ protected virtual ISpanSampler GetSpanSampler(TracerSettings settings) protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService) { var apiRequestFactory = TracesTransportStrategy.Get(settings.Exporter); - var api = new Api(apiRequestFactory, statsd, updateSampleRates, settings.Exporter.PartialFlushEnabled); + var api = GetApi(settings, statsd, updateSampleRates, apiRequestFactory, settings.Exporter.PartialFlushEnabled); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); return new AgentWriter(api, statsAggregator, statsd, maxBufferSize: settings.TraceBufferSize, batchInterval: settings.TraceBatchInterval, apmTracingEnabled: settings.ApmTracingEnabled); } + private IApi GetApi(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IApiRequestFactory apiRequestFactory, bool partialFlushEnabled) + { + if (settings.DataPipelineEnabled) + { + var telemetrySettings = TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, TelemetryFactory.Config, settings, isAgentAvailable: null); + TelemetryClientConfiguration? telemetryClientConfiguration = null; + + // We don't know how to handle telemetry in Agentless mode yet + // so we disable telemetry in this case + if (telemetrySettings.TelemetryEnabled && telemetrySettings.Agentless == null) + { + telemetryClientConfiguration = new TelemetryClientConfiguration + { + Interval = (ulong)telemetrySettings.HeartbeatInterval.Milliseconds, + RuntimeId = new CharSlice(Tracer.RuntimeId) + }; + } + + // When APM is disabled, we don't want to compute stats at all + // A common use case is in Application Security Monitoring (ASM) scenarios: + // when APM is disabled but ASM is enabled. + var clientComputedStats = !settings.StatsComputationEnabled && !settings.ApmTracingEnabled; + + var configuration = new TraceExporterConfiguration + { + Url = GetUrl(settings), + TraceVersion = TracerConstants.AssemblyVersion, + Env = settings.Environment, + Version = settings.ServiceVersion, + Service = settings.ServiceName, + Hostname = HostMetadata.Instance.Hostname, + Language = ".NET", + LanguageVersion = FrameworkDescription.Instance.ProductVersion, + LanguageInterpreter = FrameworkDescription.Instance.Name, + ComputeStats = settings.StatsComputationEnabled, + TelemetryClientConfiguration = telemetryClientConfiguration, + ClientComputedStats = clientComputedStats + }; + + return new TraceExporter(configuration); + } + + return new Api(apiRequestFactory, statsd, updateSampleRates, partialFlushEnabled); + } + + private string GetUrl(TracerSettings settings) + { + switch (settings.Exporter.TracesTransport) + { + case TracesTransportType.WindowsNamedPipe: + return $"windows://./pipe/{settings.Exporter.TracesPipeName}"; + case TracesTransportType.UnixDomainSocket: + return $"unix://{settings.Exporter.TracesUnixDomainSocketPath}"; + case TracesTransportType.Default: + default: + return settings.Exporter.AgentUri.ToString(); + } + } + protected virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) => DiscoveryService.Create(settings.Exporter); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 49e8308ac85d..c494f7f1d196 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -31,10 +32,12 @@ public AwsLambdaTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("Category", "ArmUnsupported")] [Trait("Category", "Lambda")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { // See documentation at docs/development/Serverless.md for examples and diagrams if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IsAlpine"))) @@ -43,6 +46,7 @@ public async Task SubmitsTraces() return; } + EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); using var extensionWithContext = new MockLambdaExtension(shouldSendContext: true, port: 9004, Output); using var extensionNoContext = new MockLambdaExtension(shouldSendContext: false, port: 9003, Output); using var agent = EnvironmentHelper.GetMockAgent(fixedPort: 5002); @@ -89,7 +93,13 @@ public async Task SubmitsTraces() settings.AddRegexScrubber(StackRegex, "$1 Cannot assign requested address (SocketException)$2"); settings.AddRegexScrubber(ErrorMsgRegex, "$1 Cannot assign requested address$2"); + foreach (var span in allSpans) + { + Output.WriteLine(span.ToString()); + } + await VerifyHelper.VerifySpans(allSpans, settings) + .DisableRequireUniquePrefix() .UseFileName(nameof(AwsLambdaTests)); } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs index 61a99bbdbdce..689fdb72def4 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs @@ -45,13 +45,14 @@ public static IEnumerable TestData => from behaviour in (AgentBehaviour[])Enum.GetValues(typeof(AgentBehaviour)) from transportType in Transports from metadataSchemaVersion in new[] { "v0", "v1" } - select new object[] { behaviour, transportType, metadataSchemaVersion }; + from dataPipelineEnabled in new[] { true, false } + select new object[] { behaviour, transportType, metadataSchemaVersion, dataPipelineEnabled }; [SkippableTheory] [MemberData(nameof(TestData))] [Trait("Category", "EndToEnd")] [Trait("RunOnWindows", "True")] - public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transportType, string metadataSchemaVersion) + public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transportType, string metadataSchemaVersion, bool dataPipelineEnabled) { SkipOn.Platform(SkipOn.PlatformValue.MacOs); if (transportType == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) @@ -59,7 +60,14 @@ public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transpo throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); } + if (transportType == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) + { + throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + } + EnvironmentHelper.EnableTransport(transportType); + EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + using var agent = EnvironmentHelper.GetMockAgent(); var customResponse = behaviour switch { diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs index ab6bb436df6d..998a7675bdf9 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs @@ -228,6 +228,9 @@ public async Task WhenUsingUdsAgent_UsesUdsTelemetry(bool? enableDependencies) [Trait("RunOnWindows", "True")] public async Task Telemetry_SendsMetrics() { + // telemetry metric events under test are sent only when using managed trace exporter + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, "false"); + using var agent = MockTracerAgent.Create(Output, useTelemetry: true); Output.WriteLine($"Assigned port {agent.Port} for the agentPort."); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs index 27e22e93c651..68d3b815cf36 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Threading.Tasks; using Datadog.Trace.Agent; +using Datadog.Trace.Configuration; using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -21,6 +22,15 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests [UsesVerify] public class TransportTests : TestHelper { + private static readonly TestTransports[] Transports = new[] + { + TestTransports.Tcp, + TestTransports.WindowsNamedPipe, +#if NETCOREAPP3_1_OR_GREATER + TestTransports.Uds, +#endif + }; + private readonly ITestOutputHelper _output; // Using Telemetry sample as it's simple @@ -31,23 +41,20 @@ public TransportTests(ITestOutputHelper output) } public static IEnumerable Data => - Enum.GetValues(typeof(TracesTransportType)) - .Cast() -#if !NETCOREAPP3_1_OR_GREATER - .Where(x => x != TracesTransportType.UnixDomainSocket) -#endif - .Select(x => new object[] { x }); + from transport in Transports + from dataPipelineEnabled in new[] { true, false } + select new object[] { transport, dataPipelineEnabled }; [SkippableTheory] [MemberData(nameof(Data))] [Trait("Category", "EndToEnd")] [Trait("RunOnWindows", "True")] - public async Task TransportsWorkCorrectly(Enum transport) + public async Task TransportsWorkCorrectly(TestTransports transport, bool dataPipelineEnabled) { - var transportType = (TracesTransportType)transport; + var transportType = TracesTransportTypeFromTestTransport(transport); if (transportType != TracesTransportType.WindowsNamedPipe) { - await RunTest(transportType); + await RunTest(transportType, dataPipelineEnabled); return; } @@ -58,7 +65,7 @@ public async Task TransportsWorkCorrectly(Enum transport) try { attemptsRemaining--; - await RunTest(transportType); + await RunTest(transportType, dataPipelineEnabled); return; } catch (Exception ex) when (attemptsRemaining > 0 && ex is not SkipException) @@ -68,7 +75,18 @@ public async Task TransportsWorkCorrectly(Enum transport) } } - private async Task RunTest(TracesTransportType transportType) + private TracesTransportType TracesTransportTypeFromTestTransport(TestTransports transport) + { + return transport switch + { + TestTransports.Tcp => TracesTransportType.Default, + TestTransports.WindowsNamedPipe => TracesTransportType.WindowsNamedPipe, + TestTransports.Uds => TracesTransportType.UnixDomainSocket, + _ => throw new InvalidOperationException($"Unknown transport {transport}"), + }; + } + + private async Task RunTest(TracesTransportType transportType, bool dataPipelineEnabled) { const int expectedSpanCount = 1; @@ -77,6 +95,12 @@ private async Task RunTest(TracesTransportType transportType) throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); } + if (transportType == TracesTransportType.UnixDomainSocket && !EnvironmentTools.IsLinux() && dataPipelineEnabled) + { + throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + } + + EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); EnvironmentHelper.EnableTransport(GetTransport(transportType)); using var telemetry = this.ConfigureTelemetry(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs index 266d1c628dd0..10160e301e78 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs @@ -4,6 +4,7 @@ // using System.Threading.Tasks; +using Datadog.Trace.TestHelpers; using Xunit; using Xunit.Abstractions; @@ -17,8 +18,10 @@ public DefaultTransportLargePayloadTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] - public Task SubmitsTraces() => RunTest(); + public Task SubmitsTraces(bool dataPipelineEnabled) => RunTest(TestTransports.Tcp, dataPipelineEnabled); } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs index cb6a0a190549..b9eeb17da290 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs @@ -8,6 +8,7 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; using Xunit; using Xunit.Abstractions; @@ -30,8 +31,16 @@ public LargePayloadTestBase(ITestOutputHelper output) public int ExpectedSpans => TracesToTrigger + (TracesToTrigger * SpansPerTrace); - protected async Task RunTest() + protected async Task RunTest(TestTransports transport, bool dataPipelineEnabled) { + if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) + { + throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + } + + EnvironmentHelper.EnableTransport(transport); + EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + using (var agent = EnvironmentHelper.GetMockAgent()) { using (var sample = await RunSampleAndWaitForExit(agent, arguments: $" -t {TracesToTrigger} -s {SpansPerTrace} -f {FillerTagLength}")) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs index f7900a3e330f..46737b524b60 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs @@ -20,12 +20,13 @@ public UnixDomainSocketLargePayloadTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { - EnvironmentHelper.EnableUnixDomainSockets(); - await RunTest(); + await RunTest(TestTransports.Uds, dataPipelineEnabled); } } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs index 716c750bc934..1290b908fa2f 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs @@ -21,13 +21,14 @@ public WindowsNamedPipeLargePayloadTests(ITestOutputHelper output) /// /// To be enabled when Windows Named Pipes is available in the MockTracerAgent /// - [SkippableFact(Skip = "Windows named pipes are not yet supported in the MockTracerAgent")] + [SkippableTheory(Skip = "Windows named pipes are not yet supported in the MockTracerAgent")] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] [Trait("Category", "LinuxUnsupported")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { - EnvironmentHelper.EnableWindowsNamedPipes(); - await RunTest(); + await RunTest(TestTransports.WindowsNamedPipe, dataPipelineEnabled); } } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs index 6e3d6dd25619..05cdd4fa970b 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs @@ -110,7 +110,7 @@ public async Task Sampling(bool parentTrace) { var samplingPriority = string.Empty; - if (span.Metrics.ContainsKey(Metrics.SamplingPriority)) + if (span.Metrics != null && span.Metrics.ContainsKey(Metrics.SamplingPriority)) { samplingPriority = span.Metrics[Metrics.SamplingPriority].ToString(); } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs index 1b750ae26814..0aa0a49e0801 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs @@ -31,7 +31,15 @@ public async Task Http_Headers_Contain_ContainerId() using (var agent = MockTracerAgent.Create(_output, agentPort)) { - var settings = TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" } }); + var settings = TracerSettings.Create(new() + { + { + ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" + }, + { + ConfigurationKeys.TraceDataPipelineEnabled, "false" + } + }); var tracer = new Tracer(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null); using (var scope = tracer.StartActive("operationName")) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs new file mode 100644 index 000000000000..f26b3abdfe83 --- /dev/null +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -0,0 +1,161 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.AppSec.Rasp; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using Xunit; + +namespace Datadog.Trace.IntegrationTests.LibDatadog; + +public class TraceExporterTests +{ + [SkippableTheory] + [InlineData(TestTransports.Tcp)] + [InlineData(TestTransports.Uds)] + [InlineData(TestTransports.WindowsNamedPipe)] + public async Task SendsTracesUsingDataPipeline(TestTransports transport) + { + if (EnvironmentTools.IsWindows() && Environment.Is64BitProcess == false) + { + throw new SkipException("Can't use data pipeline on x86, there is no good way to copy native libraries for x86"); + } + + if (transport == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) + { + throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + } + + if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux()) + { + throw new SkipException("Can't use Unix Domain Sockets on non-Linux with data pipeline enabled"); + } + + var pipeName = $"trace-{Guid.NewGuid()}"; + var udsPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + using var agent = GetAgent(); + var settings = GetSettings(); + var tracerSettings = TracerSettings.Create(settings); + + agent.CustomResponses[MockTracerResponseType.Traces] = new MockTracerResponse + { + StatusCode = 200, + ContentType = "application/json", + Response = """ + { + "rate_by_service": { + "service:default-service,env:test": 1.0, + "service:,env:": 0.8 + } + } + """ + }; + + var discovery = DiscoveryService.Create(tracerSettings.Exporter); + var tracer = TracerHelper.Create(tracerSettings, discoveryService: discovery); + + using var span = tracer.StartSpan("operationName"); + span.ResourceName = "resourceName"; + span.Type = "test"; + + var testMetaStruct = new TestMetaStruct + { + Foo = "foo", + Bar = 1, + }; + + var metaStructBytes = MetaStructHelper.ObjectToByteArray(testMetaStruct.ToDictionary()); + span.SetMetaStruct("test-meta-struct", metaStructBytes); + span.Finish(); + + await tracer.TracerManager.ShutdownAsync(); + var recordedSpans = agent.WaitForSpans(1); + Assert.Equal(1, recordedSpans.Count); + + var recordedSpan = recordedSpans[0]; + Assert.Equal("operationName", recordedSpan.Name); + Assert.Equal("resourceName", recordedSpan.Resource); + Assert.Equal("default-service", recordedSpan.Service); + + Assert.Single(recordedSpan.MetaStruct); + var recordedMetaStructBytes = recordedSpan.MetaStruct["test-meta-struct"]; + Assert.Equal(metaStructBytes, recordedMetaStructBytes); + + Dictionary GetSettings() + { + var settingsMap = new Dictionary + { + { ConfigurationKeys.StatsComputationEnabled, true }, + { ConfigurationKeys.ServiceName, "default-service" }, + { ConfigurationKeys.ServiceVersion, "v1" }, + { ConfigurationKeys.Environment, "test" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "true" }, + }; + + switch (transport) + { + case TestTransports.Tcp: + if (agent is MockTracerAgent.TcpUdpAgent tcpAgent) + { + settingsMap[ConfigurationKeys.AgentPort] = tcpAgent.Port; + } + else + { + throw new InvalidOperationException("Unsupported agent type " + agent.GetType()); + } + + break; + case TestTransports.WindowsNamedPipe: + settingsMap[ConfigurationKeys.TracesPipeName] = pipeName; + break; + case TestTransports.Uds: + settingsMap[ConfigurationKeys.TracesUnixDomainSocketPath] = udsPath; + break; + default: + throw new InvalidOperationException("Unsupported transport type " + transport); + } + + return settingsMap; + } + + MockTracerAgent GetAgent() + => transport switch + { + TestTransports.Tcp => MockTracerAgent.Create(null), + TestTransports.WindowsNamedPipe => MockTracerAgent.Create(null, new WindowsPipesConfig(pipeName, null)), +#if NETCOREAPP3_1_OR_GREATER + TestTransports.Uds + => MockTracerAgent.Create(null, new UnixDomainSocketConfig(udsPath, null)), +#endif + _ => throw new InvalidOperationException("Unsupported transport type " + transport), + }; + } + + internal class TestMetaStruct + { + public string Foo { get; set; } + + public int Bar { get; set; } + + public Dictionary ToDictionary() + { + return new Dictionary + { + { + "foo", Foo + }, + { + "bar", Bar + } + }; + } + } +} diff --git a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs index 9f80ad3aa5d0..2dce5a8b4e86 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs @@ -54,6 +54,7 @@ public async Task SendsStatsWithProcessing_Normalizer() { ConfigurationKeys.ServiceVersion, "v1" }, { ConfigurationKeys.Environment, "test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); var discovery = DiscoveryService.Create(settings.Exporter); @@ -198,6 +199,7 @@ public async Task SendsStatsWithProcessing_Obfuscator() { ConfigurationKeys.ServiceVersion, "v1" }, { ConfigurationKeys.Environment, "test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); var discovery = DiscoveryService.Create(settings.Exporter); @@ -356,6 +358,7 @@ private async Task SendStatsHelper(bool statsComputationEnabled, bool expectStat { ConfigurationKeys.ServiceVersion, "V" }, { ConfigurationKeys.Environment, "Test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, })); var discovery = DiscoveryService.Create(settings.Exporter); diff --git a/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs b/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs index 5d06f964fe16..fef7363ceaf2 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs @@ -6,6 +6,7 @@ using System; using System.Threading.Tasks; using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.Configuration; using Datadog.Trace.Sampling; using Datadog.Trace.Vendors.StatsdClient; @@ -23,9 +24,10 @@ public static ScopedTracer Create( IAgentWriter agentWriter = null, ITraceSampler sampler = null, IScopeManager scopeManager = null, - IDogStatsd statsd = null) + IDogStatsd statsd = null, + IDiscoveryService discoveryService = null) { - return new ScopedTracer(settings, agentWriter, sampler, scopeManager, statsd); + return new ScopedTracer(settings, agentWriter, sampler, scopeManager, statsd, discoveryService); } /// @@ -39,8 +41,14 @@ public static ScopedTracer CreateWithFakeAgent( public class ScopedTracer : Tracer, IAsyncDisposable { - public ScopedTracer(TracerSettings settings = null, IAgentWriter agentWriter = null, ITraceSampler sampler = null, IScopeManager scopeManager = null, IDogStatsd statsd = null) - : base(settings, agentWriter, sampler, scopeManager, statsd) + public ScopedTracer( + TracerSettings settings = null, + IAgentWriter agentWriter = null, + ITraceSampler sampler = null, + IScopeManager scopeManager = null, + IDogStatsd statsd = null, + IDiscoveryService discoveryService = null) + : base(settings, agentWriter, sampler, scopeManager, statsd, discoveryService: discoveryService) { } diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs index 7f9c04c69b4a..f4b404136434 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs @@ -237,6 +237,7 @@ private static async Task> SendSpan(bool tracerMetricsE { ConfigurationKeys.AgentUri, $"http://127.0.0.1:{agent.Port}" }, { ConfigurationKeys.TracerMetricsEnabled, tracerMetricsEnabled }, { ConfigurationKeys.StartupDiagnosticLogEnabled, false }, + { ConfigurationKeys.TraceDataPipelineEnabled, false }, }); await using var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json index ffefea3dd4ca..560c4c54d45f 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json @@ -671,5 +671,6 @@ "DD_TRACE_BAGGAGE_MAX_BYTES": "trace_baggage_max_bytes", "DD_TRACE_BYPASS_HTTP_REQUEST_URL_CACHING_ENABLED": "trace_bypass_http_request_url_caching_enabled", "DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED": "inferred_proxy_services_enabled", + "DD_TRACE_DATA_PIPELINE_ENABLED": "trace_data_pipeline_enabled", "DD_TRACE_INJECT_CONTEXT_INTO_STORED_PROCEDURES_ENABLED": "trace_inject_context_into_stored_procedures_enabled" } From a8c7904176e670dd72a3756542871ec3b336f69a Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Wed, 28 May 2025 17:24:00 +0200 Subject: [PATCH 02/23] fix: enable debug tel and cleanup --- .../src/Datadog.Trace/LibDatadog/ByteSlice.cs | 2 +- .../src/Datadog.Trace/LibDatadog/CharSlice.cs | 9 +++- .../TelemetryClientConfiguration.cs | 6 +-- .../Datadog.Trace/LibDatadog/TraceExporter.cs | 4 +- .../LibDatadog/TraceExporterConfiguration.cs | 43 ++++++++++++------- .../src/Datadog.Trace/TracerManagerFactory.cs | 5 ++- 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs index ab47011133a5..2f1e7f011990 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs @@ -24,5 +24,5 @@ internal struct ByteSlice /// /// Length of the slice. /// - internal UIntPtr Len; + internal nuint Len; } diff --git a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs index 7ac2a79c46cc..9f02598708d7 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs @@ -14,7 +14,7 @@ namespace Datadog.Trace.LibDatadog; /// Represents a slice of a UTF-8 encoded string in memory. /// [StructLayout(LayoutKind.Sequential)] -internal struct CharSlice +internal struct CharSlice : IDisposable { /// /// Pointer to the start of the slice. @@ -24,7 +24,7 @@ internal struct CharSlice /// /// Length of the slice. /// - internal UIntPtr Len; + internal nuint Len; /// /// Initializes a new instance of the struct. @@ -47,4 +47,9 @@ internal CharSlice(string? str) Len = (UIntPtr)bytes.Length; } } + + public void Dispose() + { + Marshal.FreeHGlobal(Ptr); + } } diff --git a/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs index e9521ea3bcb5..e9058cfd7b97 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs @@ -5,6 +5,7 @@ #nullable enable +using System; using System.Runtime.InteropServices; namespace Datadog.Trace.LibDatadog; @@ -29,12 +30,9 @@ internal struct TelemetryClientConfiguration /// public CharSlice RuntimeId; - /* - /// TODO: enable with v18 /// /// Whether to enable debug mode for telemetry. When enabled, sets the dd-telemetry-debug-enabled header to true. /// Defaults to false. /// - /// public bool DebugEnabled; - */ + public bool DebugEnabled; } diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs index cd9947a20391..b392701deeb6 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs @@ -15,7 +15,6 @@ namespace Datadog.Trace.LibDatadog; internal class TraceExporter : SafeHandle, IApi { - private readonly TraceExporterConfiguration _configuration; private readonly IDatadogLogger _log; public TraceExporter( @@ -24,10 +23,9 @@ public TraceExporter( : base(IntPtr.Zero, true) { _log = log ?? DatadogLogging.GetLoggerFor(); - _configuration = configuration; _log.Debug("Creating new TraceExporter"); - using var errPtr = NativeInterop.Exporter.New(out var ptr, _configuration); + using var errPtr = NativeInterop.Exporter.New(out var ptr, configuration); errPtr.ThrowIfError(); SetHandle(ptr); } diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs index 028bf1689e45..cc12e9e33ba3 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs @@ -15,7 +15,7 @@ namespace Datadog.Trace.LibDatadog; /// internal class TraceExporterConfiguration : SafeHandle { - private readonly IntPtr _telemetryClientConfigurationHandle; + private IntPtr _telemetryConfigPtr; public TraceExporterConfiguration() : base(IntPtr.Zero, true) @@ -30,7 +30,8 @@ public string Url { init { - using var error = NativeInterop.Config.SetUrl(this, new CharSlice(value)); + using var url = new CharSlice(value); + using var error = NativeInterop.Config.SetUrl(this, url); error.ThrowIfError(); } } @@ -39,7 +40,8 @@ public string TraceVersion { init { - using var error = NativeInterop.Config.SetTracerVersion(this, new CharSlice(value)); + using var tracerVersion = new CharSlice(value); + using var error = NativeInterop.Config.SetTracerVersion(this, tracerVersion); error.ThrowIfError(); } } @@ -48,7 +50,8 @@ public string Language { init { - using var error = NativeInterop.Config.SetLanguage(this, new CharSlice(value)); + using var language = new CharSlice(value); + using var error = NativeInterop.Config.SetLanguage(this, language); error.ThrowIfError(); } } @@ -57,7 +60,8 @@ public string LanguageVersion { init { - using var error = NativeInterop.Config.SetLanguageVersion(this, new CharSlice(value)); + using var languageVersion = new CharSlice(value); + using var error = NativeInterop.Config.SetLanguageVersion(this, languageVersion); error.ThrowIfError(); } } @@ -66,7 +70,8 @@ public string LanguageInterpreter { init { - using var error = NativeInterop.Config.SetInterpreter(this, new CharSlice(value)); + using var interpreter = new CharSlice(value); + using var error = NativeInterop.Config.SetInterpreter(this, interpreter); error.ThrowIfError(); } } @@ -75,7 +80,8 @@ public string? Hostname { init { - using var error = NativeInterop.Config.SetHostname(this, new CharSlice(value)); + using var hostname = new CharSlice(value); + using var error = NativeInterop.Config.SetHostname(this, hostname); error.ThrowIfError(); } } @@ -84,7 +90,8 @@ public string? Env { init { - using var error = NativeInterop.Config.SetEnv(this, new CharSlice(value)); + using var env = new CharSlice(value); + using var error = NativeInterop.Config.SetEnv(this, env); error.ThrowIfError(); } } @@ -93,7 +100,8 @@ public string? Version { init { - using var error = NativeInterop.Config.SetVersion(this, new CharSlice(value)); + using var version = new CharSlice(value); + using var error = NativeInterop.Config.SetVersion(this, version); error.ThrowIfError(); } } @@ -102,7 +110,8 @@ public string? Service { init { - using var error = NativeInterop.Config.SetService(this, new CharSlice(value)); + using var service = new CharSlice(value); + using var error = NativeInterop.Config.SetService(this, service); error.ThrowIfError(); } } @@ -113,10 +122,9 @@ public TelemetryClientConfiguration? TelemetryClientConfiguration { if (value.HasValue) { - _telemetryClientConfigurationHandle = Marshal.AllocHGlobal(Marshal.SizeOf(value.Value)); - Marshal.StructureToPtr(value.Value, _telemetryClientConfigurationHandle, true); - - using var error = NativeInterop.Config.EnableTelemetry(this, _telemetryClientConfigurationHandle); + _telemetryConfigPtr = Marshal.AllocHGlobal(Marshal.SizeOf(value.Value)); + Marshal.StructureToPtr(value.Value, _telemetryConfigPtr, false); + using var error = NativeInterop.Config.EnableTelemetry(this, _telemetryConfigPtr); error.ThrowIfError(); } } @@ -142,8 +150,13 @@ public bool ClientComputedStats protected override bool ReleaseHandle() { + if (_telemetryConfigPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(_telemetryConfigPtr); + _telemetryConfigPtr = IntPtr.Zero; + } + NativeInterop.Config.Free(handle); - Marshal.FreeHGlobal(_telemetryClientConfigurationHandle); return true; } } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 8696bf79b770..e1e64cc9bbb7 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -373,7 +373,8 @@ private IApi GetApi(TracerSettings settings, IDogStatsd statsd, Action Date: Fri, 30 May 2025 17:02:30 +0200 Subject: [PATCH 03/23] feat(trace-exporter): handle exceptions when creating Trace Exporter in TracerManager (#6925) --- .../Datadog.Trace/LibDatadog/TraceExporter.cs | 20 ++++-- .../src/Datadog.Trace/TracerManagerFactory.cs | 71 ++++++++++--------- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs index b392701deeb6..e42b5e9a1365 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs @@ -6,6 +6,7 @@ #nullable enable using System; +using System.Globalization; using System.Runtime.InteropServices; using System.Threading.Tasks; using Datadog.Trace.Agent; @@ -47,12 +48,21 @@ public Task SendTracesAsync(ArraySegment traces, int numberOfTraces, }; var responsePtr = IntPtr.Zero; - using var error = NativeInterop.Exporter.Send(this, tracesSlice, (UIntPtr)numberOfTraces, ref responsePtr); - if (!error.IsInvalid) + try { - var ex = error.ToException(); - _log.Error(ex, "An error occurred while sending data to the agent. Error Code: {ErrorCode}, message: {Message}", ex.ErrorCode, ex.Message); - throw ex; + using var error = NativeInterop.Exporter.Send(this, tracesSlice, (UIntPtr)numberOfTraces, ref responsePtr); + if (!error.IsInvalid) + { + var ex = error.ToException(); +#pragma warning disable DDLOG004 + _log.Error(ex, "An error occurred while sending data to the agent. Error Code: " + ex.ErrorCode + ", message: {Message}", ex.Message); +#pragma warning restore DDLOG004 + throw ex; + } + } + catch (Exception ex) when (ex is not TraceExporterException) + { + _log.Error(ex, "An error occurred while sending data to the agent."); } } } diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index e1e64cc9bbb7..af36880c0660 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -363,43 +363,50 @@ private IApi GetApi(TracerSettings settings, IDogStatsd statsd, Action Date: Tue, 3 Jun 2025 11:05:33 +0200 Subject: [PATCH 04/23] fix: update skip exception messages for transport compatibility --- .../AgentMalfunctionTests.cs | 6 +++--- .../RuntimeMetricsTests.cs | 2 +- .../TelemetryTests.cs | 2 +- .../TransportTests.cs | 4 ++-- .../Transports/LargePayload/LargePayloadTestBase.cs | 2 +- .../Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs | 2 +- .../LibDatadog/TraceExporterTests.cs | 6 +++--- .../Logging/TracerFlare/TracerFlareApiTests.cs | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs index 689fdb72def4..87a50c3c7e37 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs @@ -57,15 +57,15 @@ public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transpo SkipOn.Platform(SkipOn.PlatformValue.MacOs); if (transportType == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } if (transportType == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) { - throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported } - EnvironmentHelper.EnableTransport(transportType); + EnvironmentHelper.EnableTransport(transportType)only supported EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); using var agent = EnvironmentHelper.GetMockAgent(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs index dadb7865afa3..e60b1507fa53 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs @@ -74,7 +74,7 @@ public async Task NamedPipesSubmitsMetrics() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } EnvironmentHelper.EnableWindowsNamedPipes(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs index 998a7675bdf9..4fa6f6366e28 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs @@ -144,7 +144,7 @@ public async Task WhenUsingNamedPipesAgent_UsesNamedPipesTelemetry(bool? enableD { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } EnvironmentHelper.EnableWindowsNamedPipes(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs index 68d3b815cf36..979f5bae2bc6 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs @@ -92,12 +92,12 @@ private async Task RunTest(TracesTransportType transportType, bool dataPipelineE if (transportType == TracesTransportType.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } if (transportType == TracesTransportType.UnixDomainSocket && !EnvironmentTools.IsLinux() && dataPipelineEnabled) { - throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); } EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs index b9eeb17da290..221d0dac7b16 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs @@ -35,7 +35,7 @@ protected async Task RunTest(TestTransports transport, bool dataPipelineEnabled) { if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) { - throw new SkipException("Can't use UnixDomainSocket on non-Linux with data pipeline enabled"); + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); } EnvironmentHelper.EnableTransport(transport); diff --git a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs index 104e67bf9aad..a76189d81209 100644 --- a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs +++ b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs @@ -412,7 +412,7 @@ public async Task MethodProbeTest_NamedPipes() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } var testType = DebuggerTestHelper.FirstSupportedProbeTestType(EnvironmentHelper.GetTargetFramework()); diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index f26b3abdfe83..eba54484c561 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -26,17 +26,17 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) { if (EnvironmentTools.IsWindows() && Environment.Is64BitProcess == false) { - throw new SkipException("Can't use data pipeline on x86, there is no good way to copy native libraries for x86"); + throw new SkipException("Can't use data pipeline on win-x86, there is no good way to copy native libraries for win-x86"); } if (transport == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux()) { - throw new SkipException("Can't use Unix Domain Sockets on non-Linux with data pipeline enabled"); + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); } var pipeName = $"trace-{Guid.NewGuid()}"; diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs index 94042b97c842..4553fc8b1bc9 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs @@ -60,7 +60,7 @@ public async Task CanSendToAgent_NamedPipes() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } // named pipes is notoriously flaky From 3de4ab7a82d4b582cf54cf67000a30d67199b85c Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:05:53 +0200 Subject: [PATCH 05/23] test: enhance assertions in TraceExporterTests for clarity --- .../LibDatadog/TraceExporterTests.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index eba54484c561..ae93efc6985f 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -12,6 +12,7 @@ using Datadog.Trace.AppSec.Rasp; using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; +using FluentAssertions; using Xunit; namespace Datadog.Trace.IntegrationTests.LibDatadog; @@ -78,16 +79,16 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) await tracer.TracerManager.ShutdownAsync(); var recordedSpans = agent.WaitForSpans(1); - Assert.Equal(1, recordedSpans.Count); + recordedSpans.Should().ContainSingle(); var recordedSpan = recordedSpans[0]; - Assert.Equal("operationName", recordedSpan.Name); - Assert.Equal("resourceName", recordedSpan.Resource); - Assert.Equal("default-service", recordedSpan.Service); + recordedSpan.Name.Should().Be("operationName"); + recordedSpan.Resource.Should().Be("resourceName"); + recordedSpan.Service.Should().Be("default-service"); - Assert.Single(recordedSpan.MetaStruct); - var recordedMetaStructBytes = recordedSpan.MetaStruct["test-meta-struct"]; - Assert.Equal(metaStructBytes, recordedMetaStructBytes); + recordedSpan.MetaStruct.Should().ContainSingle(); + var recordedMetaStructBytes = recordedSpan.MetaStruct["test-meta-struct"]; + recordedMetaStructBytes.Should().BeEquivalentTo(metaStructBytes); Dictionary GetSettings() { From b09362d2c4d5034b9427b0d9ede9c6099f2d3803 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:06:22 +0200 Subject: [PATCH 06/23] cleanup --- .../AWS/AwsLambdaTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index c494f7f1d196..39e92c01841f 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -93,11 +93,6 @@ public async Task SubmitsTraces(bool dataPipelineEnabled) settings.AddRegexScrubber(StackRegex, "$1 Cannot assign requested address (SocketException)$2"); settings.AddRegexScrubber(ErrorMsgRegex, "$1 Cannot assign requested address$2"); - foreach (var span in allSpans) - { - Output.WriteLine(span.ToString()); - } - await VerifyHelper.VerifySpans(allSpans, settings) .DisableRequireUniquePrefix() .UseFileName(nameof(AwsLambdaTests)); From b493a05ee07365f98d710f23201f7176dd9c7143 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:11:01 +0200 Subject: [PATCH 07/23] fix: replace IntPtr with nint for pointer representation in ByteSlice and CharSlice --- tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs | 2 +- tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs index 2f1e7f011990..9d22aebb67fc 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs @@ -19,7 +19,7 @@ internal struct ByteSlice /// /// Pointer to the start of the slice. /// - internal IntPtr Ptr; + internal nint Ptr; /// /// Length of the slice. diff --git a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs index 9f02598708d7..728554daedfa 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs @@ -19,7 +19,7 @@ internal struct CharSlice : IDisposable /// /// Pointer to the start of the slice. /// - internal IntPtr Ptr; + internal nint Ptr; /// /// Length of the slice. From c8bbdf5cc18d3ff57e892bc49c52018df263f0f4 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:12:52 +0200 Subject: [PATCH 08/23] fix: update CharSlice length type from UIntPtr to nuint for consistency --- tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs index 728554daedfa..00622a8bbb5a 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs @@ -44,7 +44,7 @@ internal CharSlice(string? str) var bytes = System.Text.Encoding.UTF8.GetBytes(str); Ptr = Marshal.AllocHGlobal(bytes.Length); Marshal.Copy(bytes, 0, Ptr, bytes.Length); - Len = (UIntPtr)bytes.Length; + Len = (nuint)bytes.Length; } } From ede3d4a5855ace96e13343558729a5f19616635e Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:16:07 +0200 Subject: [PATCH 09/23] fix: add comment to clarify exclusion of libdatadog in debug info extraction --- tracer/build/_build/Build.Steps.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index db2ca24dedb5..922b96ff3d66 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -974,6 +974,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo .Executes(() => { // extract debug info from everything in monitoring home and copy it to the linux symbols directory + // except libdatadog since debug symbols are already stripped as part of libdatadog release. var files = MonitoringHomeDirectory.GlobFiles("linux-*/*.so") .Where(file => !Path.GetFileName(file).Contains("libdatadog_profiling.so")); From 635949c1104d9626fd716b756d140523d9e70b5c Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:20:04 +0200 Subject: [PATCH 10/23] fix: remove parameter from GetWinArchitectureAndExtension for clarity --- tracer/build/_build/Build.Steps.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 922b96ff3d66..b289b5e29e27 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -659,7 +659,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { foreach (var fmk in frameworks) { - var (arch, ext) = GetWinArchitectureAndExtension(fmk); + var (arch, ext) = GetWinArchitectureAndExtension(); var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; var dest = testBinFolder / fmk / "LibDatadog.dll"; CopyFile(source, dest, FileExistsPolicy.Overwrite); @@ -2746,7 +2746,7 @@ static LogLevel ParseNativeLogLevel(string value) (false, true) => ($"linux-musl-{UnixArchitectureIdentifier}", "so"), }; - private (string Arch, string Ext) GetWinArchitectureAndExtension(string fmk) + private (string Arch, string Ext) GetWinArchitectureAndExtension() { return RuntimeInformation.ProcessArchitecture switch { From b38f13b7e38673d3b446290b97531ac2edc328d5 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:30:44 +0200 Subject: [PATCH 11/23] fix: rename loop variable from 'fmk' to 'framework' for clarity --- tracer/build/_build/Build.Steps.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index b289b5e29e27..42f00ffef117 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -657,11 +657,11 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo if (IsWin) { - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { var (arch, ext) = GetWinArchitectureAndExtension(); var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; - var dest = testBinFolder / fmk / "LibDatadog.dll"; + var dest = testBinFolder / framework / "LibDatadog.dll"; CopyFile(source, dest, FileExistsPolicy.Overwrite); } } @@ -669,9 +669,9 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { var (arch, ext) = GetUnixArchitectureAndExtension(); var source = MonitoringHomeDirectory / arch / $"libdatadog_profiling.{ext}"; - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { - var dest = testBinFolder / fmk / $"LibDatadog.{ext}"; + var dest = testBinFolder / framework / $"LibDatadog.{ext}"; CopyFile(source, dest, FileExistsPolicy.Overwrite); } } @@ -702,9 +702,9 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { var oldVersionPath = oldVersionTempPath / "runtimes" / arch / "native" / "ddwaf.dll"; var source = MonitoringHomeDirectory / arch; - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { - var dest = testBinFolder / fmk / arch; + var dest = testBinFolder / framework / arch; CopyDirectoryRecursively(source, dest, DirectoryExistsPolicy.Merge, FileExistsPolicy.Overwrite); CopyFile(oldVersionPath, dest / $"ddwaf-{olderLibDdwafVersion}.dll", FileExistsPolicy.Overwrite); } @@ -723,7 +723,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo var oldVersionPath = oldVersionTempPath / "runtimes" / patchedArchWaf / "native" / $"libddwaf.{ext}"; await DownloadWafVersion(olderLibDdwafVersion, oldVersionTempPath); { - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { // We have to copy into the _root_ test bin folder here, not the arch sub-folder. // This is because these tests try to load the WAF. @@ -732,7 +732,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo // - The native tracer must be side-by-side with the running dll // As this is a managed-only unit test, the native tracer _must_ be in the root folder // For simplicity, we just copy all the native dlls there - var dest = testBinFolder / fmk; + var dest = testBinFolder / framework; // use the files from the monitoring native folder CopyDirectoryRecursively(MonitoringHomeDirectory / (IsOsx ? "osx" : arch), dest, DirectoryExistsPolicy.Merge, FileExistsPolicy.Overwrite); From db88db4d6d8cd334a0fee5a8784ee4377886e5d8 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:30:53 +0200 Subject: [PATCH 12/23] fix: add warning log for conflicting configuration settings in tracer --- tracer/src/Datadog.Trace/Configuration/TracerSettings.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index ff56ee9c3c7d..990ad64c75f3 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -422,6 +422,11 @@ _ when x.ToBoolean() is { } boolean => boolean, if (DataPipelineEnabled && StatsComputationEnabled) { DataPipelineEnabled = false; + Log.Warning( + "{ConfigurationKey} is enabled, but {ConfigurationKeys.StatsComputationEnabled} is enabled. Disabling {ConfigurationKeys.TraceDataPipelineEnabled}.", + ConfigurationKeys.TraceDataPipelineEnabled, + ConfigurationKeys.StatsComputationEnabled, + ConfigurationKeys.TraceDataPipelineEnabled); _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); } From f8f3a32392bab6c4656edb8e768f8f35b3631950 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 11:46:42 +0200 Subject: [PATCH 13/23] fix: format --- tracer/src/Datadog.Trace/Configuration/TracerSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 990ad64c75f3..5763f73494a7 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -423,7 +423,7 @@ _ when x.ToBoolean() is { } boolean => boolean, { DataPipelineEnabled = false; Log.Warning( - "{ConfigurationKey} is enabled, but {ConfigurationKeys.StatsComputationEnabled} is enabled. Disabling {ConfigurationKeys.TraceDataPipelineEnabled}.", + "{ConfigurationKey} is enabled, but {StatsComputationEnabled} is enabled. Disabling {TraceDataPipelineEnabled}.", ConfigurationKeys.TraceDataPipelineEnabled, ConfigurationKeys.StatsComputationEnabled, ConfigurationKeys.TraceDataPipelineEnabled); From 7ad96e9837465772b500349490676b4f46acec94 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 12:50:21 +0200 Subject: [PATCH 14/23] fix: CopyNativeFilesForTests for osx --- tracer/build/_build/Build.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index 80601a17cc8d..3529f38251f3 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -93,7 +93,7 @@ partial class Build : NukeBuild [Parameter("Enable or Disable fast developer loop")] readonly bool FastDevLoop; - + [Parameter("The directory containing the tool .nupkg file")] readonly AbsolutePath ToolSource; @@ -108,11 +108,11 @@ partial class Build : NukeBuild [Parameter("Should we build native binaries as Universal. Default to false, so we can still build native libs outside of docker.")] readonly bool AsUniversal = false; - + [Parameter("RuntimeIdentifier sets the target platform for ReadyToRun assemblies in 'PublishManagedTracerR2R'." + "See https://learn.microsoft.com/en-us/dotnet/core/rid-catalog")] string RuntimeIdentifier { get; } - + public Build() { RuntimeIdentifier = GetDefaultRuntimeIdentifier(IsAlpine); @@ -218,7 +218,7 @@ void DeleteReparsePoints(string path) .DependsOn(CreateMissingNullabilityFile) .DependsOn(CreateTrimmingFile) .DependsOn(RegenerateSolutions); - + Target BuildManagedTracerHomeR2R => _ => _ .Unlisted() .Description("Builds the native and managed src, and publishes the tracer home directory") @@ -351,6 +351,7 @@ void DeleteReparsePoints(string path) .DependsOn(CompileManagedTestHelpers) .DependsOn(CompileLinuxOrOsxIntegrationTests) .DependsOn(BuildRunnerTool) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CopyServerlessArtifacts); Target BuildAndRunOsxIntegrationTests => _ => _ From fde7edb7abc4c872a4950ad45c3e3fd07184784b Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 13:46:58 +0200 Subject: [PATCH 15/23] fix: update transport checks for Unix Domain Sockets support --- .../Configuration/TracerSettings.cs | 36 +++++++++++++------ .../AgentMalfunctionTests.cs | 7 +--- .../TransportTests.cs | 5 --- .../LargePayload/LargePayloadTestBase.cs | 5 --- .../LibDatadog/TraceExporterTests.cs | 4 +-- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index 5763f73494a7..3befa6b9d0da 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -13,6 +13,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using Datadog.Trace.Agent; using Datadog.Trace.Ci; using Datadog.Trace.Ci.CiEnvironment; using Datadog.Trace.ClrProfiler; @@ -417,17 +418,32 @@ _ when x.ToBoolean() is { } boolean => boolean, .WithKeys(ConfigurationKeys.TraceDataPipelineEnabled) .AsBool(defaultValue: true); - // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter - // as it may lead to different stats results than the managed one. - if (DataPipelineEnabled && StatsComputationEnabled) + if (DataPipelineEnabled) { - DataPipelineEnabled = false; - Log.Warning( - "{ConfigurationKey} is enabled, but {StatsComputationEnabled} is enabled. Disabling {TraceDataPipelineEnabled}.", - ConfigurationKeys.TraceDataPipelineEnabled, - ConfigurationKeys.StatsComputationEnabled, - ConfigurationKeys.TraceDataPipelineEnabled); - _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter + // as it may lead to different stats results than the managed one. + if (StatsComputationEnabled) + { + DataPipelineEnabled = false; + Log.Warning( + "{ConfigurationKey} is enabled, but {StatsComputationEnabled} is enabled. Disabling {TraceDataPipelineEnabled}.", + ConfigurationKeys.TraceDataPipelineEnabled, + ConfigurationKeys.StatsComputationEnabled, + ConfigurationKeys.TraceDataPipelineEnabled); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + + // Windows supports UnixDomainSocket https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ + // but tokio hasn't added support for it yet https://github.com/tokio-rs/tokio/issues/2201 + if (Exporter.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) + { + DataPipelineEnabled = false; + Log.Warning( + "{ConfigurationKey} is enabled, but TracesTransport is set to UnixDomainSocket which is not supported on Windows. Disabling {TraceDataPipelineEnabled}.", + ConfigurationKeys.TraceDataPipelineEnabled, + ConfigurationKeys.TraceDataPipelineEnabled); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } } // We should also be writing telemetry for OTEL_LOGS_EXPORTER similar to OTEL_METRICS_EXPORTER, but we don't have a corresponding Datadog config diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs index 87a50c3c7e37..0232159ee45c 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs @@ -60,12 +60,7 @@ public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transpo throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } - if (transportType == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) - { - throw new SkipException("Unix Domain Sockets (UDS) transport is only supported - } - - EnvironmentHelper.EnableTransport(transportType)only supported + EnvironmentHelper.EnableTransport(transportType); EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); using var agent = EnvironmentHelper.GetMockAgent(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs index 979f5bae2bc6..7d70354790ea 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs @@ -95,11 +95,6 @@ private async Task RunTest(TracesTransportType transportType, bool dataPipelineE throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } - if (transportType == TracesTransportType.UnixDomainSocket && !EnvironmentTools.IsLinux() && dataPipelineEnabled) - { - throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); - } - EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); EnvironmentHelper.EnableTransport(GetTransport(transportType)); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs index 221d0dac7b16..911564590d4a 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs @@ -33,11 +33,6 @@ public LargePayloadTestBase(ITestOutputHelper output) protected async Task RunTest(TestTransports transport, bool dataPipelineEnabled) { - if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux() && dataPipelineEnabled) - { - throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); - } - EnvironmentHelper.EnableTransport(transport); EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index ae93efc6985f..e99d9d9fc385 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -35,9 +35,9 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } - if (transport == TestTransports.Uds && !EnvironmentTools.IsLinux()) + if (transport == TestTransports.Uds && EnvironmentTools.IsWindows()) { - throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux when data pipeline is enabled"); + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux and OSX when data pipeline is enabled"); } var pipeName = $"trace-{Guid.NewGuid()}"; From 2da2b69cea4a2972deee4195fd6c25aae2b183c4 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 13:50:25 +0200 Subject: [PATCH 16/23] cleanup --- .../VersionConflict/AspNetVersionConflictTests.cs | 2 +- .../ContainerTaggingTests.cs | 8 ++------ .../LibDatadog/TraceExporterTests.cs | 4 ++-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs index 05cdd4fa970b..7041bbdef86d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs @@ -110,7 +110,7 @@ public async Task Sampling(bool parentTrace) { var samplingPriority = string.Empty; - if (span.Metrics != null && span.Metrics.ContainsKey(Metrics.SamplingPriority)) + if (span.Metrics?.ContainsKey(Metrics.SamplingPriority) == true) { samplingPriority = span.Metrics[Metrics.SamplingPriority].ToString(); } diff --git a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs index 0aa0a49e0801..f21b393cce33 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs @@ -33,12 +33,8 @@ public async Task Http_Headers_Contain_ContainerId() { var settings = TracerSettings.Create(new() { - { - ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" - }, - { - ConfigurationKeys.TraceDataPipelineEnabled, "false" - } + { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" } }); var tracer = new Tracer(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null); diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index e99d9d9fc385..3b3b140dee8f 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -61,7 +61,7 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) }; var discovery = DiscoveryService.Create(tracerSettings.Exporter); - var tracer = TracerHelper.Create(tracerSettings, discoveryService: discovery); + await using var tracer = TracerHelper.Create(tracerSettings, discoveryService: discovery); using var span = tracer.StartSpan("operationName"); span.ResourceName = "resourceName"; @@ -81,7 +81,7 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) var recordedSpans = agent.WaitForSpans(1); recordedSpans.Should().ContainSingle(); - var recordedSpan = recordedSpans[0]; + var recordedSpan = recordedSpans.Should().ContainSingle().Subject; recordedSpan.Name.Should().Be("operationName"); recordedSpan.Resource.Should().Be("resourceName"); recordedSpan.Service.Should().Be("default-service"); From fe2d531edaf16b13340196ae34074d6615b8ca59 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 13:58:35 +0200 Subject: [PATCH 17/23] fix: refactor span handling in TraceExporterTests --- .../LibDatadog/TraceExporterTests.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index 3b3b140dee8f..4d2d850718ff 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -63,19 +63,18 @@ public async Task SendsTracesUsingDataPipeline(TestTransports transport) var discovery = DiscoveryService.Create(tracerSettings.Exporter); await using var tracer = TracerHelper.Create(tracerSettings, discoveryService: discovery); - using var span = tracer.StartSpan("operationName"); - span.ResourceName = "resourceName"; - span.Type = "test"; - var testMetaStruct = new TestMetaStruct { Foo = "foo", Bar = 1, }; - var metaStructBytes = MetaStructHelper.ObjectToByteArray(testMetaStruct.ToDictionary()); - span.SetMetaStruct("test-meta-struct", metaStructBytes); - span.Finish(); + using (var span = tracer.StartSpan("operationName")) + { + span.ResourceName = "resourceName"; + span.Type = "test"; + span.SetMetaStruct("test-meta-struct", metaStructBytes); + } await tracer.TracerManager.ShutdownAsync(); var recordedSpans = agent.WaitForSpans(1); From 22c06df610b80fd7ce149347c963184e3e3921d9 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 14:05:40 +0200 Subject: [PATCH 18/23] fix: update Windows architecture handling in Build steps --- tracer/build/_build/Build.Steps.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 42f00ffef117..630fdec280d1 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -659,8 +659,8 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { foreach (var framework in frameworks) { - var (arch, ext) = GetWinArchitectureAndExtension(); - var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; + var arch = GetWinArchitecture(); + var source = MonitoringHomeDirectory / $"win-{arch}" / "datadog_profiling_ffi.dll"; var dest = testBinFolder / framework / "LibDatadog.dll"; CopyFile(source, dest, FileExistsPolicy.Overwrite); } @@ -2746,14 +2746,14 @@ static LogLevel ParseNativeLogLevel(string value) (false, true) => ($"linux-musl-{UnixArchitectureIdentifier}", "so"), }; - private (string Arch, string Ext) GetWinArchitectureAndExtension() + private string GetWinArchitecture() { return RuntimeInformation.ProcessArchitecture switch { - Architecture.X86 => ("win-x86", "dll"), - Architecture.X64 => ("win-x64", "dll"), - Architecture.Arm64 => ("win-arm64", "dll"), - _ => ("win-x86", "dll") // Default to x86 for unknown architectures + Architecture.X86 => "x86", + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + _ => throw new NotSupportedException($"Unsupported architecture: {RuntimeInformation.ProcessArchitecture}") }; } From d83d22d1c394861e94ad29acb12ab033bad4a9a0 Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 14:13:58 +0200 Subject: [PATCH 19/23] cleanup --- .../AWS/AwsLambdaTests.cs | 2 +- .../AgentMalfunctionTests.cs | 2 +- .../TransportTests.cs | 2 +- .../Transports/LargePayload/LargePayloadTestBase.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 39e92c01841f..b29b35f2c095 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -46,7 +46,7 @@ public async Task SubmitsTraces(bool dataPipelineEnabled) return; } - EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); using var extensionWithContext = new MockLambdaExtension(shouldSendContext: true, port: 9004, Output); using var extensionNoContext = new MockLambdaExtension(shouldSendContext: false, port: 9003, Output); using var agent = EnvironmentHelper.GetMockAgent(fixedPort: 5002); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs index 0232159ee45c..51328cd07615 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs @@ -61,7 +61,7 @@ public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transpo } EnvironmentHelper.EnableTransport(transportType); - EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); using var agent = EnvironmentHelper.GetMockAgent(); var customResponse = behaviour switch diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs index 7d70354790ea..c316f015e5ee 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs @@ -95,7 +95,7 @@ private async Task RunTest(TracesTransportType transportType, bool dataPipelineE throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } - EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); EnvironmentHelper.EnableTransport(GetTransport(transportType)); using var telemetry = this.ConfigureTelemetry(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs index 911564590d4a..468beb52245e 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs @@ -34,7 +34,7 @@ public LargePayloadTestBase(ITestOutputHelper output) protected async Task RunTest(TestTransports transport, bool dataPipelineEnabled) { EnvironmentHelper.EnableTransport(transport); - EnvironmentHelper.CustomEnvironmentVariables[ConfigurationKeys.TraceDataPipelineEnabled] = dataPipelineEnabled.ToString(); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); using (var agent = EnvironmentHelper.GetMockAgent()) { From 74c9f35a1129ebfde48864687a937681eea35667 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 3 Jun 2025 14:23:44 +0200 Subject: [PATCH 20/23] Try fix x86 support --- tracer/build/_build/Build.Steps.cs | 21 ++++++------------- .../LibDatadog/TraceExporterTests.cs | 4 ---- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 630fdec280d1..fb75c06a48c1 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -659,10 +659,12 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { foreach (var framework in frameworks) { - var arch = GetWinArchitecture(); - var source = MonitoringHomeDirectory / $"win-{arch}" / "datadog_profiling_ffi.dll"; - var dest = testBinFolder / framework / "LibDatadog.dll"; - CopyFile(source, dest, FileExistsPolicy.Overwrite); + foreach (var arch in WindowsArchitectureFolders) + { + var source = MonitoringHomeDirectory / $"win-{arch}" / "datadog_profiling_ffi.dll"; + var dest = testBinFolder / framework / "LibDatadog.dll"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); + } } } else @@ -2746,17 +2748,6 @@ static LogLevel ParseNativeLogLevel(string value) (false, true) => ($"linux-musl-{UnixArchitectureIdentifier}", "so"), }; - private string GetWinArchitecture() - { - return RuntimeInformation.ProcessArchitecture switch - { - Architecture.X86 => "x86", - Architecture.X64 => "x64", - Architecture.Arm64 => "arm64", - _ => throw new NotSupportedException($"Unsupported architecture: {RuntimeInformation.ProcessArchitecture}") - }; - } - // the integration tests need their own copy of the profiler, this achieved through build.props on Windows, but doesn't seem to work under Linux private void IntegrationTestLinuxOrOsxProfilerDirFudge(string project) { diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index 4d2d850718ff..4e10c5d72d1f 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -25,10 +25,6 @@ public class TraceExporterTests [InlineData(TestTransports.WindowsNamedPipe)] public async Task SendsTracesUsingDataPipeline(TestTransports transport) { - if (EnvironmentTools.IsWindows() && Environment.Is64BitProcess == false) - { - throw new SkipException("Can't use data pipeline on win-x86, there is no good way to copy native libraries for win-x86"); - } if (transport == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { From 1959affa3cb8ab54e61a2fe9f5fa67f0361d5fd0 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 3 Jun 2025 14:59:39 +0200 Subject: [PATCH 21/23] Fix typo --- tracer/build/_build/Build.Steps.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index fb75c06a48c1..86708eeab4a0 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -661,7 +661,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { foreach (var arch in WindowsArchitectureFolders) { - var source = MonitoringHomeDirectory / $"win-{arch}" / "datadog_profiling_ffi.dll"; + var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; var dest = testBinFolder / framework / "LibDatadog.dll"; CopyFile(source, dest, FileExistsPolicy.Overwrite); } From e2cad010f8e38c4f4499e372e21932703b27842c Mon Sep 17 00:00:00 2001 From: Ganesh Jangir Date: Tue, 3 Jun 2025 16:38:31 +0200 Subject: [PATCH 22/23] fix: remove unnecessary blank line in SendsTracesUsingDataPipeline test --- .../LibDatadog/TraceExporterTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs index 4e10c5d72d1f..4e6713a3267e 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -25,7 +25,6 @@ public class TraceExporterTests [InlineData(TestTransports.WindowsNamedPipe)] public async Task SendsTracesUsingDataPipeline(TestTransports transport) { - if (transport == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); From c18b698e188aa24eb223d1091a433f19a2e8c796 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Tue, 3 Jun 2025 17:43:20 +0100 Subject: [PATCH 23/23] Only copy single file --- tracer/build/_build/Build.Steps.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 86708eeab4a0..d69854e3ffb2 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -659,12 +659,9 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { foreach (var framework in frameworks) { - foreach (var arch in WindowsArchitectureFolders) - { - var source = MonitoringHomeDirectory / arch / "datadog_profiling_ffi.dll"; - var dest = testBinFolder / framework / "LibDatadog.dll"; - CopyFile(source, dest, FileExistsPolicy.Overwrite); - } + var source = MonitoringHomeDirectory / $"win-{TargetPlatform}" / "datadog_profiling_ffi.dll"; + var dest = testBinFolder / framework / "LibDatadog.dll"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); } } else