diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index b16f2bd72f57..d69854e3ffb2 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -84,7 +84,7 @@ partial class Build AbsolutePath TempDirectory => (AbsolutePath)(IsWin ? Path.GetTempPath() : "/tmp/"); - readonly string[] WafWindowsArchitectureFolders = { "win-x86", "win-x64" }; + readonly string[] WindowsArchitectureFolders = { "win-x86", "win-x64" }; Project NativeTracerProject => Solution.GetProject(Projects.ClrProfilerNative); Project NativeTracerTestsProject => Solution.GetProject(Projects.NativeTracerNativeTests); Project NativeLoaderProject => Solution.GetProject(Projects.NativeLoader); @@ -546,7 +546,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { if (IsWin) { - foreach (var architecture in WafWindowsArchitectureFolders) + foreach (var architecture in WindowsArchitectureFolders) { var source = LibDdwafDirectory() / "runtimes" / architecture / "native" / "ddwaf.dll"; var dest = MonitoringHomeDirectory / architecture; @@ -642,6 +642,41 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo } }); + Target CopyNativeFilesForTests => _ => _ + .Unlisted() + .After(Clean) + .After(BuildTracerHome) + .Executes(() => + { + foreach(var projectName in Projects.NativeFilesDependentTests) + { + var project = Solution.GetProject(projectName); + var testDir = project.Directory; + var frameworks = project.GetTargetFrameworks(); + var testBinFolder = testDir / "bin" / BuildConfiguration; + + if (IsWin) + { + foreach (var framework in frameworks) + { + var source = MonitoringHomeDirectory / $"win-{TargetPlatform}" / "datadog_profiling_ffi.dll"; + var dest = testBinFolder / framework / "LibDatadog.dll"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); + } + } + else + { + var (arch, ext) = GetUnixArchitectureAndExtension(); + var source = MonitoringHomeDirectory / arch / $"libdatadog_profiling.{ext}"; + foreach (var framework in frameworks) + { + var dest = testBinFolder / framework / $"LibDatadog.{ext}"; + CopyFile(source, dest, FileExistsPolicy.Overwrite); + } + } + } + }); + Target CopyNativeFilesForAppSecUnitTests => _ => _ .Unlisted() .After(Clean) @@ -662,13 +697,13 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo { var oldVersionTempPath = TempDirectory / $"libddwaf.{olderLibDdwafVersion}"; await DownloadWafVersion(olderLibDdwafVersion, oldVersionTempPath); - foreach (var arch in WafWindowsArchitectureFolders) + foreach (var arch in WindowsArchitectureFolders) { var oldVersionPath = oldVersionTempPath / "runtimes" / arch / "native" / "ddwaf.dll"; var source = MonitoringHomeDirectory / arch; - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { - var dest = testBinFolder / fmk / arch; + var dest = testBinFolder / framework / arch; CopyDirectoryRecursively(source, dest, DirectoryExistsPolicy.Merge, FileExistsPolicy.Overwrite); CopyFile(oldVersionPath, dest / $"ddwaf-{olderLibDdwafVersion}.dll", FileExistsPolicy.Overwrite); } @@ -687,7 +722,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo var oldVersionPath = oldVersionTempPath / "runtimes" / patchedArchWaf / "native" / $"libddwaf.{ext}"; await DownloadWafVersion(olderLibDdwafVersion, oldVersionTempPath); { - foreach (var fmk in frameworks) + foreach (var framework in frameworks) { // We have to copy into the _root_ test bin folder here, not the arch sub-folder. // This is because these tests try to load the WAF. @@ -696,7 +731,7 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo // - The native tracer must be side-by-side with the running dll // As this is a managed-only unit test, the native tracer _must_ be in the root folder // For simplicity, we just copy all the native dlls there - var dest = testBinFolder / fmk; + var dest = testBinFolder / framework; // use the files from the monitoring native folder CopyDirectoryRecursively(MonitoringHomeDirectory / (IsOsx ? "osx" : arch), dest, DirectoryExistsPolicy.Merge, FileExistsPolicy.Overwrite); @@ -938,7 +973,9 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo .Executes(() => { // extract debug info from everything in monitoring home and copy it to the linux symbols directory - var files = MonitoringHomeDirectory.GlobFiles("linux-*/*.so"); + // except libdatadog since debug symbols are already stripped as part of libdatadog release. + var files = MonitoringHomeDirectory.GlobFiles("linux-*/*.so") + .Where(file => !Path.GetFileName(file).Contains("libdatadog_profiling.so")); foreach (var file in files) { @@ -1263,6 +1300,7 @@ void PrepareMonitoringHomeLinuxForPackaging(AbsolutePath assetsDirectory, string .After(CompileManagedSrc) .After(BuildRunnerTool) .DependsOn(CopyNativeFilesForAppSecUnitTests) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CompileManagedTestHelpers) .Executes(() => { @@ -2841,7 +2879,7 @@ private void DotnetBuild( .CombineWith(projPaths, (settings, projPath) => settings.SetProjectFile(projPath))); } - + private async Task GetVcpkg() { var vcpkgFilePath = string.Empty; diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index c00abf508e07..3529f38251f3 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -93,7 +93,7 @@ partial class Build : NukeBuild [Parameter("Enable or Disable fast developer loop")] readonly bool FastDevLoop; - + [Parameter("The directory containing the tool .nupkg file")] readonly AbsolutePath ToolSource; @@ -108,11 +108,11 @@ partial class Build : NukeBuild [Parameter("Should we build native binaries as Universal. Default to false, so we can still build native libs outside of docker.")] readonly bool AsUniversal = false; - + [Parameter("RuntimeIdentifier sets the target platform for ReadyToRun assemblies in 'PublishManagedTracerR2R'." + "See https://learn.microsoft.com/en-us/dotnet/core/rid-catalog")] string RuntimeIdentifier { get; } - + public Build() { RuntimeIdentifier = GetDefaultRuntimeIdentifier(IsAlpine); @@ -218,7 +218,7 @@ void DeleteReparsePoints(string path) .DependsOn(CreateMissingNullabilityFile) .DependsOn(CreateTrimmingFile) .DependsOn(RegenerateSolutions); - + Target BuildManagedTracerHomeR2R => _ => _ .Unlisted() .Description("Builds the native and managed src, and publishes the tracer home directory") @@ -286,6 +286,7 @@ void DeleteReparsePoints(string path) .Description("Builds the integration tests for Windows") .DependsOn(CompileManagedTestHelpers) .DependsOn(CompileIntegrationTests) + .DependsOn(CopyNativeFilesForTests) .DependsOn(BuildRunnerTool); Target BuildAspNetIntegrationTests => _ => _ @@ -334,6 +335,7 @@ void DeleteReparsePoints(string path) .DependsOn(CompileLinuxOrOsxIntegrationTests) .DependsOn(CompileLinuxDdDotnetIntegrationTests) .DependsOn(BuildRunnerTool) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CopyServerlessArtifacts); Target BuildAndRunLinuxIntegrationTests => _ => _ @@ -349,6 +351,7 @@ void DeleteReparsePoints(string path) .DependsOn(CompileManagedTestHelpers) .DependsOn(CompileLinuxOrOsxIntegrationTests) .DependsOn(BuildRunnerTool) + .DependsOn(CopyNativeFilesForTests) .DependsOn(CopyServerlessArtifacts); Target BuildAndRunOsxIntegrationTests => _ => _ diff --git a/tracer/build/_build/Projects.cs b/tracer/build/_build/Projects.cs index 4a2f984ea53e..58c586e96232 100644 --- a/tracer/build/_build/Projects.cs +++ b/tracer/build/_build/Projects.cs @@ -18,10 +18,12 @@ public static class Projects public const string BenchmarksOpenTelemetryApi = "Benchmarks.OpenTelemetry.Api"; public const string BenchmarksOpenTelemetryInstrumentedApi = "Benchmarks.OpenTelemetry.InstrumentedApi"; + public const string TraceTests = "Datadog.Trace.Tests"; public const string TraceIntegrationTests = "Datadog.Trace.IntegrationTests"; public const string AppSecUnitTests = "Datadog.Trace.Security.Unit.Tests"; public const string AppSecIntegrationTests = "Datadog.Trace.Security.IntegrationTests"; public const string ClrProfilerIntegrationTests = "Datadog.Trace.ClrProfiler.IntegrationTests"; + public const string ClrProfilerManagedTests = "Datadog.Trace.ClrProfiler.Managed.Tests"; public const string DdTraceIntegrationTests = "Datadog.Trace.Tools.Runner.IntegrationTests"; public const string DdTraceArtifactsTests = "Datadog.Trace.Tools.Runner.ArtifactTests"; public const string DdDotnetIntegrationTests = "Datadog.Trace.Tools.dd_dotnet.IntegrationTests"; @@ -39,6 +41,13 @@ public static class Projects public const string DebuggerUnreferencedExternal = "Samples.Probes.Unreferenced.External"; public const string RazorPages = "Samples.AspNetCoreRazorPages"; + + public static readonly string[] NativeFilesDependentTests = { + AppSecUnitTests, + ClrProfilerManagedTests, + TraceIntegrationTests, + TraceTests + }; } public static class FileNames diff --git a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs index 24030a6fb96f..f654d833d513 100644 --- a/tracer/src/Datadog.Trace/Agent/AgentWriter.cs +++ b/tracer/src/Datadog.Trace/Agent/AgentWriter.cs @@ -186,6 +186,11 @@ await Task.WhenAny(_flushTask, Task.Delay(TimeSpan.FromSeconds(20))) await _statsAggregator.DisposeAsync().ConfigureAwait(false); } + if (_api is IDisposable disposableApi) + { + disposableApi.Dispose(); + } + if (!success) { Log.Warning("Could not flush all traces before process exit"); diff --git a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs index 668d03805ee7..2a0db80607f6 100644 --- a/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs +++ b/tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs @@ -293,6 +293,12 @@ internal static partial class ConfigurationKeys /// public const string RuntimeMetricsEnabled = "DD_RUNTIME_METRICS_ENABLED"; + /// + /// Use libdatadog data pipeline to send traces. + /// Default value is false (disabled). + /// + public const string TraceDataPipelineEnabled = "DD_TRACE_DATA_PIPELINE_ENABLED"; + /// /// Configuration key for when a standalone instance of the Trace Agent needs to be started. /// diff --git a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs index a4dbc3ab3d3f..3befa6b9d0da 100644 --- a/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs +++ b/tracer/src/Datadog.Trace/Configuration/TracerSettings.cs @@ -13,6 +13,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using Datadog.Trace.Agent; using Datadog.Trace.Ci; using Datadog.Trace.Ci.CiEnvironment; using Datadog.Trace.ClrProfiler; @@ -413,6 +414,38 @@ _ when x.ToBoolean() is { } boolean => boolean, .AsBoolResult() .OverrideWith(in otelRuntimeMetricsEnabled, ErrorLog, defaultValue: false); + DataPipelineEnabled = config + .WithKeys(ConfigurationKeys.TraceDataPipelineEnabled) + .AsBool(defaultValue: true); + + if (DataPipelineEnabled) + { + // Due to missing quantization and obfuscation in native side, we can't enable the native trace exporter + // as it may lead to different stats results than the managed one. + if (StatsComputationEnabled) + { + DataPipelineEnabled = false; + Log.Warning( + "{ConfigurationKey} is enabled, but {StatsComputationEnabled} is enabled. Disabling {TraceDataPipelineEnabled}.", + ConfigurationKeys.TraceDataPipelineEnabled, + ConfigurationKeys.StatsComputationEnabled, + ConfigurationKeys.TraceDataPipelineEnabled); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + + // Windows supports UnixDomainSocket https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ + // but tokio hasn't added support for it yet https://github.com/tokio-rs/tokio/issues/2201 + if (Exporter.TracesTransport == TracesTransportType.UnixDomainSocket && FrameworkDescription.Instance.IsWindows()) + { + DataPipelineEnabled = false; + Log.Warning( + "{ConfigurationKey} is enabled, but TracesTransport is set to UnixDomainSocket which is not supported on Windows. Disabling {TraceDataPipelineEnabled}.", + ConfigurationKeys.TraceDataPipelineEnabled, + ConfigurationKeys.TraceDataPipelineEnabled); + _telemetry.Record(ConfigurationKeys.TraceDataPipelineEnabled, false, ConfigurationOrigins.Calculated); + } + } + // We should also be writing telemetry for OTEL_LOGS_EXPORTER similar to OTEL_METRICS_EXPORTER, but we don't have a corresponding Datadog config // When we do, we can insert that here @@ -1061,6 +1094,12 @@ public bool DiagnosticSourceEnabled /// internal bool RuntimeMetricsEnabled => DynamicSettings.RuntimeMetricsEnabled ?? _runtimeMetricsEnabled; + /// + /// Gets a value indicating whether libdatadog data pipeline + /// is enabled. + /// + internal bool DataPipelineEnabled { get; } + /// /// Gets the comma separated list of url patterns to skip tracing. /// diff --git a/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs new file mode 100644 index 000000000000..9d22aebb67fc --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ByteSlice.cs @@ -0,0 +1,28 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a slice of a byte array in memory. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct ByteSlice +{ + /// + /// Pointer to the start of the slice. + /// + internal nint Ptr; + + /// + /// Length of the slice. + /// + internal nuint Len; +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs new file mode 100644 index 000000000000..00622a8bbb5a --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/CharSlice.cs @@ -0,0 +1,55 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a slice of a UTF-8 encoded string in memory. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct CharSlice : IDisposable +{ + /// + /// Pointer to the start of the slice. + /// + internal nint Ptr; + + /// + /// Length of the slice. + /// + internal nuint Len; + + /// + /// Initializes a new instance of the struct. + /// This can be further optimized if we can avoid copying the string to unmanaged memory. + /// + /// The string to copy into memory. + internal CharSlice(string? str) + { + if (str == null) + { + Ptr = IntPtr.Zero; + Len = UIntPtr.Zero; + } + else + { + // copy over str to unmanaged memory + var bytes = System.Text.Encoding.UTF8.GetBytes(str); + Ptr = Marshal.AllocHGlobal(bytes.Length); + Marshal.Copy(bytes, 0, Ptr, bytes.Length); + Len = (nuint)bytes.Length; + } + } + + public void Dispose() + { + Marshal.FreeHGlobal(Ptr); + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs b/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs new file mode 100644 index 000000000000..3669da701268 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ErrorCode.cs @@ -0,0 +1,114 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents error codes that can occur when exporting traces. +/// +internal enum ErrorCode +{ + /// + /// Address already in use + /// + AddressInUse = 0, + + /// + /// Connection aborted + /// + ConnectionAborted = 1, + + /// + /// Connection refused + /// + ConnectionRefused = 2, + + /// + /// Connection reset by peer + /// + ConnectionReset = 3, + + /// + /// Error parsing HTTP body + /// + HttpBodyFormat = 4, + + /// + /// HTTP body too long + /// + HttpBodyTooLong = 5, + + /// + /// HTTP error originated by client + /// + HttpClient = 6, + + /// + /// HTTP empty body + /// + HttpEmptyBody = 7, + + /// + /// Error while parsing HTTP message + /// + HttpParse = 8, + + /// + /// HTTP error originated by server + /// + HttpServer = 9, + + /// + /// HTTP unknown error + /// + HttpUnknown = 10, + + /// + /// HTTP wrong status number + /// + HttpWrongStatus = 11, + + /// + /// Invalid argument provided + /// + InvalidArgument = 12, + + /// + /// Invalid data payload + /// + InvalidData = 13, + + /// + /// Invalid input + /// + InvalidInput = 14, + + /// + /// Invalid URL + /// + InvalidUrl = 15, + + /// + /// Input/Output error + /// + IoError = 16, + + /// + /// Unknown network error + /// + NetworkUnknown = 17, + + /// + /// Serialization/Deserialization error + /// + Serde = 18, + + /// + /// Operation timed out + /// + TimedOut = 19, +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs b/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs new file mode 100644 index 000000000000..3a1a2f5dfd13 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/ErrorHandle.cs @@ -0,0 +1,46 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +internal class ErrorHandle : SafeHandle +{ + public ErrorHandle() + : base(IntPtr.Zero, true) + { + } + + public ErrorHandle(IntPtr handle) + : base(handle, true) + { + SetHandle(handle); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + NativeInterop.Exporter.FreeError(handle); + return true; + } + + public TraceExporterException ToException() + { + return new TraceExporterException(Marshal.PtrToStructure(handle)); + } + + public void ThrowIfError() + { + if (!IsInvalid) + { + throw ToException(); + } + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs b/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs index d7aafa9a1cab..608e27067757 100644 --- a/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs +++ b/tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs @@ -16,52 +16,61 @@ internal class NativeInterop internal static class Exporter { - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")] - // internal static extern ErrorHandle Create(out IntPtr outHandle, SafeHandle config); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_new")] + internal static extern ErrorHandle New(out IntPtr outHandle, SafeHandle config); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_error_free")] - internal static extern void ReleaseError(IntPtr error); + internal static extern void FreeError(IntPtr error); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_free")] - internal static extern void Release(IntPtr handle); + internal static extern void Free(IntPtr handle); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")] - // internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_send")] + internal static extern ErrorHandle Send(SafeHandle handle, ByteSlice trace, UIntPtr traceCount, ref IntPtr response); } internal static class Config { [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_new")] - internal static extern void Create(out IntPtr outHandle); + internal static extern void New(out IntPtr outHandle); [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_free")] - internal static extern void Release(IntPtr handle); + internal static extern void Free(IntPtr handle); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")] - // internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_url")] + internal static extern ErrorHandle SetUrl(SafeHandle config, CharSlice url); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")] - // internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_tracer_version")] + internal static extern ErrorHandle SetTracerVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")] - // internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_language")] + internal static extern ErrorHandle SetLanguage(SafeHandle config, CharSlice lang); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")] - // internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_version")] + internal static extern ErrorHandle SetLanguageVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")] - // internal static extern ErrorHandle SetInterperter(SafeHandle config, CharSlice interpreter); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_lang_interpreter")] + internal static extern ErrorHandle SetInterpreter(SafeHandle config, CharSlice interpreter); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")] - // internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_hostname")] + internal static extern ErrorHandle SetHostname(SafeHandle config, CharSlice hostname); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")] - // internal static extern ErrorHandle SetEnvironment(SafeHandle config, CharSlice env); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_env")] + internal static extern ErrorHandle SetEnv(SafeHandle config, CharSlice env); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")] - // internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_version")] + internal static extern ErrorHandle SetVersion(SafeHandle config, CharSlice version); - // [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")] - // internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service); + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_service")] + internal static extern ErrorHandle SetService(SafeHandle config, CharSlice service); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_compute_stats")] + internal static extern ErrorHandle SetComputeStats(SafeHandle config, bool isEnabled); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_enable_telemetry")] + internal static extern ErrorHandle EnableTelemetry(SafeHandle config, IntPtr telemetryConfig); + + [DllImport(DllName, EntryPoint = "ddog_trace_exporter_config_set_client_computed_stats")] + internal static extern ErrorHandle SetClientComputedStats(SafeHandle config, bool clientComputedStats); } } diff --git a/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs new file mode 100644 index 000000000000..e9058cfd7b97 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TelemetryClientConfiguration.cs @@ -0,0 +1,38 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a configuration for the telemetry client. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct TelemetryClientConfiguration +{ + /// + /// The interval at which telemetry should be sent, in milliseconds. + /// + public ulong Interval; + + /// + /// A V4 UUID that represents a tracer session. This ID should: + /// - Be generated when the tracer starts + /// - Be identical within the context of a host (i.e. multiple threads/processes + /// that belong to a single instrumented app should share the same runtime_id) + /// - Be associated with traces to allow correlation between traces and telemetry data + /// + public CharSlice RuntimeId; + + /// + /// Whether to enable debug mode for telemetry. When enabled, sets the dd-telemetry-debug-enabled header to true. + /// Defaults to false. + /// + public bool DebugEnabled; +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs new file mode 100644 index 000000000000..e42b5e9a1365 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporter.cs @@ -0,0 +1,86 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Logging; + +namespace Datadog.Trace.LibDatadog; + +internal class TraceExporter : SafeHandle, IApi +{ + private readonly IDatadogLogger _log; + + public TraceExporter( + TraceExporterConfiguration configuration, + IDatadogLogger? log = null) + : base(IntPtr.Zero, true) + { + _log = log ?? DatadogLogging.GetLoggerFor(); + + _log.Debug("Creating new TraceExporter"); + using var errPtr = NativeInterop.Exporter.New(out var ptr, configuration); + errPtr.ThrowIfError(); + SetHandle(ptr); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + public Task SendTracesAsync(ArraySegment traces, int numberOfTraces, bool statsComputationEnabled, long numberOfDroppedP0Traces, long numberOfDroppedP0Spans, bool appsecStandaloneEnabled) + { + _log.Debug("Sending {Count} traces to the Datadog Agent.", numberOfTraces); + + unsafe + { + fixed (byte* ptr = traces.Array) + { + var tracesSlice = new ByteSlice + { + Ptr = (IntPtr)ptr, + Len = (UIntPtr)traces.Count + }; + + var responsePtr = IntPtr.Zero; + try + { + using var error = NativeInterop.Exporter.Send(this, tracesSlice, (UIntPtr)numberOfTraces, ref responsePtr); + if (!error.IsInvalid) + { + var ex = error.ToException(); +#pragma warning disable DDLOG004 + _log.Error(ex, "An error occurred while sending data to the agent. Error Code: " + ex.ErrorCode + ", message: {Message}", ex.Message); +#pragma warning restore DDLOG004 + throw ex; + } + } + catch (Exception ex) when (ex is not TraceExporterException) + { + _log.Error(ex, "An error occurred while sending data to the agent."); + } + } + } + + _log.Debug("Successfully sent {Count} traces to the Datadog Agent.", numberOfTraces); + + return Task.FromResult(true); + } + + public Task SendStatsAsync(StatsBuffer stats, long bucketDuration) + { + _log.Debug("No-op: stats computation happens in the data pipeline."); + return Task.FromResult(true); + } + + protected override bool ReleaseHandle() + { + NativeInterop.Exporter.Free(handle); + return true; + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs new file mode 100644 index 000000000000..cc12e9e33ba3 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterConfiguration.cs @@ -0,0 +1,162 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents a configuration for the trace exporter. +/// +internal class TraceExporterConfiguration : SafeHandle +{ + private IntPtr _telemetryConfigPtr; + + public TraceExporterConfiguration() + : base(IntPtr.Zero, true) + { + NativeInterop.Config.New(out var ptr); + SetHandle(ptr); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + public string Url + { + init + { + using var url = new CharSlice(value); + using var error = NativeInterop.Config.SetUrl(this, url); + error.ThrowIfError(); + } + } + + public string TraceVersion + { + init + { + using var tracerVersion = new CharSlice(value); + using var error = NativeInterop.Config.SetTracerVersion(this, tracerVersion); + error.ThrowIfError(); + } + } + + public string Language + { + init + { + using var language = new CharSlice(value); + using var error = NativeInterop.Config.SetLanguage(this, language); + error.ThrowIfError(); + } + } + + public string LanguageVersion + { + init + { + using var languageVersion = new CharSlice(value); + using var error = NativeInterop.Config.SetLanguageVersion(this, languageVersion); + error.ThrowIfError(); + } + } + + public string LanguageInterpreter + { + init + { + using var interpreter = new CharSlice(value); + using var error = NativeInterop.Config.SetInterpreter(this, interpreter); + error.ThrowIfError(); + } + } + + public string? Hostname + { + init + { + using var hostname = new CharSlice(value); + using var error = NativeInterop.Config.SetHostname(this, hostname); + error.ThrowIfError(); + } + } + + public string? Env + { + init + { + using var env = new CharSlice(value); + using var error = NativeInterop.Config.SetEnv(this, env); + error.ThrowIfError(); + } + } + + public string? Version + { + init + { + using var version = new CharSlice(value); + using var error = NativeInterop.Config.SetVersion(this, version); + error.ThrowIfError(); + } + } + + public string? Service + { + init + { + using var service = new CharSlice(value); + using var error = NativeInterop.Config.SetService(this, service); + error.ThrowIfError(); + } + } + + public TelemetryClientConfiguration? TelemetryClientConfiguration + { + init + { + if (value.HasValue) + { + _telemetryConfigPtr = Marshal.AllocHGlobal(Marshal.SizeOf(value.Value)); + Marshal.StructureToPtr(value.Value, _telemetryConfigPtr, false); + using var error = NativeInterop.Config.EnableTelemetry(this, _telemetryConfigPtr); + error.ThrowIfError(); + } + } + } + + public bool ComputeStats + { + init + { + using var error = NativeInterop.Config.SetComputeStats(this, value); + error.ThrowIfError(); + } + } + + public bool ClientComputedStats + { + init + { + using var error = NativeInterop.Config.SetClientComputedStats(this, value); + error.ThrowIfError(); + } + } + + protected override bool ReleaseHandle() + { + if (_telemetryConfigPtr != IntPtr.Zero) + { + Marshal.FreeHGlobal(_telemetryConfigPtr); + _telemetryConfigPtr = IntPtr.Zero; + } + + NativeInterop.Config.Free(handle); + return true; + } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs new file mode 100644 index 000000000000..62553fabf501 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterError.cs @@ -0,0 +1,29 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents errors that can occur when exporting traces. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct TraceExporterError +{ + /// + /// The error code representing the domain of the error. + /// Consumers can use this to determine how to handle the error. + /// + internal ErrorCode Code; + + /// + /// Human-readable error message describing the error. + /// + internal IntPtr Msg; +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs new file mode 100644 index 000000000000..a45d911f1f00 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterException.cs @@ -0,0 +1,25 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +using System; +using System.Runtime.InteropServices; + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents an exception thrown by the libdatadog library. +/// +internal class TraceExporterException : Exception +{ + public TraceExporterException(TraceExporterError exporterError) + : base(Marshal.PtrToStringAnsi(exporterError.Msg)) + { + ErrorCode = exporterError.Code; + } + + public ErrorCode ErrorCode { get; } +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs new file mode 100644 index 000000000000..e8c4b237cfc6 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterInputFormat.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents the format of the input traces, as expected by the trace exporter. +/// +internal enum TraceExporterInputFormat +{ + /// + /// Used when the traces are sent to the agent without processing. The whole payload is sent as is to the agent. + /// + Proxy = 0, + + /// + /// Version 0.4 of the trace exporter format. + /// + V04 = 1, +} diff --git a/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs new file mode 100644 index 000000000000..1c55972a8af5 --- /dev/null +++ b/tracer/src/Datadog.Trace/LibDatadog/TraceExporterOutputFormat.cs @@ -0,0 +1,24 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +#nullable enable + +namespace Datadog.Trace.LibDatadog; + +/// +/// Represents the format of the output traces, as expected by the trace exporter. +/// +internal enum TraceExporterOutputFormat +{ + /// + /// Version 0.4 of the trace exporter format. + /// + V04 = 0, + + /// + /// Version 0.7 of the trace exporter format. + /// + V07 = 1, +} diff --git a/tracer/src/Datadog.Trace/TracerManagerFactory.cs b/tracer/src/Datadog.Trace/TracerManagerFactory.cs index 56b64e29e25d..af36880c0660 100644 --- a/tracer/src/Datadog.Trace/TracerManagerFactory.cs +++ b/tracer/src/Datadog.Trace/TracerManagerFactory.cs @@ -15,9 +15,11 @@ using Datadog.Trace.DataStreamsMonitoring; using Datadog.Trace.DogStatsd; using Datadog.Trace.Iast; +using Datadog.Trace.LibDatadog; using Datadog.Trace.Logging; using Datadog.Trace.Logging.DirectSubmission; using Datadog.Trace.Logging.TracerFlare; +using Datadog.Trace.PlatformHelpers; using Datadog.Trace.Processors; using Datadog.Trace.Propagators; using Datadog.Trace.RemoteConfigurationManagement; @@ -30,6 +32,7 @@ using Datadog.Trace.Vendors.StatsdClient; using ConfigurationKeys = Datadog.Trace.Configuration.ConfigurationKeys; using MetricsTransportType = Datadog.Trace.Vendors.StatsdClient.Transport.TransportType; +using NativeInterop = Datadog.Trace.ContinuousProfiler.NativeInterop; using Stopwatch = System.Diagnostics.Stopwatch; namespace Datadog.Trace @@ -349,13 +352,80 @@ protected virtual ISpanSampler GetSpanSampler(TracerSettings settings) protected virtual IAgentWriter GetAgentWriter(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IDiscoveryService discoveryService) { var apiRequestFactory = TracesTransportStrategy.Get(settings.Exporter); - var api = new Api(apiRequestFactory, statsd, updateSampleRates, settings.Exporter.PartialFlushEnabled); + var api = GetApi(settings, statsd, updateSampleRates, apiRequestFactory, settings.Exporter.PartialFlushEnabled); var statsAggregator = StatsAggregator.Create(api, settings, discoveryService); return new AgentWriter(api, statsAggregator, statsd, maxBufferSize: settings.TraceBufferSize, batchInterval: settings.TraceBatchInterval, apmTracingEnabled: settings.ApmTracingEnabled); } + private IApi GetApi(TracerSettings settings, IDogStatsd statsd, Action> updateSampleRates, IApiRequestFactory apiRequestFactory, bool partialFlushEnabled) + { + if (settings.DataPipelineEnabled) + { + try + { + var telemetrySettings = TelemetrySettings.FromSource(GlobalConfigurationSource.Instance, TelemetryFactory.Config, settings, isAgentAvailable: null); + TelemetryClientConfiguration? telemetryClientConfiguration = null; + + // We don't know how to handle telemetry in Agentless mode yet + // so we disable telemetry in this case + if (telemetrySettings.TelemetryEnabled && telemetrySettings.Agentless == null) + { + telemetryClientConfiguration = new TelemetryClientConfiguration + { + Interval = (ulong)telemetrySettings.HeartbeatInterval.Milliseconds, + RuntimeId = new CharSlice(Tracer.RuntimeId), + DebugEnabled = telemetrySettings.DebugEnabled + }; + } + + // When APM is disabled, we don't want to compute stats at all + // A common use case is in Application Security Monitoring (ASM) scenarios: + // when APM is disabled but ASM is enabled. + var clientComputedStats = !settings.StatsComputationEnabled && !settings.ApmTracingEnabled; + + using var configuration = new TraceExporterConfiguration + { + Url = GetUrl(settings), + TraceVersion = TracerConstants.AssemblyVersion, + Env = settings.Environment, + Version = settings.ServiceVersion, + Service = settings.ServiceName, + Hostname = HostMetadata.Instance.Hostname, + Language = ".NET", + LanguageVersion = FrameworkDescription.Instance.ProductVersion, + LanguageInterpreter = FrameworkDescription.Instance.Name, + ComputeStats = settings.StatsComputationEnabled, + TelemetryClientConfiguration = telemetryClientConfiguration, + ClientComputedStats = clientComputedStats + }; + + return new TraceExporter(configuration); + } + catch (Exception ex) + { + Log.Error(ex, "Failed to create native Trace Exporter, falling back to managed API"); + } + } + + return new Api(apiRequestFactory, statsd, updateSampleRates, partialFlushEnabled); + } + + private string GetUrl(TracerSettings settings) + { + switch (settings.Exporter.TracesTransport) + { + case TracesTransportType.WindowsNamedPipe: + return $"windows://./pipe/{settings.Exporter.TracesPipeName}"; + case TracesTransportType.UnixDomainSocket: + return $"unix://{settings.Exporter.TracesUnixDomainSocketPath}"; + case TracesTransportType.Default: + default: + return settings.Exporter.AgentUri.ToString(); + } + } + protected virtual IDiscoveryService GetDiscoveryService(TracerSettings settings) => DiscoveryService.Create(settings.Exporter); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 49e8308ac85d..b29b35f2c095 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -31,10 +32,12 @@ public AwsLambdaTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("Category", "ArmUnsupported")] [Trait("Category", "Lambda")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { // See documentation at docs/development/Serverless.md for examples and diagrams if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IsAlpine"))) @@ -43,6 +46,7 @@ public async Task SubmitsTraces() return; } + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); using var extensionWithContext = new MockLambdaExtension(shouldSendContext: true, port: 9004, Output); using var extensionNoContext = new MockLambdaExtension(shouldSendContext: false, port: 9003, Output); using var agent = EnvironmentHelper.GetMockAgent(fixedPort: 5002); @@ -90,6 +94,7 @@ public async Task SubmitsTraces() settings.AddRegexScrubber(ErrorMsgRegex, "$1 Cannot assign requested address$2"); await VerifyHelper.VerifySpans(allSpans, settings) + .DisableRequireUniquePrefix() .UseFileName(nameof(AwsLambdaTests)); } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs index 61a99bbdbdce..51328cd07615 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AgentMalfunctionTests.cs @@ -45,21 +45,24 @@ public static IEnumerable TestData => from behaviour in (AgentBehaviour[])Enum.GetValues(typeof(AgentBehaviour)) from transportType in Transports from metadataSchemaVersion in new[] { "v0", "v1" } - select new object[] { behaviour, transportType, metadataSchemaVersion }; + from dataPipelineEnabled in new[] { true, false } + select new object[] { behaviour, transportType, metadataSchemaVersion, dataPipelineEnabled }; [SkippableTheory] [MemberData(nameof(TestData))] [Trait("Category", "EndToEnd")] [Trait("RunOnWindows", "True")] - public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transportType, string metadataSchemaVersion) + public async Task SubmitsTraces(AgentBehaviour behaviour, TestTransports transportType, string metadataSchemaVersion, bool dataPipelineEnabled) { SkipOn.Platform(SkipOn.PlatformValue.MacOs); if (transportType == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } EnvironmentHelper.EnableTransport(transportType); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); + using var agent = EnvironmentHelper.GetMockAgent(); var customResponse = behaviour switch { diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs index dadb7865afa3..e60b1507fa53 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/RuntimeMetricsTests.cs @@ -74,7 +74,7 @@ public async Task NamedPipesSubmitsMetrics() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } EnvironmentHelper.EnableWindowsNamedPipes(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs index ab6bb436df6d..4fa6f6366e28 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TelemetryTests.cs @@ -144,7 +144,7 @@ public async Task WhenUsingNamedPipesAgent_UsesNamedPipesTelemetry(bool? enableD { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } EnvironmentHelper.EnableWindowsNamedPipes(); @@ -228,6 +228,9 @@ public async Task WhenUsingUdsAgent_UsesUdsTelemetry(bool? enableDependencies) [Trait("RunOnWindows", "True")] public async Task Telemetry_SendsMetrics() { + // telemetry metric events under test are sent only when using managed trace exporter + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, "false"); + using var agent = MockTracerAgent.Create(Output, useTelemetry: true); Output.WriteLine($"Assigned port {agent.Port} for the agentPort."); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs index 27e22e93c651..c316f015e5ee 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/TransportTests.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Threading.Tasks; using Datadog.Trace.Agent; +using Datadog.Trace.Configuration; using Datadog.Trace.Telemetry; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -21,6 +22,15 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests [UsesVerify] public class TransportTests : TestHelper { + private static readonly TestTransports[] Transports = new[] + { + TestTransports.Tcp, + TestTransports.WindowsNamedPipe, +#if NETCOREAPP3_1_OR_GREATER + TestTransports.Uds, +#endif + }; + private readonly ITestOutputHelper _output; // Using Telemetry sample as it's simple @@ -31,23 +41,20 @@ public TransportTests(ITestOutputHelper output) } public static IEnumerable Data => - Enum.GetValues(typeof(TracesTransportType)) - .Cast() -#if !NETCOREAPP3_1_OR_GREATER - .Where(x => x != TracesTransportType.UnixDomainSocket) -#endif - .Select(x => new object[] { x }); + from transport in Transports + from dataPipelineEnabled in new[] { true, false } + select new object[] { transport, dataPipelineEnabled }; [SkippableTheory] [MemberData(nameof(Data))] [Trait("Category", "EndToEnd")] [Trait("RunOnWindows", "True")] - public async Task TransportsWorkCorrectly(Enum transport) + public async Task TransportsWorkCorrectly(TestTransports transport, bool dataPipelineEnabled) { - var transportType = (TracesTransportType)transport; + var transportType = TracesTransportTypeFromTestTransport(transport); if (transportType != TracesTransportType.WindowsNamedPipe) { - await RunTest(transportType); + await RunTest(transportType, dataPipelineEnabled); return; } @@ -58,7 +65,7 @@ public async Task TransportsWorkCorrectly(Enum transport) try { attemptsRemaining--; - await RunTest(transportType); + await RunTest(transportType, dataPipelineEnabled); return; } catch (Exception ex) when (attemptsRemaining > 0 && ex is not SkipException) @@ -68,15 +75,27 @@ public async Task TransportsWorkCorrectly(Enum transport) } } - private async Task RunTest(TracesTransportType transportType) + private TracesTransportType TracesTransportTypeFromTestTransport(TestTransports transport) + { + return transport switch + { + TestTransports.Tcp => TracesTransportType.Default, + TestTransports.WindowsNamedPipe => TracesTransportType.WindowsNamedPipe, + TestTransports.Uds => TracesTransportType.UnixDomainSocket, + _ => throw new InvalidOperationException($"Unknown transport {transport}"), + }; + } + + private async Task RunTest(TracesTransportType transportType, bool dataPipelineEnabled) { const int expectedSpanCount = 1; if (transportType == TracesTransportType.WindowsNamedPipe && !EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); EnvironmentHelper.EnableTransport(GetTransport(transportType)); using var telemetry = this.ConfigureTelemetry(); diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs index 266d1c628dd0..10160e301e78 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/DefaultTransportLargePayloadTests.cs @@ -4,6 +4,7 @@ // using System.Threading.Tasks; +using Datadog.Trace.TestHelpers; using Xunit; using Xunit.Abstractions; @@ -17,8 +18,10 @@ public DefaultTransportLargePayloadTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] - public Task SubmitsTraces() => RunTest(); + public Task SubmitsTraces(bool dataPipelineEnabled) => RunTest(TestTransports.Tcp, dataPipelineEnabled); } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs index cb6a0a190549..468beb52245e 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/LargePayloadTestBase.cs @@ -8,6 +8,7 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.TestHelpers; using Xunit; using Xunit.Abstractions; @@ -30,8 +31,11 @@ public LargePayloadTestBase(ITestOutputHelper output) public int ExpectedSpans => TracesToTrigger + (TracesToTrigger * SpansPerTrace); - protected async Task RunTest() + protected async Task RunTest(TestTransports transport, bool dataPipelineEnabled) { + EnvironmentHelper.EnableTransport(transport); + SetEnvironmentVariable(ConfigurationKeys.TraceDataPipelineEnabled, dataPipelineEnabled.ToString()); + using (var agent = EnvironmentHelper.GetMockAgent()) { using (var sample = await RunSampleAndWaitForExit(agent, arguments: $" -t {TracesToTrigger} -s {SpansPerTrace} -f {FillerTagLength}")) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs index f7900a3e330f..46737b524b60 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/UnixDomainSocketLargePayloadTests.cs @@ -20,12 +20,13 @@ public UnixDomainSocketLargePayloadTests(ITestOutputHelper output) { } - [SkippableFact] + [SkippableTheory] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { - EnvironmentHelper.EnableUnixDomainSockets(); - await RunTest(); + await RunTest(TestTransports.Uds, dataPipelineEnabled); } } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs index 716c750bc934..1290b908fa2f 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Transports/LargePayload/WindowsNamedPipeLargePayloadTests.cs @@ -21,13 +21,14 @@ public WindowsNamedPipeLargePayloadTests(ITestOutputHelper output) /// /// To be enabled when Windows Named Pipes is available in the MockTracerAgent /// - [SkippableFact(Skip = "Windows named pipes are not yet supported in the MockTracerAgent")] + [SkippableTheory(Skip = "Windows named pipes are not yet supported in the MockTracerAgent")] + [InlineData(false)] + [InlineData(true)] [Trait("RunOnWindows", "True")] [Trait("Category", "LinuxUnsupported")] - public async Task SubmitsTraces() + public async Task SubmitsTraces(bool dataPipelineEnabled) { - EnvironmentHelper.EnableWindowsNamedPipes(); - await RunTest(); + await RunTest(TestTransports.WindowsNamedPipe, dataPipelineEnabled); } } } diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs index 6e3d6dd25619..7041bbdef86d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/VersionConflict/AspNetVersionConflictTests.cs @@ -110,7 +110,7 @@ public async Task Sampling(bool parentTrace) { var samplingPriority = string.Empty; - if (span.Metrics.ContainsKey(Metrics.SamplingPriority)) + if (span.Metrics?.ContainsKey(Metrics.SamplingPriority) == true) { samplingPriority = span.Metrics[Metrics.SamplingPriority].ToString(); } diff --git a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs index 104e67bf9aad..a76189d81209 100644 --- a/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs +++ b/tracer/test/Datadog.Trace.Debugger.IntegrationTests/ProbesTests.cs @@ -412,7 +412,7 @@ public async Task MethodProbeTest_NamedPipes() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } var testType = DebuggerTestHelper.FirstSupportedProbeTestType(EnvironmentHelper.GetTargetFramework()); diff --git a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs index 1b750ae26814..f21b393cce33 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/ContainerTaggingTests.cs @@ -31,7 +31,11 @@ public async Task Http_Headers_Contain_ContainerId() using (var agent = MockTracerAgent.Create(_output, agentPort)) { - var settings = TracerSettings.Create(new() { { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" } }); + var settings = TracerSettings.Create(new() + { + { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" } + }); var tracer = new Tracer(settings, agentWriter: null, sampler: null, scopeManager: null, statsd: null); using (var scope = tracer.StartActive("operationName")) diff --git a/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs new file mode 100644 index 000000000000..4e6713a3267e --- /dev/null +++ b/tracer/test/Datadog.Trace.IntegrationTests/LibDatadog/TraceExporterTests.cs @@ -0,0 +1,156 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; +using Datadog.Trace.AppSec.Rasp; +using Datadog.Trace.Configuration; +using Datadog.Trace.TestHelpers; +using FluentAssertions; +using Xunit; + +namespace Datadog.Trace.IntegrationTests.LibDatadog; + +public class TraceExporterTests +{ + [SkippableTheory] + [InlineData(TestTransports.Tcp)] + [InlineData(TestTransports.Uds)] + [InlineData(TestTransports.WindowsNamedPipe)] + public async Task SendsTracesUsingDataPipeline(TestTransports transport) + { + if (transport == TestTransports.WindowsNamedPipe && !EnvironmentTools.IsWindows()) + { + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); + } + + if (transport == TestTransports.Uds && EnvironmentTools.IsWindows()) + { + throw new SkipException("Unix Domain Sockets (UDS) transport is only supported on Linux and OSX when data pipeline is enabled"); + } + + var pipeName = $"trace-{Guid.NewGuid()}"; + var udsPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + using var agent = GetAgent(); + var settings = GetSettings(); + var tracerSettings = TracerSettings.Create(settings); + + agent.CustomResponses[MockTracerResponseType.Traces] = new MockTracerResponse + { + StatusCode = 200, + ContentType = "application/json", + Response = """ + { + "rate_by_service": { + "service:default-service,env:test": 1.0, + "service:,env:": 0.8 + } + } + """ + }; + + var discovery = DiscoveryService.Create(tracerSettings.Exporter); + await using var tracer = TracerHelper.Create(tracerSettings, discoveryService: discovery); + + var testMetaStruct = new TestMetaStruct + { + Foo = "foo", + Bar = 1, + }; + var metaStructBytes = MetaStructHelper.ObjectToByteArray(testMetaStruct.ToDictionary()); + using (var span = tracer.StartSpan("operationName")) + { + span.ResourceName = "resourceName"; + span.Type = "test"; + span.SetMetaStruct("test-meta-struct", metaStructBytes); + } + + await tracer.TracerManager.ShutdownAsync(); + var recordedSpans = agent.WaitForSpans(1); + recordedSpans.Should().ContainSingle(); + + var recordedSpan = recordedSpans.Should().ContainSingle().Subject; + recordedSpan.Name.Should().Be("operationName"); + recordedSpan.Resource.Should().Be("resourceName"); + recordedSpan.Service.Should().Be("default-service"); + + recordedSpan.MetaStruct.Should().ContainSingle(); + var recordedMetaStructBytes = recordedSpan.MetaStruct["test-meta-struct"]; + recordedMetaStructBytes.Should().BeEquivalentTo(metaStructBytes); + + Dictionary GetSettings() + { + var settingsMap = new Dictionary + { + { ConfigurationKeys.StatsComputationEnabled, true }, + { ConfigurationKeys.ServiceName, "default-service" }, + { ConfigurationKeys.ServiceVersion, "v1" }, + { ConfigurationKeys.Environment, "test" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "true" }, + }; + + switch (transport) + { + case TestTransports.Tcp: + if (agent is MockTracerAgent.TcpUdpAgent tcpAgent) + { + settingsMap[ConfigurationKeys.AgentPort] = tcpAgent.Port; + } + else + { + throw new InvalidOperationException("Unsupported agent type " + agent.GetType()); + } + + break; + case TestTransports.WindowsNamedPipe: + settingsMap[ConfigurationKeys.TracesPipeName] = pipeName; + break; + case TestTransports.Uds: + settingsMap[ConfigurationKeys.TracesUnixDomainSocketPath] = udsPath; + break; + default: + throw new InvalidOperationException("Unsupported transport type " + transport); + } + + return settingsMap; + } + + MockTracerAgent GetAgent() + => transport switch + { + TestTransports.Tcp => MockTracerAgent.Create(null), + TestTransports.WindowsNamedPipe => MockTracerAgent.Create(null, new WindowsPipesConfig(pipeName, null)), +#if NETCOREAPP3_1_OR_GREATER + TestTransports.Uds + => MockTracerAgent.Create(null, new UnixDomainSocketConfig(udsPath, null)), +#endif + _ => throw new InvalidOperationException("Unsupported transport type " + transport), + }; + } + + internal class TestMetaStruct + { + public string Foo { get; set; } + + public int Bar { get; set; } + + public Dictionary ToDictionary() + { + return new Dictionary + { + { + "foo", Foo + }, + { + "bar", Bar + } + }; + } + } +} diff --git a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs index 94042b97c842..4553fc8b1bc9 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/Logging/TracerFlare/TracerFlareApiTests.cs @@ -60,7 +60,7 @@ public async Task CanSendToAgent_NamedPipes() { if (!EnvironmentTools.IsWindows()) { - throw new SkipException("Can't use WindowsNamedPipes on non-Windows"); + throw new SkipException("WindowsNamedPipe transport is only supported on Windows"); } // named pipes is notoriously flaky diff --git a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs index 9f80ad3aa5d0..2dce5a8b4e86 100644 --- a/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs +++ b/tracer/test/Datadog.Trace.IntegrationTests/StatsTests.cs @@ -54,6 +54,7 @@ public async Task SendsStatsWithProcessing_Normalizer() { ConfigurationKeys.ServiceVersion, "v1" }, { ConfigurationKeys.Environment, "test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); var discovery = DiscoveryService.Create(settings.Exporter); @@ -198,6 +199,7 @@ public async Task SendsStatsWithProcessing_Obfuscator() { ConfigurationKeys.ServiceVersion, "v1" }, { ConfigurationKeys.Environment, "test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, }); var discovery = DiscoveryService.Create(settings.Exporter); @@ -356,6 +358,7 @@ private async Task SendStatsHelper(bool statsComputationEnabled, bool expectStat { ConfigurationKeys.ServiceVersion, "V" }, { ConfigurationKeys.Environment, "Test" }, { ConfigurationKeys.AgentUri, $"http://localhost:{agent.Port}" }, + { ConfigurationKeys.TraceDataPipelineEnabled, "false" }, })); var discovery = DiscoveryService.Create(settings.Exporter); diff --git a/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs b/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs index 5d06f964fe16..fef7363ceaf2 100644 --- a/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs +++ b/tracer/test/Datadog.Trace.TestHelpers/TracerHelper.cs @@ -6,6 +6,7 @@ using System; using System.Threading.Tasks; using Datadog.Trace.Agent; +using Datadog.Trace.Agent.DiscoveryService; using Datadog.Trace.Configuration; using Datadog.Trace.Sampling; using Datadog.Trace.Vendors.StatsdClient; @@ -23,9 +24,10 @@ public static ScopedTracer Create( IAgentWriter agentWriter = null, ITraceSampler sampler = null, IScopeManager scopeManager = null, - IDogStatsd statsd = null) + IDogStatsd statsd = null, + IDiscoveryService discoveryService = null) { - return new ScopedTracer(settings, agentWriter, sampler, scopeManager, statsd); + return new ScopedTracer(settings, agentWriter, sampler, scopeManager, statsd, discoveryService); } /// @@ -39,8 +41,14 @@ public static ScopedTracer CreateWithFakeAgent( public class ScopedTracer : Tracer, IAsyncDisposable { - public ScopedTracer(TracerSettings settings = null, IAgentWriter agentWriter = null, ITraceSampler sampler = null, IScopeManager scopeManager = null, IDogStatsd statsd = null) - : base(settings, agentWriter, sampler, scopeManager, statsd) + public ScopedTracer( + TracerSettings settings = null, + IAgentWriter agentWriter = null, + ITraceSampler sampler = null, + IScopeManager scopeManager = null, + IDogStatsd statsd = null, + IDiscoveryService discoveryService = null) + : base(settings, agentWriter, sampler, scopeManager, statsd, discoveryService: discoveryService) { } diff --git a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs index 7f9c04c69b4a..f4b404136434 100644 --- a/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs +++ b/tracer/test/Datadog.Trace.Tests/DogStatsDTests.cs @@ -237,6 +237,7 @@ private static async Task> SendSpan(bool tracerMetricsE { ConfigurationKeys.AgentUri, $"http://127.0.0.1:{agent.Port}" }, { ConfigurationKeys.TracerMetricsEnabled, tracerMetricsEnabled }, { ConfigurationKeys.StartupDiagnosticLogEnabled, false }, + { ConfigurationKeys.TraceDataPipelineEnabled, false }, }); await using var tracer = TracerHelper.Create(settings, agentWriter: null, sampler: null, scopeManager: null, statsd); diff --git a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json index ffefea3dd4ca..560c4c54d45f 100644 --- a/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json +++ b/tracer/test/Datadog.Trace.Tests/Telemetry/config_norm_rules.json @@ -671,5 +671,6 @@ "DD_TRACE_BAGGAGE_MAX_BYTES": "trace_baggage_max_bytes", "DD_TRACE_BYPASS_HTTP_REQUEST_URL_CACHING_ENABLED": "trace_bypass_http_request_url_caching_enabled", "DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED": "inferred_proxy_services_enabled", + "DD_TRACE_DATA_PIPELINE_ENABLED": "trace_data_pipeline_enabled", "DD_TRACE_INJECT_CONTEXT_INTO_STORED_PROCEDURES_ENABLED": "trace_inject_context_into_stored_procedures_enabled" }