diff --git a/Directory.Packages.props b/Directory.Packages.props index fa693f7eac..409815aa67 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -60,6 +60,7 @@ + diff --git a/Microsoft.Testing.Platform.slnf b/Microsoft.Testing.Platform.slnf index 1bcd92636a..0951decf55 100644 --- a/Microsoft.Testing.Platform.slnf +++ b/Microsoft.Testing.Platform.slnf @@ -10,6 +10,7 @@ "src\\Platform\\Microsoft.Testing.Extensions.HangDump\\Microsoft.Testing.Extensions.HangDump.csproj", "src\\Platform\\Microsoft.Testing.Extensions.HotReload\\Microsoft.Testing.Extensions.HotReload.csproj", "src\\Platform\\Microsoft.Testing.Extensions.MSBuild\\Microsoft.Testing.Extensions.MSBuild.csproj", + "src\\Platform\\Microsoft.Testing.Extensions.OpenTelemetry\\Microsoft.Testing.Extensions.OpenTelemetry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.Retry\\Microsoft.Testing.Extensions.Retry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.Telemetry\\Microsoft.Testing.Extensions.Telemetry.csproj", "src\\Platform\\Microsoft.Testing.Extensions.TrxReport.Abstractions\\Microsoft.Testing.Extensions.TrxReport.Abstractions.csproj", diff --git a/TestFx.slnx b/TestFx.slnx index 5c0fa159f4..a0fb468aec 100644 --- a/TestFx.slnx +++ b/TestFx.slnx @@ -40,6 +40,7 @@ + diff --git a/samples/Playground/Playground.csproj b/samples/Playground/Playground.csproj index c7d96bb63d..dacd0961b8 100644 --- a/samples/Playground/Playground.csproj +++ b/samples/Playground/Playground.csproj @@ -28,10 +28,12 @@ + + diff --git a/samples/Playground/Program.cs b/samples/Playground/Program.cs index fca420c9ec..41f954bd96 100644 --- a/samples/Playground/Program.cs +++ b/samples/Playground/Program.cs @@ -6,16 +6,14 @@ using Microsoft.Testing.Platform.Extensions.TestFramework; using Microsoft.Testing.Platform.Extensions.TestHostControllers; using Microsoft.Testing.Platform.Messages; + #if NETCOREAPP using Microsoft.Testing.Platform.ServerMode.IntegrationTests.Messages.V100; - using MSTest.Acceptance.IntegrationTests.Messages.V100; - #endif + using Microsoft.Extensions.AI; -using Microsoft.Testing.Extensions.AzureFoundry; using Microsoft.Testing.Platform.AI; -using Microsoft.Testing.Platform.Capabilities.TestFramework; using Microsoft.Testing.Platform.TestHost; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -54,6 +52,19 @@ public static async Task Main(string[] args) // Enable Telemetry // testApplicationBuilder.AddAppInsightsTelemetryProvider(); + + // Enable OTel + // testApplicationBuilder.AddOpenTelemetryProvider( + // tracing => + // { + // tracing.AddTestingPlatformInstrumentation(); + // tracing.AddOtlpExporter(); + // }, + // metrics => + // { + // metrics.AddTestingPlatformInstrumentation(); + // metrics.AddOtlpExporter(); + // }); using ITestApplication testApplication = await testApplicationBuilder.BuildAsync(); return await testApplication.RunAsync(); } diff --git a/samples/Playground/Tests.cs b/samples/Playground/Tests.cs index b77a79c78b..5d61d85263 100644 --- a/samples/Playground/Tests.cs +++ b/samples/Playground/Tests.cs @@ -15,5 +15,7 @@ public class TestClass [TestMethod] public void Test1() { + Thread.Sleep(5000); + Assert.IsPositive(1); } } diff --git a/samples/public/MTPOTel/MTPOTel.csproj b/samples/public/MTPOTel/MTPOTel.csproj new file mode 100644 index 0000000000..6f9c1a2484 --- /dev/null +++ b/samples/public/MTPOTel/MTPOTel.csproj @@ -0,0 +1,31 @@ + + + + Exe + net9.0 + enable + false + $(NoWarn);NETSDK1023;SA0001;EnableGenerateDocumentationFile + + true + false + $(MSBuildThisFileDirectory)..\..\..\ + + + + + + + + + + + + + + + + + + + diff --git a/samples/public/MTPOTel/Program.cs b/samples/public/MTPOTel/Program.cs new file mode 100644 index 0000000000..e045346d8d --- /dev/null +++ b/samples/public/MTPOTel/Program.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Testing.Extensions; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Messages; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestHost; + +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace MTPOTel; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder testApplicationBuilder = await TestApplication.CreateBuilderAsync(args); + + // Register our simple test framework + testApplicationBuilder.RegisterTestFramework( + _ => new TestFrameworkCapabilities(), + (capabilities, serviceProvider) => new SimpleTestFramework(serviceProvider)); + + // Enable OpenTelemetry with console exporter + testApplicationBuilder.AddOpenTelemetryProvider( + tracing => + { + tracing.AddTestingPlatformInstrumentation(); + tracing.AddConsoleExporter(); + }, + metrics => + { + metrics.AddTestingPlatformInstrumentation(); + metrics.AddConsoleExporter(); + }); + + using ITestApplication testApplication = await testApplicationBuilder.BuildAsync(); + return await testApplication.RunAsync(); + } +} diff --git a/samples/public/MTPOTel/SimpleTestFramework.cs b/samples/public/MTPOTel/SimpleTestFramework.cs new file mode 100644 index 0000000000..a782e85aca --- /dev/null +++ b/samples/public/MTPOTel/SimpleTestFramework.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Threading.Tasks; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.TestHost; + +namespace MTPOTel; + +internal sealed class SimpleTestFramework : ITestFramework, IDataProducer +{ + private readonly IServiceProvider _serviceProvider; + + public SimpleTestFramework(IServiceProvider serviceProvider) + => _serviceProvider = serviceProvider; + + public string Uid => nameof(SimpleTestFramework); + + public string Version => "1.0.0"; + + public string DisplayName => "Simple Test Framework"; + + public string Description => "A simple test framework that returns 5 tests run sequentially"; + + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + public Task IsEnabledAsync() => Task.FromResult(true); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult { IsSuccess = true }); + + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + var sessionUid = new SessionUid("SimpleTestSession"); + + // Create 5 tests to run sequentially + for (int i = 1; i <= 5; i++) + { + string testId = $"Test{i}"; + string testName = $"Simple Test {i}"; + + // Publish test node as in-progress + await context.MessageBus.PublishAsync( + this, + new TestNodeUpdateMessage( + sessionUid, + new TestNode + { + Uid = testId, + DisplayName = testName, + Properties = new PropertyBag(new InProgressTestNodeStateProperty()), + })); + + // Simulate test execution + Console.WriteLine($"Executing {testName}..."); + await Task.Delay(500); // Simulate some work + + // Determine test result (all pass for now) + bool testPassed = true; + + // Publish test node as passed or failed + IProperty resultProperty = testPassed + ? new PassedTestNodeStateProperty() + : new FailedTestNodeStateProperty(new InvalidOperationException($"{testName} failed")); + + await context.MessageBus.PublishAsync( + this, + new TestNodeUpdateMessage( + sessionUid, + new TestNode + { + Uid = testId, + DisplayName = testName, + Properties = new PropertyBag(resultProperty), + })); + + Console.WriteLine($"{testName} completed: {(testPassed ? "Passed" : "Failed")}"); + } + + context.Complete(); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.HotReload/HotReloadTestHostTestFrameworkInvoker.cs b/src/Platform/Microsoft.Testing.Extensions.HotReload/HotReloadTestHostTestFrameworkInvoker.cs index d02cc23240..4742133b50 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HotReload/HotReloadTestHostTestFrameworkInvoker.cs +++ b/src/Platform/Microsoft.Testing.Extensions.HotReload/HotReloadTestHostTestFrameworkInvoker.cs @@ -7,6 +7,7 @@ using Microsoft.Testing.Platform.OutputDevice; using Microsoft.Testing.Platform.Requests; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; namespace Microsoft.Testing.Extensions.Hosting; @@ -28,12 +29,12 @@ private static bool IsHotReloadEnabled(IEnvironment environment) => environment.GetEnvironmentVariable(EnvironmentVariableConstants.DOTNET_WATCH) == "1" || environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_HOTRELOAD_ENABLED) == "1"; - public override async Task ExecuteRequestAsync(ITestFramework testFrameworkAdapter, TestExecutionRequest request, + public override async Task ExecuteRequestAsync(ITestFramework testFramework, TestExecutionRequest request, IMessageBus messageBus, CancellationToken cancellationToken) { if (!_isHotReloadEnabled) { - await base.ExecuteRequestAsync(testFrameworkAdapter, request, messageBus, cancellationToken).ConfigureAwait(false); + await base.ExecuteRequestAsync(testFramework, request, messageBus, cancellationToken).ConfigureAwait(false); return; } @@ -44,16 +45,21 @@ public override async Task ExecuteRequestAsync(ITestFramework testFrameworkAdapt while (await hotReloadHandler.ShouldRunAsync(executionCompleted?.Task, cancellationToken).ConfigureAwait(false)) { executionCompleted = new(); - using SemaphoreSlim requestSemaphore = new(1); var hotReloadOutputDevice = ServiceProvider.GetPlatformOutputDevice() as IHotReloadPlatformOutputDevice; if (hotReloadOutputDevice is not null) { await hotReloadOutputDevice.DisplayBeforeHotReloadSessionStartAsync(cancellationToken).ConfigureAwait(false); } - await testFrameworkAdapter.ExecuteRequestAsync(new(request, messageBus, new SemaphoreSlimRequestCompleteNotifier(requestSemaphore), cancellationToken)).ConfigureAwait(false); + IPlatformOpenTelemetryService? otelService = ServiceProvider.GetPlatformOTelService(); + using (IPlatformActivity? testFrameworkActivity = otelService?.StartActivity("TestFramework", testFramework.ToOTelTags())) + { + using SemaphoreSlim requestSemaphore = new(1); + otelService?.TestFrameworkActivity = testFrameworkActivity; + await testFramework.ExecuteRequestAsync(new(request, messageBus, new SemaphoreSlimRequestCompleteNotifier(requestSemaphore), cancellationToken)).ConfigureAwait(false); + await requestSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + } - await requestSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); await ServiceProvider.GetBaseMessageBus().DrainDataAsync().ConfigureAwait(false); if (hotReloadOutputDevice is not null) { diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/ActivityWrapper.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/ActivityWrapper.cs new file mode 100644 index 0000000000..f7bc97c329 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/ActivityWrapper.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Telemetry; + +namespace Microsoft.Testing.Extensions.OpenTelemetry; + +internal sealed class ActivityWrapper(Activity activity) : IPlatformActivity +{ + public string? Id => activity.Id; + + public IPlatformActivity SetTag(string key, object? value) + { + activity.SetTag(key, value); + return this; + } + + public void Dispose() => activity.Dispose(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/BannedSymbols.txt b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/BannedSymbols.txt new file mode 100644 index 0000000000..0fcaeeec4b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/BannedSymbols.txt @@ -0,0 +1,4 @@ +M:System.String.IsNullOrEmpty(System.String); Use 'TAString.IsNullOrEmpty' instead +M:System.String.IsNullOrWhiteSpace(System.String); Use 'TAString.IsNullOrWhiteSpace' instead +M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead +M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/CounterWrapper.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/CounterWrapper.cs new file mode 100644 index 0000000000..1cde3b1643 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/CounterWrapper.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.Metrics; + +using Microsoft.Testing.Platform.Telemetry; + +namespace Microsoft.Testing.Extensions.OpenTelemetry; + +internal sealed class CounterWrapper : ICounter + where T : struct +{ + private readonly Counter _counter; + + public CounterWrapper(Counter counter) + => _counter = counter + ?? throw new ArgumentNullException(nameof(counter)); + + public void Add(T delta) => _counter.Add(delta); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/HistogramWrapper.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/HistogramWrapper.cs new file mode 100644 index 0000000000..8c6f2062b0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/HistogramWrapper.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.Metrics; + +using Microsoft.Testing.Platform.Telemetry; + +namespace Microsoft.Testing.Extensions.OpenTelemetry; + +internal sealed class HistogramWrapper : IHistogram + where T : struct +{ + private readonly Histogram _histogram; + + public HistogramWrapper(Histogram histogram) + => _histogram = histogram + ?? throw new ArgumentNullException(nameof(histogram)); + + public void Record(T value) + => _histogram.Record(value); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj new file mode 100644 index 0000000000..697db9a500 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/Microsoft.Testing.Extensions.OpenTelemetry.csproj @@ -0,0 +1,40 @@ + + + + netstandard2.0;$(SupportedNetFrameworks) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryPlatformService.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryPlatformService.cs new file mode 100644 index 0000000000..6d84e8ceb8 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryPlatformService.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.Metrics; + +using Microsoft.Testing.Platform.Telemetry; + +namespace Microsoft.Testing.Extensions.OpenTelemetry; + +internal sealed class OpenTelemetryPlatformService : IPlatformOpenTelemetryService +{ + internal const string ActivitySourceName = "Microsoft.Testing.Platform"; + internal const string MeterName = "Microsoft.Testing.Platform"; + + private readonly ActivitySource _activitySource = new(ActivitySourceName, PlatformVersion.Version); + private readonly Meter _meter = new(MeterName, PlatformVersion.Version); + + public IPlatformActivity? TestFrameworkActivity { get; set; } + + public IPlatformActivity? StartActivity([CallerMemberName] string name = "", IEnumerable>? tags = null, string? parentId = null, DateTimeOffset startTime = default) + => _activitySource.StartActivity(name, ActivityKind.Internal, tags: tags, startTime: startTime, parentId: parentId) is Activity activity + ? new ActivityWrapper(activity) + : null; + + public ICounter CreateCounter(string name, string? unit = null, string? description = null, IEnumerable>? tags = null) + where T : struct + => new CounterWrapper(_meter.CreateCounter(name, unit, description, tags!)); + + public IHistogram CreateHistogram(string name, string? unit = null, string? description = null, IEnumerable>? tags = null) + where T : struct + => new HistogramWrapper(_meter.CreateHistogram(name, unit, description, tags)); + + public void Dispose() + => _activitySource.Dispose(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProvider.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProvider.cs new file mode 100644 index 0000000000..f884b730d5 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProvider.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Telemetry; + +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace Microsoft.Testing.Extensions.OpenTelemetry; + +internal sealed class OpenTelemetryProvider : IOpenTelemetryProvider +{ + private readonly TracerProvider _tracerProvider; + private readonly MeterProvider _meterProvider; + + public OpenTelemetryProvider(Action? withTracing = null, Action? withMetrics = null) + { + TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder(); + withTracing?.Invoke(tracerProviderBuilder); + _tracerProvider = tracerProviderBuilder.Build(); + + MeterProviderBuilder meterProviderBuilder = Sdk.CreateMeterProviderBuilder(); + withMetrics?.Invoke(meterProviderBuilder); + _meterProvider = meterProviderBuilder.Build(); + } + + public void Dispose() + { + _tracerProvider.Dispose(); + _meterProvider.Dispose(); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProviderExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProviderExtensions.cs new file mode 100644 index 0000000000..d2454cf3ac --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/OpenTelemetryProviderExtensions.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.OpenTelemetry; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Services; + +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace Microsoft.Testing.Extensions; + +/// +/// Extensions for adding AppInsights telemetry provider. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public static class OpenTelemetryProviderExtensions +{ + /// + /// Adds OpenTelemetry tracing and metrics providers to the application builder, allowing customization of tracing + /// and metrics configuration. + /// + /// This method enables distributed tracing and metrics collection for the application. To + /// customize telemetry behavior, provide configuration delegates for tracing and/or metrics. If no delegates are + /// supplied, default OpenTelemetry settings are used. + /// The application builder to which the OpenTelemetry providers will be added. Cannot be null. + /// An optional delegate to configure the tracing provider. If null, default tracing configuration is applied. + /// An optional delegate to configure the metrics provider. If null, default metrics configuration is applied. + public static void AddOpenTelemetryProvider(this ITestApplicationBuilder builder, Action? withTracing = null, Action? withMetrics = null) + => ((TestApplicationBuilder)builder).Telemetry.AddOpenTelemetryProvider(serviceProvider => + { + ((ServiceProvider)serviceProvider).AddService(new OpenTelemetryPlatformService()); + return new OpenTelemetryProvider(withTracing, withMetrics); + }); + + /// + /// Enables instrumentation for the Microsoft Testing Platform by adding its activity source to the specified tracer + /// provider builder. + /// + /// Use this method to collect telemetry from components that emit activities under the + /// "Microsoft.Testing.Platform" source. This is typically required to enable distributed tracing for operations + /// performed by the Microsoft Testing Platform. + /// The tracer provider builder to which the Microsoft Testing Platform activity source will be added. + /// The tracer provider builder with the Microsoft Testing Platform activity source configured for instrumentation. + public static TracerProviderBuilder AddTestingPlatformInstrumentation(this TracerProviderBuilder builder) + => builder.AddSource(OpenTelemetryPlatformService.ActivitySourceName); + + /// + /// Adds instrumentation for the Microsoft Testing Platform to the specified + /// instance. + /// + /// Use this method to enable collection of metrics emitted by the Microsoft Testing Platform. + /// This is typically required when monitoring or analyzing test execution within applications that utilize the + /// platform. + /// The to which the Microsoft Testing Platform instrumentation will be added. + /// The same instance, configured to include metrics from the Microsoft Testing + /// Platform. + public static MeterProviderBuilder AddTestingPlatformInstrumentation(this MeterProviderBuilder builder) + => builder.AddMeter(OpenTelemetryPlatformService.MeterName); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PACKAGE.md new file mode 100644 index 0000000000..866192516c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PACKAGE.md @@ -0,0 +1,9 @@ +# Microsoft.Testing + +Microsoft Testing is a set of platform, framework and protocol intended to make it possible to run any test on any target or device. + +Documentation can be found at . + +## About + +This package provides Open Telemetry extension to Microsoft Testing Platform. diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Shipped.txt b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Shipped.txt new file mode 100644 index 0000000000..7dc5c58110 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Unshipped.txt new file mode 100644 index 0000000000..170637bd2a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.OpenTelemetry/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +#nullable enable +[TPEXP]Microsoft.Testing.Extensions.OpenTelemetryProviderExtensions +[TPEXP]static Microsoft.Testing.Extensions.OpenTelemetryProviderExtensions.AddOpenTelemetryProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder, System.Action? withTracing = null, System.Action? withMetrics = null) -> void +[TPEXP]static Microsoft.Testing.Extensions.OpenTelemetryProviderExtensions.AddTestingPlatformInstrumentation(this OpenTelemetry.Metrics.MeterProviderBuilder! builder) -> OpenTelemetry.Metrics.MeterProviderBuilder! +[TPEXP]static Microsoft.Testing.Extensions.OpenTelemetryProviderExtensions.AddTestingPlatformInstrumentation(this OpenTelemetry.Trace.TracerProviderBuilder! builder) -> OpenTelemetry.Trace.TracerProviderBuilder! diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExtensionHelper.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExtensionHelper.cs new file mode 100644 index 0000000000..0c665e500b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExtensionHelper.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Extensions; + +namespace Microsoft.Testing.Platform.Helpers; + +internal static class ExtensionHelper +{ + public static KeyValuePair[] ToOTelTags(this IExtension extension) + => [ + new("Extension.UID", extension.Uid), + new("Extension.Version", extension.Version), + new("Extension.DisplayName", extension.DisplayName), + new("Extension.Description", extension.Description), + ]; +} diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs index 73cdf1391c..bbdcc75c86 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.cs @@ -31,11 +31,17 @@ public async Task RunAsync() CancellationToken testApplicationCancellationToken = ServiceProvider.GetTestApplicationCancellationTokenSource().CancellationToken; int exitCode = ExitCodes.GenericFailure; + IPlatformOpenTelemetryService? platformOTelService = null; + IPlatformActivity? activity = null; try { + platformOTelService = ServiceProvider.GetPlatformOTelService(); + string hostType = GetHostType(); + activity = platformOTelService?.StartActivity(hostType); + if (PushOnlyProtocol is null || PushOnlyProtocol?.IsServerMode == false) { - exitCode = await RunTestAppAsync(testApplicationCancellationToken).ConfigureAwait(false); + exitCode = await RunTestAppAsync(platformOTelService, testApplicationCancellationToken).ConfigureAwait(false); if (testApplicationCancellationToken.IsCancellationRequested) { @@ -49,10 +55,10 @@ public async Task RunAsync() { RoslynDebug.Assert(PushOnlyProtocol is not null); - bool isValidProtocol = await PushOnlyProtocol.IsCompatibleProtocolAsync(GetHostType()).ConfigureAwait(false); + bool isValidProtocol = await PushOnlyProtocol.IsCompatibleProtocolAsync(hostType).ConfigureAwait(false); exitCode = isValidProtocol - ? await RunTestAppAsync(testApplicationCancellationToken).ConfigureAwait(false) + ? await RunTestAppAsync(platformOTelService, testApplicationCancellationToken).ConfigureAwait(false) : ExitCodes.IncompatibleProtocolVersion; } finally @@ -69,6 +75,9 @@ public async Task RunAsync() } finally { + // Dispose the activity + activity?.Dispose(); + await DisposeServiceProviderAsync(ServiceProvider, isProcessShutdown: true).ConfigureAwait(false); await DisposeHelper.DisposeAsync(ServiceProvider.GetService()).ConfigureAwait(false); await DisposeHelper.DisposeAsync(PushOnlyProtocol).ConfigureAwait(false); @@ -94,35 +103,44 @@ private string GetHostType() { ConsoleTestHost => "TestHost", TestHostControllersTestHost => "TestHostController", - _ => throw new InvalidOperationException("Unknown host type"), + ServerTestHost => "ServerTestHost", + _ => throw new InvalidOperationException($"Unknown host type '{GetType().FullName}'"), }; return hostType; } - private async Task RunTestAppAsync(CancellationToken testApplicationCancellationToken) + private async Task RunTestAppAsync(IPlatformOpenTelemetryService? platformOTelService, CancellationToken testApplicationCancellationToken) { if (RunTestApplicationLifeCycleCallbacks) { - // Get the test application lifecycle callbacks to be able to call the before run -#pragma warning disable CS0618 // Type or member is obsolete - foreach (ITestHostApplicationLifetime testApplicationLifecycleCallbacks in ServiceProvider.GetServicesInternal()) + using (platformOTelService?.StartActivity("BeforeRunCallbacks")) { - await testApplicationLifecycleCallbacks.BeforeRunAsync(testApplicationCancellationToken).ConfigureAwait(false); + // Get the test application lifecycle callbacks to be able to call the before run + foreach (ITestHostApplicationLifetime testApplicationLifecycleCallbacks in ServiceProvider.GetServicesInternal()) + { + using IPlatformActivity? activity = platformOTelService?.StartActivity(testApplicationLifecycleCallbacks.Uid, testApplicationLifecycleCallbacks.ToOTelTags()); + await testApplicationLifecycleCallbacks.BeforeRunAsync(testApplicationCancellationToken).ConfigureAwait(false); + } } -#pragma warning restore CS0618 // Type or member is obsolete } - int exitCode = await InternalRunAsync(testApplicationCancellationToken).ConfigureAwait(false); + int exitCode; + using (platformOTelService?.StartActivity("Run")) + { + exitCode = await InternalRunAsync(testApplicationCancellationToken).ConfigureAwait(false); + } if (RunTestApplicationLifeCycleCallbacks) { -#pragma warning disable CS0618 // Type or member is obsolete - foreach (ITestHostApplicationLifetime testApplicationLifecycleCallbacks in ServiceProvider.GetServicesInternal()) + using (platformOTelService?.StartActivity("AfterRunCallbacks")) { - await testApplicationLifecycleCallbacks.AfterRunAsync(exitCode, testApplicationCancellationToken).ConfigureAwait(false); - await DisposeHelper.DisposeAsync(testApplicationLifecycleCallbacks).ConfigureAwait(false); + foreach (ITestHostApplicationLifetime testApplicationLifecycleCallbacks in ServiceProvider.GetServicesInternal()) + { + using IPlatformActivity? activity = platformOTelService?.StartActivity(testApplicationLifecycleCallbacks.Uid, testApplicationLifecycleCallbacks.ToOTelTags()); + await testApplicationLifecycleCallbacks.AfterRunAsync(exitCode, testApplicationCancellationToken).ConfigureAwait(false); + await DisposeHelper.DisposeAsync(testApplicationLifecycleCallbacks).ConfigureAwait(false); + } } -#pragma warning restore CS0618 // Type or member is obsolete } return exitCode; @@ -137,9 +155,21 @@ protected static async Task ExecuteRequestAsync(ProxyOutputDevice outputDevice, CancellationToken cancellationToken = testSessionInfo.CancellationToken; try { - await NotifyTestSessionStartAsync(testSessionInfo, baseMessageBus, serviceProvider).ConfigureAwait(false); - await serviceProvider.GetTestAdapterInvoker().ExecuteAsync(testFramework, client, cancellationToken).ConfigureAwait(false); - await NotifyTestSessionEndAsync(testSessionInfo, baseMessageBus, serviceProvider).ConfigureAwait(false); + IPlatformOpenTelemetryService? otelService = serviceProvider.GetPlatformOTelService(); + using (otelService?.StartActivity("OnTestSessionStarting")) + { + await NotifyTestSessionStartAsync(testSessionInfo, baseMessageBus, serviceProvider, otelService).ConfigureAwait(false); + } + + using (otelService?.StartActivity("TestFrameworkInvoker")) + { + await serviceProvider.GetTestFrameworkInvoker().ExecuteAsync(testFramework, client, cancellationToken).ConfigureAwait(false); + } + + using (otelService?.StartActivity("OnTestSessionEnding")) + { + await NotifyTestSessionEndAsync(testSessionInfo, baseMessageBus, serviceProvider, otelService).ConfigureAwait(false); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -174,7 +204,7 @@ private static async Task DisplayAfterSessionEndRunAsync(ProxyOutputDevice outpu } } - private static async Task NotifyTestSessionStartAsync(ITestSessionContext testSessionContext, BaseMessageBus baseMessageBus, ServiceProvider serviceProvider) + private static async Task NotifyTestSessionStartAsync(ITestSessionContext testSessionContext, BaseMessageBus baseMessageBus, ServiceProvider serviceProvider, IPlatformOpenTelemetryService? otelService) { TestSessionLifetimeHandlersContainer? testSessionLifetimeHandlersContainer = serviceProvider.GetService(); if (testSessionLifetimeHandlersContainer is null) @@ -184,6 +214,7 @@ private static async Task NotifyTestSessionStartAsync(ITestSessionContext testSe foreach (ITestSessionLifetimeHandler testSessionLifetimeHandler in testSessionLifetimeHandlersContainer.TestSessionLifetimeHandlers) { + using IPlatformActivity? activity = otelService?.StartActivity(testSessionLifetimeHandler.Uid, testSessionLifetimeHandler.ToOTelTags()); await testSessionLifetimeHandler.OnTestSessionStartingAsync(testSessionContext).ConfigureAwait(false); } @@ -191,7 +222,7 @@ private static async Task NotifyTestSessionStartAsync(ITestSessionContext testSe await baseMessageBus.DrainDataAsync().ConfigureAwait(false); } - private static async Task NotifyTestSessionEndAsync(ITestSessionContext testSessionContext, BaseMessageBus baseMessageBus, ServiceProvider serviceProvider) + private static async Task NotifyTestSessionEndAsync(ITestSessionContext testSessionContext, BaseMessageBus baseMessageBus, ServiceProvider serviceProvider, IPlatformOpenTelemetryService? otelService) { // Drain messages generated by the test session execution before to process the session end notification. await baseMessageBus.DrainDataAsync().ConfigureAwait(false); @@ -204,7 +235,10 @@ private static async Task NotifyTestSessionEndAsync(ITestSessionContext testSess foreach (ITestSessionLifetimeHandler testSessionLifetimeHandler in serviceProvider.GetRequiredService().TestSessionLifetimeHandlers) { - await testSessionLifetimeHandler.OnTestSessionFinishingAsync(testSessionContext).ConfigureAwait(false); + using (otelService?.StartActivity(testSessionLifetimeHandler.Uid, testSessionLifetimeHandler.ToOTelTags())) + { + await testSessionLifetimeHandler.OnTestSessionFinishingAsync(testSessionContext).ConfigureAwait(false); + } // OnTestSessionFinishingAsync could produce information that needs to be handled by others. await baseMessageBus.DrainDataAsync().ConfigureAwait(false); @@ -245,7 +279,9 @@ protected static async Task DisposeServiceProviderAsync(ServiceProvider serviceP service is ITelemetryCollector or ITestHostApplicationLifetime or ITestHostApplicationLifetime or - IPushOnlyProtocol) + IPushOnlyProtocol or + IPlatformOpenTelemetryService or + IOpenTelemetryProvider) { continue; } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/ConsoleTestHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/ConsoleTestHost.cs index f0959733ff..969bcf18f8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/ConsoleTestHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/ConsoleTestHost.cs @@ -38,7 +38,6 @@ protected override async Task InternalRunAsync(CancellationToken cancellati { var consoleRunStarted = Stopwatch.StartNew(); DateTimeOffset consoleRunStart = _clock.UtcNow; - DateTimeOffset adapterLoadStart = _clock.UtcNow; // Add the ClientInfo service to the service provider @@ -49,22 +48,26 @@ protected override async Task InternalRunAsync(CancellationToken cancellati ?? new ConsoleTestExecutionFilterFactory(ServiceProvider.GetCommandLineOptions()); // Use user provided filter factory or create console default one. - ITestFrameworkInvoker testAdapterInvoker = ServiceProvider.GetService() + ITestFrameworkInvoker testFrameworkInvoker = ServiceProvider.GetService() ?? new TestHostTestFrameworkInvoker(ServiceProvider); ServiceProvider.TryAddService(new Services.TestSessionContext(cancellationToken)); - ITestFramework testFramework = await _buildTestFrameworkAsync(new( - ServiceProvider, - new ConsoleTestExecutionRequestFactory(ServiceProvider.GetCommandLineOptions(), testExecutionFilterFactory), - testAdapterInvoker, - testExecutionFilterFactory, - ServiceProvider.GetPlatformOutputDevice(), - [], - _testFrameworkManager, - _testHostManager, - new MessageBusProxy(), - ServiceProvider.GetCommandLineOptions().IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey), - false)).ConfigureAwait(false); + ITestFramework testFramework; + using (ServiceProvider.GetPlatformOTelService()?.StartActivity("CreateTestFramework")) + { + testFramework = await _buildTestFrameworkAsync(new( + ServiceProvider, + new ConsoleTestExecutionRequestFactory(ServiceProvider.GetCommandLineOptions(), testExecutionFilterFactory), + testFrameworkInvoker, + testExecutionFilterFactory, + ServiceProvider.GetPlatformOutputDevice(), + [], + _testFrameworkManager, + _testHostManager, + new MessageBusProxy(), + ServiceProvider.GetCommandLineOptions().IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey), + false)).ConfigureAwait(false); + } ITelemetryCollector telemetry = ServiceProvider.GetTelemetryCollector(); ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation(); diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/ServerTestHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/ServerTestHost.cs index f5c38c1721..edb75abd6f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/ServerTestHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/ServerTestHost.cs @@ -132,6 +132,8 @@ public void AssertInitialized() protected override async Task InternalRunAsync(CancellationToken cancellationToken) { + using IPlatformActivity? activity = ServiceProvider.GetPlatformOTelService()?.StartActivity("ServerTestHost"); + try { await _logger.LogDebugAsync("Starting server mode").ConfigureAwait(false); diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs index 1add332506..3ef2f39491 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.cs @@ -31,6 +31,7 @@ namespace Microsoft.Testing.Platform.Hosts; internal sealed class TestHostBuilder(IFileSystem fileSystem, IRuntimeFeature runtimeFeature, IEnvironment environment, IProcessHandler processHandler, ITestApplicationModuleInfo testApplicationModuleInfo) : ITestHostBuilder { + private const string BuilderHostTypeOTelKey = "HostType"; private readonly IFileSystem _fileSystem = fileSystem; private readonly ITestApplicationModuleInfo _testApplicationModuleInfo = testApplicationModuleInfo; private readonly PlatformOutputDeviceManager _outputDisplay = new(); @@ -134,6 +135,14 @@ public async Task BuildAsync( var configuration = (AggregatedConfiguration)await ((ConfigurationManager)Configuration).BuildAsync(loggingState.FileLoggerProvider, loggingState.CommandLineParseResult).ConfigureAwait(false); serviceProvider.TryAddService(configuration); + // When building the otel service, it will register a platform OTel service that we use to compensate the lack of access to System.DiagnosticsSource + IPlatformActivity? builderActivity = null; + if (((TelemetryManager)Telemetry).BuildOTelProvider(serviceProvider) is { } otelService) + { + serviceProvider.AddService(otelService); + builderActivity = serviceProvider.GetServiceInternal()?.StartActivity("TestHostBuilder", startTime: buildBuilderStart); + } + // Current test platform is picky on unhandled exception, we will tear down the process in that case. // This mode can be too aggressive especially compared to the old framework, so we allow the user to disable it if their suite // relies on unhandled exception. @@ -246,6 +255,9 @@ public async Task BuildAsync( await DisplayBannerIfEnabledAsync(loggingState, proxyOutputDevice, testFrameworkCapabilities, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); await proxyOutputDevice.DisplayAsync(commandLineHandler, new ErrorMessageOutputDeviceData(commandLineValidationResult.ErrorMessage), testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); await commandLineHandler.PrintHelpAsync(proxyOutputDevice, null, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); + + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(InformativeCommandLineHost)); + builderActivity?.Dispose(); return new InformativeCommandLineHost(ExitCodes.InvalidCommandLine, serviceProvider); } @@ -292,8 +304,9 @@ public async Task BuildAsync( // Add global telemetry service. // Add at this point or the telemetry banner appearance order will be wrong, we want the testing app banner before the telemetry banner. - ITelemetryCollector telemetryService = await ((TelemetryManager)Telemetry).BuildAsync(serviceProvider, loggerFactory, testApplicationOptions).ConfigureAwait(false); + ITelemetryCollector telemetryService = await ((TelemetryManager)Telemetry).BuildTelemetryAsync(serviceProvider, loggerFactory, testApplicationOptions).ConfigureAwait(false); serviceProvider.TryAddService(telemetryService); + AddApplicationMetadata(serviceProvider, builderMetrics); // Subscribe to the process if the option is set. @@ -310,7 +323,8 @@ public async Task BuildAsync( proxyOutputDevice, serviceProvider.GetCommandLineOptions(), serviceProvider.GetEnvironment(), - policiesService); + policiesService, + serviceProvider.GetPlatformOTelService()); serviceProvider.AddService(testApplicationResult); // Add Chat Client if AI capabilities are enabled @@ -339,6 +353,8 @@ await LogTestHostCreatedAsync( stop: systemClock.UtcNow, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(ToolsHost)); + builderActivity?.Dispose(); return toolsTestHost; } @@ -361,6 +377,8 @@ await LogTestHostCreatedAsync( await commandLineHandler.PrintHelpAsync(proxyOutputDevice, toolsInformation, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); } + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(InformativeCommandLineHost)); + builderActivity?.Dispose(); return new InformativeCommandLineHost(0, serviceProvider); } @@ -368,6 +386,8 @@ await LogTestHostCreatedAsync( if (isInfoCommand) { await commandLineHandler.PrintInfoAsync(proxyOutputDevice, toolsInformation, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(InformativeCommandLineHost)); + builderActivity?.Dispose(); return new InformativeCommandLineHost(0, serviceProvider); } @@ -383,6 +403,8 @@ await LogTestHostCreatedAsync( await ((TestHostOrchestratorManager)TestHostOrchestratorManager).BuildTestHostOrchestratorApplicationLifetimesAsync(serviceProvider).ConfigureAwait(false); serviceProvider.AddServices(orchestratorLifetimes); + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(TestHostOrchestratorHost)); + builderActivity?.Dispose(); return new TestHostOrchestratorHost(testHostOrchestratorConfiguration, serviceProvider); } @@ -429,6 +451,8 @@ await LogTestHostCreatedAsync( stop: systemClock.UtcNow, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); + builderActivity?.SetTag(BuilderHostTypeOTelKey, nameof(TestHostControllersTestHost)); + builderActivity?.Dispose(); return testHostControllersTestHost; } } @@ -485,6 +509,8 @@ await LogTestHostCreatedAsync( stop: systemClock.UtcNow, testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); + builderActivity?.SetTag(BuilderHostTypeOTelKey, testControllerConnection is not null ? nameof(TestHostControlledHost) : nameof(ServerTestHost)); + builderActivity?.Dispose(); return actualTestHost; } else @@ -532,6 +558,8 @@ await LogTestHostCreatedAsync( testApplicationCancellationTokenSource.CancellationToken).ConfigureAwait(false); #pragma warning restore SA1118 // Parameter should not span multiple lines + builderActivity?.SetTag(BuilderHostTypeOTelKey, testControllerConnection is not null ? nameof(TestHostControlledHost) : nameof(ConsoleTestHost)); + builderActivity?.Dispose(); return actualTestHost; } } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostOchestratorHost.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostOchestratorHost.cs index f4fa231042..da8ff57ef0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostOchestratorHost.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostOchestratorHost.cs @@ -5,6 +5,7 @@ using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; namespace Microsoft.Testing.Platform.Hosts; @@ -15,6 +16,7 @@ internal sealed class TestHostOrchestratorHost(TestHostOrchestratorConfiguration public async Task RunAsync() { + using IPlatformActivity? activity = _serviceProvider.GetPlatformOTelService()?.StartActivity("TestHostOrchestratorHost"); ILogger logger = _serviceProvider.GetLoggerFactory().CreateLogger(); if (_testHostOrchestratorConfiguration.TestHostOrchestrators.Length > 1) { diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 2fde16409a..4a1e732111 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -38,6 +38,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/Platform/Microsoft.Testing.Platform/Requests/TestHostTestFrameworkInvoker.cs b/src/Platform/Microsoft.Testing.Platform/Requests/TestHostTestFrameworkInvoker.cs index 1ddcdda6f5..7ddc32a1f2 100644 --- a/src/Platform/Microsoft.Testing.Platform/Requests/TestHostTestFrameworkInvoker.cs +++ b/src/Platform/Microsoft.Testing.Platform/Requests/TestHostTestFrameworkInvoker.cs @@ -12,6 +12,7 @@ using Microsoft.Testing.Platform.OutputDevice; using Microsoft.Testing.Platform.Resources; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; using Microsoft.Testing.Platform.TestHost; namespace Microsoft.Testing.Platform.Requests; @@ -36,7 +37,6 @@ internal class TestHostTestFrameworkInvoker(IServiceProvider serviceProvider) : public async Task ExecuteAsync(ITestFramework testFramework, ClientInfo client, CancellationToken cancellationToken) { ILogger logger = ServiceProvider.GetLoggerFactory().CreateLogger(); - await logger.LogInformationAsync($"Test framework UID: '{testFramework.Uid}' Version: '{testFramework.Version}' DisplayName: '{testFramework.DisplayName}' Description: '{testFramework.Description}'").ConfigureAwait(false); foreach (ICapability capability in ServiceProvider.GetTestFrameworkCapabilities().Capabilities) @@ -50,24 +50,44 @@ public async Task ExecuteAsync(ITestFramework testFramework, ClientInfo client, DateTimeOffset startTime = DateTimeOffset.UtcNow; var stopwatch = Stopwatch.StartNew(); SessionUid sessionId = ServiceProvider.GetTestSessionContext().SessionUid; - CreateTestSessionResult createTestSessionResult = await testFramework.CreateTestSessionAsync(new(sessionId, cancellationToken)).ConfigureAwait(false); - await HandleTestSessionResultAsync(createTestSessionResult.IsSuccess, createTestSessionResult.WarningMessage, createTestSessionResult.ErrorMessage, cancellationToken).ConfigureAwait(false); - ITestExecutionRequestFactory testExecutionRequestFactory = ServiceProvider.GetTestExecutionRequestFactory(); - TestExecutionRequest request = await testExecutionRequestFactory.CreateRequestAsync(new(sessionId)).ConfigureAwait(false); + IPlatformOpenTelemetryService? otelService = ServiceProvider.GetPlatformOTelService(); + using (otelService?.StartActivity("CreateTestFrameworkSession", tags: [new("SessionUid", sessionId)])) + { + CreateTestSessionResult createTestSessionResult = await testFramework.CreateTestSessionAsync(new(sessionId, cancellationToken)).ConfigureAwait(false); + await HandleTestSessionResultAsync(createTestSessionResult.IsSuccess, createTestSessionResult.WarningMessage, createTestSessionResult.ErrorMessage, cancellationToken).ConfigureAwait(false); + } + + TestExecutionRequest request; + using (otelService?.StartActivity("CreateTestRequest", tags: [new("SessionUid", sessionId)])) + { + ITestExecutionRequestFactory testExecutionRequestFactory = ServiceProvider.GetTestExecutionRequestFactory(); + request = await testExecutionRequestFactory.CreateRequestAsync(new(sessionId)).ConfigureAwait(false); + } + IMessageBus messageBus = ServiceProvider.GetMessageBus(); // Execute the test request - await ExecuteRequestAsync(testFramework, request, messageBus, cancellationToken).ConfigureAwait(false); + using (otelService?.StartActivity("ExecuteTestRequest", tags: [new("SessionUid", sessionId), new("RequestType", request.GetType().Name)])) + { + await ExecuteRequestAsync(testFramework, request, messageBus, cancellationToken).ConfigureAwait(false); + } + + using (otelService?.StartActivity("CloseTestFrameworkSession", tags: [new("SessionUid", sessionId)])) + { + CloseTestSessionResult closeTestSessionResult = await testFramework.CloseTestSessionAsync(new(sessionId, cancellationToken)).ConfigureAwait(false); + await HandleTestSessionResultAsync(closeTestSessionResult.IsSuccess, closeTestSessionResult.WarningMessage, closeTestSessionResult.ErrorMessage, cancellationToken).ConfigureAwait(false); + } - CloseTestSessionResult closeTestSessionResult = await testFramework.CloseTestSessionAsync(new(sessionId, cancellationToken)).ConfigureAwait(false); - await HandleTestSessionResultAsync(closeTestSessionResult.IsSuccess, closeTestSessionResult.WarningMessage, closeTestSessionResult.ErrorMessage, cancellationToken).ConfigureAwait(false); DateTimeOffset endTime = DateTimeOffset.UtcNow; await messageBus.PublishAsync(this, new TestRequestExecutionTimeInfo(new TimingInfo(startTime, endTime, stopwatch.Elapsed))).ConfigureAwait(false); } public virtual async Task ExecuteRequestAsync(ITestFramework testFramework, TestExecutionRequest request, IMessageBus messageBus, CancellationToken cancellationToken) { + IPlatformOpenTelemetryService? otelService = ServiceProvider.GetPlatformOTelService(); + using IPlatformActivity? testFrameworkActivity = otelService?.StartActivity("TestFramework", testFramework.ToOTelTags()); + otelService?.TestFrameworkActivity = testFrameworkActivity; using SemaphoreSlim requestSemaphore = new(0, 1); await testFramework.ExecuteRequestAsync(new(request, messageBus, new SemaphoreSlimRequestCompleteNotifier(requestSemaphore), cancellationToken)).ConfigureAwait(false); await requestSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); diff --git a/src/Platform/Microsoft.Testing.Platform/Services/ServiceProviderExtensions.cs b/src/Platform/Microsoft.Testing.Platform/Services/ServiceProviderExtensions.cs index 1c74ea6a2d..a74177c01f 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/ServiceProviderExtensions.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/ServiceProviderExtensions.cs @@ -186,7 +186,7 @@ internal static ITelemetryCollector GetTelemetryCollector(this IServiceProvider internal static ITestFramework GetTestFramework(this IServiceProvider serviceProvider) => serviceProvider.GetRequiredServiceInternal(); - internal static ITestFrameworkInvoker GetTestAdapterInvoker(this IServiceProvider serviceProvider) + internal static ITestFrameworkInvoker GetTestFrameworkInvoker(this IServiceProvider serviceProvider) => serviceProvider.GetRequiredServiceInternal(); internal static IUnhandledExceptionsPolicy GetUnhandledExceptionsPolicy(this IServiceProvider serviceProvider) @@ -206,4 +206,7 @@ internal static IPlatformInformation GetPlatformInformation(this IServiceProvide internal static IFileLoggerInformation? GetFileLoggerInformation(this IServiceProvider serviceProvider) => serviceProvider.GetServiceInternal(); + + internal static IPlatformOpenTelemetryService? GetPlatformOTelService(this IServiceProvider serviceProvider) + => serviceProvider.GetServiceInternal(); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 776eb2eb7f..3b57e124ba 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -8,6 +8,7 @@ using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.OutputDevice; using Microsoft.Testing.Platform.Resources; +using Microsoft.Testing.Platform.Telemetry; namespace Microsoft.Testing.Platform.Services; @@ -17,7 +18,17 @@ internal sealed class TestApplicationResult : ITestApplicationProcessExitCode, I private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; private readonly IStopPoliciesService _policiesService; + private readonly IPlatformOpenTelemetryService? _otelService; + private readonly ICounter? _totalDiscoveredTests; + private readonly ICounter? _totalStartedTests; + private readonly ICounter? _totalCompletedTests; + private readonly ICounter? _totalPassedTests; + private readonly ICounter? _totalFailedTests; + private readonly ICounter? _totalSkippedTests; + private readonly ICounter? _totalUnknownedTests; + private readonly IHistogram? _totalDuration; private readonly bool _isDiscovery; + private readonly Dictionary _testActivities = []; private int _failedTestsCount; private int _totalRanTests; private bool _testAdapterTestSessionFailure; @@ -26,12 +37,22 @@ public TestApplicationResult( IOutputDevice outputService, ICommandLineOptions commandLineOptions, IEnvironment environment, - IStopPoliciesService policiesService) + IStopPoliciesService policiesService, + IPlatformOpenTelemetryService? otelService) { _outputService = outputService; _commandLineOptions = commandLineOptions; _environment = environment; _policiesService = policiesService; + _otelService = otelService; + _totalDiscoveredTests = otelService?.CreateCounter("tests.discovered"); + _totalStartedTests = otelService?.CreateCounter("tests.started"); + _totalCompletedTests = otelService?.CreateCounter("tests.completed"); + _totalPassedTests = otelService?.CreateCounter("tests.passed"); + _totalFailedTests = otelService?.CreateCounter("tests.failed"); + _totalSkippedTests = otelService?.CreateCounter("tests.skipped"); + _totalUnknownedTests = otelService?.CreateCounter("tests.unknown"); + _totalDuration = otelService?.CreateHistogram("tests.duration"); _isDiscovery = _commandLineOptions.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey); } @@ -68,17 +89,62 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo return Task.CompletedTask; } - if (Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties, executionState.GetType()) != -1) + switch (executionState) + { + case DiscoveredTestNodeStateProperty: + _totalDiscoveredTests?.Add(1); + break; + + case PassedTestNodeStateProperty passed: + _totalPassedTests?.Add(1); + _totalCompletedTests?.Add(1); + HandleTestResult(message.TestNode, passed); + break; + + case FailedTestNodeStateProperty: + case ErrorTestNodeStateProperty: + case TimeoutTestNodeStateProperty: +#pragma warning disable CS0618 // Type or member is obsolete + case CancelledTestNodeStateProperty: +#pragma warning restore CS0618 // Type or member is obsolete + _totalFailedTests?.Add(1); + _totalCompletedTests?.Add(1); + HandleTestResult(message.TestNode, executionState); + break; + + case SkippedTestNodeStateProperty skipped: + _totalSkippedTests?.Add(1); + _totalCompletedTests?.Add(1); + HandleTestResult(message.TestNode, skipped); + break; + + case InProgressTestNodeStateProperty: + _totalStartedTests?.Add(1); + _testActivities.Add( + message.TestNode.Uid, + _otelService?.StartActivity( + message.TestNode.Uid, + parentId: _otelService?.TestFrameworkActivity?.Id, + tags: GetTestInitialInfo(message.TestNode, message.ParentTestNodeUid))); + break; + + default: + _totalUnknownedTests?.Add(1); + break; + } + + Type outcomeType = executionState.GetType(); + if (Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeFailedProperties, outcomeType) != -1) { _failedTestsCount++; } if (_isDiscovery - && Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeDiscoveredProperties, executionState.GetType()) != -1) + && Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeDiscoveredProperties, outcomeType) != -1) { _totalRanTests++; } - else if (Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeProperties, executionState.GetType()) != -1) + else if (Array.IndexOf(TestNodePropertiesCategories.WellKnownTestNodeTestRunOutcomeProperties, outcomeType) != -1) { _totalRanTests++; } @@ -130,4 +196,99 @@ public async Task SetTestAdapterTestSessionFailureAsync(string errorMessage, Can public Statistics GetStatistics() => new() { TotalRanTests = _totalRanTests, TotalFailedTests = _failedTestsCount }; + + private static IEnumerable> GetTestInitialInfo(TestNode testNode, TestNodeUid? parentUid) + { + yield return new("test.name", testNode.DisplayName); + yield return new("test.id", testNode.Uid.Value); + if (parentUid is not null) + { + yield return new("test.parent.id", parentUid.Value); + } + + if (testNode.Properties.SingleOrDefault() is { } identifierProperty) + { + yield return new("test.method", identifierProperty.MethodName); + yield return new("test.class", identifierProperty.TypeName); + yield return new("test.namespace", identifierProperty.Namespace); + yield return new("test.assembly", identifierProperty.AssemblyFullName); + } + + if (testNode.Properties.SingleOrDefault() is { } testLocationProperty) + { + yield return new("test.file.path", testLocationProperty.FilePath); + yield return new("test.line.start", testLocationProperty.LineSpan.Start.Line); + yield return new("test.line.end", testLocationProperty.LineSpan.End.Line); + } + + foreach (TestMetadataProperty metadata in testNode.Properties.OfType()) + { + yield return new KeyValuePair($"test.metadataProperty.{metadata.Key}", metadata.Value); + } + } + + private void HandleTestResult(TestNode testNode, TestNodeStateProperty stateProperty) + { + if (!_testActivities.TryGetValue(testNode.Uid, out IPlatformActivity? activity)) + { + return; + } + + (string result, Exception? exception, TimeSpan? timeoutTime) = stateProperty switch + { + PassedTestNodeStateProperty => ("passed", null, null), + FailedTestNodeStateProperty failed => ("failed", failed.Exception, null), + ErrorTestNodeStateProperty error => ("error", error.Exception, null), + TimeoutTestNodeStateProperty timeout => ("timeout", timeout.Exception, timeout.Timeout), +#pragma warning disable CS0618 // Type or member is obsolete + CancelledTestNodeStateProperty cancelled => ("cancelled", cancelled.Exception, null), +#pragma warning restore CS0618 // Type or member is obsolete + SkippedTestNodeStateProperty => ("skipped", null, null), + _ => ("unknown", null, null), + }; + + activity?.SetTag("test.result", result); + activity?.SetTag("test.result.explanation", stateProperty.Explanation); + if (exception is not null) + { + activity?.SetTag("test.result.exception.type", exception.GetType().FullName); + activity?.SetTag("test.result.exception.message", exception.Message); + activity?.SetTag("test.result.exception.stacktrace", exception.StackTrace); + } + + if (timeoutTime is not null) + { + activity?.SetTag("test.result.timeout.ms", timeoutTime.Value.TotalMilliseconds); + } + + if (testNode.Properties.SingleOrDefault() is { } timingProperty) + { + double totalMilliseconds = timingProperty.GlobalTiming.Duration.TotalMilliseconds; + _totalDuration?.Record(totalMilliseconds); + activity?.SetTag("test.duration.ms", totalMilliseconds); + foreach (StepTimingInfo step in timingProperty.StepTimings) + { + activity?.SetTag($"test.step{step.Id}.duration.ms", step.Timing.Duration.TotalMilliseconds); + activity?.SetTag($"test.step{step.Id}.description", step.Description); + } + } + + foreach (TestMetadataProperty metadataProperty in testNode.Properties.OfType()) + { + activity?.SetTag($"test.metadataProperty.{metadataProperty.Key}", metadataProperty.Value); + } + + activity?.SetTag("test.stdout", string.Join(_environment.NewLine, testNode.Properties.OfType().Select(x => x.StandardOutput))); + activity?.SetTag("test.stderr", string.Join(_environment.NewLine, testNode.Properties.OfType().Select(x => x.StandardError))); + + int index = 0; + foreach (FileArtifactProperty fileArtifactProperty in testNode.Properties.OfType()) + { + activity?.SetTag($"test.artifact.file[{index}].path", fileArtifactProperty.FileInfo.FullName); + index++; + } + + activity?.Dispose(); + _testActivities.Remove(testNode.Uid); + } } diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/IOpenTelemetryProvider.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/IOpenTelemetryProvider.cs new file mode 100644 index 0000000000..e03954c217 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/IOpenTelemetryProvider.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Telemetry; + +/// +/// The OpenTelemetry provider interface. +/// +internal interface IOpenTelemetryProvider : IDisposable +{ +} diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformActivity.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformActivity.cs new file mode 100644 index 0000000000..d346212907 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformActivity.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Telemetry; + +internal interface IPlatformActivity : IDisposable +{ + string? Id { get; } + + IPlatformActivity SetTag(string key, object? value); +} diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformOpenTelemetryService.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformOpenTelemetryService.cs new file mode 100644 index 0000000000..bc031b56a9 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/IPlatformOpenTelemetryService.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Telemetry; + +/// +/// Defines platform-specific services for OpenTelemetry instrumentation, including activity management and metric +/// creation. +/// +/// +/// This interface provides abstractions for starting and managing activities, as well as creating +/// counters and histograms for telemetry data collection. It is intended for internal use ONLY as the platform +/// should be dependency free and netstandard2.0 doesn't have the instrumentation APIs. +/// +internal interface IPlatformOpenTelemetryService : IDisposable +{ + IPlatformActivity? TestFrameworkActivity { get; set; } + + IPlatformActivity? StartActivity([CallerMemberName] string name = "", IEnumerable>? tags = null, string? parentId = null, DateTimeOffset startTime = default); + + ICounter CreateCounter(string name, string? unit = null, string? description = null, IEnumerable>? tags = null) + where T : struct; + + IHistogram CreateHistogram(string name, string? unit = null, string? description = null, IEnumerable>? tags = null) + where T : struct; +} diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/ITelemetryManager.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/ITelemetryManager.cs index 7ed2b3ce56..17c7c6f89e 100644 --- a/src/Platform/Microsoft.Testing.Platform/Telemetry/ITelemetryManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/ITelemetryManager.cs @@ -6,4 +6,6 @@ namespace Microsoft.Testing.Platform.Telemetry; internal interface ITelemetryManager { void AddTelemetryCollectorProvider(Func telemetryCollectorFactory); + + void AddOpenTelemetryProvider(Func openTelemetryProviderFactory); } diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/Meters.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/Meters.cs new file mode 100644 index 0000000000..e23e6d4874 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/Meters.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Telemetry; + +/// +/// Defines a generic interface for a counter that operates on value types. +/// +/// The value type that the counter tracks or manipulates. +internal interface ICounter + where T : struct +{ + /// + /// Adds the specified value to the current total or state represented by the instance. + /// + /// The value to add. The meaning of this value depends on the implementation and type parameter . + void Add(T delta); +} + +/// +/// Defines the contract for a histogram that collects and analyzes values of a specified value type. +/// +/// Implementations of this interface typically provide methods for recording values, retrieving +/// statistical summaries, and analyzing distributions. Histograms are commonly used for tracking metrics such as +/// latencies, frequencies, or other numeric measurements in performance monitoring and data analysis +/// scenarios. +/// The value type of the data points to be recorded in the histogram. Must be a struct. +internal interface IHistogram + where T : struct +{ + /// + /// Records the specified value for later processing or analysis. + /// + /// The value to record. May represent data to be stored, logged, or tracked depending on the implementation. + void Record(T value); +} diff --git a/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryManager.cs b/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryManager.cs index ec9d8b9cc3..60eb74d32b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryManager.cs +++ b/src/Platform/Microsoft.Testing.Platform/Telemetry/TelemetryManager.cs @@ -15,6 +15,7 @@ namespace Microsoft.Testing.Platform.Telemetry; internal sealed class TelemetryManager : ITelemetryManager, IOutputDeviceDataProducer { private Func? _telemetryFactory; + private Func? _openTelemetryProviderFactory; public string Uid => nameof(TelemetryManager); @@ -30,7 +31,18 @@ public void AddTelemetryCollectorProvider(Func BuildAsync(ServiceProvider serviceProvider, ILoggerFactory loggerFactory, TestApplicationOptions testApplicationOptions) + public void AddOpenTelemetryProvider(Func openTelemetryProviderFactory) + { + Guard.NotNull(openTelemetryProviderFactory); + _openTelemetryProviderFactory = openTelemetryProviderFactory; + } + + public IOpenTelemetryProvider? BuildOTelProvider(ServiceProvider serviceProvider) + => _openTelemetryProviderFactory is null + ? null + : _openTelemetryProviderFactory(serviceProvider); + + public async Task BuildTelemetryAsync(ServiceProvider serviceProvider, ILoggerFactory loggerFactory, TestApplicationOptions testApplicationOptions) { bool isTelemetryOptedOut = !testApplicationOptions.EnableTelemetry; diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs index 9613706881..0e28155f40 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/TestApplicationResultTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Testing.Platform.UnitTests; public sealed class TestApplicationResultTests { private readonly TestApplicationResult _testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object); + = new(new Mock().Object, new Mock().Object, new Mock().Object, new Mock().Object, null); [TestMethod] public async Task GetProcessExitCodeAsync_If_All_Skipped_Returns_ZeroTestsRan() @@ -77,7 +77,8 @@ public async Task GetProcessExitCodeAsync_If_Canceled_Returns_TestSessionAborted }); TestApplicationResult testApplicationResult - = new(new Mock().Object, new Mock().Object, new Mock().Object, new StopPoliciesService(testApplicationCancellationTokenSource.Object)); + = new(new Mock().Object, new Mock().Object, new Mock().Object, + new StopPoliciesService(testApplicationCancellationTokenSource.Object), null); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -114,7 +115,9 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.MinimumExpectedTestsOptionKey, ["2"]), - new Mock().Object, new Mock().Object); + new Mock().Object, + new Mock().Object, + null); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -144,7 +147,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object, null); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -164,7 +167,7 @@ TestApplicationResult testApplicationResult = new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.DiscoverTestsOptionKey, []), - new Mock().Object, new Mock().Object); + new Mock().Object, new Mock().Object, null); await testApplicationResult.ConsumeAsync(new DummyProducer(), new TestNodeUpdateMessage( default, @@ -200,12 +203,12 @@ public void GetProcessExitCodeAsync_IgnoreExitCodes(string? argument, int expect new( new Mock().Object, new CommandLineOption(PlatformCommandLineProvider.IgnoreExitCodeOptionKey, argument is null ? [] : [argument]), - new Mock().Object, new Mock().Object), + new Mock().Object, new Mock().Object, null), new( new Mock().Object, new Mock().Object, environment.Object, - new Mock().Object), + new Mock().Object, null), }) { Assert.AreEqual(expectedExitCode, testApplicationResult.GetProcessExitCode()); diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Telemetry/TelemetryManagerTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Telemetry/TelemetryManagerTests.cs index f2adaa2874..5fde9f13cb 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Telemetry/TelemetryManagerTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Telemetry/TelemetryManagerTests.cs @@ -58,7 +58,7 @@ public async Task TelemetryManager_UsingNoLogoShouldSuppressTelemetryMessage(str // Act environmentMock.Setup(e => e.GetEnvironmentVariable(variable)).Returns(value); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); // Assert if (value != "0") @@ -114,7 +114,7 @@ public async Task TelemetryManager_UsingTelemetryOptOutShouldDisableTelemetry(st // Act environmentMock.Setup(e => e.GetEnvironmentVariable(variable)).Returns(value); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); // Assert ITelemetryInformation telemetryInformation = serviceProvider.GetRequiredService(); @@ -167,7 +167,7 @@ public async Task TelemetryManager_SentinelIsWrittenPerUserAndAvoidsShowingNotic telemetryManager.AddTelemetryCollectorProvider(_ => new NopTelemetryService(false)); // Act - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); // Assert ITelemetryInformation telemetryInformation = serviceProvider.GetRequiredService(); @@ -189,7 +189,7 @@ public async Task TelemetryManager_SentinelIsWrittenPerUserAndAvoidsShowingNotic fileSystemMock.Invocations.Clear(); fileSystemMock.Setup(f => f.ExistFile(path)).Returns(true); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); fileSystemMock.Verify(f => f.ExistFile(path), Times.Once); // Message is not written to screen. @@ -235,7 +235,7 @@ public async Task TelemetryManager_SentinelIsWrittenOnlyWhenUserWouldSeeTheMessa // Act // Disable showing the telemetry message. environmentMock.Setup(s => s.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_NOBANNER)).Returns("1"); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); // Assert ITelemetryInformation telemetryInformation = serviceProvider.GetRequiredService(); @@ -262,7 +262,7 @@ public async Task TelemetryManager_SentinelIsWrittenOnlyWhenUserWouldSeeTheMessa environmentMock.Setup(s => s.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_NOBANNER)).Returns("0"); fileSystemMock.Setup(f => f.ExistFile(path)).Returns(false); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); fileSystemMock.Verify(f => f.ExistFile(path), Times.Once); // Message is written to screen. @@ -300,7 +300,7 @@ public async Task TelemetryManager_UsingNoBannerCommandLine_ShouldSuppressTeleme loggerFactoryMock.Setup(f => f.CreateLogger(It.IsAny())).Returns(new Mock().Object); TelemetryManager telemetryManager = new(); - await telemetryManager.BuildAsync(serviceProvider, loggerFactoryMock.Object, options); + await telemetryManager.BuildTelemetryAsync(serviceProvider, loggerFactoryMock.Object, options); outputDevice.Verify(c => c.DisplayAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); }