From 8ec0c034a0f8155d251de6cd4b2f734ac65c2150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 18 Feb 2022 14:53:04 +0100 Subject: [PATCH] Enable parallel discovery (#3349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Medeni Baykal <433724+Haplois@users.noreply.github.com> Co-authored-by: Sanan Yuzbashiyev Co-authored-by: Amaury Levé --- .../DesignMode/DesignModeClient.cs | 2 +- .../Discovery/DiscoveryRequest.cs | 15 +- .../RequestHelper/ITestRequestManager.cs | 10 +- .../Engine/ClientProtocol/DiscoveryStatus.cs | 25 +++ .../IParallelProxyDiscoveryManager.cs | 5 + .../ClientProtocol/IProxyDiscoveryManager.cs | 6 + .../TesthostProtocol/IDiscoveryManager.cs | 6 + .../PublicAPI/PublicAPI.Unshipped.txt | 7 + .../Interfaces/ITestRequestSender.cs | 5 + .../JsonDataSerializer.cs | 15 +- .../Messages/DiscoveryCompletePayload.cs | 15 ++ .../PublicAPI/PublicAPI.Unshipped.txt | 8 + .../TestRequestSender.cs | 22 ++- .../Client/InProcessProxyDiscoveryManager.cs | 6 + .../ParallelDiscoveryDataAggregator.cs | 37 ++++ .../ParallelDiscoveryEventsHandler.cs | 96 ++++++++-- .../Parallel/ParallelProxyDiscoveryManager.cs | 32 ++-- .../Client/ProxyDiscoveryManager.cs | 28 ++- .../Discovery/DiscoveryManager.cs | 167 +++++++++++++++++- .../EventHandlers/TestRequestHandler.cs | 13 +- .../PublicAPI/PublicAPI.Unshipped.txt | 2 + .../Events/DiscoveryCompleteEventArgs.cs | 37 +++- .../Constants.cs | 7 +- .../PublicAPI/PublicAPI.Unshipped.txt | 34 ++-- .../PublicAPI/net/PublicAPI.Unshipped.txt | 2 +- .../VsTestConsoleRequestSender.cs | 16 +- .../TestPlatformHelpers/TestRequestManager.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 23 ++- .../EventHandler/DiscoveryEventHandler.cs | 7 + .../DesignMode/DesignModeClientTests.cs | 4 +- .../Discovery/DiscoveryRequestTests.cs | 23 ++- .../ParallelDiscoveryDataAggregatorTests.cs | 38 ++++ .../ParallelDiscoveryEventsHandlerTests.cs | 23 ++- .../ParallelProxyDiscoveryManagerTests.cs | 46 +++-- .../Discovery/DiscoveryManagerTests.cs | 23 +++ .../VsTestConsoleRequestSenderTests.cs | 86 ++++++++- .../TestRequestManagerTests.cs | 32 ++-- 37 files changed, 811 insertions(+), 114 deletions(-) create mode 100644 src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 975cc1f2fd..b9ee18d3fc 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -38,7 +38,7 @@ public class DesignModeClient : IDesignModeClient private readonly ICommunicationManager _communicationManager; private readonly IDataSerializer _dataSerializer; - private readonly ProtocolConfig _protocolConfig = ObjectModel.Constants.DefaultProtocolConfig; + private readonly ProtocolConfig _protocolConfig = Constants.DefaultProtocolConfig; private readonly IEnvironment _platformEnvironment; private readonly TestSessionMessageLogger _testSessionMessageLogger; private readonly object _lockObject = new(); diff --git a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs index bab03622ff..5f3b16e855 100644 --- a/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs +++ b/src/Microsoft.TestPlatform.Client/Discovery/DiscoveryRequest.cs @@ -70,7 +70,7 @@ public void DiscoverAsync() { if (_disposed) { - throw new ObjectDisposedException("DiscoveryRequest"); + throw new ObjectDisposedException(nameof(DiscoveryRequest)); } // Reset the discovery completion event @@ -112,12 +112,21 @@ public void Abort() { if (_disposed) { - throw new ObjectDisposedException("DiscoveryRequest"); + throw new ObjectDisposedException(nameof(DiscoveryRequest)); } if (DiscoveryInProgress) { - DiscoveryManager.Abort(); + // If testhost has old version, we should use old cancel logic + // to be consistent and not create regression issues + if (Constants.DefaultProtocolConfig.Version < Constants.MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport) + { + DiscoveryManager.Abort(); + } + else + { + DiscoveryManager.Abort(this); + } } else { diff --git a/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs b/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs index 1ab92d29d9..adad3c06ad 100644 --- a/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs +++ b/src/Microsoft.TestPlatform.Client/RequestHelper/ITestRequestManager.cs @@ -22,7 +22,7 @@ public interface ITestRequestManager : IDisposable /// /// Initializes the extensions while probing additional paths. /// - /// + /// /// Paths to additional extensions. /// Skip extension filtering by name if true. void InitializeExtensions( @@ -37,7 +37,7 @@ void InitializeExtensions( /// /// Discovers tests given a list of sources and some run settings. /// - /// + /// /// Discovery payload. /// Discovery events registrar. /// Protocol related information. @@ -49,7 +49,7 @@ void DiscoverTests( /// /// Runs tests given a list of sources and some run settings. /// - /// + /// /// Test run request payload. /// Custom test host launcher for the run. /// Run events registrar. @@ -63,7 +63,7 @@ void RunTests( /// /// Processes test run attachments. /// - /// + /// /// /// Test run attachments processing payload. /// @@ -79,7 +79,7 @@ void ProcessTestRunAttachments( /// /// Starts a test session. /// - /// + /// /// The start test session payload. /// The custom test host launcher. /// The events handler. diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs new file mode 100644 index 0000000000..337b4139ff --- /dev/null +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + +/// +/// Enums for indicating discovery status of source +/// +public enum DiscoveryStatus +{ + /// + /// Indicates the sources which were not touched during discovery. + /// + NotDiscovered, + + /// + /// Indicates that we started discovery of the source but something happened (cancel/abort) and we stopped processing it. + /// + PartiallyDiscovered, + + /// + /// Indicates that source was fully discovered. + /// + FullyDiscovered, +} diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs index e65927baae..cbd40b93a0 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs @@ -12,6 +12,11 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; /// public interface IParallelProxyDiscoveryManager : IParallelOperationManager, IProxyDiscoveryManager { + /// + /// Indicates if user requested an abortion + /// + bool IsAbortRequested { get; } + /// /// Handles Partial Discovery Complete event coming from a specific concurrent proxy discovery manager /// Each concurrent proxy discovery manager will signal the parallel discovery manager when its complete diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs index 28db48338e..286fb36ce5 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IProxyDiscoveryManager.cs @@ -30,6 +30,12 @@ public interface IProxyDiscoveryManager /// void Abort(); + /// + /// Aborts discovery operation with EventHandler. + /// + /// EventHandler for handling discovery events from Engine + void Abort(ITestDiscoveryEventsHandler2 eventHandler); + /// /// Closes the current test operation. /// Send a EndSession message to close the test host and channel gracefully. diff --git a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs index a3a2c7c591..9385882eef 100644 --- a/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.Common/Interfaces/Engine/TesthostProtocol/IDiscoveryManager.cs @@ -31,4 +31,10 @@ public interface IDiscoveryManager /// Aborts the test discovery. /// void Abort(); + + /// + /// Aborts the test discovery with eventHandler. + /// + /// EventHandler for handling discovery events from Engine + void Abort(ITestDiscoveryEventsHandler2 eventHandler); } diff --git a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt index 524e7f82bb..173c7c1cd4 100644 --- a/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Unshipped.txt @@ -6,3 +6,10 @@ static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstan static Microsoft.VisualStudio.TestPlatform.Common.Telemetry.TelemetryDataConstants.StopTestSessionCompleteEvent -> string Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager.StartSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager.StopSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelProxyDiscoveryManager.IsAbortRequested.get -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.NotDiscovered = 0 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.PartiallyDiscovered = 1 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus +Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs index 6af429b1b8..7b4667997f 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/ITestRequestSender.cs @@ -89,6 +89,11 @@ public interface ITestRequestSender : IDisposable /// void SendTestRunAbort(); + /// + /// Sends the request to abort the discovery. + /// + void SendDiscoveryAbort(); + /// /// Handle client process exit /// diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs index 28a7fb72fc..c3c5413ca7 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs @@ -208,6 +208,15 @@ private JsonSerializer GetPayloadSerializer(int? version) version = 1; } + // 0: the original protocol with no versioning (Message). It is used during negotiation. + // 1: new protocol with versioning (VersionedMessage). + // 2: changed serialization because the serialization of properties in bag was too verbose, + // so common properties are considered built-in and serialized without type info. + // 3: introduced because of changes to allow attaching debugger to external process. + // 4: introduced because 3 did not update this table and ended up using the serializer for protocol v1, + // which is extremely slow. We negotiate 2 or 4, but never 3 unless the flag above is set. + // 5: ??? + // 6: accepts abort and cancel with handlers that report the status. return version switch { // 0 is used during negotiation. @@ -216,9 +225,9 @@ private JsonSerializer GetPayloadSerializer(int? version) // unless this is disabled by VSTEST_DISABLE_PROTOCOL_3_VERSION_DOWNGRADE // env variable. 0 or 1 or 3 => s_payloadSerializer, - 2 or 4 or 5 => s_payloadSerializer2, - _ => throw new NotSupportedException($"Protocol version {version} is not supported. " + - "Ensure it is compatible with the latest serializer or add a new one."), + 2 or 4 or 5 or 6 => s_payloadSerializer2, + _ => throw new NotSupportedException($"Protocol version {version} is not supported. " + + "Ensure it is compatible with the latest serializer or add a new one."), }; } } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs index 8f42162568..6d1ecfbea5 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs @@ -33,4 +33,19 @@ public class DiscoveryCompletePayload /// Gets or sets the Metrics /// public IDictionary Metrics { get; set; } + + /// + /// Gets or sets list of sources which were fully discovered. + /// + public IList FullyDiscoveredSources { get; set; } = new List(); + + /// + /// Gets or sets list of sources which were partially discovered (started discover tests, but then discovery aborted). + /// + public IList PartiallyDiscoveredSources { get; set; } = new List(); + + /// + /// Gets or sets list of sources which were not discovered at all. + /// + public IList NotDiscoveredSources { get; set; } = new List(); } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt index e69de29bb2..47a461296e 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,8 @@ +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.SendDiscoveryAbort() -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.SendDiscoveryAbort() -> void diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs index 142ebfb97d..ff818b46f5 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs @@ -60,7 +60,7 @@ public class TestRequestSender : ITestRequestSender // Must be in sync with the highest supported version in // src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs file. - private readonly int _highestSupportedVersion = 5; + private readonly int _highestSupportedVersion = 6; private readonly TestHostConnectionInfo _connectionInfo; @@ -276,6 +276,19 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve _channel.Send(message); } + + /// + public void SendDiscoveryAbort() + { + if (IsOperationComplete()) + { + EqtTrace.Verbose("TestRequestSender.SendDiscoveryAbort: Operation is already complete. Skip error message."); + return; + } + + EqtTrace.Verbose("TestRequestSender.SendDiscoveryAbort: Sending discovery abort."); + _channel?.Send(_dataSerializer.SerializeMessage(MessageType.CancelDiscovery)); + } #endregion #region Execution Protocol @@ -543,7 +556,12 @@ private void OnDiscoveryMessageReceived(ITestDiscoveryEventsHandler2 discoveryEv case MessageType.DiscoveryComplete: var discoveryCompletePayload = _dataSerializer.DeserializePayload(data); - var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(discoveryCompletePayload.TotalTests, discoveryCompletePayload.IsAborted); + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( + discoveryCompletePayload.TotalTests, + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; discoveryEventsHandler.HandleDiscoveryComplete( discoveryCompleteEventArgs, diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs index a930d71912..d11f04b25a 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/InProcessProxyDiscoveryManager.cs @@ -97,6 +97,12 @@ public void Abort() Task.Run(() => _testHostManagerFactory.GetDiscoveryManager().Abort()); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + Task.Run(() => _testHostManagerFactory.GetDiscoveryManager().Abort(eventHandler)); + } + private void InitializeExtensions(IEnumerable sources) { var extensionsFromSource = _testHostManager.GetTestPlatformExtensions(sources, Enumerable.Empty()); diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs index e68a4a47db..bd257cb56e 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryDataAggregator.cs @@ -7,6 +7,9 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using Common.Telemetry; +using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; + using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -37,6 +40,17 @@ public ParallelDiscoveryDataAggregator() /// public long TotalTests { get; private set; } + + /// + /// Dictionary which stores source with corresponding discoveryStatus + /// + private readonly ConcurrentDictionary _sourcesWithDiscoveryStatus = new(); + + /// + /// Indicates if discovery complete payload already sent back to IDE + /// + internal bool IsMessageSent { get; private set; } + /// /// Returns the Aggregated Metrics. /// @@ -120,4 +134,27 @@ public void AggregateDiscoveryDataMetrics(IDictionary metrics) } } + /// + /// Aggregate the source as fully discovered + /// + /// Fully discovered source + public void MarkSourcesWithStatus(ICollection sources, DiscoveryStatus status) + => DiscoveryManager.MarkSourcesWithStatus(sources, status, _sourcesWithDiscoveryStatus); + + /// + /// Aggregates the value indicating if we already sent message to IDE. + /// + /// Boolean value if we already sent message to IDE + public void AggregateIsMessageSent(bool isMessageSent) + { + IsMessageSent = IsMessageSent || isMessageSent; + } + + /// + /// Returns sources with particular discovery status. + /// + /// Status to filter + /// + public List GetSourcesWithStatus(DiscoveryStatus status) + => DiscoveryManager.GetSourcesWithStatus(status, _sourcesWithDiscoveryStatus); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs index 505d213f32..8faa60f75c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelDiscoveryEventsHandler.cs @@ -6,6 +6,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; using System.Collections.Generic; +using System.Linq; using Common.Telemetry; using CommunicationUtilities; @@ -35,6 +36,8 @@ internal class ParallelDiscoveryEventsHandler : ITestDiscoveryEventsHandler2 private readonly IRequestData _requestData; + private readonly object _sendMessageLock = new(); + public ParallelDiscoveryEventsHandler(IRequestData requestData, IProxyDiscoveryManager proxyDiscoveryManager, ITestDiscoveryEventsHandler2 actualDiscoveryEventsHandler, @@ -65,6 +68,12 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete var totalTests = discoveryCompleteEventArgs.TotalCount; var isAborted = discoveryCompleteEventArgs.IsAborted; + // Aggregate for final discovery complete + _discoveryDataAggregator.Aggregate(totalTests, isAborted); + + // Aggregate Discovery Data Metrics + _discoveryDataAggregator.AggregateDiscoveryDataMetrics(discoveryCompleteEventArgs.Metrics); + // we get discovery complete events from each host process // so we cannot "complete" the actual operation until all sources are consumed // We should not block last chunk results while we aggregate overall discovery data @@ -72,14 +81,13 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete { ConvertToRawMessageAndSend(MessageType.TestCasesFound, lastChunk); HandleDiscoveredTests(lastChunk); + // If we come here it means that some source was already fully discovered so we can mark it + if (!_discoveryDataAggregator.IsAborted) + { + AggregateComingSourcesAsFullyDiscovered(lastChunk, discoveryCompleteEventArgs); + } } - // Aggregate for final discovery complete - _discoveryDataAggregator.Aggregate(totalTests, isAborted); - - // Aggregate Discovery Data Metrics - _discoveryDataAggregator.AggregateDiscoveryDataMetrics(discoveryCompleteEventArgs.Metrics); - // Do not send TestDiscoveryComplete to actual test discovery handler // We need to see if there are still sources left - let the parallel manager decide var parallelDiscoveryComplete = _parallelProxyDiscoveryManager.HandlePartialDiscoveryComplete( @@ -90,13 +98,27 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete if (parallelDiscoveryComplete) { + var fullyDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered); + var partiallyDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered); + var notDiscovered = _discoveryDataAggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered); + + // As we immediately return results to IDE in case of aborting + // we need to set isAborted = true and totalTests = -1 + if (_parallelProxyDiscoveryManager.IsAbortRequested) + { + _discoveryDataAggregator.Aggregate(-1, true); + } + // In case of sequential discovery - RawMessage would have contained a 'DiscoveryCompletePayload' object // To send a raw message - we need to create raw message from an aggregated payload object var testDiscoveryCompletePayload = new DiscoveryCompletePayload() { TotalTests = _discoveryDataAggregator.TotalTests, IsAborted = _discoveryDataAggregator.IsAborted, - LastDiscoveredTests = null + LastDiscoveredTests = null, + FullyDiscoveredSources = fullyDiscovered, + PartiallyDiscoveredSources = partiallyDiscovered, + NotDiscoveredSources = notDiscovered }; // Collecting Final Discovery State @@ -106,11 +128,16 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete var aggregatedDiscoveryDataMetrics = _discoveryDataAggregator.GetAggregatedDiscoveryDataMetrics(); testDiscoveryCompletePayload.Metrics = aggregatedDiscoveryDataMetrics; - // we have to send raw messages as we block the discovery complete actual raw messages - ConvertToRawMessageAndSend(MessageType.DiscoveryComplete, testDiscoveryCompletePayload); + // Sending discovery complete message to IDE + ConvertToRawMessageAndSend(testDiscoveryCompletePayload); + + var finalDiscoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( + _discoveryDataAggregator.TotalTests, + _discoveryDataAggregator.IsAborted, + fullyDiscovered, + partiallyDiscovered, + notDiscovered); - var finalDiscoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(_discoveryDataAggregator.TotalTests, - _discoveryDataAggregator.IsAborted); finalDiscoveryCompleteEventArgs.Metrics = aggregatedDiscoveryDataMetrics; // send actual test discovery complete to clients @@ -162,4 +189,51 @@ private void ConvertToRawMessageAndSend(string messageType, object payload) var rawMessage = _dataSerializer.SerializePayload(messageType, payload); _actualDiscoveryEventsHandler.HandleRawMessage(rawMessage); } + + /// + /// Sending discovery complete message to IDE + /// + /// Discovery aggregator to know if we already sent this message + /// Discovery complete payload to send + private void ConvertToRawMessageAndSend(DiscoveryCompletePayload testDiscoveryCompletePayload) + { + // When we abort we should send raw message to IDE only once. + // All other testhosts which will finish after shouldn't send abort raw message. + if (_discoveryDataAggregator.IsMessageSent) + { + return; + } + + lock (_sendMessageLock) + { + if (!_discoveryDataAggregator.IsMessageSent) + { + // we have to send raw messages as we block the discovery complete actual raw messages + ConvertToRawMessageAndSend(MessageType.DiscoveryComplete, testDiscoveryCompletePayload); + _discoveryDataAggregator.AggregateIsMessageSent(true); + } + } + } + + /// + /// Aggregate source as fully discovered + /// + /// Aggregator to aggregate results + /// Last chunk of discovered test cases + private void AggregateComingSourcesAsFullyDiscovered(IEnumerable lastChunk, DiscoveryCompleteEventArgs discoveryCompleteEventArgs) + { + if (lastChunk is null) + { + return; + } + + // Sometimes we get lastChunk as empty list (when number of tests in project dividable by + // the chunk size, e.g. 100 tests and 10 chunk size). + // Then we will take sources from discoveryCompleteEventArgs coming from testhost. + var lastChunkSources = !lastChunk.Any() + ? discoveryCompleteEventArgs.FullyDiscoveredSources + : lastChunk.Select(testcase => testcase.Source).ToList(); + + _discoveryDataAggregator.MarkSourcesWithStatus(lastChunkSources, DiscoveryStatus.FullyDiscovered); + } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs index 7ad7a42ed0..114e6b0c44 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs @@ -25,8 +25,6 @@ internal class ParallelProxyDiscoveryManager : ParallelOperationManager /// LockObject to update discovery status in parallel /// private readonly object _discoveryStatusLockObject = new(); - #endregion - public ParallelProxyDiscoveryManager(IRequestData requestData, Func actualProxyManagerCreator, int parallelLevel, bool sharedHosts) : this(requestData, actualProxyManagerCreator, JsonDataSerializer.Instance, parallelLevel, sharedHosts) { @@ -85,16 +76,30 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve _availableTestSources = discoveryCriteria.Sources.Count(); EqtTrace.Verbose("ParallelProxyDiscoveryManager: Start discovery. Total sources: " + _availableTestSources); + + // One data aggregator per parallel discovery + _currentDiscoveryDataAggregator = new ParallelDiscoveryDataAggregator(); + + // Marking all sources as not discovered before starting actual discovery + _currentDiscoveryDataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources.ToList(), DiscoveryStatus.NotDiscovered); + DiscoverTestsPrivate(eventHandler); } /// public void Abort() { - _discoveryAbortRequested = true; + IsAbortRequested = true; DoActionOnAllManagers((proxyManager) => proxyManager.Abort(), doActionsInParallel: true); } + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + IsAbortRequested = true; + DoActionOnAllManagers((proxyManager) => proxyManager.Abort(eventHandler), doActionsInParallel: true); + } + /// public void Close() { @@ -127,7 +132,7 @@ If discovery is complete or discovery aborting was requsted by testPlatfrom(user when testhost crashed by itself and when user requested it (f.e. through TW) Schedule the clean up for managers and handlers. */ - if (allDiscoverersCompleted || _discoveryAbortRequested) + if (allDiscoverersCompleted || IsAbortRequested) { // Reset enumerators _sourceEnumerator = null; @@ -175,9 +180,6 @@ private void DiscoverTestsPrivate(ITestDiscoveryEventsHandler2 discoveryEventsHa // Reset the discovery complete data _discoveryCompletedClients = 0; - // One data aggregator per parallel discovery - _currentDiscoveryDataAggregator = new ParallelDiscoveryDataAggregator(); - foreach (var concurrentManager in GetConcurrentManagerInstances()) { var parallelEventsHandler = new ParallelDiscoveryEventsHandler( diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs index b254d2e55a..a328d53dfb 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/ProxyDiscoveryManager.cs @@ -44,7 +44,7 @@ public class ProxyDiscoveryManager : IProxyDiscoveryManager, IBaseProxy, ITestDi /// /// Initializes a new instance of the class. /// - /// + /// /// The test session info. /// The proxy operation manager creator. public ProxyDiscoveryManager( @@ -65,7 +65,7 @@ public ProxyDiscoveryManager( /// /// Initializes a new instance of the class. /// - /// + /// /// /// The request data for providing discovery services and data. /// @@ -86,11 +86,11 @@ public ProxyDiscoveryManager( /// /// Initializes a new instance of the class. /// - /// + /// /// /// Constructor with dependency injection. Used for unit testing. /// - /// + /// /// /// The request data for providing discovery services and data. /// @@ -194,6 +194,26 @@ public void Abort() Close(); } + // + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + // Do nothing if the proxy is not initialized yet. + if (_proxyOperationManager is null) + { + return; + } + + if (_baseTestDiscoveryEventsHandler is null) + { + _baseTestDiscoveryEventsHandler = eventHandler; + } + + if (_isCommunicationEstablished) + { + _proxyOperationManager.RequestSender.SendDiscoveryAbort(); + } + } + /// public void Close() { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs index e12d234986..33796d34f7 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Discovery/DiscoveryManager.cs @@ -6,6 +6,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -23,6 +24,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; using ObjectModel.Engine.TesthostProtocol; using ObjectModel.Logging; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using CrossPlatEngineResources = Resources.Resources; @@ -37,11 +39,14 @@ public class DiscoveryManager : IDiscoveryManager private ITestDiscoveryEventsHandler2 _testDiscoveryEventsHandler; private DiscoveryCriteria _discoveryCriteria; private readonly CancellationTokenSource _cancellationTokenSource = new(); + private string _previousSource; + private readonly ConcurrentDictionary _sourcesWithDiscoveryStatus = new(); /// /// Initializes a new instance of the class. /// - public DiscoveryManager(IRequestData requestData) : this(requestData, TestPlatformEventSource.Instance) + public DiscoveryManager(IRequestData requestData) + : this(requestData, TestPlatformEventSource.Instance) { } @@ -104,13 +109,14 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve foreach (var kvp in discoveryCriteria.AdapterSourceMap) { var verifiedSources = GetValidSources(kvp.Value, _sessionMessageLogger, discoveryCriteria.Package); - if (verifiedSources.Any()) + if (verifiedSources.Count > 0) { verifiedExtensionSourceMap.Add(kvp.Key, kvp.Value); + // Mark all sources as NotDiscovered before actual discovery starts + MarkSourcesWithStatus(verifiedSources, DiscoveryStatus.NotDiscovered, _sourcesWithDiscoveryStatus); } } - // If there are sources to discover if (verifiedExtensionSourceMap.Any()) { @@ -136,6 +142,9 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve if (lastChunk != null) { UpdateTestCases(lastChunk, _discoveryCriteria.Package); + // When discovery is complete then the last discovered source is still marked + // as partially discovered, so we need to mark it as fully discovered. + MarkTheLastChunkSourcesAsFullyDiscovered(lastChunk); } // Collecting Discovery State @@ -143,10 +152,20 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve // Collecting Total Tests Discovered _requestData.MetricsCollection.Add(TelemetryDataConstants.TotalTestsDiscovered, totalDiscoveredTestCount); - var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(totalDiscoveredTestCount, false) + + if (_cancellationTokenSource.IsCancellationRequested) { - Metrics = _requestData.MetricsCollection.Metrics - }; + totalDiscoveredTestCount = -1; + } + + var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs( + totalDiscoveredTestCount, + _cancellationTokenSource.IsCancellationRequested, + GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered, _sourcesWithDiscoveryStatus), + GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered, _sourcesWithDiscoveryStatus), + GetSourcesWithStatus(DiscoveryStatus.NotDiscovered, _sourcesWithDiscoveryStatus)); + + discoveryCompleteEventsArgs.Metrics = _requestData.MetricsCollection.Metrics; eventHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, lastChunk); } @@ -170,13 +189,32 @@ public void Abort() _cancellationTokenSource.Cancel(); } - private void OnReportTestCases(IEnumerable testCases) + /// + public void Abort(ITestDiscoveryEventsHandler2 eventHandler) + { + if (!_cancellationTokenSource.IsCancellationRequested) + { + Abort(); + } + + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( + -1, true, + GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered, _sourcesWithDiscoveryStatus), + GetSourcesWithStatus(DiscoveryStatus.PartiallyDiscovered, _sourcesWithDiscoveryStatus), + GetSourcesWithStatus(DiscoveryStatus.NotDiscovered, _sourcesWithDiscoveryStatus)); + + eventHandler.HandleDiscoveryComplete(discoveryCompleteEventArgs, null); + } + + private void OnReportTestCases(ICollection testCases) { UpdateTestCases(testCases, _discoveryCriteria.Package); if (_testDiscoveryEventsHandler != null) { _testDiscoveryEventsHandler.HandleDiscoveredTests(testCases); + // We need to mark sources based on already discovered testcases + MarkSourcesBasedOnDiscoveredTestCases(testCases); } else { @@ -191,7 +229,7 @@ private void OnReportTestCases(IEnumerable testCases) /// logger /// package /// The list of verified sources. - internal static IEnumerable GetValidSources(IEnumerable sources, IMessageLogger logger, string package) + internal static HashSet GetValidSources(IEnumerable sources, IMessageLogger logger, string package) { Debug.Assert(sources != null, "sources"); var verifiedSources = new HashSet(StringComparer.OrdinalIgnoreCase); @@ -235,7 +273,7 @@ void SendWarning() } // No valid source is found => we cannot discover. - if (!verifiedSources.Any()) + if (verifiedSources.Count == 0) { var sourcesString = string.Join(",", sources.ToArray()); var errorMessage = string.Format(CultureInfo.CurrentCulture, CrossPlatEngineResources.NoValidSourceFoundForDiscovery, sourcesString); @@ -284,4 +322,115 @@ private static void UpdateTestCases(IEnumerable testCases, string pack } } } + + /// + /// Mark sources based on already discovered testCases + /// + /// List of testCases which were already discovered + private void MarkSourcesBasedOnDiscoveredTestCases(ICollection testCases) + { + if (testCases is null) + { + return; + } + + foreach (var testCase in testCases) + { + string currentSource = testCase.Source; + + // If it is the first list of testCases which was discovered or if current source + // is the same as previous we mark them as partially discovered. + if (_previousSource is null || _previousSource == currentSource) + { + MarkSourceWithStatus(currentSource, DiscoveryStatus.PartiallyDiscovered); + } + // If source is changed, we need to mark previous source as already fully discovered + // and currentSource as partially discovered. + else if (currentSource != _previousSource) + { + MarkSourceWithStatus(_previousSource, DiscoveryStatus.FullyDiscovered); + MarkSourceWithStatus(currentSource, DiscoveryStatus.PartiallyDiscovered); + } + + _previousSource = currentSource; + } + } + + /// + /// Mark the last sources as fullyDiscovered + /// + /// Last chunk of testCases which were discovered + private void MarkTheLastChunkSourcesAsFullyDiscovered(IList lastChunk) + { + // When all testcases in project is dividable by 10 then lastChunk is coming as empty + // So we need to take the lastSource and mark it as FullyDiscovered. + var lastChunkSources = lastChunk.Count > 0 + ? lastChunk.Select(testcase => testcase.Source).ToList() + : new List() { _previousSource }; + + MarkSourcesWithStatus(lastChunkSources, DiscoveryStatus.FullyDiscovered, _sourcesWithDiscoveryStatus); + } + + /// + /// Mark the source with particular DiscoveryStatus + /// + /// Sources to mark + /// DiscoveryStatus to mark for source + private void MarkSourceWithStatus(string source, DiscoveryStatus status) + => MarkSourcesWithStatus(new[] { source }, status, _sourcesWithDiscoveryStatus); + + /// + /// Mark sources with particular DiscoveryStatus + /// + /// List of sources to mark + /// DiscoveryStatus to mark for list of sources + internal static void MarkSourcesWithStatus(ICollection sources, DiscoveryStatus status, + ConcurrentDictionary sourcesWithDiscoveryStatus) + { + if (sources is null) + { + return; + } + + foreach (var source in sources) + { + if (source is null) + { + continue; + } + + sourcesWithDiscoveryStatus.AddOrUpdate(source, + _ => + { + if (status != DiscoveryStatus.NotDiscovered) + { + EqtTrace.Warning($"Undiscovered {source}."); + } + + return status; + }, + (_, _) => + { + EqtTrace.Info($"Marking {source} with {status} status."); + return status; + }); + } + } + + /// + /// Returns sources with particular discovery status. + /// + /// Status to filter + /// + internal static List GetSourcesWithStatus(DiscoveryStatus discoveryStatus, + ConcurrentDictionary sourcesWithDiscoveryStatus) + { + // If by some accident SourcesWithDiscoveryStatus map is empty we will return empty list + return sourcesWithDiscoveryStatus is null || sourcesWithDiscoveryStatus.IsEmpty + ? new List() + : sourcesWithDiscoveryStatus + .Where(source => source.Value == discoveryStatus) + .Select(source => source.Key) + .ToList(); + } } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs index 8a2b4f358b..461c405002 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/EventHandlers/TestRequestHandler.cs @@ -30,7 +30,7 @@ public class TestRequestHandler : ITestRequestHandler // Must be in sync with the highest supported version in // src/Microsoft.TestPlatform.CommunicationUtilities/TestRequestSender.cs file. - private readonly int _highestSupportedVersion = 5; + private readonly int _highestSupportedVersion = 6; private readonly IDataSerializer _dataSerializer; private ITestHostManagerFactory _testHostManagerFactory; @@ -207,7 +207,10 @@ public void DiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventA TotalTests = discoveryCompleteEventArgs.TotalCount, LastDiscoveredTests = discoveryCompleteEventArgs.IsAborted ? null : lastChunk, IsAborted = discoveryCompleteEventArgs.IsAborted, - Metrics = discoveryCompleteEventArgs.Metrics + Metrics = discoveryCompleteEventArgs.Metrics, + FullyDiscoveredSources = discoveryCompleteEventArgs.FullyDiscoveredSources, + PartiallyDiscoveredSources = discoveryCompleteEventArgs.PartiallyDiscoveredSources, + NotDiscoveredSources = discoveryCompleteEventArgs.NotDiscoveredSources, }, _protocolVersion); SendData(data); @@ -478,6 +481,12 @@ public void OnMessageReceived(object sender, MessageReceivedEventArgs messageRec _onAttachDebuggerAckRecieved?.Invoke(message); break; + case MessageType.CancelDiscovery: + _jobQueue.Pause(); + _testHostManagerFactoryReady.Wait(); + _testHostManagerFactory.GetDiscoveryManager().Abort(new TestDiscoveryEventHandler(this)); + break; + case MessageType.AbortTestRun: try { diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt index f4bc1a7491..5fd225b90b 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/PublicAPI/PublicAPI.Unshipped.txt @@ -1,3 +1,5 @@ virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.TestSessionPool.KillSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.StartSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool virtual Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.ProxyTestSessionManager.StopSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData) -> bool +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.ProxyDiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void +Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Discovery.DiscoveryManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestDiscoveryEventsHandler2 eventHandler) -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs index 11e21a786b..ef1df982a6 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Client/Events/DiscoveryCompleteEventArgs.cs @@ -19,7 +19,13 @@ public class DiscoveryCompleteEventArgs : EventArgs /// /// Total tests which got discovered /// Specifies if discovery has been aborted. - public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) + /// List of fully discovered sources + /// List of partially discovered sources + /// List of not discovered sources + public DiscoveryCompleteEventArgs(long totalTests, bool isAborted, + IList fullyDiscoveredSources, + IList partiallyDiscoveredSources, + IList notDiscoveredSources) { // This event is always raised from the client side, while the total count of tests is maintained // only at the testhost end. In case of a discovery abort (various reasons including crash), it is @@ -28,6 +34,20 @@ public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) TotalCount = totalTests; IsAborted = isAborted; + + FullyDiscoveredSources = fullyDiscoveredSources ?? new List(); + PartiallyDiscoveredSources = partiallyDiscoveredSources ?? new List(); + NotDiscoveredSources = notDiscoveredSources ?? new List(); + } + + /// + /// Constructor for creating event args object + /// + /// Total tests which got discovered + /// Specifies if discovery has been aborted. + public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) + : this(totalTests, isAborted, null, null, null) + { } /// @@ -44,4 +64,19 @@ public DiscoveryCompleteEventArgs(long totalTests, bool isAborted) /// Metrics /// public IDictionary Metrics { get; set; } + + /// + /// Gets the list of sources which were fully discovered. + /// + public IList FullyDiscoveredSources { get; set; } + + /// + /// Gets the list of sources which were partially discovered (started discover tests, but then discovery aborted). + /// + public IList PartiallyDiscoveredSources { get; set; } + + /// + /// Gets the list of sources which were not discovered at all. + /// + public IList NotDiscoveredSources { get; set; } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs index 0d667b488d..ad14ac27c5 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs @@ -175,13 +175,18 @@ public static class Constants /// /// The default protocol version /// - public static readonly ProtocolConfig DefaultProtocolConfig = new() { Version = 5 }; + public static readonly ProtocolConfig DefaultProtocolConfig = new() { Version = 6 }; /// /// The minimum protocol version that has debug support /// public const int MinimumProtocolVersionWithDebugSupport = 3; + /// + /// The minimum protocol version that has debug support + /// + public const int MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport = 6; + /// /// Name of the results directory /// diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt index 67f8a4e997..da26b150f1 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Unshipped.txt @@ -1,9 +1,29 @@ -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.DiscoveryCriteria(System.Collections.Generic.IEnumerable sources, long frequencyOfDiscoveredTestsEvent, System.TimeSpan discoveredTestEventTimeout, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo) -> void +const Microsoft.VisualStudio.TestPlatform.ObjectModel.Constants.MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport = 6 -> int +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.DiscoveryCompleteEventArgs(long totalTests, bool isAborted, System.Collections.Generic.IList fullyDiscoveredSources, System.Collections.Generic.IList partiallyDiscoveredSources, System.Collections.Generic.IList notDiscoveredSources) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.FullyDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.FullyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.NotDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.NotDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCompleteEventArgs.PartiallyDiscoveredSources.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.DiscoveryCriteria(System.Collections.Generic.IEnumerable sources, long frequencyOfDiscoveredTestsEvent, System.TimeSpan discoveredTestEventTimeout, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryRequestPayload.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestPlatform.StartTestSession(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria criteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler eventsHandler) -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStartTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs eventArgs) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStopTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs eventArgs) -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionAckPayload.EventArgs.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.CollectMetrics.get -> bool +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.CollectMetrics.set -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.StopTestSessionPayload() -> void +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo +Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.TestSessionInfo.set -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs.Metrics.get -> System.Collections.Generic.IDictionary Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs.Metrics.set -> void @@ -19,15 +39,3 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEv Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.StopTestSessionCompleteEventArgs(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo testSessionInfo) -> void Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs.TestSessionInfo.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStartTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs eventArgs) -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.ITestSessionEventsHandler.HandleStopTestSessionComplete(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs eventArgs) -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCompleteEventArgs -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StartTestSessionAckPayload.EventArgs.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionAckPayload.EventArgs.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StopTestSessionCompleteEventArgs -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionAckPayload.EventArgs.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.CollectMetrics.get -> bool -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.CollectMetrics.set -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.StopTestSessionPayload() -> void -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.TestSessionInfo.get -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestSessionInfo -Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Payloads.StopTestSessionPayload.TestSessionInfo.set -> void diff --git a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt index 5f282702bb..e02abfc9b0 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt +++ b/src/Microsoft.TestPlatform.ObjectModel/PublicAPI/net/PublicAPI.Unshipped.txt @@ -1 +1 @@ - \ No newline at end of file + diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs index 2173cc9fa7..aeb9d04ae1 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleRequestSender.cs @@ -46,7 +46,7 @@ internal class VsTestConsoleRequestSender : ITranslationLayerRequestSender private bool _handShakeSuccessful; - private int _protocolVersion = 5; + private int _protocolVersion = 6; /// /// Used to cancel blocking tasks associated with the vstest.console process. @@ -67,7 +67,7 @@ public VsTestConsoleRequestSender() /// /// Initializes a new instance of the class. /// - /// + /// /// The communication manager. /// The data serializer. /// The test platform event source. @@ -974,7 +974,10 @@ private void SendMessageAndListenAndReportTestCases( var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( discoveryCompletePayload.TotalTests, - discoveryCompletePayload.IsAborted); + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); // Adding metrics from vstest.console. discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; @@ -1061,7 +1064,10 @@ private async Task SendMessageAndListenAndReportTestCasesAsync( var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs( discoveryCompletePayload.TotalTests, - discoveryCompletePayload.IsAborted); + discoveryCompletePayload.IsAborted, + discoveryCompletePayload.FullyDiscoveredSources, + discoveryCompletePayload.PartiallyDiscoveredSources, + discoveryCompletePayload.NotDiscoveredSources); // Adding Metrics from VsTestConsole discoveryCompleteEventArgs.Metrics = discoveryCompletePayload.Metrics; @@ -1089,7 +1095,7 @@ private async Task SendMessageAndListenAndReportTestCasesAsync( TestMessageLevel.Error, TranslationLayerResources.AbortedTestsDiscovery); - var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true); + var discoveryCompleteEventArgs = new DiscoveryCompleteEventArgs(-1, true, new List(), new List(), new List()); eventHandler.HandleDiscoveryComplete(discoveryCompleteEventArgs, null); // Earlier we were closing the connection with vstest.console in case of exceptions. diff --git a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs index d2fc97dc1f..dc0f43cde7 100644 --- a/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs +++ b/src/vstest.console/TestPlatformHelpers/TestRequestManager.cs @@ -566,7 +566,7 @@ public void CancelTestRun() /// public void CancelDiscovery() { - EqtTrace.Info("TestRequestManager.CancelTestDiscovery: Sending cancel request."); + EqtTrace.Info("TestRequestManager.CancelDiscovery: Sending cancel request."); _currentDiscoveryRequest?.Abort(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 5142bcb69e..f931ce34a2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -1,4 +1,5 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. + +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. #nullable disable @@ -147,6 +148,26 @@ public void DiscoverTestsUsingEventHandler1AndBatchSize(RunnerInfo runnerInfo) Assert.AreEqual(3, discoveryEventHandlerForBatchSize.BatchSize); } + [TestMethod] + [NetCoreTargetFrameworkDataSource] + [NetFullTargetFrameworkDataSource] + public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovered(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + Setup(); + + var eventHandler2 = new DiscoveryEventHandler2(); + + _vstestConsoleWrapper.DiscoverTests( + GetTestAssemblies(), + GetDefaultRunSettings(), + null, + eventHandler2); + + // Assert. + Assert.AreEqual(2, eventHandler2.FullyDiscoveredSources.Count); + } + [TestMethod] [NetFullTargetFrameworkDataSource] [NetCoreTargetFrameworkDataSource] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs index 78bb3818d1..9af416611b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/DiscoveryEventHandler.cs @@ -72,6 +72,10 @@ public class DiscoveryEventHandler2 : ITestDiscoveryEventsHandler2 /// public List DiscoveredTestCases { get; } + public IList FullyDiscoveredSources { get; private set; } + public IList PartiallyDiscoveredSources { get; private set; } + public IList NotDiscoveredSources { get; private set; } + public List TestMessages; /// @@ -103,6 +107,9 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete } Metrics = discoveryCompleteEventArgs.Metrics; + FullyDiscoveredSources = discoveryCompleteEventArgs.FullyDiscoveredSources; + PartiallyDiscoveredSources = discoveryCompleteEventArgs.PartiallyDiscoveredSources; + NotDiscoveredSources = discoveryCompleteEventArgs.NotDiscoveredSources; } public void HandleDiscoveredTests(IEnumerable discoveredTestCases) diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs index a174a93913..3896f73100 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/DesignMode/DesignModeClientTests.cs @@ -43,7 +43,7 @@ public class DesignModeClientTests private readonly DesignModeClient _designModeClient; - private readonly int _protocolVersion = 5; + private readonly int _protocolVersion = 6; private readonly AutoResetEvent _completeEvent; @@ -136,7 +136,7 @@ public void DesignModeClientConnectShouldNotSendConnectedIfServerConnectionTimes [TestMethod] public void DesignModeClientDuringConnectShouldHighestCommonVersionWhenReceivedVersionIsGreaterThanSupportedVersion() { - var verCheck = new Message { MessageType = MessageType.VersionCheck, Payload = 5 }; + var verCheck = new Message { MessageType = MessageType.VersionCheck, Payload = 6 }; var sessionEnd = new Message { MessageType = MessageType.SessionEnd }; _mockCommunicationManager.Setup(cm => cm.WaitForServerConnection(It.IsAny())).Returns(true); _mockCommunicationManager.SetupSequence(cm => cm.ReceiveMessage()).Returns(verCheck).Returns(sessionEnd); diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs b/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs index b09db35113..24a21be30c 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Discovery/DiscoveryRequestTests.cs @@ -44,6 +44,10 @@ public DiscoveryRequestTests() _discoveryRequest = new DiscoveryRequest(_mockRequestData.Object, _discoveryCriteria, _discoveryManager.Object, _loggerManager.Object, _mockDataSerializer.Object); } + public static IEnumerable ProtocolConfigVersionProvider + => Enumerable.Range(0, Constants.DefaultProtocolConfig.Version + 1) + .Select(x => new object[] { x }); + [TestMethod] public void ConstructorSetsDiscoveryCriteriaAndDiscoveryManager() { @@ -91,14 +95,27 @@ public void AbortIfDiscoveryRequestDisposedShouldThrowObjectDisposedException() Assert.ThrowsException(() => _discoveryRequest.Abort()); } - [TestMethod] - public void AbortIfDiscoveryIsinProgressShouldCallDiscoveryManagerAbort() + [DataTestMethod] + [DynamicData(nameof(ProtocolConfigVersionProvider))] + public void AbortIfDiscoveryIsinProgressShouldCallDiscoveryManagerAbort(int version) { // Just to set the IsDiscoveryInProgress flag _discoveryRequest.DiscoverAsync(); + // Set the protocol version to a version not supporting new abort overload. + Constants.DefaultProtocolConfig.Version = version; _discoveryRequest.Abort(); - _discoveryManager.Verify(dm => dm.Abort(), Times.Once); + + if (version < Constants.MinimumProtocolVersionWithCancelDiscoveryEventHandlerSupport) + { + _discoveryManager.Verify(dm => dm.Abort(), Times.Once); + _discoveryManager.Verify(dm => dm.Abort(_discoveryRequest), Times.Never); + } + else + { + _discoveryManager.Verify(dm => dm.Abort(), Times.Never); + _discoveryManager.Verify(dm => dm.Abort(_discoveryRequest), Times.Once); + } } [TestMethod] diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs index 1433a8d1a4..b13d803199 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryDataAggregatorTests.cs @@ -10,6 +10,7 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Client.Parallel; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client.Parallel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] @@ -236,4 +237,41 @@ public void GetDiscoveryDataMetricsShouldNotAddNumberOfAdapterDiscoveredIfMetric var runMetrics = aggregator.GetAggregatedDiscoveryDataMetrics(); Assert.IsFalse(runMetrics.TryGetValue(TelemetryDataConstants.NumberOfAdapterDiscoveredDuringDiscovery, out _)); } + + [TestMethod] + public void AggregateShouldAggregateMessageSentCorrectly() + { + var aggregator = new ParallelDiscoveryDataAggregator(); + + aggregator.AggregateIsMessageSent(isMessageSent: false); + Assert.IsFalse(aggregator.IsMessageSent, "Aborted must be false"); + + aggregator.AggregateIsMessageSent(isMessageSent: true); + Assert.IsTrue(aggregator.IsMessageSent, "Aborted must be true"); + + aggregator.AggregateIsMessageSent(isMessageSent: false); + Assert.IsTrue(aggregator.IsMessageSent, "Aborted must be true"); + } + + [TestMethod] + public void AggregateShouldAggregateSourcesCorrectly() + { + // Arrange + var aggregator = new ParallelDiscoveryDataAggregator(); + var sources = new List() { "sample.dll" }; + + // Act + aggregator.MarkSourcesWithStatus(sources, DiscoveryStatus.NotDiscovered); + var sourcesWithNotDiscoveredStatus = aggregator.GetSourcesWithStatus(DiscoveryStatus.NotDiscovered); + + // Assert + Assert.AreEqual(1, sourcesWithNotDiscoveredStatus.Count); + + // Act + aggregator.MarkSourcesWithStatus(sources, DiscoveryStatus.FullyDiscovered); + var sourcesWithFullyDiscoveryStatus = aggregator.GetSourcesWithStatus(DiscoveryStatus.FullyDiscovered); + + // Assert + Assert.AreEqual(1, sourcesWithFullyDiscoveryStatus.Count); + } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs index bb427b3df2..299649e29b 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelDiscoveryEventsHandlerTests.cs @@ -62,7 +62,7 @@ public void HandleDiscoveryCompleteShouldNotCallLastChunkResultsIfNotPresent() _parallelDiscoveryEventsHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); - // Raw message must be sent + // Raw message must be sent _mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleRawMessage(It.IsAny()), Times.Never); _mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleDiscoveredTests(null), Times.Never); @@ -150,6 +150,27 @@ public void HandleDiscoveryCompleteShouldCallTestDiscoveryCompleteOnActualHandle _mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleDiscoveryComplete(It.IsAny(), null), Times.Once); } + [TestMethod] + public void HandleDiscoveryCompleteShouldCallConvertToRawMessageAndSendOnceIfDiscoveryIsComplete() + { + string payload = "DiscoveryComplete"; + int totalTests = 10; + bool aborted = false; + + _mockParallelProxyDiscoveryManager.Setup(mp => mp.HandlePartialDiscoveryComplete( + _mockProxyDiscoveryManager.Object, totalTests, null, aborted)).Returns(true); + + _mockDataSerializer.Setup(mds => mds.SerializeMessage(MessageType.DiscoveryComplete)).Returns(payload); + + // Act + var discoveryCompleteEventsArgs = new DiscoveryCompleteEventArgs(totalTests, aborted, It.IsAny>(), It.IsAny>(), It.IsAny>()); + _parallelDiscoveryEventsHandler.HandleDiscoveryComplete(discoveryCompleteEventsArgs, null); + + // Verify + _mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleRawMessage(It.IsAny()), Times.Once); + _mockTestDiscoveryEventsHandler.Verify(mt => mt.HandleDiscoveryComplete(It.IsAny(), null), Times.Once); + } + [TestMethod] public void HandleDiscoveryTestsShouldJustPassOnTheEventToDiscoveryEventsHandler() { diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs index 51d3d2e45f..ff763aca3a 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/Parallel/ParallelProxyDiscoveryManagerTests.cs @@ -100,30 +100,50 @@ public void DiscoverTestsShouldProcessAllSources() AssertMissingAndDuplicateSources(_processedSources); } - /// - /// Create ParallelProxyDiscoveryManager with parallel level 1 and two source, - /// Abort in any source should not stop discovery for other sources. - /// [TestMethod] - public void DiscoveryTestsShouldProcessAllSourcesOnDiscoveryAbortsForAnySource() + public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAbortedWithEventHandler() + { + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, false); + var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); + + parallelDiscoveryManager.Abort(_mockHandler.Object); + bool isPartialDiscoveryComplete = parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: false); + + Assert.IsTrue(isPartialDiscoveryComplete); + } + + [TestMethod] + public void HandlePartialDiscoveryCompleteShouldReturnTrueIfDiscoveryWasAborted() + { + var parallelDiscoveryManager = new ParallelProxyDiscoveryManager(_mockRequestData.Object, _proxyManagerFunc, 1, false); + var proxyDiscovermanager = new ProxyDiscoveryManager(_mockRequestData.Object, new Mock().Object, new Mock().Object); + + parallelDiscoveryManager.Abort(); + bool isPartialDiscoveryComplete = parallelDiscoveryManager.HandlePartialDiscoveryComplete(proxyDiscovermanager, 20, new List(), isAborted: false); + + Assert.IsTrue(isPartialDiscoveryComplete); + } + + [TestMethod] + public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); _createdMockManagers.Add(discoveryManagerMock); var parallelDiscoveryManager = SetupDiscoveryManager(() => discoveryManagerMock.Object, 1, true, totalTests: -1); - Task.Run(() => parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object)); + Task.Run(() => + { + parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object); + parallelDiscoveryManager.Abort(); + }); Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); - Assert.AreEqual(2, _processedSources.Count, "All Sources must be processed."); + Assert.AreEqual(1, _processedSources.Count, "One source should be processed."); } - /// - /// Create ParallelProxyDiscoveryManager with parallel level 1 and two sources, - /// Overall discovery should stop, if aborting was requested - /// [TestMethod] - public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() + public void DiscoveryTestsShouldStopDiscoveryIfAbortionWithEventHandlerWasRequested() { // Since the hosts are aborted, total aggregated tests sent across will be -1 var discoveryManagerMock = new Mock(); @@ -133,7 +153,7 @@ public void DiscoveryTestsShouldStopDiscoveryIfAbortionWasRequested() Task.Run(() => { parallelDiscoveryManager.DiscoverTests(_testDiscoveryCriteria, _mockHandler.Object); - parallelDiscoveryManager.Abort(); + parallelDiscoveryManager.Abort(_mockHandler.Object); }); Assert.IsTrue(_discoveryCompleted.Wait(TaskTimeout), "Test discovery not completed."); diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs index 741e7157bf..20b1d53a3e 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Discovery/DiscoveryManagerTests.cs @@ -268,5 +268,28 @@ public void DiscoveryInitializeShouldVerifyWarningMessageIfAdapterFailedToLoad() mockLogger.Verify(rd => rd.HandleLogMessage(TestMessageLevel.Warning, "verify that the HandleLogMessage method getting invoked at least once"), Times.Once); } + [TestMethod] + public void DiscoveryTestsShouldSendAbortValuesCorrectlyIfAbortionHappened() + { + // Arrange + var sources = new List { typeof(DiscoveryManagerTests).GetTypeInfo().Assembly.Location }; + + var criteria = new DiscoveryCriteria(sources, 100, null); + var mockHandler = new Mock(); + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + mockHandler.Setup(ml => ml.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback((DiscoveryCompleteEventArgs complete, IEnumerable tests) => receivedDiscoveryCompleteEventArgs = complete); + + // Act + _discoveryManager.DiscoverTests(criteria, mockHandler.Object); + _discoveryManager.Abort(mockHandler.Object); + + // Assert + Assert.AreEqual(true, receivedDiscoveryCompleteEventArgs.IsAborted); + Assert.AreEqual(-1, receivedDiscoveryCompleteEventArgs.TotalCount); + } + #endregion } diff --git a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs index f2a2badd2a..c99d0dff36 100644 --- a/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs +++ b/test/TranslationLayer.UnitTests/VsTestConsoleRequestSenderTests.cs @@ -41,7 +41,7 @@ public class VsTestConsoleRequestSenderTests private readonly int _waitTimeout = 2000; - private readonly int _protocolVersion = 5; + private readonly int _protocolVersion = 6; private readonly IDataSerializer _serializer = JsonDataSerializer.Instance; public VsTestConsoleRequestSenderTests() @@ -432,6 +432,90 @@ public async Task DiscoverTestsAsyncShouldCompleteWithSingleTest() mockHandler.Verify(mh => mh.HandleLogMessage(It.IsAny(), It.IsAny()), Times.Never, "TestMessage event must not be called"); } + [TestMethod] + public void DiscoverTestsShouldCompleteWithSingleFullyDiscoveredSource() + { + InitializeCommunication(); + + var mockHandler = new Mock(); + + List sources = new() { "1.dll" }; + + var testCase = new TestCase("hello", new Uri("world://how"), source: sources[0]); + var testsFound = new Message() + { + MessageType = MessageType.TestCasesFound, + Payload = JToken.FromObject(new List() { testCase }) + }; + + var payload = new DiscoveryCompletePayload() { TotalTests = 1, LastDiscoveredTests = null, IsAborted = false, FullyDiscoveredSources = sources }; + var discoveryComplete = new Message() + { + MessageType = MessageType.DiscoveryComplete, + Payload = JToken.FromObject(payload) + }; + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + _mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(testsFound)); + mockHandler.Setup(mh => mh.HandleDiscoveredTests(It.IsAny>())) + .Callback(() => _mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(discoveryComplete))); + + mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback((DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable tests) => receivedDiscoveryCompleteEventArgs = discoveryCompleteEventArgs); + + _requestSender.DiscoverTests(sources, null, new TestPlatformOptions(), null, mockHandler.Object); + + mockHandler.Verify(mh => mh.HandleDiscoveryComplete(It.IsAny(), null), Times.Once, "Discovery Complete must be called"); + Assert.IsNotNull(receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources); + Assert.AreEqual(1, receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources.Count); + } + + [TestMethod] + public void DiscoverTestsShouldCompleteWithCorrectAbortedValuesIfAbortingWasRequested() + { + // Arrange + InitializeCommunication(); + + var mockHandler = new Mock(); + + List sources = new() { "1.dll" }; + + var testCase = new TestCase("hello", new Uri("world://how"), source: sources[0]); + var testsFound = new Message() + { + MessageType = MessageType.TestCasesFound, + Payload = JToken.FromObject(new List() { testCase }) + }; + + var payload = new DiscoveryCompletePayload() { TotalTests = -1, LastDiscoveredTests = null, IsAborted = true, FullyDiscoveredSources = sources }; + var discoveryComplete = new Message() + { + MessageType = MessageType.DiscoveryComplete, + Payload = JToken.FromObject(payload) + }; + + DiscoveryCompleteEventArgs receivedDiscoveryCompleteEventArgs = null; + + _mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(testsFound)); + mockHandler.Setup(mh => mh.HandleDiscoveredTests(It.IsAny>())) + .Callback(() => _mockCommunicationManager.Setup(cm => cm.ReceiveMessageAsync(It.IsAny())).Returns(Task.FromResult(discoveryComplete))); + + mockHandler.Setup(mh => mh.HandleDiscoveryComplete(It.IsAny(), It.IsAny>())) + .Callback((DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable tests) => receivedDiscoveryCompleteEventArgs = discoveryCompleteEventArgs); + + // Act + _requestSender.DiscoverTests(sources, null, new TestPlatformOptions(), null, mockHandler.Object); + _requestSender.CancelDiscovery(); + + // Assert + mockHandler.Verify(mh => mh.HandleDiscoveryComplete(It.IsAny(), null), Times.Once, "Discovery Complete must be called"); + Assert.IsNotNull(receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources); + Assert.AreEqual(1, receivedDiscoveryCompleteEventArgs.FullyDiscoveredSources.Count); + Assert.AreEqual(-1, receivedDiscoveryCompleteEventArgs.TotalCount); + Assert.AreEqual(true, receivedDiscoveryCompleteEventArgs.IsAborted); + } + [TestMethod] public void DiscoverTestsShouldReportBackTestsWithTraitsInTestsFoundMessage() { diff --git a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs index 7c827e83d1..873380890d 100644 --- a/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs +++ b/test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs @@ -234,7 +234,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); @@ -258,7 +258,7 @@ public void DiscoverTestsShouldPassSameProtocolConfigInRequestData() _testRequestManager.DiscoverTests(payload, mockDiscoveryRegistrar.Object, mockProtocolConfig); // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + Assert.AreEqual(6, actualRequestData.ProtocolConfig.Version); } @@ -284,7 +284,7 @@ public void DiscoverTestsShouldCollectMetrics() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -333,7 +333,7 @@ public void DiscoverTestsShouldCollectTargetDeviceLocalMachineIfTargetDeviceStri " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -376,7 +376,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsDevice() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -419,7 +419,7 @@ public void DiscoverTestsShouldCollectTargetDeviceIfTargetDeviceIsEmulator() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -462,7 +462,7 @@ public void DiscoverTestsShouldCollectCommands() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -517,7 +517,7 @@ public void DiscoverTestsShouldCollectTestSettings() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -564,7 +564,7 @@ public void DiscoverTestsShouldCollectVsmdiFile() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -611,7 +611,7 @@ public void DiscoverTestsShouldCollectTestRunConfigFile() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; var mockDiscoveryRegistrar = new Mock(); IRequestData actualRequestData = null; @@ -887,7 +887,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() Sources = new List() { "a" }, RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -897,7 +897,7 @@ public void RunTestsShouldPassSameProtocolConfigInRequestData() _testRequestManager.RunTests(payload, new Mock().Object, new Mock().Object, mockProtocolConfig); // Verify. - Assert.AreEqual(5, actualRequestData.ProtocolConfig.Version); + Assert.AreEqual(6, actualRequestData.ProtocolConfig.Version); } [TestMethod] @@ -911,7 +911,7 @@ public void RunTestsShouldCollectCommands() Sources = new List() { "a" }, RunSettings = DefaultRunsettings }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -975,7 +975,7 @@ public void RunTestsShouldCollectTelemetryForLegacySettings() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -1022,7 +1022,7 @@ public void RunTestsShouldCollectTelemetryForTestSettingsEmbeddedInsideRunSettin " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback( @@ -1067,7 +1067,7 @@ public void RunTestsShouldCollectMetrics() " }; - var mockProtocolConfig = new ProtocolConfig { Version = 5 }; + var mockProtocolConfig = new ProtocolConfig { Version = 6 }; IRequestData actualRequestData = null; var mockDiscoveryRequest = new Mock(); _mockTestPlatform.Setup(mt => mt.CreateTestRunRequest(It.IsAny(), It.IsAny(), It.IsAny())).Callback(