diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataSerializer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataSerializer.cs index 5704a10e14..97f52c5ca2 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataSerializer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Interfaces/IDataSerializer.cs @@ -46,5 +46,13 @@ public interface IDataSerializer /// version to be sent /// Raw Serialized message string SerializePayload(string messageType, object payload, int version); + + /// + /// Creates cloned object for given object. + /// + /// The type of object to be cloned. + /// Object to be cloned. + /// Newly cloned object. + T Clone(T obj); } } diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs index c2ea93b19b..9ac9e55038 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs @@ -152,6 +152,18 @@ public string SerializePayload(string messageType, object payload, int version) return JsonConvert.SerializeObject(message); } + /// + public T Clone(T obj) + { + if (obj == null) + { + return default(T); + } + + var stringObj = this.Serialize(obj, 2); + return this.Deserialize(stringObj, 2); + } + /// /// Serialize an object to JSON using default serialization settings. /// diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs index 9f5c62fde2..12bb65f54c 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Adapter/TestExecutionRecorder.cs @@ -64,6 +64,7 @@ internal Collection Attachments /// test case which will be started. public void RecordStart(TestCase testCase) { + EqtTrace.Verbose("TestExecutionRecorder.RecordStart: Starting test: {0}.", testCase?.FullyQualifiedName); this.testRunCache.OnTestStarted(testCase); if (this.testCaseEventsHandler != null) @@ -85,6 +86,7 @@ public void RecordStart(TestCase testCase) /// test result to the framework when the test(s) is canceled. public void RecordResult(TestResult testResult) { + EqtTrace.Verbose("TestExecutionRecorder.RecordResult: Received result for test: {0}.", testResult?.TestCase?.FullyQualifiedName); if (this.testCaseEventsHandler != null) { // Send TestCaseEnd in case RecordEnd was not called. @@ -104,6 +106,7 @@ public void RecordResult(TestResult testResult) /// outcome of the test case. public void RecordEnd(TestCase testCase, TestOutcome outcome) { + EqtTrace.Verbose("TestExecutionRecorder.RecordEnd: test: {0} execution completed.", testCase?.FullyQualifiedName); this.testRunCache.OnTestCompletion(testCase); this.SendTestCaseEnd(testCase, outcome); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs index c720d63a0f..bdf79a047d 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Execution/BaseRunTests.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution { using System; + using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; @@ -13,7 +14,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Execution using System.Linq; using System.Reflection; using System.Threading.Tasks; - + using CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework; using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; @@ -78,6 +79,11 @@ internal abstract class BaseRunTests /// private RunConfiguration runConfiguration; + /// + /// The Serializer to clone testcase object incase of user input test source is package. E.g UWP scenario(appx/build.appxrecipe). + /// + private IDataSerializer dataSerializer; + #endregion #region Constructor @@ -107,7 +113,8 @@ protected BaseRunTests(IRequestData requestData, testRunEventsHandler, testPlatformEventSource, testCaseEventsHandler as ITestEventsPublisher, - new PlatformThread()) + new PlatformThread(), + JsonDataSerializer.Instance) { } @@ -131,7 +138,8 @@ protected BaseRunTests(IRequestData requestData, ITestRunEventsHandler testRunEventsHandler, ITestPlatformEventSource testPlatformEventSource, ITestEventsPublisher testEventsPublisher, - IThread platformThread) + IThread platformThread, + IDataSerializer dataSerializer) { this.package = package; this.runSettings = runSettings; @@ -144,6 +152,7 @@ protected BaseRunTests(IRequestData requestData, this.testPlatformEventSource = testPlatformEventSource; this.testEventsPublisher = testEventsPublisher; this.platformThread = platformThread; + this.dataSerializer = dataSerializer; this.SetContext(); } @@ -557,6 +566,12 @@ private void RaiseTestRunComplete( // Collecting Number of Adapters Used to run tests. this.requestData.MetricsCollection.Add(TelemetryDataConstants.NumberOfAdapterUsedToRunTests, this.ExecutorUrisThatRanTests.Count()); + if (lastChunk.Any() && string.IsNullOrEmpty(package) == false) + { + Tuple, ICollection> updatedTestResultsAndInProgressTestCases = this.UpdateTestCaseSourceToPackage(lastChunk, null); + lastChunk = updatedTestResultsAndInProgressTestCases.Item1; + } + var testRunChangedEventArgs = new TestRunChangedEventArgs(runStats, lastChunk, Enumerable.Empty()); // Adding Metrics along with Test Run Complete Event Args @@ -569,10 +584,6 @@ private void RaiseTestRunComplete( attachments, elapsedTime); testRunCompleteEventArgs.Metrics = this.requestData.MetricsCollection.Metrics; - if (lastChunk.Any()) - { - UpdateTestResults(lastChunk, null, this.package); - } this.testRunEventsHandler.HandleTestRunComplete( testRunCompleteEventArgs, @@ -586,13 +597,19 @@ private void RaiseTestRunComplete( } } - private void OnCacheHit(TestRunStatistics testRunStats, ICollection results, ICollection inProgressTests) + private void OnCacheHit(TestRunStatistics testRunStats, ICollection results, ICollection inProgressTestCases) { if (this.testRunEventsHandler != null) { - UpdateTestResults(results, inProgressTests, this.package); + if (string.IsNullOrEmpty(package) == false) + { + Tuple, ICollection> updatedTestResultsAndInProgressTestCases + = this.UpdateTestCaseSourceToPackage(results, inProgressTestCases); + results = updatedTestResultsAndInProgressTestCases.Item1; + inProgressTestCases = updatedTestResultsAndInProgressTestCases.Item2; + } - var testRunChangedEventArgs = new TestRunChangedEventArgs(testRunStats, results, inProgressTests); + var testRunChangedEventArgs = new TestRunChangedEventArgs(testRunStats, results, inProgressTestCases); this.testRunEventsHandler.HandleTestRunStatsChange(testRunChangedEventArgs); } else @@ -624,22 +641,50 @@ private bool TryToRunInSTAThread(Action action, bool waitForCompletion) } - private static void UpdateTestResults(IEnumerable testResults, IEnumerable testCases, string package) + private Tuple, ICollection> UpdateTestCaseSourceToPackage( + ICollection testResults, + ICollection inProgressTestCases) { - // Before sending the testresults back, update the test case objects with source provided by IDE/User. - if (!string.IsNullOrEmpty(package)) + + EqtTrace.Verbose("BaseRunTests.UpdateTestCaseSourceToPackage: Update source details for testResults and testCases."); + + var updatedTestResults = UpdateTestResults(testResults, package); + + var updatedInProgressTestCases = UpdateInProgressTests(inProgressTestCases, package); + + return new Tuple, ICollection>(updatedTestResults, updatedInProgressTestCases); + } + + private ICollection UpdateTestResults(ICollection testResults, string package) + { + ICollection updatedTestResults = new List(); + + foreach (var testResult in testResults) { - foreach (var tr in testResults) - { - tr.TestCase.Source = package; - } + var updatedTestResult = this.dataSerializer.Clone(testResult); + updatedTestResult.TestCase.Source = package; + updatedTestResults.Add(updatedTestResult); + } - // TestCases can be empty, enumerate on EmptyList then - foreach (var tc in testCases ?? Enumerable.Empty()) - { - tc.Source = package; - } + return updatedTestResults; + } + + private ICollection UpdateInProgressTests(ICollection inProgressTestCases, string package) + { + if (inProgressTestCases == null) + { + return null; } + + ICollection updatedTestCases = new List(); + foreach (var inProgressTestCase in inProgressTestCases) + { + var updatedTestCase = this.dataSerializer.Clone(inProgressTestCase); + updatedTestCase.Source = package; + updatedTestCases.Add(updatedTestCase); + } + + return updatedTestCases; } #endregion diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs index e227a558b6..d0a7a9f157 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs @@ -4,12 +4,24 @@ namespace Microsoft.TestPlatform.CommunicationUtilities.UnitTests { using System; + using System.Collections.Generic; + using System.Linq; + using System.Runtime.Serialization; using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; + using VisualStudio.TestPlatform.ObjectModel; + using TestResult = VisualStudio.TestPlatform.ObjectModel.TestResult; [TestClass] public class JsonDataSerializerTests { + private JsonDataSerializer jsonDataSerializer; + + public JsonDataSerializerTests() + { + this.jsonDataSerializer = JsonDataSerializer.Instance; + } + [TestMethod] public void SerializePayloadShouldSerializeAnObjectWithSelfReferencingLoop() { @@ -17,10 +29,8 @@ public void SerializePayloadShouldSerializeAnObjectWithSelfReferencingLoop() classWithSelfReferencingLoop = new ClassWithSelfReferencingLoop(classWithSelfReferencingLoop); classWithSelfReferencingLoop.InfiniteRefernce.InfiniteRefernce = classWithSelfReferencingLoop; - var sut = JsonDataSerializer.Instance; - // This line should not throw exception - sut.SerializePayload("dummy", classWithSelfReferencingLoop); + this.jsonDataSerializer.SerializePayload("dummy", classWithSelfReferencingLoop); } [TestMethod] @@ -30,17 +40,94 @@ public void DeserializeShouldDeserializeAnObjectWhichHadSelfReferencingLoopBefor classWithSelfReferencingLoop = new ClassWithSelfReferencingLoop(classWithSelfReferencingLoop); classWithSelfReferencingLoop.InfiniteRefernce.InfiniteRefernce = classWithSelfReferencingLoop; - var sut = JsonDataSerializer.Instance; - - var json = sut.SerializePayload("dummy", classWithSelfReferencingLoop); + var json = this.jsonDataSerializer.SerializePayload("dummy", classWithSelfReferencingLoop); // This line should deserialize properly - var result = sut.Deserialize(json, 1); + var result = this.jsonDataSerializer.Deserialize(json, 1); Assert.AreEqual(typeof(ClassWithSelfReferencingLoop), result.GetType()); Assert.IsNull(result.InfiniteRefernce); } + [TestMethod] + public void CloneShouldReturnNullForNull() + { + var clonedTestCase = this.jsonDataSerializer.Clone(null); + + Assert.IsNull(clonedTestCase); + } + + [TestMethod] + public void CloneShouldWorkForValueType() + { + var i = 2; + var clonedI = this.jsonDataSerializer.Clone(i); + + Assert.AreEqual(clonedI, i); + } + + [TestMethod] + public void CloneShouldCloneTestCaseObject() + { + var testCase = JsonDataSerializerTests.GetSampleTestCase(out var expectedTrait); + + var clonedTestCase = this.jsonDataSerializer.Clone(testCase); + + VerifyTestCaseClone(clonedTestCase, testCase, expectedTrait); + } + + [TestMethod] + public void CloneShouldCloneTestResultsObject() + { + var testCase = JsonDataSerializerTests.GetSampleTestCase(out var expectedTrait); + + var testResult = new TestResult(testCase); + + var startTime = DateTimeOffset.UtcNow; + testResult.StartTime = startTime; + + var clonedTestResult = this.jsonDataSerializer.Clone(testResult); + + Assert.IsFalse(ReferenceEquals(testResult, clonedTestResult)); + + Assert.AreEqual(testResult.StartTime, clonedTestResult.StartTime); + + VerifyTestCaseClone(testResult.TestCase, clonedTestResult.TestCase, expectedTrait); + } + + private static TestCase GetSampleTestCase(out Trait expectedTrait) + { + var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); + + expectedTrait = new Trait("TraitName1", "TraitValue1"); + + testCase.Traits.Add(expectedTrait); + return testCase; + } + + private static void VerifyTestCaseClone(TestCase clonedTestCase, TestCase testCase, Trait expectedTrait) + { + Assert.IsFalse(ReferenceEquals(clonedTestCase, testCase)); + + Assert.AreEqual(testCase.FullyQualifiedName, clonedTestCase.FullyQualifiedName); + Assert.IsFalse(ReferenceEquals(testCase.FullyQualifiedName, clonedTestCase.FullyQualifiedName)); + + Assert.AreEqual(testCase.ExecutorUri, clonedTestCase.ExecutorUri); + Assert.IsFalse(ReferenceEquals(testCase.ExecutorUri, clonedTestCase.ExecutorUri)); + + Assert.AreEqual(testCase.Source, clonedTestCase.Source); + Assert.IsFalse(ReferenceEquals(testCase.Source, clonedTestCase.Source)); + + Assert.AreEqual(1, clonedTestCase.Traits.Count()); + + foreach (var trait in clonedTestCase.Traits) + { + Assert.IsFalse(ReferenceEquals(expectedTrait, trait)); + Assert.AreEqual(expectedTrait.Name, trait.Name); + Assert.AreEqual(expectedTrait.Value, trait.Value); + } + } + public class ClassWithSelfReferencingLoop { public ClassWithSelfReferencingLoop(ClassWithSelfReferencingLoop ir) diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs index 9335de51be..c4852ebc9b 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/DataCollection/DataCollectionTestRunEventsHandlerTests.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -namespace TestPlatform.CommunicationUtilities.UnitTests.ObjectModel +namespace Microsoft.TestPlatform.CrossPlatEngine.UnitTests.DataCollection { using System; using System.Collections.ObjectModel; diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Execution/BaseRunTestsTests.cs b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Execution/BaseRunTestsTests.cs index eb30d7cf08..ab30c12066 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Execution/BaseRunTestsTests.cs +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Execution/BaseRunTestsTests.cs @@ -15,6 +15,8 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Execution using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework.Utilities; using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; + using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities; + using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Tracing.Interfaces; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Adapter; using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.DataCollection.Interfaces; @@ -28,12 +30,17 @@ namespace TestPlatform.CrossPlatEngine.UnitTests.Execution using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; + using OMTestResult = Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult; + using Moq; [TestClass] public class BaseRunTestsTests { + private const string BaseRunTestsExecutorUri = "executor://BaseRunTestsExecutor/"; + private const string BadBaseRunTestsExecutorUri = "executor://BadBaseRunTestsExecutor/"; + private TestExecutionContext testExecutionContext; private Mock mockTestRunEventsHandler; @@ -46,11 +53,15 @@ public class BaseRunTestsTests private Mock mockMetricsCollection; - private const string BaseRunTestsExecutorUri = "executor://BaseRunTestsExecutor/"; - private const string BadBaseRunTestsExecutorUri = "executor://BadBaseRunTestsExecutor/"; + private Mock mockDataSerializer; - [TestInitialize] - public void TestInit() + private TestRunChangedEventArgs receivedRunStatusArgs; + private TestRunCompleteEventArgs receivedRunCompleteArgs; + private ICollection receivedattachments; + private ICollection receivedExecutorUris; + private TestCase inProgressTestCase; + + public BaseRunTestsTests() { this.testExecutionContext = new TestExecutionContext( frequencyOfRunStatsChangeEvent: 100, @@ -69,16 +80,21 @@ public void TestInit() this.mockRequestData = new Mock(); this.mockMetricsCollection = new Mock(); + this.mockThread = new Mock(); + this.mockDataSerializer = new Mock(); this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(this.mockMetricsCollection.Object); this.runTestsInstance = new TestableBaseRunTests( + this.mockRequestData.Object, null, null, - testExecutionContext, + this.testExecutionContext, null, this.mockTestRunEventsHandler.Object, this.mockTestPlatformEventSource.Object, - this.mockRequestData.Object); + null, + new PlatformThread(), + this.mockDataSerializer.Object); TestPluginCacheTests.SetupMockExtensions(new string[] { typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location }, () => { }); } @@ -136,7 +152,8 @@ public void RunTestsShouldRaiseTestRunCompleteWithAbortedAsTrueOnException() It.IsAny>(), It.IsAny>())) .Callback( - (TestRunCompleteEventArgs complete, + ( + TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection attachments, ICollection executorUris) => @@ -154,6 +171,7 @@ public void RunTestsShouldRaiseTestRunCompleteWithAbortedAsTrueOnException() public void RunTestsShouldNotThrowIfExceptionIsAFileNotFoundException() { TestRunCompleteEventArgs receivedCompleteArgs = null; + // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { throw new FileNotFoundException(); }; this.mockTestRunEventsHandler.Setup( @@ -164,7 +182,8 @@ public void RunTestsShouldNotThrowIfExceptionIsAFileNotFoundException() It.IsAny>(), It.IsAny>())) .Callback( - (TestRunCompleteEventArgs complete, + ( + TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection attachments, ICollection executorUris) => @@ -183,6 +202,7 @@ public void RunTestsShouldNotThrowIfExceptionIsAFileNotFoundException() public void RunTestsShouldNotThrowIfExceptionIsAnArgumentException() { TestRunCompleteEventArgs receivedCompleteArgs = null; + // Setup mocks. this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { throw new ArgumentException(); }; this.mockTestRunEventsHandler.Setup( @@ -193,7 +213,8 @@ public void RunTestsShouldNotThrowIfExceptionIsAnArgumentException() It.IsAny>(), It.IsAny>())) .Callback( - (TestRunCompleteEventArgs complete, + ( + TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection attachments, ICollection executorUris) => @@ -223,7 +244,8 @@ public void RunTestsShouldAbortIfExecutorUriExtensionMapIsNull() It.IsAny>(), It.IsAny>())) .Callback( - (TestRunCompleteEventArgs complete, + ( + TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection attachments, ICollection executorUris) => @@ -231,7 +253,6 @@ public void RunTestsShouldAbortIfExecutorUriExtensionMapIsNull() receivedCompleteArgs = complete; }); - // This should not throw. this.runTestsInstance.RunTests(); @@ -297,7 +318,6 @@ public void RunTestsShouldInstrumentExecutionStart() [TestMethod] public void RunTestsShouldInstrumentExecutionStop() { - this.SetupExecutorUriMock(); this.runTestsInstance.RunTests(); @@ -347,11 +367,14 @@ public void RunTestsShouldReportAWarningIfExecutorUriIsNotDefinedInExtensionAsse var expectedWarningMessageFormat = "Could not find test executor with URI '{0}'. Make sure that the test executor is installed and supports .net runtime version {1}."; - //var runtimeVersion = string.Concat(PlatformServices.Default.Runtime.RuntimeType, " ", + + // var runtimeVersion = string.Concat(PlatformServices.Default.Runtime.RuntimeType, " ", // PlatformServices.Default.Runtime.RuntimeVersion); var runtimeVersion = " "; - var expectedWarningMessage = string.Format(expectedWarningMessageFormat, "executor://nonexistent/", + var expectedWarningMessage = string.Format( + expectedWarningMessageFormat, + "executor://nonexistent/", runtimeVersion); this.mockTestRunEventsHandler.Verify( treh => treh.HandleLogMessage(TestMessageLevel.Warning, expectedWarningMessage), Times.Once); @@ -423,12 +446,14 @@ public void RunTestsShouldReportWarningIfExecutorThrowsAnException() var messageFormat = "An exception occurred while invoking executor '{0}': {1}"; var message = string.Format(messageFormat, BaseRunTestsExecutorUri.ToLower(), "Test influenced."); - this.mockTestRunEventsHandler.Verify(treh => treh.HandleLogMessage(TestMessageLevel.Error, message), + this.mockTestRunEventsHandler.Verify( + treh => treh.HandleLogMessage(TestMessageLevel.Error, message), Times.Once); // Also validate that a test run complete is called. this.mockTestRunEventsHandler.Verify( - treh => treh.HandleTestRunComplete(It.IsAny(), + treh => treh.HandleTestRunComplete( + It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>()), Times.Once); @@ -496,45 +521,7 @@ public void RunTestsShouldIterateThroughAllExecutors() [TestMethod] public void RunTestsShouldRaiseTestRunComplete() { - TestRunCompleteEventArgs receivedRunCompleteArgs = null; - TestRunChangedEventArgs receivedRunStatusArgs = null; - ICollection receivedattachments = null; - ICollection receivedExecutorUris = null; - - var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; - var executorUriExtensionMap = new List> - { - new Tuple(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), - new Tuple(new Uri(BaseRunTestsExecutorUri), assemblyLocation) - }; - - // Setup mocks. - this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return executorUriExtensionMap; }; - this.runTestsInstance.InvokeExecutorCallback = - (executor, executorUriExtensionTuple, runContext, frameworkHandle) => - { - var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); - var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); - this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); - }; - this.mockTestRunEventsHandler.Setup( - treh => - treh.HandleTestRunComplete( - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>())) - .Callback( - (TestRunCompleteEventArgs complete, - TestRunChangedEventArgs stats, - ICollection attachments, - ICollection executorUris) => - { - receivedRunCompleteArgs = complete; - receivedRunStatusArgs = stats; - receivedattachments = attachments; - receivedExecutorUris = executorUris; - }); + this.SetUpTestRunEvents(); // Act. this.runTestsInstance.RunTests(); @@ -561,56 +548,10 @@ public void RunTestsShouldRaiseTestRunComplete() } [TestMethod] - public void RunTestsTestRunCompleteShouldUpdateTestCasesWithPackageIfProvided() + public void RunTestsShouldUpdateTestResultsTestCaseSourceWithPackageIfTestSourceIsPackage() { - var package = "x.apprecipe"; - TestRunCompleteEventArgs receivedRunCompleteArgs = null; - TestRunChangedEventArgs receivedRunStatusArgs = null; - ICollection receivedattachments = null; - ICollection receivedExecutorUris = null; - this.runTestsInstance = new TestableBaseRunTests( - null, - package, - testExecutionContext, - null, - this.mockTestRunEventsHandler.Object, - this.mockTestPlatformEventSource.Object, - this.mockRequestData.Object); - - var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; - var executorUriExtensionMap = new List> - { - new Tuple(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), - new Tuple(new Uri(BaseRunTestsExecutorUri), assemblyLocation) - }; - - // Setup mocks. - this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return executorUriExtensionMap; }; - this.runTestsInstance.InvokeExecutorCallback = - (executor, executorUriExtensionTuple, runContext, frameworkHandle) => - { - var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); - var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); - this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); - }; - this.mockTestRunEventsHandler.Setup( - treh => - treh.HandleTestRunComplete( - It.IsAny(), - It.IsAny(), - It.IsAny>(), - It.IsAny>())) - .Callback( - (TestRunCompleteEventArgs complete, - TestRunChangedEventArgs stats, - ICollection attachments, - ICollection executorUris) => - { - receivedRunCompleteArgs = complete; - receivedRunStatusArgs = stats; - receivedattachments = attachments; - receivedExecutorUris = executorUris; - }); + const string package = @"C:\Projects\UnitTestApp1\AppPackages\UnitTestApp1\UnitTestApp1_1.0.0.0_Win32_Debug_Test\UnitTestApp1_1.0.0.0_Win32_Debug.appx"; + this.SetUpTestRunEvents(package); // Act. this.runTestsInstance.RunTests(); @@ -620,67 +561,26 @@ public void RunTestsTestRunCompleteShouldUpdateTestCasesWithPackageIfProvided() Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0); // verify TC.Source is updated with package - foreach(var tr in receivedRunStatusArgs.NewTestResults) + foreach (var tr in receivedRunStatusArgs.NewTestResults) { Assert.AreEqual(tr.TestCase.Source, package); } - Assert.IsTrue(receivedRunStatusArgs.ActiveTests == null || receivedRunStatusArgs.ActiveTests.Count() == 0); } [TestMethod] - public void RunTestsShouldUpdateTestCasesWithPackageWhenCacheIsHitIfProvided() + public void RunTestsShouldUpdateActiveTestCasesSourceWithPackageIfTestSourceIsPackage() { - var package = "x.apprecipe"; - this.testExecutionContext.FrequencyOfRunStatsChangeEvent = 2; - TestRunChangedEventArgs receivedRunStatusArgs = null; - this.runTestsInstance = new TestableBaseRunTests( - null, - package, - testExecutionContext, - null, - this.mockTestRunEventsHandler.Object, - this.mockTestPlatformEventSource.Object, - this.mockRequestData.Object); - - var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; - var executorUriExtensionMap = new List> - { - new Tuple(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), - new Tuple(new Uri(BaseRunTestsExecutorUri), assemblyLocation) - }; - - // Setup mocks. - this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return executorUriExtensionMap; }; - this.runTestsInstance.InvokeExecutorCallback = - (executor, executorUriExtensionTuple, runContext, frameworkHandle) => - { - var testCase = new TestCase("x.y.z1", new Uri("uri://dummy"), "x.dll"); - var inProgTestCase = new TestCase("x.y.z2", new Uri("uri://dummy"), "x.dll"); - var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); - this.runTestsInstance.GetTestRunCache.OnTestStarted(inProgTestCase); - this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); - }; - this.mockTestRunEventsHandler.Setup( treh => treh.HandleTestRunStatsChange(It.IsAny())) - .Callback((TestRunChangedEventArgs stats) => - { - receivedRunStatusArgs = stats; - }); + const string package = @"C:\Porjects\UnitTestApp3\Debug\UnitTestApp3\UnitTestApp3.build.appxrecipe"; + this.mockDataSerializer.Setup(d => d.Clone(It.IsAny())) + .Returns(t => JsonDataSerializer.Instance.Clone(t)); + this.SetUpTestRunEvents(package, setupHandleTestRunComplete:false); // Act. this.runTestsInstance.RunTests(); - // Test run changed event assertions - Assert.IsNotNull(receivedRunStatusArgs.NewTestResults); - Assert.IsTrue(receivedRunStatusArgs.NewTestResults.Count() > 0); - Assert.IsNotNull(receivedRunStatusArgs.ActiveTests); - Assert.AreEqual(receivedRunStatusArgs.ActiveTests.Count(), 1); + Assert.AreEqual(1, receivedRunStatusArgs.ActiveTests.Count()); - // verify TC.Source is updated with package - foreach (var tr in receivedRunStatusArgs.NewTestResults) - { - Assert.AreEqual(tr.TestCase.Source, package); - } foreach (var tc in receivedRunStatusArgs.ActiveTests) { @@ -688,6 +588,38 @@ public void RunTestsShouldUpdateTestCasesWithPackageWhenCacheIsHitIfProvided() } } + [TestMethod] + public void RunTestsShouldCloneTheActiveTestCaseObjectsIfTestSourceIsPackage() + { + const string package = @"C:\Porjects\UnitTestApp3\Debug\UnitTestApp3\UnitTestApp3.build.appxrecipe"; + + this.SetUpTestRunEvents(package, setupHandleTestRunComplete: false); + + // Act. + this.runTestsInstance.RunTests(); + + Assert.IsNotNull(receivedRunStatusArgs.ActiveTests); + Assert.AreEqual(1, receivedRunStatusArgs.ActiveTests.Count()); + + this.mockDataSerializer.Verify(d => d.Clone(It.IsAny()), Times.Exactly(2)); + } + + [TestMethod] + public void RunTestsShouldCloneTheTestResultsObjectsIfTestSourceIsPackage() + { + const string package = @"C:\Porjects\UnitTestApp3\Debug\UnitTestApp3\UnitTestApp3.build.appxrecipe"; + + this.SetUpTestRunEvents(package, setupHandleTestRunComplete: false); + + // Act. + this.runTestsInstance.RunTests(); + + Assert.IsNotNull(receivedRunStatusArgs.NewTestResults); + Assert.AreEqual(1, receivedRunStatusArgs.ActiveTests.Count()); + + this.mockDataSerializer.Verify(d => d.Clone(It.IsAny()), Times.Exactly(2)); + } + [TestMethod] public void RunTestsShouldNotifyItsImplementersOfAnyExceptionThrownByTheExecutors() { @@ -761,7 +693,8 @@ public void RunTestsShouldSendMetricsOnTestRunComplete() It.IsAny>(), It.IsAny>())) .Callback( - (TestRunCompleteEventArgs complete, + ( + TestRunCompleteEventArgs complete, TestRunChangedEventArgs stats, ICollection attachments, ICollection executorUris) => @@ -803,7 +736,6 @@ public void RunTestsShouldCollectMetrics() mockMetricsCollector.Setup(mc => mc.Metrics).Returns(dict); this.mockRequestData.Setup(rd => rd.MetricsCollection).Returns(mockMetricsCollector.Object); - // Act. this.runTestsInstance.RunTests(); @@ -848,7 +780,7 @@ public void RunTestsShouldRunTestsInMTAThreadWhenRunningInSTAThreadFails() public void CancelShouldCreateSTAThreadIfExecutionThreadApartmentStateIsSTA() { this.SetupForExecutionThreadApartmentStateTests(PlatformApartmentState.STA); - mockThread.Setup(mt => mt.Run(It.IsAny(), PlatformApartmentState.STA, It.IsAny())) + this.mockThread.Setup(mt => mt.Run(It.IsAny(), PlatformApartmentState.STA, It.IsAny())) .Callback((action, start, waitForCompletion) => { if (waitForCompletion) @@ -891,18 +823,21 @@ private void SetupForExecutionThreadApartmentStateTests(PlatformApartmentState a this.mockThread = new Mock(); this.runTestsInstance = new TestableBaseRunTests( + this.mockRequestData.Object, + null, $@" {apartmentState} ", - testExecutionContext, + this.testExecutionContext, null, this.mockTestRunEventsHandler.Object, this.mockTestPlatformEventSource.Object, null, this.mockThread.Object, - this.mockRequestData.Object); + this.mockDataSerializer.Object); + TestPluginCacheTests.SetupMockExtensions(new string[] { typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location }, () => { }); var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; var executorUriExtensionMap = new List> @@ -911,6 +846,98 @@ private void SetupForExecutionThreadApartmentStateTests(PlatformApartmentState a }; this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return executorUriExtensionMap; }; } + + private void SetUpTestRunEvents(string package = null, bool setupHandleTestRunComplete = true) + { + if (setupHandleTestRunComplete) + { + this.SetupHandleTestRunComplete(); + } + else + { + this.SetupHandleTestRunStatsChange(); + } + + this.SetupDataSerializer(); + + this.runTestsInstance = this.runTestsInstance = new TestableBaseRunTests( + this.mockRequestData.Object, + package, + null, + this.testExecutionContext, + null, + this.mockTestRunEventsHandler.Object, + this.mockTestPlatformEventSource.Object, + null, + new PlatformThread(), + this.mockDataSerializer.Object); + + var assemblyLocation = typeof(BaseRunTestsTests).GetTypeInfo().Assembly.Location; + var executorUriExtensionMap = new List> + { + new Tuple(new Uri(BadBaseRunTestsExecutorUri), assemblyLocation), + new Tuple(new Uri(BaseRunTestsExecutorUri), assemblyLocation) + }; + + // Setup mocks. + this.SetupExecutorCallback(executorUriExtensionMap); + } + + private void SetupExecutorCallback(List> executorUriExtensionMap) + { + this.runTestsInstance.GetExecutorUriExtensionMapCallback = (fh, rc) => { return executorUriExtensionMap; }; + this.runTestsInstance.InvokeExecutorCallback = + (executor, executorUriExtensionTuple, runContext, frameworkHandle) => + { + var testCase = new TestCase("x.y.z", new Uri("uri://dummy"), "x.dll"); + var testResult = new Microsoft.VisualStudio.TestPlatform.ObjectModel.TestResult(testCase); + this.inProgressTestCase = new TestCase("x.y.z2", new Uri("uri://dummy"), "x.dll"); + + this.runTestsInstance.GetTestRunCache.OnTestStarted(inProgressTestCase); + this.runTestsInstance.GetTestRunCache.OnNewTestResult(testResult); + }; + } + + private void SetupDataSerializer() + { + this.mockDataSerializer.Setup(d => d.Clone(It.IsAny())) + .Returns(t => JsonDataSerializer.Instance.Clone(t)); + + this.mockDataSerializer.Setup(d => d.Clone(It.IsAny())) + .Returns(t => JsonDataSerializer.Instance.Clone(t)); + } + + private void SetupHandleTestRunStatsChange() + { + this.testExecutionContext.FrequencyOfRunStatsChangeEvent = 2; + this.mockTestRunEventsHandler + .Setup(treh => treh.HandleTestRunStatsChange(It.IsAny())) + .Callback((TestRunChangedEventArgs stats) => { receivedRunStatusArgs = stats; }); + } + + private void SetupHandleTestRunComplete() + { + this.mockTestRunEventsHandler.Setup( + treh => + treh.HandleTestRunComplete( + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny>())) + .Callback( + ( + TestRunCompleteEventArgs complete, + TestRunChangedEventArgs stats, + ICollection attachments, + ICollection executorUris) => + { + receivedRunCompleteArgs = complete; + this.receivedRunStatusArgs = stats; + receivedattachments = attachments; + receivedExecutorUris = executorUris; + }); + } + #endregion #region Testable Implementation @@ -918,18 +945,8 @@ private void SetupForExecutionThreadApartmentStateTests(PlatformApartmentState a private class TestableBaseRunTests : BaseRunTests { public TestableBaseRunTests( - string runSettings, + IRequestData requestData, string package, - TestExecutionContext testExecutionContext, - ITestCaseEventsHandler testCaseEventsHandler, - ITestRunEventsHandler testRunEventsHandler, - ITestPlatformEventSource testPlatformEventSource, - IRequestData requestData) - : base(requestData, package, runSettings, testExecutionContext, testCaseEventsHandler, testRunEventsHandler, testPlatformEventSource) - { - } - - public TestableBaseRunTests( string runSettings, TestExecutionContext testExecutionContext, ITestCaseEventsHandler testCaseEventsHandler, @@ -937,11 +954,22 @@ public TestableBaseRunTests( ITestPlatformEventSource testPlatformEventSource, ITestEventsPublisher testEventsPublisher, IThread platformThread, - IRequestData requestData) - : base(requestData, null, runSettings, testExecutionContext, testCaseEventsHandler, testRunEventsHandler, testPlatformEventSource, testEventsPublisher, platformThread) + IDataSerializer dataSerializer) + : base( + requestData, + package, + runSettings, + testExecutionContext, + testCaseEventsHandler, + testRunEventsHandler, + testPlatformEventSource, + testEventsPublisher, + platformThread, + dataSerializer) { } + public Action BeforeRaisingTestRunCompleteCallback { get; set; } public Func>> GetExecutorUriExtensionMapCallback { get; set; } @@ -1001,17 +1029,14 @@ private class TestExecutor : ITestExecutor { public void Cancel() { - } public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { - } public void RunTests(IEnumerable tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { - } } diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj index 5b2fe5179b..3f46352a58 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj @@ -8,7 +8,7 @@ Exe Microsoft.TestPlatform.CrossPlatEngine.UnitTests - netcoreapp1.0;net451 + net451;netcoreapp1.0