diff --git a/src/vstest.console/Internal/ConsoleLogger.cs b/src/vstest.console/Internal/ConsoleLogger.cs index 9a2c8367d0..e046e46ecc 100644 --- a/src/vstest.console/Internal/ConsoleLogger.cs +++ b/src/vstest.console/Internal/ConsoleLogger.cs @@ -4,6 +4,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Internal { using System; + using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; @@ -85,6 +86,16 @@ internal class ConsoleLogger : ITestLoggerWithParameters /// public const string NoProgressParam = "noprogress"; + /// + /// Property Id storing the ParentExecutionId. + /// + public const string ParentExecutionIdPropertyIdentifier = "ParentExecId"; + + /// + /// Property Id storing the ExecutionId. + /// + public const string ExecutionIdPropertyIdentifier = "ExecutionId"; + #endregion internal enum Verbosity @@ -107,11 +118,8 @@ internal enum Verbosity private Verbosity verbosityLevel = Verbosity.Minimal; #endif - private int testsTotal = 0; - private int testsPassed = 0; - private int testsFailed = 0; - private int testsSkipped = 0; private bool testRunHasErrorMessages = false; + private ConcurrentDictionary leafExecutionIdAndTestOutcomePairDictionary = new ConcurrentDictionary(); #endregion @@ -371,6 +379,39 @@ private static void DisplayFullInformation(TestResult result) } } + /// + /// Returns the parent Execution id of given test result. + /// + /// + /// + private Guid GetParentExecutionId(TestResult testResult) + { + var parentExecutionIdProperty = testResult.Properties.FirstOrDefault(property => + property.Id.Equals(ParentExecutionIdPropertyIdentifier)); + return parentExecutionIdProperty == null + ? Guid.Empty + : testResult.GetPropertyValue(parentExecutionIdProperty, Guid.Empty); + } + + /// + /// Returns execution id of given test result + /// + /// + /// + private Guid GetExecutionId(TestResult testResult) + { + var executionIdProperty = testResult.Properties.FirstOrDefault(property => + property.Id.Equals(ExecutionIdPropertyIdentifier)); + var executionId = Guid.Empty; + + if (executionIdProperty != null) + { + executionId = testResult.GetPropertyValue(executionIdProperty, Guid.Empty); + } + + return executionId.Equals(Guid.Empty) ? Guid.NewGuid() : executionId; + } + #endregion #region Event Handlers @@ -468,9 +509,6 @@ private void TestResultHandler(object sender, TestResultEventArgs e) ValidateArg.NotNull(sender, "sender"); ValidateArg.NotNull(e, "e"); - // Update the test count statistics based on the result of the test. - this.testsTotal++; - var testDisplayName = e.Result.DisplayName; if (string.IsNullOrWhiteSpace(e.Result.DisplayName)) @@ -484,11 +522,20 @@ private void TestResultHandler(object sender, TestResultEventArgs e) testDisplayName = string.Format("{0} [{1}]", testDisplayName, formattedDuration); } + var executionId = GetExecutionId(e.Result); + var parentExecutionId = GetParentExecutionId(e.Result); + + if (parentExecutionId != Guid.Empty && leafExecutionIdAndTestOutcomePairDictionary.ContainsKey(parentExecutionId)) + { + leafExecutionIdAndTestOutcomePairDictionary.TryRemove(parentExecutionId, out _); + } + + leafExecutionIdAndTestOutcomePairDictionary.TryAdd(executionId, e.Result.Outcome); + switch (e.Result.Outcome) { case TestOutcome.Skipped: { - this.testsSkipped++; if (this.verbosityLevel == Verbosity.Quiet) { break; @@ -512,7 +559,6 @@ private void TestResultHandler(object sender, TestResultEventArgs e) case TestOutcome.Failed: { - this.testsFailed++; if (this.verbosityLevel == Verbosity.Quiet) { break; @@ -533,7 +579,6 @@ private void TestResultHandler(object sender, TestResultEventArgs e) case TestOutcome.Passed: { - this.testsPassed++; if (this.verbosityLevel == Verbosity.Normal || this.verbosityLevel == Verbosity.Detailed) { // Pause the progress indicator before displaying test result information @@ -617,6 +662,10 @@ private string GetFormattedDurationString(TimeSpan duration) /// private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) { + var testsTotal = 0; + var testsPassed = 0; + var testsFailed = 0; + var testsSkipped = 0; // Stop the progress indicator as we are about to print the summary this.progressIndicator?.Stop(); Output.WriteLine(string.Empty, OutputLevel.Information); @@ -636,6 +685,25 @@ private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) } } + foreach (KeyValuePair entry in leafExecutionIdAndTestOutcomePairDictionary) + { + testsTotal++; + switch (entry.Value) + { + case TestOutcome.Failed: + testsFailed++; + break; + case TestOutcome.Passed: + testsPassed++; + break; + case TestOutcome.Skipped: + testsSkipped++; + break; + default: + break; + } + } + if (e.IsCanceled) { Output.Error(false, CommandLineResources.TestRunCanceled); @@ -644,7 +712,7 @@ private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) { Output.Error(false, CommandLineResources.TestRunAborted); } - else if (this.testsFailed > 0 || this.testRunHasErrorMessages) + else if (testsFailed > 0 || this.testRunHasErrorMessages) { Output.Error(false, CommandLineResources.TestRunFailed); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs index b9e6f82917..533b64989e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/OrderedTests.cs @@ -49,7 +49,7 @@ public void OlderOrderedTestsShouldWorkFine(RunnerInfo runnerInfo) this.ValidateFailedTests("FailingTest1"); this.ValidateSkippedTests("FailingTest2"); // Parent test result should fail as inner results contain failing test. - this.ValidateSummaryStatus(2, 2, 1); + this.ValidateSummaryStatus(2, 1, 1); } } } diff --git a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs index ea4b011ec1..e7e29ed607 100644 --- a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs +++ b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs @@ -1046,6 +1046,51 @@ public void AttachmentInformationShouldBeWrittenToConsoleIfAttachmentsArePresent this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.AttachmentOutputFormat, uriDataAttachment1.Uri.LocalPath), OutputLevel.Information), Times.Once()); } + [TestMethod] + public void ResultsInHeirarchichalOrderShouldReportCorrectCount() + { + var loggerEvents = new InternalTestLoggerEvents(TestSessionMessageLogger.Instance); + loggerEvents.EnableEvents(); + var parameters = new Dictionary(); + parameters.Add("verbosity", "normal"); + this.consoleLogger.Initialize(loggerEvents, parameters); + + TestCase testCase1 = CreateTestCase("TestCase1"); + TestCase testCase2 = CreateTestCase("TestCase2"); + TestCase testCase3 = CreateTestCase("TestCase3"); + + Guid parentExecutionId = Guid.NewGuid(); + TestProperty ParentExecIdProperty = TestProperty.Register("ParentExecId", "ParentExecId", typeof(Guid), TestPropertyAttributes.Hidden, typeof(ObjectModel.TestResult)); + TestProperty ExecutionIdProperty = TestProperty.Register("ExecutionId", "ExecutionId", typeof(Guid), TestPropertyAttributes.Hidden, typeof(ObjectModel.TestResult)); + TestProperty TestTypeProperty = TestProperty.Register("TestType", "TestType" , typeof(Guid), TestPropertyAttributes.Hidden, typeof(ObjectModel.TestResult)); + + var result1 = new ObjectModel.TestResult(testCase1) { Outcome = TestOutcome.Failed }; + result1.SetPropertyValue(ExecutionIdProperty, parentExecutionId); + + var result2 = new ObjectModel.TestResult(testCase2) { Outcome = TestOutcome.Passed}; + result2.SetPropertyValue(ExecutionIdProperty, Guid.NewGuid()); + result2.SetPropertyValue(ParentExecIdProperty, parentExecutionId); + + var result3 = new ObjectModel.TestResult(testCase3) { Outcome = TestOutcome.Failed }; + result3.SetPropertyValue(ExecutionIdProperty, Guid.NewGuid()); + result3.SetPropertyValue(ParentExecIdProperty, parentExecutionId); + + loggerEvents.RaiseTestResult(new TestResultEventArgs(result1)); + loggerEvents.RaiseTestResult(new TestResultEventArgs(result2)); + loggerEvents.RaiseTestResult(new TestResultEventArgs(result3)); + + loggerEvents.CompleteTestRun(null, false, false, null, null, new TimeSpan(1, 0, 0, 0)); + + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryFailedTests, 1), OutputLevel.Information), Times.Once()); + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryPassedTests, 1), OutputLevel.Information), Times.Once()); + this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.TestRunSummaryTotalTests, 2), OutputLevel.Information), Times.Once()); + } + + private TestCase CreateTestCase(string testCaseName) + { + return new TestCase(testCaseName, new Uri("some://uri"), "DummySourceFileName"); + } + private void Setup() { this.mockRequestData = new Mock();