diff --git a/scripts/test.ps1 b/scripts/test.ps1 index 76de70f85a..8a497b9cb7 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -63,15 +63,18 @@ $Script:TPT_SkipProjects = @("testhost.UnitTests", "vstest.console.UnitTests") $Script:TPT_Pattern = $Pattern $Script:TPT_FailFast = $FailFast $Script:TPT_Parallel = $Parallel +$Script:TPT_TestResultsDir = Join-Path $env:TP_ROOT_DIR "TestResults" +$Script:TPT_DefaultTrxFileName = "TrxLogResults.trx" +$Script:TPT_ErrorMsgColor = "Red" # # Capture error state in any step globally to modify return code $Script:ScriptFailed = $false -function Write-Log ([string] $message) +function Write-Log ([string] $message, $messageColor = "Green") { $currentColor = $Host.UI.RawUI.ForegroundColor - $Host.UI.RawUI.ForegroundColor = "Green" + $Host.UI.RawUI.ForegroundColor = $messageColor if ($message) { Write-Output "... $message" @@ -84,6 +87,19 @@ function Write-VerboseLog([string] $message) Write-Verbose $message } +function Print-FailedTests($TrxFilePath) +{ + if(![System.IO.File]::Exists($TrxFilePath)){ + Write-Log "TrxFile: $TrxFilePath doesn't exists" + return + } + $xdoc = [xml] (get-content $TrxFilePath) + $FailedTestIds = $xdoc.TestRun.Results.UnitTestResult |?{$_.GetAttribute("outcome") -eq "Failed"} | %{$_.testId} + if ($FailedTestIds) { + Write-Log (".. .. . " + ($xdoc.TestRun.TestDefinitions.UnitTest | ?{ $FailedTestIds.Contains($_.GetAttribute("id")) } | %{ "$($_.TestMethod.className).$($_.TestMethod.name)"})) $Script:TPT_ErrorMsgColor + } +} + function Invoke-Test { $timer = Start-Timer @@ -147,16 +163,16 @@ function Invoke-Test if ($TPT_Parallel) { # Fill in the framework in test containers $testContainerSet = $testContainers | % { [System.String]::Format($_, $fx) } - + $trxLogFileName = [System.String]::Format("Parallel_{0}_{1}", $fx, $Script:TPT_DefaultTrxFileName) Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR){ + if($fx -eq $TPT_TargetFrameworkFullCLR) { - Write-Verbose "$vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /parallel" - $output = & $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /parallel - }else{ + Write-Verbose "$vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /parallel /logger:`"trx;LogFileName=$trxLogFileName`"" + $output = & $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /parallel /logger:"trx;LogFileName=$trxLogFileName" + } else { - Write-Verbose "$dotNetPath $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /parallel" - $output = & $dotNetPath $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /parallel + Write-Verbose "$dotNetPath $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /parallel /logger:`"trx;LogFileName=$trxLogFileName`"" + $output = & $dotNetPath $vstestConsolePath $testContainerSet /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /parallel /logger:"trx;LogFileName=$trxLogFileName" } Reset-TestEnvironment @@ -165,8 +181,8 @@ function Invoke-Test Write-Log ".. . $($output[-3])" } else { Write-Log ".. . $($output[-2])" - Write-Log ".. . Failed tests:" - Write-Log ".. . $($output -match '^Failed')" + Write-Log ".. . Failed tests:" $Script:TPT_ErrorMsgColor + Print-FailedTests (Join-Path $Script:TPT_TestResultsDir $trxLogFileName) Set-ScriptFailed @@ -179,28 +195,29 @@ function Invoke-Test $testContainers | % { # Fill in the framework in test containers $testContainer = [System.String]::Format($_, $fx) + $trxLogFileName = [System.String]::Format("{0}_{1}_{2}", ($(Get-ChildItem $testContainer).Name), $fx, $Script:TPT_DefaultTrxFileName) + Write-Log ".. Container: $testContainer" Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR){ + if($fx -eq $TPT_TargetFrameworkFullCLR) { - Write-Verbose "$vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath" - $output = & $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" - }else{ + Write-Verbose "$vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /logger:`"trx;LogFileName=$trxLogFileName`"" + $output = & $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /logger:"trx;LogFileName=$trxLogFileName" + } else { - Write-Verbose "$dotNetPath $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath" - $output = & $dotNetPath $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" + Write-Verbose "$dotNetPath $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:$testAdapterPath /logger:`"trx;LogFileName=$trxLogFileName`"" + $output = & $dotNetPath $vstestConsolePath $testContainer /platform:$testArchitecture /framework:$testFrameWork /testAdapterPath:"$testAdapterPath" /logger:"trx;LogFileName=$trxLogFileName" } Reset-TestEnvironment - if ($output[-2].Contains("Test Run Successful.")) { Write-Log ".. . $($output[-3])" } else { Write-Log ".. . $($output[-2])" - Write-Log ".. . Failed tests:" - Write-Log ".. . $($output -match '^Failed')" + Write-Log ".. . Failed tests:" $Script:TPT_ErrorMsgColor + Print-FailedTests (Join-Path $Script:TPT_TestResultsDir $trxLogFileName) Set-ScriptFailed diff --git a/src/Microsoft.TestPlatform.Common/Logging/TestLoggerManager.cs b/src/Microsoft.TestPlatform.Common/Logging/TestLoggerManager.cs index ca27486bb1..e0eb6694ba 100644 --- a/src/Microsoft.TestPlatform.Common/Logging/TestLoggerManager.cs +++ b/src/Microsoft.TestPlatform.Common/Logging/TestLoggerManager.cs @@ -385,8 +385,7 @@ private Dictionary UpdateLoggerParamters(Dictionary + /// Looks up a localized string similar to WARNING: Overwriting results file: {0}. + /// + internal static string TrxLoggerResultsFileOverwriteWarning + { + get + { + return ResourceManager.GetString("TrxLoggerResultsFileOverwriteWarning", resourceCulture); + } + } } } diff --git a/src/Microsoft.TestPlatform.Extensions.TrxLogger/Resources/TrxResource.resx b/src/Microsoft.TestPlatform.Extensions.TrxLogger/Resources/TrxResource.resx index 6b0cdf9f2e..9bfad88ae8 100644 --- a/src/Microsoft.TestPlatform.Extensions.TrxLogger/Resources/TrxResource.resx +++ b/src/Microsoft.TestPlatform.Extensions.TrxLogger/Resources/TrxResource.resx @@ -208,4 +208,7 @@ Error Details: {1}:{2} Results Not in a List + + WARNING: Overwriting results file: {0} + \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.Extensions.TrxLogger/TrxLogger.cs b/src/Microsoft.TestPlatform.Extensions.TrxLogger/TrxLogger.cs index 0312340891..b1a686f38c 100644 --- a/src/Microsoft.TestPlatform.Extensions.TrxLogger/TrxLogger.cs +++ b/src/Microsoft.TestPlatform.Extensions.TrxLogger/TrxLogger.cs @@ -28,7 +28,7 @@ namespace Microsoft.VisualStudio.TestPlatform.Extensions.TrxLogger /// [FriendlyName(TrxLogger.FriendlyName)] [ExtensionUri(TrxLogger.ExtensionUri)] - internal class TrxLogger : ITestLogger + internal class TrxLogger : ITestLoggerWithParameters { #region Constants @@ -47,14 +47,19 @@ internal class TrxLogger : ITestLogger /// public const string DataCollectorUriPrefix = "dataCollector://"; + /// + /// Log file parameter key + /// + public const string LogFileNameKey = "LogFileName"; + #endregion #region Fields /// - /// Cache the TRX filename + /// Cache the TRX file path /// - private static string trxFileName; + private string trxFilePath; private TrxLoggerObjectModel.TestRun testRun; private List results; @@ -75,35 +80,31 @@ internal class TrxLogger : ITestLogger private DateTime testRunStartTime; - #endregion + /// + /// Parameters dictionary for logger. Ex: {"LogFileName":"TestResults.trx"}. + /// + private Dictionary parametersDictionary; /// - /// Gets the directory under which trx file should be saved. + /// Gets the directory under which default trx file and test results attachements should be saved. /// - [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] - public static string TrxFileDirectory - { - get; - internal set; - } + private string testResultsDirPath; + + #endregion #region ITestLogger - /// - /// Initializes the Test Logger. - /// - /// Events that can be registered for. - /// Test Run Directory - public void Initialize(TestLoggerEvents events, string testRunDirectory) + /// + public void Initialize(TestLoggerEvents events, string testResultsDirPath) { if (events == null) { throw new ArgumentNullException(nameof(events)); } - if (string.IsNullOrEmpty(testRunDirectory)) + if (string.IsNullOrEmpty(testResultsDirPath)) { - throw new ArgumentNullException(nameof(testRunDirectory)); + throw new ArgumentNullException(nameof(testResultsDirPath)); } // Register for the events. @@ -111,11 +112,27 @@ public void Initialize(TestLoggerEvents events, string testRunDirectory) events.TestResult += this.TestResultHandler; events.TestRunComplete += this.TestRunCompleteHandler; - TrxFileDirectory = testRunDirectory; + this.testResultsDirPath = testResultsDirPath; this.InitializeInternal(); } + /// + public void Initialize(TestLoggerEvents events, Dictionary parameters) + { + if (parameters == null) + { + throw new ArgumentNullException(nameof(parameters)); + } + + if (parameters.Count == 0) + { + throw new ArgumentException("No default parameters added", nameof(parameters)); + } + + this.parametersDictionary = parameters; + this.Initialize(events, this.parametersDictionary[DefaultLoggerParameterNames.TestRunDirectory]); + } #endregion #region ForTesting @@ -257,7 +274,7 @@ internal void TestResultHandler(object sender, ObjectModel.Logging.TestResultEve // Conver the rocksteady result to MSTest result TrxLoggerObjectModel.TestOutcome testOutcome = Converter.ToOutcome(e.Result.Outcome); - TrxLoggerObjectModel.UnitTestResult testResult = Converter.ToUnitTestResult(e.Result, testElement, testOutcome, this.testRun, TrxFileDirectory); + TrxLoggerObjectModel.UnitTestResult testResult = Converter.ToUnitTestResult(e.Result, testElement, testOutcome, this.testRun, this.testResultsDirPath); // Set various counts (passtests, failed tests, total tests) this.totalTests++; @@ -331,8 +348,8 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) } List errorMessages = new List(); - List collectorEntries = Converter.ToCollectionEntries(e.AttachmentSets, this.testRun, TrxFileDirectory); - IList resultFiles = Converter.ToResultFiles(e.AttachmentSets, this.testRun, TrxFileDirectory, errorMessages); + List collectorEntries = Converter.ToCollectionEntries(e.AttachmentSets, this.testRun, this.testResultsDirPath); + IList resultFiles = Converter.ToResultFiles(e.AttachmentSets, this.testRun, this.testResultsDirPath, errorMessages); if (errorMessages.Count > 0) { @@ -358,19 +375,9 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) helper.SaveObject(runSummary, rootElement, "ResultSummary", parameters); - if (Directory.Exists(TrxFileDirectory) == false) - { - Directory.CreateDirectory(TrxFileDirectory); - } - - if (string.IsNullOrEmpty(trxFileName)) - { - // save the xml to file in testResultsFolder - // [RunDeploymentRootDirectory] Replace white space with underscore from trx file name to make it command line friendly - trxFileName = this.GetTrxFileName(TrxFileDirectory, this.testRun.RunConfiguration.RunDeploymentRootDirectory.Replace(' ', '_')); - } - - this.PopulateTrxFile(trxFileName, rootElement); + //Save results to Trx file + this.DeriveTrxFilePath(); + this.PopulateTrxFile(this.trxFilePath, rootElement); } } @@ -387,10 +394,27 @@ internal virtual void PopulateTrxFile(string trxFileName, XmlElement rootElement { try { - FileStream fs = File.OpenWrite(trxFileName); - rootElement.OwnerDocument.Save(fs); + var trxFileDirPath = Path.GetDirectoryName(trxFilePath); + if (Directory.Exists(trxFileDirPath) == false) + { + Directory.CreateDirectory(trxFileDirPath); + } + + if (File.Exists(trxFilePath)) + { + var overwriteWarningMsg = string.Format(CultureInfo.CurrentCulture, + TrxLoggerResources.TrxLoggerResultsFileOverwriteWarning, trxFileName); + Console.WriteLine(overwriteWarningMsg); + EqtTrace.Warning(overwriteWarningMsg); + } + + using (var fs = File.Open(trxFileName, FileMode.Create)) + { + rootElement.OwnerDocument.Save(fs); + } String resultsFileMessage = String.Format(CultureInfo.CurrentCulture, TrxLoggerResources.TrxLoggerResultsFile, trxFileName); Console.WriteLine(resultsFileMessage); + EqtTrace.Info(resultsFileMessage); } catch (System.UnauthorizedAccessException fileWriteException) { @@ -398,24 +422,6 @@ internal virtual void PopulateTrxFile(string trxFileName, XmlElement rootElement } } - /// - /// Get full path to trx file - /// - /// - /// The base Directory. - /// - /// - /// The trx File Name. - /// - /// - /// trx file name. - /// - [SuppressMessage("StyleCop.CSharp.DocumentationRules", "SA1650:ElementDocumentationMustBeSpelledCorrectly", Justification = "Reviewed. Suppression is OK here.")] - private string GetTrxFileName(string baseDirectory, string trxFileName) - { - return FileHelper.GetNextIterationFileName(baseDirectory, trxFileName + ".trx", false); - } - // Initializes trx logger cache. private void InitializeInternal() { @@ -454,6 +460,36 @@ private void HandleSkippedTest(ObjectModel.TestResult rsTestResult) this.AddRunLevelInformationalMessage(message); } + private void DeriveTrxFilePath() + { + if (this.parametersDictionary != null) + { + var isLogFileNameParameterExists = this.parametersDictionary.TryGetValue(TrxLogger.LogFileNameKey, out string logFileNameValue); + if (isLogFileNameParameterExists && !string.IsNullOrWhiteSpace(logFileNameValue)) + { + this.trxFilePath = Path.Combine(this.testResultsDirPath, logFileNameValue); + } + else + { + this.SetDefaultTrxFilePath(); + } + } + else + { + this.SetDefaultTrxFilePath(); + } + } + + /// + /// Sets auto generated Trx file name under test results directory. + /// + private void SetDefaultTrxFilePath() + { + // [RunDeploymentRootDirectory] Replace white space with underscore from trx file name to make it command line friendly + var defaultTrxFileName = this.testRun.RunConfiguration.RunDeploymentRootDirectory.Replace(' ', '_') + ".trx"; + this.trxFilePath = FileHelper.GetNextIterationFileName(this.testResultsDirPath, defaultTrxFileName, false); + } + #endregion } } \ No newline at end of file diff --git a/src/vstest.console/Processors/EnableLoggerArgumentProcessor.cs b/src/vstest.console/Processors/EnableLoggerArgumentProcessor.cs index 6663ff9d83..15f11e40ab 100644 --- a/src/vstest.console/Processors/EnableLoggerArgumentProcessor.cs +++ b/src/vstest.console/Processors/EnableLoggerArgumentProcessor.cs @@ -235,6 +235,7 @@ private void AddLoggerByUri(string argument, Dictionary paramete throw new CommandLineException(e.Message, e); } } + #endregion } } diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs index 387ff92ad4..4db509668a 100644 --- a/src/vstest.console/Resources/Resources.Designer.cs +++ b/src/vstest.console/Resources/Resources.Designer.cs @@ -381,12 +381,13 @@ public static string EnableDiagUsage { return ResourceManager.GetString("EnableDiagUsage", resourceCulture); } } - + /// /// Looks up a localized string similar to --logger|/logger:<Logger Uri/FriendlyName> /// Specify a logger for test results. For example, to log results into a - /// Visual Studio Test Results File (TRX) use /logger:trx. - /// To publish test results to Team Foundation Server, use TfsPublisher as shown below + /// Visual Studio Test Results File (TRX) use /logger:trx [;LogFileName=<Defaults to unique file name>] + /// Creates file in TestResults directory with given LogFileName. + /// /// Example: /logger:TfsPublisher; /// Collection=<team project collection url>; /// BuildName=<build name>; diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx index 2b5c618c5d..09b946025b 100644 --- a/src/vstest.console/Resources/Resources.resx +++ b/src/vstest.console/Resources/Resources.resx @@ -234,7 +234,9 @@ --logger|/logger:<Logger Uri/FriendlyName> Specify a logger for test results. For example, to log results into a - Visual Studio Test Results File (TRX) use /logger:trx. + Visual Studio Test Results File (TRX) use /logger:trx [;LogFileName=<Defaults to unique file name>] + Creates file in TestResults directory with given LogFileName. + To publish test results to Team Foundation Server, use TfsPublisher as shown below Example: /logger:TfsPublisher; Collection=<team project collection url>; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs new file mode 100644 index 0000000000..34ac55692e --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Xml; + +namespace Microsoft.TestPlatform.AcceptanceTests +{ + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class LoggerTests : AcceptanceTestBase + { + [CustomDataTestMethod] + [NET46TargetFramework] + [NETCORETargetFramework] + public void TrxLoggerShouldProperlyOverwriteFile(string runnerFramework, string targetFramework, string targetRuntime) + { + AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime); + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue); + var trxFileName = "TestResults.trx"; + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + this.InvokeVsTest(arguments); + + arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue); + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); + this.InvokeVsTest(arguments); + + var trxLogFilePath = Path.Combine(AppContext.BaseDirectory, "TestResults", trxFileName); + Assert.IsTrue(IsValidXml(trxLogFilePath), "Invalid content in Trx log file"); + } + + private bool IsValidXml(string xmlFilePath) + { + var reader = System.Xml.XmlReader.Create(File.OpenRead(xmlFilePath)); + try + { + while (reader.Read()) + { + } + return true; + } + catch (XmlException) + { + return false; + } + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index edaabd8963..dfc1ea5312 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -35,6 +35,7 @@ + diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Logging/TestLoggerManagerTests.cs b/test/Microsoft.TestPlatform.Common.UnitTests/Logging/TestLoggerManagerTests.cs index d55eeff355..a2d1e455f0 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Logging/TestLoggerManagerTests.cs +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Logging/TestLoggerManagerTests.cs @@ -314,6 +314,21 @@ public void AddLoggerShouldNotThrowExceptionIfUriIsNonExistent() }); } + [TestMethod] + public void AddLoggerShouldAddDefaultLoggerParameterForTestLoggerWithParameters() + { + ValidLoggerWithParameters.Reset(); + TestLoggerManager.Instance.AddLogger(new Uri("test-logger-with-parameter://logger"), new Dictionary()); + Assert.IsNotNull(ValidLoggerWithParameters.parameters, "parameters not getting passed"); + Assert.IsTrue( + ValidLoggerWithParameters.parameters.ContainsKey(DefaultLoggerParameterNames.TestRunDirectory), + $"{DefaultLoggerParameterNames.TestRunDirectory} not added to parameters"); + Assert.IsFalse( + string.IsNullOrWhiteSpace( + ValidLoggerWithParameters.parameters[DefaultLoggerParameterNames.TestRunDirectory]), + $"parameter {DefaultLoggerParameterNames.TestRunDirectory} should not be null, empty or whitespace"); + } + [TestMethod] public void DisposeShouldNotThrowExceptionIfCalledMultipleTimes() { @@ -393,6 +408,27 @@ private void TestMessageHandler(object sender, TestRunMessageEventArgs e) } + [ExtensionUri("test-logger-with-parameter://logger")] + [FriendlyName("TestLoggerWithParameterExtension")] + private class ValidLoggerWithParameters : ITestLoggerWithParameters + { + public static Dictionary parameters; + public void Initialize(TestLoggerEvents events, string testRunDirectory) + { + + } + + public void Initialize(TestLoggerEvents events, Dictionary parameters) + { + ValidLoggerWithParameters.parameters = parameters; + } + + public static void Reset() + { + ValidLoggerWithParameters.parameters = null; + } + } + internal class DummyTestLoggerManager : TestLoggerManager { public DummyTestLoggerManager() diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs index 25762f3a0e..23c438e264 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs @@ -31,6 +31,9 @@ public class TrxLoggerTests { private Mock events; private TestableTrxLogger testableTrxLogger; + private Dictionary parameters; + private static string DefaultTestRunDirectory = AppContext.BaseDirectory; + private static string DefaultLogFileNameParameterValue = "logfilevalue.trx"; [TestInitialize] public void Initialize() @@ -38,7 +41,10 @@ public void Initialize() this.events = new Mock(); this.testableTrxLogger = new TestableTrxLogger(); - this.testableTrxLogger.Initialize(this.events.Object, "dummy"); + this.parameters = new Dictionary(2); + this.parameters[DefaultLoggerParameterNames.TestRunDirectory] = TrxLoggerTests.DefaultTestRunDirectory; + this.parameters[TrxLogger.LogFileNameKey] = TrxLoggerTests.DefaultLogFileNameParameterValue; + this.testableTrxLogger.Initialize(this.events.Object, this.parameters); } [TestMethod] @@ -47,7 +53,7 @@ public void InitializeShouldThrowExceptionIfEventsIsNull() Assert.ThrowsException( () => { - this.testableTrxLogger.Initialize(null, "dummy"); + this.testableTrxLogger.Initialize(null, this.parameters); }); } @@ -55,7 +61,7 @@ public void InitializeShouldThrowExceptionIfEventsIsNull() public void InitializeShouldNotThrowExceptionIfEventsIsNotNull() { var events = new Mock(); - this.testableTrxLogger.Initialize(events.Object, "dummy"); + this.testableTrxLogger.Initialize(events.Object, this.parameters); } [TestMethod] @@ -65,7 +71,8 @@ public void InitializeShouldThrowExceptionIfTestRunDirectoryIsEmptyOrNull() () => { var events = new Mock(); - this.testableTrxLogger.Initialize(events.Object, null); + this.parameters[DefaultLoggerParameterNames.TestRunDirectory] = null; + this.testableTrxLogger.Initialize(events.Object, parameters); }); } @@ -73,7 +80,14 @@ public void InitializeShouldThrowExceptionIfTestRunDirectoryIsEmptyOrNull() public void InitializeShouldNotThrowExceptionIfTestRunDirectoryIsNeitherEmptyNorNull() { var events = new Mock(); - this.testableTrxLogger.Initialize(events.Object, "dummy"); + this.testableTrxLogger.Initialize(events.Object, this.parameters); + } + + [TestMethod] + public void InitializeShouldThrowExceptionIfParametersAreEmpty() + { + var events = new Mock(); + Assert.ThrowsException(() => this.testableTrxLogger.Initialize(events.Object, new Dictionary())); } [TestMethod] @@ -119,7 +133,7 @@ public void TestMessageHandlerShouldAddMessageInListIfItIsError() [TestMethod] public void TestResultHandlerShouldCaptureStartTimeInSummaryWithTimeStampDuringIntialize() { - ObjectModel.TestCase testCase = new ObjectModel.TestCase("dummy string", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase = CreateTestCase("dummy string"); ObjectModel.TestResult testResult = new ObjectModel.TestResult(testCase); Mock e = new Mock(testResult); @@ -131,10 +145,10 @@ public void TestResultHandlerShouldCaptureStartTimeInSummaryWithTimeStampDuringI [TestMethod] public void TestResultHandlerKeepingTheTrackOfPassedAndFailedTests() { - ObjectModel.TestCase passTestCase1 = new ObjectModel.TestCase("Pass1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase passTestCase2 = new ObjectModel.TestCase("Pass2", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase failTestCase1 = new ObjectModel.TestCase("Fail1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase skipTestCase1 = new ObjectModel.TestCase("Skip1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); + ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); + ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1"); + ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; @@ -167,10 +181,10 @@ public void TestResultHandlerKeepingTheTrackOfPassedAndFailedTests() [TestMethod] public void TestResultHandlerKeepingTheTrackOfTotalTests() { - ObjectModel.TestCase passTestCase1 = new ObjectModel.TestCase("Pass1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase passTestCase2 = new ObjectModel.TestCase("Pass2", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase failTestCase1 = new ObjectModel.TestCase("Fail1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase skipTestCase1 = new ObjectModel.TestCase("Skip1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); + ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); + ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1"); + ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; @@ -202,7 +216,7 @@ public void TestResultHandlerKeepingTheTrackOfTotalTests() [TestMethod] public void TestResultHandlerLockingAMessageForSkipTest() { - ObjectModel.TestCase skipTestCase1 = new ObjectModel.TestCase("Skip1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult skipResult1 = new ObjectModel.TestResult(skipTestCase1); skipResult1.Outcome = ObjectModel.TestOutcome.Skipped; @@ -219,8 +233,8 @@ public void TestResultHandlerLockingAMessageForSkipTest() [TestMethod] public void TestResultHandlerShouldCreateOneTestResultForEachTestCase() { - ObjectModel.TestCase testCase1 = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase testCase2 = new ObjectModel.TestCase("TestCase2", new Uri("some://uri"), "DummySourceFileName"); + var testCase1 = CreateTestCase("testCase1"); + ObjectModel.TestCase testCase2 = CreateTestCase("testCase2"); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.Outcome = ObjectModel.TestOutcome.Skipped; @@ -240,8 +254,8 @@ public void TestResultHandlerShouldCreateOneTestResultForEachTestCase() [TestMethod] public void TestResultHandlerShouldCreateOneTestEntryForEachTestCase() { - ObjectModel.TestCase testCase1 = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase testCase2 = new ObjectModel.TestCase("TestCase2", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); + ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2"); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); result1.Outcome = ObjectModel.TestOutcome.Skipped; @@ -261,8 +275,8 @@ public void TestResultHandlerShouldCreateOneTestEntryForEachTestCase() [TestMethod] public void TestResultHandlerShouldCreateOneUnitTestElementForEachTestCase() { - ObjectModel.TestCase testCase1 = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase testCase2 = new ObjectModel.TestCase("TestCase2", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); + ObjectModel.TestCase testCase2 = CreateTestCase("TestCase2"); ObjectModel.TestResult result1 = new ObjectModel.TestResult(testCase1); @@ -281,10 +295,10 @@ public void TestResultHandlerShouldCreateOneUnitTestElementForEachTestCase() [TestMethod] public void OutcomeOfRunWillBeFailIfAnyTestsFails() { - ObjectModel.TestCase passTestCase1 = new ObjectModel.TestCase("Pass1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase passTestCase2 = new ObjectModel.TestCase("Pass2", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase failTestCase1 = new ObjectModel.TestCase("Fail1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase skipTestCase1 = new ObjectModel.TestCase("Skip1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); + ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); + ObjectModel.TestCase failTestCase1 = CreateTestCase("Fail1"); + ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; @@ -308,9 +322,8 @@ public void OutcomeOfRunWillBeFailIfAnyTestsFails() this.testableTrxLogger.TestResultHandler(new object(), fail1.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan(1, 0, 0, 0)); + var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs(); - TestableTrxLogger.TrxFileDirectory = Directory.GetCurrentDirectory(); this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Failed, this.testableTrxLogger.TestResultOutcome); @@ -319,9 +332,9 @@ public void OutcomeOfRunWillBeFailIfAnyTestsFails() [TestMethod] public void OutcomeOfRunWillBeCompletedIfNoTestsFails() { - ObjectModel.TestCase passTestCase1 = new ObjectModel.TestCase("Pass1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase passTestCase2 = new ObjectModel.TestCase("Pass2", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestCase skipTestCase1 = new ObjectModel.TestCase("Skip1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase passTestCase1 = CreateTestCase("Pass1"); + ObjectModel.TestCase passTestCase2 = CreateTestCase("Pass2"); + ObjectModel.TestCase skipTestCase1 = CreateTestCase("Skip1"); ObjectModel.TestResult passResult1 = new ObjectModel.TestResult(passTestCase1); passResult1.Outcome = ObjectModel.TestOutcome.Passed; @@ -340,30 +353,44 @@ public void OutcomeOfRunWillBeCompletedIfNoTestsFails() this.testableTrxLogger.TestResultHandler(new object(), pass2.Object); this.testableTrxLogger.TestResultHandler(new object(), skip1.Object); - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan(1, 0, 0, 0)); + var testRunCompleteEventArgs = CreateTestRunCompleteEventArgs(); - TestableTrxLogger.TrxFileDirectory = Directory.GetCurrentDirectory(); this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); - Assert.AreEqual(TrxLoggerObjectModel.TestOutcome.Completed, this.testableTrxLogger.TestResultOutcome); } [TestMethod] public void TheDefaultTrxFileNameShouldNotHaveWhiteSpace() { - ObjectModel.TestCase passTestCase = new ObjectModel.TestCase("Pass1", new Uri("some://uri"), "DummySourceFileName"); - ObjectModel.TestResult passResult = new ObjectModel.TestResult(passTestCase); - Mock pass = new Mock(passResult); + // To create default trx file, log file parameter should be null. + this.parameters[TrxLogger.LogFileNameKey] = null; + this.testableTrxLogger.Initialize(this.events.Object, this.parameters); - this.testableTrxLogger.TestResultHandler(new object(), pass.Object); + this.MakeTestRunComplete(); + + bool trxFileNameContainsWhiteSpace = Path.GetFileName(this.testableTrxLogger.trxFile).Contains(' '); + Assert.IsFalse(trxFileNameContainsWhiteSpace, $"\"{this.testableTrxLogger.trxFile}\": Trx file name should not have white spaces"); + } - var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, new Collection(), new TimeSpan(1, 0, 0, 0)); + [TestMethod] + public void DefaultTrxFileShouldCreateIfLogFileNameParameterNotPassed() + { + // To create default trx file, If LogFileName parameter not passed + this.parameters.Remove(TrxLogger.LogFileNameKey); + this.testableTrxLogger.Initialize(this.events.Object, this.parameters); - TestableTrxLogger.TrxFileDirectory = Directory.GetCurrentDirectory(); - this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); + this.MakeTestRunComplete(); - bool trxFileName = Path.GetFileName(this.testableTrxLogger.trxFile).Contains(' '); + Assert.IsFalse(string.IsNullOrWhiteSpace(this.testableTrxLogger.trxFile)); + } + + [TestMethod] + public void CustomTrxFileNameShouldConstructFromLogFileParameter() + { + this.MakeTestRunComplete(); + + Assert.AreEqual(Path.Combine(TrxLoggerTests.DefaultTestRunDirectory, TrxLoggerTests.DefaultLogFileNameParameterValue), this.testableTrxLogger.trxFile, "Wrong Trx file name"); } /// @@ -372,7 +399,7 @@ public void TheDefaultTrxFileNameShouldNotHaveWhiteSpace() [TestMethod] public void GetCustomPropertyValueFromTestCaseShouldReadCategoyrAttributesFromTestCase() { - ObjectModel.TestCase testCase1 = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase1 = CreateTestCase("TestCase1"); TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject)); testCase1.SetPropertyValue(testProperty, new[] { "ClassLevel", "AsmLevel" }); @@ -392,7 +419,7 @@ public void GetCustomPropertyValueFromTestCaseShouldReadCategoyrAttributesFromTe [TestMethod] public void GetQToolsTestElementFromTestCaseShouldAssignTestCategoryOfUnitTestElement() { - ObjectModel.TestCase testCase = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase = CreateTestCase("TestCase1"); ObjectModel.TestResult result = new ObjectModel.TestResult(testCase); TestProperty testProperty = TestProperty.Register("MSTestDiscoverer.TestCategory", "String array property", string.Empty, string.Empty, typeof(string[]), null, TestPropertyAttributes.Hidden, typeof(TestObject)); @@ -411,7 +438,7 @@ public void GetQToolsTestElementFromTestCaseShouldAssignTestCategoryOfUnitTestEl [TestMethod] public void GetQToolsTestElementFromTestCaseShouldNotFailWhenThereIsNoTestCategoreis() { - ObjectModel.TestCase testCase = new ObjectModel.TestCase("TestCase1", new Uri("some://uri"), "DummySourceFileName"); + ObjectModel.TestCase testCase = CreateTestCase("TestCase1"); ObjectModel.TestResult result = new ObjectModel.TestResult(testCase); TrxLoggerObjectModel.UnitTestElement unitTestElement = Converter.GetQToolsTestElementFromTestCase(result); @@ -420,6 +447,33 @@ public void GetQToolsTestElementFromTestCaseShouldNotFailWhenThereIsNoTestCatego CollectionAssert.AreEqual(expected, unitTestElement.TestCategories.ToArray()); } + + private static TestCase CreateTestCase(string testCaseName) + { + return new ObjectModel.TestCase(testCaseName, new Uri("some://uri"), "DummySourceFileName"); + } + + private static TestRunCompleteEventArgs CreateTestRunCompleteEventArgs() + { + var testRunCompleteEventArgs = new TestRunCompleteEventArgs(null, false, false, null, + new Collection(), new TimeSpan(1, 0, 0, 0)); + return testRunCompleteEventArgs; + } + + private static Mock CreatePassTestResultEventArgsMock() + { + TestCase passTestCase = CreateTestCase("Pass1"); + var passResult = new ObjectModel.TestResult(passTestCase); + return new Mock(passResult); + } + + private void MakeTestRunComplete() + { + var pass = TrxLoggerTests.CreatePassTestResultEventArgsMock(); + this.testableTrxLogger.TestResultHandler(new object(), pass.Object); + var testRunCompleteEventArgs = TrxLoggerTests.CreateTestRunCompleteEventArgs(); + this.testableTrxLogger.TestRunCompleteHandler(new object(), testRunCompleteEventArgs); + } } internal class TestableTrxLogger : TrxLogger