From 88da72d58db89c4bb9f9e53c28b092bf6ce51c9f Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Wed, 10 Nov 2021 17:32:37 +0100 Subject: [PATCH 01/11] Adding elements to code coverage config passed via commandline, e.g. Input: --collect:"Code Coverage;CLRIEInstrumentationNetCore=true;CLRIEInstrumentationNetFramework=false" Output: true false --- .../Processors/CollectArgumentProcessor.cs | 73 +++++++++++++++---- .../EnableCodeCoverageArgumentProcessor.cs | 2 +- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/vstest.console/Processors/CollectArgumentProcessor.cs b/src/vstest.console/Processors/CollectArgumentProcessor.cs index 8e22c90768..5cf690e3c5 100644 --- a/src/vstest.console/Processors/CollectArgumentProcessor.cs +++ b/src/vstest.console/Processors/CollectArgumentProcessor.cs @@ -5,9 +5,11 @@ namespace Microsoft.VisualStudio.TestPlatform.CommandLine.Processors { using System; using System.Collections.Generic; - using System.Globalization; using System.IO; + using System.Linq; + using System.Xml; + using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.Utilities; using Microsoft.VisualStudio.TestPlatform.Common; using Microsoft.VisualStudio.TestPlatform.Common.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Utilities; @@ -108,21 +110,22 @@ public void Initialize(string argument) // 1. Disable all other data collectors. Enable only those data collectors that are explicitly specified by user. // 2. Check if Code Coverage Data Collector is specified in runsettings, if not add it and also set enable to true. + string exceptionMessage = string.Format(CultureInfo.CurrentUICulture, CommandLineResources.DataCollectorFriendlyNameInvalid, argument); + // if argument is null or doesn't contain any element, don't do anything. if (string.IsNullOrWhiteSpace(argument)) { - throw new CommandLineException( - string.Format( - CultureInfo.CurrentUICulture, - CommandLineResources.DataCollectorFriendlyNameInvalid, - argument)); + throw new CommandLineException(exceptionMessage); } + // Get collect argument list. + var collectArgumentList = ArgumentProcessorUtilities.GetArgumentList(argument, ArgumentProcessorUtilities.SemiColonArgumentSeparator, exceptionMessage); + if (InferRunSettingsHelper.IsTestSettingsEnabled(this.runSettingsManager.ActiveRunSettings.SettingsXml)) { throw new SettingsException(string.Format(CommandLineResources.CollectWithTestSettingErrorMessage, argument)); } - AddDataCollectorToRunSettings(argument, this.runSettingsManager, this.fileHelper); + AddDataCollectorToRunSettings(collectArgumentList, this.runSettingsManager, this.fileHelper, exceptionMessage); } /// @@ -149,7 +152,7 @@ public ArgumentProcessorResult Execute() return ArgumentProcessorResult.Success; } - internal static void EnableDataCollectorUsingFriendlyName(string argument, DataCollectionRunSettings dataCollectionRunSettings) + internal static DataCollectorSettings EnableDataCollectorUsingFriendlyName(string argument, DataCollectionRunSettings dataCollectionRunSettings) { DataCollectorSettings dataCollectorSettings = null; @@ -164,6 +167,38 @@ internal static void EnableDataCollectorUsingFriendlyName(string argument, DataC { dataCollectorSettings.IsEnabled = true; } + + return dataCollectorSettings; + } + + private static void AddDataCollectorConfigurations(string[] configurations, DataCollectorSettings dataCollectorSettings, string exceptionMessage) + { + if (dataCollectorSettings.Configuration == null) + { + XmlDocument doc = new XmlDocument(); + dataCollectorSettings.Configuration = doc.CreateElement("Configuration"); + } + + foreach (var configuration in configurations) + { + var keyValuePair = ArgumentProcessorUtilities.GetArgumentList(configuration, ArgumentProcessorUtilities.EqualNameValueSeparator, exceptionMessage); + + if (keyValuePair.Length == 2) + { + AddConfiguration(dataCollectorSettings.Configuration, keyValuePair[0], keyValuePair[1]); + } + else + { + throw new CommandLineException(exceptionMessage); + } + } + } + + private static void AddConfiguration(XmlElement configuration, string configurationName, string configurationValue) + { + XmlElement attribute = configuration.OwnerDocument.CreateElement(configurationName); + attribute.InnerText = configurationValue; + configuration.AppendChild(attribute); } /// @@ -209,9 +244,16 @@ private static bool DoesDataCollectorSettingsExist(string friendlyName, return false; } - internal static void AddDataCollectorToRunSettings(string argument, IRunSettingsProvider runSettingsManager, IFileHelper fileHelper) + internal static void AddDataCollectorToRunSettings(string arguments, IRunSettingsProvider runSettingsManager, IFileHelper fileHelper) { - EnabledDataCollectors.Add(argument.ToLower()); + AddDataCollectorToRunSettings(new string[] { arguments }, runSettingsManager, fileHelper, string.Empty); + } + + internal static void AddDataCollectorToRunSettings(string[] arguments, IRunSettingsProvider runSettingsManager, IFileHelper fileHelper, string exceptionMessage) + { + var collectorName = arguments[0]; + var additionalConfigurations = arguments.Skip(1).ToArray(); + EnabledDataCollectors.Add(collectorName.ToLower()); var settings = runSettingsManager.ActiveRunSettings?.SettingsXml; if (settings == null) @@ -228,14 +270,19 @@ internal static void AddDataCollectorToRunSettings(string argument, IRunSettings Constants.InProcDataCollectorSettingName); // Add data collectors if not already present, enable if already present. - EnableDataCollectorUsingFriendlyName(argument, dataCollectionRunSettings); + var dataCollectorSettings = EnableDataCollectorUsingFriendlyName(collectorName, dataCollectionRunSettings); + + if (additionalConfigurations.Length > 0) + { + AddDataCollectorConfigurations(additionalConfigurations, dataCollectorSettings, exceptionMessage); + } runSettingsManager.UpdateRunSettingsNodeInnerXml(Constants.DataCollectionRunSettingsName, dataCollectionRunSettings.ToXml().InnerXml); - if (string.Equals(argument, CoverletConstants.CoverletDataCollectorFriendlyName, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(collectorName, CoverletConstants.CoverletDataCollectorFriendlyName, StringComparison.OrdinalIgnoreCase)) { // Add in-proc data collector to runsettings if coverlet code coverage is enabled - EnableCoverletInProcDataCollector(argument, inProcDataCollectionRunSettings, runSettingsManager, fileHelper); + EnableCoverletInProcDataCollector(collectorName, inProcDataCollectionRunSettings, runSettingsManager, fileHelper); runSettingsManager.UpdateRunSettingsNodeInnerXml(Constants.InProcDataCollectionRunSettingsName, inProcDataCollectionRunSettings.ToXml().InnerXml); } } diff --git a/src/vstest.console/Processors/EnableCodeCoverageArgumentProcessor.cs b/src/vstest.console/Processors/EnableCodeCoverageArgumentProcessor.cs index fc9c490749..a77e320fce 100644 --- a/src/vstest.console/Processors/EnableCodeCoverageArgumentProcessor.cs +++ b/src/vstest.console/Processors/EnableCodeCoverageArgumentProcessor.cs @@ -99,7 +99,7 @@ internal class EnableCodeCoverageArgumentExecutor : IArgumentExecutor private CommandLineOptions commandLineOptions; private IFileHelper fileHelper; - private const string FriendlyName = "Code Coverage"; + internal const string FriendlyName = "Code Coverage"; private static string xPathSeperator = "/"; private static string[] nodeNames = new string[] { Constants.RunSettingsName, Constants.DataCollectionRunSettingsName, Constants.DataCollectorsSettingName, Constants.DataCollectorSettingName }; From 92079228ed6254f1b31d0262817017071ae47772 Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Thu, 11 Nov 2021 11:18:01 +0100 Subject: [PATCH 02/11] Adding tests, --- .../CollectArgumentProcessorTests.cs | 79 ++++++++++++++++++- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs index ae7287cb06..afe8da89b7 100644 --- a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs @@ -7,13 +7,12 @@ namespace vstest.console.UnitTests.Processors using System.IO; using Microsoft.VisualStudio.TestPlatform.CommandLine; using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors; + using Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests; using Microsoft.VisualStudio.TestPlatform.Common; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; - using Microsoft.VisualStudio.TestPlatform.CommandLine.UnitTests; using Moq; - using static Microsoft.VisualStudio.TestPlatform.CommandLine.Processors.CollectArgumentExecutor; [TestClass] @@ -531,7 +530,7 @@ public void InitializeXPlatCodeCoverageShouldAddXPlatInProcProcDataCollectoPrope fileHelper.Setup(f => f.Exists(It.IsAny())).Returns(true); CollectArgumentExecutor executor = new CollectArgumentExecutor(settingsProvider, fileHelper.Object); executor.Initialize("XPlat Code Coverage"); - + Assert.AreEqual(string.Join(Environment.NewLine, "", "", @@ -586,6 +585,80 @@ public void InitializeXPlatCodeCoverageShouldAddXPlatInProcProcDataCollectoPrope ""), this.settingsProvider.ActiveRunSettings.SettingsXml); } + [TestMethod] + public void InitializeShouldThrowExceptionWhenInvalidConfigurationsProvided() + { + var runsettingsString = string.Format(DefaultRunSettings, ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(runsettingsString); + this.settingsProvider.SetActiveRunSettings(runsettings); + + Assert.ThrowsException(() => this.executor.Initialize("MyDataCollector;SomeSetting")); + } + + [TestMethod] + public void InitializeShouldCreateConfigurationsForNewDataCollectorInRunSettings() + { + var runsettingsString = string.Format(DefaultRunSettings, ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(runsettingsString); + this.settingsProvider.SetActiveRunSettings(runsettings); + + this.executor.Initialize("MyDataCollector;SomeSetting=SomeValue;AnotherSetting=AnotherValue"); + + Assert.IsNotNull(this.settingsProvider.ActiveRunSettings); + Assert.AreEqual(string.Join(Environment.NewLine, + "", + "", + " ", + " c:\\AdapterFolderPath", + " ", + " ", + " ", + " ", + " ", + " SomeValue", + " AnotherValue", + " ", + " ", + " ", + " ", + ""), this.settingsProvider.ActiveRunSettings.SettingsXml); + } + + [TestMethod] + public void InitializeShouldCreateConfigurationsForExistingDataCollectorInRunSettings() + { + var runsettingsString = string.Format(DefaultRunSettings, + "" + + " " + + " SomeValue" + + " " + + ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(runsettingsString); + this.settingsProvider.SetActiveRunSettings(runsettings); + + this.executor.Initialize("MyDataCollector;AnotherSetting=AnotherValue"); + + Assert.AreEqual(string.Join(Environment.NewLine, + "", + "", + " ", + " c:\\AdapterFolderPath", + " ", + " ", + " ", + " ", + " ", + " SomeValue", + " AnotherValue", + " ", + " ", + " ", + " ", + ""), this.settingsProvider.ActiveRunSettings.SettingsXml); + } #endregion } } From 3041a8317e4dbfacecb4e8cca14a2eddba2348cf Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Thu, 11 Nov 2021 12:39:22 +0100 Subject: [PATCH 03/11] Update existing configuration if present, --- .../Processors/CollectArgumentProcessor.cs | 22 ++++++++++--- .../CollectArgumentProcessorTests.cs | 33 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/vstest.console/Processors/CollectArgumentProcessor.cs b/src/vstest.console/Processors/CollectArgumentProcessor.cs index 5cf690e3c5..d30772d246 100644 --- a/src/vstest.console/Processors/CollectArgumentProcessor.cs +++ b/src/vstest.console/Processors/CollectArgumentProcessor.cs @@ -185,7 +185,7 @@ private static void AddDataCollectorConfigurations(string[] configurations, Data if (keyValuePair.Length == 2) { - AddConfiguration(dataCollectorSettings.Configuration, keyValuePair[0], keyValuePair[1]); + AddOrUpdateConfiguration(dataCollectorSettings.Configuration, keyValuePair[0], keyValuePair[1]); } else { @@ -194,11 +194,23 @@ private static void AddDataCollectorConfigurations(string[] configurations, Data } } - private static void AddConfiguration(XmlElement configuration, string configurationName, string configurationValue) + private static void AddOrUpdateConfiguration(XmlElement configuration, string configurationName, string configurationValue) { - XmlElement attribute = configuration.OwnerDocument.CreateElement(configurationName); - attribute.InnerText = configurationValue; - configuration.AppendChild(attribute); + var existingConfigurations = configuration.GetElementsByTagName(configurationName); + + // Update existing configuration if present. + if (existingConfigurations.Count == 0) + { + XmlElement newConfiguration = configuration.OwnerDocument.CreateElement(configurationName); + newConfiguration.InnerText = configurationValue; + configuration.AppendChild(newConfiguration); + return; + } + + foreach (XmlNode existingConfiguration in existingConfigurations) + { + existingConfiguration.InnerText = configurationValue; + } } /// diff --git a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs index afe8da89b7..0c5f3e6e52 100644 --- a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs @@ -659,6 +659,39 @@ public void InitializeShouldCreateConfigurationsForExistingDataCollectorInRunSet " ", ""), this.settingsProvider.ActiveRunSettings.SettingsXml); } + + [TestMethod] + public void InitializeShouldUpdateConfigurationsForExistingDataCollectorInRunSettings() + { + var runsettingsString = string.Format(DefaultRunSettings, + "" + + " " + + " SomeValue" + + " " + + ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(runsettingsString); + this.settingsProvider.SetActiveRunSettings(runsettings); + + this.executor.Initialize("MyDataCollector;SomeSetting=AnotherValue"); + + Assert.AreEqual(string.Join(Environment.NewLine, + "", + "", + " ", + " c:\\AdapterFolderPath", + " ", + " ", + " ", + " ", + " ", + " AnotherValue", + " ", + " ", + " ", + " ", + ""), this.settingsProvider.ActiveRunSettings.SettingsXml); + } #endregion } } From 016cb46eb301a2500f28ea7b7f87f587e96a8309 Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Thu, 11 Nov 2021 14:40:10 +0100 Subject: [PATCH 04/11] using correct tag for specifying output file format, --- .../TranslationLayerTests/CodeCoverageTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index d8c1f1fa2d..6d4cd2d745 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -414,7 +414,7 @@ private string GetCodeCoverageRunSettings(int cpuCount, bool useClrIeInstrumenta {useClrIeInstrumentationEngine} {useClrIeInstrumentationEngine} - {outputFormat} + {outputFormat} From 6325faed4cc88abba0476907de38dbd4f486d754 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 12 Nov 2021 07:22:10 +0000 Subject: [PATCH 05/11] Update dependencies from https://dev.azure.com/devdiv/DevDiv/_git/vs-code-coverage build 20211111.1 Microsoft.Internal.CodeCoverage From Version 17.1.0-beta.21560.1 -> To Version 17.1.0-beta.21561.1 --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 59b871bd27..72f4f448eb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,9 +1,9 @@ - + https://dev.azure.com/devdiv/DevDiv/_git/vs-code-coverage - 778edda8bff7e4ba06942b971c3e9618bee2a0a5 + 3d834400e3b8cfcb3e8d97a51c7fb0b7fb4c02d9 https://github.com/dotnet/diagnostics diff --git a/eng/Versions.props b/eng/Versions.props index c3ac92f441..8fc279e3f3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -80,7 +80,7 @@ JS etc.) is risky. This can break setup authoring and cause runtime failures in workloads where Rolsyn is not installed. --> 3.8.0-3.20427.2 - 17.1.0-beta.21560.1 + 17.1.0-beta.21561.1 0.2.0-preview.21508.1 5.0.0-preview.21508.1 From 72358565fecb880ed3a97cd3061ffc235c86adbf Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Fri, 12 Nov 2021 10:01:21 +0100 Subject: [PATCH 06/11] fixing tests, --- .../Processors/CollectArgumentProcessor.cs | 6 ++ .../CodeCoverageTests.cs | 66 ++++++++++++++++++- .../CodeCoverageTests.cs | 2 +- .../CollectArgumentProcessorTests.cs | 11 ++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/vstest.console/Processors/CollectArgumentProcessor.cs b/src/vstest.console/Processors/CollectArgumentProcessor.cs index d30772d246..99f6dfebce 100644 --- a/src/vstest.console/Processors/CollectArgumentProcessor.cs +++ b/src/vstest.console/Processors/CollectArgumentProcessor.cs @@ -121,6 +121,12 @@ public void Initialize(string argument) // Get collect argument list. var collectArgumentList = ArgumentProcessorUtilities.GetArgumentList(argument, ArgumentProcessorUtilities.SemiColonArgumentSeparator, exceptionMessage); + // First argument is collector name. Remaining are key value pairs for configurations. + if (collectArgumentList[0].Contains("=")) + { + throw new CommandLineException(exceptionMessage); + } + if (InferRunSettingsHelper.IsTestSettingsEnabled(this.runSettingsManager.ActiveRunSettings.SettingsXml)) { throw new SettingsException(string.Format(CommandLineResources.CollectWithTestSettingErrorMessage, argument)); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index a30b0d139e..3d75b948c6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -16,7 +16,9 @@ public enum SettingsType { None = 0, Default = 1, - Custom = 2 + Custom = 2, + XmlOutput = 3, + CoberturaOutput = 4 } public string AssemblyName { get; set; } @@ -168,6 +170,44 @@ public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo) this.CollectCodeCoverage(runnerInfo, parameters); } + [TestMethod] + [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] + public void CollectCodeCoverageSpecifyOutputFormatXml(RunnerInfo runnerInfo) + { + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x64", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.XmlOutput, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] + public void CollectCodeCoverageSpecifyOutputFormatCobertura(RunnerInfo runnerInfo) + { + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x64", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.CoberturaOutput, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); + } + private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); @@ -185,6 +225,19 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {resultsDirectory} trxfile: {trxFilePath}"); Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); + if (testParameters.RunSettingsType == TestParameters.SettingsType.XmlOutput) + { + Assert.IsTrue(actualCoverageFile.EndsWith(".xml", StringComparison.InvariantCultureIgnoreCase)); + } + else if (testParameters.RunSettingsType == TestParameters.SettingsType.CoberturaOutput) + { + Assert.IsTrue(actualCoverageFile.EndsWith(".cobertura.xml", StringComparison.InvariantCultureIgnoreCase)); + } + else + { + Assert.IsTrue(actualCoverageFile.EndsWith(".coverage", StringComparison.InvariantCultureIgnoreCase)); + } + var coverageDocument = this.GetXmlCoverage(actualCoverageFile); if (testParameters.CheckSkipped) { @@ -232,6 +285,12 @@ private string CreateArguments( case TestParameters.SettingsType.Custom: runSettings = $" /settings:{testParameters.RunSettingsPath}"; break; + case TestParameters.SettingsType.XmlOutput: + runSettings = $" /collect:\"Code Coverage;CovrerageFileFormat=Xml\""; + break; + case TestParameters.SettingsType.CoberturaOutput: + runSettings = $" /collect:\"Code Coverage;CovrerageFileFormat=Cobertura\""; + break; } arguments = string.Concat(arguments, runSettings); @@ -262,6 +321,11 @@ private void AssertSkippedMethod(XmlDocument document) private void ValidateCoverageData(XmlDocument document, string moduleName) { var module = this.GetModuleNode(document.DocumentElement, moduleName.ToLower()); + + if (module == null) + { + module = this.GetModuleNode(document.DocumentElement, moduleName); + } Assert.IsNotNull(module); this.AssertCoverage(module, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 6d4cd2d745..49899f423d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -414,7 +414,7 @@ private string GetCodeCoverageRunSettings(int cpuCount, bool useClrIeInstrumenta {useClrIeInstrumentationEngine} {useClrIeInstrumentationEngine} - {outputFormat} + {outputFormat} diff --git a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs index 0c5f3e6e52..3afcc9f8d7 100644 --- a/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs +++ b/test/vstest.console.UnitTests/Processors/CollectArgumentProcessorTests.cs @@ -585,6 +585,17 @@ public void InitializeXPlatCodeCoverageShouldAddXPlatInProcProcDataCollectoPrope ""), this.settingsProvider.ActiveRunSettings.SettingsXml); } + [TestMethod] + public void InitializeShouldThrowExceptionWhenInvalidCollectorNameProvided() + { + var runsettingsString = string.Format(DefaultRunSettings, ""); + var runsettings = new RunSettings(); + runsettings.LoadSettingsXml(runsettingsString); + this.settingsProvider.SetActiveRunSettings(runsettings); + + Assert.ThrowsException(() => this.executor.Initialize("MyDataCollector=SomeSetting")); + } + [TestMethod] public void InitializeShouldThrowExceptionWhenInvalidConfigurationsProvided() { From 4412b4007a7af02a565cfbe4afa4d518dee3ce07 Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Fri, 12 Nov 2021 10:25:45 +0100 Subject: [PATCH 07/11] fixing parameter name, --- .../CodeCoverageTests.cs | 4 ++-- .../TranslationLayerTests/CodeCoverageTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 3d75b948c6..d1cd673a45 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -286,10 +286,10 @@ private string CreateArguments( runSettings = $" /settings:{testParameters.RunSettingsPath}"; break; case TestParameters.SettingsType.XmlOutput: - runSettings = $" /collect:\"Code Coverage;CovrerageFileFormat=Xml\""; + runSettings = $" /collect:\"Code Coverage;Format=Xml\""; break; case TestParameters.SettingsType.CoberturaOutput: - runSettings = $" /collect:\"Code Coverage;CovrerageFileFormat=Cobertura\""; + runSettings = $" /collect:\"Code Coverage;Format=Cobertura\""; break; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 49899f423d..711e5a5cb0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -414,7 +414,7 @@ private string GetCodeCoverageRunSettings(int cpuCount, bool useClrIeInstrumenta {useClrIeInstrumentationEngine} {useClrIeInstrumentationEngine} - {outputFormat} + {outputFormat} From 0cee134fe33399cce7fa58cd3aa1aa4680146b0c Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Fri, 12 Nov 2021 13:55:41 +0100 Subject: [PATCH 08/11] Adding test for overriding format from settings file using collect command, --- scripts/vstest-codecoverage2.runsettings | 15 +++++------ .../CodeCoverageAcceptanceTestBase.cs | 5 ++++ .../CodeCoverageTests.cs | 25 +++++++++++++++---- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/scripts/vstest-codecoverage2.runsettings b/scripts/vstest-codecoverage2.runsettings index 8a70b1f9ed..3e54931353 100644 --- a/scripts/vstest-codecoverage2.runsettings +++ b/scripts/vstest-codecoverage2.runsettings @@ -3,8 +3,9 @@ - - + Coverage + + .*CodeCoverage.exe$ @@ -17,11 +18,11 @@ True False - - - .*TestSign.* - - + + + .*TestSign.* + + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index 681e2c268f..9dc0f7baab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -41,6 +41,11 @@ protected XmlNode GetModuleNode(XmlNode node, string name) if (moduleNode == null) { moduleNode = this.GetNode(node, "package", name); + + if (moduleNode == null) + { + moduleNode = this.GetNode(node, "package", Path.GetFileNameWithoutExtension(name)); + } } return moduleNode; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index d1cd673a45..37afffe262 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -192,13 +192,15 @@ public void CollectCodeCoverageSpecifyOutputFormatXml(RunnerInfo runnerInfo) [TestMethod] [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] - public void CollectCodeCoverageSpecifyOutputFormatCobertura(RunnerInfo runnerInfo) + public void CollectCodeCoverageSpecifyOutputFormatCoberturaOverrideRunSettingsConfiguration(RunnerInfo runnerInfo) { var parameters = new TestParameters() { AssemblyName = "SimpleTestProject.dll", TargetPlatform = "x64", - RunSettingsPath = string.Empty, + RunSettingsPath = Path.Combine( + IntegrationTestEnvironment.TestPlatformRootDirectory, + @"scripts", "vstest-codecoverage2.runsettings"), RunSettingsType = TestParameters.SettingsType.CoberturaOutput, ExpectedPassedTests = 1, ExpectedSkippedTests = 1, @@ -244,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam this.AssertSkippedMethod(coverageDocument); } - this.ValidateCoverageData(coverageDocument, testParameters.AssemblyName); + this.ValidateCoverageData(coverageDocument, testParameters.AssemblyName, testParameters.RunSettingsType != TestParameters.SettingsType.CoberturaOutput); Directory.Delete(this.resultsDirectory, true); } @@ -287,9 +289,17 @@ private string CreateArguments( break; case TestParameters.SettingsType.XmlOutput: runSettings = $" /collect:\"Code Coverage;Format=Xml\""; + if (!string.IsNullOrWhiteSpace(testParameters.RunSettingsPath)) + { + runSettings += $" /settings:{testParameters.RunSettingsPath}"; + } break; case TestParameters.SettingsType.CoberturaOutput: runSettings = $" /collect:\"Code Coverage;Format=Cobertura\""; + if (!string.IsNullOrWhiteSpace(testParameters.RunSettingsPath)) + { + runSettings += $" /settings:{testParameters.RunSettingsPath}"; + } break; } @@ -318,7 +328,7 @@ private void AssertSkippedMethod(XmlDocument document) Assert.IsNotNull(testAbsFunction); } - private void ValidateCoverageData(XmlDocument document, string moduleName) + private void ValidateCoverageData(XmlDocument document, string moduleName, bool validateSourceFileNames) { var module = this.GetModuleNode(document.DocumentElement, moduleName.ToLower()); @@ -329,7 +339,12 @@ private void ValidateCoverageData(XmlDocument document, string moduleName) Assert.IsNotNull(module); this.AssertCoverage(module, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage); - this.AssertSourceFileName(module); + + // In case of cobertura report. Cobertura report has different format. + if (validateSourceFileNames) + { + this.AssertSourceFileName(module); + } } private void AssertSourceFileName(XmlNode module) From 33f9698192cdf9286aace77cf5c96a6b854ff966 Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Tue, 16 Nov 2021 17:04:18 +0100 Subject: [PATCH 09/11] Adding testadapterpath when additional config are provided for the following command, dotnet test --collect:"Code Coverage;arg1=val1;arg2=val2" --- .../Tasks/VSTestTask.cs | 7 ++++++- .../VsTestTaskTests.cs | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs index 2d151decb3..ecf93549da 100644 --- a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs +++ b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs @@ -368,7 +368,12 @@ private List AddArgs() { foreach (var arg in this.VSTestCollect) { - if (arg.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase)) + // Argument value could be CodeCoverage;a=b;c=d. + // Split arg with ; and compare with the first token. + var tokens = arg.Split(';'); + + if (arg.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase) || + tokens[0].Equals("Code Coverage", StringComparison.OrdinalIgnoreCase)) { isCollectCodeCoverageEnabled = true; } diff --git a/test/Microsoft.TestPlatform.Build.UnitTests/VsTestTaskTests.cs b/test/Microsoft.TestPlatform.Build.UnitTests/VsTestTaskTests.cs index e3ddc98587..c61451ec9a 100644 --- a/test/Microsoft.TestPlatform.Build.UnitTests/VsTestTaskTests.cs +++ b/test/Microsoft.TestPlatform.Build.UnitTests/VsTestTaskTests.cs @@ -266,6 +266,19 @@ public void CreateArgumentShouldAddTraceCollectorDirectoryPathAsTestAdapterForCo CollectionAssert.Contains(allArguments, expectedArg, $"Expected argument: '''{expectedArg}''' not present in [{string.Join(", ", allArguments)}]"); } + [TestMethod] + public void CreateArgumentShouldAddTraceCollectorDirectoryPathAsTestAdapterForCodeCoverageCollectWithExtraConfigurations() + { + const string traceDataCollectorDirectoryPath = @"c:\path\to\tracedata collector"; + this.vsTestTask.VSTestTraceDataCollectorDirectoryPath = traceDataCollectorDirectoryPath; + this.vsTestTask.VSTestCollect = new string[] { "code coverage;someParameter=someValue" }; + + var allArguments = this.vsTestTask.CreateArgument().ToArray(); + + const string expectedArg = "--testAdapterPath:\"c:\\path\\to\\tracedata collector\""; + CollectionAssert.Contains(allArguments, expectedArg, $"Expected argument: '''{expectedArg}''' not present in [{string.Join(", ", allArguments)}]"); + } + [TestMethod] public void CreateArgumentShouldNotAddTraceCollectorDirectoryPathAsTestAdapterForNonCodeCoverageCollect() { From 34ce0cc1d809cd8e693438657ebf3e8149cff325 Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Tue, 16 Nov 2021 17:22:21 +0100 Subject: [PATCH 10/11] updating comment, --- src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs index ecf93549da..583848a0cf 100644 --- a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs +++ b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs @@ -368,8 +368,8 @@ private List AddArgs() { foreach (var arg in this.VSTestCollect) { - // Argument value could be CodeCoverage;a=b;c=d. - // Split arg with ; and compare with the first token. + // For collecting code coverage, argument value can be either "Code Coverage" or "Code Coverage;a=b;c=d". + // Split the argument with ';' and compare first token value. var tokens = arg.Split(';'); if (arg.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase) || From a6deec21e914d8371da95111d079dc2f54de684d Mon Sep 17 00:00:00 2001 From: fhnaseer Date: Tue, 16 Nov 2021 17:26:10 +0100 Subject: [PATCH 11/11] Adding string const, --- src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs index 583848a0cf..2ded9cc953 100644 --- a/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs +++ b/src/Microsoft.TestPlatform.Build/Tasks/VSTestTask.cs @@ -20,6 +20,7 @@ public class VSTestTask : Task, ICancelableTask private VSTestForwardingApp vsTestForwardingApp; private const string vsTestAppName = "vstest.console.dll"; + private const string CodeCovergaeString = "Code Coverage"; public string TestFileFullPath { @@ -372,8 +373,8 @@ private List AddArgs() // Split the argument with ';' and compare first token value. var tokens = arg.Split(';'); - if (arg.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase) || - tokens[0].Equals("Code Coverage", StringComparison.OrdinalIgnoreCase)) + if (arg.Equals(CodeCovergaeString, StringComparison.OrdinalIgnoreCase) || + tokens[0].Equals(CodeCovergaeString, StringComparison.OrdinalIgnoreCase)) { isCollectCodeCoverageEnabled = true; }