From bf69d1cf7dc3a32d4b602bfa9792e13fbed6a160 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Mon, 21 Mar 2022 22:48:53 +0100 Subject: [PATCH 01/10] feat: improve handling of frozen VsTest run --- .../MutationTest/FullRunScenario.cs | 4 +- .../TestRunners/VsTestRunnersTest.cs | 50 +++--- .../CoverageAnalysis/CoverageAnalyser.cs | 9 ++ .../MutationTest/MutationTestProcess.cs | 10 +- .../TestRunners/VsTest/RunEventHandler.cs | 46 +++--- .../VsTest/StrykerVstestHostLauncher.cs | 69 ++------ .../TestRunners/VsTest/VsTestRunner.cs | 152 ++++++++---------- .../TestRunners/VsTest/VsTestRunnerPool.cs | 24 +-- .../CoverageCollector.cs | 2 +- 9 files changed, 154 insertions(+), 212 deletions(-) diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/MutationTest/FullRunScenario.cs b/src/Stryker.Core/Stryker.Core.UnitTest/MutationTest/FullRunScenario.cs index 3580635ead..46cf4e4347 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/MutationTest/FullRunScenario.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/MutationTest/FullRunScenario.cs @@ -147,7 +147,7 @@ private TestsGuidList GetCoveringTests(int id) return TestsGuidList.NoTest(); } - private TestRunResult GetRunResult(int id) => new(TestsGuidList.EveryTest(), GetFailedTests(id), TestsGuidList.NoTest(), string.Empty, TimeSpan.Zero); + private TestRunResult GetRunResult(int id) => new(GetCoveringTests(id), GetFailedTests(id), TestsGuidList.NoTest(), string.Empty, TimeSpan.Zero); public TestRunResult GetInitialRunResult() => GetRunResult(InitialRunID); @@ -157,7 +157,7 @@ public Mock GetTestRunnerMock() var successResult = new TestRunResult(GetGuidList(), TestsGuidList.NoTest(), TestsGuidList.NoTest(), - String.Empty, + string.Empty, TimeSpan.Zero); runnerMock.Setup(x => x.DiscoverTests()).Returns(TestSet); runnerMock.Setup(x => x.InitialTest()).Returns(GetRunResult(InitialRunID)); diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs index 5b565d014b..e4d5a375f8 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs @@ -165,10 +165,7 @@ private void MoqTestRun(ITestRunEventsHandler testRunEvents, IReadOnlyList mockVsTest, bool testResult, EventWaitHandle synchroObject) { var results = new List<(string, bool)>(_testCases.Count); - for (var i = 0; i < _testCases.Count; i++) - { - results.Add((_testCases[i].FullyQualifiedName, testResult)); - } + results.AddRange(_testCases.Select(t => (t.FullyQualifiedName, testResult))); SetupMockTestRun(mockVsTest, results, synchroObject); } @@ -176,18 +173,17 @@ private void SetupMockTestRun(Mock mockVsTest, IEnumerabl EventWaitHandle synchroObject) { var results = new List(); - foreach (var testResult in testResults) + foreach (var (testResultId, success) in testResults) { - var testResultId = testResult.id; var testCase = FindOrBuildCase(testResultId); results.Add(new TestResult(testCase) { - Outcome = testResult.success ? TestOutcome.Passed : TestOutcome.Failed, + Outcome = success ? TestOutcome.Passed : TestOutcome.Failed, ComputerName = "." }); } mockVsTest.Setup(x => - x.RunTestsWithCustomTestHost( + x.RunTestsWithCustomTestHostAsync( It.Is>(t => t.Any(source => source == _testAssemblyPath)), It.Is(settings => !settings.Contains("(o => o != null && o.TestCaseFilter == null), @@ -199,14 +195,13 @@ private void SetupMockTestRun(Mock mockVsTest, IEnumerabl // generate test results MoqTestRun(testRunEvents, results); synchroObject.Set(); - }); + }).Returns(Task.CompletedTask); } // setup a customized coverage capture run, using provided coverage results - private void SetupMockCoverageRun(Mock mockVsTest, IReadOnlyDictionary coverageResults, EventWaitHandle endProcess) - { + private void SetupMockCoverageRun(Mock mockVsTest, IReadOnlyDictionary coverageResults, EventWaitHandle endProcess) => mockVsTest.Setup(x => - x.RunTestsWithCustomTestHost( + x.RunTestsWithCustomTestHostAsync( It.Is>(t => t.Any(source => source == _testAssemblyPath)), It.Is(settings => settings.Contains("(o => o != null && o.TestCaseFilter == null), @@ -235,14 +230,12 @@ private void SetupMockCoverageRun(Mock mockVsTest, IReadO } MoqTestRun(testRunEvents, results); endProcess.Set(); - }); - } + }).Returns(Task.CompletedTask); // setup a customized partial test runs, using provided test results - private void SetupMockPartialTestRun(Mock mockVsTest, IReadOnlyDictionary results, EventWaitHandle endProcess) - { + private void SetupMockPartialTestRun(Mock mockVsTest, IReadOnlyDictionary results, EventWaitHandle endProcess) => mockVsTest.Setup(x => - x.RunTestsWithCustomTestHost( + x.RunTestsWithCustomTestHostAsync( It.IsAny>(), It.Is(s => !s.Contains("(o => o != null && o.TestCaseFilter == null), @@ -283,7 +276,7 @@ private void SetupMockPartialTestRun(Mock mockVsTest, IRe } var result = new TestResult(matchingTest) - { Outcome = strings[1] == "F" ? TestOutcome.Failed : TestOutcome.Passed, ComputerName = "." }; + { Outcome = strings[1] == "F" ? TestOutcome.Failed : TestOutcome.Passed, ComputerName = "." }; runResults.Add(result); } // setup a normal test run @@ -291,13 +284,11 @@ private void SetupMockPartialTestRun(Mock mockVsTest, IRe collector.TestSessionEnd(new TestSessionEndArgs()); endProcess.Set(); - }); - } + }).Returns(Task.CompletedTask); - private void SetupMockTimeOutTestRun(Mock mockVsTest, IReadOnlyDictionary results, string timeoutTest, EventWaitHandle endProcess) - { + private void SetupMockTimeOutTestRun(Mock mockVsTest, IReadOnlyDictionary results, string timeoutTest, EventWaitHandle endProcess) => mockVsTest.Setup(x => - x.RunTestsWithCustomTestHost( + x.RunTestsWithCustomTestHostAsync( It.IsAny>(), It.IsAny(), It.Is(o => o != null && o.TestCaseFilter == null), @@ -342,7 +333,7 @@ private void SetupMockTimeOutTestRun(Mock mockVsTest, IRe timeOutTestCase = matchingTest; } var result = new TestResult(matchingTest) - { Outcome = strings[1] == "F" ? TestOutcome.Failed : TestOutcome.Passed, ComputerName = "." }; + { Outcome = strings[1] == "F" ? TestOutcome.Failed : TestOutcome.Passed, ComputerName = "." }; runResults.Add(result); } // setup a normal test run @@ -350,8 +341,7 @@ private void SetupMockTimeOutTestRun(Mock mockVsTest, IRe collector.TestSessionEnd(new TestSessionEndArgs()); endProcess.Set(); - }); - } + }).Returns(Task.CompletedTask); private Mock BuildVsTestRunner(StrykerOptions options, WaitHandle endProcess, out VsTestRunner runner) { @@ -371,6 +361,7 @@ private Mock BuildVsTestRunner(StrykerOptions options, Wa _targetProject, null, null, + 0, _fileSystem, new Mock().Object, wrapper: mockedVsTestConsole.Object, @@ -612,11 +603,14 @@ public void RunTestsSimultaneouslyWhenPossible() _testCases.Add(new TestCase("T2", _executorUri, _testAssemblyPath)); _testCases.Add(new TestCase("T3", _executorUri, _testAssemblyPath)); + var ranTests = new TestsGuidList(_testCases.Select(t => t.Id)); + var testRunResult = new TestRunResult(ranTests, TestsGuidList.NoTest(), TestsGuidList.NoTest(), + string.Empty, TimeSpan.Zero); var input = new MutationTestInput { ProjectInfo = _targetProject, TestRunner = runner, - InitialTestRun = new InitialTestRun(new TestRunResult(true), new TimeoutValueCalculator(500)) + InitialTestRun = new InitialTestRun(testRunResult, new TimeoutValueCalculator(500)) }; foreach (var mutant in _targetProject.ProjectContents.Mutants) { @@ -779,7 +773,7 @@ public void MarkSuspiciousCoverage() } // class mocking the VsTest Host Launcher - private class MoqHost : IStrykerTestHostLauncher + private class MoqHost : ITestHostLauncher { private readonly WaitHandle _handle; diff --git a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs index c3056d676a..ffa539b765 100644 --- a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs +++ b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs @@ -72,6 +72,15 @@ private void SetCoveringTests(IReadOnlyCollection mutantsToScan) { mutant.CoveringTests = TestsGuidList.EveryTest(); } + if (mutant.CoveringTests.IsEveryTest) + { + _logger.LogDebug($"Mutant {mutant.Id} is covered by all tests."); + } + else + { + _logger.LogDebug($"Mutant {mutant.Id} is covered by {mutant.CoveringTests.Count} tests."); + _logger.LogTrace($"Tests are : {string.Join(',', mutant.CoveringTests.GetGuids())}."); + } } } } diff --git a/src/Stryker.Core/Stryker.Core/MutationTest/MutationTestProcess.cs b/src/Stryker.Core/Stryker.Core/MutationTest/MutationTestProcess.cs index 24f5c057f4..0ddad90265 100644 --- a/src/Stryker.Core/Stryker.Core/MutationTest/MutationTestProcess.cs +++ b/src/Stryker.Core/Stryker.Core/MutationTest/MutationTestProcess.cs @@ -228,7 +228,7 @@ private IEnumerable> BuildMutantGroupsForTest(IReadOnlyCollection m.MustRunAgainstAllTests).Select(m => new List { m })); mutantsToGroup.RemoveAll(m => m.MustRunAgainstAllTests); - var testsCount = mutantsToGroup.Sum(m => m.CoveringTests.Count); + var testsCount = Input.InitialTestRun.Result.RanTests.Count; mutantsToGroup = mutantsToGroup.OrderBy(m => m.CoveringTests.Count).ToList(); for (var i = 0; i < mutantsToGroup.Count; i++) { @@ -238,11 +238,15 @@ private IEnumerable> BuildMutantGroupsForTest(IReadOnlyCollection testsCount || - nextSet.ContainsAny(usedTests)) + if (nextSet.Count + usedTests.Count > testsCount) + { + break; + } + if (nextSet.ContainsAny(usedTests)) { continue; } + // add this mutant to the block nextBlock.Add(currentMutant); // remove the mutant from the list of mutants to group diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/RunEventHandler.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/RunEventHandler.cs index 9cbfba3ba7..626f2ab60e 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/RunEventHandler.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/RunEventHandler.cs @@ -33,10 +33,7 @@ public bool AddResult(TestResult result) return _results.Count >= _testDescription.NbSubCases; } - public bool IsComplete() - { - return _results.Count >= _testDescription.NbSubCases; - } + public bool IsComplete() => _results.Count >= _testDescription.NbSubCases; public TestResult Result() { @@ -78,7 +75,7 @@ public sealed class RunEventHandler : ITestRunEventsHandler, IDisposable, IRunRe private readonly string _runnerId; private readonly IDictionary _vsTests; private readonly IDictionary _runs = new Dictionary(); - private readonly Dictionary _inProgress = new Dictionary(); + private readonly Dictionary _inProgress = new(); public event EventHandler VsTestFailed; public event EventHandler ResultsUpdated; @@ -193,24 +190,36 @@ public void HandleTestRunComplete( _waitHandle.Set(); } - public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) - { - throw new NotSupportedException(); - } + public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) => throw new NotSupportedException(); - public void HandleRawMessage(string rawMessage) - { - _logger.LogTrace($"{_runnerId}: {rawMessage} [RAW]"); - } + public void HandleRawMessage(string rawMessage) => _logger.LogTrace($"{_runnerId}: {rawMessage} [RAW]"); - public void WaitEnd() + public bool WaitEnd(int? timeOut) { - _waitHandle.WaitOne(); + if (timeOut == null) + { + return _waitHandle.WaitOne(); + } + else + { + var delay = 0; + const int Unit = 500; + while (delay LogLevel.Debug, TestMessageLevel.Warning => LogLevel.Warning, @@ -220,9 +229,6 @@ public void HandleLogMessage(TestMessageLevel level, string message) _logger.LogTrace($"{_runnerId}: [{levelFinal}] {message}"); } - public void Dispose() - { - _waitHandle.Dispose(); - } + public void Dispose() => _waitHandle.Dispose(); } } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs index 4cd67fd442..c7d52a397a 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs @@ -2,34 +2,20 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Stryker.Core.Logging; -using System; using System.Diagnostics; using System.Threading; namespace Stryker.Core.TestRunners.VsTest { - public interface IStrykerTestHostLauncher : ITestHostLauncher + public class StrykerVsTestHostLauncher : ITestHostLauncher { - bool WaitProcessExit(); - } - - public class StrykerVsTestHostLauncher : IStrykerTestHostLauncher - { - private Process _currentProcess; - private readonly object _lck = new object(); private readonly int _id; private static ILogger Logger { get; } - static StrykerVsTestHostLauncher() - { - Logger = ApplicationLogging.LoggerFactory.CreateLogger(); - } + static StrykerVsTestHostLauncher() => Logger = ApplicationLogging.LoggerFactory.CreateLogger(); - public StrykerVsTestHostLauncher(int id) - { - _id = id; - } + public StrykerVsTestHostLauncher(int id) => _id = id; public bool IsDebug => false; @@ -53,49 +39,20 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell ["Verify_DisableClipboard"] = "true", }, }; - _currentProcess = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; + var currentProcess = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; - _currentProcess.Exited += CurrentProcess_Exited; + currentProcess.OutputDataReceived += Process_OutputDataReceived; + currentProcess.ErrorDataReceived += Process_ErrorDataReceived; - _currentProcess.OutputDataReceived += Process_OutputDataReceived; - _currentProcess.ErrorDataReceived += Process_ErrorDataReceived; - - if (!_currentProcess.Start()) + if (!currentProcess.Start()) { Logger.LogError($"Runner {_id}: Failed to start process {processInfo.Arguments}."); } - _currentProcess.BeginOutputReadLine(); - _currentProcess.BeginErrorReadLine(); - - return _currentProcess.Id; - } + currentProcess.BeginOutputReadLine(); + currentProcess.BeginErrorReadLine(); - private void CurrentProcess_Exited(object sender, EventArgs e) - { - lock (_lck) - { - Monitor.Pulse(_lck); - } - } - - public bool WaitProcessExit() - { - if (_currentProcess == null) - { - return false; - } - - while (!_currentProcess.HasExited) - { - lock (_lck) - { - Monitor.Wait(_lck, 500); - } - } - - _currentProcess.Dispose(); - return true; + return currentProcess.Id; } private void Process_ErrorDataReceived(object sender, DataReceivedEventArgs e) @@ -114,9 +71,7 @@ private void Process_OutputDataReceived(object sender, DataReceivedEventArgs e) } } - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) - { - return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); - } + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) => + LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); } } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index 296f269507..31859f15d2 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.IO.Abstractions; using System.Linq; using System.Threading; @@ -10,6 +9,7 @@ using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using NuGet.Frameworks; using Serilog.Events; using Stryker.Core.Exceptions; @@ -26,16 +26,14 @@ namespace Stryker.Core.TestRunners.VsTest { public sealed class VsTestRunner : ITestRunner { - private static int count; - private IVsTestConsoleWrapper _vsTestConsole; private readonly IFileSystem _fileSystem; private readonly StrykerOptions _options; private readonly ProjectInfo _projectInfo; - private readonly Func _hostBuilder; + private readonly Func _hostBuilder; private readonly IVsTestHelper _vsTestHelper; private readonly bool _ownHelper; - private readonly List _messages = new List(); + private readonly List _messages = new(); private IDictionary _vsTests; private ICollection _sources; private bool _disposedValue; // To detect redundant calls @@ -52,31 +50,35 @@ public VsTestRunner(StrykerOptions options, ProjectInfo projectInfo, IDictionary tests, TestSet testSet, + int id, IFileSystem fileSystem = null, IVsTestHelper helper = null, ILogger logger = null, IVsTestConsoleWrapper wrapper = null, - Func hostBuilder = null) + Func hostBuilder = null) { _logger = logger ?? ApplicationLogging.LoggerFactory.CreateLogger(); _fileSystem = fileSystem ?? new FileSystem(); _options = options; _projectInfo = projectInfo; - _hostBuilder = hostBuilder ?? (id => new StrykerVsTestHostLauncher(id)); + _hostBuilder = hostBuilder ?? (xId => new StrykerVsTestHostLauncher(xId)); SetListOfTests(tests); _tests = testSet ?? new TestSet(); _ownHelper = helper == null; _vsTestHelper = helper ?? new VsTestHelper(); - _vsTestConsole = wrapper ?? PrepareVsTestConsole(); - _id = count++; - InitializeVsTestConsole(); + _id = id; + PrepareVsTestEnvironment(); + _vsTestConsole = wrapper; + if (_vsTestConsole == null) + { + PrepareVsTestConsole(); + } + _vsTests = DiscoverTests(null).Item1; } - private bool CantUseStrykerDataCollector() - { - return _projectInfo.TestProjectAnalyzerResults.Select(x => x.GetNuGetFramework()).Any(t => + private bool CantUseStrykerDataCollector() => + _projectInfo.TestProjectAnalyzerResults.Select(x => x.GetNuGetFramework()).Any(t => t.Framework == FrameworkConstants.FrameworkIdentifiers.NetCoreApp && t.Version.Major < 2); - } public TestRunResult InitialTest() { @@ -95,17 +97,14 @@ public TestRunResult InitialTest() return BuildTestRunResult(testResults, int.MaxValue, false); } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) - { - return TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - } + public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList mutants, TestUpdateHandler update) { var mutantTestsMap = new Dictionary(); var needAll = true; ICollection testCases; - int? timeOutMs = timeoutCalc?.DefaultTimeout; + var timeOutMs = timeoutCalc?.DefaultTimeout; if (mutants != null) { @@ -159,7 +158,7 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR testCases = null; } - var numberTestCases = testCases == null ? 0 : testCases.Count; + var numberTestCases = testCases?.Count ?? 0; var expectedTests = needAll ? DiscoverTests().Count : numberTestCases; void HandleUpdate(IRunResults handler) @@ -186,9 +185,11 @@ void HandleUpdate(IRunResults handler) _aborted = true; } - _logger.LogDebug("Using {0} ms as testrun timeout", timeOutMs.ToString() ?? "no"); + if (timeOutMs.HasValue) + _logger.LogDebug("Using {0} ms as test run timeout", timeOutMs); - var testResults = RunTestSession(mutantTestsMap, needAll, GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), HandleUpdate); + var testResults = RunTestSession(mutantTestsMap, needAll, + GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), timeOutMs, HandleUpdate); return BuildTestRunResult(testResults, expectedTests); } @@ -225,10 +226,7 @@ private void SetListOfTests(IDictionary tests) DetectTestFramework(_vsTests?.Values); } - public TestSet DiscoverTests() - { - return DiscoverTests(null).Item2; - } + public TestSet DiscoverTests() => DiscoverTests(null).Item2; public (IDictionary, TestSet) DiscoverTests(string runSettings) { @@ -287,8 +285,7 @@ public TestRunResult CaptureCoverage(IEnumerable mutants) _logger.LogDebug($"{RunnerId}: Capturing coverage."); if (CantUseStrykerDataCollector()) { - _logger.LogDebug($"{RunnerId}: project does not support StrykerDataCollector. Coverage data is simulated. Upgrade test proj" + - $" to@ NetCore 2.0+"); + _logger.LogDebug($"{RunnerId}: project does not support StrykerDataCollector. Coverage data is simulated. Upgrade test proj to NetCore 2.0+"); // can't capture coverage info foreach (var mutant in mutants) { @@ -348,7 +345,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume // we have coverage data seenTestCases.Add(testDescription.Id); - var propertyPairValue = (value as string); + var propertyPairValue = value as string; if (string.IsNullOrWhiteSpace(propertyPairValue)) { _logger.LogDebug($"{RunnerId}: Test {testResult.TestCase.DisplayName} does not cover any mutation."); @@ -369,10 +366,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume map[id].Add(testDescription.Description); } - foreach (var id in staticMutants) - { - staticMutantLists.Add(id); - } + staticMutantLists.UnionWith(staticMutants); } var (testProperty, mutantOutsideTests) = testResult.GetProperties() .FirstOrDefault(x => x.Key.Id == CoverageCollector.OutOfTestsPropertyName); @@ -384,10 +378,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume ? Enumerable.Empty() : propertyPairValue.Split(',').Select(int.Parse); _logger.LogWarning("Some mutations were executed outside any test (mutation ids: {0}).", propertyPairValue); - foreach (var id in coveredMutants) - { - staticMutantLists.Add(id); - } + staticMutantLists.UnionWith(coveredMutants); } } } @@ -414,6 +405,7 @@ public void CoverageForOneTest(Guid test, IEnumerable mutants) private IRunResults RunTestSession(Dictionary mutantTestsMap, bool runAllTests, string runSettings, + int? timeOut = null, Action updateHandler = null, int retries = 0) { @@ -429,50 +421,48 @@ private IRunResults RunTestSession(Dictionary mutantTestsMap, var options = new TestPlatformOptions { TestCaseFilter = _options.TestCaseFilter }; if (runAllTests) { - _vsTestConsole.RunTestsWithCustomTestHost(_sources, runSettings, options, eventHandler, + _vsTestConsole.RunTestsWithCustomTestHostAsync(_sources, runSettings, options, eventHandler, strykerVsTestHostLauncher); } else { - _vsTestConsole.RunTestsWithCustomTestHost(mutantTestsMap.SelectMany(m => m.Value.GetGuids()).Select(t => _vsTests[t].Case), runSettings, + _vsTestConsole.RunTestsWithCustomTestHostAsync(mutantTestsMap.SelectMany(m => m.Value.GetGuids()).Select(t => _vsTests[t].Case), runSettings, options, eventHandler, strykerVsTestHostLauncher); } - // Test host exited signal comes after the run completed - strykerVsTestHostLauncher.WaitProcessExit(); - - // At this point, run must have complete. Check signal for true - eventHandler.WaitEnd(); + // Wait for test completed report + if (!eventHandler.WaitEnd(timeOut)) + { + _logger.LogWarning($"{RunnerId}: VsTest did not report the end of test session in due time, it may have hang. Retrying"); + _vsTestConsole.AbortTestRun(); + _vsTestFailed = true; + } eventHandler.ResultsUpdated -= HandlerUpdate; eventHandler.VsTestFailed -= HandlerVsTestFailed; - if (!_vsTestFailed || retries > 10) + if (!_vsTestFailed || retries > 5) { return eventHandler; } - _vsTestConsole = PrepareVsTestConsole(); + PrepareVsTestConsole(); _vsTestFailed = false; - return RunTestSession(mutantTestsMap, true, runSettings, updateHandler, ++retries); + return RunTestSession(mutantTestsMap, runAllTests, runSettings, timeOut, updateHandler, retries+1); } private ConsoleParameters DetermineConsoleParameters() { - if (_options.LogOptions.LogToFile) - { - var vsTestLogPath = Path.Combine(_options.OutputPath, "logs", "VsTest-log.txt"); - _fileSystem.Directory.CreateDirectory(Path.GetDirectoryName(vsTestLogPath)); - - _logger.LogTrace("{1}: Logging VsTest output to: {0}", vsTestLogPath, RunnerId); - return new ConsoleParameters - { - TraceLevel = DetermineTraceLevel(), - LogFilePath = vsTestLogPath - }; - } + if (!_options.LogOptions.LogToFile) return new ConsoleParameters(); + var vsTestLogPath = _fileSystem.Path.Combine(_options.OutputPath, "logs", $"{RunnerId}_VsTest-log.txt"); + _fileSystem.Directory.CreateDirectory(_fileSystem.Path.GetDirectoryName(vsTestLogPath)); - return new ConsoleParameters(); + _logger.LogTrace("{1}: Logging VsTest output to: {0}", vsTestLogPath, RunnerId); + return new ConsoleParameters + { + TraceLevel = DetermineTraceLevel(), + LogFilePath = vsTestLogPath + }; } private TraceLevel DetermineTraceLevel() @@ -508,7 +498,7 @@ private string GenerateRunSettings(int? timeout, bool forMutantTesting, bool for if (_testFramework.HasFlag(TestFramework.xUnit)) { - settingsForCoverage += "true"; + settingsForCoverage += "true"; } var timeoutSettings = timeout != null ? $"{timeout}" + Environment.NewLine : string.Empty; @@ -536,12 +526,10 @@ private string GenerateRunSettings(int? timeout, bool forMutantTesting, bool for return runSettings; } - private bool NeedCoverage() - { - return _options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest) || _options.OptimizationMode.HasFlag(OptimizationModes.SkipUncoveredMutants); - } + private bool NeedCoverage() => _options.OptimizationMode.HasFlag(OptimizationModes.CoverageBasedTest) + || _options.OptimizationMode.HasFlag(OptimizationModes.SkipUncoveredMutants); - private IVsTestConsoleWrapper PrepareVsTestConsole() + private void PrepareVsTestConsole() { if (_vsTestConsole != null) { @@ -550,16 +538,25 @@ private IVsTestConsoleWrapper PrepareVsTestConsole() _vsTestConsole.EndSession(); } catch { /*Ignore exception. vsTestConsole has been disposed outside of our control*/ } - _vsTestConsole = null; } - return new VsTestConsoleWrapper(_vsTestHelper.GetCurrentPlatformVsTestToolPath(), DetermineConsoleParameters()); + _vsTestConsole = new VsTestConsoleWrapper(_vsTestHelper.GetCurrentPlatformVsTestToolPath(), DetermineConsoleParameters()); + try + { + // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes + Environment.SetEnvironmentVariable("DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX", "2"); + _vsTestConsole.StartSession(); + _vsTestConsole.InitializeExtensions(_sources.Select( _fileSystem.Path.GetDirectoryName)); + } + catch (Exception e) + { + throw new GeneralStrykerException("Stryker failed to connect to vstest.console", e); + } } - private void InitializeVsTestConsole() + private void PrepareVsTestEnvironment() { var testBinariesPaths = _projectInfo.TestProjectAnalyzerResults.Select(testProject => testProject.GetAssemblyPath()).ToList(); - var testBinariesLocations = new List(); _sources = new List(); foreach (var path in testBinariesPaths) @@ -569,23 +566,8 @@ private void InitializeVsTestConsole() throw new GeneralStrykerException($"The test project binaries could not be found at {path}, exiting..."); } - testBinariesLocations.Add(Path.GetDirectoryName(path)); _sources.Add(path); } - - try - { - // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes - Environment.SetEnvironmentVariable("DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX", "2"); - _vsTestConsole.StartSession(); - _vsTestConsole.InitializeExtensions(testBinariesLocations); - } - catch (Exception e) - { - throw new GeneralStrykerException("Stryker failed to connect to vstest.console", e); - } - - _vsTests = DiscoverTests(null).Item1; } #region IDisposable Support diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs index 168448b703..c4bbb1893e 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs @@ -15,8 +15,8 @@ namespace Stryker.Core.TestRunners.VsTest public sealed class VsTestRunnerPool : ITestRunner { private readonly StrykerOptions _options; - private readonly AutoResetEvent _runnerAvailableHandler = new AutoResetEvent(false); - private readonly ConcurrentBag _availableRunners = new ConcurrentBag(); + private readonly AutoResetEvent _runnerAvailableHandler = new(false); + private readonly ConcurrentBag _availableRunners = new(); private readonly IDictionary _vsTests; private readonly TestSet _tests; private readonly VsTestHelper _helper = new(); @@ -26,14 +26,12 @@ public VsTestRunnerPool(StrykerOptions options, ProjectInfo projectInfo) { _options = options; _logger = ApplicationLogging.LoggerFactory.CreateLogger(); - using var runner = new VsTestRunner(options, projectInfo, null, new TestSet(), helper: _helper); + var runner = new VsTestRunner(options, projectInfo, null, new TestSet(), 0, helper: _helper); (_vsTests, _tests) = runner.DiscoverTests(null); _availableRunners.Add(runner); - Parallel.For(1, options.Concurrency, (i, loopState) => - { - _availableRunners.Add(new VsTestRunner(options, projectInfo, _vsTests, _tests, helper: _helper)); - }); + Parallel.For(1, options.Concurrency, + (i, _) => _availableRunners.Add(new VsTestRunner(options, projectInfo, _vsTests, _tests, i, helper: _helper))); } public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList mutants, TestUpdateHandler update) @@ -50,10 +48,7 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR } } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) - { - return TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - } + public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); public TestRunResult InitialTest() { @@ -104,7 +99,7 @@ private TestRunResult CaptureCoveragePerIsolatedTests(IEnumerable mutant } catch (Exception e) { - _logger.LogError("Something went wrong while capturing coverage", e); + _logger.LogError("Something went wrong while capturing coverage: {0}", e); } finally { @@ -141,9 +136,6 @@ public void Dispose() _runnerAvailableHandler.Dispose(); } - public TestSet DiscoverTests() - { - return _tests; - } + public TestSet DiscoverTests() => _tests; } } diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs index 2c8a53df31..f4f22ad3ba 100644 --- a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs +++ b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs @@ -173,7 +173,7 @@ private void ReadConfiguration(string configuration) } var nameSpaceNode = node.SelectSingleNode("//Parameters/MutantControl"); - if (nameSpaceNode != null && nameSpaceNode.Attributes != null) + if (nameSpaceNode?.Attributes != null) { _controlClassName = nameSpaceNode.Attributes["name"].Value; } From 0f21634b9398b5884acec01987bf59c0de835fa6 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Mon, 28 Mar 2022 23:53:07 +0200 Subject: [PATCH 02/10] fix: restore support for xUnit + add better error detection when VsTest fails on start --- .../TestRunners/VsTestRunnersTest.cs | 26 ++++++++++++------ .../CoverageAnalysis/CoverageAnalyser.cs | 9 ++++--- .../VsTest/StrykerVstestHostLauncher.cs | 27 ++++++++++++------- .../TestRunners/VsTest/VsTestRunner.cs | 26 ++++++++++-------- 4 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs index e4d5a375f8..732d611e52 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs @@ -16,6 +16,7 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollector.InProcDataCollector; using Moq; using Shouldly; +using Stryker.Core.Exceptions; using Stryker.Core.Initialisation; using Stryker.Core.MutantFilters; using Stryker.Core.Mutants; @@ -343,7 +344,7 @@ private void SetupMockTimeOutTestRun(Mock mockVsTest, IRe endProcess.Set(); }).Returns(Task.CompletedTask); - private Mock BuildVsTestRunner(StrykerOptions options, WaitHandle endProcess, out VsTestRunner runner) + private Mock BuildVsTestRunner(StrykerOptions options, WaitHandle endProcess, out VsTestRunner runner, bool suceed = true) { var mockedVsTestConsole = new Mock(MockBehavior.Strict); mockedVsTestConsole.Setup(x => x.StartSession()); @@ -365,7 +366,7 @@ private Mock BuildVsTestRunner(StrykerOptions options, Wa _fileSystem, new Mock().Object, wrapper: mockedVsTestConsole.Object, - hostBuilder: _ => new MoqHost(endProcess, false)); + hostBuilder: _ => new MoqHost(endProcess, suceed, false)); return mockedVsTestConsole; } @@ -390,6 +391,17 @@ public void RunInitialTests() } + [Fact] + public void HandleVsTestCreationFailure() + { + using var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset); + var mockVsTest = BuildVsTestRunner(new StrykerOptions(), endProcess, out var runner, false); + SetupMockTestRun(mockVsTest, new[] { ("T0", false), ("T1", true) }, endProcess); + + Action action = () => runner.InitialTest(); + action.ShouldThrow(); + } + [Fact] public void RunTests() { @@ -773,13 +785,14 @@ public void MarkSuspiciousCoverage() } // class mocking the VsTest Host Launcher - private class MoqHost : ITestHostLauncher + private class MoqHost : IStrykerTestHostLauncher { private readonly WaitHandle _handle; - public MoqHost(WaitHandle handle, bool isDebug) + public MoqHost(WaitHandle handle, bool succeed, bool isDebug) { _handle = handle; + IsProcessCreated = succeed; IsDebug = isDebug; } @@ -795,10 +808,7 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell public bool IsDebug { get; } - public bool WaitProcessExit() - { - return _handle == null || _handle.WaitOne(); - } + public bool IsProcessCreated { get; } } } } diff --git a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs index ffa539b765..92cf230146 100644 --- a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs +++ b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs @@ -2,7 +2,6 @@ using System.Linq; using Microsoft.Extensions.Logging; using Stryker.Core.Exceptions; -using Stryker.Core.Initialisation.Buildalyzer; using Stryker.Core.Logging; using Stryker.Core.Mutants; using Stryker.Core.MutationTest; @@ -74,13 +73,17 @@ private void SetCoveringTests(IReadOnlyCollection mutantsToScan) } if (mutant.CoveringTests.IsEveryTest) { - _logger.LogDebug($"Mutant {mutant.Id} is covered by all tests."); + _logger.LogDebug($"Mutant {mutant.Id} will be tested agaisnt all tests."); } - else + else if (mutant.CoveringTests.Count>0) { _logger.LogDebug($"Mutant {mutant.Id} is covered by {mutant.CoveringTests.Count} tests."); _logger.LogTrace($"Tests are : {string.Join(',', mutant.CoveringTests.GetGuids())}."); } + else + { + _logger.LogDebug($"Mutant {mutant.Id} is not covered by any test."); + } } } } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs index c7d52a397a..87a34b74f6 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/StrykerVstestHostLauncher.cs @@ -7,9 +7,16 @@ namespace Stryker.Core.TestRunners.VsTest { - public class StrykerVsTestHostLauncher : ITestHostLauncher + + public interface IStrykerTestHostLauncher : ITestHostLauncher + { + bool IsProcessCreated { get; } + } + + public class StrykerVsTestHostLauncher : IStrykerTestHostLauncher { private readonly int _id; + private Process _currentProcess; private static ILogger Logger { get; } @@ -19,6 +26,8 @@ public class StrykerVsTestHostLauncher : ITestHostLauncher public bool IsDebug => false; + public bool IsProcessCreated => _currentProcess != null; + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) { var processInfo = new ProcessStartInfo(defaultTestHostStartInfo.FileName, defaultTestHostStartInfo.Arguments) @@ -39,27 +48,27 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell ["Verify_DisableClipboard"] = "true", }, }; - var currentProcess = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; + _currentProcess = new Process { StartInfo = processInfo, EnableRaisingEvents = true }; - currentProcess.OutputDataReceived += Process_OutputDataReceived; - currentProcess.ErrorDataReceived += Process_ErrorDataReceived; + _currentProcess.OutputDataReceived += Process_OutputDataReceived; + _currentProcess.ErrorDataReceived += Process_ErrorDataReceived; - if (!currentProcess.Start()) + if (!_currentProcess.Start()) { Logger.LogError($"Runner {_id}: Failed to start process {processInfo.Arguments}."); } - currentProcess.BeginOutputReadLine(); - currentProcess.BeginErrorReadLine(); + _currentProcess.BeginOutputReadLine(); + _currentProcess.BeginErrorReadLine(); - return currentProcess.Id; + return _currentProcess.Id; } private void Process_ErrorDataReceived(object sender, DataReceivedEventArgs e) { if (e.Data != null) { - Logger.LogTrace($"Runner {_id}: {e.Data} (VsTest error)"); + Logger.LogDebug($"Runner {_id}: {e.Data} (VsTest error)"); } } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index 31859f15d2..502b45e7cd 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -9,7 +9,6 @@ using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using NuGet.Frameworks; using Serilog.Events; using Stryker.Core.Exceptions; @@ -30,7 +29,7 @@ public sealed class VsTestRunner : ITestRunner private readonly IFileSystem _fileSystem; private readonly StrykerOptions _options; private readonly ProjectInfo _projectInfo; - private readonly Func _hostBuilder; + private readonly Func _hostBuilder; private readonly IVsTestHelper _vsTestHelper; private readonly bool _ownHelper; private readonly List _messages = new(); @@ -55,7 +54,7 @@ public VsTestRunner(StrykerOptions options, IVsTestHelper helper = null, ILogger logger = null, IVsTestConsoleWrapper wrapper = null, - Func hostBuilder = null) + Func hostBuilder = null) { _logger = logger ?? ApplicationLogging.LoggerFactory.CreateLogger(); _fileSystem = fileSystem ?? new FileSystem(); @@ -89,7 +88,7 @@ public TestRunResult InitialTest() if (!_vsTests.ContainsKey(result.TestCase.Id)) { _vsTests[result.TestCase.Id] = new VsTestDescription(result.TestCase); - _logger.LogWarning($"Initial test run encounter a unexpected test case ({result.TestCase.DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts."); + _logger.LogWarning($"{RunnerId}: Initial test run encounter a unexpected test case ({result.TestCase.DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts."); } _vsTests[result.TestCase.Id].RegisterInitialTestResult(result); } @@ -186,7 +185,7 @@ void HandleUpdate(IRunResults handler) } if (timeOutMs.HasValue) - _logger.LogDebug("Using {0} ms as test run timeout", timeOutMs); + _logger.LogDebug($"{RunnerId}: Using {timeOutMs} ms as test run timeout"); var testResults = RunTestSession(mutantTestsMap, needAll, GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), timeOutMs, HandleUpdate); @@ -318,7 +317,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume var (key, value) = testResult.GetProperties().FirstOrDefault(x => x.Key.Id == CoverageCollector.PropertyName); if (!_vsTests.ContainsKey(testResult.TestCase.Id)) { - _logger.LogWarning($"Coverage analysis run encountered a unexpected test case ({testResult.TestCase.DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts."); + _logger.LogWarning($"{RunnerId}: Coverage analysis run encountered a unexpected test case ({testResult.TestCase.DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts."); _vsTests.Add(testResult.TestCase.Id, new VsTestDescription(testResult.TestCase)); } var testDescription = _vsTests[testResult.TestCase.Id]; @@ -377,7 +376,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume var coveredMutants = string.IsNullOrEmpty(propertyPairValue) ? Enumerable.Empty() : propertyPairValue.Split(',').Select(int.Parse); - _logger.LogWarning("Some mutations were executed outside any test (mutation ids: {0}).", propertyPairValue); + _logger.LogWarning($"{RunnerId}: Some mutations were executed outside any test (mutation ids: {propertyPairValue})."); staticMutantLists.UnionWith(coveredMutants); } } @@ -438,6 +437,11 @@ private IRunResults RunTestSession(Dictionary mutantTestsMap, _vsTestFailed = true; } + if (!strykerVsTestHostLauncher.IsProcessCreated) + { + throw new GeneralStrykerException("*** Failed to create a TestRunner, Stryker cannot recover from this!***"); + } + eventHandler.ResultsUpdated -= HandlerUpdate; eventHandler.VsTestFailed -= HandlerVsTestFailed; @@ -457,7 +461,7 @@ private ConsoleParameters DetermineConsoleParameters() var vsTestLogPath = _fileSystem.Path.Combine(_options.OutputPath, "logs", $"{RunnerId}_VsTest-log.txt"); _fileSystem.Directory.CreateDirectory(_fileSystem.Path.GetDirectoryName(vsTestLogPath)); - _logger.LogTrace("{1}: Logging VsTest output to: {0}", vsTestLogPath, RunnerId); + _logger.LogTrace($"{RunnerId}: Logging VsTest output to: {vsTestLogPath}"); return new ConsoleParameters { TraceLevel = DetermineTraceLevel(), @@ -478,7 +482,7 @@ private TraceLevel DetermineTraceLevel() _ => TraceLevel.Off } : TraceLevel.Off; - _logger.LogTrace("{0}: VsTest logging set to {1}", RunnerId, traceLevel); + _logger.LogTrace($"{RunnerId}: VsTest logging set to {traceLevel}"); return traceLevel; } @@ -498,7 +502,7 @@ private string GenerateRunSettings(int? timeout, bool forMutantTesting, bool for if (_testFramework.HasFlag(TestFramework.xUnit)) { - settingsForCoverage += "true"; + settingsForCoverage += "true"; } var timeoutSettings = timeout != null ? $"{timeout}" + Environment.NewLine : string.Empty; @@ -521,7 +525,7 @@ private string GenerateRunSettings(int? timeout, bool forMutantTesting, bool for {dataCollectorSettings} "; - _logger.LogTrace("VsTest run settings set to: {0}", runSettings); + _logger.LogTrace($"{RunnerId}: VsTest run settings set to: {runSettings}"); return runSettings; } From 8eb3459bb32682c2f89aca90ddaa8e8e42445119 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Tue, 29 Mar 2022 10:40:06 +0200 Subject: [PATCH 03/10] ci: restore integration tests --- azure-pipelines.yml | 8 ++++---- .../ValidationProject/ValidateStrykerResults.cs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8b42b02a2e..76ecfb5f50 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -200,7 +200,7 @@ stages: inputs: command: test projects: '**/Validation.csproj' - arguments: --filter Traits=SingleTestProject + arguments: --filter Category=SingleTestProject - job: LinuxTests displayName: Run tests on Linux @@ -217,7 +217,7 @@ stages: inputs: command: test projects: '**/Validation.csproj' - arguments: --filter Traits=SingleTestProject + arguments: --filter Category=SingleTestProject - job: MacOsTests displayName: Run tests on Mac OS @@ -234,7 +234,7 @@ stages: inputs: command: test projects: '**/Validation.csproj' - arguments: --filter Traits=SingleTestProject + arguments: --filter Category=SingleTestProject - template: pipeline-templates/run-integration-test-steps.yml parameters: testName: 'macos-netcore-multi-test-project' @@ -244,7 +244,7 @@ stages: inputs: command: test projects: '**/Validation.csproj' - arguments: --filter Traits=MultipleTestProjects + arguments: --filter Category=MultipleTestProjects - stage: PublishMasterReleaseArtifact displayName: Publish Release Artifact diff --git a/integrationtest/ValidationProject/ValidateStrykerResults.cs b/integrationtest/ValidationProject/ValidateStrykerResults.cs index 42affe909f..a71515a1bf 100644 --- a/integrationtest/ValidationProject/ValidateStrykerResults.cs +++ b/integrationtest/ValidationProject/ValidateStrykerResults.cs @@ -30,7 +30,7 @@ public void NetFullFramework() var report = JsonConvert.DeserializeObject(strykerRunOutput); - CheckReportMutantCounts(report, total: 12, ignored: 0, survived: 1, killed: 4, timeout: 0, nocoverage: 7); + CheckReportMutantCounts(report, total: 24, ignored: 0, survived: 1, killed: 8, timeout: 0, nocoverage: 15); } } @@ -49,7 +49,7 @@ public void NetCore() var report = JsonConvert.DeserializeObject(strykerRunOutput); - CheckReportMutantCounts(report, total: 63, ignored: 21, survived: 2, killed: 2, timeout: 2, nocoverage: 34); + CheckReportMutantCounts(report, total: 103, ignored: 36, survived: 4, killed: 9, timeout: 2, nocoverage: 50); } [Fact] @@ -67,7 +67,7 @@ public void NetCoreWithTwoTestProjects() var report = JsonConvert.DeserializeObject(strykerRunOutput); - CheckReportMutantCounts(report, total: 63, ignored: 21, survived: 2, killed: 2, timeout: 2, nocoverage: 34); + CheckReportMutantCounts(report, total: 103, ignored: 6, survived: 9, killed: 11, timeout: 2, nocoverage: 73); } private void CheckReportMutantCounts(JsonReport report, int total, int ignored, int survived, int killed, int timeout, int nocoverage) From 290f20a1d034b54ab054cf50b122d8c5dd85ea36 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Tue, 29 Mar 2022 11:47:32 +0200 Subject: [PATCH 04/10] chore: address PR remarks --- .../InitialisationProcessTests.cs | 3 -- .../TestRunners/VsTestRunnersTest.cs | 32 +++++++++--------- .../Stryker.Core/TestRunners/ITestRunner.cs | 2 -- .../TestRunners/VsTest/VsTestRunner.cs | 2 -- .../TestRunners/VsTest/VsTestRunnerPool.cs | 6 ++-- .../CoverageCollector.cs | 33 +++++-------------- 6 files changed, 28 insertions(+), 50 deletions(-) diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs b/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs index 6ab1756f72..0005a8d4f4 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs @@ -26,8 +26,6 @@ public void InitialisationProcess_ShouldCallNeededResolvers() var initialTestProcessMock = new Mock(MockBehavior.Strict); var assemblyReferenceResolverMock = new Mock(MockBehavior.Strict); - testRunnerMock.Setup(x => x.RunAll(It.IsAny(), null, null)) - .Returns(new TestRunResult(true)); // testrun is successful testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet()); testRunnerMock.Setup(x => x.Dispose()); var projectContents = new CsharpFolderComposite(); @@ -80,7 +78,6 @@ public void InitialisationProcess_ShouldThrowOnFailedInitialTestRun() var initialTestProcessMock = new Mock(MockBehavior.Strict); var assemblyReferenceResolverMock = new Mock(MockBehavior.Strict); - testRunnerMock.Setup(x => x.RunAll(It.IsAny(), null, null)); var folder = new CsharpFolderComposite(); folder.Add(new CsharpFileLeaf()); diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs index 732d611e52..a78a9b4484 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs @@ -408,7 +408,7 @@ public void RunTests() using var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset); var mockVsTest = BuildVsTestRunner(new StrykerOptions(), endProcess, out var runner); SetupMockTestRun(mockVsTest, true, endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // tests are successful => run should be successful result.FailingTests.IsEmpty.ShouldBeTrue(); } @@ -421,7 +421,7 @@ public void DetectTestsErrors() { var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner); SetupMockTestRun(mockVsTest, false, endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // run is failed result.FailingTests.IsEmpty.ShouldBeFalse(); } @@ -445,7 +445,7 @@ public void DetectTimeout() SetupMockTimeOutTestRun(mockVsTest, new Dictionary { ["0"] = "T0=S" }, "T0", endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.TimedOutTests.IsEmpty.ShouldBeFalse(); } } @@ -462,7 +462,7 @@ public void AbortOnError() mockVsTest.Setup(x => x.CancelTestRun()).Verifiable(); SetupMockTestRun(mockVsTest, false, endProcess); - var result = runner.RunAll(null, _mutant, ((_, _, _, _) => false)); + var result = runner.TestMultipleMutants(null, new []{_mutant}, ((_, _, _, _) => false)); // verify Abort has been called Mock.Verify(mockVsTest); // and test run is failed @@ -556,7 +556,7 @@ public void RunOnlyUsefulTest() SetupMockPartialTestRun(mockVsTest, new Dictionary { ["0"] = "T0=S" }, endProcess); // process coverage information - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // verify Abort has been called Mock.Verify(mockVsTest); // verify only one test has been run @@ -585,11 +585,11 @@ public void NotRunTestWhenNotCovered() SetupMockTestRun(mockVsTest, false, endProcess); // mutant 0 is covered _mutant.IsStaticValue.ShouldBeTrue(); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // mutant is killed result.FailingTests.IsEmpty.ShouldBeFalse(); // mutant 1 is not covered - result = runner.RunAll(null, _otherMutant, null); + result = runner.TestMultipleMutants(null, new []{_otherMutant}, null); // tests are ok result.RanTests.IsEmpty.ShouldBeTrue(); } @@ -658,9 +658,9 @@ public void RunRelevantTestsOnStaticWhenPerTestCoverage() runner.CaptureCoverage(_targetProject.ProjectContents.Mutants); SetupMockPartialTestRun(mockVsTest, new Dictionary { ["0"] = "T0=F", ["1"] = "T0=S" }, endProcess); - var result = runner.RunAll(null, _otherMutant, null); + var result = runner.TestMultipleMutants(null, new []{_otherMutant}, null); result.FailingTests.IsEmpty.ShouldBeTrue(); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.FailingTests.IsEmpty.ShouldBeFalse(); } } @@ -678,7 +678,7 @@ public void HandleMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.FailingTests.IsEmpty.ShouldBeTrue(); result.RanTests.IsEveryTest.ShouldBeTrue(); } @@ -696,13 +696,14 @@ public void HandleFailureWithinMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail on test 1 SetupMockTestRun(mockVsTest, new[] { ("T0", false), ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); // test session will fail on the other test result SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", false), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -721,7 +722,8 @@ public void HandleTimeOutWithMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); + result.FailingTests.IsEmpty.ShouldBeTrue(); result.TimedOutTests.Count.ShouldBe(1); result.TimedOutTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -741,7 +743,7 @@ public void HandleFailureWhenExtraMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", true), ("T0", false), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -760,7 +762,7 @@ public void HandleUnexpectedTestResult() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T2", true), ("T1", true), ("T0", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeTrue(); } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs index 798fcdb377..d21c10ab47 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs @@ -12,8 +12,6 @@ public delegate bool TestUpdateHandler(IReadOnlyList testedMutants, public interface ITestRunner : IDisposable { - TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update); - TestSet DiscoverTests(); TestRunResult InitialTest(); diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index 502b45e7cd..a8de5012be 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -96,8 +96,6 @@ public TestRunResult InitialTest() return BuildTestRunResult(testResults, int.MaxValue, false); } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList mutants, TestUpdateHandler update) { var mutantTestsMap = new Dictionary(); diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs index c4bbb1893e..c4b09294e3 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs @@ -40,7 +40,7 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR try { - return mutants == null ? runner.RunAll(timeoutCalc, null, update) : runner.TestMultipleMutants(timeoutCalc, mutants, update); + return runner.TestMultipleMutants(timeoutCalc, mutants, update); } finally { @@ -48,8 +48,6 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR } } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - public TestRunResult InitialTest() { var runner = TakeRunner(); @@ -77,7 +75,7 @@ public TestRunResult CaptureCoverage(IEnumerable mutants) try { - result = needCoverage ? runner.CaptureCoverage(mutants) : runner.RunAll(null, null, null); + result = needCoverage ? runner.CaptureCoverage(mutants) : new TestRunResult(true); } finally { diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs index f4f22ad3ba..fe0a013a16 100644 --- a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs +++ b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs @@ -66,20 +66,11 @@ public static string GetVsTestSettings(bool needCoverage, IEnumerable<(int, IEnu return string.Format(TemplateForConfiguration, line, configuration); } - public void Initialize(IDataCollectionSink dataCollectionSink) - { - this._dataSink = dataCollectionSink; - } + public void Initialize(IDataCollectionSink dataCollectionSink) => this._dataSink = dataCollectionSink; - public void SetLogger(Action logger) - { - _logger = logger; - } + public void SetLogger(Action logger) => _logger = logger; - public void Log(string message) - { - _logger?.Invoke(message); - } + public void Log(string message) => _logger?.Invoke(message); // called before any test is run public void TestSessionStart(TestSessionStartArgs testSessionStartArgs) @@ -233,22 +224,16 @@ private void PublishCoverageData(TestCaseEndArgs testCaseEndArgs) } _dataSink.SendData(testCaseEndArgs.DataCollectionContext, PropertyName, coverData); - if (_mutationCoveredOutsideTests.Count > 0) - { - // report any mutations covered before this test executed - _dataSink.SendData(testCaseEndArgs.DataCollectionContext, OutOfTestsPropertyName, string.Join(",", _mutationCoveredOutsideTests)); - _mutationCoveredOutsideTests.Clear(); - } + if (_mutationCoveredOutsideTests.Count <= 0) return; + // report any mutations covered before this test executed + _dataSink.SendData(testCaseEndArgs.DataCollectionContext, OutOfTestsPropertyName, string.Join(",", _mutationCoveredOutsideTests)); + _mutationCoveredOutsideTests.Clear(); } public IList[] RetrieveCoverData() - { - return (IList[])_getCoverageData.Invoke(null, new object[] { }); - } + => (IList[])_getCoverageData.Invoke(null, new object[] { }); public void TestSessionEnd(TestSessionEndArgs testSessionEndArgs) - { - Log($"TestSession ends."); - } + => Log($"TestSession ends."); } } From 88c9017225ab60d177461c26dfe058cfb33ff232 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Wed, 30 Mar 2022 14:29:19 +0200 Subject: [PATCH 05/10] Update src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs Co-authored-by: Rouke Broersma --- .../Stryker.DataCollector/CoverageCollector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs index fe0a013a16..54c2dd29f1 100644 --- a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs +++ b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs @@ -224,7 +224,7 @@ private void PublishCoverageData(TestCaseEndArgs testCaseEndArgs) } _dataSink.SendData(testCaseEndArgs.DataCollectionContext, PropertyName, coverData); - if (_mutationCoveredOutsideTests.Count <= 0) return; + if (_mutationCoveredOutsideTests.Count <= 0) { return; } // report any mutations covered before this test executed _dataSink.SendData(testCaseEndArgs.DataCollectionContext, OutOfTestsPropertyName, string.Join(",", _mutationCoveredOutsideTests)); _mutationCoveredOutsideTests.Clear(); From 04502116c05c52b67155e7c4afd8e80bbc4bb301 Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Wed, 30 Mar 2022 14:29:30 +0200 Subject: [PATCH 06/10] Update src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs Co-authored-by: Rouke Broersma --- .../Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs index c4b09294e3..e8e92ece5b 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs @@ -30,8 +30,8 @@ public VsTestRunnerPool(StrykerOptions options, ProjectInfo projectInfo) (_vsTests, _tests) = runner.DiscoverTests(null); _availableRunners.Add(runner); - Parallel.For(1, options.Concurrency, - (i, _) => _availableRunners.Add(new VsTestRunner(options, projectInfo, _vsTests, _tests, i, helper: _helper))); + Parallel.For(1, options.Concurrency, (i, _) => + _availableRunners.Add(new VsTestRunner(options, projectInfo, _vsTests, _tests, i, helper: _helper))); } public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList mutants, TestUpdateHandler update) From 9ec10eb580e54eb29769d44edcd8583cf719679a Mon Sep 17 00:00:00 2001 From: Cyrille DUPUYDAUBY Date: Tue, 29 Mar 2022 11:47:32 +0200 Subject: [PATCH 07/10] chore: address PR remarks --- .../ExampleLibrary/ExampleLibrary.csproj | 3 +- .../FullFrameworkApp.Test.csproj | 3 +- .../FullFrameworkApp.Test/packages.lock.json | 2 +- .../NetFramework/FullFrameworkApp/App.config | 4 +-- .../FullFrameworkApp/FullFrameworkApp.csproj | 7 ++-- .../InitialisationProcessTests.cs | 3 -- .../TestRunners/VsTestRunnersTest.cs | 32 +++++++++--------- .../Stryker.Core/TestRunners/ITestRunner.cs | 2 -- .../TestRunners/VsTest/VsTestRunner.cs | 2 -- .../TestRunners/VsTest/VsTestRunnerPool.cs | 6 ++-- .../CoverageCollector.cs | 33 +++++-------------- 11 files changed, 39 insertions(+), 58 deletions(-) diff --git a/integrationtest/TargetProjects/NetFramework/ExampleLibrary/ExampleLibrary.csproj b/integrationtest/TargetProjects/NetFramework/ExampleLibrary/ExampleLibrary.csproj index e8f330f759..119f5e3d39 100644 --- a/integrationtest/TargetProjects/NetFramework/ExampleLibrary/ExampleLibrary.csproj +++ b/integrationtest/TargetProjects/NetFramework/ExampleLibrary/ExampleLibrary.csproj @@ -9,9 +9,10 @@ Properties ExampleLibrary ExampleLibrary - v4.6.1 + v4.7.2 512 true + true diff --git a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/FullFrameworkApp.Test.csproj b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/FullFrameworkApp.Test.csproj index 99c6da0f33..21230428b6 100644 --- a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/FullFrameworkApp.Test.csproj +++ b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/FullFrameworkApp.Test.csproj @@ -10,7 +10,7 @@ Properties FullFrameworkApp.Test FullFrameworkApp.Test - v4.6.1 + v4.7.2 512 {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 15.0 @@ -21,6 +21,7 @@ UnitTest + true diff --git a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/packages.lock.json b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/packages.lock.json index 3033d77d78..447cf68e4c 100644 --- a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/packages.lock.json +++ b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp.Test/packages.lock.json @@ -1,7 +1,7 @@ { "version": 1, "dependencies": { - ".NETFramework,Version=v4.6.1": { + ".NETFramework,Version=v4.7.2": { "MSTest.TestAdapter": { "type": "Direct", "requested": "[2.2.8, 2.2.8]", diff --git a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/App.config b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/App.config index 7eb8d24f98..a932133fee 100644 --- a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/App.config +++ b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/App.config @@ -1,6 +1,6 @@ - + - + diff --git a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/FullFrameworkApp.csproj b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/FullFrameworkApp.csproj index 6dd98208da..2412f954c6 100644 --- a/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/FullFrameworkApp.csproj +++ b/integrationtest/TargetProjects/NetFramework/FullFrameworkApp/FullFrameworkApp.csproj @@ -1,4 +1,4 @@ - + @@ -8,9 +8,10 @@ Exe FullFrameworkApp FullFrameworkApp - v4.6.1 + v4.7.2 512 true + AnyCPU @@ -60,4 +61,4 @@ - + \ No newline at end of file diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs b/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs index 6ab1756f72..0005a8d4f4 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/Initialisation/InitialisationProcessTests.cs @@ -26,8 +26,6 @@ public void InitialisationProcess_ShouldCallNeededResolvers() var initialTestProcessMock = new Mock(MockBehavior.Strict); var assemblyReferenceResolverMock = new Mock(MockBehavior.Strict); - testRunnerMock.Setup(x => x.RunAll(It.IsAny(), null, null)) - .Returns(new TestRunResult(true)); // testrun is successful testRunnerMock.Setup(x => x.DiscoverTests()).Returns(new TestSet()); testRunnerMock.Setup(x => x.Dispose()); var projectContents = new CsharpFolderComposite(); @@ -80,7 +78,6 @@ public void InitialisationProcess_ShouldThrowOnFailedInitialTestRun() var initialTestProcessMock = new Mock(MockBehavior.Strict); var assemblyReferenceResolverMock = new Mock(MockBehavior.Strict); - testRunnerMock.Setup(x => x.RunAll(It.IsAny(), null, null)); var folder = new CsharpFolderComposite(); folder.Add(new CsharpFileLeaf()); diff --git a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs index 732d611e52..a78a9b4484 100644 --- a/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs +++ b/src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs @@ -408,7 +408,7 @@ public void RunTests() using var endProcess = new EventWaitHandle(false, EventResetMode.ManualReset); var mockVsTest = BuildVsTestRunner(new StrykerOptions(), endProcess, out var runner); SetupMockTestRun(mockVsTest, true, endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // tests are successful => run should be successful result.FailingTests.IsEmpty.ShouldBeTrue(); } @@ -421,7 +421,7 @@ public void DetectTestsErrors() { var mockVsTest = BuildVsTestRunner(options, endProcess, out var runner); SetupMockTestRun(mockVsTest, false, endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // run is failed result.FailingTests.IsEmpty.ShouldBeFalse(); } @@ -445,7 +445,7 @@ public void DetectTimeout() SetupMockTimeOutTestRun(mockVsTest, new Dictionary { ["0"] = "T0=S" }, "T0", endProcess); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.TimedOutTests.IsEmpty.ShouldBeFalse(); } } @@ -462,7 +462,7 @@ public void AbortOnError() mockVsTest.Setup(x => x.CancelTestRun()).Verifiable(); SetupMockTestRun(mockVsTest, false, endProcess); - var result = runner.RunAll(null, _mutant, ((_, _, _, _) => false)); + var result = runner.TestMultipleMutants(null, new []{_mutant}, ((_, _, _, _) => false)); // verify Abort has been called Mock.Verify(mockVsTest); // and test run is failed @@ -556,7 +556,7 @@ public void RunOnlyUsefulTest() SetupMockPartialTestRun(mockVsTest, new Dictionary { ["0"] = "T0=S" }, endProcess); // process coverage information - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // verify Abort has been called Mock.Verify(mockVsTest); // verify only one test has been run @@ -585,11 +585,11 @@ public void NotRunTestWhenNotCovered() SetupMockTestRun(mockVsTest, false, endProcess); // mutant 0 is covered _mutant.IsStaticValue.ShouldBeTrue(); - var result = runner.RunAll(null, _mutant, null); + var result = runner.TestMultipleMutants(null, new []{_mutant}, null); // mutant is killed result.FailingTests.IsEmpty.ShouldBeFalse(); // mutant 1 is not covered - result = runner.RunAll(null, _otherMutant, null); + result = runner.TestMultipleMutants(null, new []{_otherMutant}, null); // tests are ok result.RanTests.IsEmpty.ShouldBeTrue(); } @@ -658,9 +658,9 @@ public void RunRelevantTestsOnStaticWhenPerTestCoverage() runner.CaptureCoverage(_targetProject.ProjectContents.Mutants); SetupMockPartialTestRun(mockVsTest, new Dictionary { ["0"] = "T0=F", ["1"] = "T0=S" }, endProcess); - var result = runner.RunAll(null, _otherMutant, null); + var result = runner.TestMultipleMutants(null, new []{_otherMutant}, null); result.FailingTests.IsEmpty.ShouldBeTrue(); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.FailingTests.IsEmpty.ShouldBeFalse(); } } @@ -678,7 +678,7 @@ public void HandleMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.FailingTests.IsEmpty.ShouldBeTrue(); result.RanTests.IsEveryTest.ShouldBeTrue(); } @@ -696,13 +696,14 @@ public void HandleFailureWithinMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail on test 1 SetupMockTestRun(mockVsTest, new[] { ("T0", false), ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); // test session will fail on the other test result SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", false), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -721,7 +722,8 @@ public void HandleTimeOutWithMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); + result.FailingTests.IsEmpty.ShouldBeTrue(); result.TimedOutTests.Count.ShouldBe(1); result.TimedOutTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -741,7 +743,7 @@ public void HandleFailureWhenExtraMultipleTestResults() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T0", true), ("T0", false), ("T1", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeFalse(); result.FailingTests.GetGuids().ShouldContain(_testCases[0].Id); @@ -760,7 +762,7 @@ public void HandleUnexpectedTestResult() result.FailingTests.IsEmpty.ShouldBeTrue(); // test session will fail SetupMockTestRun(mockVsTest, new[] { ("T0", true), ("T2", true), ("T1", true), ("T0", true) }, endProcess); - result = runner.RunAll(null, _mutant, null); + result = runner.TestMultipleMutants(null, new []{_mutant}, null); result.RanTests.IsEveryTest.ShouldBeTrue(); result.FailingTests.IsEmpty.ShouldBeTrue(); } diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs index 798fcdb377..d21c10ab47 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/ITestRunner.cs @@ -12,8 +12,6 @@ public delegate bool TestUpdateHandler(IReadOnlyList testedMutants, public interface ITestRunner : IDisposable { - TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update); - TestSet DiscoverTests(); TestRunResult InitialTest(); diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index 502b45e7cd..a8de5012be 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -96,8 +96,6 @@ public TestRunResult InitialTest() return BuildTestRunResult(testResults, int.MaxValue, false); } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IReadOnlyList mutants, TestUpdateHandler update) { var mutantTestsMap = new Dictionary(); diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs index c4bbb1893e..c4b09294e3 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunnerPool.cs @@ -40,7 +40,7 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR try { - return mutants == null ? runner.RunAll(timeoutCalc, null, update) : runner.TestMultipleMutants(timeoutCalc, mutants, update); + return runner.TestMultipleMutants(timeoutCalc, mutants, update); } finally { @@ -48,8 +48,6 @@ public TestRunResult TestMultipleMutants(ITimeoutValueCalculator timeoutCalc, IR } } - public TestRunResult RunAll(ITimeoutValueCalculator timeoutMs, Mutant activeMutant, TestUpdateHandler update) => TestMultipleMutants(timeoutMs, activeMutant == null ? null : new List { activeMutant }, update); - public TestRunResult InitialTest() { var runner = TakeRunner(); @@ -77,7 +75,7 @@ public TestRunResult CaptureCoverage(IEnumerable mutants) try { - result = needCoverage ? runner.CaptureCoverage(mutants) : runner.RunAll(null, null, null); + result = needCoverage ? runner.CaptureCoverage(mutants) : new TestRunResult(true); } finally { diff --git a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs index f4f22ad3ba..fe0a013a16 100644 --- a/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs +++ b/src/Stryker.DataCollector/Stryker.DataCollector/CoverageCollector.cs @@ -66,20 +66,11 @@ public static string GetVsTestSettings(bool needCoverage, IEnumerable<(int, IEnu return string.Format(TemplateForConfiguration, line, configuration); } - public void Initialize(IDataCollectionSink dataCollectionSink) - { - this._dataSink = dataCollectionSink; - } + public void Initialize(IDataCollectionSink dataCollectionSink) => this._dataSink = dataCollectionSink; - public void SetLogger(Action logger) - { - _logger = logger; - } + public void SetLogger(Action logger) => _logger = logger; - public void Log(string message) - { - _logger?.Invoke(message); - } + public void Log(string message) => _logger?.Invoke(message); // called before any test is run public void TestSessionStart(TestSessionStartArgs testSessionStartArgs) @@ -233,22 +224,16 @@ private void PublishCoverageData(TestCaseEndArgs testCaseEndArgs) } _dataSink.SendData(testCaseEndArgs.DataCollectionContext, PropertyName, coverData); - if (_mutationCoveredOutsideTests.Count > 0) - { - // report any mutations covered before this test executed - _dataSink.SendData(testCaseEndArgs.DataCollectionContext, OutOfTestsPropertyName, string.Join(",", _mutationCoveredOutsideTests)); - _mutationCoveredOutsideTests.Clear(); - } + if (_mutationCoveredOutsideTests.Count <= 0) return; + // report any mutations covered before this test executed + _dataSink.SendData(testCaseEndArgs.DataCollectionContext, OutOfTestsPropertyName, string.Join(",", _mutationCoveredOutsideTests)); + _mutationCoveredOutsideTests.Clear(); } public IList[] RetrieveCoverData() - { - return (IList[])_getCoverageData.Invoke(null, new object[] { }); - } + => (IList[])_getCoverageData.Invoke(null, new object[] { }); public void TestSessionEnd(TestSessionEndArgs testSessionEndArgs) - { - Log($"TestSession ends."); - } + => Log($"TestSession ends."); } } From 2aa989f0dddc7a87093b25cee882a82b99d1ba09 Mon Sep 17 00:00:00 2001 From: Rouke Broersma Date: Fri, 1 Apr 2022 09:09:13 +0200 Subject: [PATCH 08/10] Update src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs --- .../Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs index 92cf230146..95dc678f0c 100644 --- a/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs +++ b/src/Stryker.Core/Stryker.Core/CoverageAnalysis/CoverageAnalyser.cs @@ -73,7 +73,7 @@ private void SetCoveringTests(IReadOnlyCollection mutantsToScan) } if (mutant.CoveringTests.IsEveryTest) { - _logger.LogDebug($"Mutant {mutant.Id} will be tested agaisnt all tests."); + _logger.LogDebug($"Mutant {mutant.Id} will be tested against all tests."); } else if (mutant.CoveringTests.Count>0) { From f1aba1fdb2244d43ded4c74c4fab7428aad996a3 Mon Sep 17 00:00:00 2001 From: Rouke Broersma Date: Fri, 1 Apr 2022 09:13:19 +0200 Subject: [PATCH 09/10] Update src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs --- .../Stryker.Core/TestRunners/VsTest/VsTestRunner.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index a8de5012be..a20fc25311 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -183,7 +183,9 @@ void HandleUpdate(IRunResults handler) } if (timeOutMs.HasValue) + { _logger.LogDebug($"{RunnerId}: Using {timeOutMs} ms as test run timeout"); + } var testResults = RunTestSession(mutantTestsMap, needAll, GenerateRunSettings(timeOutMs, mutants != null, false, mutantTestsMap), timeOutMs, HandleUpdate); From 24c824091f4175a941f9340b83ba87d5f51add7e Mon Sep 17 00:00:00 2001 From: Rouke Broersma Date: Mon, 11 Apr 2022 23:06:03 +0200 Subject: [PATCH 10/10] Some codestyle cleanup --- .../TestRunners/VsTest/VsTestRunner.cs | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs index a20fc25311..3fb6d08893 100644 --- a/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs +++ b/src/Stryker.Core/Stryker.Core/TestRunners/VsTest/VsTestRunner.cs @@ -88,7 +88,8 @@ public TestRunResult InitialTest() if (!_vsTests.ContainsKey(result.TestCase.Id)) { _vsTests[result.TestCase.Id] = new VsTestDescription(result.TestCase); - _logger.LogWarning($"{RunnerId}: Initial test run encounter a unexpected test case ({result.TestCase.DisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts."); + _logger.LogWarning("{RunnerId}: Initial test run encounter a unexpected test case ({TestCaseDisplayName}), mutation tests may be inaccurate. Disable coverage analysis if your have doubts.", + RunnerId, result.TestCase.DisplayName); } _vsTests[result.TestCase.Id].RegisterInitialTestResult(result); } @@ -395,7 +396,7 @@ private void ParseResultsForCoverage(IEnumerable testResults, IEnume public void CoverageForOneTest(Guid test, IEnumerable mutants) { - _logger.LogDebug($"{RunnerId}: Capturing coverage for {_vsTests[test].Case.FullyQualifiedName}."); + _logger.LogDebug("{RunnerId}: Capturing coverage for {TestCaseFullyQualifiedName}.", RunnerId, _vsTests[test].Case.FullyQualifiedName); var map = new Dictionary(1) { [-1] = new WrappedGuidsEnumeration(new[] { test }) }; var testResults = RunTestSession(map, true, GenerateRunSettings(null, false, true, null)); ParseResultsForCoverage(testResults.TestResults.Where(x => x.TestCase.Id == test), mutants); @@ -409,8 +410,16 @@ private IRunResults RunTestSession(Dictionary mutantTestsMap, int retries = 0) { using var eventHandler = new RunEventHandler(_vsTests, _logger, RunnerId); - void HandlerVsTestFailed(object sender, EventArgs e) => _vsTestFailed = true; - void HandlerUpdate(object sender, EventArgs e) => updateHandler?.Invoke(eventHandler); + void HandlerVsTestFailed(object sender, EventArgs e) + { + _vsTestFailed = true; + } + + void HandlerUpdate(object sender, EventArgs e) + { + updateHandler?.Invoke(eventHandler); + } + var strykerVsTestHostLauncher = _hostBuilder(_id); eventHandler.VsTestFailed += HandlerVsTestFailed; @@ -452,16 +461,19 @@ private IRunResults RunTestSession(Dictionary mutantTestsMap, PrepareVsTestConsole(); _vsTestFailed = false; - return RunTestSession(mutantTestsMap, runAllTests, runSettings, timeOut, updateHandler, retries+1); + return RunTestSession(mutantTestsMap, runAllTests, runSettings, timeOut, updateHandler, retries + 1); } private ConsoleParameters DetermineConsoleParameters() { - if (!_options.LogOptions.LogToFile) return new ConsoleParameters(); + if (!_options.LogOptions.LogToFile) + { + return new ConsoleParameters(); + } var vsTestLogPath = _fileSystem.Path.Combine(_options.OutputPath, "logs", $"{RunnerId}_VsTest-log.txt"); _fileSystem.Directory.CreateDirectory(_fileSystem.Path.GetDirectoryName(vsTestLogPath)); - _logger.LogTrace($"{RunnerId}: Logging VsTest output to: {vsTestLogPath}"); + _logger.LogTrace("{RunnerId}: Logging VsTest output to: {vsTestLogPath}", RunnerId, vsTestLogPath); return new ConsoleParameters { TraceLevel = DetermineTraceLevel(), @@ -471,7 +483,7 @@ private ConsoleParameters DetermineConsoleParameters() private TraceLevel DetermineTraceLevel() { - var traceLevel = _options.LogOptions.LogToFile ? _options.LogOptions.LogLevel switch + var traceLevel = _options.LogOptions.LogLevel switch { LogEventLevel.Debug => TraceLevel.Verbose, LogEventLevel.Verbose => TraceLevel.Verbose, @@ -480,9 +492,9 @@ private TraceLevel DetermineTraceLevel() LogEventLevel.Warning => TraceLevel.Warning, LogEventLevel.Information => TraceLevel.Info, _ => TraceLevel.Off - } : TraceLevel.Off; + }; - _logger.LogTrace($"{RunnerId}: VsTest logging set to {traceLevel}"); + _logger.LogTrace("{RunnerId}: VsTest logging set to {traceLevel}", RunnerId, traceLevel); return traceLevel; } @@ -525,7 +537,7 @@ private string GenerateRunSettings(int? timeout, bool forMutantTesting, bool for {dataCollectorSettings} "; - _logger.LogTrace($"{RunnerId}: VsTest run settings set to: {runSettings}"); + _logger.LogTrace("{RunnerId}: VsTest run settings set to: {runSettings}", RunnerId, runSettings); return runSettings; } @@ -550,7 +562,7 @@ private void PrepareVsTestConsole() // Set roll forward on no candidate fx so vstest console can start on incompatible dotnet core runtimes Environment.SetEnvironmentVariable("DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX", "2"); _vsTestConsole.StartSession(); - _vsTestConsole.InitializeExtensions(_sources.Select( _fileSystem.Path.GetDirectoryName)); + _vsTestConsole.InitializeExtensions(_sources.Select(_fileSystem.Path.GetDirectoryName)); } catch (Exception e) {