Skip to content

Commit

Permalink
fix: add external timeout support for VsTEst
Browse files Browse the repository at this point in the history
  • Loading branch information
@dupdob
dupdob committed Mar 17, 2022
1 parent ad5f8f9 commit 18ea19b
Show file tree
Hide file tree
Showing 7 changed files with 232 additions and 253 deletions.
53 changes: 16 additions & 37 deletions src/Stryker.Core/Stryker.Core.UnitTest/TestRunners/VsTestRunnersTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ private void SetupMockTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IEnumerabl
});
}
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
x.RunTestsWithCustomTestHostAsync(
It.Is<IEnumerable<string>>(t => t.Any(source => source == _testAssemblyPath)),
It.Is<string>(settings => !settings.Contains("<Coverage")),
It.Is<TestPlatformOptions>(o => o != null && o.TestCaseFilter == null),
Expand All @@ -199,14 +199,14 @@ private void SetupMockTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IEnumerabl
// generate test results
MoqTestRun(testRunEvents, results);
synchroObject.Set();
});
}).Returns(Task.CompletedTask);
}

// setup a customized coverage capture run, using provided coverage results
private void SetupMockCoverageRun(Mock<IVsTestConsoleWrapper> mockVsTest, IReadOnlyDictionary<string, string> coverageResults, EventWaitHandle endProcess)
{
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
x.RunTestsWithCustomTestHostAsync(
It.Is<IEnumerable<string>>(t => t.Any(source => source == _testAssemblyPath)),
It.Is<string>(settings => settings.Contains("<Coverage")),
It.Is<TestPlatformOptions>(o => o != null && o.TestCaseFilter == null),
Expand Down Expand Up @@ -235,14 +235,14 @@ private void SetupMockCoverageRun(Mock<IVsTestConsoleWrapper> mockVsTest, IReadO
}
MoqTestRun(testRunEvents, results);
endProcess.Set();
});
}).Returns(Task.CompletedTask);
}

// setup a customized partial test runs, using provided test results
private void SetupMockPartialTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IReadOnlyDictionary<string, string> results, EventWaitHandle endProcess)
{
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
x.RunTestsWithCustomTestHostAsync(
It.IsAny<IEnumerable<TestCase>>(),
It.Is<string>(s => !s.Contains("<Coverage")),
It.Is<TestPlatformOptions>(o => o != null && o.TestCaseFilter == null),
Expand Down Expand Up @@ -291,13 +291,13 @@ private void SetupMockPartialTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IRe
collector.TestSessionEnd(new TestSessionEndArgs());

endProcess.Set();
});
}).Returns(Task.CompletedTask);
}

private void SetupMockTimeOutTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IReadOnlyDictionary<string, string> results, string timeoutTest, EventWaitHandle endProcess)
{
mockVsTest.Setup(x =>
x.RunTestsWithCustomTestHost(
x.RunTestsWithCustomTestHostAsync(
It.IsAny<IEnumerable<TestCase>>(),
It.IsAny<string>(),
It.Is<TestPlatformOptions>(o => o != null && o.TestCaseFilter == null),
Expand Down Expand Up @@ -350,7 +350,7 @@ private void SetupMockTimeOutTestRun(Mock<IVsTestConsoleWrapper> mockVsTest, IRe
collector.TestSessionEnd(new TestSessionEndArgs());

endProcess.Set();
});
}).Returns(Task.CompletedTask);
}

private Mock<IVsTestConsoleWrapper> BuildVsTestRunner(StrykerOptions options, WaitHandle endProcess, out VsTestRunner runner)
Expand All @@ -359,13 +359,11 @@ private Mock<IVsTestConsoleWrapper> BuildVsTestRunner(StrykerOptions options, Wa
mockedVsTestConsole.Setup(x => x.StartSession());
mockedVsTestConsole.Setup(x => x.InitializeExtensions(It.IsAny<List<string>>()));
mockedVsTestConsole.Setup(x => x.AbortTestRun());
mockedVsTestConsole.Setup(x =>
x.DiscoverTests(It.Is<IEnumerable<string>>(d => d.Any(e => e == _testAssemblyPath)),
It.IsAny<string>(),
It.IsAny<ITestDiscoveryEventsHandler>())).Callback(
mockedVsTestConsole.Setup(x => x.DiscoverTestsAsync(It.Is<IEnumerable<string>>(d => d.Any(e => e == _testAssemblyPath)),
It.IsAny<string>(),
It.IsAny<ITestDiscoveryEventsHandler>())).Callback(
(IEnumerable<string> _, string _, ITestDiscoveryEventsHandler discoveryEventsHandler) =>
DiscoverTests(discoveryEventsHandler, _testCases, false));

DiscoverTests(discoveryEventsHandler, _testCases, false)).Returns(Task.CompletedTask);
runner = new VsTestRunner(
options,
_targetProject,
Expand All @@ -374,7 +372,7 @@ private Mock<IVsTestConsoleWrapper> BuildVsTestRunner(StrykerOptions options, Wa
_fileSystem,
new Mock<IVsTestHelper>().Object,
wrapper: mockedVsTestConsole.Object,
hostBuilder: _ => new MoqHost(endProcess, false));
hostBuilder: _ => new MoqHost());
return mockedVsTestConsole;
}

Expand Down Expand Up @@ -779,32 +777,13 @@ public void MarkSuspiciousCoverage()
}

// class mocking the VsTest Host Launcher
private class MoqHost : IStrykerTestHostLauncher
private class MoqHost : ITestHostLauncher
{
private readonly WaitHandle _handle;
public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) => throw new NotImplementedException();

public MoqHost(WaitHandle handle, bool isDebug)
{
_handle = handle;
IsDebug = isDebug;
}

public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo)
{
throw new NotImplementedException();
}

public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) => throw new NotImplementedException();

public bool IsDebug { get; }

public bool WaitProcessExit()
{
return _handle == null || _handle.WaitOne();
}
}
}
}
10 changes: 2 additions & 8 deletions src/Stryker.Core/Stryker.Core/TestRunners/VsTest/DiscoveryEventHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,8 @@ public void HandleDiscoveryComplete(long totalTests, IEnumerable<TestCase> lastC
_waitHandle.Set();
}

public void HandleRawMessage(string rawMessage)
{
_messages.Add("Test Discovery Raw Message: " + rawMessage);
}
public void HandleRawMessage(string rawMessage) => _messages.Add("Test Discovery Raw Message: " + rawMessage);

public void HandleLogMessage(TestMessageLevel level, string message)
{
_messages.Add("Test Discovery Message: " + message);
}
public void HandleLogMessage(TestMessageLevel level, string message) => _messages.Add("Test Discovery Message: " + message);
}
}
184 changes: 77 additions & 107 deletions src/Stryker.Core/Stryker.Core/TestRunners/VsTest/RunEventHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,60 +16,6 @@ public interface IRunResults
List<TestResult> TestResults { get; }
IReadOnlyList<TestCase> TestsInTimeout { get; }
}
internal class TestRun
{
private readonly VsTestDescription _testDescription;
private readonly IList<TestResult> _results;

public TestRun(VsTestDescription testDescription)
{
_testDescription = testDescription;
_results = new List<TestResult>(testDescription.NbSubCases);
}

public bool AddResult(TestResult result)
{
_results.Add(result);
return _results.Count >= _testDescription.NbSubCases;
}

public bool IsComplete()
{
return _results.Count >= _testDescription.NbSubCases;
}

public TestResult Result()
{
var result = _results.Aggregate((TestResult)null, (acc, next) =>
{
if (acc == null)
{
return next;
}
if (next.Outcome == TestOutcome.Failed || acc.Outcome == TestOutcome.None)
{
acc.Outcome = next.Outcome;
}
if (acc.StartTime > next.StartTime)
{
acc.StartTime = next.StartTime;
}
if (acc.EndTime < next.EndTime)
{
acc.EndTime = next.EndTime;
}

foreach (var message in next.Messages)
{
acc.Messages.Add(message);
}

acc.Duration = acc.EndTime - acc.StartTime;
return acc;
});
return result;
}
}

public sealed class RunEventHandler : ITestRunEventsHandler, IDisposable, IRunResults
{
Expand All @@ -78,7 +24,9 @@ public sealed class RunEventHandler : ITestRunEventsHandler, IDisposable, IRunRe
private readonly string _runnerId;
private readonly IDictionary<Guid, VsTestDescription> _vsTests;
private readonly IDictionary<Guid, TestRun> _runs = new Dictionary<Guid, TestRun>();
private readonly Dictionary<Guid, TestCase> _inProgress = new Dictionary<Guid, TestCase>();
private readonly Dictionary<Guid, TestCase> _inProgress = new();
private bool _disposed;
private readonly object _lck = new();
public event EventHandler VsTestFailed;
public event EventHandler ResultsUpdated;

Expand Down Expand Up @@ -125,21 +73,26 @@ private void CaptureTestResults(IEnumerable<TestResult> results)

public void HandleTestRunStatsChange(TestRunChangedEventArgs testRunChangedArgs)
{
if (testRunChangedArgs.ActiveTests != null)
lock (_lck)
{
foreach (var activeTest in testRunChangedArgs.ActiveTests)
if (_disposed)
return;
if (testRunChangedArgs.ActiveTests != null)
{
_inProgress[activeTest.Id] = activeTest;
foreach (var activeTest in testRunChangedArgs.ActiveTests)
{
_inProgress[activeTest.Id] = activeTest;
}
}
}

if (testRunChangedArgs.NewTestResults == null || !testRunChangedArgs.NewTestResults.Any())
{
return;
}
if (testRunChangedArgs.NewTestResults == null || !testRunChangedArgs.NewTestResults.Any())
{
return;
}

CaptureTestResults(testRunChangedArgs.NewTestResults);
ResultsUpdated?.Invoke(this, EventArgs.Empty);
CaptureTestResults(testRunChangedArgs.NewTestResults);
ResultsUpdated?.Invoke(this, EventArgs.Empty);
}
}

public void HandleTestRunComplete(
Expand All @@ -148,69 +101,82 @@ public void HandleTestRunComplete(
ICollection<AttachmentSet> runContextAttachments,
ICollection<string> executorUris)
{
if (lastChunkArgs?.ActiveTests != null)
_logger.LogTrace($"{_runnerId}: Test Completed");
lock (_lck)
{
foreach (var activeTest in lastChunkArgs.ActiveTests)
if (_disposed)
{
_inProgress[activeTest.Id] = activeTest;
return;
}
}

if (lastChunkArgs?.NewTestResults != null)
{
CaptureTestResults(lastChunkArgs.NewTestResults);
}

if (!testRunCompleteArgs.IsCanceled && (_inProgress.Any() || _runs.Values.Any(t => !t.IsComplete())))
{
// report ongoing tests and test case with missing results as timeouts.
TestsInTimeout = _inProgress.Values.Union(_runs.Values.Where(t => !t.IsComplete()).Select(t => t.Result().TestCase)).ToList();
if (TestsInTimeout.Count > 0)
if (lastChunkArgs?.ActiveTests != null)
{
TimeOut = true;
foreach (var activeTest in lastChunkArgs.ActiveTests)
{
_inProgress[activeTest.Id] = activeTest;
}
}
}

ResultsUpdated?.Invoke(this, EventArgs.Empty);

if (testRunCompleteArgs.Error != null)
{
if (testRunCompleteArgs.Error.GetType() == typeof(TransationLayerException))
if (lastChunkArgs?.NewTestResults != null)
{
_logger.LogDebug(testRunCompleteArgs.Error, $"{_runnerId}: VsTest may have crashed, triggering VsTest restart!");
VsTestFailed?.Invoke(this, EventArgs.Empty);
CaptureTestResults(lastChunkArgs.NewTestResults);
}
else if (testRunCompleteArgs.Error.InnerException is IOException sock)

if (!testRunCompleteArgs.IsCanceled && (_inProgress.Any() || _runs.Values.Any(t => !t.IsComplete())))
{
_logger.LogWarning(sock, $"{_runnerId}: Test session ended unexpectedly.");
// report ongoing tests and test case with missing results as timeouts.
TestsInTimeout = _inProgress.Values.Union(_runs.Values.Where(t => !t.IsComplete()).Select(t => t.Result().TestCase)).ToList();
if (TestsInTimeout.Count > 0)
{
TimeOut = true;
}
}
else if (!CancelRequested)

ResultsUpdated?.Invoke(this, EventArgs.Empty);

if (testRunCompleteArgs.Error != null)
{
_logger.LogDebug(testRunCompleteArgs.Error, $"{_runnerId}: VsTest error:");
if (testRunCompleteArgs.Error.GetType() == typeof(TransationLayerException))
{
_logger.LogDebug(testRunCompleteArgs.Error, $"{_runnerId}: VsTest may have crashed, triggering VsTest restart!");
VsTestFailed?.Invoke(this, EventArgs.Empty);
}
else if (testRunCompleteArgs.Error.InnerException is IOException sock)
{
_logger.LogDebug(sock, $"{_runnerId}: Test session ended unexpectedly.");
}
else if (!CancelRequested)
{
_logger.LogDebug(testRunCompleteArgs.Error, $"{_runnerId}: VsTest error:");
VsTestFailed?.Invoke(this, EventArgs.Empty);
}
}
}

_waitHandle.Set();
_waitHandle.Set();
}
}

public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo)
{
throw new NotSupportedException();
}
public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessStartInfo) => throw new NotSupportedException();

public void HandleRawMessage(string rawMessage)
{
_logger.LogTrace($"{_runnerId}: {rawMessage} [RAW]");
}
public void HandleRawMessage(string rawMessage) => _logger.LogTrace($"{_runnerId}: {rawMessage} [RAW]");

public void WaitEnd()
public void WaitEnd(DateTime deadLine)
{
_waitHandle.WaitOne();
var timeout = deadLine - DateTime.Now;
if (timeout < TimeSpan.Zero)
{
timeout = TimeSpan.Zero;
}
if (!_waitHandle.WaitOne(timeout))
{
_logger.LogDebug($"{_runnerId}: VsTest client did not report run completion.");
HandleTestRunComplete(new TestRunCompleteEventArgs(null, false, false, null, null, TimeSpan.Zero),
null, null, null);
}
}

public void HandleLogMessage(TestMessageLevel level, string message)
{
LogLevel levelFinal = level switch
var levelFinal = level switch
{
TestMessageLevel.Informational => LogLevel.Debug,
TestMessageLevel.Warning => LogLevel.Warning,
Expand All @@ -222,7 +188,11 @@ public void HandleLogMessage(TestMessageLevel level, string message)

public void Dispose()
{
_waitHandle.Dispose();
lock (_lck)
{
_waitHandle.Dispose();
_disposed = true;
}
}
}
}
Loading

0 comments on commit 18ea19b

Please sign in to comment.