Skip to content

Commit

Permalink
tests which were skipped due to a crashed test now link to that test (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
csoltenborn committed Feb 28, 2016
1 parent b91e961 commit af3bb23
Show file tree
Hide file tree
Showing 10 changed files with 63 additions and 26 deletions.
16 changes: 13 additions & 3 deletions GoogleTestAdapter/Core/Runners/SequentialTestRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -106,15 +106,25 @@ private List<TestResult> CollectTestResults(IEnumerable<TestCase> testCasesRun,

if (testResults.Count < testCasesRunAsArray.Length)
{
string errorMessage, errorStackTrace = null;
if (consoleParser.CrashedTestCase == null)
{
errorMessage = "";
}
else
{
errorMessage = "reason is probably a crash of test " + consoleParser.CrashedTestCase.DisplayName;
errorStackTrace = ErrorMessageParser.GetStackTraceEntry("crash suspect",
consoleParser.CrashedTestCase.CodeFilePath, consoleParser.CrashedTestCase.LineNumber.ToString());
}
foreach (TestCase testCase in testCasesRunAsArray.Where(tc => !testResults.Exists(tr => tr.TestCase.FullyQualifiedName == tc.FullyQualifiedName)))
{
string errorMsg = consoleParser.CrashedTestCase == null ? ""
: "reason is probably a crash of test " + consoleParser.CrashedTestCase.DisplayName;
testResults.Add(new TestResult(testCase)
{
ComputerName = Environment.MachineName,
Outcome = TestOutcome.Skipped,
ErrorMessage = errorMsg
ErrorMessage = errorMessage,
ErrorStackTrace = errorStackTrace
});
}
}
Expand Down
7 changes: 6 additions & 1 deletion GoogleTestAdapter/Core/TestResults/ErrorMessageParser.cs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ private void HandleMultipleFailures()
ErrorStackTrace = string.Join("", finalStackTraces);
}

public static string GetStackTraceEntry(string label, string fullFileName, string lineNumber)
{
return $"at {label} in {fullFileName}:line {lineNumber}{Environment.NewLine}";
}

private void CreateErrorMessageAndStacktrace(ref string errorMessage, out string stackTrace, int msgId = 0)
{
Match match = ParseRegex.Match(errorMessage);
Expand All @@ -138,7 +143,7 @@ private void CreateErrorMessageAndStacktrace(ref string errorMessage, out string

string msgReference = msgId == 0 ? "" : $"#{msgId} - ";

stackTrace = $"at {msgReference}{fileName}:{lineNumber} in {fullFileName}:line {lineNumber}{Environment.NewLine}";
stackTrace = GetStackTraceEntry($"{msgReference}{fileName}:{lineNumber}", fullFileName, lineNumber);
errorMessage = errorMessage.Replace(match.Value, "").Trim();

match = ScopedTraceStartRegex.Match(errorMessage);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Threading;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
using GoogleTestAdapter.Framework;
using GoogleTestAdapter.Model;
using GoogleTestAdapter.TestAdapter.Helpers;

namespace GoogleTestAdapter.TestAdapter.Framework
Expand Down Expand Up @@ -58,22 +59,22 @@ public void ReportTestsStarted(IEnumerable<Model.TestCase> testCases)
{
lock (Lock)
{
foreach (Model.TestCase testCase in testCases)
foreach (TestCase testCase in testCases)
{
FrameworkHandle.RecordStart(DataConversionExtensions.ToVsTestCase(testCase));
}
}
}

public void ReportTestResults(IEnumerable<Model.TestResult> testResults)
public void ReportTestResults(IEnumerable<TestResult> testResults)
{
lock (Lock)
{
foreach (Model.TestResult testResult in testResults)
foreach (TestResult testResult in testResults)
{
if (IsRunningInsideVisualStudio)
if (IsRunningInsideVisualStudio && (testResult.Outcome == TestOutcome.Failed || testResult.Outcome == TestOutcome.Skipped))
testResult.ErrorMessage = Environment.NewLine + testResult.ErrorMessage;
else if (testResult.ErrorStackTrace != null)
if (!IsRunningInsideVisualStudio && testResult.ErrorStackTrace != null)
testResult.ErrorStackTrace = testResult.ErrorStackTrace.Trim();

ReportTestResult(testResult);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,11 @@ at crashingtests.cpp:12 in ${BaseDir}\sampletests\crashingtests\crashingtests.cp
!! This is probably the test that crashed !!
Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Test Run Failed.
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Test Run Failed.
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ at crashingtests.cpp:12 in ${BaseDir}\sampletests\crashingtests\crashingtests.cp
!! This is probably the test that crashed !!
Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Test Run Failed.
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Test Run Failed.

Google Test Adapter Coverage Marker
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,11 @@ at crashingtests.cpp:12 in ${BaseDir}\sampletests\crashingtests\crashingtests.cp
!! This is probably the test that crashed !!
Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Test Run Failed.
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Test Run Failed.
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ at crashingtests.cpp:12 in ${BaseDir}\sampletests\crashingtests\crashingtests.cp
!! This is probably the test that crashed !!
Error Message:
reason is probably a crash of test Crashing.TheCrash
Error Message:
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Error Message:
reason is probably a crash of test Crashing.TheCrash
Test Run Failed.
Stack Trace:
at crash suspect in ${BaseDir}\sampletests\crashingtests\crashingtests.cpp:line 21Test Run Failed.

Google Test Adapter Coverage Marker
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,11 @@ public static string RunExecutableAndGetOutput(string solutionFile, string argum

public static bool IsRunningOnBuildServer()
{
return Environment.GetEnvironmentVariable("APPVEYOR") != null;
#pragma warning disable 162
// ReSharper disable once RedundantLogicalConditionalExpressionOperand
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
return ResultChecker.OverwriteTestResults || Environment.GetEnvironmentVariable("APPVEYOR") != null;
#pragma warning restore 162
}

private static string NormalizeOutput(string resultString, string baseDir)
Expand Down
4 changes: 2 additions & 2 deletions GoogleTestAdapter/VsPackage.Tests/ResultChecker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ namespace GoogleTestAdapter.VsPackage
{
public class ResultChecker
{
private const bool overwriteTestResults = false;
internal const bool OverwriteTestResults = false;

private readonly string goldenFilesDirectory;
private readonly string testErrorsDirectory;
Expand Down Expand Up @@ -40,7 +40,7 @@ public void CheckResults(string testResults, string typeName, [CallerMemberName]
{
#pragma warning disable CS0162 // Unreachable code (because overwriteTestResults is compile time constant)
// ReSharper disable once ConditionIsAlwaysTrueOrFalse
if (overwriteTestResults)
if (OverwriteTestResults)
{
File.WriteAllText(expectationFile, testResults);
Assert.Inconclusive("Test results changed and have been overwritten. Differences: " + msg);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ Expected: 1000</Error>
<Name>Crashing.TheCrash</Name>
<Result>Test Failed - Crashing.TheCrash</Result>
<Source>crashingtests.cpp line 21</Source>
<Error>Message: !! This is probably the test that crashed !!</Error>
<Error>Message:
!! This is probably the test that crashed !!</Error>
</TestCase>
<TestCase>
<Name>InstantiationName/ParameterizedTests.Simple/1 [(1,!)]</Name>
Expand Down Expand Up @@ -461,19 +462,25 @@ Expected: (this-&gt;container[0] + this-&gt;container[1]) != (this-&gt;container
<Name>Crashing.AddFailsAfterCrash</Name>
<Result>Test Skipped - Crashing.AddFailsAfterCrash</Result>
<Source>crashingtests.cpp line 26</Source>
<Error>Message: reason is probably a crash of test Crashing.TheCrash</Error>
<Error>Message:
reason is probably a crash of test Crashing.TheCrash</Error>
<Stacktrace>crash suspect</Stacktrace>
</TestCase>
<TestCase>
<Name>Crashing.AddPassesAfterCrash</Name>
<Result>Test Skipped - Crashing.AddPassesAfterCrash</Result>
<Source>crashingtests.cpp line 31</Source>
<Error>Message: reason is probably a crash of test Crashing.TheCrash</Error>
<Error>Message:
reason is probably a crash of test Crashing.TheCrash</Error>
<Stacktrace>crash suspect</Stacktrace>
</TestCase>
<TestCase>
<Name>Crashing.LongRunning</Name>
<Result>Test Skipped - Crashing.LongRunning</Result>
<Source>crashingtests.cpp line 36</Source>
<Error>Message: reason is probably a crash of test Crashing.TheCrash</Error>
<Error>Message:
reason is probably a crash of test Crashing.TheCrash</Error>
<Stacktrace>crash suspect</Stacktrace>
</TestCase>
</TestGroup>
<TestGroup>
Expand Down

0 comments on commit af3bb23

Please sign in to comment.