Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[xharness] Improve parsing of NUnitV2 xml reports to fix an issue with failures in parameterized tests. #9259

Merged
merged 1 commit into from
Aug 3, 2020
Merged

[xharness] Improve parsing of NUnitV2 xml reports to fix an issue with failures in parameterized tests. #9259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ...rosoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
<None Remove="Samples\TouchUnitSample.xml" />
<None Remove="Samples\xUnitSample.xml" />
<None Remove="Samples\NUnitV2SampleFailure.xml" />
<None Remove="Samples\TestCaseFailures.xml" />
</ItemGroup>

<ItemGroup>
Expand Down Expand Up @@ -52,6 +53,7 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</EmbeddedResource>
<EmbeddedResource Include="Samples\NUnitV2SampleFailure.xml" />
<EmbeddedResource Include="Samples\TestCaseFailures.xml" />
</ItemGroup>

<ItemGroup>
Expand Down
62 changes: 62 additions & 0 deletions tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<test-results name="mtouch.dll" total="440" errors="4" failures="19" not-run="6" inconclusive="0" ignored="6" skipped="0" invalid="0" date="2020-07-31" time="03:17:46">
<environment nunit-version="3.6.0.0" clr-version="4.0.30319.42000" os-version="Unix 19.5.0.0" platform="Unix" cwd="..." machine-name="..." user="..." user-domain="..." />
<culture-info current-culture="en-US" current-uiculture="en-US" />
<test-suite type="Assembly" name="mtouch.dll" executed="True" result="Failure" success="False" time="3393.268" asserts="7560">
<properties>
<property name="_PID" value="72474" />
<property name="_APPDOMAIN" value="domain-" />
</properties>
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
<stack-trace />
</failure>
<results>
<test-suite type="SetUpFixture" name="[default namespace]" executed="True" result="Failure" success="False" time="3393.255" asserts="7560">
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
<stack-trace />
</failure>
<results>
<test-suite type="TestSuite" name="Xamarin" executed="True" result="Failure" success="False" time="3393.250" asserts="7560">
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
<stack-trace />
</failure>
<results>
<test-suite type="TestFixture" name="MTouch" executed="True" result="Failure" success="False" time="2720.409" asserts="576">
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
<stack-trace />
</failure>
<results>
<test-suite type="ParameterizedTest" name="FastDev_LinkAll" executed="True" result="Failure" success="False" time="16.633" asserts="2">
<failure>
<message><![CDATA[One or more child tests had errors]]></message>
<stack-trace />
</failure>
<results>
<test-case name="Xamarin.MTouch.FastDev_LinkAll(iOS)" executed="True" result="Failure" success="False" time="8.162" asserts="1">
<failure>
<message>message</message>
<stack-trace>stacktrace</stack-trace>
</failure>
</test-case>
</results>
</test-suite>
<test-case name="Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode" executed="True" result="Failure" success="False" time="0.443" asserts="0">
<failure>
<message>message</message>
<stack-trace>stacktrace</stack-trace>
</failure>
</test-case>
</results>
</test-suite>
</results>
</test-suite>
</results>
</test-suite>
</results>
</test-suite>
</test-results>

26 changes: 24 additions & 2 deletions tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,9 +361,31 @@ public void NUnitV2GenerateTestReport ()
message</div>
</li>
<li>
ErrorTest2: SingleLineErrorMessage</li>
<li>
NUnit.Tests.Assemblies.MockTestFixture.FailingTest: Intentional failure</li>
<li>
NUnit.Tests.Assemblies.MockTestFixture.TestWithException: System.ApplicationException : Intentional Exception</li>
</ul>
</div>
";
Assert.AreEqual (expectedOutput, writer.ToString (), "Output");
}


[Test]
public void NUnitV2GenerateTestReportWithTestCaseFailures ()
{
using var writer = new StringWriter ();
using var stream = GetType ().Assembly.GetManifestResourceStream ("Microsoft.DotNet.XHarness.iOS.Shared.Tests.Samples.TestCaseFailures.xml");
using var reader = new StreamReader (stream);
resultParser.GenerateTestReport (writer, reader, XmlResultJargon.NUnitV2);
Console.WriteLine (writer.ToString ());
var expectedOutput =
@"<div style='padding-left: 15px;'>
<ul>
<li>
Xamarin.MTouch.FastDev_LinkAll(iOS): message</li>
<li>
Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode: message</li>
</ul>
</div>
";
Expand Down
27 changes: 18 additions & 9 deletions tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,14 @@ static void GenerateNUnitV2TestReport (TextWriter writer, XmlReader reader)
void write_failure ()
{
var name = reader ["name"];
var message = reader.ReadToDescendant ("message") ? reader.ReadElementContentAsString () : null;
string? message = null;
var depth = reader.Depth;
if (reader.ReadToDescendant ("message")) {
message = reader.ReadElementContentAsString ();
// ReadToParent
while (depth > reader.Depth && reader.Read ()) {
}
}
var message_block = message?.IndexOf ('\n') >= 0;
writer.WriteLine ("<li>");
writer.Write (name.AsHtml ());
Expand All @@ -544,16 +551,18 @@ void write_failure ()
if (result == "Error")
write_failure ();

if (!reader.ReadToDescendant ("test-case"))
continue;
var depth = reader.Depth;

do {
result = reader ["result"];
if (result != "Error" && result != "Failure")
continue;
while (reader.Read ()) {
if (reader.NodeType != XmlNodeType.Element || reader.Name != "test-case")
continue;

write_failure ();
} while (reader.ReadToNextSibling ("test-case"));
result = reader ["result"];
if (result == "Error" || result == "Failure")
write_failure ();
if (reader.Depth < depth)
break;
}
}

writer.WriteLine ("</ul>");
Expand Down