Add support for junit results

src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs

@@ -88,6 +88,16 @@ protected Task RetryAsync(Func<Task> function)
 
         }
 
+        protected Task<T> RetryAsync<T>(Func<Task<T>> function)
+        {
+            // Grab the retry logic from the helix api client
+            return ApiFactory.GetAnonymous()
+                .RetryAsync(
+                    async () => await function(),
+                    ex => Log.LogMessage(MessageImportance.Low, $"Azure Dev Ops Operation failed: {ex}\nRetrying..."));
+
+        }
+
         protected async Task LogFailedRequest(HttpRequestMessage req, HttpResponseMessage res)
         {
             Log.LogError($"Request to {req.RequestUri} returned failed status {(int)res.StatusCode} {res.ReasonPhrase}\n\n{(res.Content != null ? await res.Content.ReadAsStringAsync() : "")}");

src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestRun.cs

@@ -11,34 +11,79 @@ public class CheckAzurePipelinesTestRun : AzureDevOpsTask
         [Required]
         public int TestRunId { get; set; }
 
+        public ITaskItem[] ExpectedTestFailures { get; set; }
+
         protected override async Task ExecuteCoreAsync(HttpClient client)
         {
-            await RetryAsync(
+            if (ExpectedTestFailures?.Length > 0)
+            {
+                await ValidateExpectedTestFailuresAsync(client);
+                return;
+            }
+            var data = await RetryAsync(
+                async () =>
+                {
+                    using (var req = new HttpRequestMessage(
+                        HttpMethod.Get,
+                        $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}?api-version=5.0"))
+                    {
+                        using (var res = await client.SendAsync(req))
+                        {
+                            return await ParseResponseAsync(req, res);
+                        }
+                    }
+                });
+            if (data != null && data["runStatistics"] is JArray runStatistics)
+            {
+                var failed = runStatistics.Children()
+                    .FirstOrDefault(stat => stat["outcome"]?.ToString() == "Failed");
+                if (failed != null)
+                {
+                    Log.LogError($"Test run {TestRunId} has one or more failing tests.");
+                }
+                else
+                {
+                    Log.LogMessage(MessageImportance.Low, $"Test run {TestRunId} has not failed.");
+                }
+            }
+        }
+
+        private async Task ValidateExpectedTestFailuresAsync(HttpClient client)
+        {
+            var data = await RetryAsync(
                 async () =>
                 {
                     using (var req = new HttpRequestMessage(
                         HttpMethod.Get,
-                        $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}?api-version=5.0-preview.2"))
+                        $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}/results?api-version=5.0&outcomes=Failed"))
                     {
                         using (var res = await client.SendAsync(req))
                         {
-                            var data = await ParseResponseAsync(req, res);
-                            if (data != null && data["runStatistics"] is JArray runStatistics)
-                            {
-                                var failed = runStatistics.Children()
-                                    .FirstOrDefault(stat => stat["outcome"]?.ToString() == "Failed");
-                                if (failed != null)
-                                {
-                                    Log.LogError($"Test run {TestRunId} has one or more failing tests.");
-                                }
-                                else
-                                {
-                                    Log.LogMessage(MessageImportance.Low, $"Test run {TestRunId} has not failed.");
-                                }
-                            }
+                            return await ParseResponseAsync(req, res);
                         }
                     }
                 });
+
+            var failedResults = (JArray) data["value"];
+            var expectedFailures = ExpectedTestFailures.Select(i => i.GetMetadata("Identity")).ToHashSet();
+            foreach (var failedResult in failedResults)
+            {
+                var testName = (string) failedResult["automatedTestName"];
+                if (expectedFailures.Contains(testName))
+                {
+                    expectedFailures.Remove(testName);
+                    Log.LogMessage($"TestRun {TestRunId}: Test {testName} has failed and was expected to fail.");
+                }
+                else
+                {
+                    Log.LogError($"TestRun {TestRunId}: Test {testName} has failed and is not expected to fail.");
+                }
+            }
+
+            foreach (var expectedFailure in expectedFailures)
+            {
+                Log.LogError($"TestRun {TestRunId}: Test {expectedFailure} was expected to fail but did not fail.");
+            }
         }
     }
 }

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets

@@ -36,6 +36,6 @@
           BeforeTargets="AfterTest"
           Condition="$(FailOnTestFailure)"
           Outputs="%(HelixTargetQueue.Identity)">
-    <CheckAzurePipelinesTestRun TestRunId="%(HelixTargetQueue.TestRunId)"/>
+    <CheckAzurePipelinesTestRun TestRunId="%(HelixTargetQueue.TestRunId)" ExpectedTestFailures="@(AzurePipelinesExpectedTestFailure)"/>
   </Target>
 </Project>

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/__init__.py

@@ -1,8 +1,10 @@
 from typing import List
 from result_format import ResultFormat
 from xunit import XUnitFormat
+from junit import JUnitFormat
 
 
 all_formats = [
-    XUnitFormat()
+    XUnitFormat(),
+    JUnitFormat()
 ]  # type: List[ResultFormat]

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py

@@ -0,0 +1,73 @@
+import xml.etree.ElementTree
+from result_format import ResultFormat
+from defs import TestResult, TestResultAttachment
+
+
+class JUnitFormat(ResultFormat):
+
+    def __init__(self):
+        super(JUnitFormat, self).__init__()
+        pass
+
+    @property
+    def name(self):
+        return 'junit'
+
+    @property
+    def acceptable_file_names(self):
+        yield 'junit-results.xml'
+        yield 'junitresults.xml'
+
+    def read_results(self, path):
+        for (_, element) in xml.etree.ElementTree.iterparse(path, events=['end']):
+            if element.tag == 'testcase':
+                test_name = element.get("name")
+                classname = element.get("classname")
+                name = classname + "." + test_name
+                type_name = classname
+                method = test_name
+                duration = float(element.get("time"))
+                result = "Pass"
+                exception_type = None
+                failure_message = None
+                stack_trace = None
+                skip_reason = None
+                attachments = []
+
+
+                failure_element = element.find("failure")
+                if failure_element is None:
+                    failure_element = element.find("error")
+
+                if failure_element is not None:
+                    result = "Fail"
+                    exception_type = failure_element.get("type")
+                    failure_message = failure_element.get("message")
+                    stack_trace = failure_element.text
+
+                    stdout_element = element.find("system-out")
+                    if stdout_element is not None:
+                        attachments.append(TestResultAttachment(
+                            name=u"Console_Output.log",
+                            text=stdout_element.text,
+                        ))
+
+                    stderr_element = element.find("system-err")
+                    if stderr_element is not None:
+                        attachments.append(TestResultAttachment(
+                            name=u"Error_Output.log",
+                            text=stderr_element.text,
+                        ))
+
+                skipped_element = element.find("skipped")
+                if skipped_element is not None:
+                    result = "Skip"
+                    skip_reason = skipped_element.text or u""
+
+
+                res = TestResult(name, u'junit', type_name, method, duration, result, exception_type, failure_message, stack_trace,
+                                 skip_reason, attachments)
+                yield res
+                # remove the element's content so we don't keep it around too long.
+                element.clear()
+

src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py

@@ -6,6 +6,9 @@
 class ResultFormat:
     __metaclass__ = ABCMeta
 
+    def __init__(self):
+        pass
+
     @abstractproperty
     def name(self):
         pass

tests/Junit/junit-results.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+   <testsuite name="Pizza" errors="0" skipped="1" tests="3" failures="1" time="0.006" timestamp="2013-05-24T10:23:58">
+      <testcase classname="PizzaTests" name="cheese" time="0.006">
+         <failure message="test failure">Assertion failed</failure>
+      </testcase>
+      <testcase classname="PizzaTests" name="cheese2" time="0.006">
+         <error message="test failure">Assertion failed</error>
+      </testcase>
+      <testcase classname="PizzaTests" name="pepperoni" time="0">
+         <skipped />
+      </testcase>
+      <testcase classname="PizzaTests" name="anchovy" time="0" />
+   </testsuite>
+</testsuites>

tests/UnitTests.proj

@@ -19,6 +19,17 @@
     <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
   </PropertyGroup>
 
+  <ItemGroup>
+    <HelixWorkItem Include="JUnitTest">
+      <Command>echo 'done!'</Command>
+      <PayloadDirectory>$(MSBuildThisFileDirectory)Junit</PayloadDirectory>
+    </HelixWorkItem>
+  </ItemGroup>
+  <ItemGroup>
+    <AzurePipelinesExpectedTestFailure Include="PizzaTests.cheese"/>
+    <AzurePipelinesExpectedTestFailure Include="PizzaTests.cheese2"/>
+  </ItemGroup>
+
   <ItemGroup>
     <XUnitProject Include="..\src\**\*.Tests.csproj"/>