diff --git a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs index 1eafd462470..eb27e65a7b9 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/AzureDevOpsTask.cs @@ -88,6 +88,16 @@ protected Task RetryAsync(Func function) } + protected Task RetryAsync(Func> function) + { + // Grab the retry logic from the helix api client + return ApiFactory.GetAnonymous() + .RetryAsync( + async () => await function(), + ex => Log.LogMessage(MessageImportance.Low, $"Azure Dev Ops Operation failed: {ex}\nRetrying...")); + + } + protected async Task LogFailedRequest(HttpRequestMessage req, HttpResponseMessage res) { Log.LogError($"Request to {req.RequestUri} returned failed status {(int)res.StatusCode} {res.ReasonPhrase}\n\n{(res.Content != null ? await res.Content.ReadAsStringAsync() : "")}"); diff --git a/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestRun.cs b/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestRun.cs index dd901571fd8..a791115a5cc 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestRun.cs +++ b/src/Microsoft.DotNet.Helix/Sdk/CheckAzurePipelinesTestRun.cs @@ -11,34 +11,79 @@ public class CheckAzurePipelinesTestRun : AzureDevOpsTask [Required] public int TestRunId { get; set; } + public ITaskItem[] ExpectedTestFailures { get; set; } + protected override async Task ExecuteCoreAsync(HttpClient client) { - await RetryAsync( + if (ExpectedTestFailures?.Length > 0) + { + await ValidateExpectedTestFailuresAsync(client); + return; + } + var data = await RetryAsync( + async () => + { + using (var req = new HttpRequestMessage( + HttpMethod.Get, + $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}?api-version=5.0")) + { + using (var res = await client.SendAsync(req)) + { + return await ParseResponseAsync(req, res); + } + } + }); + if (data != null && data["runStatistics"] is JArray runStatistics) + { + var failed = runStatistics.Children() + .FirstOrDefault(stat => stat["outcome"]?.ToString() == "Failed"); + if (failed != null) + { + Log.LogError($"Test run {TestRunId} has one or more failing tests."); + } + else + { + Log.LogMessage(MessageImportance.Low, $"Test run {TestRunId} has not failed."); + } + } + } + + private async Task ValidateExpectedTestFailuresAsync(HttpClient client) + { + var data = await RetryAsync( async () => { using (var req = new HttpRequestMessage( HttpMethod.Get, - $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}?api-version=5.0-preview.2")) + $"{CollectionUri}{TeamProject}/_apis/test/runs/{TestRunId}/results?api-version=5.0&outcomes=Failed")) { using (var res = await client.SendAsync(req)) { - var data = await ParseResponseAsync(req, res); - if (data != null && data["runStatistics"] is JArray runStatistics) - { - var failed = runStatistics.Children() - .FirstOrDefault(stat => stat["outcome"]?.ToString() == "Failed"); - if (failed != null) - { - Log.LogError($"Test run {TestRunId} has one or more failing tests."); - } - else - { - Log.LogMessage(MessageImportance.Low, $"Test run {TestRunId} has not failed."); - } - } + return await ParseResponseAsync(req, res); } } }); + + var failedResults = (JArray) data["value"]; + var expectedFailures = ExpectedTestFailures.Select(i => i.GetMetadata("Identity")).ToHashSet(); + foreach (var failedResult in failedResults) + { + var testName = (string) failedResult["automatedTestName"]; + if (expectedFailures.Contains(testName)) + { + expectedFailures.Remove(testName); + Log.LogMessage($"TestRun {TestRunId}: Test {testName} has failed and was expected to fail."); + } + else + { + Log.LogError($"TestRun {TestRunId}: Test {testName} has failed and is not expected to fail."); + } + } + + foreach (var expectedFailure in expectedFailures) + { + Log.LogError($"TestRun {TestRunId}: Test {expectedFailure} was expected to fail but did not fail."); + } } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets index 527ffe182cf..66754fadf5f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.MultiQueue.targets @@ -36,6 +36,6 @@ BeforeTargets="AfterTest" Condition="$(FailOnTestFailure)" Outputs="%(HelixTargetQueue.Identity)"> - + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/__init__.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/__init__.py index 81b2342faa9..658bece1167 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/__init__.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/__init__.py @@ -1,8 +1,10 @@ from typing import List from result_format import ResultFormat from xunit import XUnitFormat +from junit import JUnitFormat all_formats = [ - XUnitFormat() + XUnitFormat(), + JUnitFormat() ] # type: List[ResultFormat] diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py new file mode 100644 index 00000000000..a1442c1349e --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/junit.py @@ -0,0 +1,73 @@ +import xml.etree.ElementTree +from result_format import ResultFormat +from defs import TestResult, TestResultAttachment + + +class JUnitFormat(ResultFormat): + + def __init__(self): + super(JUnitFormat, self).__init__() + pass + + @property + def name(self): + return 'junit' + + @property + def acceptable_file_names(self): + yield 'junit-results.xml' + yield 'junitresults.xml' + + def read_results(self, path): + for (_, element) in xml.etree.ElementTree.iterparse(path, events=['end']): + if element.tag == 'testcase': + test_name = element.get("name") + classname = element.get("classname") + name = classname + "." + test_name + type_name = classname + method = test_name + duration = float(element.get("time")) + result = "Pass" + exception_type = None + failure_message = None + stack_trace = None + skip_reason = None + attachments = [] + + + failure_element = element.find("failure") + if failure_element is None: + failure_element = element.find("error") + + if failure_element is not None: + result = "Fail" + exception_type = failure_element.get("type") + failure_message = failure_element.get("message") + stack_trace = failure_element.text + + stdout_element = element.find("system-out") + if stdout_element is not None: + attachments.append(TestResultAttachment( + name=u"Console_Output.log", + text=stdout_element.text, + )) + + stderr_element = element.find("system-err") + if stderr_element is not None: + attachments.append(TestResultAttachment( + name=u"Error_Output.log", + text=stderr_element.text, + )) + + skipped_element = element.find("skipped") + if skipped_element is not None: + result = "Skip" + skip_reason = skipped_element.text or u"" + + + res = TestResult(name, u'junit', type_name, method, duration, result, exception_type, failure_message, stack_trace, + skip_reason, attachments) + yield res + # remove the element's content so we don't keep it around too long. + element.clear() + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py index f0b0081c411..bf1f58ad80a 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/formats/result_format.py @@ -6,6 +6,9 @@ class ResultFormat: __metaclass__ = ABCMeta + def __init__(self): + pass + @abstractproperty def name(self): pass diff --git a/tests/Junit/junit-results.xml b/tests/Junit/junit-results.xml new file mode 100644 index 00000000000..6155073457e --- /dev/null +++ b/tests/Junit/junit-results.xml @@ -0,0 +1,15 @@ + + + + + Assertion failed + + + Assertion failed + + + + + + + diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index fee861a6897..b1d420d3bb7 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -19,6 +19,17 @@ true + + + echo 'done!' + $(MSBuildThisFileDirectory)Junit + + + + + + +