Adds repeat test attribute for flaky tests

Also attempt to fix test publishing #196
QutEcoacoustics · Apr 2, 2020 · b1cf305 · b1cf305
1 parent db5118a
commit b1cf305
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 7 deletions.
diff --git a/build/azure-pipelines-build.yml b/build/azure-pipelines-build.yml
@@ -109,7 +109,7 @@ jobs:
               nugetConfigPath: NuGet.config
             displayName: Restore solution dependencies
 
-          - pwsh: src/git_version.ps1 -build_type ${{ configuration }} -set_ci | Tee-Object '$(Build.ArtifactStagingDirectory)/AP_version_vars.txt'
+          - pwsh: src/git_version.ps1 -configuration ${{ configuration }} -self_contained $(selfContainedArgument) -runtime_identifier $(runtimeArgument) -set_ci | Tee-Object '$(Build.ArtifactStagingDirectory)/AP_version_vars.txt'
             displayName: run git_version.ps1 script
             name: git_version
 
@@ -142,7 +142,8 @@ jobs:
           - task: PublishTestResults@2
             inputs:
               testResultFormat: VSTest
-              testResultFiles: '$(Agent.TempDirectory)/Acoustics.Test_Results/**/*.trx'
+              testResultFiles: '**/*.trx'
+              searchFolder: $(Agent.TempDirectory)/Acoustics.Test_Results
               testRunTitle: "Acoustics.Test for ${{ platform.rid }} ${{ configuration }}"
               buildConfig: ${{ configuration }}
               buildPlatform: $(platformTag)
@@ -169,7 +170,8 @@ jobs:
           - task: PublishTestResults@2
             inputs:
               testResultFormat: VSTest
-              testResultFiles: '$(Agent.TempDirectory)/AED.Test_Results/**/*.trx'
+              testResultFiles: '**/*.trx'
+              searchFolder: $(Agent.TempDirectory)/AED.Test_Results
               testRunTitle: ".Test for ${{ platform.rid }} ${{ configuration }}"
               buildConfig: ${{ configuration }}
               buildPlatform: $(platformTag)

diff --git a/tests/Acoustics.Test/.runsettings b/tests/Acoustics.Test/.runsettings
@@ -3,6 +3,15 @@
   <RunConfiguration>
     <ResultsDirectory>../../tests/Results</ResultsDirectory>
   </RunConfiguration>
+  <LoggerRunSettings>
+    <Loggers>
+      <Logger friendlyName="console">
+        <Configuration>
+          <verbosity>detailed</verbosity>
+        </Configuration>
+      </Logger>
+    </Loggers>
+  </LoggerRunSettings>
   <DataCollectionRunSettings>
     <DataCollectors>
       <DataCollector friendlyName="XPlat code coverage">

diff --git a/tests/Acoustics.Test/Shared/ProcessRunnerTests.cs b/tests/Acoustics.Test/Shared/ProcessRunnerTests.cs
@@ -5,6 +5,7 @@
 namespace Acoustics.Test.Shared
 {
     using System;
+    using System.Diagnostics;
     using System.Linq;
     using System.Runtime.InteropServices;
     using Acoustics.Shared;
@@ -16,29 +17,29 @@ public class ProcessRunnerTests : OutputDirectoryTest
     {
         public const string TestFile = "very_large_file_20170522-180007Z.flac";
 
-        [TestMethod]
+        [RetryTestMethod(2)]
         public void ProcessRunnerDoesNotDeadlock()
         {
             var result = Enumerable.Range(0, 100).AsParallel().Select(this.RunFfprobe).ToArray();
 
             Assert.IsTrue(result.All());
         }
 
-        [TestMethod]
+        [RetryTestMethod(2)]
         public void ProcessRunnerSimple()
         {
             this.RunFfprobe(0);
         }
 
-        [TestMethod]
+        [RetryTestMethod(2)]
         public void ProcessRunnerTimeOutDoesNotDeadlock()
         {
             var result = Enumerable.Range(0, 100).AsParallel().Select(this.RunFfprobeIndefinite).ToArray();
 
             Assert.IsTrue(result.All());
         }
 
-        [TestMethod]
+        [RetryTestMethod(2)]
         public void ProcessRunnerTimeOutSimple()
         {
             this.RunFfprobeIndefinite(0);

diff --git a/tests/Acoustics.Test/TestHelpers/RetryTestMethodAttribute.cs b/tests/Acoustics.Test/TestHelpers/RetryTestMethodAttribute.cs
@@ -0,0 +1,61 @@
+// <copyright file="RetryTestMethodAttribute.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
+
+namespace Acoustics.Test.TestHelpers
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Diagnostics;
+    using System.Linq;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class RetryTestMethodAttribute : TestMethodAttribute
+    {
+        public RetryTestMethodAttribute(int retryCount)
+        {
+            if (retryCount < 0)
+            {
+                throw new ArgumentException($"{nameof(retryCount)} cannot be less than 0", nameof(retryCount));
+            }
+
+            this.Retrycount = retryCount;
+        }
+
+        public int Retrycount { get; }
+
+        public override TestResult[] Execute(ITestMethod testMethod)
+        {
+            // always run at least once, and then rety n times if fails
+            int retryCount = this.Retrycount + 1;
+
+            var result = new List<TestResult>();
+
+            var success = false;
+            for (int count = 0; count < retryCount; count++)
+            {
+                var testResults = base.Execute(testMethod);
+                result.AddRange(testResults);
+
+                var failed = testResults.FirstOrDefault((tr) => tr.Outcome == UnitTestOutcome.Failed);
+                if (failed != null)
+                {
+                    failed.TestContextMessages += $"Test iteration  {count + 1} of {retryCount} failed. Attempting to run again.";
+                    failed.Outcome = UnitTestOutcome.Inconclusive;
+                    continue;
+                }
+
+                success = true;
+                break;
+            }
+
+            if (!success)
+            {
+                result[^1].Outcome = UnitTestOutcome.Failed;
+            }
+
+            return result.ToArray();
+        }
+    }
+}