[ci] Automatically retry failed emulator tests.

Author: jpobst

build-tools/automation/yaml-templates/install-dotnet-test-slicer.yaml

@@ -1,5 +1,5 @@
 parameters:
-  version: '0.1.0-alpha2'
+  version: '0.1.0-alpha5'
   condition: succeeded()
   continueOnError: true
 

build-tools/automation/yaml-templates/run-msbuild-tests.yaml

@@ -48,7 +48,7 @@ jobs:
       testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
       testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
       testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
-      testResultsTitle: TestResult-MSBuildTests-${{ parameters.jobName }}
+      retryFailedTests: false
 
   - template: upload-results.yaml
     parameters:

build-tools/automation/yaml-templates/run-sliced-nunit-tests.yaml

@@ -2,22 +2,55 @@ parameters:
   testAssembly:           # NUnit test assembly to run
   testFilter:             # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
   testRunTitle:           # Title of the test run
-  testResultsTitle:       # Title used to construct test results file name
+  retryFailedTests: true  # Retry failed tests once
 
 steps:
 - pwsh: |
     dotnet-test-slicer `
+      slice `
       --test-assembly="${{ parameters.testAssembly }}" `
       --test-filter="${{ parameters.testFilter }}" `
       --slice-number=$(System.JobPositionInPhase) `
       --total-slices=$(System.TotalJobsInPhase) `
       --outfile="${{ parameters.testAssembly }}.runsettings"
   displayName: Slice unit tests
 
-- template: run-nunit-tests.yaml
-  parameters:
-    useDotNet: true
-    testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
-    testAssembly: ${{ parameters.testAssembly }}
-    dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
-    testResultsFile: ${{ parameters.testResultsTitle }}-$(System.JobPositionInPhase)-$(XA.Build.Configuration).xml
+- ${{ if eq(parameters.retryFailedTests, 'false') }}:
+  # If we aren't using auto-retry logic, then this is just a simple template call
+  - template: run-nunit-tests.yaml
+    parameters:
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
+      testAssembly: ${{ parameters.testAssembly }}
+      dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"
+
+- ${{ if eq(parameters.retryFailedTests, 'true') }}:
+  # We need a custom dotnet test invocation here that does not trigger a task failure on failed tests
+  - pwsh: |
+      dotnet `
+        test `
+        ${{ parameters.testAssembly }} `
+        --settings "${{ parameters.testAssembly }}.runsettings" `
+        --logger trx --results-directory $(Agent.TempDirectory) `
+        -- NUnit.NumberOfTestWorkers=$(NUnit.NumberOfTestWorkers)
+    displayName: Run tests
+    ignoreLASTEXITCODE: true
+      
+  - pwsh: |
+      dotnet-test-slicer `
+        retry `
+        --trx="$(Agent.TempDirectory)" `
+        --outfile="${{ parameters.testAssembly }}.runsettings"
+    displayName: Look for failed tests
+
+    # dotnet-test-slicer removed the failed tests from our results file, so it's safe to publish it now
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: VSTest
+      testResultsFiles: $(Agent.TempDirectory)/*.trx
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase)
+
+  - template: run-nunit-tests.yaml
+    parameters:
+      testRunTitle: ${{ parameters.testRunTitle }}-$(System.JobPositionInPhase) (Auto-Retry)
+      testAssembly: ${{ parameters.testAssembly }}
+      dotNetTestExtraArgs: --settings "${{ parameters.testAssembly }}.runsettings"

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -57,7 +57,6 @@ stages:
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
         testFilter: cat != TimeZoneInfo & cat != Localization $(ExcludedNUnitCategories)
         testRunTitle: MSBuildDeviceIntegration On Device - macOS
-        testResultsTitle: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}
 
     - template: start-stop-emulator.yaml
       parameters:

build-tools/automation/yaml-templates/start-stop-emulator.yaml

@@ -5,7 +5,7 @@ parameters:
   avdApiLevel:            # Device API level, like '30', required if 'specificImage' is 'true'
   avdAbi:                 # Device ABI, like 'x86', required if 'specificImage' is 'true'
   avdType:                # Device AVD, like 'android-wear', required if 'specificImage' is 'true'
-  launchTimeoutMin: 15    # Minutes to wait for the emulator to start
+  launchTimeoutMin: 20    # Minutes to wait for the emulator to start
   xaSourcePath: $(System.DefaultWorkingDirectory) # working directory
 
 steps: