Outerloop enhancements (#8706)

Addendum to #8618

Author: RussKie

.github/workflows/tests-outerloop.yml

@@ -38,7 +38,7 @@ jobs:
         with:
           name: logs-runsheet
           path: |
-            ${{ github.workspace }}/artifacts/log/*/*.binlog
+            ${{ github.workspace }}/artifacts/log/*/runsheet.binlog
             ${{ github.workspace }}/artifacts/log/*/TestLogs/**
             ${{ github.workspace }}/artifacts/tmp/*/combined_runsheet.json
           retention-days: 5
@@ -100,6 +100,11 @@ jobs:
               }
           }
 
+          if ($testResults.Length -lt 1) {
+              Write-Host "::notice::Tests Summary: no quaratined tests found"
+              return;
+          }
+
           # Sort the test results by test name
           $testResults = $testResults | Sort-Object -Property TestName
 
@@ -130,7 +135,7 @@ jobs:
           Write-Host "Test results saved to $outputPath"
 
       - name: Upload logs, and test results
-        if: failure()
+        if: always()
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: logs-${{ matrix.tests.os }}-${{ matrix.tests.project }}

eng/QuarantinedTestRunsheetBuilder/QuarantinedTestRunsheetBuilder.targets

@@ -133,6 +133,12 @@
       <_TestRunnerLinux>./eng/build.sh</_TestRunnerLinux>
       <_TestCommand>-restore -build -test -projects &quot;$(_RelativeTestProjectPath)&quot; /bl:&quot;$(_RelativeTestBinLog)&quot; -c $(Configuration) -ci /p:RunQuarantinedTests=true /p:CI=false</_TestCommand>
 
+      <!--
+        Some quarantinted test may only be executable on Windows or Linux, however we can't possibly know that at this time.
+        The MTP runner will return exit code 8 if no tests are found, and we need to ignore it instead of failing the test.
+        -->
+      <_TestCommand>$(_TestCommand) /p:IgnoreZeroTestResult=true</_TestCommand>
+
       <!-- Replace \ with /, and then escape " with \", so we have a compliant JSON -->
       <_TestCommand>$([System.String]::Copy($(_TestCommand)).Replace("\", "/").Replace('&quot;', '\&quot;'))</_TestCommand>
 

eng/Testing.props

@@ -19,6 +19,7 @@
     <_BlameArgs>--hangdump --hangdump-timeout $(BlameHangTimeout) --crashdump</_BlameArgs>
 
     <TestRunnerAdditionalArguments>$(TestRunnerAdditionalArguments) --filter-not-trait &quot;category=failing&quot;</TestRunnerAdditionalArguments>
+    <TestRunnerAdditionalArguments Condition=" '$(IgnoreZeroTestResult)' == 'true' ">$(TestRunnerAdditionalArguments) --ignore-exit-code 8</TestRunnerAdditionalArguments>
 
     <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' != 'true'">$(TestRunnerAdditionalArguments) $(_NonQuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>
     <TestRunnerAdditionalArguments Condition="'$(RunQuarantinedTests)' == 'true'">$(TestRunnerAdditionalArguments) $(_QuarantinedTestRunAdditionalArgs) $(_BlameArgs)</TestRunnerAdditionalArguments>