[tests] PR validation speed ups (#8227)

Improves `Tests` workflow time from ~14 mins to ~10 mins.
### Build packages once on Linux, and use with tests

This reduces the run time by avoiding package builds on Windows, and reduces total compute consumption.

### Merge github action enumerate-template-tests into enumerate-tests

These two tasks were called sequentially, and each would
checkout-code+install-dotnet before doing any work. By combining the
tasks, we get an improved run time.

### Always use system dotnet for EndToEnd tests, thus avoiding the need for test sdks to be installed

### Avoid building test projects when TestNoBuild!=true

Saves on build unnecessary build times.

* address review feedback from @ joperezr

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: radical

.github/actions/enumerate-template-tests/action.yml

@@ -1,9 +1,22 @@
 name: 'Enumerate test projects'
 description: 'Enumerate list of test projects'
+inputs:
+  includeIntegrations:
+    required: false
+    type: boolean
+    default: false
+  includeTemplates:
+    required: false
+    type: boolean
+    default: false
+
 outputs:
   integrations_tests_matrix:
-    description: tests matrix
-    value: ${{ steps.generate_test_matrix.outputs.integrations_tests_matrix }}
+    description: Integration tests matrix
+    value: ${{ steps.generate_integrations_matrix.outputs.integrations_tests_matrix }}
+  templates_tests_matrix:
+    description: Templates tests matrix
+    value: ${{ steps.generate_templates_matrix.outputs.templates_tests_matrix }}
 runs:
   using: "composite"
   steps:
@@ -16,7 +29,8 @@ runs:
         dotnet-version: |
           9.x
 
-    - name: Get list of tests
+    - name: Get list of integration tests
+      if: ${{ inputs.includeIntegrations }}
       env:
         CI: false
       shell: pwsh
@@ -25,8 +39,23 @@ runs:
         /p:TestsListOutputPath=${{ github.workspace }}/artifacts/TestsForGithubActions.list
         /p:ContinuousIntegrationBuild=true
 
+    - name: Generate list of template tests
+      if: ${{ inputs.includeTemplates }}
+      env:
+        CI: false
+      shell: pwsh
+      run: >
+          dotnet build ${{ github.workspace }}/tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj
+          "/t:Build;ExtractTestClassNames"
+          /bl:${{ github.workspace }}/artifacts/log/Debug/BuildTemplatesTests.binlog
+          -p:ExtractTestClassNamesForHelix=true
+          -p:ArchiveTests=true
+          -p:ExtractTestClassNamesPrefix=Aspire.Templates.Tests
+          -p:InstallBrowsersForPlaywright=false
+
     - name: Generate tests matrix
-      id: generate_test_matrix
+      id: generate_integrations_matrix
+      if: ${{ inputs.includeIntegrations }}
       shell: pwsh
       env:
         CI: false
@@ -39,3 +68,25 @@ runs:
         $jsonString = ConvertTo-Json $jsonObject -Compress
         "integrations_tests_matrix=$jsonString"
         "integrations_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT
+
+    - name: Generate tests matrix
+      id: generate_templates_matrix
+      if: ${{ inputs.includeTemplates }}
+      shell: pwsh
+      env:
+        CI: false
+      run: |
+        $inputFilePath = "${{ github.workspace }}/artifacts/helix/templates-tests/Aspire.Templates.Tests.tests.list"
+        $lines = Get-Content $inputFilePath
+
+        $prefix = "Aspire.Templates.Tests."
+        $lines = Get-Content $inputFilePath | ForEach-Object {
+            $_ -replace "^$prefix", ""
+        }
+
+        $jsonObject = @{
+            "shortname" = $lines | Sort-Object
+        }
+        $jsonString = ConvertTo-Json $jsonObject -Compress
+        "templates_tests_matrix=$jsonString"
+        "templates_tests_matrix=$jsonString" | Out-File -FilePath $env:GITHUB_OUTPUT

.github/actions/enumerate-tests/action.yml

@@ -24,10 +24,18 @@ on:
       extraTestArgs:
         required: false
         type: string
+      # Triggers downloading the built nugets, and running the tests
+      # from the archive outside the repo
       requiresNugets:
         required: false
         type: boolean
         default: false
+      # Installs the sdks used for test via tests/workloads.proj
+      # Useful for tests that can't use system dotnet
+      requiresTestSdk:
+        required: false
+        type: boolean
+        default: false
       os:
         required: false
         type: string
@@ -83,14 +91,21 @@ jobs:
         if: inputs.os == 'ubuntu-latest'
         run: docker info
 
-      - name: Build with packages
+      - name: Download built nugets
         if: ${{ inputs.requiresNugets }}
-        env:
-          CI: false
-        run: ${{ env.BUILD_SCRIPT }} -restore -build -ci -pack -testnobuild /bl
+        uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+        with:
+          pattern: built-nugets
+          path: ${{ github.workspace }}/artifacts/packages
 
-      - name: Install sdk for nuget based testing
+      - name: Copy nugets to the correct location
         if: ${{ inputs.requiresNugets }}
+        shell: pwsh
+        run:
+          Move-Item -Path "${{ github.workspace }}/artifacts/packages/built-nugets/Debug" -Destination "${{ github.workspace }}/artifacts/packages"
+
+      - name: Install sdk for nuget based testing
+        if: ${{ inputs.requiresTestSdk }}
         env:
           CI: false
         run: >

.github/workflows/run-tests.yml

@@ -19,31 +19,49 @@ jobs:
     name: Setup for tests (Linux)
     runs-on: ubuntu-latest
     outputs:
-      integrations_tests_matrix: ${{ steps.generate_integrations_test_matrix.outputs.integrations_tests_matrix }}
-      templates_tests_matrix: ${{ steps.generate_templates_test_matrix.outputs.templates_tests_matrix }}
+      integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
+      templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: ./.github/actions/enumerate-template-tests
-        id: generate_templates_test_matrix
-
       - uses: ./.github/actions/enumerate-tests
-        id: generate_integrations_test_matrix
+        id: generate_tests_matrix
+        with:
+          includeIntegrations: true
+          includeTemplates: true
 
   setup_for_tests_win:
     name: Setup for tests (Windows)
     runs-on: windows-latest
     outputs:
-      integrations_tests_matrix: ${{ steps.generate_integrations_test_matrix.outputs.integrations_tests_matrix }}
-      templates_tests_matrix: ${{ steps.generate_templates_test_matrix.outputs.templates_tests_matrix }}
+      integrations_tests_matrix: ${{ steps.generate_tests_matrix.outputs.integrations_tests_matrix }}
+      templates_tests_matrix: ${{ steps.generate_tests_matrix.outputs.templates_tests_matrix }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: ./.github/actions/enumerate-template-tests
-        id: generate_templates_test_matrix
-
       - uses: ./.github/actions/enumerate-tests
-        id: generate_integrations_test_matrix
+        id: generate_tests_matrix
+        with:
+          includeIntegrations: true
+          includeTemplates: true
+
+  build_packages:
+    name: Build packages
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Build with packages
+        env:
+          CI: false
+        run: ./build.sh -restore -build -ci -pack -testnobuild /bl /p:InstallBrowsersForPlaywright=false
+
+      - name: Upload built NuGets
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: built-nugets
+          path: artifacts/packages
 
   integrations_test_lin:
     uses: ./.github/workflows/run-tests.yml
@@ -56,7 +74,6 @@ jobs:
     with:
       testShortName: ${{ matrix.shortname }}
       os: "ubuntu-latest"
-      requiresNugets: ${{ matrix.shortname == 'EndToEnd' }}
 
   integrations_test_win:
     uses: ./.github/workflows/run-tests.yml
@@ -69,12 +86,11 @@ jobs:
     with:
       testShortName: ${{ matrix.shortname }}
       os: "windows-latest"
-      requiresNugets: ${{ matrix.shortname == 'EndToEnd' }}
 
   templates_test_lin:
     name: Templates Linux
     uses: ./.github/workflows/run-tests.yml
-    needs: setup_for_tests_lin
+    needs: [setup_for_tests_lin, build_packages]
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.setup_for_tests_lin.outputs.templates_tests_matrix) }}
@@ -87,11 +103,12 @@ jobs:
       # append '.' to the name so only the test class with exactly that name is run
       extraTestArgs: "--filter FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}."
       requiresNugets: true
+      requiresTestSdk: true
 
   templates_test_win:
     name: Templates Windows
     uses: ./.github/workflows/run-tests.yml
-    needs: setup_for_tests_win
+    needs: [setup_for_tests_win, build_packages]
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.setup_for_tests_win.outputs.templates_tests_matrix) }}
@@ -104,12 +121,24 @@ jobs:
       # append '.' to the name so only the test class with exactly that name is run
       extraTestArgs: "--filter FullyQualifiedName~Aspire.Templates.Tests.${{ matrix.shortname }}."
       requiresNugets: true
+      requiresTestSdk: true
+
+  endtoend_tests:
+    name: EndToEnd ${{ matrix.os }}
+    uses: ./.github/workflows/run-tests.yml
+    needs: build_packages
+    with:
+      testShortName: EndToEnd
+      # EndToEnd is not run on Windows due to missing Docker support
+      os: ubuntu-latest
+      testProjectPath: tests/Aspire.EndToEnd.Tests/Aspire.EndToEnd.Tests.csproj
+      requiresNugets: true
 
   results: # This job is used for branch protection. It ensures all the above tests passed
     if: ${{ always() }}
     runs-on: ubuntu-latest
     name: Final Results
-    needs: [ integrations_test_lin, integrations_test_win, templates_test_lin, templates_test_win ]
+    needs: [ integrations_test_lin, integrations_test_win, templates_test_lin, templates_test_win, endtoend_tests ]
     steps:
       # get all the test-job-result* artifacts into a single directory
       - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9

.github/workflows/tests.yml

@@ -4,7 +4,7 @@
     <ProjectToBuild Include="$(RepoRoot)eng\dcppack\**\*.csproj" />
     <ProjectToBuild Include="$(RepoRoot)eng\dashboardpack\**\*.csproj" />
     <ProjectToBuild Include="$(RepoRoot)playground\**\*.csproj" />
-    <ProjectToBuild Include="$(RepoRoot)tests\**\*.csproj"  />
+    <ProjectToBuild Include="$(RepoRoot)tests\**\*.csproj" Condition="'$(TestNoBuild)' != 'true'" />
   </ItemGroup>
 
   <!-- When building from source, we want to use the live repo contents as opposed to cloning the repo. -->

eng/Build.props

@@ -38,7 +38,7 @@ public IntegrationServicesFixture(IMessageSink diagnosticMessageSink)
     {
         _diagnosticMessageSink = diagnosticMessageSink;
         _testOutput = new TestOutputWrapper(messageSink: _diagnosticMessageSink);
-        BuildEnvironment = new(useSystemDotNet: !TestsRunningOutsideOfRepo);
+        BuildEnvironment = new(useSystemDotNet: true);
         if (TestsRunningOutsideOfRepo)
         {
             BuildEnvironment.EnvVars["TestsRunningOutsideOfRepo"] = "true";

tests/Aspire.EndToEnd.Tests/IntegrationServicesFixture.cs

@@ -137,6 +137,7 @@ await app.ResourceNotifications.PublishUpdateAsync(resource.Resource, s => s wit
     }
 
     [Fact]
+    [ActiveIssue("https://github.com/dotnet/aspire/issues/8103")]
     public async Task ResourcesWithHealthCheck_StopsAndRestartsMonitoringWithResource()
     {
         using var builder = TestDistributedApplicationBuilder.Create(testOutputHelper);

tests/Aspire.Hosting.Tests/Health/ResourceHealthCheckServiceTests.cs

@@ -19,6 +19,8 @@
       <_TestProjectsToExclude Include="$(RepoRoot)tests\testproject\**\*Tests.csproj" />
       <_TestProjectsToExclude Include="$(RepoRoot)tests\TestingAppHost1\**\*Tests.csproj" />
 
+      <!-- This runs in a separate job -->
+      <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.EndToEnd.Tests\**\*Tests.csproj" />
       <!-- This runs in a separate job -->
       <_TestProjectsToExclude Include="$(RepoRoot)tests\Aspire.Templates.Tests\**\*Tests.csproj" />