Skip to content

Commit

Permalink
[ci] Parallelize and reduce overhead of MSBuild test stage. (dotnet#7850
Browse files Browse the repository at this point in the history
)

Bring the AzDO parallelization from dotnet#7804 and environment setup improvements from dotnet#7832 to `Xamarin.Android.Build.Tests.csproj` based test suites.  This includes both the main `MSBuild` stage and the `Smoke Tests` jobs.

Increases parallelization of all jobs as many were approaching ~90 minutes.

As there is no longer a place in the `MSBuild` stage to run `Xamarin.Android.Tools.Aidl-Tests` tests, it was moved to the `macOS > Tests > APKs .NET` stage.  This suite should be fine to only run on Mac and not Windows.  (Note this test assembly was also updated to .NET 7. This required moving it from `Xamarin.Android-Tests.sln` which is currently built with Mono which cannot build .NET 7+ projects. It now is built via `Xamarin.Android.sln`.)
  • Loading branch information
jpobst authored Mar 10, 2023
1 parent 03fff8c commit 928e7b9
Show file tree
Hide file tree
Showing 60 changed files with 160 additions and 342 deletions.
14 changes: 0 additions & 14 deletions Xamarin.Android-Tests.sln
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Java.Interop-Tests", "tests
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop", "external\Java.Interop\src\Java.Interop\Java.Interop.csproj", "{94BD81F7-B06F-4295-9636-F8A3B6BDC762}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl-Tests", "tests\Xamarin.Android.Tools.Aidl-Tests\Xamarin.Android.Tools.Aidl-Tests.csproj", "{883941C8-C4ED-428D-A7D2-26F2EEA23F92}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl", "src\Xamarin.Android.Tools.Aidl\Xamarin.Android.Tools.Aidl.csproj", "{302D9D2E-1F98-4374-9B6B-922F78620C4B}"
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
tests\Mono.Android-Tests\Mono.Android-Test.Shared.projitems*{0ab4956e-6fb9-4da0-9d49-ab65a3ff403a}*SharedItemsImports = 13
Expand Down Expand Up @@ -237,14 +233,6 @@ Global
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Debug|Any CPU.Build.0 = Debug|Any CPU
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Release|Any CPU.ActiveCfg = Release|Any CPU
{94BD81F7-B06F-4295-9636-F8A3B6BDC762}.Release|Any CPU.Build.0 = Release|Any CPU
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Debug|Any CPU.Build.0 = Debug|Any CPU
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Release|Any CPU.ActiveCfg = Release|Any CPU
{883941C8-C4ED-428D-A7D2-26F2EEA23F92}.Release|Any CPU.Build.0 = Release|Any CPU
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{302D9D2E-1F98-4374-9B6B-922F78620C4B}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -281,8 +269,6 @@ Global
{D1243BAB-23CA-4566-A2A3-3ADA2C2DC3AF} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
{6CB00820-A66B-43E5-8785-ED456C6E9F39} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
{94BD81F7-B06F-4295-9636-F8A3B6BDC762} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
{883941C8-C4ED-428D-A7D2-26F2EEA23F92} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
{302D9D2E-1F98-4374-9B6B-922F78620C4B} = {EFBC4DC0-DBFF-4DAA-B0B8-6D0CB02A25F5}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8643CD20-B195-4919-8135-27549488237E}
Expand Down
7 changes: 7 additions & 0 deletions Xamarin.Android.sln
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Runtime.Environment",
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "create-android-api", "build-tools\create-android-api\create-android-api.csproj", "{BA4D889D-066B-4C2C-A973-09E319CBC396}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Xamarin.Android.Tools.Aidl-Tests", "tests\Xamarin.Android.Tools.Aidl-Tests\Xamarin.Android.Tools.Aidl-Tests.csproj", "{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|AnyCPU = Debug|AnyCPU
Expand Down Expand Up @@ -424,6 +426,10 @@ Global
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Debug|AnyCPU.Build.0 = Debug|Any CPU
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.ActiveCfg = Release|Any CPU
{BA4D889D-066B-4C2C-A973-09E319CBC396}.Release|AnyCPU.Build.0 = Release|Any CPU
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Debug|AnyCPU.ActiveCfg = Debug|Any CPU
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Debug|AnyCPU.Build.0 = Debug|Any CPU
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Release|AnyCPU.ActiveCfg = Release|Any CPU
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708}.Release|AnyCPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -495,6 +501,7 @@ Global
{D8E14B43-E929-4C18-9FA6-2C3DC47EFC17} = {864062D3-A415-4A6F-9324-5820237BA058}
{C0E44558-FEE3-4DD3-986A-3F46DD1BF41B} = {04E3E11E-B47D-4599-8AFC-50515A95E715}
{BA4D889D-066B-4C2C-A973-09E319CBC396} = {E351F97D-EA4F-4E7F-AAA0-8EBB1F2A4A62}
{A39B6D7C-6616-40D6-8AE4-C6CEE93D2708} = {CAB438D8-B0F5-4AF0-BEBD-9E2ADBD7B483}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {53A1F287-EFB2-4D97-A4BB-4A5E145613F6}
Expand Down
144 changes: 35 additions & 109 deletions build-tools/automation/azure-pipelines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ variables:
value: $[or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/'))]
- name: DotNetNUnitCategories
value: '& TestCategory != DotNetIgnore & TestCategory != HybridAOT & TestCategory != MkBundle & TestCategory != MonoSymbolicate & TestCategory != PackagesConfig & TestCategory != StaticProject & TestCategory != SystemApplication'
- name: ExcludedNUnitCategories
value: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != PackagesConfig & cat != StaticProject & cat != SystemApplication'
- ${{ if eq(variables['Build.DefinitionName'], 'Xamarin.Android-Private') }}:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-MSRC-Storage
Expand Down Expand Up @@ -315,6 +317,11 @@ stages:
parameters:
displayName: install emulator
arguments: --s=EmulatorTestDependencies

- task: DownloadPipelineArtifact@2
inputs:
artifactName: $(TestAssembliesArtifactName)
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)

# Set up dependencies to run tests in both debug and release configurations
- task: DotNetCoreCLI@2
Expand Down Expand Up @@ -398,6 +405,12 @@ stages:
artifactSource: bin/Test$(XA.Build.Configuration)/$(DotNetTargetFramework)-android/Mono.Android.NET_Tests-Signed.aab
artifactFolder: $(DotNetTargetFramework)-AotLlvm

- template: yaml-templates/run-nunit-tests.yaml
parameters:
testRunTitle: Xamarin.Android.Tools.Aidl-Tests - macOS
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Tools.Aidl-Tests.dll
testResultsFile: TestResult-Aidl-Tests-macOS-$(XA.Build.Configuration).xml

- task: MSBuild@1
displayName: shut down emulator
inputs:
Expand Down Expand Up @@ -440,23 +453,25 @@ stages:

- template: yaml-templates/fail-on-issue.yaml

# Xamarin.Android (Smoke Tests MSBuild - Mac-0)
- template: yaml-templates/run-msbuild-mac-tests.yaml
# MSBuild Smoke Tests - Mac
- template: yaml-templates/run-msbuild-tests.yaml
parameters:
job_name: mac_dotnet_tests_0
job_suffix: One .NET
nunit_categories: '| (TestCategory = SmokeTests $(DotNetNUnitCategories))'
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

# Xamarin.Android (Smoke Tests MSBuild - Win-0)
- template: yaml-templates\run-msbuild-win-tests.yaml
testOS: macOS
jobName: mac_smoke_msbuild_tests
jobDisplayName: macOS > Tests > MSBuild
agentCount: 2
testFilter: cat = SmokeTests
installApkDiff: true

# MSBuild Smoke Tests - Win
- template: yaml-templates/run-msbuild-tests.yaml
parameters:
job_name: win_dotnet_tests_0
job_suffix: One .NET
nunit_categories: '| (TestCategory = SmokeTests $(DotNetNUnitCategories))'
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}
testOS: Windows
jobName: win_smoke_msbuild_tests
jobDisplayName: Windows > Tests > MSBuild
agentCount: 2
testFilter: cat = SmokeTests
installApkDiff: true

# Check - "Xamarin.Android (macOS > Tests > MSBuild+Emulator)"
- job: mac_msbuilddevice_tests
Expand Down Expand Up @@ -550,103 +565,14 @@ stages:
artifactName: Test Results - MSBuild Smoke - Linux

- template: yaml-templates/fail-on-issue.yaml

- stage: msbuild_dotnet
displayName: One .NET Tests
dependsOn: mac_build
condition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuild')))
jobs:
# Xamarin.Android (Test MSBuild One .NET - macOS)
- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 1
job_name: mac_dotnet_tests_1
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 2
job_name: mac_dotnet_tests_2
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 3
job_name: mac_dotnet_tests_3
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 4
job_name: mac_dotnet_tests_4
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 5
job_name: mac_dotnet_tests_5
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-mac-tests.yaml
parameters:
node_id: 6
job_name: mac_dotnet_tests_6
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

# Xamarin.Android (Test MSBuild One .NET - Windows)
- template: yaml-templates\run-msbuild-win-tests.yaml
parameters:
node_id: 1
additional_node_id: 4
job_name: win_dotnet_tests_1
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-win-tests.yaml
parameters:
node_id: 2
additional_node_id: 5
job_name: win_dotnet_tests_2
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}

- template: yaml-templates\run-msbuild-win-tests.yaml
parameters:
node_id: 3
additional_node_id: 6
job_name: win_dotnet_tests_3
job_suffix: One .NET
nunit_categories: $(DotNetNUnitCategories) & TestCategory != SmokeTests
target_framework: $(DotNetStableTargetFramework)
provisionatorChannel: ${{ parameters.provisionatorChannel }}


- template: yaml-templates/stage-msbuild-tests.yaml
parameters:
stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuild')))

- template: yaml-templates/stage-msbuild-emulator-tests.yaml
parameters:
provisionatorChannel: ${{ parameters.provisionatorChannel }}
stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuildDevice')))
nunit_categories: '& cat != DotNetIgnore & cat != HybridAOT & cat != MkBundle & cat != MonoSymbolicate & cat != PackagesConfig & cat != StaticProject & cat != SystemApplication'

- stage: bcl_tests
displayName: BCL Emulator Tests
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
parameters:
version: '0.1.0-alpha1'
version: '0.1.0-alpha2'
condition: succeeded()
continueOnError: true

Expand Down
66 changes: 0 additions & 66 deletions build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml

This file was deleted.

54 changes: 54 additions & 0 deletions build-tools/automation/yaml-templates/run-msbuild-tests.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
parameters:
testOS: # 'macOS' or 'Windows'
jobName: # Name of the job
jobDisplayName: # Display name of the job
agentCount: # Number of build agents to run in parallel
testFilter: # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
installApkDiff: false # Smoke tests use apkdiff, others do not

jobs:
- job: ${{ parameters.jobName }}
strategy:
parallel: ${{ parameters.agentCount }}
displayName: ${{ parameters.jobDisplayName }}
${{ if eq(parameters.testOS, 'Windows') }}:
pool: $(1ESWindowsPool)
${{ if eq(parameters.testOS, 'macOS') }}:
pool:
vmImage: $(HostedMacImage)
timeoutInMinutes: 180
cancelTimeoutInMinutes: 5
steps:
- ${{ if eq(parameters.testOS, 'Windows') }}:
- script: netsh int ipv4 set global sourceroutingbehavior=drop

- template: kill-processes.yaml

- template: clean.yaml

- template: setup-test-environment.yaml
parameters:
provisionClassic: false
installTestSlicer: true
installApkDiff: ${{ parameters.installApkDiff }}
installLegacyDotNet: false
restoreNUnitConsole: false
updateMono: false

- task: DownloadPipelineArtifact@2
inputs:
artifactName: $(TestAssembliesArtifactName)
downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)

- template: run-sliced-nunit-tests.yaml
parameters:
testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll
testFilter: ${{ parameters.testFilter }} $(ExcludedNUnitCategories)
testRunTitle: Xamarin.Android.Build.Tests - ${{ parameters.testOS }}
testResultsTitle: TestResult-MSBuildTests-${{ parameters.jobName }}

- template: upload-results.yaml
parameters:
artifactName: Test Results - MSBuild - ${{ parameters.testOS }}-$(System.JobPositionInPhase)

- template: fail-on-issue.yaml
Loading

0 comments on commit 928e7b9

Please sign in to comment.