[ci] Split MSBuildDeviceIntegration tests across multiple build agents.

build-tools/automation/azure-pipelines.yaml

@@ -1092,14 +1092,46 @@ stages:
  # Check - "Xamarin.Android (MSBuild Emulator Tests macOS - Legacy)"
  - template: yaml-templates/run-msbuild-device-tests.yaml
  parameters:
- job_name: mac_msbuilddevice_tests
+ node_id: 1
+ job_name: mac_msbuilddevice_tests_1
+ job_suffix: Legacy
+ jdkTestFolder: $(XA.Jdk11.Folder)
+
+ - template: yaml-templates/run-msbuild-device-tests.yaml
+ parameters:
+ node_id: 2
+ job_name: mac_msbuilddevice_tests_2
+ job_suffix: Legacy
+ jdkTestFolder: $(XA.Jdk11.Folder)
+
+ - template: yaml-templates/run-msbuild-device-tests.yaml
+ parameters:
+ node_id: 3
+ job_name: mac_msbuilddevice_tests_3
  job_suffix: Legacy
  jdkTestFolder: $(XA.Jdk11.Folder)
 
  # Check - "Xamarin.Android (MSBuild Emulator Tests macOS - One .NET)"
  - template: yaml-templates/run-msbuild-device-tests.yaml
  parameters:
- job_name: mac_dotnetdevice_tests
+ node_id: 1
+ job_name: mac_dotnetdevice_tests_1
+ job_suffix: One .NET
+ nunit_categories: $(DotNetNUnitCategories)
+ target_framework: 'net6.0'
+
+ - template: yaml-templates/run-msbuild-device-tests.yaml
+ parameters:
+ node_id: 2
+ job_name: mac_dotnetdevice_tests_2
+ job_suffix: One .NET
+ nunit_categories: $(DotNetNUnitCategories)
+ target_framework: 'net6.0'
+
+ - template: yaml-templates/run-msbuild-device-tests.yaml
+ parameters:
+ node_id: 3
+ job_name: mac_dotnetdevice_tests_3
  job_suffix: One .NET
  nunit_categories: $(DotNetNUnitCategories)
  target_framework: 'net6.0'

build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml

@@ -1,6 +1,7 @@
 # Runs MSBuild tests against a device running on macOS
 
 parameters:
+ node_id: 0
  job_name: ''
  job_suffix: ''
  nunit_categories: ''
@@ -9,10 +10,10 @@ parameters:
 
 jobs:
  - job: ${{ parameters.job_name }}
- displayName: MSBuild With Emulator - macOS - ${{ parameters.job_suffix }}
+ displayName: MSBuild With Emulator - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
  pool:
  vmImage: $(HostedMacImage)
- timeoutInMinutes: 150
+ timeoutInMinutes: 90
  cancelTimeoutInMinutes: 5
  workspace:
  clean: all
@@ -29,7 +30,8 @@ jobs:
  artifactName: $(TestAssembliesArtifactName)
  downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
 
- - ${{ if eq(parameters.job_suffix, 'Legacy') }}:
+ # Only run on Legacy Node-1
+ - ${{ if and(eq(parameters.job_suffix, 'Legacy'), eq(parameters.node_id, 1)) }}:
  - task: MSBuild@1
  displayName: build check-boot-times.csproj
  inputs:
@@ -56,12 +58,23 @@ jobs:
  - template: run-nunit-tests.yaml
  parameters:
  useDotNet: ${{ eq(parameters.target_framework, 'net6.0') }}
- testRunTitle: MSBuildDeviceIntegration On Device - macOS - ${{ parameters.job_suffix }}
+ testRunTitle: MSBuildDeviceIntegration On Device - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
  testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll
- nunitConsoleExtraArgs: --where "cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}"
- dotNetTestExtraArgs: --filter "TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}"
+ nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} && cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}"
+ dotNetTestExtraArgs: --filter "TestCategory = Node-${{ parameters.node_id }} & TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}"
  testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-$(XA.Build.Configuration).xml
 
+ # Tests with no "Node" category. This should be empty, but just in case! Only run these tests on node 1
+ - ${{ if eq(parameters.node_id, 1) }}:
+ - template: run-nunit-tests.yaml
+ parameters:
+ useDotNet: ${{ eq(parameters.target_framework, 'net6.0') }}
+ testRunTitle: MSBuildDeviceIntegration On Device - macOS-NoNode - ${{ parameters.job_suffix }}
+ testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll
+ nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3 && cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}"
+ dotNetTestExtraArgs: --filter "TestCategory != Node-1 & TestCategory != Node-2 & TestCategory != Node-3 & TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}"
+ testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-NoNode-$(XA.Build.Configuration).xml
+
  - task: MSBuild@1
  displayName: shut down emulator
  inputs:
@@ -74,6 +87,6 @@ jobs:
 
  - template: upload-results.yaml
  parameters:
- artifactName: Test Results - MSBuild With Emulator - macOS - ${{ parameters.job_suffix }}
+ artifactName: Test Results - MSBuild With Emulator - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
 
  - template: fail-on-issue.yaml

tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs

@@ -5,7 +5,7 @@
 
 namespace Xamarin.Android.Build.Tests
 {
- [Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT")]
+ [Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT"), Category ("Node-3")]
  public class AotProfileTests : DeviceTest
  {
 

tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs

@@ -10,7 +10,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("Node-2")]
+ [Category ("Node-1")]
  public class BundleToolTests : DeviceTest
  {
  static readonly string [] Abis = new [] { "armeabi-v7a", "arm64-v8a", "x86" };

tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs

@@ -13,7 +13,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice")]
+ [Category ("UsesDevice"), Category ("Node-3")]
  public class DebuggingTest : DeviceTest {
  [TearDown]
  public void ClearDebugProperties ()

tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs

@@ -7,7 +7,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("DotNetIgnore")] // .csproj files are legacy projects that won't build under dotnet
+ [Category ("DotNetIgnore"), Category ("Node-1")] // .csproj files are legacy projects that won't build under dotnet
  public class DeleteBinObjTest : DeviceTest
  {
  const string BaseUrl = "https://xamjenkinsartifact.azureedge.net/mono-jenkins/xamarin-android-test/";

tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs

@@ -19,7 +19,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice")]
+ [Category ("UsesDevice"), Category ("Node-2")]
  public class DeploymentTest : DeviceTest {
 
  static ProjectBuilder builder;

tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs

@@ -8,7 +8,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice")]
+ [Category ("UsesDevice"), Category ("Node-2")]
  public class InstallAndRunTests : DeviceTest
  {
  static ProjectBuilder builder;

tests/MSBuildDeviceIntegration/Tests/InstallTests.cs

@@ -12,7 +12,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("Commercial"), Category ("UsesDevice")]
+ [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-1")]
  public class InstallTests : DeviceTest
  {
  string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCommand = true)

tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs

@@ -7,7 +7,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("Commercial"), Category ("UsesDevice")]
+ [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-3")]
  public class InstantRunTest : DeviceTest
  {
  [Test]

tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs

@@ -12,7 +12,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice")]
+ [Category ("UsesDevice"), Category ("Node-1")]
  public class MonoAndroidExportTest : DeviceTest {
 #pragma warning disable 414
  static object [] MonoAndroidExportTestCases = new object [] {

tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs

@@ -11,6 +11,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
+ [Category ("Node-2")]
  public class PerformanceTest : DeviceTest
  {
  const int Retry = 2;

tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs

@@ -11,7 +11,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("Commercial"), Category ("UsesDevice")]
+ [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-1")]
  public class SystemApplicationTests : DeviceTest
  {
  // All Tests here require the emulator to be started with -writable-system

tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs

@@ -8,7 +8,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice")]
+ [Category ("UsesDevice"), Category ("Node-3")]
  public class UncaughtExceptionTests : DeviceTest
  {
  class LogcatLine

tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs

@@ -14,7 +14,7 @@
 namespace Xamarin.Android.Build.Tests
 {
  [TestFixture]
- [Category ("UsesDevice"), Category ("SmokeTests")]
+ [Category ("UsesDevice"), Category ("SmokeTests"), Category ("Node-3")]
  public class XASdkDeployTests : DeviceTest
  {
  static object [] DotNetInstallAndRunSource = new object [] {