diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index f284eca4d79..3e1449fe63a 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -1092,14 +1092,46 @@ stages: # Check - "Xamarin.Android (MSBuild Emulator Tests macOS - Legacy)" - template: yaml-templates/run-msbuild-device-tests.yaml parameters: - job_name: mac_msbuilddevice_tests + node_id: 1 + job_name: mac_msbuilddevice_tests_1 + job_suffix: Legacy + jdkTestFolder: $(XA.Jdk11.Folder) + + - template: yaml-templates/run-msbuild-device-tests.yaml + parameters: + node_id: 2 + job_name: mac_msbuilddevice_tests_2 + job_suffix: Legacy + jdkTestFolder: $(XA.Jdk11.Folder) + + - template: yaml-templates/run-msbuild-device-tests.yaml + parameters: + node_id: 3 + job_name: mac_msbuilddevice_tests_3 job_suffix: Legacy jdkTestFolder: $(XA.Jdk11.Folder) # Check - "Xamarin.Android (MSBuild Emulator Tests macOS - One .NET)" - template: yaml-templates/run-msbuild-device-tests.yaml parameters: - job_name: mac_dotnetdevice_tests + node_id: 1 + job_name: mac_dotnetdevice_tests_1 + job_suffix: One .NET + nunit_categories: $(DotNetNUnitCategories) + target_framework: 'net6.0' + + - template: yaml-templates/run-msbuild-device-tests.yaml + parameters: + node_id: 2 + job_name: mac_dotnetdevice_tests_2 + job_suffix: One .NET + nunit_categories: $(DotNetNUnitCategories) + target_framework: 'net6.0' + + - template: yaml-templates/run-msbuild-device-tests.yaml + parameters: + node_id: 3 + job_name: mac_dotnetdevice_tests_3 job_suffix: One .NET nunit_categories: $(DotNetNUnitCategories) target_framework: 'net6.0' diff --git a/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml index 8f5ece7abde..3aa870aa59a 100644 --- a/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml +++ b/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml @@ -1,6 +1,7 @@ # Runs MSBuild tests against a device running on macOS parameters: + node_id: 0 job_name: '' job_suffix: '' nunit_categories: '' @@ -9,10 +10,10 @@ parameters: jobs: - job: ${{ parameters.job_name }} - displayName: MSBuild With Emulator - macOS - ${{ parameters.job_suffix }} + displayName: MSBuild With Emulator - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }} pool: vmImage: $(HostedMacImage) - timeoutInMinutes: 150 + timeoutInMinutes: 90 cancelTimeoutInMinutes: 5 workspace: clean: all @@ -29,7 +30,8 @@ jobs: artifactName: $(TestAssembliesArtifactName) downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) - - ${{ if eq(parameters.job_suffix, 'Legacy') }}: + # Only run on Legacy Node-1 + - ${{ if and(eq(parameters.job_suffix, 'Legacy'), eq(parameters.node_id, 1)) }}: - task: MSBuild@1 displayName: build check-boot-times.csproj inputs: @@ -56,12 +58,23 @@ jobs: - template: run-nunit-tests.yaml parameters: useDotNet: ${{ eq(parameters.target_framework, 'net6.0') }} - testRunTitle: MSBuildDeviceIntegration On Device - macOS - ${{ parameters.job_suffix }} + testRunTitle: MSBuildDeviceIntegration On Device - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }} testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll - nunitConsoleExtraArgs: --where "cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}" - dotNetTestExtraArgs: --filter "TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}" + nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} && cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}" + dotNetTestExtraArgs: --filter "TestCategory = Node-${{ parameters.node_id }} & TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}" testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-$(XA.Build.Configuration).xml + # Tests with no "Node" category. This should be empty, but just in case! Only run these tests on node 1 + - ${{ if eq(parameters.node_id, 1) }}: + - template: run-nunit-tests.yaml + parameters: + useDotNet: ${{ eq(parameters.target_framework, 'net6.0') }} + testRunTitle: MSBuildDeviceIntegration On Device - macOS-NoNode - ${{ parameters.job_suffix }} + testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll + nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3 && cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}" + dotNetTestExtraArgs: --filter "TestCategory != Node-1 & TestCategory != Node-2 & TestCategory != Node-3 & TestCategory != TimeZoneInfo ${{ parameters.nunit_categories }}" + testResultsFile: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}-NoNode-$(XA.Build.Configuration).xml + - task: MSBuild@1 displayName: shut down emulator inputs: @@ -74,6 +87,6 @@ jobs: - template: upload-results.yaml parameters: - artifactName: Test Results - MSBuild With Emulator - macOS - ${{ parameters.job_suffix }} + artifactName: Test Results - MSBuild With Emulator - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }} - template: fail-on-issue.yaml diff --git a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs index 8885879417c..ce0e18f2700 100644 --- a/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/AotProfileTests.cs @@ -5,7 +5,7 @@ namespace Xamarin.Android.Build.Tests { - [Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT")] + [Category ("UsesDevice"), Category ("AOT"), Category ("ProfiledAOT"), Category ("Node-3")] public class AotProfileTests : DeviceTest { diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index 9c0deb624fc..eac880827ed 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -10,7 +10,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("Node-2")] + [Category ("Node-1")] public class BundleToolTests : DeviceTest { static readonly string [] Abis = new [] { "armeabi-v7a", "arm64-v8a", "x86" }; diff --git a/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs b/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs index a2510d32ec3..541aed580e3 100755 --- a/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/DebuggingTest.cs @@ -13,7 +13,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice")] + [Category ("UsesDevice"), Category ("Node-3")] public class DebuggingTest : DeviceTest { [TearDown] public void ClearDebugProperties () diff --git a/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs b/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs index 3d83813f668..b536634bc2c 100644 --- a/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/DeleteBinObjTest.cs @@ -7,7 +7,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("DotNetIgnore")] // .csproj files are legacy projects that won't build under dotnet + [Category ("DotNetIgnore"), Category ("Node-1")] // .csproj files are legacy projects that won't build under dotnet public class DeleteBinObjTest : DeviceTest { const string BaseUrl = "https://xamjenkinsartifact.azureedge.net/mono-jenkins/xamarin-android-test/"; diff --git a/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs b/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs index f9726745410..6dd7363f301 100644 --- a/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs @@ -19,7 +19,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice")] + [Category ("UsesDevice"), Category ("Node-2")] public class DeploymentTest : DeviceTest { static ProjectBuilder builder; diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 8052f1e7c60..12868a91178 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -8,7 +8,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice")] + [Category ("UsesDevice"), Category ("Node-2")] public class InstallAndRunTests : DeviceTest { static ProjectBuilder builder; diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index 25ab81fcea4..642b9b5f3de 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -12,7 +12,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("Commercial"), Category ("UsesDevice")] + [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-1")] public class InstallTests : DeviceTest { string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCommand = true) diff --git a/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs b/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs index 3c1e8fb8cf8..0582154f5ec 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstantRunTest.cs @@ -7,7 +7,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("Commercial"), Category ("UsesDevice")] + [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-3")] public class InstantRunTest : DeviceTest { [Test] diff --git a/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs b/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs index d2c08b719b6..371fbdf105a 100644 --- a/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/MonoAndroidExportTest.cs @@ -12,7 +12,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice")] + [Category ("UsesDevice"), Category ("Node-1")] public class MonoAndroidExportTest : DeviceTest { #pragma warning disable 414 static object [] MonoAndroidExportTestCases = new object [] { diff --git a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs index 9870d54d8f1..6838021f81b 100644 --- a/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs +++ b/tests/MSBuildDeviceIntegration/Tests/PerformanceTest.cs @@ -11,6 +11,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] + [Category ("Node-2")] public class PerformanceTest : DeviceTest { const int Retry = 2; diff --git a/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs b/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs index 90651a1dfa4..736c2c507aa 100644 --- a/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/SystemApplicationTests.cs @@ -11,7 +11,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("Commercial"), Category ("UsesDevice")] + [Category ("Commercial"), Category ("UsesDevice"), Category ("Node-1")] public class SystemApplicationTests : DeviceTest { // All Tests here require the emulator to be started with -writable-system diff --git a/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs b/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs index 609ebbf6486..368353f8dfa 100644 --- a/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/UncaughtExceptionTests.cs @@ -8,7 +8,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice")] + [Category ("UsesDevice"), Category ("Node-3")] public class UncaughtExceptionTests : DeviceTest { class LogcatLine diff --git a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs index f877509af65..418c09e15db 100644 --- a/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/XASdkDeployTests.cs @@ -14,7 +14,7 @@ namespace Xamarin.Android.Build.Tests { [TestFixture] - [Category ("UsesDevice"), Category ("SmokeTests")] + [Category ("UsesDevice"), Category ("SmokeTests"), Category ("Node-3")] public class XASdkDeployTests : DeviceTest { static object [] DotNetInstallAndRunSource = new object [] {