[ci] Remove separate SmokeTests jobs. (#7872)

Context: #7850 (comment)

With the re-parallelization of our test suites (#7804, #7850), combined with the fact that most PR's end up running the full test suite anyways, the separate "Smoke Tests" jobs aren't really saving us much, if any, CI time.  We feel we would be better off using those additional agents to run the full test suite faster.

This PR removes the separate "Smoke Tests" jobs and runs all tests in the same jobs.

Additionally, as the `[Category ("SmokeTests")]` NUnit attribute is now only used by the Windows build stage to run in-tree tests, it has been cut back to the absolute bare minimum, saving ~45 minutes.  Note that all the tests will continue to be run against the Windows installer, this only runs fewer tests for the Windows in-tree tests.

build-tools/automation/azure-pipelines.yaml

@@ -93,7 +93,7 @@ stages:
 - template: yaml-templates/build-linux.yaml
 
 - stage: smoke_tests
- displayName: Smoke Tests
+ displayName: Package Tests
  dependsOn: mac_build
  jobs:
  # Check - "Xamarin.Android (macOS > Tests > APKs Classic)"
@@ -453,80 +453,6 @@ stages:
 
  - template: yaml-templates/fail-on-issue.yaml
 
- # MSBuild Smoke Tests - Mac
- - template: yaml-templates/run-msbuild-tests.yaml
- parameters:
- testOS: macOS
- jobName: mac_smoke_msbuild_tests
- jobDisplayName: macOS > Tests > MSBuild
- agentCount: 2
- testFilter: cat = SmokeTests
- installApkDiff: true
-
- # MSBuild Smoke Tests - Win
- - template: yaml-templates/run-msbuild-tests.yaml
- parameters:
- testOS: Windows
- jobName: win_smoke_msbuild_tests
- jobDisplayName: Windows > Tests > MSBuild
- agentCount: 2
- testFilter: cat = SmokeTests
- installApkDiff: true
-
- # Check - "Xamarin.Android (macOS > Tests > MSBuild+Emulator)"
- - job: mac_msbuilddevice_tests
- displayName: macOS > Tests > MSBuild+Emulator One .NET
- pool:
- vmImage: $(HostedMacImage)
- timeoutInMinutes: 180
- workspace:
- clean: all
- steps:
- - template: yaml-templates/setup-test-environment.yaml
- parameters:
- provisionClassic: false
- provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
- - template: yaml-templates/run-xaprepare.yaml
- parameters:
- displayName: install emulator
- arguments: --s=EmulatorTestDependencies
-
- - task: DownloadPipelineArtifact@2
- inputs:
- artifactName: $(TestAssembliesArtifactName)
- downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)
-
- - task: MSBuild@1
- displayName: start emulator
- inputs:
- solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
- configuration: $(XA.Build.Configuration)
- msbuildArguments: /t:AcquireAndroidTarget /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/start-emulator.binlog
-
- - template: yaml-templates/run-nunit-tests.yaml
- parameters:
- testRunTitle: MSBuildDeviceIntegration Smoke - macOS
- testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
- dotNetTestExtraArgs: --filter "TestCategory = SmokeTests $(DotNetNUnitCategories)"
- testResultsFile: TestResult-MSBuildDeviceIntegrationSmoke-$(XA.Build.Configuration).xml
-
- - task: MSBuild@1
- displayName: shut down emulator
- inputs:
- solution: tests/Mono.Android-Tests/Mono.Android-Tests.csproj
- configuration: $(XA.Build.Configuration)
- msbuildArguments: >-
- /t:AcquireAndroidTarget,ReleaseAndroidTarget
- /bl:$(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/shutdown-emulator.binlog
- condition: always()
-
- - template: yaml-templates/upload-results.yaml
- parameters:
- artifactName: Test Results - MSBuild Smoke With Emulator - macOS - One .NET
-
- - template: yaml-templates/fail-on-issue.yaml
-
 - stage: linux_tests
  displayName: Linux Tests
  dependsOn:
@@ -567,18 +493,14 @@ stages:
  - template: yaml-templates/fail-on-issue.yaml
 
 - template: yaml-templates/stage-msbuild-tests.yaml
- parameters:
- stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuild')))
 
 - template: yaml-templates/stage-msbuild-emulator-tests.yaml
- parameters:
- stageCondition: and(succeeded(), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'MSBuildDevice')))
 
 - stage: bcl_tests
  displayName: BCL Emulator Tests
  dependsOn: mac_build
  # Disabled on .NET release branches
- condition: and(succeeded(), eq(variables.IsRelOrTargetingRel, 'False'), or(eq(variables['RunAllTests'], true), contains(dependencies.mac_build.outputs['mac_build_create_installers.TestConditions.TestAreas'], 'BCL')))
+ condition: and(succeeded(), eq(variables.IsRelOrTargetingRel, 'False'))
  jobs:
  # Check - "Xamarin.Android (macOS > Tests > BCL (Emulator))"
  - job: mac_bcl_tests

build-tools/automation/yaml-templates/build-macos.yaml

@@ -80,11 +80,3 @@ stages:
  xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
  artifactName: ${{ parameters.buildResultArtifactName }}
  includeBuildResults: true
-
- - template: run-xaprepare.yaml
- parameters:
- arguments: --s=DetermineApplicableTests
- xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
- displayName: determine which test stages to run
- name: TestConditions
- condition: and(succeeded(), eq(variables['Build.DefinitionName'], 'Xamarin.Android-PR'))

build-tools/automation/yaml-templates/run-msbuild-tests.yaml

@@ -4,7 +4,6 @@ parameters:
  jobDisplayName: # Display name of the job
  agentCount: # Number of build agents to run in parallel
  testFilter: # Filter used to select tests (NUnit test selection language, not dotnet test filter language)
- installApkDiff: false # Smoke tests use apkdiff, others do not
 
 jobs:
 - job: ${{ parameters.jobName }}
@@ -30,7 +29,6 @@ jobs:
  parameters:
  provisionClassic: false
  installTestSlicer: true
- installApkDiff: ${{ parameters.installApkDiff }}
  installLegacyDotNet: false
  restoreNUnitConsole: false
  updateMono: false

build-tools/automation/yaml-templates/stage-msbuild-emulator-tests.yaml

@@ -2,7 +2,7 @@
 
 parameters:
  job_name: 'mac_dotnetdevice_tests'
- agent_count: 6
+ agent_count: 8
  stageCondition: succeeded()
 
 stages:
@@ -50,7 +50,7 @@ stages:
  - template: run-sliced-nunit-tests.yaml
  parameters:
  testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/$(DotNetStableTargetFramework)/MSBuildDeviceIntegration.dll
- testFilter: cat != TimeZoneInfo & cat != Localization & cat != SmokeTests $(ExcludedNUnitCategories)
+ testFilter: cat != TimeZoneInfo & cat != Localization $(ExcludedNUnitCategories)
  testRunTitle: MSBuildDeviceIntegration On Device - macOS
  testResultsTitle: TestResult-MSBuildDeviceIntegration-${{ parameters.job_name }}
 

build-tools/automation/yaml-templates/stage-msbuild-tests.yaml

@@ -14,13 +14,13 @@ stages:
  testOS: macOS
  jobName: mac_msbuild_tests
  jobDisplayName: macOS > Tests > MSBuild
- agentCount: 8
- testFilter: cat != SmokeTests
+ agentCount: 10
+ testFilter: cat != Dummy # This is because $(ExcludedNUnitCategories) gets appended which starts with '&'
 
  - template: run-msbuild-tests.yaml
  parameters:
  testOS: Windows
  jobName: win_msbuild_tests
  jobDisplayName: Windows > Tests > MSBuild
- agentCount: 4
- testFilter: cat != SmokeTests
+ agentCount: 6
+ testFilter: cat != Dummy # This is because $(ExcludedNUnitCategories) gets appended which starts with '&'

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs

@@ -87,7 +87,6 @@ public void BuildAppWithSystemNamespace ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void DesignTimeBuild ([Values(false, true)] bool isRelease, [Values (false, true)] bool useManagedParser, [Values (false, true)] bool useAapt2)
  {
  AssertAaptSupported (useAapt2);

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs

@@ -57,7 +57,7 @@ void AssertProfiledAotBuildMessages(ProjectBuilder b)
  StringAssertEx.ContainsRegex (@$"Method.*emitted at", b.LastBuildOutput, "Should contain verbose AOT compiler output", RegexOptions.IgnoreCase);
  }
 
- [Test, Category ("SmokeTests"), Category ("ProfiledAOT")]
+ [Test, Category ("ProfiledAOT")]
  public void BuildBasicApplicationReleaseProfiledAot ([Values (true, false)] bool enableLLVM)
  {
  if (Builder.UseDotNet && enableLLVM) {
@@ -75,7 +75,7 @@ public void BuildBasicApplicationReleaseProfiledAot ([Values (true, false)] bool
  AssertProfiledAotBuildMessages (b);
  }
 
- [Test, Category ("SmokeTests"), Category ("ProfiledAOT")]
+ [Test, Category ("ProfiledAOT")]
  public void BuildBasicApplicationReleaseWithCustomAotProfile ()
  {
  var proj = new XamarinAndroidApplicationProject () {
@@ -255,7 +255,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL
 
  [Test]
  [NonParallelizable]
- [Category ("SmokeTests"), Category ("XamarinBuildDownload")]
+ [Category ("XamarinBuildDownload")]
  public void BuildAMassiveApp ()
  {
  var testPath = Path.Combine ("temp", "BuildAMassiveApp");

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs

@@ -22,7 +22,6 @@ public class BindingBuildTest : BaseTest {
 
  [Test]
  [TestCaseSource (nameof (ClassParseOptions))]
- [Category ("SmokeTests")]
  public void BuildBasicBindingLibrary (string classParser)
  {
  var targets = new List<string> {
@@ -154,7 +153,6 @@ public void BuildAarBindigLibraryWithNuGetPackageOfJar (string classParser)
  [Test]
  [TestCaseSource (nameof (ClassParseOptions))]
  [NonParallelizable]
- [Category ("SmokeTests")]
  public void BuildLibraryZipBindigLibraryWithAarOfJar (string classParser)
  {
  var proj = new XamarinAndroidBindingProject () {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs

@@ -100,7 +100,6 @@ public void CheckMonoComponentsMask (bool enableProfiler, bool useInterpreter, b
  [Test]
  [TestCaseSource (nameof (CheckAssemblyCountsSource))]
  [NonParallelizable]
- [Category ("SmokeTests")]
  public void CheckAssemblyCounts (bool isRelease, bool aot)
  {
  var proj = new XamarinFormsAndroidApplicationProject {
@@ -145,7 +144,6 @@ public void CheckAssemblyCounts (bool isRelease, bool aot)
  // DotNet fails, see https://github.com/dotnet/runtime/issues/65484
  // Enable the commented out signature (and AOT) once the above is fixed
  [Test]
- [Category ("SmokeTests")]
  public void SmokeTestBuildWithSpecialCharacters ([Values (false, true)] bool forms, [Values (false /*, true*/)] bool aot)
  {
  var testName = "テスト";
@@ -281,7 +279,6 @@ public void CheckSequencePointGeneration (bool isRelease, bool monoSymbolArchive
 
  [Test]
  [NonParallelizable]
- [Category ("SmokeTests")]
  public void BuildWithNativeLibraries ([Values (true, false)] bool isRelease)
  {
  var dll = new XamarinAndroidLibraryProject () {
@@ -409,7 +406,6 @@ public void BuildWithNativeLibraryUnknownAbi ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void BuildWithExternalJavaLibrary ()
  {
  var path = Path.Combine ("temp", TestName);
@@ -777,7 +773,7 @@ public interface OnFooListener
  }
 
  [Test]
- [Category ("SmokeTests"), Category ("AOT")]
+ [Category ("AOT")]
  [NonParallelizable]
  public void BuildApplicationWithSpacesInPath ([Values (true, false)] bool enableMultiDex, [Values ("", "r8")] string linkTool)
  {
@@ -1041,7 +1037,6 @@ public void CompileBeforeUpgradingNuGet ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  [Category ("DotNetIgnore")] // .mdb and non-portable .pdb files not supported in .NET 5+
  public void BuildBasicApplicationCheckPdb ()
  {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs

@@ -117,7 +117,6 @@ public void BuildBasicApplication ([ValueSource (nameof (SupportedTargetFramewor
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void BuildBasicApplicationThenMoveIt ([Values (true, false)] bool isRelease)
  {
  string path = Path.Combine (Root, "temp", TestName, "App1");
@@ -157,7 +156,6 @@ public static string GetLinkedPath (ProjectBuilder builder, bool isRelease, stri
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void BuildReleaseArm64 ([Values (false, true)] bool forms)
  {
  var proj = forms ?
@@ -511,7 +509,7 @@ public void DuplicateRJavaOutput ()
  }
 
  [Test]
- [Category ("SmokeTests"), Category ("XamarinBuildDownload")]
+ [Category ("XamarinBuildDownload")]
  [NonParallelizable] // parallel NuGet restore causes failures
  public void BuildXamarinFormsMapsApplication ([Values (true, false)] bool multidex)
  {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DeferredBuildTest.cs

@@ -8,7 +8,6 @@ namespace Xamarin.Android.Build.Tests
  public class DeferredBuildTest : BaseTest
  {
  [Test]
- [Category ("SmokeTests")]
  public void SelectivelyRunUpdateAndroidResources ()
  {
  var path = Path.Combine ("temp", TestName);

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DesignerTests.cs

@@ -137,7 +137,6 @@ public CustomTextView(Context context, IAttributeSet attributes) : base(context,
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void IncrementalDesignTimeBuild ()
  {
  var proj = new XamarinFormsAndroidApplicationProject ();

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/EnvironmentContentTests.cs

@@ -15,7 +15,6 @@ public class EnvironmentContentTests : BaseTest
  {
  [Test]
  [NonParallelizable]
- [Category ("SmokeTests")]
  public void BuildApplicationWithMonoEnvironment ([Values ("", "Normal", "Offline")] string sequencePointsMode)
  {
  const string supportedAbis = "armeabi-v7a;x86";

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs

@@ -42,7 +42,6 @@ public void BasicApplicationRepetitiveBuild ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void BasicApplicationRepetitiveReleaseBuild ()
  {
  var proj = new XamarinAndroidApplicationProject () { IsRelease = true };
@@ -422,7 +421,6 @@ public Class2 ()
 
  //https://github.com/xamarin/xamarin-android/issues/2247
  [Test]
- [Category ("SmokeTests")]
  [NonParallelizable] // Do not run timing sensitive tests in parallel
  public void AppProjectTargetsDoNotBreak ()
  {
@@ -566,7 +564,6 @@ public void ManifestMergerIncremental ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void ProduceReferenceAssembly ()
  {
  var path = Path.Combine ("temp", TestName);

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs

@@ -771,7 +771,6 @@ public class TestActivity2 : FragmentActivity {
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void AllActivityAttributeProperties ([Values ("legacy", "manifestmerger.jar")] string manifestMerger)
  {
  var proj = new XamarinAndroidApplicationProject {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs

@@ -350,7 +350,6 @@ public void CheckMetadataSkipItemsAreProcessedCorrectly ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void CheckSignApk ([Values(true, false)] bool useApkSigner, [Values(true, false)] bool perAbiApk)
  {
  string ext = Environment.OSVersion.Platform != PlatformID.Unix ? ".bat" : "";
@@ -483,7 +482,6 @@ public void CheckAapt2WarningsDoNotGenerateErrors ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void CheckAppBundle ([Values (true, false)] bool isRelease)
  {
  var proj = new XamarinAndroidApplicationProject () {
@@ -512,7 +510,7 @@ public void CheckAppBundle ([Values (true, false)] bool isRelease)
  }
 
  [Test]
- [Category ("SmokeTests"), Category ("DotNetIgnore")] // Xamarin.Forms version is too old, uses net45 MSBuild tasks
+ [Category ("DotNetIgnore")] // Xamarin.Forms version is too old, uses net45 MSBuild tasks
  public void NetStandardReferenceTest ()
  {
  var netStandardProject = new DotNetStandard () {

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/ManagedResourceParserTests.cs

@@ -512,7 +512,6 @@ public void UpdateLayoutIdIsIncludedInDesigner ([Values(true, false)] bool useRt
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void RtxtGeneratorOutput ()
  {
  var path = Path.Combine ("temp", TestName);
@@ -545,7 +544,6 @@ public void RtxtGeneratorOutput ()
  }
 
  [Test]
- [Category ("SmokeTests")]
  public void CompareAapt2AndManagedParserOutput ()
  {
  var path = Path.Combine ("temp", TestName);