Run UI tests per groups of categories in CI

eng/devices/android.cake

@@ -18,9 +18,6 @@ var testAppInstrumentation = Argument("instrumentation", EnvironmentVariable("AN
 var testResultsPath = Argument("results", EnvironmentVariable("ANDROID_TEST_RESULTS") ?? GetTestResultsDirectory()?.FullPath);
 var deviceCleanupEnabled = Argument("cleanup", true);
 
-// Test where clause
-string testWhere = Argument("where", EnvironmentVariable("NUNIT_TEST_WHERE") ?? "");
-
 // Device details
 var deviceSkin = Argument("skin", EnvironmentVariable("ANDROID_TEST_SKIN") ?? "Nexus 5X");
 var androidAvd = "DEVICE_TESTS_EMULATOR";
@@ -96,7 +93,6 @@ Task("uitest-build")
 
 	});
 Task("uitest")
-	.IsDependentOn("uitest-build")
 	.Does(() =>
 	{
 		ExecuteUITests(projectPath, testAppProjectPath, testAppPackageName, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, "", androidVersion, dotnetToolPath, testAppInstrumentation);
@@ -305,7 +301,7 @@ void ExecuteUITests(string project, string app, string appPackageName, string de
 	var name = System.IO.Path.GetFileNameWithoutExtension(project);
 	var binlog = $"{binDir}/{name}-{config}-{platform}.binlog";
 	var appiumLog = $"{binDir}/appium_{platform}.log";
-	var resultsFileName = $"{name}-{config}-{platform}";
+	var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-{platform}-{testFilter}");
 
 	DotNetBuild(project, new DotNetBuildSettings
 	{

eng/devices/catalyst.cake

@@ -58,7 +58,6 @@ Task("uitest-build")
 	});
 
 Task("uitest")
-	.IsDependentOn("uitest-build")
 	.Does(() =>
 	{
 		ExecuteUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, dotnetToolPath);
@@ -131,21 +130,20 @@ void ExecuteUITests(string project, string app, string device, string resultsDir
 	}
 
 	// Launch the app so it can be found by the test runner
-	DotNetBuild(app, new DotNetBuildSettings
-	{
-		Configuration = config,
-		Framework = tfm,
-		ToolPath = toolPath,
-		ArgumentCustomization = args => args
-			.Append("/t:Run")
-	});
+	StartProcess("chmod", $"+x {testApp}/Contents/MacOS/Controls.TestCases.HostApp");
+
+	var p = new System.Diagnostics.Process();
+	p.StartInfo.UseShellExecute = true;
+	p.StartInfo.FileName = "open";
+	p.StartInfo.Arguments = testApp;
+	p.Start();
 
 	Information("Build UITests project {0}", project);
 
 	var name = System.IO.Path.GetFileNameWithoutExtension(project);
 	var binlog = $"{binDir}/{name}-{config}-mac.binlog";
 	var appiumLog = $"{binDir}/appium_mac.log";
-	var resultsFileName = $"{name}-{config}-catalyst";
+	var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-catalyst-{testFilter}");
 
 	DotNetBuild(project, new DotNetBuildSettings
 	{

eng/devices/devices-shared.cake

@@ -45,7 +45,7 @@ IEnumerable<string> GetTestApplications(string project, string device, string co
     const string artifactsDir = "../../artifacts/bin/";
     bool isAndroid = tfm.Contains("android");
 
-    var binDir = new DirectoryPath(project).Combine($"{binDirBase}/{config}/{tfm}/{rid}");
+    var binDir = new DirectoryPath(project).Combine($"{config}/{tfm}/{rid}");
     IEnumerable<string> applications;
 
     if (isAndroid)
@@ -222,3 +222,10 @@ void ExecuteWithRetries(Func<int> action, int retries)
         System.Threading.Thread.Sleep(1000);
     }
 }
+
+string SanitizeTestResultsFilename(string input)
+{
+    string resultFilename = input.Replace("|", "_").Replace("TestCategory=", "");
+
+    return resultFilename;
+}

eng/devices/ios.cake

@@ -18,9 +18,6 @@ var platform = testDevice.ToLower().Contains("simulator") ? "iPhoneSimulator" :
 var runtimeIdentifier = Argument("rid", EnvironmentVariable("IOS_RUNTIME_IDENTIFIER") ?? GetDefaultRuntimeIdentifier(testDevice));
 var deviceCleanupEnabled = Argument("cleanup", true);
 
-// Test where clause
-string testWhere = Argument("where", EnvironmentVariable("NUNIT_TEST_WHERE") ?? "");
-
 // Device details
 var udid = Argument("udid", EnvironmentVariable("IOS_SIMULATOR_UDID") ?? "");
 var iosVersion = Argument("apiversion", EnvironmentVariable("IOS_PLATFORM_VERSION") ?? DefaultVersion);
@@ -87,7 +84,6 @@ Task("uitest-build")
 	});
 
 Task("uitest")
-	.IsDependentOn("uitest-build")
 	.Does(() =>
 	{
 		ExecuteUITests(projectPath, testAppProjectPath, testDevice, testResultsPath, binlogDirectory, configuration, targetFramework, runtimeIdentifier, iosVersion, dotnetToolPath);
@@ -212,7 +208,7 @@ void ExecuteUITests(string project, string app, string device, string resultsDir
 	var name = System.IO.Path.GetFileNameWithoutExtension(project);
 	var binlog = $"{binDir}/{name}-{config}-ios.binlog";
 	var appiumLog = $"{binDir}/appium_ios.log";
-	var resultsFileName = $"{name}-{config}-ios";
+	var resultsFileName = SanitizeTestResultsFilename($"{name}-{config}-ios-{testFilter}");
 
 	DotNetBuild(project, new DotNetBuildSettings
 	{

eng/devices/windows.cake

@@ -212,7 +212,15 @@ Task("Test")
 	Information("Cleaned directories");
 
 	var testResultsPath = MakeAbsolute((DirectoryPath)TEST_RESULTS).FullPath.Replace("/", "\\");
-	var testResultsFile = testResultsPath + $"\\TestResults-{PACKAGEID.Replace(".", "_")}.xml";
+	var testResultsFile = testResultsPath + $"\\TestResults-{PACKAGEID.Replace(".", "_")}";
+
+	if (!string.IsNullOrWhiteSpace(testFilter))
+	{
+		testResultsFile += SanitizeTestResultsFilename($"-{testFilter}");
+	}
+
+	testResultsFile += ".xml";
+
 	var testsToRunFile = MakeAbsolute((DirectoryPath)TEST_RESULTS).FullPath.Replace("/", "\\") + $"\\devicetestcategories.txt";
 
 	Information($"Test Results File: {testResultsFile}");
@@ -432,7 +440,7 @@ Task("SetupTestPaths")
 		}
 
 		var winVersion = $"{dotnetVersion}-windows{windowsVersion}";
-		var binDir = TEST_APP_PROJECT.GetDirectory().Combine("bin").Combine(CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM);
+		var binDir = TEST_APP_PROJECT.GetDirectory().Combine("Controls.TestCases.HostApp").Combine(CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM);
 		Information("BinDir: {0}", binDir);
 		var apps = GetFiles(binDir + "/*.exe").Where(c => !c.FullPath.EndsWith("createdump.exe"));
 		if (apps.Count() == 0) 
@@ -464,7 +472,7 @@ Task("SetupTestPaths")
 				binDir = MakeAbsolute(new DirectoryPath(arcadeBin + "/Essentials.DeviceTests/" + CONFIGURATION + "/" + winVersion).Combine(DOTNET_PLATFORM));
 			}
 
-			Information("Looking for .app in arcade binDir {0}", binDir);
+			Information("Looking for .exe in arcade binDir {0}", binDir);
 			apps = GetFiles(binDir + "/*.exe").Where(c => !c.FullPath.EndsWith("createdump.exe"));
 			if(apps.Count() == 0 )
 			{

eng/pipelines/common/ui-tests-build-sample.yml

@@ -0,0 +1,69 @@
+parameters:
+  platform: '' # [ android, ios, windows, catalyst ]
+  path: '' # path to csproj
+  device: '' # the xharness device to use
+  cakeArgs: '' # additional cake args
+  app: '' #path to app to test
+  version: '' #the iOS version'
+  provisionatorChannel: 'latest'
+  agentPoolAccessToken: ''
+  configuration : "Release"
+  testFilter: ''
+
+steps:
+  - ${{ if eq(parameters.platform, 'ios')}}:
+    - bash: |
+        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
+      displayName: 'Clean bot'
+      continueOnError: true
+      timeoutInMinutes: 60
+
+  - template: provision.yml
+    parameters:
+      skipProvisioning:  ${{ eq(parameters.platform, 'windows') }}
+      skipAndroidSdks: ${{ ne(parameters.platform, 'android') }}
+      skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }}
+      provisionatorChannel: ${{ parameters.provisionatorChannel }}
+
+  - task: PowerShell@2
+    condition: ne('${{ parameters.platform }}' , 'windows')
+    inputs:
+      targetType: 'inline'
+      script: |
+        defaults write -g NSAutomaticCapitalizationEnabled -bool false
+        defaults write -g NSAutomaticTextCompletionEnabled -bool false
+        defaults write -g NSAutomaticSpellingCorrectionEnabled -bool false
+    displayName: "Modify defaults"
+    continueOnError: true
+
+  - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic
+    displayName: 'Install .NET'
+    retryCountOnTaskFailure: 2
+    env:
+      DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token)
+      PRIVATE_BUILD: $(PrivateBuild)
+
+  - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)"
+    displayName: 'Add .NET to PATH'
+
+  - pwsh: ./build.ps1 --target=dotnet-buildtasks --configuration="${{ parameters.configuration }}"
+    displayName: 'Build the MSBuild Tasks'
+
+  - pwsh: ./build.ps1 --target=dotnet-samples --configuration="${{ parameters.configuration }}" --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false
+    displayName: 'Build the samples'
+
+  - bash: |
+      if [ -f "$HOME/Library/Logs/CoreSimulator/*" ]; then rm -r $HOME/Library/Logs/CoreSimulator/*; fi
+      if [ -f "$HOME/Library/Logs/DiagnosticReports/*" ]; then rm -r $HOME/Library/Logs/DiagnosticReports/*; fi
+    displayName: Delete Old Simulator Logs
+    condition: ${{ eq(parameters.platform, 'ios') }}
+    continueOnError: true
+
+  - publish: $(System.DefaultWorkingDirectory)/artifacts/bin
+    condition: ne('${{ parameters.platform }}' , 'windows')
+    artifact: ui-tests-samples
+
+  - publish: $(System.DefaultWorkingDirectory)/artifacts/bin
+    condition: eq('${{ parameters.platform }}' , 'windows')
+    artifact: ui-tests-samples-windows

eng/pipelines/common/ui-tests-compatibility-steps.yml

@@ -8,9 +8,9 @@ parameters:
   provisionatorChannel: 'latest'
   agentPoolAccessToken: ''
   configuration : "Release"
-  where : "cat==Issues"
   targetSample: "dotnet-legacy-controlgallery"
   provisionPlatform: "windows"
+  testFilter: ''
 
 steps:
   - bash: |
@@ -56,7 +56,7 @@ steps:
     condition: ${{ eq(parameters.platform, 'ios') }}
     continueOnError: true
 
-  - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic --where="${{ parameters.where }}"
+  - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=cg-uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic
     displayName: $(Agent.JobName)
     # retryCountOnTaskFailure: 2
 

eng/pipelines/common/ui-tests-steps.yml

@@ -8,33 +8,28 @@ parameters:
   provisionatorChannel: 'latest'
   agentPoolAccessToken: ''
   configuration : "Release"
+  testFilter: ''
 
 steps:
-  - ${{ if eq(parameters.platform, 'ios')}}:
-    - bash: |
-        chmod +x $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
-        $(System.DefaultWorkingDirectory)/eng/scripts/clean-bot.sh
-      displayName: 'Clean bot'
-      continueOnError: true
-      timeoutInMinutes: 60
-
-  - template: provision.yml
-    parameters:
-      skipProvisioning:  ${{ eq(parameters.platform, 'windows') }}
-      skipAndroidSdks: ${{ ne(parameters.platform, 'android') }}
-      skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }}
-      provisionatorChannel: ${{ parameters.provisionatorChannel }}
-
-  - task: PowerShell@2
+  - task: DownloadPipelineArtifact@2
     condition: ne('${{ parameters.platform }}' , 'windows')
     inputs:
-      targetType: 'inline'
-      script: |
-        defaults write -g NSAutomaticCapitalizationEnabled -bool false
-        defaults write -g NSAutomaticTextCompletionEnabled -bool false
-        defaults write -g NSAutomaticSpellingCorrectionEnabled -bool false
-    displayName: "Modify defaults"
-    continueOnError: true
+      artifact: ui-tests-samples
+
+  - task: DownloadPipelineArtifact@2
+    condition: eq('${{ parameters.platform }}' , 'windows')
+    inputs:
+      artifact: ui-tests-samples-windows
+
+  - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic
+    displayName: 'Install .NET'
+    retryCountOnTaskFailure: 2
+    env:
+      DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token)
+      PRIVATE_BUILD: $(PrivateBuild)
+
+  - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)"
+    displayName: 'Add .NET to PATH'
 
   # AzDO hosted agents default to 1024x768; set something bigger for Windows UI tests
   - task: ScreenResolutionUtility@1
@@ -60,30 +55,25 @@ steps:
     env:
       APPIUM_HOME: $(APPIUM_HOME)
 
-  - pwsh: ./build.ps1 --target=dotnet --configuration="${{ parameters.configuration }}" --verbosity=diagnostic
-    displayName: 'Install .NET'
-    retryCountOnTaskFailure: 2
-    env:
-      DOTNET_TOKEN: $(dotnetbuilds-internal-container-read-token)
-      PRIVATE_BUILD: $(PrivateBuild)
-
-  - pwsh: echo "##vso[task.prependpath]$(DotNet.Dir)"
-    displayName: 'Add .NET to PATH'
+  - pwsh: |
+      $command = "./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project=""${{ parameters.path }}"""
+      $command += " --appproject=""${{ parameters.app }}"" --device=""${{ parameters.device }}"" --apiversion=""${{ parameters.version }}"" --configuration=""${{ parameters.configuration }}"""
+      $command += " --results=""$(TestResultsDirectory)"" --binlog=""$(LogDirectory)"" ${{ parameters.cakeArgs }} --verbosity=diagnostic"
+
+      $testFilter = ""
 
-  - pwsh: ./build.ps1 --target=dotnet-buildtasks --configuration="${{ parameters.configuration }}"
-    displayName: 'Build the MSBuild Tasks'
+      "${{ parameters.testFilter }}".Split(",") | ForEach {
+        $testFilter += "TestCategory=" + $_ + "|"
+      }
 
-  - pwsh: ./build.ps1 --target=dotnet-samples --configuration="${{ parameters.configuration }}" --${{ parameters.platform }} --verbosity=diagnostic --usenuget=false
-    displayName: 'Build the samples'
+      $testFilter = $testFilter.TrimEnd("|")
 
-  - bash: |
-      if [ -f "$HOME/Library/Logs/CoreSimulator/*" ]; then rm -r $HOME/Library/Logs/CoreSimulator/*; fi
-      if [ -f "$HOME/Library/Logs/DiagnosticReports/*" ]; then rm -r $HOME/Library/Logs/DiagnosticReports/*; fi
-    displayName: Delete Old Simulator Logs
-    condition: ${{ eq(parameters.platform, 'ios') }}
-    continueOnError: true
+      # Cake does not allow empty parameters, so check if our filter is empty before adding it
+      if ($testFilter) {
+        $command += " --test-filter ""$testFilter"""
+      }
 
-  - pwsh: ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --target=uitest --project="${{ parameters.path }}" --appproject="${{ parameters.app }}" --device="${{ parameters.device }}" --apiversion="${{ parameters.version }}" --configuration="${{ parameters.configuration }}" --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }} --verbosity=diagnostic
+      Invoke-Expression $command
     displayName: $(Agent.JobName)
     ${{ if ne(parameters.platform, 'android')}}:
       retryCountOnTaskFailure: 1