Run Categories Separately (#29362)

* Run Categories Separately

* - fix windows

* - cleanup

* - wait for last event to fire before finishing

* - keep retry for windows and android

Author: PureWeen

eng/devices/catalyst.cake

@@ -85,32 +85,26 @@ void ExecuteBuild(string project, string binDir, string config, string rid, stri
 	});
 }
 
+
 void ExecuteTests(string project, string device, string resultsDir, string config, string tfm, string rid, string toolPath)
 {
 	CleanResults(resultsDir);
 
 	var testApp = GetTestApplications(project, device, config, tfm, rid).FirstOrDefault();
-
 	Information($"Testing App: {testApp}");
-	var settings = new DotNetToolSettings
-	{
-		DiagnosticOutput = true,
-		ToolPath = toolPath,
-		ArgumentCustomization = args => args.Append($"run xharness apple test --app=\"{testApp}\" --targets=\"{device}\" --output-directory=\"{resultsDir}\" --verbosity=\"Debug\" ")
-	};
 
-	bool testsFailed = true;
-	try
-	{
-		DotNetTool("tool", settings);
-		testsFailed = false;
-	}
-	finally
+	RunMacAndiOSTests(project, device, resultsDir, config, tfm, rid, toolPath, projectPath, (category) =>
 	{
-		HandleTestResults(resultsDir, testsFailed, true);
-	}
-
-	Information("Testing completed.");
+		return new DotNetToolSettings
+		{
+			DiagnosticOutput = true,
+			ToolPath = toolPath,
+			ArgumentCustomization = args => 
+				args.Append($"run xharness apple test --app=\"{testApp}\" --targets=\"{device}\" --output-directory=\"{resultsDir}\" " + 
+					" --verbosity=\"Debug\" " + 
+					$"--set-env=\"TestFilter={category}\" ")
+		};
+	});
 }
 
 void ExecuteUITests(string project, string app, string device, string resultsDir, string binDir, string config, string tfm, string rid, string toolPath)

eng/devices/devices-shared.cake

@@ -133,7 +133,72 @@ void CleanResults(string resultsDir)
     }
 }
 
-void HandleTestResults(string resultsDir, bool testsFailed, bool needsNameFix)
+List<string> GetTestCategoriesToRunSeparately(string projectPath)
+{
+
+    if (!string.IsNullOrEmpty(testFilter))
+    {
+        return new List<string> { testFilter };
+    }
+
+    if (!projectPath.EndsWith("Controls.DeviceTests.csproj") && !projectPath.EndsWith("Core.DeviceTests.csproj"))
+    {
+        return new List<string>
+        {
+            ""
+        };
+    }
+
+	var file = Context.GetCallerInfo().SourceFilePath;
+	var directoryPath = file.GetDirectory().FullPath;
+	Information($"Directory: {directoryPath}");
+	Information(directoryPath);
+
+	// Search for files that match the pattern
+	List<FilePath> dllFilePath = null;
+    
+    if (projectPath.EndsWith("Controls.DeviceTests.csproj"))
+        dllFilePath = GetFiles($"{directoryPath}/../../**/Microsoft.Maui.Controls.DeviceTests.dll").ToList();
+
+    if (projectPath.EndsWith("Core.DeviceTests.csproj"))
+        dllFilePath = GetFiles($"{directoryPath}/../../**/Microsoft.Maui.Core.DeviceTests.dll").ToList();
+
+    System.Reflection.Assembly loadedAssembly = null;
+
+    foreach (var filePath in dllFilePath)
+    {
+        try
+        {
+            loadedAssembly = System.Reflection.Assembly.LoadFrom(filePath.FullPath);
+            Information($"Loaded assembly from {filePath}: {loadedAssembly.FullName}");
+            break; // Exit the loop if the assembly is loaded successfully
+        }
+        catch (Exception ex)
+        {
+            Warning($"Failed to load assembly from {filePath}: {ex.Message}");
+        }
+    }
+
+    if (loadedAssembly == null)
+    {
+        throw new Exception("No test assembly found.");
+    }
+	var testCategoryType = loadedAssembly.GetType("Microsoft.Maui.DeviceTests.TestCategory");
+
+	var values = new List<string>();
+
+	foreach (var field in testCategoryType.GetFields(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Static))
+	{
+		if (field.FieldType == typeof(string))
+		{
+			values.Add($"Category={(string)field.GetValue(null)}");
+		}
+	}
+	
+	return values.ToList();
+}
+
+void HandleTestResults(string resultsDir, bool testsFailed, bool needsNameFix, string suffix = null)
 {
     Information($"Handling test results: {resultsDir}");
 
@@ -145,10 +210,24 @@ void HandleTestResults(string resultsDir, bool testsFailed, bool needsNameFix)
         {
             throw new Exception("No test results found.");
         }
+        
         if (FileExists(resultsFile))
         {
             Information($"Test results found on {resultsDir}");
-            CopyFile(resultsFile, resultsFile.GetDirectory().CombineWithFilePath("TestResults.xml"));
+            MoveFile(resultsFile, resultsFile.GetDirectory().CombineWithFilePath($"TestResults{suffix}.xml"));
+            var logFiles = GetFiles($"{resultsDir}/*.log");
+
+            foreach (var logFile in logFiles)
+            {
+                if (logFile.GetFilename().ToString().StartsWith("TestResults"))
+                {
+                    // These are log files that have already been renamed
+                    continue;
+                }
+
+                Information($"Log file found: {logFile.GetFilename().ToString()}");
+                MoveFile(logFile, resultsFile.GetDirectory().CombineWithFilePath($"TestResults{suffix}-{logFile.GetFilename()}"));
+            }
         }
     }
 
@@ -158,12 +237,21 @@ void HandleTestResults(string resultsDir, bool testsFailed, bool needsNameFix)
         EnsureDirectoryExists(failurePath);
         // The tasks will retry the tests and overwrite the failed results each retry
         // we want to retain the failed results for diagnostic purposes
-        CopyFiles($"{resultsDir}/*.*", failurePath);
+
+        var searchQuery = "*.*";
+
+        if (!string.IsNullOrWhiteSpace(suffix))
+        {
+            searchQuery = $"*{suffix}*.*";
+        }
+
+        // Only copy files from this suffix set of failures
+        CopyFiles($"{resultsDir}/{searchQuery}", failurePath);
 
         // We don't want these to upload
-        MoveFile($"{failurePath}/TestResults.xml", $"{failurePath}/Results.xml");
+        MoveFile($"{failurePath}/TestResults{suffix}.xml", $"{failurePath}/Results{suffix}.xml");
     }
-    FailRunOnOnlyInconclusiveTests($"{resultsDir}/TestResults.xml");
+    FailRunOnOnlyInconclusiveTests($"{resultsDir}/TestResults{suffix}.xml");
 }
 
 DirectoryPath DetermineBinlogDirectory(string projectPath, string binlogArg)
@@ -185,6 +273,67 @@ DirectoryPath DetermineBinlogDirectory(string projectPath, string binlogArg)
     }
 }
 
+void RunMacAndiOSTests(
+    string project, string device, string resultsDir, string config, string tfm, string rid, string toolPath, string projectPath,
+    Func<string, DotNetToolSettings> getSettings)
+{
+    Exception exception = null;
+	foreach (var category in GetTestCategoriesToRunSeparately(projectPath))
+	{
+	    bool testsFailed = true;
+		Information($"Running tests for category: {category}");
+		var settings = getSettings(category);
+        var suffix = category.Split('=').Skip(1).FirstOrDefault();
+
+		try
+		{
+			for(int i = 0; i < 2; i++)
+			{
+				Information($"Running test attempt {i}");
+				try
+				{
+					DotNetTool("tool", settings);
+					testsFailed = false;
+					break;
+				}
+				catch (Exception ex)
+				{
+					Information($"Test attempt {i} failed: {ex.Message}");
+					if (i == 1)
+                    {
+						throw;
+                    }
+                    else
+                    {
+                        // delete any log files created so it's fresh for the rerun
+			            HandleTestResults(resultsDir, false, true, "-" + suffix);
+                        var logFiles = GetFiles($"{resultsDir}/*-{suffix}*");
+
+                        foreach (var logFile in logFiles)
+                        {
+                            DeleteFile(logFile);
+                        }
+                    }
+				}
+			}
+		}
+		catch (Exception ex)
+		{
+			exception = ex;
+		}
+		finally
+		{
+			HandleTestResults(resultsDir, testsFailed, true, "-" + suffix);
+		}
+	}
+
+	Information("Testing completed.");
+	if (exception is not null)
+	{
+		throw exception;
+	}
+}
+
 void LogSetupInfo(string toolPath)
 {
     Information($"DOTNET_TOOL_PATH: {toolPath}");

eng/devices/ios.cake

@@ -173,45 +173,38 @@ void ExecuteTests(string project, string device, string resultsDir, string confi
 
 	Information($"Testing App: {testApp}");
 
-	var settings = new DotNetToolSettings
+
+
+	RunMacAndiOSTests(project, device, resultsDir, config, tfm, rid, toolPath, projectPath, (category) =>
 	{
-		ToolPath = toolPath,
-		DiagnosticOutput = true,
-		ArgumentCustomization = args =>
+		return new DotNetToolSettings
 		{
-			args.Append("run xharness apple test " +
-				$"--app=\"{testApp}\" " +
-				$"--targets=\"{device}\" " +
-				$"--output-directory=\"{resultsDir}\" " +
-				$"--timeout=01:15:00 " +
-				$"--launch-timeout=00:06:00 " +
-				xcode_args +
-				$"--verbosity=\"Debug\" ");
-
-			if (device.Contains("device"))
+			ToolPath = toolPath,
+			DiagnosticOutput = true,
+			ArgumentCustomization = args =>
 			{
-				if (string.IsNullOrEmpty(DEVICE_UDID))
+				args.Append("run xharness apple test " +
+					$"--app=\"{testApp}\" " +
+					$"--targets=\"{device}\" " +
+					$"--output-directory=\"{resultsDir}\" " +
+					$"--timeout=01:15:00 " +
+					$"--launch-timeout=00:06:00 " +
+					xcode_args +
+					$"--verbosity=\"Debug\" " +
+					$"--set-env=\"TestFilter={category}\" ");
+
+				if (device.Contains("device"))
 				{
-					throw new Exception("No device was found to install the app on. See the Setup method for more details.");
+					if (string.IsNullOrEmpty(DEVICE_UDID))
+					{
+						throw new Exception("No device was found to install the app on. See the Setup method for more details.");
+					}
+					args.Append($"--device=\"{DEVICE_UDID}\" ");
 				}
-				args.Append($"--device=\"{DEVICE_UDID}\" ");
+				return args;
 			}
-			return args;
-		}
-	};
-
-	bool testsFailed = true;
-	try
-	{
-		DotNetTool("tool", settings);
-		testsFailed = false;
-	}
-	finally
-	{
-		HandleTestResults(resultsDir, testsFailed, true);
-	}
-	
-	Information("Testing completed.");
+		};
+	});
 }
 
 void ExecutePrepareUITests(string project, string app, string device, string resultsDir, string binDir, string config, string tfm, string rid, string ver, string toolPath)

eng/pipelines/common/device-tests-steps.yml

@@ -145,7 +145,8 @@ steps:
     displayName: Execute Test Run
     workingDirectory: ${{ parameters.checkoutDirectory }}
     condition: and(succeeded(), ne('${{ parameters.buildType }}', 'buildOnly'))
-    retryCountOnTaskFailure: 1
+    ${{ if or(eq(parameters.buildType, 'windows'), eq(parameters.platform, 'android')) }}:
+      retryCountOnTaskFailure: 1
 
 
   ##################################################

src/Controls/tests/DeviceTests/ControlsHandlerTestBase.cs

@@ -140,6 +140,7 @@ protected Task CreateHandlerAndAddToWindow<THandler>(IElement view, Action<THand
 		protected Task CreateHandlerAndAddToWindow<THandler>(IElement view, Func<THandler, Task> action, IMauiContext mauiContext = null, TimeSpan? timeOut = null)
 			where THandler : class, IElementHandler
 		{
+			var stopwatch = System.Diagnostics.Stopwatch.StartNew();
 			mauiContext ??= MauiContext;
 
 			if (System.Diagnostics.Debugger.IsAttached)
@@ -275,7 +276,11 @@ void OnBatchCommitted(object sender, Controls.Internals.EventArg<VisualElement>
 				finally
 				{
 					_takeOverMainContentSempahore.Release();
-					TestRunnerLogger.LogDebug($"Finished Running Test");
+					stopwatch.Stop();
+					TestRunnerLogger.LogDebug($"Finished Running Test: {stopwatch.Elapsed}");
+
+					if (stopwatch.ElapsedMilliseconds > 15000)
+						TestRunnerLogger.LogError($"Test took longer than 15 seconds to complete.");
 				}
 			});
 		}

src/Controls/tests/DeviceTests/Elements/Button/ButtonTests.iOS.cs

@@ -17,7 +17,7 @@ UIButton GetPlatformButton(ButtonHandler buttonHandler) =>
 
 		Task<string> GetPlatformText(ButtonHandler buttonHandler)
 		{
-			return InvokeOnMainThreadAsync(() => GetPlatformButton(buttonHandler).CurrentTitle);
+			return InvokeOnMainThreadAsync(() => GetPlatformButton(buttonHandler).CurrentTitle!);
 		}
 
 		UILineBreakMode GetPlatformLineBreakMode(ButtonHandler buttonHandler) =>

src/Controls/tests/DeviceTests/Elements/Editor/EditorTests.cs

@@ -326,7 +326,6 @@ await InvokeOnMainThreadAsync(async () =>
 		}
 
 		[Category(TestCategory.Editor)]
-		[Category(TestCategory.TextInput)]
 		[Collection(RunInNewWindowCollection)]
 		public class EditorTextInputTests : TextInputTests<EditorHandler, Editor>
 		{
@@ -340,4 +339,4 @@ protected override Task<string> GetPlatformText(EditorHandler handler) =>
 				EditorTests.GetPlatformText(handler);
 		}
 	}
-}
+}

src/Controls/tests/DeviceTests/Elements/Entry/EntryTests.cs

@@ -239,7 +239,6 @@ await InvokeOnMainThreadAsync(async () =>
 		}
 
 		[Category(TestCategory.Entry)]
-		[Category(TestCategory.TextInput)]
 		[Collection(RunInNewWindowCollection)]
 		public class EntryTextInputTests : TextInputTests<EntryHandler, Entry>
 		{

src/Controls/tests/DeviceTests/Elements/Entry/EntryTests.iOS.cs

@@ -66,6 +66,7 @@ Task<bool> GetPlatformIsVisible(EntryHandler entryHandler)
 		}
 
 		[Collection(ControlsHandlerTestBase.RunInNewWindowCollection)]
+		[Category(TestCategory.Entry)]
 		public class ScrollTests : ControlsHandlerTestBase
 		{
 			[Fact]
@@ -126,6 +127,7 @@ await contentViewHandler.PlatformView.AttachAndRun(async () =>
 		}
 
 		[Collection(ControlsHandlerTestBase.RunInNewWindowCollection)]
+		[Category(TestCategory.Entry)]
 		public class NextKeyboardTests : ControlsHandlerTestBase
 		{
 			void SetupNextBuilder()

src/Controls/tests/DeviceTests/Elements/Page/PageTests.Android.cs

@@ -9,6 +9,7 @@
 
 namespace Microsoft.Maui.DeviceTests
 {
+	[Category(TestCategory.Page)]
 	public partial class PageTests : ControlsHandlerTestBase
 	{
 		//src/Compatibility/Core/tests/Android/EmbeddingTests.cs