From e73255b108b5ee376ea614a269c418cb8525ce90 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Mon, 6 Jan 2025 12:37:11 -0800 Subject: [PATCH 1/5] add retry logic to StartAndVerifyProcessesAreRunning --- .../B2CWebAppCallsWebApiLocally.cs | 6 +++--- .../E2E Tests/WebAppUiTests/UiTestHelpers.cs | 21 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs index f2ee2c21a..9f7765d6d 100644 --- a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs @@ -42,7 +42,7 @@ public B2CWebAppCallsWebApiLocally(ITestOutputHelper output) [Fact] [SupportedOSPlatform("windows")] - public async Task Susi_B2C_LocalAccount_TodoAppFucntionsCorrectlyAsync() + public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { // Web app and api environmental variable setup. DefaultAzureCredential azureCred = new(); @@ -79,9 +79,9 @@ public async Task Susi_B2C_LocalAccount_TodoAppFucntionsCorrectlyAsync() // The delay before starting client prevents transient devbox issue where the client fails to load the first time after rebuilding. serviceProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListServicePath, TC.s_todoListServiceExe, serviceEnvVars); await Task.Delay(3000); - clientProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListClientPath, TC.s_todoListClientExe, clientEnvVars); + clientProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListClientPath, TC.s_todoListClientExe, clientEnvVars, 5); - if (!UiTestHelpers.ProcessesAreAlive(new List() { clientProcess, serviceProcess })) + if (!UiTestHelpers.ProcessesAreAlive([clientProcess, serviceProcess])) { Assert.Fail(TC.WebAppCrashedString); } diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index d80a39d16..6b4015795 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -142,8 +142,13 @@ await page.Context.Tracing.StartAsync(new() /// The name of the executable that launches the process. /// The port for the process to listen on. /// If the launch URL is http or https. Default is https. + /// Optionally, maximum number of retries if the process exited prematurely. /// The started process. - public static Process StartProcessLocally(string testAssemblyLocation, string appLocation, string executableName, Dictionary? environmentVariables = null) + public static Process StartProcessLocally( + string testAssemblyLocation, + string appLocation, string executableName, + Dictionary? environmentVariables = null, + int maxRetries = 0) { string applicationWorkingDirectory = GetApplicationWorkingDirectory(testAssemblyLocation, appLocation); ProcessStartInfo processStartInfo = new ProcessStartInfo(applicationWorkingDirectory + executableName) @@ -161,7 +166,12 @@ public static Process StartProcessLocally(string testAssemblyLocation, string ap } } - Process? process = Process.Start(processStartInfo); + var currentAttempt = 1; + Process? process; + do + { + process = Process.Start(processStartInfo); + } while (currentAttempt++ <= maxRetries && ProcessIsAlive(process)); if (process == null) { @@ -275,9 +285,9 @@ public static bool ProcessesAreAlive(List processes) /// /// Process to check /// True if alive false if not - public static bool ProcessIsAlive(Process process) + public static bool ProcessIsAlive(Process? process) { - return !process.HasExited; + return process != null && !process.HasExited; } /// @@ -321,7 +331,8 @@ internal static bool StartAndVerifyProcessesAreRunning(List processDataEntry.TestAssemblyLocation, processDataEntry.AppLocation, processDataEntry.ExecutableName, - processDataEntry.EnvironmentVariables); + processDataEntry.EnvironmentVariables, + 5); processes.Add(processDataEntry.ExecutableName, process); Thread.Sleep(5000); From d82fdea4c48fc408041ac1c3d0fad7417fb46b3d Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Mon, 6 Jan 2025 14:19:05 -0800 Subject: [PATCH 2/5] revert change that caused builds to fail again --- tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 6b4015795..fb55ae8e2 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -331,8 +331,7 @@ internal static bool StartAndVerifyProcessesAreRunning(List processDataEntry.TestAssemblyLocation, processDataEntry.AppLocation, processDataEntry.ExecutableName, - processDataEntry.EnvironmentVariables, - 5); + processDataEntry.EnvironmentVariables); processes.Add(processDataEntry.ExecutableName, process); Thread.Sleep(5000); From 9dbcc4c17f08347add337a93591ccc1c56d8eeae Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Mon, 6 Jan 2025 16:25:24 -0800 Subject: [PATCH 3/5] change requests --- tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index fb55ae8e2..503a45201 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -170,6 +170,7 @@ public static Process StartProcessLocally( Process? process; do { + Thread.Sleep(1000 * currentAttempt++); // Exponential backoff process = Process.Start(processStartInfo); } while (currentAttempt++ <= maxRetries && ProcessIsAlive(process)); @@ -179,6 +180,13 @@ public static Process StartProcessLocally( } else { + // Log the output and error streams + process.OutputDataReceived += (sender, e) => Console.WriteLine(e.Data); + process.ErrorDataReceived += (sender, e) => Console.Error.WriteLine(e.Data); + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + return process; } } From 204e4aa753062239487524a4ae465db23c165146 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Mon, 6 Jan 2025 16:31:26 -0800 Subject: [PATCH 4/5] fix spacing --- tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 503a45201..02dc7693d 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -146,7 +146,8 @@ await page.Context.Tracing.StartAsync(new() /// The started process. public static Process StartProcessLocally( string testAssemblyLocation, - string appLocation, string executableName, + string appLocation, + string executableName, Dictionary? environmentVariables = null, int maxRetries = 0) { From e11c87ec13e35b6c37d667c2928116905bc79569 Mon Sep 17 00:00:00 2001 From: Kelly Song Date: Mon, 6 Jan 2025 16:52:50 -0800 Subject: [PATCH 5/5] address comments --- .../WebAppUiTests/B2CWebAppCallsWebApiLocally.cs | 5 ++--- .../WebAppUiTests/TestingWebAppLocally.cs | 2 +- tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs | 16 +++++++++------- .../WebAppCallsApiCallsGraphLocally.cs | 10 ++++------ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs index 9f7765d6d..a8f58f4e6 100644 --- a/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/B2CWebAppCallsWebApiLocally.cs @@ -6,7 +6,6 @@ using System.Diagnostics; using System.IO; using System.Runtime.Versioning; -using System.Threading; using System.Threading.Tasks; using Azure.Identity; using Microsoft.Identity.Lab.Api; @@ -77,9 +76,9 @@ public async Task Susi_B2C_LocalAccount_TodoAppFunctionsCorrectlyAsync() { // Start the web app and api processes. // The delay before starting client prevents transient devbox issue where the client fails to load the first time after rebuilding. - serviceProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListServicePath, TC.s_todoListServiceExe, serviceEnvVars); + serviceProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListServicePath, TC.s_todoListServiceExe, _output, serviceEnvVars); await Task.Delay(3000); - clientProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListClientPath, TC.s_todoListClientExe, clientEnvVars, 5); + clientProcess = UiTestHelpers.StartProcessLocally(_testAssemblyPath, _devAppPath + TC.s_todoListClientPath, TC.s_todoListClientExe, _output, clientEnvVars, 5); if (!UiTestHelpers.ProcessesAreAlive([clientProcess, serviceProcess])) { diff --git a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs index 91ba12984..4959fa83f 100644 --- a/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs +++ b/tests/E2E Tests/WebAppUiTests/TestingWebAppLocally.cs @@ -76,7 +76,7 @@ private async Task ExecuteWebAppCallsGraphFlowAsync(string upn, string credentia try { - process = UiTestHelpers.StartProcessLocally(_uiTestAssemblyLocation, _devAppPath, _devAppExecutable, clientEnvVars); + process = UiTestHelpers.StartProcessLocally(_uiTestAssemblyLocation, _devAppPath, _devAppExecutable, _output, clientEnvVars); if (!UiTestHelpers.ProcessIsAlive(process)) { Assert.Fail(TC.WebAppCrashedString); } diff --git a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs index 02dc7693d..403e09e5b 100644 --- a/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs +++ b/tests/E2E Tests/WebAppUiTests/UiTestHelpers.cs @@ -13,7 +13,6 @@ using Azure.Core; using Azure.Security.KeyVault.Secrets; using Microsoft.Identity.Web.Test.Common; -using Microsoft.IdentityModel.Tokens; using Microsoft.Playwright; using Xunit.Abstractions; @@ -148,6 +147,7 @@ public static Process StartProcessLocally( string testAssemblyLocation, string appLocation, string executableName, + ITestOutputHelper output, Dictionary? environmentVariables = null, int maxRetries = 0) { @@ -171,7 +171,7 @@ public static Process StartProcessLocally( Process? process; do { - Thread.Sleep(1000 * currentAttempt++); // Exponential backoff + Thread.Sleep(1000 * currentAttempt++); // linear backoff process = Process.Start(processStartInfo); } while (currentAttempt++ <= maxRetries && ProcessIsAlive(process)); @@ -182,8 +182,8 @@ public static Process StartProcessLocally( else { // Log the output and error streams - process.OutputDataReceived += (sender, e) => Console.WriteLine(e.Data); - process.ErrorDataReceived += (sender, e) => Console.Error.WriteLine(e.Data); + process.OutputDataReceived += (sender, e) => output.WriteLine(e.Data); + process.ErrorDataReceived += (sender, e) => output.WriteLine(e.Data); process.BeginOutputReadLine(); process.BeginErrorReadLine(); @@ -329,7 +329,7 @@ internal static async Task GetValueFromKeyvaultWitDefaultCredsAsync(Uri return (await client.GetSecretAsync(keyvaultSecretName)).Value.Value; } - internal static bool StartAndVerifyProcessesAreRunning(List processDataEntries, out Dictionary processes) + internal static bool StartAndVerifyProcessesAreRunning(List processDataEntries, ITestOutputHelper output, out Dictionary processes) { processes = new Dictionary(); @@ -340,6 +340,7 @@ internal static bool StartAndVerifyProcessesAreRunning(List processDataEntry.TestAssemblyLocation, processDataEntry.AppLocation, processDataEntry.ExecutableName, + output, processDataEntry.EnvironmentVariables); processes.Add(processDataEntry.ExecutableName, process); @@ -351,7 +352,7 @@ internal static bool StartAndVerifyProcessesAreRunning(List { if (!UiTestHelpers.ProcessesAreAlive(processes.Values.ToList())) { - RestartProcesses(processes, processDataEntries); + RestartProcesses(processes, processDataEntries , output); } } @@ -363,7 +364,7 @@ internal static bool StartAndVerifyProcessesAreRunning(List return true; } - static void RestartProcesses(Dictionary processes, List processDataEntries) + static void RestartProcesses(Dictionary processes, List processDataEntries, ITestOutputHelper output) { //attempt to restart failed processes foreach (KeyValuePair processEntry in processes) @@ -375,6 +376,7 @@ static void RestartProcesses(Dictionary processes, List { /*grpcProcessOptions,*/ serviceProcessOptions, clientProcessOptions }, out processes); + bool areProcessesRunning = UiTestHelpers.StartAndVerifyProcessesAreRunning(new List { /*grpcProcessOptions,*/ serviceProcessOptions, clientProcessOptions }, _output, out processes); if (!areProcessesRunning) { @@ -219,7 +217,7 @@ public async Task ChallengeUser_MicrosoftIdFlow_LocalApp_ValidEmailPasswordCreds // The delay before starting client prevents transient devbox issue where the client fails to load the first time after rebuilding. var serviceProcessOptions = new ProcessStartOptions(_testAssemblyLocation, _devAppPathCiam + TC.s_myWebApiPath, TC.s_myWebApiExe, serviceEnvVars); var clientProcessOptions = new ProcessStartOptions(_testAssemblyLocation, _devAppPathCiam + TC.s_myWebAppPath, TC.s_myWebAppExe, clientEnvVars); - bool areProcessesRunning = UiTestHelpers.StartAndVerifyProcessesAreRunning(new List { serviceProcessOptions, clientProcessOptions }, out processes); + bool areProcessesRunning = UiTestHelpers.StartAndVerifyProcessesAreRunning(new List { serviceProcessOptions, clientProcessOptions }, _output, out processes); if (!areProcessesRunning) {