From a4d89b0d4d4ae24a94fad5e323a6599907313884 Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Mon, 3 Jun 2024 14:21:14 +0200 Subject: [PATCH 1/2] fix: e2e long loading --- tests/e2e/compare/compare.ts | 6 +++--- tests/e2e/config.ts | 4 ++++ tests/e2e/testRunner.ts | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/e2e/compare/compare.ts b/tests/e2e/compare/compare.ts index 6cd1feec47b5..c30900526bc2 100644 --- a/tests/e2e/compare/compare.ts +++ b/tests/e2e/compare/compare.ts @@ -50,7 +50,7 @@ function buildCompareEntry(name: string, compare: Stats, baseline: Stats): Entry /** * Compare results between baseline and current entries and categorize. */ -function compareResults(compareEntries: Metric | string, baselineEntries: Metric | string) { +function compareResults(baselineEntries: Metric | string, compareEntries: Metric | string = baselineEntries) { // Unique test scenario names const baselineKeys = Object.keys(baselineEntries ?? {}); const names = Array.from(new Set([...baselineKeys])); @@ -81,8 +81,8 @@ function compareResults(compareEntries: Metric | string, baselineEntries: Metric } export default (main: Metric | string, delta: Metric | string, outputFile: string, outputFormat = 'all') => { - // IMPORTANT NOTE: make sure you are passing the delta/compare results first, then the main/baseline results: - const outputData = compareResults(delta, main); + // IMPORTANT NOTE: make sure you are passing the main/baseline results first, then the delta/compare results: + const outputData = compareResults(main, delta); if (outputFormat === 'console' || outputFormat === 'all') { printToConsole(outputData); diff --git a/tests/e2e/config.ts b/tests/e2e/config.ts index 6fe88d45c2e9..d453a8f8ff2c 100644 --- a/tests/e2e/config.ts +++ b/tests/e2e/config.ts @@ -73,6 +73,7 @@ export default { }, [TEST_NAMES.OpenChatFinderPage]: { name: TEST_NAMES.OpenChatFinderPage, + requiresDoubleWarmup: true, }, // TODO: Fix text and enable again // [TEST_NAMES.ReportTyping]: { @@ -82,11 +83,13 @@ export default { // }, // // Crowded Policy (Do Not Delete) Report, has a input bar available: // reportID: '8268282951170052', + // requiresDoubleWarmup: true, // }, [TEST_NAMES.ChatOpening]: { name: TEST_NAMES.ChatOpening, // #announce Chat with many messages reportID: '5421294415618529', + requiresDoubleWarmup: true, }, // TODO: fix and enable again // [TEST_NAMES.Linking]: { @@ -98,6 +101,7 @@ export default { // reportID: '8268282951170052', // linkedReportID: '5421294415618529', // linkedReportActionID: '2845024374735019929', + // requiresDoubleWarmup: true, // }, }, }; diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index ee8cf45df191..f678c7df8b60 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -156,6 +156,10 @@ const runTests = async (): Promise => { for (let testIndex = 0; testIndex < tests.length; testIndex++) { const test = Object.values(config.TESTS_CONFIG)[testIndex]; + // re-instal app for each new test suite + await installApp(config.MAIN_APP_PACKAGE, mainAppPath); + await installApp(config.DELTA_APP_PACKAGE, deltaAppPath); + // check if we want to skip the test if (args.includes('--includes')) { const includes = args[args.indexOf('--includes') + 1]; @@ -177,11 +181,15 @@ const runTests = async (): Promise => { const warmupText = `Warmup for test '${test.name}' [${testIndex + 1}/${tests.length}]`; - // Warmup the main app: - await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}`); + const requiresDoubleWarmup = test.requiresDoubleWarmup ?? false; + const iterations = requiresDoubleWarmup ? 2 : 1; + for (let i = 0; i < iterations; i++) { + // Warmup the main app: + await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`); - // Warmup the delta app: - await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}`); + // Warmup the delta app: + await runTestIteration(config.DELTA_APP_PACKAGE, `[DELTA] ${warmupText}. Iteration ${i + 1}/${iterations}`); + } // For each test case we allow the test to fail three times before we stop the test run: const errorCountRef = { From 50ad7b8f2d142e46637ea7f45032d9061bbd0076 Mon Sep 17 00:00:00 2001 From: kirillzyusko Date: Mon, 3 Jun 2024 15:23:54 +0200 Subject: [PATCH 2/2] chore: changes after review --- tests/e2e/config.ts | 6 +----- tests/e2e/testRunner.ts | 6 ++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/e2e/config.ts b/tests/e2e/config.ts index d453a8f8ff2c..4120019de692 100644 --- a/tests/e2e/config.ts +++ b/tests/e2e/config.ts @@ -68,12 +68,11 @@ export default { TESTS_CONFIG: { [TEST_NAMES.AppStartTime]: { name: TEST_NAMES.AppStartTime, - + warmupRuns: 1, // ... any additional config you might need }, [TEST_NAMES.OpenChatFinderPage]: { name: TEST_NAMES.OpenChatFinderPage, - requiresDoubleWarmup: true, }, // TODO: Fix text and enable again // [TEST_NAMES.ReportTyping]: { @@ -83,13 +82,11 @@ export default { // }, // // Crowded Policy (Do Not Delete) Report, has a input bar available: // reportID: '8268282951170052', - // requiresDoubleWarmup: true, // }, [TEST_NAMES.ChatOpening]: { name: TEST_NAMES.ChatOpening, // #announce Chat with many messages reportID: '5421294415618529', - requiresDoubleWarmup: true, }, // TODO: fix and enable again // [TEST_NAMES.Linking]: { @@ -101,7 +98,6 @@ export default { // reportID: '8268282951170052', // linkedReportID: '5421294415618529', // linkedReportActionID: '2845024374735019929', - // requiresDoubleWarmup: true, // }, }, }; diff --git a/tests/e2e/testRunner.ts b/tests/e2e/testRunner.ts index f678c7df8b60..fcf93d525f8e 100644 --- a/tests/e2e/testRunner.ts +++ b/tests/e2e/testRunner.ts @@ -181,8 +181,10 @@ const runTests = async (): Promise => { const warmupText = `Warmup for test '${test.name}' [${testIndex + 1}/${tests.length}]`; - const requiresDoubleWarmup = test.requiresDoubleWarmup ?? false; - const iterations = requiresDoubleWarmup ? 2 : 1; + // by default we do 2 warmups: + // - first warmup to pass a login flow + // - second warmup to pass an actual flow and cache network requests + const iterations = test.warmupRuns ?? 2; for (let i = 0; i < iterations; i++) { // Warmup the main app: await runTestIteration(config.MAIN_APP_PACKAGE, `[MAIN] ${warmupText}. Iteration ${i + 1}/${iterations}`);