Merge pull request #7127 from Sage/playwright-pipeline-investigation

perf(pipeline): flaky Playwright investigations
Sage · Dec 17, 2024 · 4359844 · 4359844
2 parents 4b63924 + 00c12d5
commit 4359844
Show file tree

Hide file tree

Showing 8 changed files with 481 additions and 294 deletions.
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -12,7 +12,7 @@ jobs:
     timeout-minutes: 30
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/playwright:v1.47.0-jammy
+      image: mcr.microsoft.com/playwright:v1.49.1-noble
     strategy:
       fail-fast: false
       matrix:

diff --git a/playwright-ct.config.ts b/playwright-ct.config.ts
@@ -1,4 +1,5 @@
 import { defineConfig, devices } from "@playwright/experimental-ct-react17";
+
 import { resolve } from "path";
 
 const playwrightDir = resolve(__dirname, "./playwright");
@@ -30,7 +31,7 @@ export default defineConfig({
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: "on-first-retry",
+    trace: "retain-on-failure",
     /* Port to use for Playwright component endpoint. */
     ctPort: 3100,
     /* Custom config for internal bundler Playwright uses for component tests. See https://playwright.dev/docs/test-components#under-the-hood */

diff --git a/src/components/split-button/split-button.pw.tsx b/src/components/split-button/split-button.pw.tsx
@@ -580,6 +580,8 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<WithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
     await expect(additionalButton(page, 0)).toBeVisible();
     await expect(additionalButton(page, 1)).toBeVisible();
     await expect(additionalButton(page, 2)).toBeVisible();
@@ -592,14 +594,20 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
-    await expect(additionalButton(page, 0)).toBeFocused();
-    await page.keyboard.press("Tab");
-    await expect(additionalButton(page, 1)).toBeFocused();
-    await page.keyboard.press("Tab");
-    await expect(additionalButton(page, 2)).toBeFocused();
-    await page.keyboard.press("Tab");
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
 
-    await expect(additionalButton(page, 0)).not.toBeVisible();
+    const firstButton = buttonList.getByRole("button").first();
+    const secondButton = buttonList.getByRole("button").nth(1);
+    const thirdButton = buttonList.getByRole("button").last();
+
+    await expect(firstButton).toBeFocused();
+    await firstButton.press("Tab");
+    await expect(secondButton).toBeFocused();
+    await secondButton.press("Tab");
+    await expect(thirdButton).toBeFocused();
+    await thirdButton.press("Tab");
+    await expect(buttonList).toBeHidden();
     await expect(mainButton(page).nth(1)).toBeFocused();
   });
 
@@ -610,7 +618,9 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
-    await page.waitForTimeout(1000);
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
+
     await page.keyboard.press("ArrowDown");
     await page.keyboard.press("ArrowDown");
     await page.keyboard.press("ArrowDown");
@@ -625,10 +635,13 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
+
     await additionalButton(page, 1).focus();
-    await page.keyboard.press("Shift+Tab");
+    await page.keyboard.press("Shift+Tab", { delay: 1000 });
     await expect(additionalButton(page, 0)).toBeFocused();
-    await page.keyboard.press("Shift+Tab");
+    await page.keyboard.press("Shift+Tab", { delay: 1000 });
 
     await expect(additionalButton(page, 0)).not.toBeVisible();
     await expect(splitToggleButton(page).nth(0)).toBeFocused();
@@ -641,12 +654,14 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
     await additionalButton(page, 2).focus();
-    await page.waitForTimeout(1000);
+
     await page.keyboard.press("ArrowUp");
     await page.keyboard.press("ArrowUp");
     await page.keyboard.press("ArrowUp");
-    await page.waitForTimeout(1000);
+
     await expect(additionalButton(page, 0)).toBeFocused();
   });
 
@@ -657,6 +672,8 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
     await additionalButton(page, 2).focus();
     await page.keyboard.down("Meta");
     await page.keyboard.press("ArrowUp");
@@ -670,6 +687,8 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
     await additionalButton(page, 2).focus();
     await page.keyboard.down("Control");
     await page.keyboard.press("ArrowUp");
@@ -695,6 +714,7 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<TwoButtonsWithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+
     await page.keyboard.down("Meta");
     await page.keyboard.press("ArrowDown");
     await expect(additionalButton(page, 2)).toBeFocused();
@@ -730,6 +750,7 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<WithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+
     await additionalButton(page, 0).click();
     await expect(additionalButtonsContainer(page)).toHaveCount(0);
   });
@@ -741,6 +762,8 @@ test.describe("Functional tests in a custom component", () => {
     await mount(<WithWrapper />);
 
     await splitToggleButton(page).nth(0).click();
+    const buttonList = page.getByRole("list");
+    await buttonList.waitFor();
     await additionalButton(page, 1).focus();
     await page.keyboard.press("Escape");
     await expect(additionalButtonsContainer(page)).toHaveCount(0);

diff --git a/src/components/tabs/tabs.pw.tsx b/src/components/tabs/tabs.pw.tsx
@@ -454,121 +454,6 @@ test.describe("Tabs component", () => {
     });
   });
 
-  test.describe("check events for Tabs component", () => {
-    test.describe("when position is top", () => {
-      // TODO: Skipped due to flaky focus behaviour. To review in FE-6428
-      test.skip("should call onTabChange callback when click event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsComponent
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).click();
-        expect(callbackCount).toBe(1);
-      });
-
-      test("should call onTabChange callback when enter key event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsComponent
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).press("Enter");
-        expect(callbackCount).toBe(1);
-      });
-    });
-
-    test.describe("when position is left", () => {
-      // TODO: Skipped due to flaky focus behaviour. To review in FE-6428
-      test.skip("should call onTabChange callback when click event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsComponent
-            position="left"
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).click();
-        expect(callbackCount).toBe(1);
-      });
-
-      test("should call onTabChange callback when enter key event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsComponent
-            position="left"
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).press("Enter");
-        expect(callbackCount).toBe(1);
-      });
-    });
-
-    test.describe("when in Sidebar", () => {
-      // TODO: Skipped due to flaky focus behaviour. To review in FE-6428
-      test.skip("should call onTabChange callback when click event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsInSidebar
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).click();
-        expect(callbackCount).toBe(1);
-      });
-
-      test("should call onTabChange callback when enter key event is triggered", async ({
-        mount,
-        page,
-      }) => {
-        let callbackCount = 0;
-        await mount(
-          <TabsInSidebar
-            onTabChange={() => {
-              callbackCount += 1;
-            }}
-          />,
-        );
-
-        await tabById(page, 2).press("Enter");
-        expect(callbackCount).toBe(1);
-      });
-    });
-  });
-
   test.describe("Accessibility tests for Tabs component", () => {
     test("should pass accessibility tests for Tabs component", async ({
       mount,