fix e2e test flakiness due to workshop page fixture

end-to-end-tests/fixtures/modDefinitions.ts

@@ -70,7 +70,6 @@ export const test = pageContextFixture.extend<{
   async _workshopPage({ context, extensionId }, use) {
     const newPage = await context.newPage();
     const workshopPage = new WorkshopPage(newPage, extensionId);
-    await workshopPage.goto();
     await use(workshopPage);
     await newPage.close();
   },
@@ -121,6 +120,8 @@ export const test = pageContextFixture.extend<{
       mode?: "diff" | "current";
       prevModId?: string;
     }) => {
+      // Reload to ensure the latest mod definitions are fetched.
+      await workshopPage.reload();
       await workshopPage.goto();
       const editPage = await workshopPage.findAndSelectMod(modId);
 

end-to-end-tests/pageObjects/extensionConsole/modsPage.ts

@@ -18,7 +18,6 @@
 import { expect, type Page } from "@playwright/test";
 import { getBaseExtensionConsoleUrl } from "../constants";
 import { BasePageObject } from "../basePageObject";
-import { ensureVisibility } from "../../utils";
 import { validateRegistryId } from "@/types/helpers";
 import { API_PATHS, UI_PATHS } from "@/data/service/urlPaths";
 import { DEFAULT_TIMEOUT } from "../../../playwright.config";
@@ -169,12 +168,19 @@ export class ActivateModPage extends BasePageObject {
 
   async goto() {
     await this.page.goto(this.activateModUrl);
-
-    await expect(
-      this.getByRole("heading", { name: "Activate " }),
-    ).toBeVisible();
-    // Loading the mod details may take a long time. Using ensureVisibility because the modId may be attached and hidden
-    await ensureVisibility(this.getByText(this.modId));
+    // Wrapped in toPass due to flakiness with the page not loading ex:
+    // https://github.com/pixiebrix/pixiebrix-extension/actions/runs/11373118427?pr=9286
+    await expect(async () => {
+      await this.getByRole("heading", { name: "Activate " }).waitFor({
+        timeout: 10_000,
+      });
+      try {
+        await this.getByText(this.modId).waitFor({ timeout: 10_000 });
+      } catch (error) {
+        await this.page.reload();
+        throw error;
+      }
+    }).toPass({ timeout: 30_000 });
   }
 
   async getIntegrationConfigField(index: number) {

end-to-end-tests/pageObjects/extensionConsole/workshop/workshopPage.ts

@@ -34,10 +34,9 @@ export class WorkshopPage extends BasePageObject {
   }
 
   async goto() {
-    await this.page.goto(this.extensionConsoleUrl);
-    await this.getByRole("link", {
-      name: "Workshop",
-    }).click();
+    await this.page.goto(`${this.extensionConsoleUrl}#/workshop`);
+    await this.getByRole("heading", { name: "Workshop" }).waitFor();
+    await this.getByRole("cell").first().waitFor();
   }
 
   async findAndSelectMod(modId: string) {

end-to-end-tests/tests/pageEditor/addStarterBrick.spec.ts

@@ -19,11 +19,7 @@ import { test, expect } from "../../fixtures/testBase";
 
 // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only
 import { type Page, test as base } from "@playwright/test";
-import {
-  getSidebarPage,
-  isMsEdge,
-  PRE_RELEASE_BROWSER_WORKFLOW_NAME,
-} from "../../utils";
+import { getSidebarPage, isMsEdge } from "../../utils";
 
 test("Add new mod with different starter brick components", async ({
   page,
@@ -95,6 +91,11 @@ test("Add new mod with different starter brick components", async ({
   });
 
   await test.step("Add new Sidebar Panel starter brick", async () => {
+    // eslint-disable-next-line playwright/no-conditional-in-test -- MSedge won't open the sidebar unless the target page is focused
+    if (isMsEdge(chromiumChannel)) {
+      await page.bringToFront();
+    }
+
     const { modComponentNameMatcher } =
       await pageEditorPage.modListingPanel.addNewMod({
         starterBrickName: "Sidebar Panel",
@@ -108,12 +109,8 @@ test("Add new mod with different starter brick components", async ({
       }),
     ).toHaveValue(modComponentNameMatcher);
 
-    /* eslint-disable playwright/no-conditional-in-test, playwright/no-conditional-expect -- Edge bug, see https://github.com/pixiebrix/pixiebrix-extension/issues/9011 */
-    if (!isMsEdge(chromiumChannel)) {
-      const sidebarPage = await getSidebarPage(page, extensionId);
-      await expect(sidebarPage.getByText("Example Document")).toBeVisible();
-    }
-    /* eslint-enable playwright/no-conditional-in-test, playwright/no-conditional-expect */
+    const sidebarPage = await getSidebarPage(page, extensionId);
+    await expect(sidebarPage.getByText("Example Document")).toBeVisible();
   });
 
   await test.step("Add new Trigger starter brick", async () => {
@@ -205,7 +202,7 @@ test("Add starter brick to mod", async ({
     ).toHaveValue("My pbx.vercel.app button");
 
     await modListItem.select();
-    await modListItem.saveButton.click();
+    await pageEditorPage.saveActiveMod();
     await verifyModDefinitionSnapshot({
       modId,
       snapshotName: "add-button-starter-brick-to-mod",
@@ -226,7 +223,7 @@ test("Add starter brick to mod", async ({
     ).toHaveValue("Context menu item");
 
     await modListItem.select();
-    await modListItem.saveButton.click();
+    await pageEditorPage.saveActiveMod();
     await verifyModDefinitionSnapshot({
       modId,
       snapshotName: "add-context-menu-starter-brick-to-mod",
@@ -247,7 +244,7 @@ test("Add starter brick to mod", async ({
     ).toHaveValue("Quick Bar item");
 
     await modListItem.select();
-    await modListItem.saveButton.click();
+    await pageEditorPage.saveActiveMod();
     await verifyModDefinitionSnapshot({
       modId,
       snapshotName: "add-quick-bar-starter-brick-to-mod",
@@ -256,6 +253,11 @@ test("Add starter brick to mod", async ({
   });
 
   await test.step("Add Sidebar Panel starter brick to mod", async () => {
+    // eslint-disable-next-line playwright/no-conditional-in-test -- MSedge won't open the sidebar unless the target page is focused
+    if (isMsEdge(chromiumChannel)) {
+      await page.bringToFront();
+    }
+
     const modActionMenu = await openModActionMenu();
     await modActionMenu.addStarterBrick("Sidebar Panel");
 
@@ -272,7 +274,7 @@ test("Add starter brick to mod", async ({
     await expect(sidebarPage.getByText("Example Document")).toBeVisible();
 
     await modListItem.select();
-    await modListItem.saveButton.click();
+    await pageEditorPage.saveActiveMod();
     await verifyModDefinitionSnapshot({
       modId,
       snapshotName: "add-sidebar-panel-starter-brick-to-mod",
@@ -281,12 +283,6 @@ test("Add starter brick to mod", async ({
   });
 
   await test.step("Add Trigger starter brick to mod", async () => {
-    test.fixme(
-      process.env.GITHUB_WORKFLOW === PRE_RELEASE_BROWSER_WORKFLOW_NAME &&
-        isMsEdge(chromiumChannel),
-      "Skipping test for MS Edge in pre-release workflow, see https://github.com/pixiebrix/pixiebrix-extension/issues/9125",
-    );
-
     const modActionMenu = await openModActionMenu();
     await modActionMenu.addStarterBrick("Trigger");
 
@@ -299,7 +295,7 @@ test("Add starter brick to mod", async ({
     ).toHaveValue("My pbx.vercel.app trigger");
 
     await modListItem.select();
-    await modListItem.saveButton.click();
+    await pageEditorPage.saveActiveMod();
     await verifyModDefinitionSnapshot({
       modId,
       snapshotName: "add-trigger-starter-brick-to-mod",

end-to-end-tests/tests/pageEditor/copyMod.spec.ts

@@ -23,12 +23,7 @@ import { uuidv4 } from "@/types/helpers";
 import { type Serializable } from "playwright-core/types/structs";
 import path from "node:path";
 import { FloatingActionButton } from "end-to-end-tests/pageObjects/floatingActionButton";
-import {
-  getSidebarPage,
-  runModViaQuickBar,
-  isMsEdge,
-  PRE_RELEASE_BROWSER_WORKFLOW_NAME,
-} from "end-to-end-tests/utils";
+import { getSidebarPage, runModViaQuickBar } from "end-to-end-tests/utils";
 import { VALID_UUID_REGEX } from "@/types/stringTypes";
 
 test("copying a mod that uses the PixieBrix API is copied correctly", async ({
@@ -86,14 +81,7 @@ test("run a copied mod with a built-in integration", async ({
   context,
   newPageEditorPage,
   verifyModDefinitionSnapshot,
-  chromiumChannel,
 }) => {
-  test.fixme(
-    process.env.GITHUB_WORKFLOW === PRE_RELEASE_BROWSER_WORKFLOW_NAME &&
-      isMsEdge(chromiumChannel),
-    "Skipping test for MS Edge in pre-release workflow, see https://github.com/pixiebrix/pixiebrix-extension/issues/9125",
-  );
-
   let giphyRequestPostData: Serializable;
   // The giphy search request is proxied through the PixieBrix server, which is kicked off in the background/service
   // worker. Playwright experimentally supports mocking service worker requests, see

end-to-end-tests/tests/pageEditor/liveEditing.spec.ts

@@ -20,7 +20,6 @@ import { expect, test } from "../../fixtures/testBase";
 import { test as base } from "@playwright/test";
 import { ActivateModPage } from "../../pageObjects/extensionConsole/modsPage";
 import { getSidebarPage, isMsEdge, isSidebarOpen } from "../../utils";
-import { FloatingActionButton } from "../../pageObjects/floatingActionButton";
 
 test("live editing behavior", async ({
   page,
@@ -67,6 +66,14 @@ test("live editing behavior", async ({
   });
 
   await test.step("Verify render panel functionality", async () => {
+    /* eslint-disable-next-line playwright/no-conditional-in-test -- MS Edge has a bug where the page editor
+     * cannot open the sidebar, unless the target page is already focused.
+     * https://www.loom.com/share/fbad85e901794161960b737b27a13677
+     */
+    if (isMsEdge(chromiumChannel)) {
+      await page.bringToFront();
+    }
+
     await pageEditor.modListingPanel
       .getModStarterBrick("Live Editing Test", "Sidebar Panel")
       .select();
@@ -75,16 +82,6 @@ test("live editing behavior", async ({
 
     expect(isSidebarOpen(page, extensionId)).toBe(false);
 
-    /* eslint-disable-next-line playwright/no-conditional-in-test -- MS Edge has a bug where the page editor
-     * cannot open the sidebar, so we need to open it manually.
-     * https://www.loom.com/share/fbad85e901794161960b737b27a13677
-     */
-    if (isMsEdge(chromiumChannel)) {
-      await page.bringToFront();
-      const floatingActionButton = new FloatingActionButton(page);
-      await floatingActionButton.toggleSidebar();
-    }
-
     await pageEditor.editorPane.renderPanelButton.click();
     const sidebar = await getSidebarPage(page, extensionId);
     await expect(

end-to-end-tests/tests/regressions/doNotCloseSidebarOnPageEditorSave.spec.ts

@@ -19,7 +19,6 @@ import { test, expect } from "../../fixtures/testBase";
 // @ts-expect-error -- https://youtrack.jetbrains.com/issue/AQUA-711/Provide-a-run-configuration-for-Playwright-tests-in-specs-with-fixture-imports-only
 import { type Page, test as base } from "@playwright/test";
 import { getSidebarPage, isMsEdge } from "../../utils";
-import { FloatingActionButton } from "../../pageObjects/floatingActionButton";
 
 test("#8104: Do not automatically close the sidebar when saving in the Page Editor", async ({
   page,
@@ -30,20 +29,18 @@ test("#8104: Do not automatically close the sidebar when saving in the Page Edit
   await page.goto("/");
   const pageEditorPage = await newPageEditorPage(page.url());
 
-  await pageEditorPage.modListingPanel.addNewMod({
-    starterBrickName: "Sidebar Panel",
-  });
-
   /* eslint-disable-next-line playwright/no-conditional-in-test -- MS Edge has a bug where the page editor
-   * cannot open the sidebar, so we need to open it manually.
+   * cannot open the sidebar unless it is already focused.
    * https://www.loom.com/share/fbad85e901794161960b737b27a13677
    */
   if (isMsEdge(chromiumChannel)) {
     await page.bringToFront();
-    const floatingActionButton = new FloatingActionButton(page);
-    await floatingActionButton.toggleSidebar();
   }
 
+  await pageEditorPage.modListingPanel.addNewMod({
+    starterBrickName: "Sidebar Panel",
+  });
+
   const sidebar = await getSidebarPage(page, extensionId);
   await expect(
     sidebar.getByRole("tab", { name: "Sidebar Panel" }),

playwright.config.ts

@@ -98,14 +98,15 @@ export default defineConfig<{ chromiumChannel: string }>({
     ["json", { outputFile: "./end-to-end-tests/.report/report.json" }],
   ],
   // /* Repeat each test N times. Useful for catching flaky test. */
-  // repeatEach: 6,
+  // repeatEach: 3,
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Base URL to use in actions like `await page.goto('/')`. */
     baseURL: "https://pbx.vercel.app",
 
     /* Collect trace when retrying the failed test in CI, and always on failure when running locally. See https://playwright.dev/docs/trace-viewer */
-    trace: CI ? "on-first-retry" : "retain-on-failure",
+    // trace: CI ? "on-first-retry" : "retain-on-failure",
+    trace: "retain-on-failure",
 
     /* Set the default timeout for actions such as `click` */
     actionTimeout: DEFAULT_TIMEOUT,