ensure test run from the IDE

.github/workflows/tests.yml

@@ -61,7 +61,7 @@ jobs:
                   path: playwright-report/
                   retention-days: 30
             - if: ${{ failure() && steps.screenshot_tests.conclusion == 'failure' }}
-              run: npm run test.int -- --grep @screenshots --update-snapshots
+              run: npm run test.int -- --grep @screenshots --update-snapshots --last-failed
             - if: ${{ failure() && steps.screenshot_tests.conclusion == 'failure' }}
               name: Upload screens
               uses: actions/upload-artifact@v4

.github/workflows/update-snapshots.yml

@@ -1,40 +1,75 @@
 name: Update Snapshots on a PR
 
 on:
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: "Pull Request Number"
-        required: true
-        type: string
+    workflow_dispatch:
+        inputs:
+            pr_number:
+                description: 'Pull Request Number (Warning: This action will push a commit to the referenced PR)'
+                required: true
+                type: string
 
 permissions:
-  pull-requests: write
-  contents: write
+    pull-requests: write
+    contents: write
 
 jobs:
-  update-pr:
-    name: Update PR Snapshots
-    runs-on: macos-latest
-    steps:
-      - name: Checkout PR
-        if: github.event_name == 'workflow_dispatch'
-        run: gh pr checkout ${{ github.event.inputs.pr_number }}
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Run command
-        if: github.event_name == 'workflow_dispatch'
-        run: |
-          echo "Running A command here command..."
-
-      # Step 5: Commit and push changes back to the PR branch
-#      - name: Commit and push changes
-#        env:
-#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#        run: |
-#          git config user.name "github-actions[bot]"
-#          git config user.email "github-actions[bot]@users.noreply.github.com"
-#          git add .
-#          git commit -m "Automated updates from GitHub Actions"
-#          git push origin ${{ env.PR_BRANCH }}
+    update-pr:
+        name: Update PR Snapshots
+        runs-on: macos-latest
+        steps:
+            - uses: actions/checkout@v3
+            - name: Checkout PR ${{ github.event.inputs.pr_number }}
+              if: github.event_name == 'workflow_dispatch'
+              run: gh pr checkout ${{ github.event.inputs.pr_number }}
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Use Node.js from .nvmrc
+              uses: actions/setup-node@v3
+              with:
+                  node-version-file: '.nvmrc'
+
+            - name: Cache node modules
+              id: cache-npm
+              uses: actions/cache@v3
+              env:
+                  cache-name: cache-node-modules
+              with:
+                  path: node_modules
+                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+
+            - name: Install dependencies
+              run: npm ci
+
+            - name: Build all
+              run: npm run build && npm run build.debug
+
+            - name: Fetch fonts
+              run: npm run fetch-fonts
+
+            - name: Install Playwright Browsers
+              run: npx playwright install --with-deps
+
+            - name: Run Screenshot tests
+              id: screenshot_tests
+              run: npm run test.int -- screenshots.js --grep @screenshots
+
+            - if: ${{ failure() && steps.screenshot_tests.conclusion == 'success' }}
+              run: |
+                  echo "nothing to update - tests all passed"
+
+            - name: Re-Running Playwright to update snapshots
+              id: screenshot_tests_update
+              if: ${{ failure() && steps.screenshot_tests.conclusion == 'failure' }}
+              run: npm run test.int -- screenshots.js --grep @screenshots --update-snapshots --last-failed
+
+            - name: Commit the updated files to the PR branch
+              if: ${{ failure() && steps.screenshot_tests_update.conclusion == 'success' }}
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: |
+                  git config user.name "github-actions[bot]"
+                  git config user.email "github-actions[bot]@users.noreply.github.com"
+                  git add integration-tests/*
+                  git commit -m "Updated snapshots via workflow"
+                  git push origin HEAD

.gitignore

@@ -19,3 +19,4 @@ build/
 build/app_backup
 bslive.yml
 .vscode
+/json-reports

integration-tests/DashboardPage.js

@@ -2,7 +2,7 @@ import { expect } from '@playwright/test';
 import { forwardConsole, playTimeline } from './helpers';
 import { Mocks } from './Mocks';
 import { Nav } from './Nav';
-import { testDataStates } from '../shared/js/ui/views/tests/states-with-fixtures';
+import { testDataStates } from './utils/states-with-fixtures';
 import { mockBrowserApis } from '../shared/js/browser/utils/communication-mocks.mjs';
 import { Extension } from './Extension';
 import toggleReportScreen from '../schema/__fixtures__/toggle-report-screen.json';
@@ -490,7 +490,8 @@ export class DashboardPage {
      * @param {{ telemetry?: boolean }} [options]
      */
     async selectsCategory(text, category, options = { telemetry: false }) {
-        await this.page.getByLabel(text).click();
+        // await this.page.pause();
+        await this.page.getByLabel(text).click({ timeout: 5000 });
 
         if (options.telemetry) {
             await this.mocks.didSendTelemetry({
@@ -831,6 +832,21 @@ export class DashboardPage {
         await this.page.getByText('Submitting an anonymous').waitFor({ timeout: 5000 });
     }
 
+    // todo: remove
+    async waitForRouterToSettle() {
+        const { page } = this;
+        // wait for it to start animating
+        await page.waitForFunction(() => {
+            const element = document.getElementById('popup-container');
+            return element && element.classList.contains('sliding-subview-v2--animating');
+        });
+        // then wait for it to finish
+        await page.waitForFunction(() => {
+            const element = document.getElementById('popup-container');
+            return element && !element.classList.contains('sliding-subview-v2--animating');
+        });
+    }
+
     async skipsToBreakageFormWhenDisliked() {
         await this.page.getByLabel('I dislike the content on this').click();
         await this.page.getByRole('button', { name: 'Send Report' }).click();

integration-tests/Extension.js

@@ -32,7 +32,7 @@ export class Extension {
         await page.evaluate(() => {
             window.__playwright.onDisconnect?.();
         });
-        const callCountAfter = await this.dash.mocks.outgoing({ names: ['getToggleReportOptions'] });
+        const callCountAfter = await this.dash.mocks.waitFor({ name: 'getToggleReportOptions', count: 2 });
         expect(callCountAfter).toHaveLength(2);
     }
 

integration-tests/Mocks.js

@@ -65,6 +65,24 @@ export class Mocks {
         return values;
     }
 
+    /**
+     * @param {{name: string, count: number}} opts
+     * @returns {Promise<any[]>}
+     */
+    async waitFor(opts) {
+        await this.page.waitForFunction(
+            (opts) => {
+                const current = window.__playwright.mocks.outgoing;
+                return current.filter(([name]) => name === opts.name).length >= opts.count;
+            },
+            opts,
+            { timeout: 5000 }
+        );
+
+        const values = await this.page.evaluate(() => window.__playwright.mocks.outgoing);
+        return values.filter(([name]) => opts.name === name);
+    }
+
     async calledForShowBreakageForm() {
         // only on ios/android
         if (!['android', 'ios'].includes(this.platform.name)) return;

integration-tests/Nav.js

@@ -16,6 +16,7 @@ export class Nav {
 
     async goesBackToPrimaryScreenFromBreakageScreen() {
         const { page } = this.dash;
+        await page.getByRole('textbox', { name: 'Please describe the issue you' }).waitFor({ timeout: 1000 });
         await page.getByTestId('subview-breakageFormFinalStep').getByLabel('Back', { exact: true }).click();
         await page.getByTestId('subview-breakageFormCategorySelection').getByLabel('Back', { exact: true }).click();
         await page.getByTestId('subview-breakageForm').getByLabel('Back', { exact: true }).click();

integration-tests/android.spec-int.js

@@ -1,5 +1,5 @@
 import { test } from '@playwright/test';
-import { testDataStates } from '../shared/js/ui/views/tests/states-with-fixtures';
+import { testDataStates } from './utils/states-with-fixtures';
 import { DashboardPage } from './DashboardPage';
 import { toggleFlows } from './utils/common-flows';
 
@@ -197,6 +197,7 @@ test.describe('breakage form', () => {
     test('shows empty description warning', { tag: '@screenshots' }, async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.android(page, { screen: 'breakageForm' });
+        await dash.reducedMotion();
         await dash.addState([testDataStates.google]);
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Something else', 'other');
@@ -209,6 +210,7 @@ test.describe('breakage form', () => {
     test('submits form with description', { tag: '@screenshots' }, async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.android(page, { screen: 'breakageForm' });
+        await dash.reducedMotion();
         await dash.addState([testDataStates.google]);
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Something else', 'other');
@@ -274,8 +276,11 @@ test.describe('stack based router', () => {
         await dash.addState([testDataStates['webBreakageForm-enabled']]);
 
         await dash.clicksWebsiteNotWorking();
+        await dash.waitForRouterToSettle();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
+        await dash.waitForRouterToSettle();
         await dash.selectsCategory('Site layout broken', 'layout');
+        await dash.waitForRouterToSettle();
         await dash.nav.goesBackToPrimaryScreenFromBreakageScreen();
     });
     test('goes back and forward generally', async ({ page }) => {
@@ -360,6 +365,7 @@ test.describe('Android screenshots', { tag: '@screenshots' }, () => {
             test(name, async ({ page }) => {
                 await page.emulateMedia({ reducedMotion: 'reduce' });
                 const dash = await DashboardPage.android(page);
+                await dash.reducedMotion();
                 await dash.screenshotEachScreenForState(name, state);
             });
         }
@@ -374,6 +380,7 @@ test.describe('Android screenshots', { tag: '@screenshots' }, () => {
         for (const { name, state } of states) {
             test(name, async ({ page }) => {
                 const dash = await DashboardPage.android(page);
+                await dash.reducedMotion();
                 await dash.addState([state]);
                 await dash.showsPrimaryScreen();
                 await dash.screenshot(`${name}.png`);
@@ -400,6 +407,7 @@ test.describe('Android screenshots', { tag: '@screenshots' }, () => {
             });
             test('secondary screen', async ({ page }) => {
                 const dash = await DashboardPage.android(page);
+                await dash.reducedMotion();
                 await dash.addState([testDataStates['consent-managed-configurable']]);
                 await dash.viewCookiePromptManagement();
                 await dash.screenshot('consent-managed-configurable-secondary.png');
@@ -417,6 +425,7 @@ test.describe('Android screenshots', { tag: '@screenshots' }, () => {
             test('secondary screen', async ({ page }) => {
                 const dash = await DashboardPage.android(page);
                 await dash.addState([testDataStates['consent-managed-configurable-cosmetic']]);
+                await dash.reducedMotion();
                 await dash.viewCookiePromptManagement();
                 await dash.screenshot('consent-managed-configurable-secondary-cosmetic.png');
                 await dash.disableCookiesInSettings();

integration-tests/browser.spec-int.js

@@ -1,5 +1,5 @@
 import { test } from '@playwright/test';
-import { testDataStates } from '../shared/js/ui/views/tests/states-with-fixtures';
+import { testDataStates } from './utils/states-with-fixtures';
 import { DashboardPage } from './DashboardPage';
 import { toggleFlows } from './utils/common-flows';
 import { forwardConsole } from './helpers';
@@ -25,6 +25,7 @@ test.describe('breakage form', () => {
             screen: 'breakageForm',
             randomisedCategories: 'false',
         });
+        await dash.reducedMotion();
         await dash.screenshot('category-type-selection.png');
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.screenshot('category-selection.png');
@@ -42,6 +43,7 @@ test.describe('breakage form', () => {
             screen: 'breakageForm',
             randomisedCategories: 'true',
         });
+        await dash.reducedMotion();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.categoryIsLast('Something else');
     });
@@ -52,6 +54,7 @@ test.describe('breakage form', () => {
             screen: 'breakageForm',
             randomisedCategories: 'false',
         });
+        await dash.reducedMotion();
         await dash.selectsCategoryType('I dislike the content on this site', 'dislike');
         await dash.breakageFormIsVisible('I dislike the content');
         await dash.descriptionPromptIsNotVisible();
@@ -61,13 +64,15 @@ test.describe('breakage form', () => {
     test('skips to breakage form when disliked', async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.browser(page, testDataStates.google, { screen: 'breakageForm' });
+        await dash.reducedMotion();
         await dash.showsCategoryTypeSelectionForExtension();
         await dash.skipsToBreakageFormWhenDisliked();
     });
 
     test('shows empty description warning', { tag: '@screenshots' }, async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.browser(page, testDataStates.google, { screen: 'breakageForm' });
+        await dash.reducedMotion();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Something else', 'other');
         await dash.breakageFormIsVisible();
@@ -79,16 +84,18 @@ test.describe('breakage form', () => {
     test('submits form with description', { tag: '@screenshots' }, async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.browser(page, testDataStates.google, { screen: 'breakageForm' });
+        await dash.reducedMotion();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Something else', 'other');
         await dash.breakageFormIsVisible();
         await dash.submitOtherFeedbackFormWithDescription('something happened');
         await dash.screenshot('screen-breakage-form-success.png');
     });
 
-    test('goes back to primary screen from success screen', { tag: '@screenshots' }, async ({ page }) => {
+    test('goes back to primary screen from success screen', async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.browser(page, testDataStates.google, { opener: 'dashboard' });
+        await dash.reducedMotion();
         await dash.clicksWebsiteNotWorking();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Site layout broken', 'layout');
@@ -97,9 +104,10 @@ test.describe('breakage form', () => {
         await dash.nav.goesBackToPrimaryScreenFromSuccessScreen();
     });
 
-    test('hides back button in success screen when invoked from menu', { tag: '@screenshots' }, async ({ page }) => {
+    test('hides back button in success screen when invoked from menu', async ({ page }) => {
         /** @type {DashboardPage} */
         const dash = await DashboardPage.browser(page, testDataStates.google, { opener: 'menu' });
+        await dash.reducedMotion();
         await dash.clicksWebsiteNotWorking();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
         await dash.selectsCategory('Site layout broken', 'layout');
@@ -126,10 +134,12 @@ test.describe('stack based router', () => {
     test('goes back and forward in categorySelection flow', async ({ page }) => {
         const dash = await DashboardPage.browser(page, testDataStates.google);
         // await dash.reducedMotion(); // TODO: Removed because back button was going back two steps rather than one
-
         await dash.clicksWebsiteNotWorking();
+        await dash.waitForRouterToSettle();
         await dash.selectsCategoryType('The site is not working as expected', 'notWorking');
+        await dash.waitForRouterToSettle();
         await dash.selectsCategory('Site layout broken', 'layout');
+        await dash.waitForRouterToSettle();
         await dash.nav.goesBackToPrimaryScreenFromBreakageScreen();
     });
     test('goes back and forward generally', async ({ page }) => {
@@ -168,6 +178,7 @@ test.describe('Protections toggle -> simple report screen', () => {
     test('shows toggle report + accepts it', async ({ page }) => {
         forwardConsole(page);
         const dash = await DashboardPage.browser(page, testDataStates.protectionsOn);
+        await dash.reducedMotion();
         await dash.showsPrimaryScreen();
         await dash.toggleProtectionsOff();
         await dash.mocks.calledForToggleAllowList();
@@ -178,6 +189,7 @@ test.describe('Protections toggle -> simple report screen', () => {
     test('handles disconnect + fetches new data for toggle-report', async ({ page }) => {
         forwardConsole(page);
         const dash = await DashboardPage.browser(page, testDataStates.protectionsOn);
+        await dash.reducedMotion();
         await dash.showsPrimaryScreen();
         await dash.toggleProtectionsOff();
         await dash.mocks.calledForToggleAllowList();
@@ -190,6 +202,7 @@ test.describe('Protections toggle -> simple report screen', () => {
     test('shows toggle report + rejects it', async ({ page }) => {
         forwardConsole(page);
         const dash = await DashboardPage.browser(page, testDataStates.protectionsOn);
+        await dash.reducedMotion();
         await dash.showsPrimaryScreen();
         await dash.toggleProtectionsOff();
         await dash.mocks.calledForToggleAllowList();
@@ -300,6 +313,7 @@ test.describe('screenshots', { tag: '@screenshots' }, () => {
     for (const { name, state } of states) {
         test(name, async ({ page }) => {
             const dash = await DashboardPage.browser(page, state);
+            await dash.reducedMotion();
             await dash.screenshotEachScreenForState(name, state, { skipInCI: true });
         });
     }