Add visual tests using Percy in Cypress (#8610)

* Add basic Percy tests

* Run Percy in CI

* Fix snapshot timing

* Fix bad selector

* Hide another bit of dynamic text

* Add docs

.github/workflows/element-build-and-test.yaml

@@ -67,14 +67,20 @@ jobs:
       - name: Run Cypress tests
         uses: cypress-io/github-action@v2
         with:
-          # The built in Electron runner seems to grind to a halt trying
+          # The built-in Electron runner seems to grind to a halt trying
           # to run the tests, so use chrome.
           browser: chrome
           start: npx serve -p 8080 webapp
           record: true
+          command-prefix: 'yarn percy exec --'
         env:
           # pass the Dashboard record key as an environment variable
           CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          # pass the Percy token as an environment variable
+          PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+          # Use existing chromium rather than downloading another
+          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
+          PERCY_BROWSER_EXECUTABLE: /usr/bin/chromium-browser
           # pass GitHub token to allow accurately detecting a build vs a re-run build
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.percy.yml

@@ -0,0 +1,5 @@
+version: 2
+snapshot:
+  widths:
+    - 1024
+    - 1920

README.md

@@ -2,6 +2,7 @@
 ![Tests](https://github.com/matrix-org/matrix-react-sdk/actions/workflows/tests.yml/badge.svg)
 ![Static Analysis](https://github.com/matrix-org/matrix-react-sdk/actions/workflows/static_analysis.yaml/badge.svg)
 [![matrix-react-sdk](https://img.shields.io/endpoint?url=https://dashboard.cypress.io/badge/simple/ppvnzg/develop&style=flat&logo=cypress)](https://dashboard.cypress.io/projects/ppvnzg/runs)
+[![This project is using Percy.io for visual regression testing.](https://percy.io/static/images/percy-badge.svg)](https://percy.io/dfde73bd/matrix-react-sdk)
 [![Weblate](https://translate.element.io/widgets/element-web/-/matrix-react-sdk/svg-badge.svg)](https://translate.element.io/engage/element-web/)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=matrix-react-sdk&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=matrix-react-sdk)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=matrix-react-sdk&metric=coverage)](https://sonarcloud.io/summary/new_code?id=matrix-react-sdk)

cypress/integration/1-register/register.spec.ts

@@ -34,22 +34,37 @@ describe("Registration", () => {
 
     it("registers an account and lands on the home screen", () => {
         cy.get(".mx_ServerPicker_change", { timeout: 15000 }).click();
+        cy.get(".mx_ServerPickerDialog_continue").should("be.visible");
+        cy.percySnapshot("Server Picker");
+
         cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);
         cy.get(".mx_ServerPickerDialog_continue").click();
         // wait for the dialog to go away
         cy.get('.mx_ServerPickerDialog').should('not.exist');
 
+        cy.get("#mx_RegistrationForm_username").should("be.visible");
+        // Hide the server text as it contains the randomly allocated Synapse port
+        const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }";
+        cy.percySnapshot("Registration", { percyCSS });
+
         cy.get("#mx_RegistrationForm_username").type("alice");
         cy.get("#mx_RegistrationForm_password").type("totally a great password");
         cy.get("#mx_RegistrationForm_passwordConfirm").type("totally a great password");
         cy.startMeasuring("create-account");
         cy.get(".mx_Login_submit").click();
 
+        cy.get(".mx_RegistrationEmailPromptDialog").should("be.visible");
+        cy.percySnapshot("Registration email prompt", { percyCSS });
         cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
+
         cy.stopMeasuring("create-account");
+        cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy").should("be.visible");
+        cy.percySnapshot("Registration terms prompt", { percyCSS });
+
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
         cy.startMeasuring("from-submit-to-home");
         cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
+
         cy.url().should('contain', '/#/home');
         cy.stopMeasuring("from-submit-to-home");
     });

cypress/integration/2-login/login.spec.ts

@@ -41,7 +41,10 @@ describe("Login", () => {
         });
 
         it("logs in with an existing account and lands on the home screen", () => {
-            cy.get(".mx_ServerPicker_change", { timeout: 15000 }).click();
+            cy.get("#mx_LoginForm_username", { timeout: 15000 }).should("be.visible");
+            cy.percySnapshot("Login");
+
+            cy.get(".mx_ServerPicker_change").click();
             cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);
             cy.get(".mx_ServerPickerDialog_continue").click();
             // wait for the dialog to go away

cypress/support/index.ts

@@ -16,6 +16,8 @@ limitations under the License.
 
 /// <reference types="cypress" />
 
+import "@percy/cypress";
+
 import "./performance";
 import "./synapse";
 import "./login";

cypress/tsconfig.json

@@ -2,7 +2,7 @@
     "compilerOptions": {
         "target": "es2016",
         "lib": ["es2020", "dom"],
-        "types": ["cypress"],
+        "types": ["cypress", "@percy/cypress"],
         "moduleResolution": "node"
     },
     "include": ["**/*.ts"]

docs/cypress.md

@@ -6,6 +6,7 @@ It aims to cover:
  * How to run the tests yourself
  * How the tests work
  * How to write great Cypress tests
+ * Visual testing
 
 ## Running the Tests
 Our Cypress tests run automatically as part of our CI along with our other tests,
@@ -166,3 +167,14 @@ already familiar with Cypress.
 
 This is a small selection - the Cypress best practices guide, linked above, has more good advice, and we
 should generally try to adhere to them.
+
+## Percy Visual Testing
+We also support visual testing via Percy, this extracts the DOM from Cypress and renders it using custom renderers
+for Safari, Firefox, Chrome & Edge, allowing us to spot visual regressions before they become release regressions.
+Right now we run it as part of the standard Pull Request CI automation but due to only having 25k screenshots/month,
+and each `cy.percySnapshot()` call results in 8 screenshots (4 browsers, 2 sizes) this could quickly be exhausted and
+at that point we would likely run it on a CRON interval or before releases.
+
+To record a snapshot use `cy.percySnapshot()`, you may have to pass `percyCSS` into the 2nd argument to hide certain
+elements which contain dynamic/generated data to avoid them cause false positives in the Percy screenshot diffs.
+

package.json

@@ -133,6 +133,8 @@
     "@babel/traverse": "^7.12.12",
     "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
     "@peculiar/webcrypto": "^1.1.4",
+    "@percy/cli": "^1.1.4",
+    "@percy/cypress": "^3.1.1",
     "@sentry/types": "^6.10.0",
     "@sinonjs/fake-timers": "^9.1.2",
     "@types/classnames": "^2.2.11",