[test] Add visual regression tests (#1081)

.circleci/config.yml

@@ -23,35 +23,69 @@ defaults: &defaults
 
 commands:
   install_js:
+    parameters:
+      browsers:
+        type: boolean
+        default: false
+        description: 'Set to true if you intend to any browser (e.g. with playwright).'
+
     steps:
       - run:
           name: View install environment
           command: |
             node --version
             yarn --version
       - restore_cache:
+          name: Restore yarn cache
           keys:
-            - v6-yarn-sha-{{ checksum "yarn.lock" }}
-            - v6-yarn-sha-
+            - v7-yarn-{{ checksum "yarn.lock" }}
+            - v7-yarn-
       - run:
           name: Set yarn cache folder
           command: |
-            # Keep path in sync with `save_cache` for key "v6-yarn-sha-"
-            yarn config set cache-folder ~/.cache/yarn
+            # Keep path in sync with `save_cache` for key "v7-yarn-"
+            yarn config set cache-folder /tmp/yarn-cache
             # Debug information
             yarn cache dir
             yarn cache list
+      - when:
+          condition: << parameters.browsers >>
+          steps:
+            - run:
+                name: Prepare playwright hash
+                command: yarn --json list --pattern playwright > /tmp/playwright_info.json
+            - store_artifacts:
+                name: Debug playwright hash
+                path: /tmp/playwright_info.json
+            - restore_cache:
+                name: Restore playwright cache
+                keys:
+                  - v5-playwright-{{ arch }}-{{ checksum "/tmp/playwright_info.json" }}
+                  - v5-playwright-{{ arch }}-
+                  - v5-playwright-
       - run:
           name: Install js dependencies
           command: yarn install --verbose
-  prepare_chrome_headless:
-    steps:
-      - run:
-          name: Install dependencies for Chrome Headless
-          # From https://github.com/GoogleChrome/puppeteer/blob/811415bc8c47f7882375629b57b3fe186ad61ed4/docs/troubleshooting.md#chrome-headless-doesnt-launch
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y --force-yes gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget
+          environment:
+            PLAYWRIGHT_BROWSERS_PATH: /tmp/pw-browsers
+            PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: <<# parameters.browsers >>0<</ parameters.browsers >><<^ parameters.browsers >>1<</ parameters.browsers >>
+      - save_cache:
+          name: Save yarn cache
+          key: v7-yarn-{{ checksum "yarn.lock" }}
+          paths:
+            # Keep path in sync with "Set yarn cache folder"
+            # Can't use environment variables for `save_cache` paths (tested in https://app.circleci.com/pipelines/github/mui-org/material-ui/37813/workflows/5b1e207f-ac8b-44e7-9ba4-d0f9a01f5c55/jobs/223370)
+            - /tmp/yarn-cache
+      - when:
+          condition: << parameters.browsers >>
+          steps:
+            - save_cache:
+                name: Save playwright cache
+                key: v5-playwright-{{ arch }}-{{ checksum "/tmp/playwright_info.json" }}
+                paths:
+                  # Keep path in sync with "Install js dependencies"
+                  # Can't use environment variables for `save_cache` paths (tested in https://app.circleci.com/pipelines/github/mui-org/material-ui/37813/workflows/5b1e207f-ac8b-44e7-9ba4-d0f9a01f5c55/jobs/223370)
+                  - /tmp/pw-browsers
 
 jobs:
   checkout:
@@ -105,10 +139,14 @@ jobs:
           command: yarn jsonlint
   test_browser:
     <<: *defaults
+    docker:
+      - image: mcr.microsoft.com/playwright:bionic
+        environment:
+          NODE_ENV: development # Needed if playwright is in `devDependencies`
     steps:
       - checkout
-      - install_js
-      - prepare_chrome_headless
+      - install_js:
+          browsers: true
       - run:
           name: Tests real browsers
           command: yarn test:karma
@@ -130,6 +168,22 @@ jobs:
       - run:
           name: Tests TypeScript definitions
           command: yarn typescript
+  test_regressions:
+    <<: *defaults
+    docker:
+      - image: mcr.microsoft.com/playwright:bionic
+        environment:
+          NODE_ENV: development # Needed if playwright is in `devDependencies`
+    steps:
+      - checkout
+      - install_js:
+          browsers: true
+      - run:
+          name: Run visual regression tests
+          command: xvfb-run yarn test:regressions
+      - run:
+          name: Upload screenshots to Argos CI
+          command: yarn test:argos
 workflows:
   version: 2
   pipeline:
@@ -147,3 +201,6 @@ workflows:
       - test_types:
           requires:
             - checkout
+      - test_regressions:
+          requires:
+            - checkout

.gitignore

@@ -11,6 +11,7 @@ __diff_output__
 /coverage
 /docs/.next
 /docs/export
+/test/regressions/screenshots
 build
 # TODO `dist` should be replaced with `build`, per convention
 dist

package.json

@@ -28,6 +28,12 @@
     "test:karma": "cross-env NODE_ENV=test karma start test/karma.conf.js",
     "test:unit": "cross-env NODE_ENV=test mocha 'packages/**/*.test.tsx' --exclude '**/node_modules/**'",
     "test:watch": "yarn test:unit --watch",
+    "test:regressions": "cross-env NODE_ENV=production yarn test:regressions:build && concurrently --success first --kill-others \"yarn test:regressions:run\" \"yarn test:regressions:server\"",
+    "test:regressions:build": "webpack --config test/regressions/webpack.config.js",
+    "test:regressions:dev": "concurrently \"yarn test:regressions:build --watch\" \"yarn test:regressions:server\"",
+    "test:regressions:run": "mocha --config test/regressions/.mocharc.js --delay 'test/regressions/**/*.test.js'",
+    "test:regressions:server": "serve test/regressions",
+    "test:argos": "node ./scripts/pushArgos.js",
     "lint": "yarn eslint && yarn jsonlint",
     "eslint": "eslint . --report-unused-disable-directives --ext .js,.ts,.tsx",
     "eslint:ci": "eslint . --report-unused-disable-directives --ext .js,.ts,.tsx",
@@ -64,12 +70,14 @@
     "@types/styled-components": "^5.0.1",
     "@typescript-eslint/eslint-plugin": "^4.11.1",
     "@typescript-eslint/parser": "^4.11.1",
+    "argos-cli": "^0.3.0",
     "babel-plugin-module-resolver": "^4.0.0",
     "babel-plugin-optimize-clsx": "^2.6.1",
     "babel-plugin-transform-rename-import": "^2.3.0",
     "chai": "^4.2.0",
     "chai-dom": "^1.8.2",
     "compression-webpack-plugin": "^6.0.2",
+    "concurrently": "^5.3.0",
     "cross-env": "^7.0.2",
     "enzyme": "^3.11.0",
     "enzyme-adapter-react-16": "^1.15.2",
@@ -92,12 +100,12 @@
     "karma-browserstack-launcher": "^1.6.0",
     "karma-chrome-launcher": "^3.1.0",
     "karma-mocha": "^2.0.1",
-    "karma-mocha-reporter": "^2.2.5",
     "karma-sourcemap-loader": "^0.3.7",
     "karma-webpack": "^4.0.2",
     "lerna": "^3.20.2",
     "mocha": "^8.0.1",
     "nyc": "^15.1.0",
+    "playwright": "^1.6.1",
     "prettier": "^2.0.5",
     "pretty-format-v24": "npm:pretty-format@24",
     "react": "^16.13.1",
@@ -111,6 +119,7 @@
     "rollup-plugin-terser": "^7.0.2",
     "rollup-plugin-typescript2": "^0.30.0",
     "sinon": "^9.0.2",
+    "serve": "^11.3.2",
     "stylelint": "^13.7.2",
     "stylelint-config-standard": "^20.0.0",
     "stylelint-processor-styled-components": "^1.10.0",

packages/demo-app/src/app/app.tsx

@@ -33,12 +33,6 @@ const GlobalStyle = createGlobalStyle`
     width: 100%;
   }
 
-  .main-container {
-    position: relative;
-    flex-grow: 1;
-    box-sizing: border-box;
-  }
-
   .fill-space,
   .fill-abs,
   .fill {
@@ -49,16 +43,6 @@ const GlobalStyle = createGlobalStyle`
     right: 0;
   }
 
-  .flex-col {
-    display: flex;
-    flex-direction: column;
-  }
-
-  .flex-row {
-    display: flex;
-    flex-direction: row;
-  }
-
   .grow {
     flex-grow: 1;
   }
@@ -67,45 +51,9 @@ const GlobalStyle = createGlobalStyle`
     justify-content: center;
   }
 
-  .action-bar {
-    display: flex;
-    justify-content: center;
-
-    button {
-      margin-right: 20px;
-    }
-  }
-
   .storybook-img {
     width: 30px;
   }
-
-  @media screen and (max-width: 768px) {
-    .big-device {
-      display: none !important;
-    }
-  }
-
-  .negative {
-    color: #d20704;
-  }
-
-  .positive {
-    color: #008508;
-  }
-
-  .info-icon {
-    color: #2196f3;
-    align-self: center;
-  }
-
-  .bad {
-    background: rgba(255, 128, 171, 0.4);
-  }
-
-  .good {
-    background: rgba(185, 246, 202, 0.4);
-  }
 `;
 
 const StyledApp = styled.div`

packages/grid/x-grid-data-generator/src/commodities.columns.tsx

@@ -1,4 +1,3 @@
-// import './style/real-data-stories.css';
 import {
   generateTotalPrice,
   randomCommodity,

packages/grid/x-grid-data-generator/src/employees.columns.tsx

@@ -1,4 +1,3 @@
-// import './style/real-data-stories.css';
 import {
   generateName,
   randomAvatar,

packages/grid/x-grid-data-generator/src/services/random-generator.ts

@@ -11,7 +11,14 @@ import {
   TAXCODE_OPTIONS,
 } from './static-data';
 
-const chance = globalChance();
+const chanceId = globalChance();
+let chance;
+
+if (process.env.DISABLE_CHANCE_RANDOM) {
+  chance = globalChance(() => 0.5);
+} else {
+  chance = chanceId;
+}
 
 function dateFuture(years?: number, refDate?: string) {
   let date = new Date();
@@ -67,17 +74,17 @@ function datePast(years?: number, refDate?: string) {
   return date;
 }
 
-export const random = (min: number, max: number): number => Math.random() * (max - min) + min;
+export const random = (min: number, max: number): number => chance.random() * (max - min) + min;
 export const randomInt = (min: number, max: number): number => Number(random(min, max).toFixed());
 export const randomPrice = (min = 0, max = 100000): number => random(min, max);
 export const randomRate = (): number => random(0, 1);
 export const randomDate = (start, end) =>
-  new Date(start.getTime() + Math.random() * (end.getTime() - start.getTime()));
+  new Date(start.getTime() + chance.random() * (end.getTime() - start.getTime()));
 export const getDate = () => randomDate(new Date(2012, 0, 1), new Date());
 export const randomArrayItem = (arr: any[]) => arr[random(0, arr.length - 1).toFixed()];
 
 export const randomColor = () => randomArrayItem(COLORS);
-export const randomId = () => chance.guid();
+export const randomId = () => chanceId.guid();
 export const randomDesk = () => `D-${chance.integer({ min: 0, max: 10000 })}`;
 export const randomCommodity = () => randomArrayItem(COMMODITY_OPTIONS);
 export const randomTraderName = () => chance.name();

packages/grid/x-grid-data-generator/src/useDemoData.ts

@@ -49,14 +49,6 @@ function extrapolateSeed(rowLength, columns, data) {
   return { ...data, rows };
 }
 
-async function sleep(duration: number) {
-  return new Promise<void>((resolve) => {
-    setTimeout(() => {
-      resolve();
-    }, duration);
-  });
-}
-
 function deepFreeze(object) {
   // Retrieve the property names defined on object
   const propNames = Object.getOwnPropertyNames(object);
@@ -94,7 +86,6 @@ export const useDemoData = (options: DemoDataOptions): DemoDataReturnType => {
     let active = true;
 
     (async () => {
-      await sleep(100);
       let columns = options.dataSet === 'Commodity' ? getCommodityColumns() : getEmployeeColumns();
 
       if (options.maxColumns) {
@@ -105,7 +96,6 @@ export const useDemoData = (options: DemoDataOptions): DemoDataReturnType => {
 
       if (rowLength > 1000) {
         newData = await getRealData(1000, columns);
-        await sleep(100);
         newData = extrapolateSeed(rowLength, columns, newData);
       } else {
         newData = await getRealData(rowLength, columns);

packages/storybook/src/stories/grid-pagination.stories.tsx

@@ -24,7 +24,7 @@ export default {
 } as Meta;
 
 export function PaginationDefault() {
-  const data = useData(2000, 200);
+  const data = useData(200, 20);
 
   return (
     <div className="grid-container">
@@ -34,7 +34,7 @@ export function PaginationDefault() {
 }
 
 export function PageSize100() {
-  const data = useData(2000, 200);
+  const data = useData(2000, 20);
 
   return (
     <div className="grid-container">
@@ -44,7 +44,7 @@ export function PageSize100() {
 }
 
 export const PaginationArgs: Story = (args) => {
-  const data = useData(2000, 200);
+  const data = useData(200, 20);
 
   return <XGrid rows={data.rows} columns={data.columns} {...args} />;
 };
@@ -61,7 +61,7 @@ PaginationArgs.args = {
 };
 
 export function HiddenPagination() {
-  const data = useData(2000, 200);
+  const data = useData(200, 20);
 
   return (
     <div className="grid-container">