[FEATURE] Add support for BackstopJS Visual Regression testing.

Add support for easily setting up BackstopJS through docker. Idea is
that tests can easily be added via `backstop.json` and then easily run
and approved with some basic scripts.

Also added a dedicated docs section on testing with a run through of the
different testing types supported and how to run each one.

.gitignore

@@ -37,6 +37,7 @@ debug.log
 !/bin/makepot
 !/bin/pot2po
 !/bin/po2mo
+!/bin/backstop_report
 
 .env
 /.env.local

README.md

@@ -75,6 +75,8 @@ Admin:
 
 🚩 - First class support for feature flags using [flagpole](https://github.com/jamesrwilliams/flagpole)
 
+✅ - Unit, Integration and Visual Regression testing support
+
 > You can read more about all of these features in [this post on the Box UK blog](https://www.boxuk.com/insight/how-we-develop-wordpress-sites/).
 
 
@@ -96,6 +98,7 @@ Admin:
 * [Working with a styleguide](docs/dev/styleguide.md)
 * [i18n](docs/dev/i18n.md)
 * [Tools](docs/dev/tools.md)
+* [Testing](docs/dev/testing.md)
 
 
 

bin/backstop_report

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+ENVIRONMENT="${@:-local}"
+
+# TODO: Add Windows support.
+
+if [ ! -d "tests/visual-regression/output/${ENVIRONMENT}/html_report/" ]; then
+    exit 1;
+fi
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # Mac OS.
+    open tests/visual-regression/output/${ENVIRONMENT}/html_report/index.html;
+else
+    # Linux.
+    xdg-open tests/visual-regression/output/${ENVIRONMENT}/html_report/index.html;
+fi

bin/docker/backstop

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+docker-compose -f docker-compose.yml run --rm --workdir /src backstop node ./bin/backstop.js "$@"

bin/docker/setup_backstop

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+ENVIRONMENT="${@:-local}"
+
+docker-compose -f docker-compose.yml run --rm --workdir /src backstop cp config/${ENVIRONMENT}/cookies.json.dist config/${ENVIRONMENT}/cookies.json;
+docker-compose -f docker-compose.yml run --rm --workdir /src backstop cp config/${ENVIRONMENT}/secrets.json.dist config/${ENVIRONMENT}/secrets.json;
+
+docker-compose -f docker-compose.yml run --rm --workdir /src backstop npm install;

docker-compose.yml

@@ -53,7 +53,7 @@ services:
             context: docker/nginx
             dockerfile: Dockerfile
         volumes:
-            - '.:/var/www/html/'
+            - '.:/var/www/html:cached'
             - './docker/nginx/nginx.conf:/etc/nginx/conf.d/default.conf'
             - './docker/nginx/conf:/etc/nginx/conf.d/conf'
             - './docker/nginx/insecure_https.crt:/etc/pki/tls/certs/insecure_https.crt:delegated'
@@ -87,6 +87,21 @@ services:
             - memcached
             - -I 5m # Increase item size from the default 1mb to 5mb
 
+    backstop:
+        build:
+            context: docker/backstop
+            dockerfile: Dockerfile
+        volumes:
+            - './tests/visual-regression:/src:cached'
+            - vr_yarn_cache:/usr/local/share/.cache/yarn/v6:delegated
+            - vr_node_modules:/src/app/node_modules:delegated
+        extra_hosts:
+            - '${LOOPBACK_HOST_NAME}:192.168.35.10' # IP should match the IP address set for nginx above
+
+volumes:
+    vr_yarn_cache:
+    vr_node_modules:
+
 networks:
     default:
         external: true

docker/backstop/Dockerfile

@@ -0,0 +1,40 @@
+FROM node:14.18.1
+
+ARG BACKSTOPJS_VERSION
+
+ENV \
+    BACKSTOPJS_VERSION=$BACKSTOPJS_VERSION
+
+# Base packages
+RUN apt-get update && \
+    apt-get install -y git sudo software-properties-common --no-install-recommends; \
+    rm -rf /var/lib/apt/lists/*
+
+#RUN sudo npm install -g --unsafe-perm=true --allow-root backstopjs@${BACKSTOPJS_VERSION}
+
+RUN wget --progress=dot:giga https://dl-ssl.google.com/linux/linux_signing_key.pub && apt-key add linux_signing_key.pub; \
+    add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main"
+
+# RUN	apt-get -y update && apt-get -y install google-chrome-stable
+
+# RUN apt-get install -y firefox-esr
+
+# gconf-service libxext6.... added for https://github.com/garris/BackstopJS/issues/1225
+
+RUN apt-get -qqy update \
+  && apt-get -qqy --no-install-recommends install \
+    libfontconfig \
+    libfreetype6 \
+    xfonts-cyrillic \
+    xfonts-scalable \
+    fonts-liberation \
+    fonts-ipafont-gothic \
+    fonts-wqy-zenhei \
+    gconf-service libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxss1 libxtst6 libappindicator1 libnss3 libasound2 libatk1.0-0 libc6 ca-certificates fonts-liberation lsb-release xdg-utils wget \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get -qyy clean
+
+
+WORKDIR /src
+
+#ENTRYPOINT ["backstop"]

docs/dev/testing.md

@@ -0,0 +1,83 @@
+# Testing
+
+Testing WordPress sites can be a challenge. We should strive to make use of different testing techniques to ensure we're getting good test coverage across our sites. Here are the different types of tests we recommend and how to achieve each.
+
+* [Unit Testing](#unit-testing)
+* [Integration Testing](#integration-testing)
+* [Visual Regression Testing](#visual-regression-testing)
+* [Acceptance Testing](#acceptance-testing)
+
+## Unit Testing
+
+Unit tests should be added for every `(mu-)plugin`/`theme` we create. We should always aim to decouple the core logic from WordPress, for this to be true we should avoid any use of global functions or hidden dependencies in any classes/functions we use for our core logic. We should then be able to unit test these easily.
+
+We use [PHPUnit](https://phpunit.de/) for unit testing.
+
+## Integration Testing
+
+Integration tests should also be added wherever appropriate. For any part of your `(mu-)plugin`/`theme` that interacts with WordPress this should be covered by an integration test.
+
+We also use [PHPUnit](https://phpunit.de/) for integration testing along with [WP PHPUnit](https://github.com/wp-phpunit/wp-phpunit) which will allow you to interact with the WordPress install easily. It also provides a number of helpers to allow for easy generation of synthetic data for testing.
+
+### Running the unit & integration tests
+
+Both the unit tests and integration tests follow the same structure, tests are designed to be added within each `mu-plugin` you create within a `tests` directory. The test runner sits outside though in the root of this repo. It works by looping over each `mu-plugin` and running its test.
+
+> It's important each `mu-plugin` follows the format `plugin-name/plugin-name.php`
+
+`bin/docker/phpunit`
+
+(or `bin/phpunit` if not using docker)
+
+## Visual Regression Testing
+
+Sometimes hard to find bugs can be present even though all our automated tests are passing, this could be due to something that's fallen through the cracks, a JS/CSS issue, an update to a plugin or theme, an update to a browser or something else entirely. It's therefore useful to run Visual Regression (VR) tests to catch anything that may have changed how the site appears to the end user.
+
+VR tests work by taking screenshots of given scenarios (typically defined pages/posts), approving the screenshots to a known state and then comparing future runs against this approved state to see if anything has changed. Tests will then fail if the difference between the screenshots is above a set tolerance.
+
+We use [BackstopJS](https://github.com/garris/BackstopJS/) for VR testing.
+
+### Running the tests
+
+Tests (or scenarios) should be added to `tests/visual-regression/backstop.json` within the `scenarios` section. For example:
+
+```json
+    "scenarios": [
+        {
+            "label": "Page or post to test",
+            "cookiePath": "config/%env_name%/cookies.json",
+            "url": "%base_url%/page-or-post-slug",
+            "delay": 500,
+            "misMatchThreshold": 0.1,
+            "requireSameDimensions": true
+        }
+    ]
+```
+
+You can configure things like cookies, env vars and secrets within the `config` directory. You can do so for each environment you want to test, `local` is set up currently that will allow you to test the local environment.
+
+* Setup backstop
+
+`bin/docker/setup_backstop`
+
+* Run backstop tests
+
+`bin/docker/backstop test [environment]`
+
+* Approve detected changes
+
+`bin/docker/backstop approve [environment]`
+
+> [environment] is the environment you want to test against, falls back to `local`.
+
+* Open report
+
+`bin/backstop_report`
+
+> This will open the test report in your default browser.
+
+## Acceptance testing
+
+As well as the other tests we also want to run Acceptance Tests, so we can test our `(mu-)plugin`/`theme` does what it's supposed to do for the end user. For this we typically want to run some automated browser based testing to make sure clicking links, navigating between sections works the way it should.
+
+We use... [coming soon]

docs/dev/usage.md

@@ -8,16 +8,6 @@ This one is optional but highly recommended, if you want to use memcache for obj
 
 `cp wp-content/plugins/memcached/object-cache.php wp-content/object-cache.php`
 
-## Running the tests
-
-Tests are designed to be added within each `mu-plugin` you create within a `tests` directory. The test runner sits outside though in the root of this repo. It works by looping over each `mu-plugin` and running it's test.
-
-> It's important each `mu-plugin` follows the format `plugin-name/plugin-name.php`
-
-`bin/docker/phpunit`
-
-(or `bin/phpunit` if not using docker)
-
 ## Running the code sniffs
 
 We use the [WordPress VIP coding standards](https://github.com/Automattic/VIP-Coding-Standards) to ensure the code is adhering to the best possible performance and security practices. We also use the core [WordPress standards](https://github.com/WordPress/WordPress-Coding-Standards) and the [Neutron standards](https://github.com/Automattic/phpcs-neutron-standard) to ensure for consistency for a modern WordPress code base. You can check your code against all of this by running the following command:

fixtures.yml

@@ -58,6 +58,8 @@ Hellonico\Fixtures\Entity\Post:
         post_category: '1x @category*->term_id'
         tax_input:
             post_tag: '5x @tag*->term_id'
+    post_sample (extends default):
+        post_title: Sample Post
 
 ## Menu
 Hellonico\Fixtures\Entity\NavMenu:

tests/visual-regression/.gitignore

@@ -0,0 +1,6 @@
+node_modules
+output
+!output/.gitkeep
+config/**/cookies.json
+config/**/secrets.json
+config/profile.json

tests/visual-regression/backstop.json

@@ -0,0 +1,73 @@
+{
+  "docker": "%use_docker%",
+  "config": {
+      "id": "backstop_default",
+      "viewports": [
+          {
+              "label": "phone",
+              "width": 320,
+              "height": 480
+          },
+          {
+              "label": "tablet",
+              "width": 1024,
+              "height": 768
+          },
+          {
+              "label": "desktop",
+              "width": 1920,
+              "height": 1080
+          }
+      ],
+      "onBeforeScript": "puppet/onBefore.js",
+      "onReadyScript": "puppet/onReady.js",
+      "scenarios": [
+          {
+              "label": "Homepage",
+              "cookiePath": "config/%env_name%/cookies.json",
+              "url": "%base_url%/",
+              "delay": 500,
+              "misMatchThreshold": 0.1,
+              "requireSameDimensions": true
+          },
+          {
+              "label": "About page",
+              "cookiePath": "config/%env_name%/cookies.json",
+              "url": "%base_url%/about",
+              "delay": 500,
+              "misMatchThreshold": 0.1,
+              "requireSameDimensions": true
+          },
+
+          {
+              "label": "Sample post",
+              "cookiePath": "config/%env_name%/cookies.json",
+              "url": "%base_url%/sample-post",
+              "delay": 500,
+              "misMatchThreshold": 0.1,
+              "requireSameDimensions": true
+          }
+      ],
+      "paths": {
+          "bitmaps_reference": "output/%env_name%/bitmaps_reference",
+          "bitmaps_test": "output/%env_name%/bitmaps_test",
+          "engine_scripts": "scripts",
+          "html_report": "output/%env_name%/html_report",
+          "ci_report": "output/%env_name%/ci_report"
+      },
+      "report": [
+          "html"
+      ],
+      "engine": "puppeteer",
+      "engineOptions": {
+          "ignoreHTTPSErrors": true,
+          "args": [
+              "--no-sandbox"
+          ]
+      },
+      "asyncCaptureLimit": 5,
+      "asyncCompareLimit": 50,
+      "debug": false,
+      "debugWindow": false
+  }
+}

tests/visual-regression/bin/backstop.js

@@ -0,0 +1,52 @@
+const backstop = require('backstopjs');
+const envVars = require('../scripts/envVars.js');
+const utils = require('../scripts/utils.js');
+const { exec } = require("child_process");
+
+const validOperations = [ 'test', 'approve' ];
+
+const myArgs = process.argv.slice( 2 );
+const operation = myArgs[ 0 ];
+const env = myArgs[ 1 ] || 'local';
+
+const configFolderPath = 'config/';
+const envVarsFilePath = configFolderPath + env + '/env_vars.json';
+const envSecretsFilePath = configFolderPath + env + '/secrets.json';
+const profilePath = configFolderPath + 'profile.json';
+const profilePathFallback = configFolderPath + 'profile.json.dist';
+const backstopFilePath = 'backstop.json';
+
+if ( ! utils.isOperationValid( operation, validOperations ) ) {
+	utils.exitWithReason( '"' + operation + '" is not a valid operation. Valid operations are: ' + validOperations.join( ', ' ) );
+}
+
+if ( ! utils.doesFileExist( envVarsFilePath ) ) {
+	utils.exitWithReason( 'Required file not found at ' + envVarsFilePath );
+}
+
+if ( ! utils.doesFileExist( envSecretsFilePath ) ) {
+	utils.exitWithReason( 'Required file not found at ' + envSecretsFilePath );
+}
+
+if ( ! utils.doesFileExist( backstopFilePath ) ) {
+	utils.exitWithReason( 'Required file not found at ' + backstopFilePath );
+}
+
+console.log( 'Using environment variables file ' + envVarsFilePath );
+console.log( 'Using environment secrets file ' + envSecretsFilePath );
+console.log( 'Using backstop file ' + backstopFilePath );
+
+envVars.setVars( utils.parseJsonFile( envVarsFilePath ) );
+envVars.setVars( utils.parseJsonFile( envSecretsFilePath ) );
+
+const profilePathToUse = utils.doesFileExist( profilePath ) ? profilePath : profilePathFallback;
+console.log( 'Using profile file ' + profilePathToUse );
+
+envVars.setVars( utils.parseJsonFile( profilePathToUse ) );
+
+// Add env_name to the configuration object, so it can be referenced from within the config.
+envVars.setVar( 'env_name', env );
+
+const config = utils.parseJsonFile( backstopFilePath, envVars.getVars() );
+
+backstop( operation, config );

tests/visual-regression/config/local/cookies.json.dist

@@ -0,0 +1,14 @@
+[
+    {
+        "domain": ".boxuk-wp-skeleton.local",
+        "path": "/",
+        "name": "yourCookieName",
+        "value": "yourCookieValue",
+        "expirationDate": 1798790400,
+        "hostOnly": false,
+        "httpOnly": false,
+        "secure": false,
+        "session": false,
+        "sameSite": "no_restriction"
+    }
+]

tests/visual-regression/config/local/env_vars.json

@@ -0,0 +1,3 @@
+{
+    "base_url": "https://boxuk-wp-skeleton.local"
+}

tests/visual-regression/config/local/secrets.json.dist

@@ -0,0 +1 @@
+{}

tests/visual-regression/config/profile.json.dist

@@ -0,0 +1,3 @@
+{
+  "use_docker": false
+}