Testing: Add partial support for Axe verification in e2e tests (#15018)

* Testing: Add partial support for Axe verification in e2e tests * Execute build earlier on Travis to make ensure jest-puppeteer-axe can be used * Exclude TinyMCE toolbar from Axe verification in e2e tests * Update tests setup to make all tests pass * Respond to the test instagram URL with a 404 * Revert "Respond to the test instagram URL with a 404" This reverts commit 35b5a5c5b6714d08694a0fae5ce1ba3fc81a0e2f. * Update index.js
WordPress · May 6, 2019 · a797ec5 · a797ec5
1 parent ccd5317
commit a797ec5
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 28 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -69,72 +69,73 @@ jobs:
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Admin with plugins) (2/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false POPULAR_PLUGINS=true
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Admin with plugins) (3/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false POPULAR_PLUGINS=true
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Admin with plugins) (4/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false POPULAR_PLUGINS=true
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Author without plugins) (1/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false E2E_ROLE=author
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Author without plugins) (2/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false E2E_ROLE=author
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Author without plugins) (3/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false E2E_ROLE=author
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests )
 
     - name: E2E tests (Author without plugins) (4/4)
       env: WP_VERSION=latest SCRIPT_DEBUG=false E2E_ROLE=author
       install:
         - ./bin/setup-local-env.sh
       script:
-        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run build
+        - $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         - npm run test-e2e -- --ci --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests )
+
   allow_failures:
     - name: PHP unit tests (PHP 5.3)
       env: WP_VERSION=latest SWITCH_TO_PHP=5.3

diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/e2e-tests/CHANGELOG.md b/packages/e2e-tests/CHANGELOG.md
@@ -1,3 +1,9 @@
+## Unreleased
+
+### New features
+
+- Added Axe (the Accessibility Engine) API integration with e2e tests suite.
+
 ## 1.0.0 (2019-03-06)
 
 -   Initial release.
diff --git a/packages/e2e-tests/config/setup-test-framework.js b/packages/e2e-tests/config/setup-test-framework.js
@@ -6,15 +6,14 @@ import { get } from 'lodash';
 /**
  * WordPress dependencies
  */
-import '@wordpress/jest-console';
 import {
+	activatePlugin,
 	clearLocalStorage,
 	enablePageDialogAccept,
 	setBrowserViewport,
-	visitAdminPage,
-	activatePlugin,
 	switchUserToAdmin,
 	switchUserToTest,
+	visitAdminPage,
 } from '@wordpress/e2e-test-utils';
 
 /**
@@ -148,6 +147,40 @@ function observeConsoleLogging() {
 	} );
 }
 
+/**
+ * Runs Axe tests when the block editor is found on the current page.
+ *
+ * @return {?Promise} Promise resolving once Axe texts are finished.
+ */
+async function runAxeTestsForBlockEditor() {
+	if ( ! await page.$( '.block-editor' ) ) {
+		return;
+	}
+
+	await expect( page ).toPassAxeTests( {
+		// Temporary disabled rules to enable initial integration.
+		// See: https://github.com/WordPress/gutenberg/pull/15018.
+		disabledRules: [
+			'aria-allowed-role',
+			'aria-valid-attr-value',
+			'button-name',
+			'color-contrast',
+			'dlitem',
+			'duplicate-id',
+			'label',
+			'link-name',
+			'listitem',
+			'region',
+		],
+		exclude: [
+			// Ignores elements created by metaboxes.
+			'.edit-post-layout__metaboxes',
+			// Ignores elements created by TinyMCE.
+			'.mce-container',
+		],
+	} );
+}
+
 // Before every test suite run, delete all content created by the test. This ensures
 // other posts/comments/etc. aren't dirtying tests and tests don't depend on
 // each other's side-effects.
@@ -162,6 +195,7 @@ beforeAll( async () => {
 } );
 
 afterEach( async () => {
+	await runAxeTestsForBlockEditor();
 	await setupBrowser();
 } );
 

diff --git a/packages/e2e-tests/jest.config.js b/packages/e2e-tests/jest.config.js
@@ -5,6 +5,8 @@ module.exports = {
 	],
 	setupFilesAfterEnv: [
 		'<rootDir>/config/setup-test-framework.js',
+		'@wordpress/jest-console',
+		'@wordpress/jest-puppeteer-axe',
 		'expect-puppeteer',
 	],
 };
diff --git a/packages/e2e-tests/package.json b/packages/e2e-tests/package.json
@@ -24,6 +24,7 @@
 	"dependencies": {
 		"@wordpress/e2e-test-utils": "file:../e2e-test-utils",
 		"@wordpress/jest-console": "file:../jest-console",
+		"@wordpress/jest-puppeteer-axe": "file:../jest-puppeteer-axe",
 		"@wordpress/scripts": "file:../scripts",
 		"expect-puppeteer": "^4.0.0",
 		"lodash": "^4.17.11"

diff --git a/packages/e2e-tests/plugins/format-api/index.js b/packages/e2e-tests/plugins/format-api/index.js
@@ -18,7 +18,7 @@
 									props.value, {
 										type: 'my-plugin/link',
 										attributes: {
-											url: '#test',
+											url: 'https://example.com',
 										}
 									}
 								)

diff --git a/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap b/packages/e2e-tests/specs/plugins/__snapshots__/format-api.test.js.snap
@@ -2,6 +2,6 @@
 
 exports[`Using Format API Clicking the control wraps the selected text properly with HTML code 1`] = `
 "<!-- wp:paragraph -->
-<p>First <a href=\\"#test\\" class=\\"my-plugin-link\\">paragraph</a></p>
+<p>First <a href=\\"https://example.com\\" class=\\"my-plugin-link\\">paragraph</a></p>
 <!-- /wp:paragraph -->"
 `;
diff --git a/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js b/packages/e2e-tests/specs/plugins/meta-attribute-block.test.js
@@ -41,13 +41,14 @@ describe( 'Block with a meta attribute', () => {
 		await page.keyboard.type( 'Meta Value' );
 
 		const inputs = await page.$$( '.my-meta-input' );
-		await inputs.forEach( async ( input ) => {
+		await Promise.all( inputs.map( async ( input ) => {
 			// Clicking the input selects the block,
 			// and selecting the block enables the sync data mode
-			// as otherwise the asynchronous rerendering of unselected blocks
+			// as otherwise the asynchronous re-rendering of unselected blocks
 			// may cause the input to have not yet been updated for the other blocks
 			await input.click();
-			expect( await input.getProperty( 'value' ) ).toBe( 'Meta Value' );
-		} );
+			const inputValue = await input.getProperty( 'value' );
+			expect( await inputValue.jsonValue() ).toBe( 'Meta Value' );
+		} ) );
 	} );
 } );
diff --git a/packages/jest-puppeteer-axe/CHANGELOG.md b/packages/jest-puppeteer-axe/CHANGELOG.md
@@ -1,3 +1,9 @@
+## Unreleased
+
+### New features
+
+- Added optional `disabledRules` option to use with `toPassAxeTests` matcher.
+
 ## 1.0.0 (2019-03-06)
 
 - Initial release.
diff --git a/packages/jest-puppeteer-axe/README.md b/packages/jest-puppeteer-axe/README.md
@@ -38,8 +38,9 @@ test( 'checks the test page with Axe', async () => {
 ```
 
 It is also possible to pass optional Axe API options to perform customized check:
-- `include` - CSS selector to to add the list of elements to include in analysis.
-- `exclude` - CSS selector to to add the list of elements to exclude from analysis.
+- `include` - CSS selector(s) to to add the list of elements to include in analysis.
+- `exclude` - CSS selector(s) to to add the list of elements to exclude from analysis.
+- `disabledRules` - the list of [Axe rules](https://github.com/dequelabs/axe-core/blob/master/doc/rule-descriptions.md) to skip from verification.
 
 ```js
 test( 'checks the test component with Axe excluding some button', async () => {
@@ -50,6 +51,7 @@ test( 'checks the test component with Axe excluding some button', async () => {
 	await expect( page ).toPassAxeTests( {
 		include: '.test-component',
 		exclude: '.some-button',
+		disabledRules: [ 'aria-allowed-role' ],
 	} );
 } );
 ```

diff --git a/packages/jest-puppeteer-axe/src/index.js b/packages/jest-puppeteer-axe/src/index.js
@@ -11,8 +11,9 @@ import AxePuppeteer from 'axe-puppeteer';
  * @return {string} The user friendly message to display when the matcher fails.
  */
 function formatViolations( violations ) {
-	return violations.map( ( { help, id, nodes } ) => {
-		let output = `Rule: ${ id } (${ help })\n` +
+	return violations.map( ( { help, helpUrl, id, nodes } ) => {
+		let output = `Rule: "${ id }" (${ help })\n` +
+			`Help: ${ helpUrl }\n` +
 			'Affected Nodes:\n';
 
 		nodes.forEach( ( node ) => {
@@ -52,16 +53,17 @@ function formatViolations( violations ) {
  *
  * @see https://github.com/dequelabs/axe-puppeteer
  *
- * @param {Page}    page             Puppeteer's page instance.
- * @param {?Object} params           Optional Axe API options.
- * @param {?string} params.include   CSS selector to add to the list of elements
- *                                   to include in analysis.
- * @param {?string} params.exclude   CSS selector to add to the list of elements
- *                                   to exclude from analysis.
+ * @param {Page}          page                 Puppeteer's page instance.
+ * @param {?Object}       params               Optional Axe API options.
+ * @param {?string|Array} params.include       CSS selector(s) to add to the list of elements
+ *                                             to include in analysis.
+ * @param {?string|Array} params.exclude       CSS selector(s) to add to the list of elements
+ *                                             to exclude from analysis.
+ * @param {?Array}        params.disabledRules The list of Axe rules to skip from verification.
  *
  * @return {Object} A matcher object with two keys `pass` and `message`.
  */
-async function toPassAxeTests( page, { include, exclude } = {} ) {
+async function toPassAxeTests( page, { include, exclude, disabledRules } = {} ) {
 	const axe = new AxePuppeteer( page );
 
 	if ( include ) {
@@ -72,6 +74,10 @@ async function toPassAxeTests( page, { include, exclude } = {} ) {
 		axe.exclude( exclude );
 	}
 
+	if ( disabledRules ) {
+		axe.disableRules( disabledRules );
+	}
+
 	const { violations } = await axe.analyze();
 
 	const pass = violations.length === 0;