Add infrastructure for writing Playwright E2E tests (WordPress#38570)

* Add basic infrastructure of using @playwright/test

* Fix comment of visitAdminPage

* Add initial guideline for the migration

* Mark e2e-test-utils-playwright as private for now

* Fix link name

* Mark e2e-tests as private and deprecate puppeteer packages

Co-authored-by: Robert Anderson <robert@noisysocks.com>

.eslintrc.js

@@ -225,14 +225,39 @@ module.exports = {
 			},
 		},
 		{
-			files: [ 'packages/jest*/**/*.js' ],
+			files: [ 'packages/jest*/**/*.js', '**/test/**/*.js' ],
+			excludedFiles: [ 'test/e2e/**/*.js' ],
 			extends: [ 'plugin:@wordpress/eslint-plugin/test-unit' ],
 		},
 		{
-			files: [ 'packages/e2e-test*/**/*.js' ],
+			files: [
+				'packages/e2e-tests/**/*.js',
+				'packages/e2e-test-utils/**/*.js',
+			],
 			extends: [ 'plugin:@wordpress/eslint-plugin/test-e2e' ],
+		},
+		{
+			files: [
+				'test/e2e/**/*.[tj]s',
+				'packages/e2e-test-utils-playwright/**/*.[tj]s',
+			],
+			extends: [ 'plugin:@wordpress/eslint-plugin/test-e2e-playwright' ],
 			rules: {
-				'jest/expect-expect': 'off',
+				'@wordpress/no-global-active-element': 'off',
+				'@wordpress/no-global-get-selection': 'off',
+				'no-restricted-syntax': [
+					'error',
+					{
+						selector: 'CallExpression[callee.name="$"]',
+						message:
+							'`$` is discouraged, please use `locator` instead',
+					},
+					{
+						selector: 'CallExpression[callee.name="$$"]',
+						message:
+							'`$$` is discouraged, please use `locator` instead',
+					},
+				],
 			},
 		},
 		{

.github/report-flaky-tests/index.js

@@ -46,6 +46,7 @@ const metaData = {
 
 	for ( const flakyTest of flakyTests ) {
 		const {
+			runner: testRunner = 'jest-circus',
 			title: testTitle,
 			path: testPath,
 			results: testResults,
@@ -92,7 +93,11 @@ const metaData = {
 					body.indexOf( TEST_RESULTS_LIST.close )
 				) +
 				[
-					renderTestErrorMessage( { testPath, testResults } ),
+					renderTestErrorMessage( {
+						testRunner,
+						testPath,
+						testResults,
+					} ),
 					TEST_RESULTS_LIST.close,
 				].join( '\n' );
 
@@ -121,6 +126,7 @@ const metaData = {
 				title: issueTitle,
 				body: renderIssueBody( {
 					meta,
+					testRunner,
 					testTitle,
 					testPath,
 					testResults,
@@ -187,10 +193,16 @@ function getIssueTitle( testTitle ) {
 	return `[Flaky Test] ${ testTitle }`;
 }
 
-function renderIssueBody( { meta, testTitle, testPath, testResults } ) {
+function renderIssueBody( {
+	meta,
+	testRunner,
+	testTitle,
+	testPath,
+	testResults,
+} ) {
 	return (
 		renderIssueDescription( { meta, testTitle, testPath } ) +
-		renderTestResults( { testPath, testResults } )
+		renderTestResults( { testRunner, testPath, testResults } )
 	);
 }
 
@@ -211,14 +223,41 @@ ${ testTitle }
 `;
 }
 
-function renderTestResults( { testPath, testResults } ) {
+function renderTestResults( { testRunner, testPath, testResults } ) {
 	return `${ TEST_RESULTS_LIST.open }
-${ renderTestErrorMessage( { testPath, testResults } ) }
+${ renderTestErrorMessage( { testRunner, testPath, testResults } ) }
 ${ TEST_RESULTS_LIST.close }
 `;
 }
 
-function renderTestErrorMessage( { testPath, testResults } ) {
+function renderTestResults( { testRunner, testResults, testPath } ) {
+	switch ( testRunner ) {
+		case '@playwright/test': {
+			// Could do a slightly better formatting than this.
+			return stripAnsi(
+				testResults.map( ( result ) => result.error.stack ).join( '\n' )
+			);
+		}
+		case 'jest-circus':
+		default: {
+			return stripAnsi(
+				formatResultsErrors(
+					testResults,
+					{
+						rootDir: path.join(
+							process.cwd(),
+							'packages/e2e-tests'
+						),
+					},
+					{},
+					testPath
+				)
+			);
+		}
+	}
+}
+
+function renderTestErrorMessage( { testRunner, testPath, testResults } ) {
 	const date = new Date().toISOString();
 	// It will look something like this without formatting:
 	// ▶ [2021-08-31T16:15:19.875Z] Test passed after 2 failed attempts on trunk
@@ -231,14 +270,7 @@ function renderTestErrorMessage( { testPath, testResults } ) {
 </summary>
 
 \`\`\`
-${ stripAnsi(
-	formatResultsErrors(
-		testResults,
-		{ rootDir: path.join( process.cwd(), 'packages/e2e-tests' ) },
-		{},
-		testPath
-	)
-) }
+${ renderTestResults( { testRunner, testPath, testResults } ) }
 \`\`\`
 </details>${ TEST_RESULT.close }`;
 }

.github/workflows/end2end-test-playwright.yml

@@ -0,0 +1,68 @@
+name: End-to-End Tests Playwright
+
+on:
+    pull_request:
+    push:
+        branches:
+            - trunk
+            - 'release/**'
+            - 'wp/**'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+    # The concurrency group contains the workflow name and the branch name for pull requests
+    # or the commit hash for any other events.
+    group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+    cancel-in-progress: true
+
+jobs:
+    e2e:
+        name: E2E Tests
+        runs-on: ubuntu-latest
+        if: ${{ github.repository == 'WordPress/gutenberg' || github.event_name == 'pull_request' }}
+        strategy:
+            fail-fast: false
+            matrix:
+                node: ['14']
+
+        steps:
+            - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f # v2.3.4
+
+            - name: Use desired version of NodeJS
+              uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f # v2.2.2
+              with:
+                  node-version: ${{ matrix.node }}
+                  cache: npm
+
+            - name: Npm install and build
+              run: |
+                  npm ci
+                  npm run build
+
+            - name: Install Playwright dependencies
+              run: |
+                  npx playwright install chromium --with-deps
+
+            - name: Install WordPress and start the server
+              run: |
+                  npm run wp-env start
+
+            - name: Run the tests
+              run: |
+                  npm run test-e2e:playwright
+
+            - name: Archive debug artifacts (screenshots, traces)
+              uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2
+              if: always()
+              with:
+                  name: failures-artifacts
+                  path: artifacts
+                  if-no-files-found: ignore
+
+            - name: Archive flaky tests report
+              uses: actions/upload-artifact@e448a9b857ee2131e752b06002bf0e093c65e571 # v2.2.2
+              if: always()
+              with:
+                  name: flaky-tests-report
+                  path: flaky-tests
+                  if-no-files-found: ignore

.github/workflows/flaky-tests.yml

@@ -2,6 +2,8 @@ name: Report Flaky Tests
 
 on:
     workflow_run:
+        # We should also add 'End-to-End Tests Playwright' here but that
+        # wil run this workflow whenever either one of them completes.
         workflows: ['End-to-End Tests']
         types:
             - completed

bin/packages/build.js

@@ -224,6 +224,7 @@ if ( files.length ) {
 				`**/benchmark/**`,
 				`**/{__mocks__,__tests__,test}/**`,
 				`**/{storybook,stories}/**`,
+				`**/e2e-test-utils-playwright/**`,
 			],
 			onlyFiles: true,
 		}

bin/packages/watch.js

@@ -64,7 +64,7 @@ function isSourceFile( filename ) {
 	return (
 		/\/src\/.+\.(js|json|scss|ts|tsx)$/.test( relativePath ) &&
 		! [
-			/\/(benchmark|__mocks__|__tests__|test|storybook|stories)\/.+/,
+			/\/(benchmark|__mocks__|__tests__|test|storybook|stories|e2e-test-utils-playwright)\/.+/,
 			/.\.(spec|test)\.js$/,
 		].some( ( regex ) => regex.test( relativePath ) )
 	);

docs/contributors/code/testing-overview.md

@@ -485,7 +485,9 @@ There is an ongoing effort to add integration tests to the native mobile project
 
 ## End-to-end Testing
 
-End-to-end tests use [Puppeteer](https://github.com/puppeteer/puppeteer) as a headless Chromium driver, and are otherwise still run by a [Jest](https://jestjs.io/) test runner.
+End-to-end tests currently use [Puppeteer](https://github.com/puppeteer/puppeteer) as a headless Chromium driver to run the tests in `packages/e2e-tests`, and are otherwise still run by a [Jest](https://jestjs.io/) test runner.
+
+> There's a ongoing [project](https://github.com/WordPress/gutenberg/issues/38851) to migrate them from Puppeteer to Playwright. See the [README](https://github.com/WordPress/gutenberg/tree/HEAD/test/e2e/README.md) of the new E2E tests for the updated guideline and best practices.
 
 ### Using wp-env
 

docs/manifest.json

@@ -1505,12 +1505,6 @@
 		"markdown_source": "../packages/e2e-test-utils/README.md",
 		"parent": "packages"
 	},
-	{
-		"title": "@wordpress/e2e-tests",
-		"slug": "packages-e2e-tests",
-		"markdown_source": "../packages/e2e-tests/README.md",
-		"parent": "packages"
-	},
 	{
 		"title": "@wordpress/edit-post",
 		"slug": "packages-edit-post",