Migrate performance tests to Playwright (#51084)

.eslintrc.js

@@ -367,7 +367,7 @@ module.exports = {
 		},
 		{
 			files: [ 'packages/jest*/**/*.js', '**/test/**/*.js' ],
-			excludedFiles: [ 'test/e2e/**/*.js' ],
+			excludedFiles: [ 'test/e2e/**/*.js', 'test/performance/**/*.js' ],
 			extends: [ 'plugin:@wordpress/eslint-plugin/test-unit' ],
 		},
 		{
@@ -377,6 +377,7 @@ module.exports = {
 				'packages/react-native-*/**/*.[tj]s?(x)',
 				'test/native/**/*.[tj]s?(x)',
 				'test/e2e/**/*.[tj]s?(x)',
+				'test/performance/**/*.[tj]s?(x)',
 				'test/storybook-playwright/**/*.[tj]s?(x)',
 			],
 			extends: [
@@ -396,6 +397,7 @@ module.exports = {
 		{
 			files: [
 				'test/e2e/**/*.[tj]s',
+				'test/performance/**/*.[tj]s',
 				'packages/e2e-test-utils-playwright/**/*.[tj]s',
 			],
 			extends: [
@@ -406,6 +408,7 @@ module.exports = {
 				tsconfigRootDir: __dirname,
 				project: [
 					'./test/e2e/tsconfig.json',
+					'./test/performance/tsconfig.json',
 					'./packages/e2e-test-utils-playwright/tsconfig.json',
 				],
 			},

package.json

@@ -311,6 +311,7 @@
 		"test:e2e:watch": "npm run test:e2e -- --watch",
 		"test:performance": "wp-scripts test-e2e --config packages/e2e-tests/jest.performance.config.js",
 		"test:performance:debug": "wp-scripts --inspect-brk test-e2e --runInBand --no-cache --verbose --config packages/e2e-tests/jest.performance.config.js --puppeteer-devtools",
+		"test:performance:playwright": "playwright test --config test/performance/playwright.config.ts",
 		"test:php": "npm-run-all lint:php test:unit:php",
 		"test:php:watch": "wp-env run --env-cwd='wp-content/plugins/gutenberg' tests-cli composer run-script test:watch",
 		"test:unit": "wp-scripts test-unit-js --config test/unit/jest.config.js",

test/performance/assets/small-post-with-containers.html

@@ -0,0 +1,77 @@
+<!-- wp:columns {"align":"full"} -->
+<div class="wp-block-columns alignfull"><!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column -->
+
+<!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column -->
+
+<!-- wp:column -->
+<div class="wp-block-column"><!-- wp:heading {"level":3} -->
+<h3 class="wp-block-heading">Heading</h3>
+<!-- /wp:heading -->
+
+<!-- wp:paragraph -->
+<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>
+<!-- /wp:paragraph -->
+
+<!-- wp:list -->
+<ul><!-- wp:list-item -->
+<li>one</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>two</li>
+<!-- /wp:list-item -->
+
+<!-- wp:list-item -->
+<li>three</li>
+<!-- /wp:list-item --></ul>
+<!-- /wp:list --></div>
+<!-- /wp:column --></div>
+<!-- /wp:columns -->
+
+<!-- wp:paragraph -->
+<p></p>
+<!-- /wp:paragraph -->

test/performance/config/global-setup.ts

@@ -0,0 +1,31 @@
+/**
+ * External dependencies
+ */
+import { request } from '@playwright/test';
+import type { FullConfig } from '@playwright/test';
+
+/**
+ * WordPress dependencies
+ */
+import { RequestUtils } from '@wordpress/e2e-test-utils-playwright';
+
+async function globalSetup( config: FullConfig ) {
+	const { storageState, baseURL } = config.projects[ 0 ].use;
+	const storageStatePath =
+		typeof storageState === 'string' ? storageState : undefined;
+
+	const requestContext = await request.newContext( {
+		baseURL,
+	} );
+
+	const requestUtils = new RequestUtils( requestContext, {
+		storageStatePath,
+	} );
+
+	// Authenticate and save the storageState to disk.
+	await requestUtils.setupRest();
+
+	await requestContext.dispose();
+}
+
+export default globalSetup;

test/performance/config/performance-reporter.ts

@@ -0,0 +1,174 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+import chalk from 'chalk';
+import { readFileSync, existsSync } from 'fs';
+import type { Reporter, TestCase } from '@playwright/test/reporter';
+
+/**
+ * Internal dependencies
+ */
+import { average, round } from '../utils';
+
+const title = chalk.bold;
+const success = chalk.bold.green;
+
+class PerformanceReporter implements Reporter {
+	onTestEnd( test: TestCase ) {
+		const basename = path.basename( test.location.file, '.js' );
+		const filepath = path.join(
+			process.env.WP_ARTIFACTS_PATH as string,
+			basename + '.performance-results.json'
+		);
+
+		if ( ! existsSync( filepath ) ) {
+			return;
+		}
+
+		const results = readFileSync( filepath, 'utf8' );
+		const {
+			serverResponse,
+			firstPaint,
+			domContentLoaded,
+			loaded,
+			firstContentfulPaint,
+			firstBlock,
+			type,
+			typeContainer,
+			focus,
+			listViewOpen,
+			inserterOpen,
+			inserterHover,
+			inserterSearch,
+		} = JSON.parse( results );
+
+		if ( serverResponse && serverResponse.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Loading Time:' ) }
+Average time to server response (subtracted from client side metrics): ${ success(
+				round( average( serverResponse ) ) + 'ms'
+			) }
+Average time to first paint: ${ success(
+				round( average( firstPaint ) ) + 'ms'
+			) }
+Average time to DOM content load: ${ success(
+				round( average( domContentLoaded ) ) + 'ms'
+			) }
+Average time to load: ${ success( round( average( loaded ) ) + 'ms' ) }
+Average time to first contentful paint: ${ success(
+				round( average( firstContentfulPaint ) ) + 'ms'
+			) }
+Average time to first block: ${ success(
+				round( average( firstBlock ) ) + 'ms'
+			) }` );
+		}
+
+		if ( type && type.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Typing:' ) }
+Average time to type character: ${ success( round( average( type ) ) + 'ms' ) }
+Slowest time to type character: ${ success(
+				round( Math.max( ...type ) ) + 'ms'
+			) }
+Fastest time to type character: ${ success(
+				round( Math.min( ...type ) ) + 'ms'
+			) }` );
+		}
+
+		if ( typeContainer && typeContainer.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Typing within a container:' ) }
+Average time to type within a container: ${ success(
+				round( average( typeContainer ) ) + 'ms'
+			) }
+Slowest time to type within a container: ${ success(
+				round( Math.max( ...typeContainer ) ) + 'ms'
+			) }
+Fastest time to type within a container: ${ success(
+				round( Math.min( ...typeContainer ) ) + 'ms'
+			) }` );
+		}
+
+		if ( focus && focus.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Block Selection:' ) }
+Average time to select a block: ${ success( round( average( focus ) ) + 'ms' ) }
+Slowest time to select a block: ${ success(
+				round( Math.max( ...focus ) ) + 'ms'
+			) }
+Fastest time to select a block: ${ success(
+				round( Math.min( ...focus ) ) + 'ms'
+			) }` );
+		}
+
+		if ( listViewOpen && listViewOpen.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Opening List View:' ) }
+Average time to open list view: ${ success(
+				round( average( listViewOpen ) ) + 'ms'
+			) }
+Slowest time to open list view: ${ success(
+				round( Math.max( ...listViewOpen ) ) + 'ms'
+			) }
+Fastest time to open list view: ${ success(
+				round( Math.min( ...listViewOpen ) ) + 'ms'
+			) }` );
+		}
+
+		if ( inserterOpen && inserterOpen.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Opening Global Inserter:' ) }
+Average time to open global inserter: ${ success(
+				round( average( inserterOpen ) ) + 'ms'
+			) }
+Slowest time to open global inserter: ${ success(
+				round( Math.max( ...inserterOpen ) ) + 'ms'
+			) }
+Fastest time to open global inserter: ${ success(
+				round( Math.min( ...inserterOpen ) ) + 'ms'
+			) }` );
+		}
+
+		if ( inserterSearch && inserterSearch.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Inserter Search:' ) }
+Average time to type the inserter search input: ${ success(
+				round( average( inserterSearch ) ) + 'ms'
+			) }
+Slowest time to type the inserter search input: ${ success(
+				round( Math.max( ...inserterSearch ) ) + 'ms'
+			) }
+Fastest time to type the inserter search input: ${ success(
+				round( Math.min( ...inserterSearch ) ) + 'ms'
+			) }` );
+		}
+
+		if ( inserterHover && inserterHover.length ) {
+			// eslint-disable-next-line no-console
+			console.log( `
+${ title( 'Inserter Block Item Hover:' ) }
+Average time to move mouse between two block item in the inserter: ${ success(
+				round( average( inserterHover ) ) + 'ms'
+			) }
+Slowest time to move mouse between two block item in the inserter: ${ success(
+				round( Math.max( ...inserterHover ) ) + 'ms'
+			) }
+Fastest time to move mouse between two block item in the inserter: ${ success(
+				round( Math.min( ...inserterHover ) ) + 'ms'
+			) }` );
+		}
+
+		// eslint-disable-next-line no-console
+		console.log( '' );
+	}
+}
+
+export default PerformanceReporter;

test/performance/playwright.config.ts

@@ -0,0 +1,64 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { defineConfig, devices } from '@playwright/test';
+
+process.env.WP_ARTIFACTS_PATH ??= path.join( process.cwd(), 'artifacts' );
+process.env.STORAGE_STATE_PATH ??= path.join(
+	process.env.WP_ARTIFACTS_PATH,
+	'storage-states/admin.json'
+);
+process.env.ASSETS_PATH = path.join( __dirname, 'assets' );
+
+const config = defineConfig( {
+	reporter: process.env.CI
+		? undefined // We're using another reporter in CI.
+		: [ [ 'list' ], [ './config/performance-reporter.ts' ] ],
+	forbidOnly: !! process.env.CI,
+	fullyParallel: false,
+	workers: 1,
+	retries: 0,
+	timeout: parseInt( process.env.TIMEOUT || '', 10 ) || 600_000, // Defaults to 10 minutes.
+	testDir: fileURLToPath( new URL( './specs', 'file:' + __filename ).href ),
+	outputDir: path.join( process.env.WP_ARTIFACTS_PATH, 'test-results' ),
+	snapshotPathTemplate:
+		'{testDir}/{testFileDir}/__snapshots__/{arg}-{projectName}{ext}',
+	globalSetup: fileURLToPath(
+		new URL( './config/global-setup.ts', 'file:' + __filename ).href
+	),
+	use: {
+		baseURL: process.env.WP_BASE_URL || 'http://localhost:8889',
+		headless: true,
+		viewport: {
+			width: 960,
+			height: 700,
+		},
+		ignoreHTTPSErrors: true,
+		locale: 'en-US',
+		contextOptions: {
+			reducedMotion: 'reduce',
+			strictSelectors: true,
+		},
+		storageState: process.env.STORAGE_STATE_PATH,
+		actionTimeout: 10_000, // 10 seconds.
+		trace: 'retain-on-failure',
+		screenshot: 'only-on-failure',
+		video: 'off',
+	},
+	webServer: {
+		command: 'npm run wp-env start',
+		port: 8889,
+		timeout: 120_000, // 120 seconds.
+		reuseExistingServer: true,
+	},
+	projects: [
+		{
+			name: 'chromium',
+			use: { ...devices[ 'Desktop Chrome' ] },
+		},
+	],
+} );
+
+export default config;