diff --git a/.gitignore b/.gitignore index 5b69fc33deb..e5910ecfea4 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ local.env # Screenshots created locally when running e2e tests tests/e2e/screenshots + +# E2E Performance test results +tests/e2e/reports \ No newline at end of file diff --git a/changelog/add-1021-performance-measure b/changelog/add-1021-performance-measure new file mode 100644 index 00000000000..b6ae9927fa7 --- /dev/null +++ b/changelog/add-1021-performance-measure @@ -0,0 +1,4 @@ +Significance: patch +Type: add + +Add E2E test to measure checkout page performance diff --git a/package.json b/package.json index 950654fead7..68fd12148f6 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "test:e2e-reset": "npm run test:e2e-down && npm run test:e2e-cleanup", "test:e2e": "NODE_CONFIG_DIR='tests/e2e/config' wp-scripts test-e2e --config tests/e2e/config/jest.config.js", "test:e2e-dev": "NODE_CONFIG_DIR='tests/e2e/config' JEST_PUPPETEER_CONFIG='tests/e2e/config/jest-puppeteer.config.js' wp-scripts test-e2e --config tests/e2e/config/jest.config.js --puppeteer-interactive", + "test:e2e-performance": "NODE_CONFIG_DIR='tests/e2e/config' wp-scripts test-e2e --config tests/e2e/config/jest.performance.config.js", "test:update-snapshots": "npm run test:js -- --updateSnapshot", "test:php": "./bin/run-tests.sh", "watch": "webpack --watch", diff --git a/tests/e2e/config/jest.performance.config.js b/tests/e2e/config/jest.performance.config.js new file mode 100644 index 00000000000..5d3c11ab1cc --- /dev/null +++ b/tests/e2e/config/jest.performance.config.js @@ -0,0 +1,15 @@ +const path = require( 'path' ); +const { useE2EJestConfig } = require( '@woocommerce/e2e-environment' ); + +// eslint-disable-next-line react-hooks/rules-of-hooks +const testConfig = useE2EJestConfig( { + setupFiles: [], + rootDir: path.resolve( __dirname, '../../../' ), + roots: [ path.resolve( __dirname, '../specs/performance' ) ], + testSequencer: path.resolve( + __dirname, + '../config/jest-custom-sequencer.js' + ), +} ); + +module.exports = testConfig; diff --git a/tests/e2e/specs/performance/payment-methods.spec.js b/tests/e2e/specs/performance/payment-methods.spec.js new file mode 100644 index 00000000000..63425346c5a --- /dev/null +++ b/tests/e2e/specs/performance/payment-methods.spec.js @@ -0,0 +1,115 @@ +/** + * External dependencies + */ +import config from 'config'; +const { merchant } = require( '@woocommerce/e2e-utils' ); + +/** + * Internal dependencies + */ +import { setupProductCheckout } from '../../utils/payments'; +import { shopperWCP, merchantWCP } from '../../utils'; +import { + recreatePerformanceFile, + logPerformanceResult, + measureCheckoutMetrics, + averageMetrics, +} from '../../utils/performance'; + +describe( 'Checkout page performance', () => { + beforeAll( async () => { + // Start a new file for every run. + recreatePerformanceFile(); + } ); + + describe( 'Stripe element', () => { + beforeEach( async () => { + await setupProductCheckout( + config.get( 'addresses.customer.billing' ) + ); + } ); + + afterEach( async () => { + // Clear the cart at the end so it's ready for another test + await shopperWCP.emptyCart(); + } ); + + it( 'measures averaged page load metrics', async () => { + const results = await measureCheckoutMetrics( + '#wcpay-card-element iframe' + ); + logPerformanceResult( + 'Stripe element: Average', + averageMetrics( results ) + ); + } ); + } ); + + describe( 'UPE', () => { + beforeEach( async () => { + // Activate UPE + await merchant.login(); + await merchantWCP.activateUpe(); + await merchant.logout(); + + // Setup cart + await setupProductCheckout( + config.get( 'addresses.customer.billing' ) + ); + } ); + + afterEach( async () => { + // Clear the cart at the end so it's ready for another test + await shopperWCP.emptyCart(); + + // Deactivate UPE + await merchant.login(); + await merchantWCP.deactivateUpe(); + await merchant.logout(); + } ); + + it( 'measures averaged page load metrics', async () => { + const results = await measureCheckoutMetrics( + '#wcpay-upe-element iframe' + ); + logPerformanceResult( + 'Stripe UPE: Average', + averageMetrics( results ) + ); + } ); + } ); + + describe( 'WooPay without UPE', () => { + beforeEach( async () => { + // Activate UPE + await merchant.login(); + await merchantWCP.activateWooPay(); + await merchant.logout(); + + // Setup cart + await setupProductCheckout( + config.get( 'addresses.customer.billing' ) + ); + } ); + + afterEach( async () => { + // Clear the cart at the end so it's ready for another test + await shopperWCP.emptyCart(); + + // Deactivate UPE + await merchant.login(); + await merchantWCP.deactivateWooPay(); + await merchant.logout(); + } ); + + it( 'measures averaged page load metrics', async () => { + const results = await measureCheckoutMetrics( + '#wcpay-card-element iframe' + ); + logPerformanceResult( + 'WooPay: Average', + averageMetrics( results ) + ); + } ); + } ); +} ); diff --git a/tests/e2e/utils/constants.js b/tests/e2e/utils/constants.js new file mode 100644 index 00000000000..90d328360c4 --- /dev/null +++ b/tests/e2e/utils/constants.js @@ -0,0 +1,9 @@ +/** + * Constants used for E2E tests. + * + * @type {string} + */ +export const PERFORMANCE_REPORT_DIR = __dirname + '/../reports/'; +export const PERFORMANCE_REPORT_FILENAME = + PERFORMANCE_REPORT_DIR + 'checkout-performance.txt'; +export const NUMBER_OF_TRIALS = 3; diff --git a/tests/e2e/utils/flows.js b/tests/e2e/utils/flows.js index d670c3a7565..0a1edea53b6 100644 --- a/tests/e2e/utils/flows.js +++ b/tests/e2e/utils/flows.js @@ -494,4 +494,62 @@ export const merchantWCP = { await checkbox.click(); } }, + + activateWooPay: async () => { + await page.goto( WCPAY_DEV_TOOLS, { + waitUntil: 'networkidle0', + } ); + + if ( + ! ( await page.$( '#override_platform_checkout_eligible:checked' ) ) + ) { + await expect( page ).toClick( 'label', { + text: + 'Override the platform_checkout_eligible flag in the account cache', + } ); + } + + if ( + ! ( await page.$( + '#override_platform_checkout_eligible_value:checked' + ) ) + ) { + await expect( page ).toClick( 'label', { + text: + 'Set platform_checkout_eligible flag to true, false otherwise', + } ); + } + + await expect( page ).toClick( 'input[type="submit"]' ); + await page.waitForNavigation( { + waitUntil: 'networkidle0', + } ); + }, + + deactivateWooPay: async () => { + await page.goto( WCPAY_DEV_TOOLS, { + waitUntil: 'networkidle0', + } ); + + if ( await page.$( '#override_platform_checkout_eligible:checked' ) ) { + await expect( page ).toClick( 'label', { + text: + 'Override the platform_checkout_eligible flag in the account cache', + } ); + } + + if ( + await page.$( '#override_platform_checkout_eligible_value:checked' ) + ) { + await expect( page ).toClick( 'label', { + text: + 'Set platform_checkout_eligible flag to true, false otherwise', + } ); + } + + await expect( page ).toClick( 'input[type="submit"]' ); + await page.waitForNavigation( { + waitUntil: 'networkidle0', + } ); + }, }; diff --git a/tests/e2e/utils/performance.js b/tests/e2e/utils/performance.js new file mode 100644 index 00000000000..24a3e996b77 --- /dev/null +++ b/tests/e2e/utils/performance.js @@ -0,0 +1,138 @@ +/** + * External dependencies + */ +import { appendFileSync, existsSync, mkdirSync, truncateSync } from 'fs'; + +/** + * Internal dependencies + */ +import { + PERFORMANCE_REPORT_DIR, + PERFORMANCE_REPORT_FILENAME, + NUMBER_OF_TRIALS, +} from './constants'; + +async function getLoadingDurations() { + return await page.evaluate( () => { + const { + requestStart, + responseStart, + responseEnd, + domContentLoadedEventEnd, + loadEventEnd, + } = performance.getEntriesByType( 'navigation' )[ 0 ]; + const paintTimings = performance.getEntriesByType( 'paint' ); + + let firstPaintTimings, firstContentfulPaintTimings; + + paintTimings.forEach( ( item ) => { + if ( 'first-paint' === item.name ) { + firstPaintTimings = item; + } + if ( 'first-contentful-paint' === item.name ) { + firstContentfulPaintTimings = item; + } + } ); + + // Returns metrics in milliseconds (10^-3). Spec uses DOMHighResTimeStamp https://www.w3.org/TR/hr-time-2/#sec-domhighrestimestamp. + return { + // Server side metric. + serverResponse: responseStart - requestStart, + // For client side metrics, consider the end of the response (the + // browser receives the HTML) as the start time (0). + firstPaint: firstPaintTimings.startTime - responseEnd, + domContentLoaded: domContentLoadedEventEnd - responseEnd, + loaded: loadEventEnd - responseEnd, + firstContentfulPaint: + firstContentfulPaintTimings.startTime - responseEnd, + // This is evaluated right after Puppeteer found the block selector. + firstBlock: performance.now() - responseEnd, + }; + } ); +} + +/** + * Writes a line to the e2e performance result. + * + * @param {string} description A title that describe this metric + * @param {Object} metrics array of metrics to record. + */ +export const logPerformanceResult = ( description, metrics ) => { + appendFileSync( + PERFORMANCE_REPORT_FILENAME, + JSON.stringify( { description, ...metrics } ) + '\n' + ); +}; + +/** + * Wipe the existing performance file. Also make sure the "report" folder exists. + */ +export const recreatePerformanceFile = () => { + if ( ! existsSync( PERFORMANCE_REPORT_DIR ) ) { + mkdirSync( PERFORMANCE_REPORT_DIR ); + } + + if ( existsSync( PERFORMANCE_REPORT_FILENAME ) ) { + truncateSync( PERFORMANCE_REPORT_FILENAME ); + } +}; + +/** + * Takes the metric object and for each of the property, reduce to the average. + * + * @param {Object} metrics An object containing multiple trials' data. + * @return {Object} The averaged results. + */ +export const averageMetrics = ( metrics ) => { + const results = {}; + for ( const [ key, value ] of Object.entries( metrics ) ) { + results[ key ] = + value.reduce( ( prev, curr ) => prev + curr ) / NUMBER_OF_TRIALS; + } + return results; +}; + +/** + * This helper function goes to checkout page *i* times. Wait + * for the given card selector to load, retrieve all the metrics + * and find the average. + * + * @param {string} selector CSS selector. + * @param {number} numberOfTrials The number of trials we would like to do. + * @return {Object} The averaged results. + */ +export const measureCheckoutMetrics = async ( selector ) => { + await expect( page ).toMatch( 'Checkout' ); + + // Run performance tests a few times, then take the average. + const results = { + serverResponse: [], + firstPaint: [], + domContentLoaded: [], + loaded: [], + firstContentfulPaint: [], + firstBlock: [], + }; + + let i = NUMBER_OF_TRIALS; + while ( i-- ) { + await page.reload(); + await page.waitForSelector( selector ); + const { + serverResponse, + firstPaint, + domContentLoaded, + loaded, + firstContentfulPaint, + firstBlock, + } = await getLoadingDurations(); + + results.serverResponse.push( serverResponse ); + results.firstPaint.push( firstPaint ); + results.domContentLoaded.push( domContentLoaded ); + results.loaded.push( loaded ); + results.firstContentfulPaint.push( firstContentfulPaint ); + results.firstBlock.push( firstBlock ); + } + return results; +};