Add visual regression tests

.github/workflows/pr-closed.yml

@@ -13,4 +13,6 @@ jobs:
  host: ${{ secrets.PREVIEW_REMOTE_HOST }}
  username: ${{ secrets.PREVIEW_REMOTE_USER }}
  key: ${{ secrets.PREVIEW_SSH_KEY }}
- script: bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
+ script: |
+ bash /home/thephpfoundation/scripts/pr_closed.sh web-php ${{ github.event.number }}
+ bash /home/thephpfoundation/scripts/pr_closed.sh web-php-regression-report ${{ github.event.number }}

.github/workflows/pr-preview.yml

@@ -32,4 +32,99 @@ jobs:
  comment-id: ${{ steps.fc.outputs.comment-id }}
  edit-mode: 'replace'
  body: |
- 🚀 Commit ${{ github.sha }} Deployed on https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
+ 🚀 Preview for commit ${{ github.sha }} can be found at https://web-php-pr-${{ github.event.number }}.preview.thephp.foundation
+
+ tests_visual:
+ name: "Visual Tests"
+
+ runs-on: "ubuntu-latest"
+ if: github.repository_owner == 'php'
+
+ strategy:
+ matrix:
+ php-version:
+ - "8.2"
+ node-version:
+ - "22.x"
+
+ env:
+ HTTP_HOST: "localhost:8080"
+
+ steps:
+ - name: "Checkout"
+ uses: actions/checkout@v4
+ with:
+ ref: "refs/pull/${{ github.event.number }}/merge"
+
+ - name: "Set up PHP"
+ uses: "shivammathur/setup-php@v2"
+ with:
+ coverage: "none"
+ extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
+ php-version: "${{ matrix.php-version }}"
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+
+ - name: "Set up problem matchers for PHP"
+ run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
+
+ - name: "Set up problem matchers for phpunit/phpunit"
+ run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
+
+ - name: "Determine composer cache directory"
+ run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
+
+ - name: "Cache dependencies installed with composer"
+ uses: "actions/cache@v3"
+ with:
+ path: "${{ env.COMPOSER_CACHE_DIR }}"
+ key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
+ restore-keys: "php-${{ matrix.php-version }}-composer-"
+
+ - name: "Install dependencies with composer"
+ run: "composer install --ansi --no-interaction --no-progress"
+
+ - name: "Install dependencies"
+ run: "yarn install"
+
+ - name: "Install Playwright"
+ run: "npx playwright install"
+
+ - name: "Run visual tests"
+ run: "make tests_visual"
+
+ - uses: actions/upload-artifact@v4
+ if: ${{ !cancelled() }}
+ with:
+ name: playwright-report
+ path: playwright-report/
+ retention-days: 30
+
+ - uses: easingthemes/ssh-deploy@main
+ if: ${{ !cancelled() }}
+ with:
+ REMOTE_HOST: ${{ secrets.PREVIEW_REMOTE_HOST }}
+ REMOTE_USER: ${{ secrets.PREVIEW_REMOTE_USER }}
+ SSH_PRIVATE_KEY: ${{ secrets.PREVIEW_SSH_KEY }}
+ SOURCE: "playwright-report/"
+ TARGET: "/home/thephpfoundation/preview/web-php-regression-report-pr-${{ github.event.number }}/public"
+ SCRIPT_BEFORE: bash /home/thephpfoundation/scripts/pr_created_pre.sh web-php-regression-report ${{ github.event.number }}
+
+ - uses: peter-evans/find-comment@v3
+ if: ${{ !cancelled() }}
+ id: snapshot
+ with:
+ issue-number: ${{ github.event.number }}
+ comment-author: 'github-actions[bot]'
+
+ - uses: peter-evans/create-or-update-comment@v4
+ if: ${{ !cancelled() }}
+ with:
+ issue-number: ${{ github.event.number }}
+ comment-id: ${{ steps.snapshot.outputs.comment-id }}
+ edit-mode: 'replace'
+ body: |
+ 🚀 Regression report for commit ${{ github.sha }} is at https://web-php-regression-report-pr-${{ github.event.number }}.preview.thephp.foundation

.github/workflows/update-screenshots.yaml

@@ -0,0 +1,74 @@
+# https://docs.github.com/en/actions
+
+name: "Update screenshots"
+
+on:
+ workflow_dispatch:
+
+jobs:
+ tests_update_snapshots:
+ name: "Update Tests snapshots"
+
+ runs-on: "ubuntu-latest"
+
+ strategy:
+ matrix:
+ php-version:
+ - "8.2"
+ node-version:
+ - "22.x"
+
+ env:
+ HTTP_HOST: "localhost:8080"
+
+ steps:
+ - name: "Checkout"
+ uses: "actions/checkout@v4"
+
+ - name: "Set up PHP"
+ uses: "shivammathur/setup-php@v2"
+ with:
+ coverage: "none"
+ extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter"
+ php-version: "${{ matrix.php-version }}"
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+
+ - name: "Set up problem matchers for PHP"
+ run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
+
+ - name: "Set up problem matchers for phpunit/phpunit"
+ run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
+
+ - name: "Determine composer cache directory"
+ run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
+
+ - name: "Cache dependencies installed with composer"
+ uses: "actions/cache@v3"
+ with:
+ path: "${{ env.COMPOSER_CACHE_DIR }}"
+ key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
+ restore-keys: "php-${{ matrix.php-version }}-composer-"
+
+ - name: "Install dependencies with composer"
+ run: "composer install --ansi --no-interaction --no-progress"
+
+ - name: "Install dependencies"
+ run: "yarn install"
+
+ - name: "Install Playwright"
+ run: "npx playwright install"
+
+ - name: "Run visual tests"
+ run: "make tests_update_snapshots"
+
+ - name: Update snapshots
+ uses: test-room-7/action-update-file@v1
+ if: ${{ !cancelled() }}
+ with:
+ file-path: tests/Visual/**/*
+ commit-msg: Update snapshots
+ github-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -5,3 +5,8 @@ backend/mirror.gif
 backend/mirror.png
 backend/mirror.jpg
 backend/GeoIP.dat
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/

.htaccess

@@ -1,4 +1,4 @@
-<FilesMatch "\.(inc|sql)$">
+<FilesMatch "\.(inc|sql|lock)$">
  Order allow,deny
  Deny from all
 </FilesMatch>

Makefile

@@ -1,6 +1,7 @@
 .EXPORT_ALL_VARIABLES:
 
 HTTP_HOST:=localhost:8080
+CORES?=$(shell (nproc || sysctl -n hw.ncpu) 2> /dev/null)
 
 .PHONY: it
 it: coding-standards tests ## Runs all the targets
@@ -21,7 +22,19 @@ help: ## Displays this list of targets with descriptions
 tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit
  vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=unit
  rm -rf tests/server.log
- tests/server start; vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end; tests/server stop
+ tests/server start;
+ vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end;
+ tests/server stop
+
+tests_visual:
+ tests/server start;
+ npx playwright test --workers=$(CORES)
+ tests/server stop
+
+tests_update_snapshots:
+ tests/server start;
+ npx playwright test --update-snapshots
+ tests/server stop
 
 vendor: composer.json composer.lock
  composer validate --strict

package.json

@@ -0,0 +1,8 @@
+{
+ "name": "php.net",
+ "devDependencies": {
+ "@playwright/test": "^1.44.0",
+ "@types/node": "^20.12.11"
+ },
+ "scripts": {}
+}

playwright.config.ts

@@ -0,0 +1,34 @@
+import {defineConfig, devices} from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+ testDir: './tests/Visual',
+ /* Run tests in files in parallel */
+ fullyParallel: true,
+ /* Fail the build on CI if you accidentally left test.only in the source code. */
+ forbidOnly: !!process.env.CI,
+ /* Retry on CI only */
+ retries: process.env.CI ? 2 : 0,
+ /* Opt out of parallel tests on CI. */
+ workers: process.env.CI ? 1 : undefined,
+ /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+ reporter: 'html',
+ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+ use: {
+ /* Base URL to use in actions like `await page.goto('/')`. */
+ // baseURL: 'http://127.0.0.1:3000',
+
+ /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+ trace: 'on-first-retry',
+ },
+ timeout: 0,
+
+ projects: [
+ {
+ name: 'chromium',
+ use: {...devices['Desktop Chrome']},
+ },
+ ],
+});

tests/Visual/SmokeTest.spec.ts

@@ -0,0 +1,63 @@
+import {expect, test} from '@playwright/test';
+
+export type TestPageOptions = {
+ path: string
+ options?: object
+ evaluate?: () => any
+}
+
+const items: TestPageOptions[] = [
+ {
+ path: 'index.php',
+ evaluate: () => {
+ const selector = document.querySelector('.elephpants');
+ selector.remove()
+ }
+ },
+ {
+ path: 'archive/1998.php',
+ evaluate: () => {
+ const selector = document.querySelector('.elephpants');
+ selector.remove()
+ }
+ },
+ {path: 'releases/8_3_6.php'},
+ {path: 'releases/8.0/index.php'},
+ {path: 'releases/8.1/index.php'},
+ {path: 'releases/8.2/index.php'},
+ {path: 'releases/8.3/index.php'},
+ {path: 'manual/index.php'},
+ {path: 'manual/php5.php'},
+ {
+ path: 'conferences/index.php',
+ options: {
+ fullPage: false,
+ }
+ },
+]
+
+for (const item of items) {
+ test(`testing with ${item.path}`, async ({page}, testInfo) => {
+ testInfo.snapshotSuffix = '';
+
+ const httpHost = process.env.HTTP_HOST
+
+ if (typeof httpHost !== 'string') {
+ throw new Error('Environment variable "HTTP_HOST" is not set.')
+ }
+
+ await page.goto(`${httpHost}/${item.path}`)
+
+ if (typeof item.evaluate === 'function') {
+ await page.evaluate(item.evaluate)
+ }
+
+ await expect(page).toHaveScreenshot(
+ `tests/screenshots/${item.path}.png`,
+ item.options ?? {
+ fullPage: true,
+ timeout: 10000,
+ }
+ )
+ })
+}

yarn.lock

@@ -0,0 +1,41 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@playwright/test@^1.44.0":
+ version "1.44.0"
+ resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.44.0.tgz#ac7a764b5ee6a80558bdc0fcbc525fcb81f83465"
+ integrity sha512-rNX5lbNidamSUorBhB4XZ9SQTjAqfe5M+p37Z8ic0jPFBMo5iCtQz1kRWkEMg+rYOKSlVycpQmpqjSFq7LXOfg==
+ dependencies:
+ playwright "1.44.0"
+
+"@types/node@^20.12.11":
+ version "20.12.11"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.12.11.tgz#c4ef00d3507000d17690643278a60dc55a9dc9be"
+ integrity sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==
+ dependencies:
+ undici-types "~5.26.4"
+
+fsevents@2.3.2:
+ version "2.3.2"
+ resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
+ integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
+
+playwright-core@1.44.0:
+ version "1.44.0"
+ resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.44.0.tgz#316c4f0bca0551ffb88b6eb1c97bc0d2d861b0d5"
+ integrity sha512-ZTbkNpFfYcGWohvTTl+xewITm7EOuqIqex0c7dNZ+aXsbrLj0qI8XlGKfPpipjm0Wny/4Lt4CJsWJk1stVS5qQ==
+
+playwright@1.44.0:
+ version "1.44.0"
+ resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.44.0.tgz#22894e9b69087f6beb639249323d80fe2b5087ff"
+ integrity sha512-F9b3GUCLQ3Nffrfb6dunPOkE5Mh68tR7zN32L4jCk4FjQamgesGay7/dAAe1WaMEGV04DkdJfcJzjoCKygUaRQ==
+ dependencies:
+ playwright-core "1.44.0"
+ optionalDependencies:
+ fsevents "2.3.2"
+
+undici-types@~5.26.4:
+ version "5.26.5"
+ resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
+ integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==