Use WordPress Playground

.github/workflows/test-basic.yml

@@ -1,4 +1,4 @@
-name: 'build-test'
+name: 'Basic Tests'
 on: # rebuild any PRs and main branch changes
   pull_request:
   push:
@@ -8,6 +8,7 @@ on: # rebuild any PRs and main branch changes
 
 jobs:
   basic:
+    name: 'Run tests'
     runs-on: ubuntu-latest
     steps:
     - name: Checkout

.github/workflows/test-comparison.yml

@@ -0,0 +1,87 @@
+name: 'Test with comparison'
+on: # rebuild any PRs and main branch changes
+  pull_request:
+  push:
+    branches:
+    - main
+    - 'releases/*'
+
+jobs:
+  comparison-same:
+    name: 'Run tests with same version'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Run performance tests (before)
+      id: before
+      uses: ./
+      with:
+        urls: |
+          /
+          /sample-page/
+        plugins: |
+          ./tests/dummy-plugin
+        blueprint: './tests/blueprint-complex.json'
+        iterations: 5
+        repetitions: 1
+        wp-version: 'latest'
+        print-results: false
+        upload-artifacts: false
+
+    - name: Run performance tests (after)
+      id: after
+      uses: ./
+      with:
+        urls: |
+          /
+          /sample-page/
+        plugins: |
+          ./tests/dummy-plugin
+        blueprint: './tests/blueprint-complex.json'
+        iterations: 5
+        repetitions: 1
+        wp-version: 'latest'
+        previous-results: ${{ steps.before.outputs.results }}
+        print-results: true
+        upload-artifacts: false
+  comparison-different:
+    name: 'Run tests with different versions'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Run performance tests (before)
+      id: before
+      uses: ./
+      with:
+        urls: |
+          /
+          /sample-page/
+        plugins: |
+          ./tests/dummy-plugin
+        blueprint: './tests/blueprint-complex.json'
+        iterations: 5
+        repetitions: 1
+        wp-version: '6.5'
+        print-results: false
+        upload-artifacts: false
+
+    - name: Run performance tests (after)
+      id: after
+      uses: ./
+      with:
+        urls: |
+          /
+          /sample-page/
+        plugins: |
+          ./tests/dummy-plugin
+        blueprint: './tests/blueprint-complex.json'
+        iterations: 5
+        repetitions: 1
+        wp-version: '6.6'
+        previous-results: ${{ steps.before.outputs.results }}
+        print-results: true
+        upload-artifacts: false

.github/workflows/test-complex.yml

@@ -1,4 +1,4 @@
-name: 'build-test'
+name: 'Complex Tests'
 on: # rebuild any PRs and main branch changes
   pull_request:
   push:
@@ -8,6 +8,7 @@ on: # rebuild any PRs and main branch changes
 
 jobs:
   complex:
+    name: 'Run tests'
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
@@ -20,9 +21,8 @@ jobs:
           /
           /sample-page/
         plugins: |
-          ./dummy-plugin
-          https://downloads.wordpress.org/plugin/performant-translations.zip
-          https://downloads.wordpress.org/plugin/wordpress-seo.zip
+          ./tests/dummy-plugin
+        blueprint: './tests/blueprint-complex.json'
         iterations: 5
         repetitions: 1
         wp-version: 'trunk'

.github/workflows/test-matrix.yml

@@ -1,4 +1,4 @@
-name: 'build-test'
+name: 'Matrix Tests'
 on: # rebuild any PRs and main branch changes
   pull_request:
   push:
@@ -8,6 +8,7 @@ on: # rebuild any PRs and main branch changes
 
 jobs:
   matrix:
+    name: 'Run tests'
     timeout-minutes: 60
     runs-on: ubuntu-latest
     strategy:
@@ -25,10 +26,11 @@ jobs:
           /
           /sample-page/
         plugins: |
-          ./dummy-plugin
+          ./tests/dummy-plugin
         shard: ${{ matrix.shard }}
 
   merge-reports:
+    name: 'Merge reports'
     # Merge reports after playwright-tests, even if some shards have failed
     if: always()
     needs: [matrix]

.github/workflows/test-pr-comment.yml

@@ -1,4 +1,4 @@
-name: 'test-pr-comment'
+name: 'Test with PR comment'
 on: # rebuild any PRs and main branch changes
   pull_request:
   push:
@@ -8,6 +8,7 @@ on: # rebuild any PRs and main branch changes
 
 jobs:
   basic:
+    name: 'Run tests'
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
@@ -21,3 +22,4 @@ jobs:
           /sample-page/
         create-comment: true
         github-token: ${{ secrets.PR_COMMENT_TOKEN }}
+        repetitions: 5

README.md

@@ -4,7 +4,11 @@ A GitHub action to measure performance metrics of WordPress sites.
 
 Results are posted as comments to pull requests and as [GitHub Action job summaries](https://github.blog/2022-05-09-supercharging-github-actions-with-job-summaries/).
 
-It collects data from the `Server-Timing` header and runs Lighthouse on a given set of URLs. 
+It collects data from the `Server-Timing` header and runs Lighthouse on a given set of URLs.
+
+**Note:** Tests are run using [WordPress Playground](https://wordpress.org/playground/), which means you can use [blueprints](https://wordpress.github.io/wordpress-playground/blueprints) to prepare the test environment suitable to your needs.
+
+## Example
 
 <img width="1154" alt="Screenshot of a GitHub Action job summary output by this action" src="https://github.com/swissspidy/wp-performance-action/assets/841956/bb543ba2-a142-49d5-bb45-77d11f121824">
 
@@ -24,59 +28,78 @@ See [action.yml](action.yml)
     github-token: ''
 
     # Whether to create PR comments with performance results.
+    #
     # Might require a custom `github-token` to be set.
     #
     # Default: false
     create-comment: ''
 
+    # Whether to add results to the workflow summary.
+    #
+    # Default: true
+    print-results: ''
+
+    # Whether to upload any artifacts.
+    #
+    # Default: true
+    upload-artifacts: ''
+
     # Whether to log additional debugging information
     #
     # Default: ${{ runner.debug == '1' }}
     debug: ''
 
     # List of URLs on the WordPress site to test.
+    #
     # Each URL should be separated with new lines.
     #
     # Default: ''
     urls: ''
 
-    # List of plugins to install.
+    # List of plugin directories to mount.
+    #
     # Each plugin should be separated with new lines.
-    # Supports paths to local directories or ZIP URLs.
-    # Performance Lab (performance-lab) is always installed.
+    # Needs to be a path to a local directory.
+    # For installing plugins from the plugin directory
+    # or a ZIP file, use a blueprint.
     #
     # Default: ''
     plugins: ''
 
-    # List of themes to install.
+    # List of theme directories to mount.
+    #
     # Each theme should be separated with new lines.
-    # Supports paths to local directories or ZIP URLs.
-    # Twenty Twenty-One (twentytwentyone) and Twenty Twenty-Three (twentytwentythree)
-    # are always installed.
+    # Needs to be a path to a local directory.
+    # For installing themes from the theme directory
+    # or a ZIP file, use a blueprint.
     #
     # Default: ''
     themes: ''
 
-    # Theme to activate on the site.
-    # Theme needs to be already installed.
+    # Blueprint to use for setting up the environment.
+    #
+    # Use this to install or activate additional plugins, defining constants,
+    # and much more.
     #
-    # Default: 'twentytwentyone'
-    active-theme: ''
+    # See https://wordpress.github.io/wordpress-playground/blueprints for more information.
+    #
+    # Default: ''
+    blueprint: ''
 
     # WordPress version to use.
-    # Supports aliases such as latest, nightly, or trunk.
-    # Also supports ZIP URLs or a Git reference from https://github.com/WordPress/wordpress
-    # to install a specific version.
+    #
+    # Loads the specified WordPress version.
+    # Accepts the last four major WordPress versions.
+    # You can also use the generic values 'latest', 'nightly', or 'beta'.
     #
     # Default: 'latest'
     wp-version: ''
 
     # PHP version to use.
-    # Defaults to whatever version is the default
-    # in the Docker-maintained WordPress image
-    # (currently 8.0 as of November 2023)
     #
-    # Default: 'auto'
+    # Accepts 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2, 8.3.
+    #
+    # Default: 'latest'
     php-version: ''
 
     # Number of times the tests should be repeated.
@@ -133,6 +156,8 @@ steps:
 
 ### Advanced
 
+Add a workflow (`.github/workflows/build-test.yml`):
+
 ```yaml
 steps:
   - name: Checkout
@@ -146,12 +171,37 @@ steps:
         /sample-page/
       plugins: |
         ./my-awesome-plugin
-        https://downloads.wordpress.org/plugin/performant-translations.zip
-        https://downloads.wordpress.org/plugin/wordpress-seo.zip
+      blueprint: ./my-custom-blueprint.json
       iterations: 5
       repetitions: 1
 ```
 
+Add a blueprint (`my-custom-blueprint.json`):
+
+```json
+{
+  "$schema": "https://playground.wordpress.net/blueprint-schema.json",
+  "plugins": [
+    "performant-translations",
+    "akismet"
+  ],
+  "steps": [
+    {
+      "step": "defineWpConfigConsts",
+      "consts": {
+        "WP_DEBUG": true
+      }
+    },
+    {
+      "step": "activatePlugin",
+      "pluginName": "My Awesome Plugin",
+      "pluginPath": "/wordpress/wp-content/plugins/my-awesome-plugin"
+    }
+  ]
+}
+
+```
+
 ### Running tests in parallel (sharding)
 
 ```yaml