NEW e2e environment using playground cli

.github/workflows/e2e-tests.yml

@@ -1,43 +1,176 @@
 name: E2E Tests
+
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
+  pull_request:
+    paths:
+    - '.github/workflows/e2e-tests.yml'
+    - 'build/**'
+    - 'includes/**'
+    - 'src/**'
+    - 'test/e2e/**'
+    - '*.php'
+    - 'package.*'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name for pull requests
+  # or the commit hash for any other events.
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
 
 jobs:
-  playwright:
-    name: Playwright Tests
+  e2e-tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 1 # Prevent parallel runs to make use of the caching for node_modules and playwright browsers
+      matrix:
+          php: [
+            '7.4',
+            '8.3'
+          ]
+
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Log debug information
-        run: |
-          npm --version
-          node --version
-          git --version
-          php --version
-          composer --version
-
-      - name: Install NodeJS
-        uses: actions/setup-node@v4
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - uses: actions/cache@v4
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }}
+
+    - name: Setup Node
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: '.nvmrc'
+        # Enable built-in functionality for caching and restoring dependencies, which is disabled by default.
+        # The actions/setup-node uses actions/cache under the hood.
+        # https://github.com/actions/setup-node#caching-global-packages-data
+        cache: 'npm'
+
+    # Restoring the short lived node_modules cache
+    # to be used across all workflows running on the last commit.
+    # https://github.com/actions/cache/blob/main/caching-strategies.md#creating-a-short-lived-cache
+    - uses: actions/cache/restore@v4
+      id: node_modules-cache
+      with:
+        path: |
+          ./node_modules
+        key: ${{ runner.os }}-node_modules-${{ github.sha }}-${{ hashFiles('package-lock.json') }}
+
+    - name: NPM install
+      if: steps.node_modules-cache.outputs.cache-hit != 'true'
+      run: npm ci --legacy-peer-deps
+
+    # Creating a short lived node_modules cache
+    - uses: actions/cache/save@v4
+      if: steps.node_modules-cache.outputs.cache-hit != 'true'
+      with:
+        path: |
+          ./node_modules
+        key: ${{ steps.node_modules-cache.outputs.cache-primary-key }}
+
+    - name: Install Playwright dependencies
+      run: npx playwright install --with-deps
+      if: steps.playwright-cache.outputs.cache-hit != 'true'
+    - run: npx playwright install-deps
+      if: steps.playwright-cache.outputs.cache-hit == 'true'
+
+    - name: Prepare preferredVersions for blueprint
+      # playground/cli ignores --php & --wp flags when --blueprint is set
+      #
+      # I created an issue at:
+      # https://github.com/WordPress/playground-tools/issues/352
+      run: |
+        preferredVersions='{
+          "php": "${{ matrix.php }}",
+          "wp": "latest"
+        }'
+        echo "Use jq to prepend the preferred versions to the existing blueprint JSON file."
+        jq --argjson preferredVersions "$preferredVersions" '{preferredVersions: $preferredVersions} + .' ./test/e2e/blueprint.json > temp.blueprint.json
+
+    - name: Starting Playground, staring Playwright & running the tests
+      # This is kind of a hack PART 1/2,
+      # to make sure Playwright DOES NOT start the webserver on its own.
+      #
+      # Part 2/2 is the "command: undefined," declaration
+      # in test/e2e/playwright.config.ts
+      #
+      # While auto-loading the webserver when needed sounded nice, it introduced a race-condition
+      # between the setup of Playground and Playwrights own start event.
+      # Playwright listens for the availability of the webserver relatively simple,
+      # as soon as there is a status code 200, Playwright starts all engines.
+      #
+      # Unfortunately Playground is not ready at this point, it hast started WordPress
+      # and is going to start stepping through the blueprint, but hasn't loaded GatherPress nor imported any data;
+      # Resulting in failing tests.
+      #
+      # It was not possible (for me) to keep the setup of Playground in a separate step,
+      # why this "run > sleep > run" became necessary.
+      # The setup process usually takes about 20sec, so 60 is just a good extra, to not run into errors.
+      #
+      # The sleep step, should help making sure Playwright runs only, when this is DONE & READY.
+      #
+      # Because it seems to be a problem to "wait on webServer.command" https://github.com/microsoft/playwright/issues/11811
+      # & "it seems that globalSetup runs before webServer is started." https://github.com/microsoft/playwright/issues/11811#issuecomment-1040732201
+
+      run: |
+        npm run playground:mount -- --blueprint=./temp.blueprint.json & \
+        sleep 60 && \
+        echo 'Playground is ready now, lets take some pictures.' && \
+        # DEBUG=pw:api,pw:webserver \
+        xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- \
+        npm run test:e2e
+
+
+    - name: Archive debug artifacts (screenshots, traces)
+      uses: actions/upload-artifact@v4
+      if: ${{ ! cancelled() }}
+      with:
+        name: failures-artifacts--${{ matrix.php }}
+        path: artifacts/test-results/
+        if-no-files-found: ignore
+
+
+    - name: Archive flaky tests report
+      uses: actions/upload-artifact@v4
+      if: ${{ ! cancelled() }}
+      with:
+          name: flaky-tests-report--${{ matrix.php }}
+          path: flaky-tests
+          if-no-files-found: ignore
+
+
+  merge-artifacts:
+    name: Merge Artifacts
+    if: ${{ ! cancelled() }}
+    needs: [e2e-tests]
+    runs-on: ubuntu-latest
+    outputs:
+      has-flaky-test-report: ${{ !!steps.merge-flaky-tests-reports.outputs.artifact-id }}
+    steps:
+      - name: Merge failures artifacts
+        uses: actions/upload-artifact/merge@v4
+        # Don't fail the job if there aren't any artifacts to merge.
+        continue-on-error: true
+        with:
+          name: failures-artifacts
+          # Retain the merged artifacts in case of a rerun.
+          pattern: failures-artifacts*
+          delete-merged: true
+
+      - name: Merge flaky tests reports
+        id: merge-flaky-tests-reports
+        uses: actions/upload-artifact/merge@v4
+        continue-on-error: true
         with:
-          node-version-file: '.nvmrc'
-          # Enable built-in functionality for caching and restoring dependencies, which is disabled by default.
-          # The actions/setup-node uses actions/cache under the hood.
-          # https://github.com/actions/setup-node#caching-global-packages-data
-          cache: 'npm'
-
-      - name: NPM install
-        run: npm ci --legacy-peer-deps
-
-      - name: Playwright install
-        run: npx playwright install --with-deps
-
-      - name: Run Playwright
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          WP_ADMIN_PASSWORD: ${{ secrets.WP_ADMIN_PASSWORD }}
-        run: npm run test:e2e
-        if: ${{ success() || failure() }}
+          name: flaky-tests-report
+          pattern: flaky-tests-report*
+          delete-merged: true

docs/contributor/e2e-tests/README.md

@@ -0,0 +1,109 @@
+# End-to-end Testing using Playwright & Playground
+
+GatherPress allows to run **automated & manual end-to-end tests** (e2e), while sharing the same, [`wp-playground/cli`](https://github.com/WordPress/wordpress-playground/pull/1289) powered, setup. The started playground imports the [`GatherPress/demo-data`](https://github.com/GatherPress/demo-data), that can be used instead of mocks or fixtures.
+
+## Automated tests
+
+Check the results of the [*e2e-tests action workflow*](https://github.com/GatherPress/gatherpress/actions/workflows/e2e-tests.yml) at  `https://github.com/GatherPress/gatherpress/actions/workflows/e2e-tests.yml`.
+
+## Manual testing
+
+The e2e-setup can also be used to **manually run the test suite**. In general, only a Playground instance is needed.
+
+### Install dependencies
+
+To run the e2e tests you will have to install playwright using the following command:
+
+```bash
+npx playwright install --with-deps
+```
+
+> [!NOTE]
+> You also need to use Node.js 20 or later
+
+Install the dependencies to create the Playground testing instance, using the following command:
+
+```bash
+npm ci --legacy-peer-deps
+```
+
+### Start the Playground
+
+A call to `npm run playground:mount` will automatically setup a `wp-playground/cli` powered WordPress instance, already prepared to mount GatherPress from the current directory.
+
+
+```bash
+npm run playground:mount -- --blueprint=./test/e2e/blueprint.json
+```
+
+The testing website is reachable at `http://127.0.0.1:9400`, the user is `admin` and the password is `password`. 
+
+### Run the e2e tests
+
+Now while Playground is running in the background, it's possible to start the testing.
+
+#### *Choose one of the following options*
+
+1. For the *headless* mode, use the following command:
+
+   ```bash
+   npm run test:e2e
+   ```
+
+2. Run Playwright *visually* (to run tests in isolation and change what's happening), use:
+
+   ```bash
+   npm run test:e2e:ui
+   ```
+
+   ![grafik](https://github.com/user-attachments/assets/1627dff7-363e-447e-9981-adac610ac888)
+
+
+3. For *debug* mode (which will open the browser along with Playwright Editor and allows you to record what's happening), use the following command:
+
+   ```bash
+   npm run test:e2e:debug
+   ```
+
+   Run all the tests against a specific project.
+
+   ```bash
+   npm run test:e2e:debug -- project=webkit
+   ```
+
+   Run files that have *events.spec* in the file name.
+
+   ```bash
+   npm run test:e2e:debug -- events.spec
+   ```
+
+   > [!NOTE]
+   > When writing a test, using the debug mode is recommended since it will allow you to see the browser and the test in action.
+
+4. Run Tests independently *AND* visually using the [Playwright VSCode extension](https://playwright.dev/docs/getting-started-vscode)
+
+
+### More about e2e testing
+
+#### Start here
+
+- [Playwright Documentation](https://playwright.dev/docs/intro)
+- [End-To-End Playwright test utils for WordPress](https://github.com/WordPress/gutenberg/blob/trunk/packages/e2e-test-utils-playwright/README.md)
+
+#### from the WordPress handbooks
+
+- [End-to-End Testing – Block Editor Handbook | Developer.WordPress.org](https://developer.wordpress.org/block-editor/contributors/code/testing-overview/e2e/)
+- [Overusing snapshots – Block Editor Handbook | Developer.WordPress.org](https://developer.wordpress.org/block-editor/contributors/code/testing-overview/e2e/overusing-snapshots/)
+
+### More about `wp-playground/cli`, as the testing environment
+
+Examples with great documentation:
+
+- [Playground CLI · WordPress/wordpress-playground#1289](https://github.com/WordPress/wordpress-playground/pull/1289)
+- [PoC: Run e2e tests with WP Playground · WordPress/gutenberg#62692](https://github.com/WordPress/gutenberg/pull/62692)
+- [Use WordPress Playground · swissspidy/wp-performance-action#173](https://github.com/swissspidy/wp-performance-action/pull/173)
+
+To see more examples of e2e tests, check the Gutenberg repository: [github.com/WordPress/gutenberg/tree/trunk/test/e2e](https://github.com/WordPress/gutenberg/tree/trunk/test/e2e)
+
+> [!NOTE]
+> If you are out of ideas on who to test, check the Gutenberg repository. It has a lot of examples of e2e tests that you can use as a reference.

package.json

@@ -87,7 +87,10 @@
 		"screenshots:wporg": "WP_BASE_URL='http://127.0.0.1:9400/' wp-scripts test-playwright --config .github/scripts/wordpress-org-screenshots/playwright.config.ts",
 		"screenshots:wporg:debug": "npm run screenshots:wporg -- --debug",
 		"screenshots:wporg:ui": "npm run screenshots:wporg -- --ui",
-		"test:e2e": "playwright test --config=playwright.config.js",
+		"pretest:e2e": "npm run build",
+		"test:e2e": "WP_BASE_URL='http://127.0.0.1:9400/' wp-scripts test-playwright --config test/e2e/playwright.config.ts",
+		"test:e2e:debug": "npm run test:e2e -- --debug",
+		"test:e2e:ui": "npm run test:e2e -- --ui",
 		"test:unit:js": "wp-scripts test-unit-js --coverage --testResultsProcessor=jest-sonar-reporter",
 		"pretest:unit:php": "wp-env start --xdebug",
 		"test:unit:php": "wp-env run tests-wordpress php -dxdebug.mode=coverage /var/www/html/wp-content/plugins/gatherpress/vendor/bin/phpunit -c /var/www/html/wp-content/plugins/gatherpress/phpunit.xml.dist --coverage-clover=coverage.xml --coverage-html=build/coverage-report",