build-test-measure.yml

name: Build, test & measure

on:
  push:
    branches:
      - develop
      # Include all release branches.
      - '[0-9]+.[0-9]+'
  pull_request:
    # Run workflow whenever a PR is opened, updated (synchronized), or marked ready for review.
    types: [opened, synchronize, ready_for_review]

# Cancel previous workflow run groups that have not completed.
concurrency:
  # Group workflow runs by workflow name, along with the head branch ref of the pull request
  # or otherwise the branch or tag ref.
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.ref }}
  cancel-in-progress: true

# Disable permissions for all available scopes.
# Enable permissions for specific scopes as needed on job level.
permissions: {}

jobs:
  pre-run:
    name: 'Pre run'
    runs-on: ubuntu-latest
    outputs:
      changed-file-count: ${{ steps.determine-file-counts.outputs.count }}
      changed-php-count: ${{ steps.determine-file-counts.outputs.php-count }}
      changed-css-count: ${{ steps.determine-file-counts.outputs.css-count }}
      changed-js-count: ${{ steps.determine-file-counts.outputs.js-count }}
      changed-gha-workflow-count: ${{ steps.determine-file-counts.outputs.gha-workflow-count }}
    steps:
      - name: Checkout including last 2 commits
        # Fetch last 2 commits if it's not a PR, so that we can determine the list of modified files.
        if: ${{ github.base_ref == null }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2
        with:
          fetch-depth: 2

      - name: Checkout
        # Do usual checkout if it's a PR.
        if: ${{ github.base_ref != null }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Determine modified files
        id: determine-file-counts
        uses: ./.github/actions/determine-changed-files

#-----------------------------------------------------------------------------------------------------------------------

  lint-css:
    needs: pre-run
    if: needs.pre-run.outputs.changed-css-count > 0 || needs.pre-run.outputs.changed-gha-workflow-count > 0
    name: 'Lint: CSS'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup NodeJS and NPM
        uses: ./.github/actions/setup-node-npm

      - name: Detect coding standard violations (stylelint)
        run: npm run lint:css

#-----------------------------------------------------------------------------------------------------------------------

  lint-js:
    name: 'Lint: JS'
    needs: pre-run
    if: needs.pre-run.outputs.changed-js-count > 0 || needs.pre-run.outputs.changed-gha-workflow-count > 0
    runs-on: ubuntu-latest
    permissions:
      checks: write
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup NodeJS and NPM
        uses: ./.github/actions/setup-node-npm

      - name: Validate package.json
        run: npm run lint:pkg-json

      - name: Detect ESLint coding standard violations
        if: >
          github.event.pull_request.head.repo.fork == true ||
          github.event.pull_request.user.login == 'dependabot[bot]'
        run: npm run lint:js

      - name: Generate ESLint coding standard violations report
        # Prevent generating the ESLint report if PR is from a fork or authored by Dependabot.
        if: >
          ! ( github.event.pull_request.head.repo.fork == true ||
          github.event.pull_request.user.login == 'dependabot[bot]' )
        run: npm run lint:js:report
        continue-on-error: true

      - name: Annotate code linting results
        # The action cannot annotate the PR when run from a PR fork or was authored by Dependabot.
        if: >
          ! ( github.event.pull_request.head.repo.fork == true ||
          github.event.pull_request.user.login == 'dependabot[bot]' )
        uses: ataylorme/eslint-annotate-action@d57a1193d4c59cbfbf3f86c271f42612f9dbd9e9 #3.0.0
        with:
          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
          report-json: 'lint-js-report.json'

#-----------------------------------------------------------------------------------------------------------------------

  lint-php:
    name: 'Lint: PHP'
    needs: pre-run
    if: needs.pre-run.outputs.changed-php-count > 0 || needs.pre-run.outputs.changed-gha-workflow-count > 0
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup PHP and Composer
        uses: ./.github/actions/setup-php-composer
        with:
          tools: 'composer,cs2pr,composer-normalize'
          php-version: '8.1'

      - name: Detect coding standard violations (PHPCS)
        run: vendor/bin/phpcs -q --report=checkstyle --runtime-set ignore_errors_on_exit 1 --runtime-set ignore_warnings_on_exit 1 | cs2pr --graceful-warnings

      - name: Validate composer.json
        run: composer --no-interaction validate --no-check-all

      - name: Normalize composer.json
        run: |
          composer config --no-interaction --no-plugins allow-plugins.ergebnis/composer-normalize true
          composer-normalize --dry-run --diff

#-----------------------------------------------------------------------------------------------------------------------

  static-analysis-php:
    name: 'Static Analysis: PHP'
    runs-on: ubuntu-latest
    needs: pre-run
    if: needs.pre-run.outputs.changed-php-count > 0 || needs.pre-run.outputs.changed-gha-workflow-count > 0
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup PHP and Composer
        uses: ./.github/actions/setup-php-composer
        with:
          tools: 'composer,phpstan'
          php-version: '8.1'

      - name: Static Analysis (PHPStan)
        run: phpstan analyze

#-----------------------------------------------------------------------------------------------------------------------

  unit-test-js:
    name: 'Unit test: JS'
    runs-on: ubuntu-latest
    needs: pre-run
    if: needs.pre-run.outputs.changed-js-count > 0 || needs.pre-run.outputs.changed-gha-workflow-count > 0
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup NodeJS and NPM
        uses: ./.github/actions/setup-node-npm

      - name: Setup Jest cache
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a #v4.1.2
        with:
          path: ~/.jest-cache
          key: ${{ runner.os }}-jest

      - name: Run unit tests (with coverage)
        run: npm run test:js -- --ci --cacheDirectory="$HOME/.jest-cache" --collectCoverage

      - name: Upload code coverage report
        if: github.actor != 'dependabot[bot]'
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 #v4.6.0
        with:
          file: build/logs/lcov.info
          flags: javascript
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

#-----------------------------------------------------------------------------------------------------------------------

  e2e-test-js:
    name: 'E2E test: JS'
    needs: pre-run
    if: needs.pre-run.outputs.changed-file-count > 0
    runs-on: ubuntu-latest
    permissions:
      actions: write
    env:
      WP_BASE_URL: 'http://127.0.0.1:8890'
      JEST_PUPPETEER_CONFIG: './tests/e2e/puppeteer.config.js'
    strategy:
      fail-fast: false
      matrix:
          part: [1, 2]
          totalParts: [2]
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup NodeJS and NPM
        uses: ./.github/actions/setup-node-npm

      - name: Setup PHP and Composer
        uses: ./.github/actions/setup-php-composer

      - name: Build plugin
        uses: ./.github/actions/plugin-build

      - name: Setup Jest cache
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a #v4.1.2
        with:
          path: ~/.jest-cache
          key: ${{ runner.os }}-jest-e2e-${{ matrix.part }}

      - name: Start Docker environment
        run: npm run env:start:ci
        env:
          COMPOSE_INTERACTIVE_NO_CLI: true

        # Avoid adding any logs to stdout/stderr, while getting the list of E2E tests.
      - name: Set NPM log level
        run: export npm_config_loglevel=silent

      - name: Get E2E test files
        run: npx wp-scripts test-e2e --config=tests/e2e/jest-ci.config.js --listTests > $RUNNER_TEMP/.jest-e2e-tests

      - name: Unset NPM log level
        run: unset npm_config_loglevel

      - name: Run E2E tests
        run: npx wp-scripts test-e2e --config=tests/e2e/jest-ci.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % ${{ matrix.totalParts }} == ${{ matrix.part }} - 1' < $RUNNER_TEMP/.jest-e2e-tests )

      - name: Stop Docker environment
        run: npm run env:stop:ci
        if: always()
        env:
          COMPOSE_INTERACTIVE_NO_CLI: true

      - name: Archive debug artifacts (screenshots, HTML snapshots)
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 #v4.4.3
        if: always()
        with:
          name: e2e-failure-artifacts
          path: artifacts
          if-no-files-found: ignore

#-----------------------------------------------------------------------------------------------------------------------

  # Adapted from workflow for running PHP unit tests on google/web-stories-wp.
  # See https://github.com/google/web-stories-wp/blob/cb2ebada48039171e25c279bdb27d3712dd70b22/.github/workflows/continuous-integration-unit-php.yml
  unit-test-php:
    name: "Unit test${{ matrix.coverage && ' (with coverage)' || '' }}: PHP ${{ matrix.php }}, WP ${{ matrix.wp }}"
    runs-on: ubuntu-latest
    needs: pre-run
    env:
      WP_CORE_DIR: /tmp/wordpress
      WP_TESTS_DIR: /tmp/wordpress-tests-lib
      WP_ENVIRONMENT_TYPE: local
    services:
      mysql:
        image: mariadb:latest
        env:
          MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: true
          MARIADB_DATABASE: wordpress_test
          MARIADB_MYSQL_LOCALHOST_USER: 1
          MARIADB_MYSQL_LOCALHOST_GRANTS: USAGE
        ports:
          - 3306
        options: --health-cmd="healthcheck.sh --su-mysql --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
    continue-on-error: ${{ matrix.experimental == true }}
    strategy:
      fail-fast: false
      matrix:
        php: ['8.0']
        wp: ['latest']
        external-http: [false]
        install-pwa-plugin: [true]
        coverage: [false]
        include:
          - php: '8.4'
            wp: 'trunk'
            phpunit: '9.6'
            experimental: true

          - php: '8.3'
            wp: 'trunk'
            phpunit: '9.6'
            experimental: true

          - php: '8.2'
            wp: 'trunk'
            phpunit: '9.6'

          - php: '8.1'
            wp: 'trunk'
            phpunit: '9.6'

          - php: '8.0'
            wp: 'trunk'
            phpunit: '9.3'
            multisite: true

          - php: '8.0'
            wp: 'latest'
            phpunit: '9.3'
            coverage: true

          - php: '8.0'
            wp: 'latest'
            phpunit: '9.3'
            external-http: true

          - php: '7.4'
            wp: 'latest'
            phpunit: '9.3'
            multisite: true

          - php: '7.4'
            wp: 'latest'
            phpunit: '9.3'
            external-http: true

          - php: '7.4'
            wp: '6.5'
            phpunit: '7'

          - php: '7.4'
            wp: '6.5'
            phpunit: '7'
            external-http: true
    steps:
      # Note: The repeated `needs.pre-run.outputs.changed-php-count > 0` checks would be avoided if a step could short-
      # circuit per <https://github.com/actions/runner/issues/662>. The reason why the if statement can't be put on the
      # job as a whole is because the name is variable based on the matrix, and if the condition is not met then the
      # name won't be interpolated in order to match the required jobs set up in branch protection.
      - name: Notice
        if: needs.pre-run.outputs.changed-php-count == 0
        run: echo "No PHP files were changed so no PHP unit tests will run"

      - name: Shutdown default MySQL service
        if: needs.pre-run.outputs.changed-php-count > 0
        run: sudo service mysql stop

      - name: Verify MariaDB connection
        if: needs.pre-run.outputs.changed-php-count > 0
        run: |
          retry_count=0
          max_retries=5

          while [ $retry_count -lt $max_retries ]; do
            if mysqladmin ping -h"127.0.0.1" -P"${{ job.services.mysql.ports[3306] }}" --silent; then
              break
            else
              ((retry_count++))
              sleep 5
            fi
          done

          if [ $retry_count -ge $max_retries ]; then
            echo "::error::Failed to establish a MariaDB connection after $max_retries retries."
            exit 1
          fi

      - name: Checkout
        if: needs.pre-run.outputs.changed-php-count > 0
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup PHP and Composer
        if: needs.pre-run.outputs.changed-php-count > 0
        uses: ./.github/actions/setup-php-composer
        with:
          php-version: ${{ matrix.php }}
          coverage: ${{ matrix.coverage && 'xdebug' || 'none' }}
          tools: 'composer,phpunit:${{ matrix.phpunit }}'

      - name: Setup NodeJS and NPM
        if: needs.pre-run.outputs.changed-php-count > 0
        uses: ./.github/actions/setup-node-npm

      - name: Build plugin
        if: needs.pre-run.outputs.changed-php-count > 0
        uses: ./.github/actions/plugin-build

      # Avoid conflicts with globally installed PHPUnit.
      - name: Remove locally installed PHPUnit
        if: needs.pre-run.outputs.changed-php-count > 0
        run: |
          rm -rf vendor/phpunit
          composer dump-autoload -o

      # Scan the logs for failing tests and surface that information by creating annotations and log file decorations.
      - name: Setup problem matcher to provide annotations for PHPUnit
        if: needs.pre-run.outputs.changed-php-count > 0
        # The JSON file is provided by the `shivammathur/setup-php` action. See https://github.com/shivammathur/setup-php#problem-matchers.
        run: echo "::add-matcher::${{ runner.tool_cache }}/phpunit.json"

      - name: Install WP tests
        if: needs.pre-run.outputs.changed-php-count > 0
        run: bash bin/ci/install-wp-tests.sh wordpress_test root '' 127.0.0.1:${{ job.services.mysql.ports['3306'] }} ${{ matrix.wp }} true

      - name: Post install of WP tests
        if: needs.pre-run.outputs.changed-php-count > 0
        run: bash bin/ci/after-wp-install.sh ${{ matrix.wp }} ${{ matrix.install-pwa-plugin == true }}

      - name: Copy plugin to WP plugins directory
        if: needs.pre-run.outputs.changed-php-count > 0
        run: cp -r "$PWD" "$WP_CORE_DIR/src/wp-content/plugins/amp"

      - name: Override default PHPUnit configuration
        if: ${{ matrix.experimental == true && needs.pre-run.outputs.changed-php-count > 0 }}
        run: |
          cp phpunit.xml.dist phpunit.xml

          # Avoid converting deprecations, errors, notices, and warnings to exceptions in experimental mode.
          sed -i 's/convertDeprecationsToExceptions="true"/convertDeprecationsToExceptions="false"/g' phpunit.xml
          sed -i 's/convertErrorsToExceptions="true"/convertErrorsToExceptions="false"/g' phpunit.xml
          sed -i 's/convertNoticesToExceptions="true"/convertNoticesToExceptions="false"/g' phpunit.xml
          sed -i 's/convertWarningsToExceptions="true"/convertWarningsToExceptions="false"/g' phpunit.xml
        working-directory: ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp

      - name: Run Single Site tests
        if: ${{ matrix.multisite == false && needs.pre-run.outputs.changed-php-count > 0 }}
        run: |
            if [[ ${{ matrix.external-http == false }} == true && ${{ matrix.coverage == false }} == true ]]; then
              phpunit --verbose
            elif [[ ${{ matrix.external-http == true }} == true && ${{ matrix.coverage == false }} == true ]]; then
              phpunit --testsuite external-http --verbose
            elif [[ ${{ matrix.external-http == false }} == true && ${{ matrix.coverage == true }} == true ]]; then
              phpunit --verbose --coverage-clover ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp/build/logs/clover.xml
            elif [[ ${{ matrix.external-http == true }} == true && ${{ matrix.coverage == true }} == true ]]; then
              phpunit --testsuite external-http --verbose --coverage-clover ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp/build/logs/clover.xml
            fi
        working-directory: ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp

      - name: Run multisite tests
        if: ${{ matrix.multisite == true && needs.pre-run.outputs.changed-php-count > 0 }}
        run: |
          if [[ ${{ matrix.external-http == false }} == true && ${{ matrix.coverage == false }} == true ]]; then
            phpunit --verbose
          elif [[ ${{ matrix.external-http == true }} == true && ${{ matrix.coverage == false }} == true ]]; then
            phpunit --testsuite external-http --verbose
          elif [[ ${{ matrix.external-http == false }} == true && ${{ matrix.coverage == true }} == true ]]; then
            phpunit --verbose --coverage-clover ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp/build/logs/clover.xml
          elif [[ ${{ matrix.external-http == true }} == true && ${{ matrix.coverage == true }} == true ]]; then
            phpunit --testsuite external-http --verbose --coverage-clover ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp/build/logs/clover.xml
          fi
        working-directory: ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp
        env:
          WP_MULTISITE: 1

      - name: Upload code coverage report
        if: ${{ matrix.coverage == true && needs.pre-run.outputs.changed-php-count > 0 && github.actor != 'dependabot[bot]' }}
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 #v4.6.0
        with:
          file: ${{ env.WP_CORE_DIR }}/src/wp-content/plugins/amp/build/logs/clover.xml
          flags: php,unit
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}

#-----------------------------------------------------------------------------------------------------------------------

  feature-test-php:
    name: "Feature tests: PHP ${{ matrix.php }}, WP ${{ matrix.wp }}"
    needs: pre-run
    if: needs.pre-run.outputs.changed-php-count > 0
    runs-on: ubuntu-latest
    env:
      # @see <https://github.com/wp-cli/wp-cli-tests?tab=readme-ov-file#wordpress-version>.
      WP_VERSION: ${{ matrix.wp }}
    services:
      mysql:
        image: mariadb:latest
        env:
          MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: true
          MARIADB_DATABASE: wordpress_test
          MARIADB_MYSQL_LOCALHOST_USER: 1
          MARIADB_MYSQL_LOCALHOST_GRANTS: USAGE
        ports:
          - 3306
        options: --health-cmd="healthcheck.sh --su-mysql --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
    continue-on-error: ${{ matrix.experimental == true }}
    strategy:
      fail-fast: false
      matrix:
        php: ['7.4']
        wp: ['latest']
        include:
          - php: '8.4'
            wp: 'trunk'
            experimental: true

          - php: '8.3'
            wp: 'trunk'
            experimental: true

          - php: '8.2'
            wp: 'trunk'
            experimental: true

          - php: '8.1'
            wp: 'trunk'

          - php: '8.0'
            wp: 'latest'

          - php: '7.4'
            wp: '6.5'
    steps:
      - name: Shutdown default MySQL service
        if: needs.pre-run.outputs.changed-php-count > 0
        run: sudo service mysql stop

      - name: Verify MariaDB connection
        if: needs.pre-run.outputs.changed-php-count > 0
        run: |
          retry_count=0
          max_retries=5

          while [ $retry_count -lt $max_retries ]; do
            if mysqladmin ping -h"127.0.0.1" -P"${{ job.services.mysql.ports[3306] }}" --silent; then
              break
            else
              ((retry_count++))
              sleep 5
            fi
          done

          if [ $retry_count -ge $max_retries ]; then
            echo "::error::Failed to establish a MariaDB connection after $max_retries retries."
            exit 1
          fi

      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #v4.2.2

      - name: Setup PHP and Composer
        uses: ./.github/actions/setup-php-composer
        with:
          php-version: ${{ matrix.php }}

      - name: Setup NodeJS and NPM
        uses: ./.github/actions/setup-node-npm

      - name: Build plugin
        uses: ./.github/actions/plugin-build

      - name: Configure DB environment
        run: |
          echo "MYSQL_HOST=127.0.0.1" >> $GITHUB_ENV
          echo "MYSQL_TCP_PORT=${{ job.services.mysql.ports['3306'] }}" >> $GITHUB_ENV
          echo "WP_CLI_TEST_DBROOTUSER=root" >> $GITHUB_ENV
          echo "WP_CLI_TEST_DBROOTPASS=" >> $GITHUB_ENV
          echo "WP_CLI_TEST_DBUSER=wp_cli_test" >> $GITHUB_ENV
          echo "WP_CLI_TEST_DBPASS=password1" >> $GITHUB_ENV
          echo "WP_CLI_TEST_DBHOST=127.0.0.1:${{ job.services.mysql.ports['3306'] }}" >> $GITHUB_ENV

      - name: Prepare test database
        run: composer prepare-tests

      - name: Run tests
        run: ./vendor/bin/behat