Smoke test daily #169
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Smoke test daily | |
| on: | |
| schedule: | |
| - cron: '25 3 * * *' | |
| workflow_dispatch: | |
| env: | |
| API_ARTIFACT: api-daily--run-${{ github.run_number }} | |
| E2E_ARTIFACT: e2e-daily--run-${{ github.run_number }} | |
| FORCE_COLOR: 1 | |
| PLUGIN_SLACK_BLOCKS_ARTIFACT: plugin-blocks | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: {} | |
| jobs: | |
| api-tests: | |
| name: API tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run-api-composite-action.outputs.result }} | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/api-core-tests/test-results/allure-report | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Load docker images and start containers. | |
| working-directory: plugins/woocommerce | |
| env: | |
| ENABLE_HPOS: 0 | |
| run: pnpm --filter=@woocommerce/plugin-woocommerce env:test | |
| - name: Run API tests | |
| id: run-api-composite-action | |
| uses: ./.github/actions/tests/run-api-tests | |
| with: | |
| report-name: ${{ env.API_ARTIFACT }} | |
| env: | |
| BASE_URL: http://localhost:8086 | |
| USER_KEY: admin | |
| USER_SECRET: password | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| - name: Generate Playwright API Test report. | |
| id: generate_api_report | |
| if: | | |
| always() && | |
| ( | |
| steps.run-api-composite-action.conclusion != 'cancelled' || | |
| steps.run-api-composite-action.conclusion != 'skipped' | |
| ) | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive Playwright API test report | |
| if: | | |
| always() && | |
| steps.generate_api_report.conclusion == 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 20 | |
| e2e-tests: | |
| name: E2E tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| outputs: | |
| test-result: ${{ steps.run_playwright_e2e_tests.outputs.result }} | |
| # needs: [api-tests] | |
| env: | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Load docker images and start containers. | |
| working-directory: plugins/woocommerce | |
| env: | |
| WP_ENV_PHP_VERSION: 7.4 | |
| run: pnpm env:test | |
| - name: Download and install Chromium browser. | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright install chromium | |
| - name: Run E2E tests | |
| timeout-minutes: 60 | |
| id: run_playwright_e2e_tests | |
| env: | |
| USE_WP_ENV: 1 | |
| E2E_MAX_FAILURES: 15 | |
| FORCE_COLOR: 1 | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-report | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/test-results/allure-results | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec playwright test --config=tests/e2e-pw/playwright.config.js | |
| - name: Generate Test report. | |
| if: success() || ( failure() && steps.run_playwright_e2e_tests.conclusion == 'failure' ) | |
| working-directory: plugins/woocommerce | |
| run: pnpm exec allure generate --clean ${{ env.ALLURE_RESULTS_DIR }} --output ${{ env.ALLURE_REPORT_DIR }} | |
| - name: Archive test report | |
| if: success() || ( failure() && steps.run_playwright_e2e_tests.conclusion == 'failure' ) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: | | |
| ${{ env.ALLURE_RESULTS_DIR }} | |
| ${{ env.ALLURE_REPORT_DIR }} | |
| retention-days: 20 | |
| k6-tests: | |
| name: k6 tests on nightly build | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| if: success() || failure() | |
| outputs: | |
| test-result: ${{ steps.run-k6-tests.conclusion }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| - name: Update performance test site with E2E test | |
| id: update-perf-site | |
| continue-on-error: true | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: k6-daily-update-site--run-${{ github.run_number }} | |
| tests: update-woocommerce.spec.js | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| BASE_URL: ${{ secrets.SMOKE_TEST_PERF_URL }}/ | |
| ADMIN_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| ADMIN_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| CUSTOMER_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| CUSTOMER_PASSWORD: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| DEFAULT_TIMEOUT_OVERRIDE: 120000 | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| UPDATE_WC: nightly | |
| - name: Install k6 | |
| run: | | |
| curl https://github.com/grafana/k6/releases/download/v0.33.0/k6-v0.33.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1 | |
| - name: Run k6 smoke tests | |
| id: run-k6-tests | |
| env: | |
| URL: ${{ secrets.SMOKE_TEST_PERF_URL }} | |
| HOST: ${{ secrets.SMOKE_TEST_PERF_HOST }} | |
| A_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| A_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| C_USER: ${{ secrets.SMOKE_TEST_PERF_ADMIN_USER }} | |
| C_PW: ${{ secrets.SMOKE_TEST_PERF_ADMIN_PASSWORD }} | |
| P_ID: 22733 | |
| run: | | |
| ./k6 run plugins/woocommerce/tests/performance/tests/gh-action-daily-ext-requests.js | |
| test-plugins: | |
| name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [api-tests] | |
| env: | |
| ALLURE_RESULTS_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-results | |
| ALLURE_REPORT_DIR: ${{ github.workspace }}/plugins/woocommerce/tests/e2e-pw/allure-report | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooPayments' | |
| repo: 'automattic/woocommerce-payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| repo: 'woocommerce/woocommerce-paypal-payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| repo: 'automattic/woocommerce-services' | |
| slug: woocommerce-services | |
| - plugin: 'Woo Subscriptions' | |
| repo: WC_SUBSCRIPTIONS_REPO | |
| private: true | |
| slug: woocommerce-subscriptions | |
| - plugin: 'Gutenberg' | |
| repo: 'WordPress/gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| repo: 'bph/gutenberg' | |
| slug: gutenberg-nightly | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup WooCommerce Monorepo | |
| uses: ./.github/actions/setup-woocommerce-monorepo | |
| with: | |
| install: '@woocommerce/plugin-woocommerce...' | |
| build: '@woocommerce/plugin-woocommerce' | |
| - name: Setup local test environment | |
| uses: ./.github/actions/tests/setup-local-test-environment | |
| with: | |
| test-type: e2e | |
| - name: Run 'Upload plugin' test | |
| id: run-upload-plugin-test | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| tests: upload-plugin.spec.js | |
| env: | |
| PLUGIN_REPOSITORY: ${{ matrix.private && secrets[matrix.repo] || matrix.repo }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| PLUGIN_SLUG: ${{ matrix.slug }} | |
| GITHUB_TOKEN: ${{ secrets.E2E_GH_TOKEN }} | |
| - name: Run the rest of E2E tests | |
| id: run-e2e-composite-action | |
| timeout-minutes: 60 | |
| uses: ./.github/actions/tests/run-e2e-tests | |
| with: | |
| playwright-config: ignore-plugin-tests.playwright.config.js | |
| report-name: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| env: | |
| E2E_MAX_FAILURES: 15 | |
| - name: Create context block and save as JSON file | |
| if: success() || failure() | |
| id: create-block-json | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require( './.github/actions/tests/slack-summary-daily/scripts/create-blocks-plugin-tests.js' ) | |
| script( { core } ); | |
| env: | |
| UPLOAD_RESULT: ${{ steps.run-upload-plugin-test.outputs.result }} | |
| E2E_RESULT: ${{ steps.run-e2e-composite-action.outputs.result }} | |
| PLUGIN_NAME: ${{ matrix.plugin }} | |
| PLUGIN_SLUG: ${{ matrix.slug }} | |
| - name: Upload JSON file as artifact | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: ${{ steps.create-block-json.outputs.path }} | |
| trunk-results: | |
| name: Publish report on smoke tests on nightly build | |
| if: | | |
| ( success() || failure() ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| steps: | |
| - name: Create dirs | |
| run: | | |
| mkdir -p repo | |
| mkdir -p artifacts/api | |
| mkdir -p artifacts/e2e | |
| mkdir -p output | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| path: repo | |
| - name: Download API test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.API_ARTIFACT }} | |
| path: artifacts/api | |
| - name: Download E2E test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.E2E_ARTIFACT }} | |
| path: artifacts/e2e | |
| - name: Post test summary | |
| uses: actions/github-script@v6 | |
| env: | |
| API_SUMMARY_PATH: ${{ github.workspace }}/artifacts/api/allure-report/widgets/summary.json | |
| E2E_PW_SUMMARY_PATH: ${{ github.workspace }}/artifacts/e2e/allure-report/widgets/summary.json | |
| with: | |
| result-encoding: string | |
| script: | | |
| const script = require( './repo/.github/workflows/scripts/prepare-test-summary-daily.js' ) | |
| return await script( { core } ) | |
| - name: Publish report | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| gh workflow run publish-test-reports-daily.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f api_artifact="$API_ARTIFACT" \ | |
| -f e2e_artifact="$E2E_ARTIFACT" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| plugins-results: | |
| name: Publish report on Smoke tests on trunk with plugins | |
| if: | | |
| ( success() || failure() ) && | |
| ( needs.test-plugins.result != 'skipped' ) && | |
| ! github.event.pull_request.head.repo.fork | |
| runs-on: ubuntu-20.04 | |
| needs: [e2e-tests, test-plugins, k6-tests] | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.REPORTS_TOKEN }} | |
| RUN_ID: ${{ github.run_id }} | |
| ARTIFACT: Smoke tests on trunk with ${{ matrix.plugin }} plugin installed (run ${{ github.run_number }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - plugin: 'WooCommerce Payments' | |
| slug: woocommerce-payments | |
| - plugin: 'WooCommerce PayPal Payments' | |
| slug: woocommerce-paypal-payments | |
| - plugin: 'WooCommerce Shipping & Tax' | |
| slug: woocommerce-services | |
| - plugin: 'Woo Subscriptions' | |
| slug: woocommerce-subscriptions | |
| - plugin: 'Gutenberg' | |
| slug: gutenberg | |
| - plugin: 'Gutenberg - Nightly' | |
| slug: gutenberg-nightly | |
| steps: | |
| - name: Download test report artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.ARTIFACT }} | |
| - name: Publish reports | |
| run: | | |
| gh workflow run publish-test-reports-daily-plugins.yml \ | |
| -f run_id=$RUN_ID \ | |
| -f artifact="${{ env.ARTIFACT }}" \ | |
| -f plugin="${{ matrix.plugin }}" \ | |
| -f slug="${{ matrix.slug }}" \ | |
| -f s3_root=public \ | |
| --repo woocommerce/woocommerce-test-reports | |
| post-slack-summary: | |
| name: Post Slack summary | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| if: | | |
| github.ref_name == 'trunk' && | |
| success() || ( | |
| failure() && contains( needs.*.result, 'failure' ) | |
| ) | |
| needs: | |
| - api-tests | |
| - e2e-tests | |
| - k6-tests | |
| - test-plugins | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Download Slack blocks from plugin tests | |
| if: needs.test-plugins.result != 'skipped' | |
| id: download-plugin-blocks | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.PLUGIN_SLACK_BLOCKS_ARTIFACT }} | |
| path: /tmp/plugin-blocks | |
| - name: Construct Slack payload | |
| id: construct-slack-payload | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const script = require('./.github/actions/tests/slack-summary-daily/scripts/construct-slack-payload.js'); | |
| await script( { context, core, github } ); | |
| env: | |
| API_RESULT: ${{ needs.api-tests.outputs.test-result }} | |
| E2E_RESULT: ${{ needs.e2e-tests.outputs.test-result || needs.e2e-tests.result }} | |
| k6_RESULT: ${{ needs.k6-tests.outputs.test-result || needs.k6-tests.result }} | |
| PLUGINS_BLOCKS_PATH: ${{ steps.download-plugin-blocks.outputs.download-path }} | |
| PLUGIN_TESTS_RESULT: ${{ needs.test-plugins.result }} | |
| - name: Send Slack message | |
| id: send-slack-message | |
| uses: slackapi/slack-github-action@v1.23.0 | |
| with: | |
| channel-id: ${{ secrets.DAILY_TEST_SLACK_CHANNEL }} | |
| payload: ${{ steps.construct-slack-payload.outputs.payload }} | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.E2E_SLACK_TOKEN }} |