From c7290f6c782a2e07e954c64fe92036dc8ca62fc7 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 10:30:02 -0500 Subject: [PATCH 1/6] Ignore test results --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 70749a9e4..6f46035ff 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,4 @@ docs/source/reference/ # Developer scratch area _dev/ tests/playwright/deploys/**/requirements.txt +test-results/ From ef6573e6e5e8524ecc0195b0354358a99bed2804 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 10:31:14 -0500 Subject: [PATCH 2/6] Add video and tracing to debug test results --- pytest.ini | 4 +--- tests/playwright/playwright-pytest.ini | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index e6879b461..7efe317fc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,4 @@ [pytest] asyncio_mode=strict testpaths=tests/pytest/ -; ; Debug version of options -; addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vv -addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto +addopts = --strict-markers --durations=6 --durations-min=5.0 --browser webkit --browser firefox --browser chromium --numprocesses auto --tracing=retain-on-failure --video=retain-on-failure diff --git a/tests/playwright/playwright-pytest.ini b/tests/playwright/playwright-pytest.ini index 74013315d..8eb2dbd0d 100644 --- a/tests/playwright/playwright-pytest.ini +++ b/tests/playwright/playwright-pytest.ini @@ -6,8 +6,9 @@ asyncio_mode=strict # --browser : browser type to run on playwright # --numprocesses auto: number of testing workers. auto is number of (virtual) cores # --video=retain-on-failure: playwright saves recording of any failed test +# --tracing=retain-on-failure: playwright saves trace of any failed test # -vv: Extra extra verbose output # # --headed: Headed browser testing # # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings... # --maxfail=1: Stop after 1 failure has occurred -addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto --video=retain-on-failure -vvv --maxfail=1 --headed +addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=retain-on-failure --video=retain-on-failure From 57702af57cf51d6edd0f916ada0960bef6e7168d Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 10:47:23 -0500 Subject: [PATCH 3/6] Upload failed test trace / video for 5 days --- .github/workflows/pytest.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index f7c9085d2..58344b235 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -73,6 +73,12 @@ jobs: timeout-minutes: 20 run: | make playwright-shiny SUB_FILE=". -vv" + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: "playright-shiny-${{ matrix.os }}-${{ matrix.python-version }}-results" + path: test-results/ + retention-days: 5 playwright-examples: runs-on: ${{ matrix.os }} @@ -105,6 +111,12 @@ jobs: timeout-minutes: 20 run: | make playwright-examples SUB_FILE=". -vv" + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: "playright-examples-${{ matrix.os }}-${{ matrix.python-version }}-results" + path: test-results/ + retention-days: 5 playwright-deploys: # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR) @@ -146,6 +158,13 @@ jobs: run: | make playwright-deploys SUB_FILE=". -vv --numprocesses 12" + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: "playright-deploys-${{ matrix.os }}-${{ matrix.python-version }}-results" + path: test-results/ + retention-days: 5 + pypi: name: "Deploy to PyPI" runs-on: ubuntu-latest From cd53f2048cc2b5cbe7c3a8879397a092614b6ea6 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 10:47:41 -0500 Subject: [PATCH 4/6] Update Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 41035d377..600377a22 100644 --- a/Makefile +++ b/Makefile @@ -123,6 +123,8 @@ playwright-examples: install-playwright ## end-to-end tests on examples with pla playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE) +playwright-show-trace: ## Show trace of failed tests + npm playwright show-trace test-results/*/trace.zip testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml From a170e7d051529e9fafa08cb5e41acb4893760d9c Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 10:53:18 -0500 Subject: [PATCH 5/6] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 600377a22..5b0b6796e 100644 --- a/Makefile +++ b/Makefile @@ -124,7 +124,7 @@ playwright-examples: install-playwright ## end-to-end tests on examples with pla playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE) playwright-show-trace: ## Show trace of failed tests - npm playwright show-trace test-results/*/trace.zip + npm playwright show-trace test-results/*/trace.zip testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml From 99ac9c4d238d6d2c666eada5a4832e064a811e7f Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Mon, 4 Mar 2024 11:32:07 -0500 Subject: [PATCH 6/6] Spacing. Using `npx` instead of `npm` --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5b0b6796e..76c205aa3 100644 --- a/Makefile +++ b/Makefile @@ -123,8 +123,9 @@ playwright-examples: install-playwright ## end-to-end tests on examples with pla playwright-debug: install-playwright ## All end-to-end tests, chrome only, headed pytest -c tests/playwright/playwright-pytest.ini tests/playwright/$(SUB_FILE) + playwright-show-trace: ## Show trace of failed tests - npm playwright show-trace test-results/*/trace.zip + npx playwright show-trace test-results/*/trace.zip testrail-junit: install-playwright install-trcli ## end-to-end tests with playwright and generate junit report pytest tests/playwright/shiny/$(SUB_FILE) --junitxml=report.xml