Provide historical summary of tests and PR comment on failed tests

.github/workflows/_test.yml

@@ -49,8 +49,9 @@ jobs:
         with:
           python-version: ${{ inputs.python-version }}
           pip-install: ".[dev]"
+
       - name: Run tests
-        run: tox -e tests
+        run: tox -e tests -- --ctrf report.json
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -59,3 +60,19 @@ jobs:
           files: cov.xml
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Comment on failing PR tests
+        run: npx github-actions-ctrf summary report.json --pr-comment --on-fail-only
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish CTRF Historical results table
+        run: npx github-actions-ctrf historical report.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.python-version }}-${{ inputs.runs-on }}-ctrf-report
+          path: report.json

pyproject.toml

@@ -64,6 +64,7 @@ dev = [
     "pytest-forked",
     "pytest-rerunfailures",
     "pytest-timeout",
+    "pytest-json-ctrf",
     "ruff",
     "sphinx<7.4.0",              # https://github.com/bluesky/ophyd-async/issues/459
     "sphinx-autobuild",