From 2f7c6d5c11975423b4f3220d193af389a8d953c1 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Nov 2025 21:10:18 +0200 Subject: [PATCH 1/6] ci: generate github release notes separately from creating github release Since creating the github release is security sensitive, better to isolate the part of generating the markdown release notes in its own job, such that if e.g. pip/tox/pandoc is compromised it could not in turn compromise the release files. --- .github/workflows/deploy.yml | 76 ++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index eb3940b6fef..5e2ebad315b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -35,9 +35,47 @@ jobs: with: attest-build-provenance-github: 'true' + generate-gh-release-notes: + needs: [package] + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + + - name: Generate release notes + env: + VERSION: ${{ github.event.inputs.version }} + run: | + sudo apt-get install pandoc + tox -e generate-gh-release-notes -- "$VERSION" gh-release-notes.md + + - name: Upload release notes + uses: actions/upload-artifact@v4 + with: + name: release-notes + path: gh-release-notes.md + retention-days: 1 + deploy: if: github.repository == 'pytest-dev/pytest' - needs: [package] + # Need generate-gh-release-notes only for ordering. + # Don't want to release to PyPI if generating GitHub release notes fails. + needs: [package, generate-gh-release-notes] runs-on: ubuntu-latest environment: deploy timeout-minutes: 30 @@ -69,48 +107,28 @@ jobs: git tag --annotate --message=v"$VERSION" "$VERSION" ${{ github.sha }} git push origin "$VERSION" - release-notes: - - # todo: generate the content in the build job - # the goal being of using a github action script to push the release data - # after success instead of creating a complete python/tox env - needs: [deploy] + create-github-release: + needs: [generate-gh-release-notes, deploy] runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 10 permissions: contents: write steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - persist-credentials: false - - name: Download Package uses: actions/download-artifact@v6 with: name: Packages path: dist - - name: Set up Python - uses: actions/setup-python@v6 + - name: Download release notes + uses: actions/download-artifact@v6 with: - python-version: "3.11" - - - name: Install tox - run: | - python -m pip install --upgrade pip - pip install --upgrade tox - - - name: Generate release notes - env: - VERSION: ${{ github.event.inputs.version }} - run: | - sudo apt-get install pandoc - tox -e generate-gh-release-notes -- "$VERSION" scripts/latest-release-notes.md + name: release-notes + path: . - name: Publish GitHub Release env: VERSION: ${{ github.event.inputs.version }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create --notes-file scripts/latest-release-notes.md --verify-tag "$VERSION" dist/* + gh release create --notes-file gh-release-notes.md --verify-tag "$VERSION" dist/* From 1abff440a8209e1fdbfbfbb842fec5047e3d165a Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Nov 2025 23:02:19 +0200 Subject: [PATCH 2/6] tox: use `pypandoc_binary` instead of `pypandoc` This installs pandoc from a binary wheel instead of requiring it in the system (apt install in CI). While system pandoc may be more trustworthy, we already trust pypandoc so might as well trust the binary wheels (which from the same project) to make it easier to run. --- .github/workflows/deploy.yml | 1 - tox.ini | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5e2ebad315b..d3eb137f105 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -61,7 +61,6 @@ jobs: env: VERSION: ${{ github.event.inputs.version }} run: | - sudo apt-get install pandoc tox -e generate-gh-release-notes -- "$VERSION" gh-release-notes.md - name: Upload release notes diff --git a/tox.ini b/tox.ini index 1203bfe2352..0e6edf6df97 100644 --- a/tox.ini +++ b/tox.ini @@ -212,5 +212,5 @@ commands = python scripts/prepare-release-pr.py {posargs} description = generate release notes that can be published as GitHub Release usedevelop = True deps = - pypandoc + pypandoc_binary commands = python scripts/generate-gh-release-notes.py {posargs} From 74e22f5bbeef160ea1342e4babaeed2996a7bfae Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Nov 2025 23:04:14 +0200 Subject: [PATCH 3/6] scripts/generate-gh-release-notes: use github flavor markdown This fixes the `{.interpreted-text role="func"}` stuff that shows up with the `md` format which github doesn't support. See comparison gist: https://gist.github.com/bluetech/365f5b20f6ed1258439d65768714ddc0 --- scripts/generate-gh-release-notes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate-gh-release-notes.py b/scripts/generate-gh-release-notes.py index b6d92d085e1..d293a3bb695 100644 --- a/scripts/generate-gh-release-notes.py +++ b/scripts/generate-gh-release-notes.py @@ -43,7 +43,7 @@ def extract_changelog_entries_for(version: str) -> str: def convert_rst_to_md(text: str) -> str: result = pypandoc.convert_text( - text, "md", format="rst", extra_args=["--wrap=preserve"] + text, "gfm", format="rst", extra_args=["--wrap=preserve"] ) assert isinstance(result, str), repr(result) return result From dfd796fb2ff6356af116f76d307f853dc11a10b2 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Nov 2025 22:43:53 +0200 Subject: [PATCH 4/6] ci: move running update-plugin-list script to tox We run all scripts through tox, that's debatable but consistency is more important. --- .github/workflows/update-plugin-list.yml | 7 +++---- tox.ini | 12 ++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml index aca22a6dd95..788dd470509 100644 --- a/.github/workflows/update-plugin-list.yml +++ b/.github/workflows/update-plugin-list.yml @@ -29,7 +29,6 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.11" - cache: pip - name: requests-cache uses: actions/cache@v4 @@ -38,13 +37,13 @@ jobs: key: plugins-http-cache-${{ github.run_id }} # Can use time based key as well restore-keys: plugins-http-cache- - - name: Install dependencies + - name: Install tox run: | python -m pip install --upgrade pip - pip install packaging requests tabulate[widechars] tqdm requests-cache platformdirs + pip install --upgrade tox - name: Update Plugin List - run: python scripts/update-plugin-list.py + run: tox -e update-plugin-list - name: Create Pull Request id: pr diff --git a/tox.ini b/tox.ini index 0e6edf6df97..b6fcecc886a 100644 --- a/tox.ini +++ b/tox.ini @@ -214,3 +214,15 @@ usedevelop = True deps = pypandoc_binary commands = python scripts/generate-gh-release-notes.py {posargs} + +[testenv:update-plugin-list] +description = update the plugin list +skip_install = True +deps = + packaging + requests + tabulate[widechars] + tqdm + requests-cache + platformdirs +commands = python scripts/update-plugin-list.py {posargs} From ebc152f84e40796ae88fadb71e4fd95c2946bfc3 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Mon, 3 Nov 2025 23:14:08 +0200 Subject: [PATCH 5/6] ci: update setup python's from 3.11 or 3.* to 3.13 Better to use a recent version, but didn't go for 3.14 maybe it's still too new for some dependencies. --- .github/workflows/deploy.yml | 2 +- .github/workflows/doc-check-links.yml | 2 +- .github/workflows/prepare-release-pr.yml | 2 +- .github/workflows/update-plugin-list.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d3eb137f105..fce5de3fc4b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.13" - name: Install tox run: | diff --git a/.github/workflows/doc-check-links.yml b/.github/workflows/doc-check-links.yml index bf88c2b3c6c..497ec73500a 100644 --- a/.github/workflows/doc-check-links.yml +++ b/.github/workflows/doc-check-links.yml @@ -25,7 +25,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.13" cache: pip - name: Install dependencies diff --git a/.github/workflows/prepare-release-pr.yml b/.github/workflows/prepare-release-pr.yml index e46a307d0ef..9dcfea7bae5 100644 --- a/.github/workflows/prepare-release-pr.yml +++ b/.github/workflows/prepare-release-pr.yml @@ -36,7 +36,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.x" + python-version: "3.13" - name: Install dependencies run: | diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml index 788dd470509..b396d6e19d4 100644 --- a/.github/workflows/update-plugin-list.yml +++ b/.github/workflows/update-plugin-list.yml @@ -28,7 +28,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.13" - name: requests-cache uses: actions/cache@v4 From a250954723eda5ae2cb60396a516762b08fa0644 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Tue, 4 Nov 2025 09:10:22 +0200 Subject: [PATCH 6/6] ci: split publish-to-pypi and push-tag jobs This way each job only gets the permissions it needs. --- .github/workflows/deploy.yml | 21 ++++++++++++++------- RELEASING.rst | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fce5de3fc4b..176daa1b4d7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,7 +70,7 @@ jobs: path: gh-release-notes.md retention-days: 1 - deploy: + publish-to-pypi: if: github.repository == 'pytest-dev/pytest' # Need generate-gh-release-notes only for ordering. # Don't want to release to PyPI if generating GitHub release notes fails. @@ -80,12 +80,7 @@ jobs: timeout-minutes: 30 permissions: id-token: write - contents: write steps: - - uses: actions/checkout@v5 - with: - persist-credentials: true - - name: Download Package uses: actions/download-artifact@v6 with: @@ -97,6 +92,18 @@ jobs: with: attestations: true + push-tag: + needs: [publish-to-pypi] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: true + - name: Push tag env: VERSION: ${{ github.event.inputs.version }} @@ -107,7 +114,7 @@ jobs: git push origin "$VERSION" create-github-release: - needs: [generate-gh-release-notes, deploy] + needs: [push-tag, generate-gh-release-notes] runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/RELEASING.rst b/RELEASING.rst index 57e1e176a80..79b4e2f764d 100644 --- a/RELEASING.rst +++ b/RELEASING.rst @@ -133,7 +133,7 @@ Releasing Both automatic and manual processes described above follow the same steps from this point onward. -#. After all tests pass and the PR has been approved, trigger the ``deploy`` job +#. After all tests pass and the PR has been approved, trigger the ``deploy`` workflow in https://github.com/pytest-dev/pytest/actions/workflows/deploy.yml, using the ``release-MAJOR.MINOR.PATCH`` branch as source.