From 983635c2ea6c1334dc58d792b5d574564b1aeeac Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:02:20 +0100 Subject: [PATCH 01/11] Generate PEP 740 attestations for PyPI --- .github/workflows/create-release.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2291cd35862..cbf14ba5eb5 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -29,10 +29,16 @@ jobs: run: | pip install -U pip pip install build twine + # resolution fails without betterproto and protobuf-specs + pip install "pypi-attestations~=0.0.12" "sigstore-protobuf-specs==0.3.2" "betterproto==2.0.0b6" - name: Build distribution run: python -m build + - name: Check distribution + run: | + twine check dist/* + - name: Mint PyPI API token id: mint-token uses: actions/github-script@v7 @@ -59,14 +65,26 @@ jobs: core.setSecret(api_token) core.setOutput('api-token', api_token) + - name: Generate PEP 740 attestations + run: | + python -m pypi_attestations sign dist/* + + - name: Inspect PEP 740 attestations + run: | + python -m pypi_attestations inspect dist/*.publish.attestation + + - name: Verify PEP 740 attestations + run: | + python -m pypi_attestations verify dist/*.whl --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} + python -m pypi_attestations verify dist/*.tar.gz --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} + - name: Upload to PyPI env: TWINE_NON_INTERACTIVE: "true" TWINE_USERNAME: "__token__" TWINE_PASSWORD: "${{ steps.mint-token.outputs.api-token }}" run: | - twine check dist/* - twine upload dist/* + twine upload dist/* --attestations github-release: runs-on: ubuntu-latest From 7c703469c02c5ac6eb16ee57b46af448692a5d0d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:04:55 +0100 Subject: [PATCH 02/11] Remove uploads for testing --- .github/workflows/create-release.yml | 60 +++++++++++++--------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index cbf14ba5eb5..e01a4b35e1d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -2,9 +2,6 @@ name: Create release on: push: - tags: - - "v*.*.*" - workflow_dispatch: permissions: contents: read @@ -13,7 +10,6 @@ jobs: publish-pypi: runs-on: ubuntu-latest name: PyPI Release - environment: release permissions: id-token: write # for PyPI trusted publishing steps: @@ -78,31 +74,31 @@ jobs: python -m pypi_attestations verify dist/*.whl --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} python -m pypi_attestations verify dist/*.tar.gz --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} - - name: Upload to PyPI - env: - TWINE_NON_INTERACTIVE: "true" - TWINE_USERNAME: "__token__" - TWINE_PASSWORD: "${{ steps.mint-token.outputs.api-token }}" - run: | - twine upload dist/* --attestations - - github-release: - runs-on: ubuntu-latest - name: GitHub release - environment: release - permissions: - contents: write # for softprops/action-gh-release to create GitHub release - steps: - - uses: actions/checkout@v4 - - name: Get release version - id: get_version - uses: actions/github-script@v7 - with: - script: core.setOutput('version', context.ref.replace("refs/tags/v", "")) - - - name: Create GitHub release - uses: softprops/action-gh-release@v2 - if: startsWith(github.ref, 'refs/tags/') - with: - name: "Sphinx ${{ steps.get_version.outputs.version }}" - body: "Changelog: https://www.sphinx-doc.org/en/master/changes.html" +# - name: Upload to PyPI +# env: +# TWINE_NON_INTERACTIVE: "true" +# TWINE_USERNAME: "__token__" +# TWINE_PASSWORD: "${{ steps.mint-token.outputs.api-token }}" +# run: | +# twine upload dist/* --attestations +# +# github-release: +# runs-on: ubuntu-latest +# name: GitHub release +# environment: release +# permissions: +# contents: write # for softprops/action-gh-release to create GitHub release +# steps: +# - uses: actions/checkout@v4 +# - name: Get release version +# id: get_version +# uses: actions/github-script@v7 +# with: +# script: core.setOutput('version', context.ref.replace("refs/tags/v", "")) +# +# - name: Create GitHub release +# uses: softprops/action-gh-release@v2 +# if: startsWith(github.ref, 'refs/tags/') +# with: +# name: "Sphinx ${{ steps.get_version.outputs.version }}" +# body: "Changelog: https://www.sphinx-doc.org/en/master/changes.html" From 54be53418376164da0b68a596c3c1ae2c1ade59c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:05:02 +0100 Subject: [PATCH 03/11] Remove other workflows for testing --- .github/workflows/builddoc.yml | 46 ----- .github/workflows/lint.yml | 146 ---------------- .github/workflows/lock.yml | 20 --- .github/workflows/main.yml | 289 -------------------------------- .github/workflows/nodejs.yml | 43 ----- .github/workflows/transifex.yml | 78 --------- 6 files changed, 622 deletions(-) delete mode 100644 .github/workflows/builddoc.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/lock.yml delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/nodejs.yml delete mode 100644 .github/workflows/transifex.yml diff --git a/.github/workflows/builddoc.yml b/.github/workflows/builddoc.yml deleted file mode 100644 index 5c1736d3c5b..00000000000 --- a/.github/workflows/builddoc.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Render documentation - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: "1" - UV_SYSTEM_PYTHON: "1" # make uv do global installs - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install .[docs] - - name: Render the documentation - run: > - sphinx-build - -M html ./doc ./build/sphinx - -vv - --jobs=auto - --show-traceback - --fail-on-warning diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 96a7ef718bd..00000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Lint source code - -on: - push: - pull_request: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: "1" - UV_SYSTEM_PYTHON: "1" # make uv do global installs - -jobs: - # If you update any of these commands, don't forget to update the equivalent - # tox environment - ruff: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Get Ruff version from pyproject.toml - run: | - RUFF_VERSION=$(awk -F'[="]' '/\[project\.optional-dependencies\]/ {p=1} /ruff/ {if (p) print $4}' pyproject.toml) - echo "RUFF_VERSION=$RUFF_VERSION" >> $GITHUB_ENV - - - name: Install Ruff - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - --write-out "%{stderr}Downloaded: %{url}\n" - "https://astral.sh/ruff/$RUFF_VERSION/install.sh" - | sh - - - name: Lint with Ruff - run: ruff check . --output-format github - - - name: Format with Ruff - run: ruff format . --diff - - flake8: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install --upgrade "flake8>=6.0" - - name: Lint with flake8 - run: flake8 . - - mypy: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install ".[lint,test]" - - name: Type check with mypy - run: mypy - - pyright: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install ".[lint,test]" - - name: Type check with pyright - run: pyright - - docs-lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install --upgrade sphinx-lint - - name: Lint documentation with sphinx-lint - run: make doclinter - - twine: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install --upgrade twine build - - name: Lint with twine - run: | - python -m build . - twine check dist/* diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml deleted file mode 100644 index d86f4b36282..00000000000 --- a/.github/workflows/lock.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Lock old threads - -on: - schedule: - - cron: "0 0 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - action: - if: github.repository_owner == 'sphinx-doc' - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@v3 - with: - github-token: ${{ github.token }} - issue-inactive-days: "30" - pr-inactive-days: "30" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index b38a8d70983..00000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,289 +0,0 @@ -name: CI - -on: - push: - paths: - - ".github/workflows/main.yml" - - "sphinx/**" - - "tests/**" - pull_request: - paths: - - ".github/workflows/main.yml" - - "sphinx/**" - - "tests/**" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: "1" - PYTHONDEVMODE: "1" # -X dev - PYTHONWARNDEFAULTENCODING: "1" # -X warn_default_encoding - UV_SYSTEM_PYTHON: "1" # make uv do global installs - -jobs: - ubuntu: - runs-on: ubuntu-latest - name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}) - strategy: - fail-fast: false - matrix: - python: - - "3.10" - - "3.11" - - "3.12" - docutils: - - "0.20" - - "0.21" -# include: -# # test every supported Docutils version for the latest supported Python -# - python: "3.12" -# docutils: "0.20" - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install .[test] - - name: Install Docutils ${{ matrix.docutils }} - run: uv pip install --upgrade "docutils~=${{ matrix.docutils }}.0" - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - - deadsnakes: - runs-on: ubuntu-latest - name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}) - strategy: - fail-fast: false - matrix: - python: - - "3.13-dev" - docutils: - - "0.20" - - "0.21" - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python }} (deadsnakes) - uses: deadsnakes/action@v3.2.0 - with: - python-version: ${{ matrix.python }} - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install .[test] - - name: Install Docutils ${{ matrix.docutils }} - run: python -m pip install --upgrade "docutils~=${{ matrix.docutils }}.0" - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - - deadsnakes-free-threraded: - runs-on: ubuntu-latest - name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}; free-threaded) - strategy: - fail-fast: false - matrix: - python: - - "3.13-dev" - docutils: - - "0.20" - - "0.21" - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python }} (deadsnakes) - uses: deadsnakes/action@v3.2.0 - with: - python-version: ${{ matrix.python }} - nogil: true - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install .[test] - - name: Install Docutils ${{ matrix.docutils }} - run: python -m pip install --upgrade "docutils~=${{ matrix.docutils }}.0" - # markupsafe._speedups has not declared that it can run safely without the GIL - - name: Remove markupsafe._speedups - run: rm -rf "$(python -c 'from markupsafe._speedups import __file__ as f; print(f)')" - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - - windows: - runs-on: windows-2019 - name: Windows - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: choco install --no-progress graphviz - - name: Install uv - run: > - Invoke-WebRequest -Uri "https://astral.sh/uv/install.ps1" - | Invoke-Expression - - name: Install dependencies - run: uv pip install .[test] - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - - docutils-latest: - runs-on: ubuntu-latest - name: Docutils HEAD - - steps: - - name: Install epubcheck - run: | - EPUBCHECK_VERSION="5.1.0" - mkdir /tmp/epubcheck && cd /tmp/epubcheck - wget --no-verbose https://github.com/w3c/epubcheck/releases/download/v${EPUBCHECK_VERSION}/epubcheck-${EPUBCHECK_VERSION}.zip - unzip epubcheck-${EPUBCHECK_VERSION}.zip - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install .[test] - - name: Install Docutils' HEAD - run: uv pip install "docutils @ git+https://repo.or.cz/docutils.git#subdirectory=docutils" - - name: Test with pytest - run: python -m pytest -vv - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - DO_EPUBCHECK: "1" - EPUBCHECK_PATH: "/tmp/epubcheck/epubcheck-5.1.0/epubcheck.jar" - - oldest-supported: - runs-on: ubuntu-latest - name: Oldest supported - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: | - uv pip install .[test] --resolution lowest-direct - uv pip install alabaster==1.0.0 - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - - latex: - runs-on: ubuntu-latest - name: LaTeX - container: - image: ghcr.io/sphinx-doc/sphinx-ci - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Check Python version - run: python --version --version - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install .[test] - - name: Test with pytest - run: python -m pytest -vv --durations 25 - env: - PYTHONWARNINGS: "error" # treat all warnings as errors - DO_EPUBCHECK: "1" - - coverage: - if: github.event_name == 'push' && github.repository_owner == 'sphinx-doc' - runs-on: ubuntu-latest - name: Coverage - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Check Python version - run: python --version --version - - name: Install graphviz - run: sudo apt-get install graphviz - - name: Install uv - run: > - curl --no-progress-meter --location --fail - --proto '=https' --tlsv1.2 - "https://astral.sh/uv/install.sh" - | sh - - name: Install dependencies - run: uv pip install .[test] pytest-cov - - name: Test with pytest - run: python -m pytest -vv --cov . --cov-append --cov-config pyproject.toml - env: - VIRTUALENV_SYSTEM_SITE_PACKAGES: "1" - - name: codecov - uses: codecov/codecov-action@v4 diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml deleted file mode 100644 index 29359f90958..00000000000 --- a/.github/workflows/nodejs.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: CI (node.js) - -on: - push: - paths: - - ".github/workflows/nodejs.yml" - - "sphinx/themes/**.js" - - "tests/js/**" - - "package.json" - - "package-lock.json" - pull_request: - paths: - - ".github/workflows/nodejs.yml" - - "sphinx/themes/**.js" - - "tests/js/**" - - "package.json" - - "package-lock.json" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - FORCE_COLOR: "1" - -jobs: - build: - runs-on: ubuntu-latest - env: - node-version: "20" - - steps: - - uses: actions/checkout@v4 - - name: Use Node.js ${{ env.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ env.node-version }} - cache: "npm" - - run: npm install - - run: npm test diff --git a/.github/workflows/transifex.yml b/.github/workflows/transifex.yml deleted file mode 100644 index 78e74efcf5a..00000000000 --- a/.github/workflows/transifex.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Synchronise translations - -on: - schedule: - # 22:38 GMT, every Sunday. Chosen to be a random time. - - cron: "38 22 * * SUN" - workflow_dispatch: - -permissions: - contents: read - -jobs: - push: - if: github.repository_owner == 'sphinx-doc' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install transifex client - run: | - mkdir -p /tmp/tx_cli && cd $_ - curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash - shell: bash - - name: Install dependencies - run: pip install --upgrade babel jinja2 - - name: Extract translations from source code - run: python utils/babel_runner.py extract - - name: Push translations to transifex.com - run: | - cd sphinx/locale - /tmp/tx_cli/tx push --source --use-git-timestamps --workers 10 - env: - TX_TOKEN: ${{ secrets.TX_TOKEN }} - - pull: - permissions: - contents: write # for peter-evans/create-pull-request to create branch - pull-requests: write # for peter-evans/create-pull-request to create a PR - if: github.repository_owner == 'sphinx-doc' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3" - - name: Install transifex client - run: | - mkdir -p /tmp/tx_cli && cd $_ - curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash - shell: bash - - name: Install dependencies - run: pip install --upgrade babel jinja2 - - name: Extract translations from source code - run: python utils/babel_runner.py extract - - name: Pull translations from transifex.com - run: | - cd sphinx/locale - /tmp/tx_cli/tx pull --translations --all --force --use-git-timestamps --workers 10 - env: - TX_TOKEN: ${{ secrets.TX_TOKEN }} - - name: Compile message catalogs - run: python utils/babel_runner.py compile - - name: Create Pull Request - uses: peter-evans/create-pull-request@v7 - with: - add-paths: | - sphinx/locale - commit-message: "[internationalisation] Update translations" - branch: bot/pull-translations - title: "[bot]: Update message catalogues" - labels: "internals:internationalisation" - body-path: babel_compile.txt From d0f708a7c3e823b376660b6ef6c20390caefcfdc Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:06:19 +0100 Subject: [PATCH 04/11] Add concurrency and env --- .github/workflows/create-release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index e01a4b35e1d..f36d6028a59 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -6,6 +6,13 @@ on: permissions: contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + jobs: publish-pypi: runs-on: ubuntu-latest From 6471a81d4e851a8ca9e17c4c56e5fd03bc2bbd43 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:09:06 +0100 Subject: [PATCH 05/11] Run on PR --- .github/workflows/create-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f36d6028a59..d0b1b03e549 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -2,6 +2,7 @@ name: Create release on: push: + pull_request: permissions: contents: read From e4f02a672b357d4c48b33bfc816f1aed798241e8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:10:49 +0100 Subject: [PATCH 06/11] Revert "Run on PR" This reverts commit 6471a81d4e851a8ca9e17c4c56e5fd03bc2bbd43. --- .github/workflows/create-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index d0b1b03e549..f36d6028a59 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -2,7 +2,6 @@ name: Create release on: push: - pull_request: permissions: contents: read From 62d627086414280c2ebb18944a896a3d7b58c58f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:10:49 +0100 Subject: [PATCH 07/11] Revert "Remove other workflows for testing" This reverts commit 54be53418376164da0b68a596c3c1ae2c1ade59c. --- .github/workflows/builddoc.yml | 46 +++++ .github/workflows/lint.yml | 146 ++++++++++++++++ .github/workflows/lock.yml | 20 +++ .github/workflows/main.yml | 289 ++++++++++++++++++++++++++++++++ .github/workflows/nodejs.yml | 43 +++++ .github/workflows/transifex.yml | 78 +++++++++ 6 files changed, 622 insertions(+) create mode 100644 .github/workflows/builddoc.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/lock.yml create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/nodejs.yml create mode 100644 .github/workflows/transifex.yml diff --git a/.github/workflows/builddoc.yml b/.github/workflows/builddoc.yml new file mode 100644 index 00000000000..5c1736d3c5b --- /dev/null +++ b/.github/workflows/builddoc.yml @@ -0,0 +1,46 @@ +name: Render documentation + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + UV_SYSTEM_PYTHON: "1" # make uv do global installs + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install .[docs] + - name: Render the documentation + run: > + sphinx-build + -M html ./doc ./build/sphinx + -vv + --jobs=auto + --show-traceback + --fail-on-warning diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..96a7ef718bd --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,146 @@ +name: Lint source code + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + UV_SYSTEM_PYTHON: "1" # make uv do global installs + +jobs: + # If you update any of these commands, don't forget to update the equivalent + # tox environment + ruff: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Get Ruff version from pyproject.toml + run: | + RUFF_VERSION=$(awk -F'[="]' '/\[project\.optional-dependencies\]/ {p=1} /ruff/ {if (p) print $4}' pyproject.toml) + echo "RUFF_VERSION=$RUFF_VERSION" >> $GITHUB_ENV + + - name: Install Ruff + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + --write-out "%{stderr}Downloaded: %{url}\n" + "https://astral.sh/ruff/$RUFF_VERSION/install.sh" + | sh + + - name: Lint with Ruff + run: ruff check . --output-format github + + - name: Format with Ruff + run: ruff format . --diff + + flake8: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install --upgrade "flake8>=6.0" + - name: Lint with flake8 + run: flake8 . + + mypy: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install ".[lint,test]" + - name: Type check with mypy + run: mypy + + pyright: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install ".[lint,test]" + - name: Type check with pyright + run: pyright + + docs-lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install --upgrade sphinx-lint + - name: Lint documentation with sphinx-lint + run: make doclinter + + twine: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install --upgrade twine build + - name: Lint with twine + run: | + python -m build . + twine check dist/* diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 00000000000..d86f4b36282 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,20 @@ +name: Lock old threads + +on: + schedule: + - cron: "0 0 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + action: + if: github.repository_owner == 'sphinx-doc' + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v3 + with: + github-token: ${{ github.token }} + issue-inactive-days: "30" + pr-inactive-days: "30" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000..b38a8d70983 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,289 @@ +name: CI + +on: + push: + paths: + - ".github/workflows/main.yml" + - "sphinx/**" + - "tests/**" + pull_request: + paths: + - ".github/workflows/main.yml" + - "sphinx/**" + - "tests/**" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + PYTHONDEVMODE: "1" # -X dev + PYTHONWARNDEFAULTENCODING: "1" # -X warn_default_encoding + UV_SYSTEM_PYTHON: "1" # make uv do global installs + +jobs: + ubuntu: + runs-on: ubuntu-latest + name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}) + strategy: + fail-fast: false + matrix: + python: + - "3.10" + - "3.11" + - "3.12" + docutils: + - "0.20" + - "0.21" +# include: +# # test every supported Docutils version for the latest supported Python +# - python: "3.12" +# docutils: "0.20" + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install .[test] + - name: Install Docutils ${{ matrix.docutils }} + run: uv pip install --upgrade "docutils~=${{ matrix.docutils }}.0" + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + + deadsnakes: + runs-on: ubuntu-latest + name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}) + strategy: + fail-fast: false + matrix: + python: + - "3.13-dev" + docutils: + - "0.20" + - "0.21" + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} (deadsnakes) + uses: deadsnakes/action@v3.2.0 + with: + python-version: ${{ matrix.python }} + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install .[test] + - name: Install Docutils ${{ matrix.docutils }} + run: python -m pip install --upgrade "docutils~=${{ matrix.docutils }}.0" + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + + deadsnakes-free-threraded: + runs-on: ubuntu-latest + name: Python ${{ matrix.python }} (Docutils ${{ matrix.docutils }}; free-threaded) + strategy: + fail-fast: false + matrix: + python: + - "3.13-dev" + docutils: + - "0.20" + - "0.21" + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} (deadsnakes) + uses: deadsnakes/action@v3.2.0 + with: + python-version: ${{ matrix.python }} + nogil: true + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install .[test] + - name: Install Docutils ${{ matrix.docutils }} + run: python -m pip install --upgrade "docutils~=${{ matrix.docutils }}.0" + # markupsafe._speedups has not declared that it can run safely without the GIL + - name: Remove markupsafe._speedups + run: rm -rf "$(python -c 'from markupsafe._speedups import __file__ as f; print(f)')" + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + + windows: + runs-on: windows-2019 + name: Windows + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: choco install --no-progress graphviz + - name: Install uv + run: > + Invoke-WebRequest -Uri "https://astral.sh/uv/install.ps1" + | Invoke-Expression + - name: Install dependencies + run: uv pip install .[test] + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + + docutils-latest: + runs-on: ubuntu-latest + name: Docutils HEAD + + steps: + - name: Install epubcheck + run: | + EPUBCHECK_VERSION="5.1.0" + mkdir /tmp/epubcheck && cd /tmp/epubcheck + wget --no-verbose https://github.com/w3c/epubcheck/releases/download/v${EPUBCHECK_VERSION}/epubcheck-${EPUBCHECK_VERSION}.zip + unzip epubcheck-${EPUBCHECK_VERSION}.zip + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install .[test] + - name: Install Docutils' HEAD + run: uv pip install "docutils @ git+https://repo.or.cz/docutils.git#subdirectory=docutils" + - name: Test with pytest + run: python -m pytest -vv + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + DO_EPUBCHECK: "1" + EPUBCHECK_PATH: "/tmp/epubcheck/epubcheck-5.1.0/epubcheck.jar" + + oldest-supported: + runs-on: ubuntu-latest + name: Oldest supported + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: | + uv pip install .[test] --resolution lowest-direct + uv pip install alabaster==1.0.0 + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + + latex: + runs-on: ubuntu-latest + name: LaTeX + container: + image: ghcr.io/sphinx-doc/sphinx-ci + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Check Python version + run: python --version --version + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install .[test] + - name: Test with pytest + run: python -m pytest -vv --durations 25 + env: + PYTHONWARNINGS: "error" # treat all warnings as errors + DO_EPUBCHECK: "1" + + coverage: + if: github.event_name == 'push' && github.repository_owner == 'sphinx-doc' + runs-on: ubuntu-latest + name: Coverage + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Check Python version + run: python --version --version + - name: Install graphviz + run: sudo apt-get install graphviz + - name: Install uv + run: > + curl --no-progress-meter --location --fail + --proto '=https' --tlsv1.2 + "https://astral.sh/uv/install.sh" + | sh + - name: Install dependencies + run: uv pip install .[test] pytest-cov + - name: Test with pytest + run: python -m pytest -vv --cov . --cov-append --cov-config pyproject.toml + env: + VIRTUALENV_SYSTEM_SITE_PACKAGES: "1" + - name: codecov + uses: codecov/codecov-action@v4 diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml new file mode 100644 index 00000000000..29359f90958 --- /dev/null +++ b/.github/workflows/nodejs.yml @@ -0,0 +1,43 @@ +name: CI (node.js) + +on: + push: + paths: + - ".github/workflows/nodejs.yml" + - "sphinx/themes/**.js" + - "tests/js/**" + - "package.json" + - "package-lock.json" + pull_request: + paths: + - ".github/workflows/nodejs.yml" + - "sphinx/themes/**.js" + - "tests/js/**" + - "package.json" + - "package-lock.json" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + +jobs: + build: + runs-on: ubuntu-latest + env: + node-version: "20" + + steps: + - uses: actions/checkout@v4 + - name: Use Node.js ${{ env.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ env.node-version }} + cache: "npm" + - run: npm install + - run: npm test diff --git a/.github/workflows/transifex.yml b/.github/workflows/transifex.yml new file mode 100644 index 00000000000..78e74efcf5a --- /dev/null +++ b/.github/workflows/transifex.yml @@ -0,0 +1,78 @@ +name: Synchronise translations + +on: + schedule: + # 22:38 GMT, every Sunday. Chosen to be a random time. + - cron: "38 22 * * SUN" + workflow_dispatch: + +permissions: + contents: read + +jobs: + push: + if: github.repository_owner == 'sphinx-doc' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install transifex client + run: | + mkdir -p /tmp/tx_cli && cd $_ + curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash + shell: bash + - name: Install dependencies + run: pip install --upgrade babel jinja2 + - name: Extract translations from source code + run: python utils/babel_runner.py extract + - name: Push translations to transifex.com + run: | + cd sphinx/locale + /tmp/tx_cli/tx push --source --use-git-timestamps --workers 10 + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + + pull: + permissions: + contents: write # for peter-evans/create-pull-request to create branch + pull-requests: write # for peter-evans/create-pull-request to create a PR + if: github.repository_owner == 'sphinx-doc' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + - name: Install transifex client + run: | + mkdir -p /tmp/tx_cli && cd $_ + curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash + shell: bash + - name: Install dependencies + run: pip install --upgrade babel jinja2 + - name: Extract translations from source code + run: python utils/babel_runner.py extract + - name: Pull translations from transifex.com + run: | + cd sphinx/locale + /tmp/tx_cli/tx pull --translations --all --force --use-git-timestamps --workers 10 + env: + TX_TOKEN: ${{ secrets.TX_TOKEN }} + - name: Compile message catalogs + run: python utils/babel_runner.py compile + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + add-paths: | + sphinx/locale + commit-message: "[internationalisation] Update translations" + branch: bot/pull-translations + title: "[bot]: Update message catalogues" + labels: "internals:internationalisation" + body-path: babel_compile.txt From 126cfaefd7fc61b6730d69939b5157a1c15c35ce Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 6 Oct 2024 18:10:50 +0100 Subject: [PATCH 08/11] Revert "Remove uploads for testing" This reverts commit 7c703469c02c5ac6eb16ee57b46af448692a5d0d. --- .github/workflows/create-release.yml | 60 +++++++++++++++------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f36d6028a59..31d528ce002 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -2,6 +2,9 @@ name: Create release on: push: + tags: + - "v*.*.*" + workflow_dispatch: permissions: contents: read @@ -17,6 +20,7 @@ jobs: publish-pypi: runs-on: ubuntu-latest name: PyPI Release + environment: release permissions: id-token: write # for PyPI trusted publishing steps: @@ -81,31 +85,31 @@ jobs: python -m pypi_attestations verify dist/*.whl --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} python -m pypi_attestations verify dist/*.tar.gz --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} -# - name: Upload to PyPI -# env: -# TWINE_NON_INTERACTIVE: "true" -# TWINE_USERNAME: "__token__" -# TWINE_PASSWORD: "${{ steps.mint-token.outputs.api-token }}" -# run: | -# twine upload dist/* --attestations -# -# github-release: -# runs-on: ubuntu-latest -# name: GitHub release -# environment: release -# permissions: -# contents: write # for softprops/action-gh-release to create GitHub release -# steps: -# - uses: actions/checkout@v4 -# - name: Get release version -# id: get_version -# uses: actions/github-script@v7 -# with: -# script: core.setOutput('version', context.ref.replace("refs/tags/v", "")) -# -# - name: Create GitHub release -# uses: softprops/action-gh-release@v2 -# if: startsWith(github.ref, 'refs/tags/') -# with: -# name: "Sphinx ${{ steps.get_version.outputs.version }}" -# body: "Changelog: https://www.sphinx-doc.org/en/master/changes.html" + - name: Upload to PyPI + env: + TWINE_NON_INTERACTIVE: "true" + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: "${{ steps.mint-token.outputs.api-token }}" + run: | + twine upload dist/* --attestations + + github-release: + runs-on: ubuntu-latest + name: GitHub release + environment: release + permissions: + contents: write # for softprops/action-gh-release to create GitHub release + steps: + - uses: actions/checkout@v4 + - name: Get release version + id: get_version + uses: actions/github-script@v7 + with: + script: core.setOutput('version', context.ref.replace("refs/tags/v", "")) + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + if: startsWith(github.ref, 'refs/tags/') + with: + name: "Sphinx ${{ steps.get_version.outputs.version }}" + body: "Changelog: https://www.sphinx-doc.org/en/master/changes.html" From 54a203fdbfa58c76dccc098980eabd55c1901961 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:48:45 +0100 Subject: [PATCH 09/11] Use actions/attest --- .github/workflows/create-release.yml | 45 ++++++++++++++++++++-------- utils/convert_attestations.py | 42 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 utils/convert_attestations.py diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 31d528ce002..21edc163dde 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -22,6 +22,7 @@ jobs: name: PyPI Release environment: release permissions: + attestations: write # for actions/attest id-token: write # for PyPI trusted publishing steps: - uses: actions/checkout@v4 @@ -46,6 +47,37 @@ jobs: run: | twine check dist/* + - uses: actions/attest@v1 + id: attest + with: + subject-path: "dist/*" + predicate-type: "https://docs.pypi.org/attestations/publish/v1" + predicate: "null" + show-summary: "true" + + - name: Convert attestations to PEP 740 + # workflow_ref example: sphinx-doc/sphinx/.github/workflows/create-release.yml@refs/heads/master + run: > + python utils/convert_attestations.py + "${{ steps.attest.outputs.bundle-path }}" + "https://github.com/${{ github.workflow_ref }}" + + - name: Inspect PEP 740 attestations + run: | + python -m pypi_attestations inspect dist/*.publish.attestation + + - name: Prepare attestation bundles for uploading + run: | + mkdir -p /tmp/attestation-bundles + cp "${{ steps.attest.outputs.bundle-path }}" /tmp/attestation-bundles/ + cp dist/*.publish.attestation /tmp/attestation-bundles/ + + - name: Upload attestation bundles + uses: actions/upload-artifact@v4 + with: + name: attestation-bundles + path: /tmp/attestation-bundles/ + - name: Mint PyPI API token id: mint-token uses: actions/github-script@v7 @@ -72,19 +104,6 @@ jobs: core.setSecret(api_token) core.setOutput('api-token', api_token) - - name: Generate PEP 740 attestations - run: | - python -m pypi_attestations sign dist/* - - - name: Inspect PEP 740 attestations - run: | - python -m pypi_attestations inspect dist/*.publish.attestation - - - name: Verify PEP 740 attestations - run: | - python -m pypi_attestations verify dist/*.whl --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} - python -m pypi_attestations verify dist/*.tar.gz --identity https://github.com/${{ github.repository }}/.github/workflows/create-release.yml@${{ github.ref }} - - name: Upload to PyPI env: TWINE_NON_INTERACTIVE: "true" diff --git a/utils/convert_attestations.py b/utils/convert_attestations.py new file mode 100644 index 00000000000..d359c1593b3 --- /dev/null +++ b/utils/convert_attestations.py @@ -0,0 +1,42 @@ +"""Convert Sigstore attestations to PEP 740. + +See https://github.com/trailofbits/pypi-attestations. +""" + +import json +import sys +from base64 import b64decode +from pathlib import Path + +from pypi_attestations import Attestation, Distribution +from sigstore.models import Bundle +from sigstore.verify import Verifier +from sigstore.verify.policy import Identity + +DIST = Path('dist') +bundle_path = Path(sys.argv[1]) +signer_identity = sys.argv[2] + +for line in bundle_path.read_bytes().splitlines(): + dsse_envelope_payload = json.loads(line)['dsseEnvelope']['payload'] + subjects = json.loads(b64decode(dsse_envelope_payload))['subject'] + for subject in subjects: + filename = subject['name'] + assert (DIST / filename).is_file() + + # Convert attestation from Sigstore to PEP 740 + print(f'Converting attestation for {filename}') + sigstore_bundle = Bundle.from_json(line) + attestation = Attestation.from_bundle(sigstore_bundle) + attestation_path = DIST / f'{filename}.publish.attestation' + attestation_path.write_text(attestation.model_dump_json()) + print(f'Attestation for {filename} written to {attestation_path}') + print() + + # Validate attestation + dist = Distribution.from_file(DIST / filename) + attestation = Attestation.model_validate_json(attestation_path.read_bytes()) + verifier = Verifier.production() + policy = Identity(identity=signer_identity) + attestation.verify(verifier, policy, dist) + print(f'Verified {attestation_path}') From dcf9ceb2546aa33fd52c210f29d7984f9bcef3f8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:54:14 +0100 Subject: [PATCH 10/11] Require Twine 5.1 or later --- .github/workflows/create-release.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 21edc163dde..7eb61211756 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -36,7 +36,7 @@ jobs: - name: Install build dependencies (pypa/build, twine) run: | pip install -U pip - pip install build twine + pip install build "twine>=5.1" # resolution fails without betterproto and protobuf-specs pip install "pypi-attestations~=0.0.12" "sigstore-protobuf-specs==0.3.2" "betterproto==2.0.0b6" @@ -47,7 +47,8 @@ jobs: run: | twine check dist/* - - uses: actions/attest@v1 + - name: Create Sigstore attestations for built distributions + uses: actions/attest@v1 id: attest with: subject-path: "dist/*" From 7c52db7895b9d5912501a52f11c66f0a01d4408a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 8 Oct 2024 04:58:15 +0100 Subject: [PATCH 11/11] Exclude convert_attestations --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 49f2531a0e0..2342e593395 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -199,6 +199,7 @@ exclude = [ "^tests/test_util/test_util_typing\\.py$", "^tests/test_util/typing_test_data\\.py$", # tests/test_writers + "^utils/convert_attestations\\.py$", ] python_version = "3.10" strict = true