Skip to content
CI

Small debugging enhancements (#1119) #4296

Sign in to view logs

Small debugging enhancements (#1119)

Small debugging enhancements (#1119) #4296

Workflow file for this run

.github/workflows/ci.yaml at 5d337b6
name: CI
on:
merge_group:
pull_request:
push:
# We need to explicitly include tags because otherwise when adding
# `branches-ignore` it will only trigger on branches.
tags:
- '*'
branches-ignore:
# Ignore pushes to merge queues.
# We only want to test the merge commit (`merge_group` event), the hashes
# in the push were already tested by the PR checks
- 'gh-readonly-queue/**'
- 'dependabot/**'
workflow_dispatch:
env:
# Please make sure this version is included in the `matrix`, as the
# `matrix` section can't use `env`, so it must be entered manually
DEFAULT_PYTHON_VERSION: '3.11'
# It would be nice to be able to also define a DEFAULT_UBUNTU_VERSION
# but sadly `env` can't be used either in `runs-on`.
jobs:
nox:
name: Test with nox
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04
python:
- "3.11"
- "3.12"
nox-session:
# To speed things up a bit we use the special ci_checks_max session
# that uses the same venv to run multiple linting sessions
- "ci_checks_max"
- "pytest_min"
runs-on: ${{ matrix.os }}
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Print environment (debug)
run: env
- name: Fetch sources
uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: 'pip'
- name: Install required Python packages
run: |
python -m pip install --upgrade pip
python -m pip install -e .[dev-noxfile]
pip freeze
- name: Create nox venv
env:
NOX_SESSION: ${{ matrix.nox-session }}
run: nox --install-only -e "$NOX_SESSION"
- name: Print pip freeze for nox venv (debug)
env:
NOX_SESSION: ${{ matrix.nox-session }}
run: |
. ".nox/$NOX_SESSION/bin/activate"
pip freeze
deactivate
- name: Run nox
env:
NOX_SESSION: ${{ matrix.nox-session }}
run: nox -R -e "$NOX_SESSION"
timeout-minutes: 10
# This job runs if all the `nox` matrix jobs ran and succeeded.
# It is only used to have a single job that we can require in branch
# protection rules, so we don't have to update the protection rules each time
# we add or remove a job from the matrix.
nox-all:
# The job name should match the name of the `nox` job.
name: Test with nox
needs: ["nox"]
# We skip this job only if nox was also skipped
if: always() && needs.nox.result != 'skipped'
runs-on: ubuntu-20.04
env:
DEPS_RESULT: ${{ needs.nox.result }}
steps:
- name: Check matrix job result
run: test "$DEPS_RESULT" = "success"
nox-cross-arch:
name: Cross-arch tests with nox
if: github.event_name != 'pull_request'
strategy:
fail-fast: false
# Before adding new items to this matrix, make sure that a dockerfile
# exists for the combination of items in the matrix.
# Refer to .github/containers/nox-cross-arch/README.md to learn how to
# add and name new dockerfiles.
matrix:
arch:
- arm64
os:
- ubuntu-20.04
python:
- "3.11"
- "3.12"
nox-session:
- "pytest_min"
- "pytest_max"
runs-on: ${{ matrix.os }}
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Fetch sources
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/${{ matrix.arch }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# This is a workaround to prevent the cache from growing indefinitely.
# https://docs.docker.com/build/ci/github-actions/cache/#local-cache
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Cache container layers
uses: actions/cache@v4
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-nox-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python }}
- name: Build image
uses: docker/build-push-action@v6
with:
context: .github/containers/nox-cross-arch
file: .github/containers/nox-cross-arch/${{ matrix.arch }}-${{ matrix.os }}-python.Dockerfile
platforms: linux/${{ matrix.arch }}
build-args: |
PYTHON_VERSION=${{ matrix.python }}
tags: localhost/nox-cross-arch:latest
push: false
load: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
# Refer to the workaround mentioned above
- name: Move cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
# Cache pip downloads
- name: Cache pip downloads
uses: actions/cache@v4
with:
path: /tmp/pip-cache
key: nox-${{ matrix.nox-session }}-${{ matrix.arch }}-${{ matrix.os }}-${{ matrix.python }}-${{ hashFiles('pyproject.toml') }}
# This ensures that the docker container has access to the pip cache.
# Changing the user in the docker-run step causes it to fail due to
# incorrect permissions. Setting the ownership of the pip cache to root
# before running is a workaround to this issue.
- name: Set pip cache owners to root for docker
run: if [[ -e /tmp/pip-cache ]]; then sudo chown -R root:root /tmp/pip-cache; fi
- name: Run nox
run: |
docker run \
--rm \
-v $(pwd):/${{ github.workspace }} \
-v /tmp/pip-cache:/root/.cache/pip \
-w ${{ github.workspace }} \
--net=host \
--platform linux/${{ matrix.arch }} \
localhost/nox-cross-arch:latest \
bash -c "pip install -e .[dev-noxfile]; nox --install-only -e ${{ matrix.nox-session }}; pip freeze; nox -e ${{ matrix.nox-session }}"
timeout-minutes: 30
# This ensures that the runner has access to the pip cache.
- name: Reset pip cache ownership
if: always()
run: sudo chown -R $USER:$USER /tmp/pip-cache
# This job runs if all the `nox-cross-arch` matrix jobs ran and succeeded.
# As the `nox-all` job, its main purpose is to provide a single point of
# reference in branch protection rules, similar to how `nox-all` operates.
# However, there's a crucial difference: the `nox-cross-arch` job is omitted
# in PRs. Without the `nox-cross-arch-all` job, the inner matrix wouldn't be
# expanded in such scenarios. This would lead to the CI indefinitely waiting
# for these jobs to complete due to the branch protection rules, essentially
# causing it to hang. This behavior is tied to a recognized GitHub matrices
# issue when certain jobs are skipped. For a deeper understanding, refer to:
# https://github.com/orgs/community/discussions/9141
nox-cross-arch-all:
# The job name should match the name of the `nox-cross-arch` job.
name: Cross-arch tests with nox
needs: ["nox-cross-arch"]
# We skip this job only if nox-cross-arch was also skipped
if: always() && needs.nox-cross-arch.result != 'skipped'
runs-on: ubuntu-20.04
env:
DEPS_RESULT: ${{ needs.nox-cross-arch.result }}
steps:
- name: Check matrix job result
run: test "$DEPS_RESULT" = "success"
build:
name: Build distribution packages
runs-on: ubuntu-20.04
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Fetch sources
uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: 'pip'
- name: Install required Python packages
run: |
python -m pip install -U pip
python -m pip install -U build
pip freeze
- name: Build the source and binary distribution
run: python -m build
- name: Upload distribution files
uses: actions/upload-artifact@v4
with:
name: dist-packages
path: dist/
if-no-files-found: error
test-installation:
name: Test package installation in different architectures
needs: ["build"]
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04
python:
- "3.11"
- "3.12"
runs-on: ${{ matrix.os }}
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Fetch sources
uses: actions/checkout@v4
- name: Download package
uses: actions/download-artifact@v4
with:
name: dist-packages
path: dist
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up docker-buildx
uses: docker/setup-buildx-action@v3
- name: Test Installation
uses: docker/build-push-action@v6
with:
context: .
file: .github/containers/test-installation/Dockerfile
platforms: linux/amd64,linux/arm64
build-args: |
PYTHON_VERSION=${{ matrix.python }}
tags: localhost/test-installation
push: false
# This job runs if all the `test-installation` matrix jobs ran and succeeded.
# It is only used to have a single job that we can require in branch
# protection rules, so we don't have to update the protection rules each time
# we add or remove a job from the matrix.
test-installation-all:
# The job name should match the name of the `test-installation` job.
name: Test package installation in different architectures
needs: ["test-installation"]
# We skip this job only if test-installation was also skipped
if: always() && needs.test-installation.result != 'skipped'
runs-on: ubuntu-20.04
env:
DEPS_RESULT: ${{ needs.test-installation.result }}
steps:
- name: Check matrix job result
run: test "$DEPS_RESULT" = "success"
test-docs:
name: Test documentation website generation
if: github.event_name != 'push'
runs-on: ubuntu-20.04
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Fetch sources
uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: 'pip'
- name: Install build dependencies
run: |
python -m pip install -U pip
python -m pip install .[dev-mkdocs]
pip freeze
- name: Generate the documentation
env:
MIKE_VERSION: gh-${{ github.job }}
run: |
mike deploy $MIKE_VERSION
mike set-default $MIKE_VERSION
- name: Upload site
uses: actions/upload-artifact@v4
with:
name: docs-site
path: site/
if-no-files-found: error
publish-docs:
name: Publish documentation website to GitHub pages
needs: ["nox-all", "nox-cross-arch-all", "test-installation"]
if: github.event_name == 'push'
runs-on: ubuntu-20.04
permissions:
contents: write
steps:
- name: Setup Git
uses: frequenz-floss/gh-action-setup-git@v0.x.x
- name: Fetch sources
uses: actions/checkout@v4
with:
submodules: true
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: 'pip'
- name: Install build dependencies
run: |
python -m pip install -U pip
python -m pip install .[dev-mkdocs]
pip freeze
- name: Calculate and check version
id: mike-version
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPO: ${{ github.repository }}
GIT_REF: ${{ github.ref }}
GIT_SHA: ${{ github.sha }}
run: |
python -m frequenz.repo.config.cli.version.mike.info
- name: Fetch the gh-pages branch
if: steps.mike-version.outputs.version
run: git fetch origin gh-pages --depth=1
- name: Build site
if: steps.mike-version.outputs.version
env:
VERSION: ${{ steps.mike-version.outputs.version }}
TITLE: ${{ steps.mike-version.outputs.title }}
ALIASES: ${{ steps.mike-version.outputs.aliases }}
# This is not ideal, we need to define all these variables here
# because we need to calculate all the repository version information
# to be able to show the correct versions in the documentation when
# building it.
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPO: ${{ github.repository }}
GIT_REF: ${{ github.ref }}
GIT_SHA: ${{ github.sha }}
run: |
mike deploy --update-aliases --title "$TITLE" "$VERSION" $ALIASES
- name: Sort site versions
if: steps.mike-version.outputs.version
run: |
git checkout gh-pages
python -m frequenz.repo.config.cli.version.mike.sort versions.json
git commit -a -m "Sort versions.json"
- name: Publish site
if: steps.mike-version.outputs.version
run: |
git push origin gh-pages
create-github-release:
name: Create GitHub release
needs: ["publish-docs"]
# Create a release only on tags creation
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
permissions:
# We need write permissions on contents to create GitHub releases and on
# discussions to create the release announcement in the discussion forums
contents: write
discussions: write
runs-on: ubuntu-20.04
steps:
- name: Download distribution files
uses: actions/download-artifact@v4
with:
name: dist-packages
path: dist
- name: Download RELEASE_NOTES.md
run: |
set -ux
gh api \
-X GET \
-f ref=$REF \
-H "Accept: application/vnd.github.raw" \
"/repos/$REPOSITORY/contents/RELEASE_NOTES.md" \
> RELEASE_NOTES.md
env:
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create GitHub release
run: |
set -ux
extra_opts=
if echo "$REF_NAME" | grep -- -; then extra_opts=" --prerelease"; fi
gh release create \
-R "$REPOSITORY" \
--notes-file RELEASE_NOTES.md \
--generate-notes \
$extra_opts \
$REF_NAME \
dist/*
env:
REF_NAME: ${{ github.ref_name }}
REPOSITORY: ${{ github.repository }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
publish-to-pypi:
name: Publish packages to PyPI
needs: ["create-github-release"]
runs-on: ubuntu-20.04
permissions:
# For trusted publishing. See:
# https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
id-token: write
steps:
- name: Download distribution files
uses: actions/download-artifact@v4
with:
name: dist-packages
path: dist
- name: Publish the Python distribution to PyPI
uses: pypa/gh-action-pypi-publish@release/v1