Skip to content

[3.12] gh-111062: CI: Move OS test jobs to reusable workflows #111568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 16, 2023
Merged

[3.12] gh-111062: CI: Move OS test jobs to reusable workflows #111568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6b1203c
CI: Move OS test jobs to reusable workflows
hugovk Oct 31, 2023
4bb8440
Merge branch '3.12' into 3.12-ci-free-threaded
corona10 Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 14 additions & 147 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ jobs:
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- name: Install dependencies
run: |
Expand Down Expand Up @@ -208,159 +208,32 @@ jobs:
if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME
run: make check-c-globals

build_win32:
name: 'Windows (x86)'
runs-on: windows-latest
timeout-minutes: 60
build_windows:
name: 'Windows'
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p Win32
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0

build_win_amd64:
name: 'Windows (x64)'
runs-on: windows-latest
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Register MSVC problem matcher
run: echo "::add-matcher::.github/problem-matchers/msvc.json"
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p x64
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0

build_win_arm64:
name: 'Windows (arm64)'
runs-on: windows-latest
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Register MSVC problem matcher
run: echo "::add-matcher::.github/problem-matchers/msvc.json"
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p arm64
uses: ./.github/workflows/reusable-windows.yml

build_macos:
name: 'macOS'
runs-on: macos-latest
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTO_UPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1
PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- uses: actions/checkout@v4
- name: Restore config.cache
uses: actions/cache@v3
with:
path: config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
- name: Install Homebrew dependencies
run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
- name: Configure CPython
run: |
GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \
./configure \
--config-cache \
--with-pydebug \
--prefix=/opt/python-dev \
--with-openssl="$(brew --prefix openssl@3.0)"
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Tests
run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
uses: ./.github/workflows/reusable-macos.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}

build_ubuntu:
name: 'Ubuntu'
runs-on: ubuntu-20.04
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
env:
OPENSSL_VER: 3.0.11
PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- uses: actions/checkout@v4
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Configure OpenSSL env vars
run: |
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
- name: 'Restore OpenSSL build'
id: cache-openssl
uses: actions/cache@v3
with:
path: ./multissl/openssl/${{ env.OPENSSL_VER }}
key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- name: Setup directory envs for out-of-tree builds
run: |
echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
- name: Create directories for read-only out-of-tree builds
run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
- name: Bind mount sources read-only
run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
- name: Restore config.cache
uses: actions/cache@v3
with:
path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}
- name: Configure CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: |
uses: ./.github/workflows/reusable-ubuntu.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
options: |
../cpython-ro-srcdir/configure \
--config-cache \
--with-pydebug \
--with-openssl=$OPENSSL_DIR
- name: Build CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make -j4
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- name: Remount sources writable for tests
# some tests write to srcdir, lack of pyc files slows down testing
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
- name: Tests
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"

build_ubuntu_ssltests:
name: 'Ubuntu SSL tests with OpenSSL'
Expand Down Expand Up @@ -429,7 +302,7 @@ jobs:
- uses: actions/checkout@v4
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install Dependencies
- name: Install dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Configure OpenSSL env vars
run: |
Expand Down Expand Up @@ -586,12 +459,10 @@ jobs:
- check_source # Transitive dependency, needed to access `run_tests` value
- check-docs
- check_generated_files
- build_win32
- build_win_amd64
- build_win_arm64
- build_macos
- build_ubuntu
- build_ubuntu_ssltests
- build_windows
- test_hypothesis
- build_asan

Expand All @@ -604,8 +475,6 @@ jobs:
allowed-failures: >-
build_macos,
build_ubuntu_ssltests,
build_win32,
build_win_arm64,
test_hypothesis,
allowed-skips: >-
${{
Expand All @@ -619,12 +488,10 @@ jobs:
needs.check_source.outputs.run_tests != 'true'
&& '
check_generated_files,
build_win32,
build_win_amd64,
build_win_arm64,
build_macos,
build_ubuntu,
build_ubuntu_ssltests,
build_windows,
build_asan,
'
|| ''
Expand Down
46 changes: 46 additions & 0 deletions .github/workflows/reusable-macos.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string
free-threaded:
required: false
type: boolean
default: false

jobs:
build_macos:
name: 'build and test'
runs-on: macos-latest
timeout-minutes: 60
env:
HOMEBREW_NO_ANALYTICS: 1
HOMEBREW_NO_AUTO_UPDATE: 1
HOMEBREW_NO_INSTALL_CLEANUP: 1
PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- uses: actions/checkout@v4
- name: Restore config.cache
uses: actions/cache@v3
with:
path: config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
- name: Install Homebrew dependencies
run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk
- name: Configure CPython
run: |
GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \
./configure \
--config-cache \
--with-pydebug \
${{ inputs.free-threaded && '--disable-gil' || '' }} \
--prefix=/opt/python-dev \
--with-openssl="$(brew --prefix openssl@3.0)"
- name: Build CPython
run: make -j4
- name: Display build info
run: make pythoninfo
- name: Tests
run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
71 changes: 71 additions & 0 deletions .github/workflows/reusable-ubuntu.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
on:
workflow_call:
inputs:
config_hash:
required: true
type: string
options:
required: true
type: string

jobs:
build_ubuntu_reusable:
name: 'build and test'
timeout-minutes: 60
runs-on: ubuntu-20.04
env:
OPENSSL_VER: 3.0.11
PYTHONSTRICTEXTENSIONBUILD: 1
steps:
- uses: actions/checkout@v4
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Configure OpenSSL env vars
run: |
echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV
echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV
- name: 'Restore OpenSSL build'
id: cache-openssl
uses: actions/cache@v3
with:
path: ./multissl/openssl/${{ env.OPENSSL_VER }}
key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
- name: Install OpenSSL
if: steps.cache-openssl.outputs.cache-hit != 'true'
run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
- name: Add ccache to PATH
run: |
echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
- name: Configure ccache action
uses: hendrikmuhs/ccache-action@v1.2
- name: Setup directory envs for out-of-tree builds
run: |
echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
- name: Create directories for read-only out-of-tree builds
run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
- name: Bind mount sources read-only
run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
- name: Restore config.cache
uses: actions/cache@v3
with:
path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
key: ${{ github.job }}-${{ runner.os }}-${{ inputs.config_hash }}
- name: Configure CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: ${{ inputs.options }}
- name: Build CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make -j4
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
- name: Remount sources writable for tests
# some tests write to srcdir, lack of pyc files slows down testing
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
- name: Tests
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
53 changes: 53 additions & 0 deletions .github/workflows/reusable-windows.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
on:
workflow_call:
inputs:
free-threaded:
required: false
type: boolean
default: false

jobs:
build_win32:
name: 'build and test (x86)'
runs-on: windows-latest
timeout-minutes: 60
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p Win32 ${{ inputs.free-threaded && '--disable-gil' || '' }}
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0

build_win_amd64:
name: 'build and test (x64)'
runs-on: windows-latest
timeout-minutes: 60
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Register MSVC problem matcher
run: echo "::add-matcher::.github/problem-matchers/msvc.json"
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p x64 ${{ inputs.free-threaded && '--disable-gil' || '' }}
- name: Display build info
run: .\python.bat -m test.pythoninfo
- name: Tests
run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0

build_win_arm64:
name: 'build (arm64)'
runs-on: windows-latest
timeout-minutes: 60
env:
IncludeUwp: 'true'
steps:
- uses: actions/checkout@v4
- name: Register MSVC problem matcher
run: echo "::add-matcher::.github/problem-matchers/msvc.json"
- name: Build CPython
run: .\PCbuild\build.bat -e -d -p arm64 ${{ inputs.free-threaded && '--disable-gil' || '' }}