Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

try using tests/benchmarks as PGO input #686

Closed
wants to merge 3 commits into from
Closed

try using tests/benchmarks as PGO input #686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d2d6c48
try pgo for benchmarks run
davidhewitt Jun 20, 2023
8ef5de6
add pgo builds for release artifacts on ubuntu
davidhewitt Jun 28, 2023
bef03cb
try using tests/benchmarks as PGO input
davidhewitt Jun 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 98 additions & 15 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,10 +287,25 @@ jobs:
jobs: ${{ toJSON(needs) }}
allowed-failures: coverage

build-sdist:
name: build sdist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v3
with:
name: pypi_files
path: dist

build:
name: build on ${{ matrix.platform || matrix.os }} (${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
# only run on push to main and on release
if: "success() && (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build'))"
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -345,6 +360,11 @@ jobs:
container: messense/manylinux_2_24-cross:s390x
interpreter: 3.7 3.8 3.9 3.10 3.11
exclude:
# Optimized PGO builds for manylinux follow a different matrix, maybe in future
# maturin-action can support this automatically
- os: ubuntu
target: x86_64
manylinux: auto
# Windows on arm64 only supports Python 3.11+
- os: windows
target: aarch64
Expand All @@ -359,19 +379,11 @@ jobs:
python-version: '3.11'
architecture: ${{ matrix.python-architecture || 'x64' }}

- run: pip install -U twine 'black>=22.3.0,<23' typing_extensions
- run: pip install -U 'black>=22.3.0,<23' typing_extensions

# generate self-schema now, so we don't have to do so inside docker in maturin build
- run: python generate_self_schema.py

- name: build sdist
if: ${{ matrix.os == 'ubuntu' && matrix.target == 'x86_64' && matrix.manylinux == 'auto' }}
uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
rust-toolchain: stable

- name: build wheels
uses: PyO3/maturin-action@v1
with:
Expand All @@ -384,15 +396,86 @@ jobs:

- run: ${{ matrix.ls || 'ls -lh' }} dist/

- run: twine check --strict dist/*
- uses: actions/upload-artifact@v3
with:
name: pypi_files
path: dist

build-pgo:
name: build pgo-optimized on ${{ matrix.platform || matrix.os }} (${{ matrix.interpreter}} - ${{ matrix.target }} - ${{ matrix.manylinux || 'auto' }})
# only run on push to main and on release
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
strategy:
fail-fast: false
matrix:
os: [ubuntu]
platform: [linux]
target: [x86_64]
manylinux: [auto]
interpreter: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8", "pypy3.9"]

runs-on: ${{ matrix.os }}-latest
steps:
- uses: actions/checkout@v3

- name: set up python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.interpreter }}
architecture: ${{ matrix.python-architecture || 'x64' }}

- name: install rust stable
id: rust-toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools

- run: pip install -U 'black>=22.3.0,<23' typing_extensions

# generate self-schema now, so we don't have to do so inside docker in maturin build
- run: python generate_self_schema.py

- name: build initial wheel
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux || 'auto' }}
args: >
--release
--out pgo-wheel
--interpreter ${{ matrix.interpreter }}
-- -Cprofile-generate=${{ github.workspace }}/profdata
rust-toolchain: stable

- name: generate pgo data
run: |
pip install -U pip
pip install -r tests/requirements.txt
pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
pytest tests/benchmarks
rustup run stable bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'

- name: build pgo-optimized wheel
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux || 'auto' }}
args: >
--release
--out dist
--interpreter ${{ matrix.interpreter }}
-- -Cprofile-use=${{ github.workspace }}/merged.profdata
rust-toolchain: stable

- run: ${{ matrix.ls || 'ls -lh' }} dist/

- uses: actions/upload-artifact@v3
with:
name: pypi_files
path: dist

inspect-pypi-assets:
needs: [build]
needs: [build, build-sdist, build-pgo]
runs-on: ubuntu-latest

steps:
Expand Down Expand Up @@ -474,7 +557,7 @@ jobs:

test-builds-os:
name: test build on ${{ matrix.os }}
needs: [build]
needs: [build, build-pgo]

strategy:
fail-fast: false
Expand Down Expand Up @@ -502,8 +585,8 @@ jobs:
- run: pytest --ignore=tests/test_docstrings.py

release:
needs: [test-builds-arch, test-builds-os, check]
if: "success() && startsWith(github.ref, 'refs/tags/')"
needs: [test-builds-arch, test-builds-os, build-sdist, check]
if: success() && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest

steps:
Expand Down
21 changes: 20 additions & 1 deletion .github/workflows/codspeed.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,37 @@ jobs:
if: steps.cache-py.outputs.cache-hit != 'true'

- name: install rust stable
id: rust-toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools

- name: cache rust
uses: Swatinem/rust-cache@v2

- name: Install pydantic-core
- name: Compile pydantic-core for profiling
# --no-default-features to avoid using mimalloc
run: |
pip install -e . --config-settings=build-args='--no-default-features --verbose' -v
python -c 'import pydantic_core; assert pydantic_core._pydantic_core.__pydantic_core_default_allocator__'
env:
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"

- name: Gather pgo data
run: pytest tests/benchmarks

- name: Prepare merged pgo data
run: rustup run stable bash -c '$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/x86_64-unknown-linux-gnu/bin/llvm-profdata merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata'

- name: Compile pydantic-core for benchmarking
# --no-default-features to avoid using mimalloc
run: |
pip install -e . --config-settings=build-args='--no-default-features --verbose' -v
python -c 'import pydantic_core; assert pydantic_core._pydantic_core.__pydantic_core_default_allocator__'
env:
CONST_RANDOM_SEED: 0 # Fix the compile time RNG seed
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"

- name: Run CodSpeed benchmarks
uses: CodSpeedHQ/action@v1
Expand Down