Skip to content
Back to pull request #1125

fix memory leak with recursive definitions creating reference cycles #4101

Sign in to view logs

fix memory leak with recursive definitions creating reference cycles

fix memory leak with recursive definitions creating reference cycles #4101

Workflow file for this run

.github/workflows/ci.yml at f9d1e1a
name: ci
on:
push:
branches:
- main
tags:
- '**'
pull_request: {}
env:
COLUMNS: 150
jobs:
coverage:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- name: install rust nightly
uses: dtolnay/rust-toolchain@nightly
- id: cache-rust
name: cache rust
uses: Swatinem/rust-cache@v2
with:
key: coverage-v2
- run: cargo install rustfilt coverage-prepare
if: steps.cache-rust.outputs.cache-hit != 'true'
- run: rustup component add llvm-tools-preview
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- run: pip install -r tests/requirements.txt
- run: rustc --version --verbose
- run: pip install -e .
env:
RUST_BACKTRACE: 1
RUSTFLAGS: '-C instrument-coverage'
- run: pip freeze
- run: coverage run -m pytest
- run: ls -lha
- run: coverage xml
- run: coverage-prepare lcov python/pydantic_core/*.so
- uses: codecov/codecov-action@v3
# See https://github.com/PyO3/pyo3/discussions/2781
# tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job
test-python:
name: test ${{ matrix.python-version }}
strategy:
fail-fast: false
matrix:
python-version:
- '3.7'
- '3.8'
- '3.9'
- '3.10'
- '3.11'
- '3.12'
- 'pypy3.7'
- 'pypy3.8'
- 'pypy3.9'
- 'pypy3.10'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: install rust stable
uses: dtolnay/rust-toolchain@stable
- name: cache rust
uses: Swatinem/rust-cache@v2
with:
key: test-v3
- name: set up python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- run: pip install -r tests/requirements.txt
- run: pip install -e .
env:
RUST_BACKTRACE: 1
- run: pip freeze
- run: pytest
env:
HYPOTHESIS_PROFILE: slow
test-os:
name: test on ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
runs-on: ${{ matrix.os }}-latest
steps:
- uses: actions/checkout@v4
- name: install rust stable
uses: dtolnay/rust-toolchain@stable
- name: cache rust
uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.os }}-v1
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- run: pip install -r tests/requirements.txt
- run: pip install -e .
env:
RUST_BACKTRACE: 1
- run: pip freeze
- run: pytest
- run: cargo test
# test with a debug build as it picks up errors which optimised release builds do not
test-debug:
name: test-debug ${{ matrix.python-version }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version:
- '3.11'
- 'pypy3.10'
steps:
- uses: actions/checkout@v4
- name: set up python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: install rust stable
uses: dtolnay/rust-toolchain@stable
- name: cache rust
uses: Swatinem/rust-cache@v2
with:
key: test-debug
- run: pip install -r tests/requirements.txt
- run: make build-dev
- run: pip freeze
- run: pytest
test-pydantic-integration:
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@v4
with:
repository: pydantic/pydantic
path: pydantic
- uses: actions/checkout@v4
with:
path: pydantic-core
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: install rust stable
uses: dtolnay/rust-toolchain@stable
- name: cache rust
uses: Swatinem/rust-cache@v2
with:
key: test-pydantic-integration
- name: install deps
run: |
pip install pdm maturin
pdm venv create --with-pip
pdm install -G testing -G email
pdm run pip install maturin
pdm run bash -c 'cd ../pydantic-core && make build-dev'
working-directory: pydantic
- run: pdm info && pdm list
working-directory: pydantic
# Run pytest with lax xfail because we often add tests to pydantic
# which xfail on a pending release of pydantic-core
- run: pdm run pytest --override-ini=xfail_strict=False
working-directory: pydantic
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: install rust stable
uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
- name: cache rust
uses: Swatinem/rust-cache@v2
- uses: actions/setup-python@v4
with:
python-version: '3.11'
# used to lint js code
- uses: actions/setup-node@v4
with:
node-version: '18'
- uses: actions/cache@v3
id: cache-py
name: cache python
with:
path: ${{ env.pythonLocation }}
key: >
py
${{ env.pythonLocation }}
${{ hashFiles('tests/requirements-linting.txt') }}
${{ hashFiles('pyproject.toml') }}
- run: pip install -r tests/requirements-linting.txt
if: steps.cache-py.outputs.cache-hit != 'true'
- run: make build-dev
- run: pip freeze
- run: make lint
- run: make pyright
- run: npm install
- run: npm run lint
bench:
name: rust benchmarks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: install rust nightly
uses: dtolnay/rust-toolchain@nightly
- name: cache rust
uses: Swatinem/rust-cache@v2
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- run: pip install typing_extensions
- run: cargo bench
build-wasm-emscripten:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: set up python
id: setup-python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: install rust nightly
uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
targets: wasm32-unknown-emscripten
- name: cache rust
uses: Swatinem/rust-cache@v2
- uses: mymindstorm/setup-emsdk@v13
with:
# NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version
# in node_modules/pyodide/repodata.json, to get the version, run:
# `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform`
version: '3.1.32'
actions-cache-folder: emsdk-cache
- run: pip install 'maturin>=1,<2' 'ruff==0.1.3' typing_extensions
- name: build wheels
run: make build-wasm
- uses: actions/setup-node@v4
with:
node-version: '18'
- run: npm install
- run: npm run test
- run: |
ls -lh dist/
ls -l dist/
- uses: actions/upload-artifact@v3
with:
name: wasm_wheels
path: dist
# https://github.com/marketplace/actions/alls-green#why used for branch protection checks
check:
if: always()
needs: [coverage, test-python, test-os, test-debug, lint, bench, build-wasm-emscripten]
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
allowed-failures: coverage
build-sdist:
name: build sdist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
rust-toolchain: stable
- uses: actions/upload-artifact@v3
with:
name: pypi_files
path: dist
build:
name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }})
# only run on push to main and on release
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
strategy:
fail-fast: false
matrix:
os: [linux, macos, windows]
target: [x86_64, aarch64]
manylinux: [auto]
include:
# manylinux for various platforms, plus x86_64 pypy
- os: linux
manylinux: auto
target: i686
- os: linux
manylinux: auto
target: aarch64
- os: linux
manylinux: auto
target: armv7
interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
- os: linux
manylinux: auto
target: ppc64le
interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
- os: linux
manylinux: auto
target: s390x
interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
- os: linux
manylinux: auto
target: x86_64
interpreter: pypy3.7 pypy3.8 pypy3.9 pypy3.10
# musllinux
- os: linux
manylinux: musllinux_1_1
target: x86_64
- os: linux
manylinux: musllinux_1_1
target: aarch64
# macos;
# all versions x86_64
# arm pypy and older pythons which can't be run on the arm hardware for PGO
- os: macos
target: x86_64
- os: macos
target: aarch64
interpreter: 3.7 3.8 3.9 pypy3.8 pypy3.9 pypy3.10
# windows;
# x86_64 pypy builds are not PGO optimized
# i686 not supported by pypy
# aarch64 only 3.11 and up, also not PGO optimized
- os: windows
target: x86_64
interpreter: pypy3.8 pypy3.9 pypy3.10
- os: windows
target: i686
python-architecture: x86
interpreter: 3.7 3.8 3.9 3.10 3.11 3.12
- os: windows
target: aarch64
interpreter: 3.11 3.12
runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest
steps:
- uses: actions/checkout@v4
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
architecture: ${{ matrix.python-architecture || 'x64' }}
- run: pip install -U twine 'ruff==0.1.3' typing_extensions
# generate self-schema now, so we don't have to do so inside docker in maturin build
- run: python generate_self_schema.py
- name: build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
manylinux: ${{ matrix.manylinux == 'manylinux' && 'auto' || matrix.manylinux }}
args: --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 3.12 pypy3.7 pypy3.8 pypy3.9 pypy3.10' }}
rust-toolchain: stable
docker-options: -e CI
- run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} dist/
- run: twine check --strict dist/*
- uses: actions/upload-artifact@v3
with:
name: pypi_files
path: dist
build-pgo:
name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }}
# only run on push to main and on release
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
strategy:
fail-fast: false
matrix:
os: [linux, windows, macos]
interpreter: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
include:
# standard runners with override for macos arm
- os: linux
runs-on: ubuntu-latest
- os: windows
ls: dir
runs-on: windows-latest
- os: macos
runs-on: macos-latest-xlarge
exclude:
# macos arm only supported from 3.10 and up
- os: macos
interpreter: '3.7'
- os: macos
interpreter: '3.8'
- os: macos
interpreter: '3.9'
runs-on: ${{ matrix.runs-on }}
steps:
- uses: actions/checkout@v4
- name: set up python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.interpreter }}
- name: install rust stable
id: rust-toolchain
uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools
- run: pip install -U 'ruff==0.1.3' typing_extensions
# generate self-schema now, so we don't have to do so inside docker in maturin build
- run: python generate_self_schema.py
- run: rustc --version --verbose
- name: build initial wheel
uses: PyO3/maturin-action@v1
with:
manylinux: auto
args: >
--release
--out pgo-wheel
--interpreter ${{ matrix.interpreter }}
rust-toolchain: stable
docker-options: -e CI
env:
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
- name: detect rust host
run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
shell: bash
- name: generate pgo data
run: |
pip install -U pip
pip install -r tests/requirements.txt
pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall
pytest tests/benchmarks
rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
- name: merge pgo data
run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
- name: build pgo-optimized wheel
uses: PyO3/maturin-action@v1
with:
manylinux: auto
args: >
--release
--out dist
--interpreter ${{ matrix.interpreter }}
rust-toolchain: stable
docker-options: -e CI
env:
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
- run: ${{ matrix.ls || 'ls -lh' }} dist/
- uses: actions/upload-artifact@v3
with:
name: pypi_files_pgo
path: dist
inspect-pypi-assets:
needs: [build, build-sdist, build-pgo]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: get dist artifacts
uses: actions/download-artifact@v3
with:
name: pypi_files
path: dist
- name: list dist files before PGO builds
run: |
ls -lh dist/
ls -l dist/
echo "`ls dist | wc -l` files"
- name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
uses: actions/download-artifact@v3
with:
name: pypi_files_pgo
path: dist
- name: list dist files with PGO builds
run: |
ls -lh dist/
ls -l dist/
echo "`ls dist | wc -l` files"
- name: extract and list sdist file
run: |
mkdir sdist-files
tar -xvf dist/*.tar.gz -C sdist-files
tree -a sdist-files
- name: extract and list wheel file
run: |
ls dist/*cp310-manylinux*x86_64.whl | head -n 1
python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1`
test-builds-arch:
name: test build on ${{ matrix.target }}-${{ matrix.distro }}
needs: [build]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
target: [aarch64, armv7, s390x, ppc64le]
distro: ['ubuntu22.04']
include:
- target: aarch64
distro: alpine_latest
steps:
- uses: actions/checkout@v4
- name: get dist artifacts
uses: actions/download-artifact@v3
with:
name: pypi_files
path: dist
- name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
uses: actions/download-artifact@v3
with:
name: pypi_files_pgo
path: dist
- uses: uraimo/run-on-arch-action@v2.6.0
name: install & test
with:
arch: ${{ matrix.target }}
distro: ${{ matrix.distro }}
githubToken: ${{ github.token }}
install: |
set -x
if command -v apt-get &> /dev/null; then
echo "installing python & pip with apt-get..."
apt-get update
apt-get install -y --no-install-recommends python3 python3-pip python3-venv
else
echo "installing python & pip with apk..."
apk update
apk add python3 py3-pip
fi
run: |
set -x
# typing-extensions isn't automatically installed because of `--no-index --no-deps`
python3 -m venv venv
source venv/bin/activate
python3 -m pip install -U pip typing-extensions
python3 -m pip install -r tests/requirements.txt
python3 -m pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall
python3 -m pytest --ignore=tests/test_docstrings.py
python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)'
test-builds-os:
name: test build on ${{ matrix.os }}
needs: [build, build-pgo]
strategy:
fail-fast: false
matrix:
os: [ubuntu, macos, windows]
runs-on: ${{ matrix.os }}-latest
steps:
- uses: actions/checkout@v4
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: get dist artifacts
uses: actions/download-artifact@v3
with:
name: pypi_files
path: dist
- name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
uses: actions/download-artifact@v3
with:
name: pypi_files_pgo
path: dist
- run: pip install typing-extensions
- run: pip install -r tests/requirements.txt
- run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall
- run: pytest --ignore=tests/test_docstrings.py
release:
needs: [test-builds-arch, test-builds-os, build-sdist, check]
if: success() && startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: set up python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- run: pip install -U twine
- name: check package version
run: python .github/check_version.py
- name: get dist artifacts
uses: actions/download-artifact@v3
with:
name: pypi_files
path: dist
- name: get PGO dist artifacts (comes after "get dist artifacts" to so these files override the non-PGO builds)
uses: actions/download-artifact@v3
with:
name: pypi_files_pgo
path: dist
- run: twine check --strict dist/*
- name: upload to pypi
run: twine upload dist/*
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.pypi_token }}
- name: get wasm dist artifacts
uses: actions/download-artifact@v3
with:
name: wasm_wheels
path: wasm
- name: upload to github release
uses: softprops/action-gh-release@v1
with:
files: |
wasm/*.whl
prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }}