Simplify shared union serializer logic #5343
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| pull_request: {} | |
| env: | |
| COLUMNS: 150 | |
| jobs: | |
| coverage: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - id: cache-rust | |
| name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: cargo install rustfilt coverage-prepare | |
| if: steps.cache-rust.outputs.cache-hit != 'true' | |
| - run: rustup component add llvm-tools-preview | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -r tests/requirements.txt | |
| - run: rustc --version --verbose | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: '-C instrument-coverage' | |
| - run: pip freeze | |
| - run: coverage run -m pytest | |
| - run: ls -lha | |
| - run: coverage xml | |
| - run: coverage-prepare lcov python/pydantic_core/*.so | |
| - uses: codecov/codecov-action@v4 | |
| # See https://github.com/PyO3/pyo3/discussions/2781 | |
| # tests intermittently segfault with pypy and cpython 3.7 when using `coverage run ...`, hence separate job | |
| test-python: | |
| name: test ${{ matrix.python-version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - '3.13' | |
| - 'pypy3.9' | |
| - 'pypy3.10' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-v3 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| allow-prereleases: true | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| env: | |
| HYPOTHESIS_PROFILE: slow | |
| test-os: | |
| name: test on ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| - run: cargo test | |
| test-msrv: | |
| name: test MSRV | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: resolve MSRV | |
| id: resolve-msrv | |
| run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT | |
| - name: install rust MSRV | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| toolchain: ${{ steps.resolve-msrv.outputs.MSRV }} | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install -e . | |
| env: | |
| RUST_BACKTRACE: 1 | |
| - run: pip freeze | |
| - run: pytest | |
| - run: cargo test | |
| # test with a debug build as it picks up errors which optimised release builds do not | |
| test-debug: | |
| name: test-debug ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - '3.13' | |
| - 'pypy3.10' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - run: pip install -r tests/requirements.txt | |
| - run: make build-dev | |
| - run: pip freeze | |
| - run: pytest | |
| test-pydantic-integration: | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: pydantic/pydantic | |
| path: pydantic | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: pydantic-core | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - name: install deps | |
| run: | | |
| pip install pdm maturin | |
| pdm venv create --with-pip | |
| pdm install -G testing -G email | |
| pdm run pip install maturin | |
| pdm run bash -c 'cd ../pydantic-core && make build-dev' | |
| working-directory: pydantic | |
| - run: pdm info && pdm list | |
| working-directory: pydantic | |
| # Run pytest with lax xfail because we often add tests to pydantic | |
| # which xfail on a pending release of pydantic-core | |
| - run: pdm run pytest --override-ini=xfail_strict=False | |
| working-directory: pydantic | |
| env: | |
| PYDANTIC_PRIVATE_ALLOW_UNHANDLED_SCHEMA_TYPES: 1 | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust stable | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| # used to lint js code | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - uses: actions/cache@v4 | |
| id: cache-py | |
| name: cache python | |
| with: | |
| path: ${{ env.pythonLocation }} | |
| key: > | |
| py | |
| ${{ env.pythonLocation }} | |
| ${{ hashFiles('tests/requirements-linting.txt') }} | |
| ${{ hashFiles('pyproject.toml') }} | |
| - run: pip install -r tests/requirements-linting.txt | |
| if: steps.cache-py.outputs.cache-hit != 'true' | |
| - run: make build-dev | |
| - run: pip freeze | |
| - run: make lint | |
| - run: make pyright | |
| - run: npm install | |
| - run: npm run lint | |
| bench: | |
| name: rust benchmarks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install typing_extensions | |
| - run: cargo bench | |
| build-wasm-emscripten: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: install rust nightly | |
| uses: dtolnay/rust-toolchain@nightly | |
| with: | |
| components: rust-src | |
| targets: wasm32-unknown-emscripten | |
| - name: cache rust | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: mymindstorm/setup-emsdk@v14 | |
| with: | |
| # NOTE!: as per https://github.com/pydantic/pydantic-core/pull/149 this version needs to match the version | |
| # in node_modules/pyodide/repodata.json, to get the version, run: | |
| # `cat node_modules/pyodide/repodata.json | python -m json.tool | rg platform` | |
| version: '3.1.58' | |
| actions-cache-folder: emsdk-cache | |
| - run: pip install 'maturin>=1,<2' 'ruff==0.5.0' typing_extensions | |
| - name: build wheels | |
| run: make build-wasm | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - run: npm install | |
| - run: npm run test | |
| - run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: wasm_wheels | |
| path: dist | |
| # https://github.com/marketplace/actions/alls-green#why used for branch protection checks | |
| check: | |
| if: always() | |
| needs: [coverage, test-python, test-os, test-debug, lint, bench, build-wasm-emscripten] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| jobs: ${{ toJSON(needs) }} | |
| allowed-failures: coverage | |
| build-sdist: | |
| name: build sdist | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| rust-toolchain: stable | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_sdist | |
| path: dist | |
| build: | |
| name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }}) | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, macos, windows] | |
| target: [x86_64, aarch64] | |
| manylinux: [auto] | |
| include: | |
| # manylinux for various platforms, plus x86_64 pypy | |
| - os: linux | |
| manylinux: auto | |
| target: i686 | |
| - os: linux | |
| manylinux: auto | |
| target: aarch64 | |
| - os: linux | |
| manylinux: auto | |
| target: armv7 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: ppc64le | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: s390x | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: linux | |
| manylinux: auto | |
| target: x86_64 | |
| interpreter: pypy3.9 pypy3.10 | |
| # musllinux | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: x86_64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: aarch64 | |
| - os: linux | |
| manylinux: musllinux_1_1 | |
| target: armv7 | |
| # macos; | |
| # all versions x86_64 | |
| # arm pypy and older pythons which can't be run on the arm hardware for PGO | |
| - os: macos | |
| target: x86_64 | |
| - os: macos | |
| target: aarch64 | |
| interpreter: 3.8 3.9 pypy3.9 pypy3.10 | |
| # windows; | |
| # x86_64 pypy builds are not PGO optimized | |
| # i686 not supported by pypy | |
| # aarch64 only 3.11 and up, also not PGO optimized | |
| - os: windows | |
| target: x86_64 | |
| interpreter: pypy3.9 pypy3.10 | |
| - os: windows | |
| target: i686 | |
| python-architecture: x86 | |
| interpreter: 3.8 3.9 3.10 3.11 3.12 3.13 | |
| - os: windows | |
| target: aarch64 | |
| interpreter: 3.11 3.12 3.13 | |
| exclude: | |
| # See above; disabled for now. | |
| - os: windows | |
| target: aarch64 | |
| runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| architecture: ${{ matrix.python-architecture || 'x64' }} | |
| - run: pip install -U twine 'ruff==0.5.0' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - name: build wheels | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| manylinux: ${{ matrix.manylinux }} | |
| args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13 pypy3.9 pypy3.10' }} | |
| # Limit windows builds to 1.77 to keep Windows 7 support. | |
| # FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10) | |
| rust-toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }} | |
| docker-options: -e CI | |
| - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} dist/ | |
| - run: twine check --strict dist/* | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }} | |
| path: dist | |
| build-pgo: | |
| name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }} | |
| # only run on push to main and on release | |
| if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [linux, windows, macos] | |
| interpreter: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] | |
| include: | |
| # standard runners with override for macos arm | |
| - os: linux | |
| runs-on: ubuntu-latest | |
| - os: windows | |
| ls: dir | |
| runs-on: windows-latest | |
| - os: macos | |
| runs-on: macos-latest-xlarge | |
| exclude: | |
| # macos arm only supported from 3.10 and up | |
| - os: macos | |
| interpreter: '3.8' | |
| - os: macos | |
| interpreter: '3.9' | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.interpreter }} | |
| allow-prereleases: true | |
| - name: install rust stable | |
| id: rust-toolchain | |
| uses: dtolnay/rust-toolchain@master | |
| with: | |
| components: llvm-tools | |
| # Limit windows builds to 1.77 to keep Windows 7 support. | |
| # FIXME: Unpin when Python 3.8 support is dropped. (3.9 requires Windows 10) | |
| toolchain: ${{ (matrix.os == 'windows' && '1.77') || 'stable' }} | |
| - run: pip install -U 'ruff==0.5.0' typing_extensions | |
| # generate self-schema now, so we don't have to do so inside docker in maturin build | |
| - run: python generate_self_schema.py | |
| - name: build initial wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out pgo-wheel | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: ${{ steps.rust-toolchain.outputs.name }} | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: '-Cprofile-generate=${{ github.workspace }}/profdata' | |
| - name: detect rust host | |
| run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: generate pgo data | |
| run: | | |
| pip install -U pip | |
| pip install -r tests/requirements.txt | |
| pip install pydantic-core --no-index --no-deps --find-links pgo-wheel --force-reinstall | |
| pytest tests/benchmarks | |
| rustup run ${{ steps.rust-toolchain.outputs.name }} bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"' | |
| - name: merge pgo data | |
| run: ${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata | |
| - name: build pgo-optimized wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: auto | |
| args: > | |
| --release | |
| --out dist | |
| --interpreter ${{ matrix.interpreter }} | |
| rust-toolchain: ${{steps.rust-toolchain.outputs.name}} | |
| docker-options: -e CI | |
| env: | |
| RUSTFLAGS: '-Cprofile-use=${{ github.workspace }}/merged.profdata' | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }} | |
| path: dist | |
| inspect-pypi-assets: | |
| needs: [build, build-sdist, build-pgo] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - name: list dist files | |
| run: | | |
| ls -lh dist/ | |
| ls -l dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp310-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1` | |
| test-builds-arch: | |
| name: test build on ${{ matrix.target }}-${{ matrix.distro }} | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: [aarch64, armv7, s390x, ppc64le] | |
| distro: ['ubuntu22.04'] | |
| include: | |
| - target: aarch64 | |
| distro: alpine_latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_linux_* | |
| merge-multiple: true | |
| path: dist | |
| - uses: uraimo/run-on-arch-action@v2.8.1 | |
| name: install & test | |
| with: | |
| arch: ${{ matrix.target }} | |
| distro: ${{ matrix.distro }} | |
| githubToken: ${{ github.token }} | |
| install: | | |
| set -x | |
| if command -v apt-get &> /dev/null; then | |
| echo "installing python & pip with apt-get..." | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-pip python3-venv git | |
| else | |
| echo "installing python & pip with apk..." | |
| apk update | |
| apk add python3 py3-pip git | |
| fi | |
| run: | | |
| set -x | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| python3 -m pip install -r tests/requirements.txt | |
| python3 -m pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| python3 -m pytest --ignore=tests/test_docstrings.py | |
| python3 -c 'import pydantic_core._pydantic_core; print(pydantic_core._pydantic_core.__version__)' | |
| test-builds-os: | |
| name: test build on ${{ matrix.os }} | |
| needs: [build, build-pgo] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos, windows] | |
| runs-on: ${{ matrix.os }}-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: pip install -r tests/requirements.txt | |
| - run: pip install pydantic-core --no-index --no-deps --find-links dist --force-reinstall | |
| - run: pytest --ignore=tests/test_docstrings.py | |
| release: | |
| needs: [test-builds-arch, test-builds-os, build-sdist, check] | |
| if: success() && startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.13' | |
| - run: pip install -U twine | |
| - name: check package version | |
| run: python .github/check_version.py | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: pypi_files_* | |
| merge-multiple: true | |
| path: dist | |
| - run: twine check --strict dist/* | |
| - name: upload to pypi | |
| run: twine upload dist/* | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.pypi_token }} | |
| - name: get wasm dist artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wasm_wheels | |
| path: wasm | |
| - name: upload to github release | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: | | |
| wasm/*.whl | |
| prerelease: ${{ contains(github.ref, 'alpha') || contains(github.ref, 'beta') }} |