feat: more economic data skipping with datafusion #7121
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: python_build | |
| on: | |
| merge_group: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| defaults: | |
| run: | |
| working-directory: ./python | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Check Python | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| pip install ruff==0.5.2 mypy==1.10.1 types-dataclasses typing-extensions | |
| make check-python | |
| - name: Check Rust | |
| run: make check-rust | |
| test-minimal: | |
| name: Python Build (Python 3.8 PyArrow 16.0.0) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: 3.8 | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| # Install minimum PyArrow version | |
| pip install -e .[pandas,devel] pyarrow==16.0.0 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make unit-test | |
| test: | |
| name: Python Build (Python 3.10 PyArrow latest) | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Start emulated services | |
| run: docker compose up -d | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop | |
| - name: Download Data Acceptance Tests (DAT) files | |
| run: make setup-dat | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| python -m pytest -m '((s3 or azure) and integration) or not integration and not benchmark' --doctest-modules | |
| - name: Test without pandas | |
| run: | | |
| source venv/bin/activate | |
| pip uninstall --yes pandas | |
| python -m pytest -m "not pandas and not integration and not benchmark" | |
| pip install pandas | |
| benchmark: | |
| name: Python Benchmark | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - name: Build deltalake in release mode | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| MATURIN_EXTRA_ARGS=--release make develop | |
| # Download previous benchmark result from cache (if exists) | |
| - name: Download previous benchmark data | |
| uses: actions/cache@v2 | |
| with: | |
| path: ./cache | |
| key: ${{ runner.os }}-benchmark | |
| - name: Run benchmark | |
| run: | | |
| source venv/bin/activate | |
| pytest tests/test_benchmark.py -m benchmark --benchmark-json output.json | |
| - name: Store benchmark result | |
| uses: benchmark-action/github-action-benchmark@v1 | |
| with: | |
| tool: "pytest" | |
| output-file-path: python/output.json | |
| external-data-json-path: ./cache/benchmark-data.json | |
| fail-on-alert: true | |
| test-pyspark: | |
| name: PySpark Integration Tests | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| - uses: actions/setup-java@v2 | |
| with: | |
| distribution: "zulu" | |
| java-version: "11" | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make develop-pyspark | |
| - name: Run tests | |
| run: | | |
| source venv/bin/activate | |
| make test-pyspark | |
| multi-python-running: | |
| name: Running with Python ${{ matrix.python-version }} | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTFLAGS: "-C debuginfo=line-tables-only" | |
| CARGO_INCREMENTAL: 0 | |
| strategy: | |
| matrix: | |
| python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Environment | |
| uses: ./.github/actions/setup-env | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build and install deltalake | |
| run: | | |
| python -m venv venv | |
| source venv/bin/activate | |
| make setup | |
| maturin develop | |
| - name: Run deltalake | |
| run: | | |
| source venv/bin/activate | |
| python -c 'import deltalake' |