From a5fccad2f35dbb79336da7d6262ca3a17b2b8987 Mon Sep 17 00:00:00 2001 From: Brendan Ward Date: Wed, 4 Sep 2024 23:23:40 -0700 Subject: [PATCH] BLD/RLS: Update wheels to include GDAL 3.9.2 and add manylinux_2_28 wheels (#466) --- .circleci/config.yml | 2 +- .github/workflows/docker-gdal.yml | 8 +- .github/workflows/release.yml | 113 +++++++++++++----- CHANGES.md | 17 +-- ....cmake => x64-linux-dynamic-release.cmake} | 0 ...ic.cmake => x64-osx-dynamic-release.cmake} | 0 ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile | 10 +- ...nylinux_2_28_aarch64-vcpkg-gdal.Dockerfile | 4 +- ...anylinux_2_28_x86_64-vcpkg-gdal.Dockerfile | 33 +++++ ci/vcpkg.json | 4 +- pyproject.toml | 30 +++-- 11 files changed, 159 insertions(+), 62 deletions(-) rename ci/custom-triplets/{x64-linux-dynamic.cmake => x64-linux-dynamic-release.cmake} (100%) rename ci/custom-triplets/{x64-osx-dynamic.cmake => x64-osx-dynamic-release.cmake} (100%) create mode 100644 ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b207874..af135db3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: - run: name: Build the Linux aarch64 wheels. command: | - python3 -m pip install --user cibuildwheel==2.17.0 + python3 -m pip install --user cibuildwheel==2.20.0 python3 -m cibuildwheel --output-dir wheelhouse - run: name: Test the wheels diff --git a/.github/workflows/docker-gdal.yml b/.github/workflows/docker-gdal.yml index 601edbe4..189954a0 100644 --- a/.github/workflows/docker-gdal.yml +++ b/.github/workflows/docker-gdal.yml @@ -20,8 +20,9 @@ jobs: fail-fast: false matrix: container: - - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.10.6 - - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.3" # python 3.10.12 + - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.12.3 + - "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" # python 3.12.3 + - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12 - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6 - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.8.10 @@ -38,6 +39,9 @@ jobs: - uses: actions/checkout@v4 - name: Create virtual environment + # install uv and use it to create a virtual environment, then add it to + # environment variables so that it is automatically activated and can be + # used for tests below run: | curl -LsSf https://astral.sh/uv/install.sh | sh . $HOME/.cargo/env diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 131d78e7..56324590 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,36 +54,60 @@ jobs: needs: [build-sdist] runs-on: ubuntu-latest container: - image: "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" + image: "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" steps: + - name: Install packages + run: | + apt-get update && apt-get install -y build-essential python3-dev + + - name: Create virtual environment + # install uv and use it to create a virtual environment, then add it to + # environment variables so that it is automatically activated and can be + # used for tests below + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + . $HOME/.cargo/env + uv venv .venv + echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV + echo "$PWD/.venv/bin" >> $GITHUB_PATH + - name: Download sdist from artifacts uses: actions/download-artifact@v4 with: name: pyogrio-sdist path: wheelhouse - - name: Install pip - run: | - apt-get update && apt-get install -y python3-pip - - name: Build from sdist and install test dependencies shell: bash run: | - python3 -m pip install --no-cache-dir wheelhouse/*.tar.gz - python3 -m pip install --no-cache-dir pytest pandas pyproj shapely>=2 - python3 -m pip install --no-cache-dir --no-deps geopandas - python3 -m pip list + uv pip install --no-cache wheelhouse/*.tar.gz + uv pip install pytest pandas pyproj shapely>=2 + uv pip install --no-deps geopandas + uv pip list - name: Run tests shell: bash + # virtual environment is automatically activated run: | cd .. - python3 -m pytest --pyargs pyogrio.tests -v + uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" + uv run python -m pytest --pyargs pyogrio.tests -v build-wheels-linux: name: Build wheels on Linux - runs-on: "ubuntu-20.04" + runs-on: "ubuntu-latest" + strategy: + fail-fast: false + matrix: + include: + # use manylinux2014 for older glibc platforms until discontinued + - wheel_name: "pyogrio-wheel-linux-manylinux2014_x86_64" + container: "ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile" + + # use manylinux_2_28 for any platforms with glibc>=2.28 + - wheel_name: "pyogrio-wheel-linux-manylinux_2_28_x86_64" + container: "ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile" steps: - name: Checkout @@ -103,7 +127,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile + file: ${{ matrix.container }} tags: manylinux-vcpkg-gdal:latest push: false load: true @@ -117,7 +141,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: pyogrio-wheel-linux + name: ${{ matrix.wheel_name }} path: ./wheelhouse/*.whl compression-level: 0 @@ -129,7 +153,7 @@ jobs: matrix: include: - os: "macos-12" - triplet: "x64-osx-dynamic" + triplet: "x64-osx-dynamic-release" arch: x86_64 vcpkg_cache: "/Users/runner/.cache/vcpkg/archives" vcpkg_logs: "/usr/local/share/vcpkg/buildtrees/**/*.log" @@ -160,7 +184,7 @@ jobs: path: | ${{ matrix.vcpkg_cache }} # bump the last digit to avoid using previous build cache - key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.8.5-cache0 + key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache0 # MacOS build requires aclocal, which is part of automake, but appears # to be missing in default image @@ -176,9 +200,9 @@ jobs: cd $VCPKG_INSTALLATION_ROOT # on mac the clone is not clean, otherwise git pull fails git reset --hard - # pull latest version to ensure the required commit with GDAL 3.8 is available + # pull specific commit with desired GDAL version git pull - git checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git checkout 73794ce5f63fd138fab999a22959ca7c6305d93c - name: Install GDAL env: @@ -215,17 +239,28 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-20.04", "windows-latest", "macos-12", "macos-latest"] + os: + [ + "ubuntu-latest", + "ubuntu-20.04", + "windows-latest", + "macos-12", + "macos-latest", + ] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] include: + - os: "ubuntu-latest" + artifact: pyogrio-wheel-linux-manylinux2014_x86_64 + - os: "ubuntu-latest" + artifact: pyogrio-wheel-linux-manylinux_2_28_x86_64 - os: "ubuntu-20.04" - artifact: pyogrio-wheel-linux + artifact: pyogrio-wheel-linux-manylinux_2_28_x86_64 - os: "windows-latest" artifact: pyogrio-wheel-x64-windows-dynamic-release - os: "macos-12" - artifact: pyogrio-wheel-x64-osx-dynamic + artifact: pyogrio-wheel-x64-osx-dynamic-release - os: "macos-latest" - artifact: pyogrio-wheel-x64-osx-dynamic + artifact: pyogrio-wheel-arm64-osx-dynamic-release steps: - name: Checkout @@ -236,8 +271,27 @@ jobs: with: python-version: ${{ matrix.python-version }} allow-prereleases: true - cache: "pip" - cache-dependency-path: "ci/requirements-wheel-test.txt" + + - name: Create virtual environment (Linux / MacOS) + # install uv and use it to create a virtual environment, then add it to + # environment variables so that it is automatically activated and can be + # used for tests below + if: ${{ runner.os != 'Windows' }} + shell: bash + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + . $HOME/.cargo/env + uv venv .venv + echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV + echo "$PWD/.venv/bin" >> $GITHUB_PATH + + - name: Create virtual environment (Windows) + if: ${{ runner.os == 'Windows' }} + run: | + irm https://astral.sh/uv/install.ps1 | iex + uv venv .venv + "VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append + "$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append - name: Download wheels from artifacts uses: actions/download-artifact@v4 @@ -248,19 +302,20 @@ jobs: - name: Install dependencies and pyogrio wheel shell: bash run: | - python -m pip install -r ci/requirements-wheel-test.txt + uv pip install -r ci/requirements-wheel-test.txt + uv pip install --no-cache --pre --no-index --find-links wheelhouse pyogrio if [ ${{ matrix.python-version }} != "3.12" ]; then - python -m pip install --no-deps geopandas + uv pip install --no-deps geopandas fi - python -m pip install --pre --find-links wheelhouse pyogrio - python -m pip list + uv pip list - name: Run tests shell: bash + # virtual environment is automatically activated run: | cd .. - python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" - python -m pytest --pyargs pyogrio.tests -v + uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')" + uv run python -m pytest --pyargs pyogrio.tests -v publish: name: Publish pyogrio to GitHub / PyPI diff --git a/CHANGES.md b/CHANGES.md index 626f3eb2..242cf518 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,13 @@ # CHANGELOG -## 0.9.1 (yyyy-mm-dd) +## 0.10.0 (yyyy-mm-dd) + +### Bug fixes + +- Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435). +- Enable mask & bbox filter when geometry column not read (#431). +- Raise NotImplmentedError when user attempts to write to an open file handle (#442). +- Prevent seek on read from compressed inputs (#443). ### Packaging @@ -10,13 +17,7 @@ packages if needed: more info [here](https://gdal.org/download.html#conda). This also leads to `pyproj` becoming an optional dependency; you will need to install `pyproj` in order to support spatial reference systems (#452). - -### Bug fixes - -- Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435). -- Enable mask & bbox filter when geometry column not read (#431). -- Raise NotImplmentedError when user attempts to write to an open file handle (#442). -- Prevent seek on read from compressed inputs (#443). +- The GDAL library included in the wheels is updated from 3.8.5 to GDAL 3.9.2 (#466). ## 0.9.0 (2024-06-17) diff --git a/ci/custom-triplets/x64-linux-dynamic.cmake b/ci/custom-triplets/x64-linux-dynamic-release.cmake similarity index 100% rename from ci/custom-triplets/x64-linux-dynamic.cmake rename to ci/custom-triplets/x64-linux-dynamic-release.cmake diff --git a/ci/custom-triplets/x64-osx-dynamic.cmake b/ci/custom-triplets/x64-osx-dynamic-release.cmake similarity index 100% rename from ci/custom-triplets/x64-osx-dynamic.cmake rename to ci/custom-triplets/x64-osx-dynamic-release.cmake diff --git a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile index 99485463..bb1a7843 100644 --- a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile +++ b/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile @@ -3,16 +3,16 @@ FROM quay.io/pypa/manylinux2014_x86_64:2024-08-12-7fde9b1 # building openssl needs IPC-Cmd (https://github.com/microsoft/vcpkg/issues/24988) RUN yum install -y curl unzip zip tar perl-IPC-Cmd -# require python >= 3.7 (python 3.6 is default on base image) for meson +# require python >= 3.7 (python 3.6 is default on base image) for meson RUN ln -s /opt/python/cp38-cp38/bin/python3 /usr/bin/python3 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \ - git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg" ENV PATH="${PATH}:/opt/vcpkg" -ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic" +ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic-release" # mkdir & touch -> workaround for https://github.com/microsoft/vcpkg/issues/27786 RUN bootstrap-vcpkg.sh && \ @@ -21,11 +21,11 @@ RUN bootstrap-vcpkg.sh && \ vcpkg integrate install && \ vcpkg integrate bash -COPY ci/custom-triplets/x64-linux-dynamic.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic.cmake +COPY ci/custom-triplets/x64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic-release.cmake COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/ COPY ci/vcpkg.json opt/vcpkg/ -ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic/lib" +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic-release/lib" RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \ --overlay-ports=opt/vcpkg/custom-ports \ --feature-flags="versions,manifests" \ diff --git a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile index 568143e4..174b37a1 100644 --- a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile +++ b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile @@ -4,7 +4,7 @@ FROM quay.io/pypa/manylinux_2_28_aarch64:2024-08-12-7fde9b1 RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \ - git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049 + git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg" ENV PATH="${PATH}:/opt/vcpkg" @@ -24,7 +24,6 @@ RUN bootstrap-vcpkg.sh && \ vcpkg integrate bash COPY ci/custom-triplets/arm64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/arm64-linux-dynamic-release.cmake -COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/ COPY ci/vcpkg.json opt/vcpkg/ # temporary workaround for https://github.com/microsoft/vcpkg/issues/36094 @@ -33,7 +32,6 @@ RUN git -C /opt/vcpkg apply vcpkg_linux_crosscompiling.patch ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/arm64-linux-dynamic-release/lib" RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \ - --overlay-ports=opt/vcpkg/custom-ports \ --feature-flags="versions,manifests" \ --x-manifest-root=opt/vcpkg \ --x-install-root=opt/vcpkg/installed && \ diff --git a/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile new file mode 100644 index 00000000..6a6ef05c --- /dev/null +++ b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile @@ -0,0 +1,33 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64:2024-08-12-7fde9b1 + +# building openssl needs IPC-Cmd (https://github.com/microsoft/vcpkg/issues/24988) +RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd + +RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \ + git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c + +ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg" +ENV PATH="${PATH}:/opt/vcpkg" + +ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic-release" + +# mkdir & touch -> workaround for https://github.com/microsoft/vcpkg/issues/27786 +RUN bootstrap-vcpkg.sh && \ + mkdir -p /root/.vcpkg/ $HOME/.vcpkg && \ + touch /root/.vcpkg/vcpkg.path.txt $HOME/.vcpkg/vcpkg.path.txt && \ + vcpkg integrate install && \ + vcpkg integrate bash + +COPY ci/custom-triplets/x64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic-release.cmake +COPY ci/vcpkg.json opt/vcpkg/ + +ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic-release/lib" +RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \ + --feature-flags="versions,manifests" \ + --x-manifest-root=opt/vcpkg \ + --x-install-root=opt/vcpkg/installed && \ + vcpkg list + +# setting git safe directory is required for properly building wheels when +# git >= 2.35.3 +RUN git config --global --add safe.directory "*" diff --git a/ci/vcpkg.json b/ci/vcpkg.json index 95ef73bf..ea331f18 100644 --- a/ci/vcpkg.json +++ b/ci/vcpkg.json @@ -1,6 +1,6 @@ { "name": "pyogrio", - "version": "0.8.0", + "version": "0.10.0", "dependencies": [ { "name": "gdal", @@ -8,5 +8,5 @@ "features": ["recommended-features", "curl", "geos", "iconv"] } ], - "builtin-baseline": "4f4a1821b2e8c7a2863e4df65a4d514f84144049" + "builtin-baseline": "73794ce5f63fd138fab999a22959ca7c6305d93c" } diff --git a/pyproject.toml b/pyproject.toml index 3ed6f129..53084902 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ build-verbosity = 3 VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/$VCPKG_DEFAULT_TRIPLET" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj" @@ -33,7 +33,7 @@ repair-wheel-command = [ VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/$VCPKG_DEFAULT_TRIPLET" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj" @@ -47,7 +47,7 @@ repair-wheel-command = "delvewheel repair --add-path C:/vcpkg/installed/x64-wind VCPKG_INSTALL = "$VCPKG_INSTALLATION_ROOT/installed/x64-windows-dynamic-release" GDAL_INCLUDE_PATH = "$VCPKG_INSTALL/include" GDAL_LIBRARY_PATH = "$VCPKG_INSTALL/lib" -GDAL_VERSION = "3.8.5" +GDAL_VERSION = "3.9.2" PYOGRIO_PACKAGE_DATA = 1 GDAL_DATA = "$VCPKG_INSTALL/share/gdal" PROJ_LIB = "$VCPKG_INSTALL/share/proj" @@ -144,7 +144,7 @@ ignore = [ # collection-literal-concatenation "RUF005", # Mutable class attributes should be annotated with `typing.ClassVar`, - "RUF012" + "RUF012", ] [tool.ruff.lint.per-file-ignores] @@ -156,15 +156,21 @@ combine-as-imports = true extra-standard-library = ["packaging"] section-order = [ - "future", - "standard-library", - "third-party", - "geo", - "first-party", - "local-folder", - "testing" + "future", + "standard-library", + "third-party", + "geo", + "first-party", + "local-folder", + "testing", ] [tool.ruff.lint.isort.sections] "geo" = ["shapely", "pyproj"] -"testing" = ["pytest", "pandas.testing", "numpy.testing", "geopandas.tests", "geopandas.testing"] +"testing" = [ + "pytest", + "pandas.testing", + "numpy.testing", + "geopandas.tests", + "geopandas.testing", +]