From aaacdda8dfc0f5c06a1ac8d036810eff6957fe49 Mon Sep 17 00:00:00 2001 From: Sean Gillies Date: Thu, 12 Oct 2023 13:22:49 -0600 Subject: [PATCH] Updates (#40) * Add Python 3.12 and update curl to 8.4.0 Resolves #39 Resolves #38 * Install test dependencies like pytz * Add pytz * Remove conditional install of shapely * Update change log --------- Co-authored-by: Sean Gillies --- .cirrus.yml | 15 +++++++++------ .github/workflows/build.sh | 3 ++- .github/workflows/wheels.yaml | 22 ++++++++++----------- .github/workflows/win-wheels.yaml | 14 ++++++++------ CHANGES.md | 11 +++++++++++ Fiona | 2 +- config.sh | 32 +++++++++++++++---------------- env_vars.sh | 8 ++++---- multibuild | 2 +- test_fiona_issue383.py | 9 +++++---- vcpkg.json | 4 ++-- 11 files changed, 69 insertions(+), 53 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 549586b..84235bb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,26 +5,29 @@ wheel_build_task: env: BUILD_PREFIX: /tmp/local REPO_DIR: Fiona - BUILD_COMMIT: 1.9.4 + BUILD_COMMIT: 1.9.5 CIRRUS_CLONE_SUBMODULES: true PATH: /opt/homebrew/opt/python@3.10/bin:$PATH PLAT: "arm64" - BUILD_DEPENDS: "oldest-supported-numpy cython>=0.29 wheel==0.34.2" - TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz" + BUILD_DEPENDS: "setuptools oldest-supported-numpy cython~=3.0.2 wheel" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" MACOSX_DEPLOYMENT_TARGET: "11.0" matrix: + - MB_PYTHON_VERSION: "3.12" - MB_PYTHON_VERSION: "3.11" - MB_PYTHON_VERSION: "3.10" - MB_PYTHON_VERSION: "3.9" - MB_PYTHON_VERSION: "3.8" install_pre_requirements_script: - - brew install python@3.10 + - brew install pkg-config python@3.10 - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python brew_remove_pkgs_script: - - brew remove -f --ignore-dependencies webp zstd xz libtiff libxcb curl php lcms2 xmlto ghostscript + - brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg build_wheel_script: - mkdir -p $BUILD_PREFIX/include - - mkdir -p $BUILD_PREFIX/lib + - mkdir -p $BUILD_PREFIX/lib/pkgconfig + - mkdir -p $BUILD_PREFIX/bin + - mkdir -p $BUILD_PREFIX/share/doc - mkdir -p $BUILD_PREFIX/share/man/man{1,3,5,7} - source multibuild/common_utils.sh - source multibuild/travis_steps.sh diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh index d3ece68..4d11638 100755 --- a/.github/workflows/build.sh +++ b/.github/workflows/build.sh @@ -3,7 +3,8 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # webp, zstd, xz, libtiff cause a conflict with building webp and libtiff # curl from brew requires zstd, use system curl # if php is installed, brew tries to reinstall these after installing openblas - brew remove --ignore-dependencies webp zstd xz libtiff curl php + brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg + brew install pkg-config fi if [[ "$MB_PYTHON_VERSION" == pypy3* ]]; then diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml index ff46386..80c6c35 100644 --- a/.github/workflows/wheels.yaml +++ b/.github/workflows/wheels.yaml @@ -6,7 +6,6 @@ on: [push, pull_request, workflow_dispatch] env: REPO_DIR: Fiona WHEEL_SDIR: wheelhouse - SETUPTOOLS_USE_DISTUTILS: stdlib jobs: build: @@ -15,37 +14,36 @@ jobs: strategy: fail-fast: false matrix: - os: [ "ubuntu-20.04", "macos-11" ] - python: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + os: [ "ubuntu-20.04", "macos-12" ] + python: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ] platform: [ "x86_64" ] macos-target: [ "10.10" ] exclude: - - os: "macos-11" + - os: "macos-12" platform: "i686" include: - - os: "macos-11" + - os: "macos-12" os-name: "osx" - os: "ubuntu-20.04" os-name: "focal" env: - BUILD_COMMIT: 1.9.4 + BUILD_COMMIT: 1.9.5 PLAT: "${{ matrix.platform }}" MB_PYTHON_VERSION: "${{ matrix.python }}" MB_ML_VER: "2014" - BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.32 wheel==0.34.2" - TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz" + TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz" TRAVIS_OS_NAME: ${{ matrix.os-name }} MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.x" - name: Build Wheel run: .github/workflows/build.sh - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 with: name: wheels path: wheelhouse/*.whl diff --git a/.github/workflows/win-wheels.yaml b/.github/workflows/win-wheels.yaml index 90a367f..e91f725 100644 --- a/.github/workflows/win-wheels.yaml +++ b/.github/workflows/win-wheels.yaml @@ -5,7 +5,7 @@ name: Windows wheels on: [push, pull_request, workflow_dispatch] env: - REF: 1.9.4 + REF: 1.9.5 # cancel running jobs on new commit to PR concurrency: @@ -80,7 +80,7 @@ jobs: shell: bash - name: Build wheels - uses: pypa/cibuildwheel@v2.10.2 + uses: pypa/cibuildwheel@v2.16.0 env: CIBW_SKIP: "*-win32 pp*" CIBW_ENVIRONMENT_WINDOWS: @@ -90,7 +90,7 @@ jobs: PACKAGE_DATA=1 GDAL_DATA="$VCPKG_INSTALL/share/gdal" PROJ_LIB="$VCPKG_INSTALL/share/proj/data" - GDAL_VERSION="3.5.2" + GDAL_VERSION="3.6.4" CIBW_BEFORE_BUILD_WINDOWS: pip install delvewheel CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin -w {dest_dir} {wheel} @@ -106,7 +106,7 @@ jobs: fail-fast: false matrix: os: ["windows-latest"] - python-version: ["3.8", "3.9", "3.10", "3.11-dev"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -118,9 +118,10 @@ jobs: path: ./Fiona - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Download wheels from artifacts uses: actions/download-artifact@v2 @@ -129,7 +130,8 @@ jobs: - name: Install dependencies and Fiona wheel shell: bash run: | - python -m pip install shapely oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz + python -m pip install oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz + python -m pip install "shapely ; python_version < '3.12'" python -m pip install --pre --find-links wheelhouse/artifact fiona python -m pip list diff --git a/CHANGES.md b/CHANGES.md index 178e7ce..bda6e98 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,17 @@ Changes ======= +## 2023-10-12 + +Library version changes: + +* Update curl to 8.4.0 to address CVE-2023-38545 and CVE-2023-38546 (#112). +* Update vcpkg to commit b40de44891dc1cab11d4722094ae44807a837b98 (#112). + +Python version changes: + +Python 3.12 has been added. + ## 2023-05-16 Library version changes: diff --git a/Fiona b/Fiona index d36e0c8..ca2fe82 160000 --- a/Fiona +++ b/Fiona @@ -1 +1 @@ -Subproject commit d36e0c897c545e0e51fe759e540c85c117bf3fc1 +Subproject commit ca2fe824239612ac4fa15bbf14980ff28169ec70 diff --git a/config.sh b/config.sh index a4f5f37..b6b9cd9 100644 --- a/config.sh +++ b/config.sh @@ -164,7 +164,7 @@ function build_openssl { fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} (cd ${OPENSSL_ROOT} \ - && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \ + && ./config no-ssl2 -fPIC --prefix=$BUILD_PREFIX \ && make -j4 \ && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi) touch openssl-stamp @@ -175,8 +175,8 @@ function build_curl { if [ -e curl-stamp ]; then return; fi CFLAGS="$CFLAGS -g -O2" CXXFLAGS="$CXXFLAGS -g -O2" - build_nghttp2 build_openssl + build_nghttp2 local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl" # fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz (cd curl-${CURL_VERSION} \ @@ -268,13 +268,13 @@ function pre_build { #fi local cmake=$(get_modern_cmake) + suppress build_openssl suppress build_nghttp2 if [ -n "$IS_OSX" ]; then rm /usr/local/lib/libpng* || true fi - suppress build_openssl fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz @@ -314,43 +314,43 @@ function run_tests { apt-get install -y ca-certificates fi cp -R ../Fiona/tests ./tests + python -m pip install "shapely;python_version<'3.12'" $TEST_DEPENDS GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]" fio --version fio env --formats - if [[ $MB_PYTHON_VERSION != "3.10" ]]; then - pip install shapely && python ../test_fiona_issue383.py - fi + python ../test_fiona_issue383.py } function build_wheel_cmd { - # Update the container's auditwheel with our patched version. - if [ -n "$IS_OSX" ]; then - : - else # manylinux - /opt/python/cp37-cp37m/bin/pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel" - fi - - local cmd=${1:-pip_wheel_cmd} + local cmd=${1:-build_cmd} local repo_dir=${2:-$REPO_DIR} [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) start_spinner if [ -n "$(is_function "pre_build")" ]; then pre_build; fi stop_spinner + pip install -U pip + pip install -U build if [ -n "$BUILD_DEPENDS" ]; then pip install $(pip_opts) $BUILD_DEPENDS fi - (cd $repo_dir && GDAL_VERSION=3.5.3 $cmd $wheelhouse) + (cd $repo_dir && GDAL_VERSION=3.6.4 $cmd $wheelhouse) if [ -n "$IS_OSX" ]; then : else # manylinux - /opt/python/cp37-cp37m/bin/pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel" + pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel" fi repair_wheelhouse $wheelhouse } +function build_cmd { + local abs_wheelhouse=$1 + python -m build -o $abs_wheelhouse +} + + function macos_arm64_native_build_setup { # Setup native build for single arch arm_64 wheels export PLAT="arm64" diff --git a/env_vars.sh b/env_vars.sh index 57354f0..6509a27 100644 --- a/env_vars.sh +++ b/env_vars.sh @@ -6,16 +6,16 @@ JSONC_VERSION=0.15 SQLITE_VERSION=3330000 PROJ_VERSION=9.0.1 GDAL_VERSION=3.6.4 -CURL_VERSION=7.80.0 +CURL_VERSION=8.4.0 NGHTTP2_VERSION=1.46.0 EXPAT_VERSION=2.2.6 TIFF_VERSION=4.3.0 OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/ -OPENSSL_ROOT=openssl-1.1.1l -OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 +OPENSSL_ROOT=openssl-1.1.1w +OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 export MACOSX_DEPLOYMENT_TARGET=10.15 export GDAL_CONFIG=/usr/local/bin/gdal-config export PACKAGE_DATA=1 export PROJ_LIB=/usr/local/share/proj export AUDITWHEEL_EXTRA_LIB_NAME_TAG=fiona -export SETUPTOOLS_USE_DISTUTILS=stdlib +export TEST_DEPENDS="oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz" diff --git a/multibuild b/multibuild index bb32cfe..509e63a 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit bb32cfec4f755cb146332a0490abcf3187ce61d1 +Subproject commit 509e63a6d8ab4705a500264d396b0e675798b6fc diff --git a/test_fiona_issue383.py b/test_fiona_issue383.py index 81f6718..6bbfc0c 100644 --- a/test_fiona_issue383.py +++ b/test_fiona_issue383.py @@ -407,7 +407,8 @@ }, } - -from shapely.geometry import shape - -print(shape(d["geometry"])) +try: + from shapely.geometry import shape + print(shape(d["geometry"])) +except ImportError: + print("Shapely not installed.") diff --git a/vcpkg.json b/vcpkg.json index 93e35b2..226f24f 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,8 +1,8 @@ { "name": "fiona", - "version": "1.9.4", + "version": "1.9.5", "dependencies": [ "gdal" ], - "builtin-baseline": "8fdb0f2e58ad31d02b314ed45e6a53b19302472d", + "builtin-baseline": "b40de44891dc1cab11d4722094ae44807a837b98", "overrides": [ { "name": "gdal", "version": "3.6.4" }, { "name": "proj", "version": "9.0.1" }