Updates (#40)

* Add Python 3.12 and update curl to 8.4.0 Resolves #39 Resolves #38 * Install test dependencies like pytz * Add pytz * Remove conditional install of shapely * Update change log --------- Co-authored-by: Sean Gillies <seangillies@Seans-MacBook-Air.local>
sgillies · Oct 12, 2023 · aaacdda · aaacdda
1 parent b4cbf1e
commit aaacdda
Show file tree

Hide file tree

Showing 11 changed files with 69 additions and 53 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -5,26 +5,29 @@ wheel_build_task:
   env:
     BUILD_PREFIX: /tmp/local
     REPO_DIR: Fiona
-    BUILD_COMMIT: 1.9.4
+    BUILD_COMMIT: 1.9.5
     CIRRUS_CLONE_SUBMODULES: true
     PATH: /opt/homebrew/opt/python@3.10/bin:$PATH
     PLAT: "arm64"
-    BUILD_DEPENDS: "oldest-supported-numpy cython>=0.29 wheel==0.34.2"
-    TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz"
+    BUILD_DEPENDS: "setuptools oldest-supported-numpy cython~=3.0.2 wheel"
+    TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz"
     MACOSX_DEPLOYMENT_TARGET: "11.0"
     matrix:
+      - MB_PYTHON_VERSION: "3.12"
       - MB_PYTHON_VERSION: "3.11"
       - MB_PYTHON_VERSION: "3.10"
       - MB_PYTHON_VERSION: "3.9"
       - MB_PYTHON_VERSION: "3.8"
   install_pre_requirements_script:
-    - brew install python@3.10
+    - brew install pkg-config python@3.10
     - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python
   brew_remove_pkgs_script:
-    - brew remove -f --ignore-dependencies webp zstd xz libtiff libxcb curl php lcms2 xmlto ghostscript
+    - brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg
   build_wheel_script:
     - mkdir -p $BUILD_PREFIX/include
-    - mkdir -p $BUILD_PREFIX/lib
+    - mkdir -p $BUILD_PREFIX/lib/pkgconfig
+    - mkdir -p $BUILD_PREFIX/bin
+    - mkdir -p $BUILD_PREFIX/share/doc
     - mkdir -p $BUILD_PREFIX/share/man/man{1,3,5,7}
     - source multibuild/common_utils.sh
     - source multibuild/travis_steps.sh

diff --git a/.github/workflows/build.sh b/.github/workflows/build.sh
@@ -3,7 +3,8 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
   # webp, zstd, xz, libtiff cause a conflict with building webp and libtiff
   # curl from brew requires zstd, use system curl
   # if php is installed, brew tries to reinstall these after installing openblas
-  brew remove --ignore-dependencies webp zstd xz libtiff curl php
+  brew remove -f --ignore-dependencies iliblerc webp zstd libtiff libxcb libxdcmp curl php lcms2 xmlto ghostscript lz4 openjpeg
+  brew install pkg-config
 fi
 
 if [[ "$MB_PYTHON_VERSION" == pypy3* ]]; then

diff --git a/.github/workflows/wheels.yaml b/.github/workflows/wheels.yaml
@@ -6,7 +6,6 @@ on: [push, pull_request, workflow_dispatch]
 env:
   REPO_DIR: Fiona
   WHEEL_SDIR: wheelhouse
-  SETUPTOOLS_USE_DISTUTILS: stdlib
 
 jobs:
   build:
@@ -15,37 +14,36 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ "ubuntu-20.04", "macos-11" ]
-        python: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+        os: [ "ubuntu-20.04", "macos-12" ]
+        python: [ "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ]
         platform: [ "x86_64" ]
         macos-target: [ "10.10" ]
         exclude:
-          - os: "macos-11"
+          - os: "macos-12"
             platform: "i686"
         include:
-          - os: "macos-11"
+          - os: "macos-12"
             os-name: "osx"
           - os: "ubuntu-20.04"
             os-name: "focal"
     env:
-      BUILD_COMMIT: 1.9.4
+      BUILD_COMMIT: 1.9.5
       PLAT: "${{ matrix.platform }}"
       MB_PYTHON_VERSION: "${{ matrix.python }}"
       MB_ML_VER: "2014"
-      BUILD_DEPENDS: "oldest-supported-numpy cython==0.29.32 wheel==0.34.2"
-      TEST_DEPENDS: "oldest-supported-numpy attrs==21.4.0 pytest==6.2.5 click==8.0.3 mock boto3 packaging hypothesis==4.38.1 wheel==0.34.2 pytz"
+      TEST_DEPENDS: "oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis wheel pytz"
       TRAVIS_OS_NAME: ${{ matrix.os-name }}
       MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-target }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: "3.x"
       - name: Build Wheel
         run: .github/workflows/build.sh
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: wheels
           path: wheelhouse/*.whl

diff --git a/.github/workflows/win-wheels.yaml b/.github/workflows/win-wheels.yaml
@@ -5,7 +5,7 @@ name: Windows wheels
 on: [push, pull_request, workflow_dispatch]
 
 env:
-  REF: 1.9.4
+  REF: 1.9.5
 
 # cancel running jobs on new commit to PR
 concurrency:
@@ -80,7 +80,7 @@ jobs:
         shell: bash
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.10.2
+        uses: pypa/cibuildwheel@v2.16.0
         env:
           CIBW_SKIP: "*-win32 pp*"
           CIBW_ENVIRONMENT_WINDOWS:
@@ -90,7 +90,7 @@ jobs:
             PACKAGE_DATA=1
             GDAL_DATA="$VCPKG_INSTALL/share/gdal"
             PROJ_LIB="$VCPKG_INSTALL/share/proj/data"
-            GDAL_VERSION="3.5.2"
+            GDAL_VERSION="3.6.4"
           CIBW_BEFORE_BUILD_WINDOWS: pip install delvewheel
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: delvewheel repair --add-path C:/vcpkg/installed/x64-windows/bin -w {dest_dir} {wheel}
 
@@ -106,7 +106,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["windows-latest"]
-        python-version: ["3.8", "3.9", "3.10", "3.11-dev"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3
@@ -118,9 +118,10 @@ jobs:
           path: ./Fiona
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
 
       - name: Download wheels from artifacts
         uses: actions/download-artifact@v2
@@ -129,7 +130,8 @@ jobs:
       - name: Install dependencies and Fiona wheel
         shell: bash
         run: |
-          python -m pip install shapely oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz
+          python -m pip install oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz
+          python -m pip install "shapely ; python_version < '3.12'"
           python -m pip install --pre --find-links wheelhouse/artifact fiona
           python -m pip list
 

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,6 +1,17 @@
 Changes
 =======
 
+## 2023-10-12
+
+Library version changes:
+
+* Update curl to 8.4.0 to address CVE-2023-38545 and CVE-2023-38546 (#112).
+* Update vcpkg to commit b40de44891dc1cab11d4722094ae44807a837b98 (#112).
+
+Python version changes:
+
+Python 3.12 has been added.
+
 ## 2023-05-16
 
 Library version changes:

diff --git a/Fiona b/Fiona
diff --git a/config.sh b/config.sh
@@ -164,7 +164,7 @@ function build_openssl {
     fetch_unpack ${OPENSSL_DOWNLOAD_URL}/${OPENSSL_ROOT}.tar.gz
     check_sha256sum $ARCHIVE_SDIR/${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH}
     (cd ${OPENSSL_ROOT} \
-        && ./config no-ssl2 no-shared -fPIC --prefix=$BUILD_PREFIX \
+        && ./config no-ssl2 -fPIC --prefix=$BUILD_PREFIX \
         && make -j4 \
         && if [ -n "$IS_OSX" ]; then sudo make install; else make install; fi)
     touch openssl-stamp
@@ -175,8 +175,8 @@ function build_curl {
     if [ -e curl-stamp ]; then return; fi
     CFLAGS="$CFLAGS -g -O2"
     CXXFLAGS="$CXXFLAGS -g -O2"
-    build_nghttp2
     build_openssl
+    build_nghttp2
     local flags="--prefix=$BUILD_PREFIX --with-nghttp2=$BUILD_PREFIX --with-libz --with-ssl"
     #    fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
     (cd curl-${CURL_VERSION} \
@@ -268,13 +268,13 @@ function pre_build {
     #fi
 
     local cmake=$(get_modern_cmake)
+    suppress build_openssl
     suppress build_nghttp2
 
     if [ -n "$IS_OSX" ]; then
         rm /usr/local/lib/libpng* || true
     fi
 
-    suppress build_openssl
 
     fetch_unpack https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz
 
@@ -314,43 +314,43 @@ function run_tests {
         apt-get install -y ca-certificates
     fi
     cp -R ../Fiona/tests ./tests
+    python -m pip install "shapely;python_version<'3.12'" $TEST_DEPENDS
     GDAL_ENABLE_DEPRECATED_DRIVER_GTM=YES python -m pytest -vv tests -k "not test_collection_zip_http and not test_mask_polygon_triangle and not test_show_versions and not test_append_or_driver_error and not [PCIDSK] and not cannot_append[FlatGeobuf]"
     fio --version
     fio env --formats
-    if [[ $MB_PYTHON_VERSION != "3.10" ]]; then
-        pip install shapely && python ../test_fiona_issue383.py
-    fi
+    python ../test_fiona_issue383.py
 }
 
 
 function build_wheel_cmd {
-    # Update the container's auditwheel with our patched version.
-    if [ -n "$IS_OSX" ]; then
-	:
-    else  # manylinux
-        /opt/python/cp37-cp37m/bin/pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel"
-    fi
-
-    local cmd=${1:-pip_wheel_cmd}
+    local cmd=${1:-build_cmd}
     local repo_dir=${2:-$REPO_DIR}
     [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
     local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
     start_spinner
     if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
     stop_spinner
+    pip install -U pip
+    pip install -U build
     if [ -n "$BUILD_DEPENDS" ]; then
         pip install $(pip_opts) $BUILD_DEPENDS
     fi
-    (cd $repo_dir && GDAL_VERSION=3.5.3 $cmd $wheelhouse)
+    (cd $repo_dir && GDAL_VERSION=3.6.4 $cmd $wheelhouse)
     if [ -n "$IS_OSX" ]; then
 	:
     else  # manylinux
-        /opt/python/cp37-cp37m/bin/pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel"
+        pip install -I "git+https://github.com/sgillies/auditwheel.git#egg=auditwheel"
     fi
     repair_wheelhouse $wheelhouse
 }
 
 
+function build_cmd {
+    local abs_wheelhouse=$1
+    python -m build -o $abs_wheelhouse
+}
+
+
 function macos_arm64_native_build_setup {
     # Setup native build for single arch arm_64 wheels
     export PLAT="arm64"

diff --git a/env_vars.sh b/env_vars.sh
@@ -6,16 +6,16 @@ JSONC_VERSION=0.15
 SQLITE_VERSION=3330000
 PROJ_VERSION=9.0.1
 GDAL_VERSION=3.6.4
-CURL_VERSION=7.80.0
+CURL_VERSION=8.4.0
 NGHTTP2_VERSION=1.46.0
 EXPAT_VERSION=2.2.6
 TIFF_VERSION=4.3.0
 OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source/
-OPENSSL_ROOT=openssl-1.1.1l
-OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
+OPENSSL_ROOT=openssl-1.1.1w
+OPENSSL_HASH=cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8
 export MACOSX_DEPLOYMENT_TARGET=10.15
 export GDAL_CONFIG=/usr/local/bin/gdal-config
 export PACKAGE_DATA=1
 export PROJ_LIB=/usr/local/share/proj
 export AUDITWHEEL_EXTRA_LIB_NAME_TAG=fiona
-export SETUPTOOLS_USE_DISTUTILS=stdlib
+export TEST_DEPENDS="oldest-supported-numpy attrs pytest click mock boto3 packaging hypothesis pytz"
diff --git a/multibuild b/multibuild
diff --git a/test_fiona_issue383.py b/test_fiona_issue383.py
@@ -407,7 +407,8 @@
     },
 }
 
-
-from shapely.geometry import shape
-
-print(shape(d["geometry"]))
+try:
+    from shapely.geometry import shape
+    print(shape(d["geometry"]))
+except ImportError:
+    print("Shapely not installed.")
diff --git a/vcpkg.json b/vcpkg.json
@@ -1,8 +1,8 @@
 {
     "name": "fiona",
-    "version": "1.9.4",
+    "version": "1.9.5",
     "dependencies": [ "gdal" ],
-    "builtin-baseline": "8fdb0f2e58ad31d02b314ed45e6a53b19302472d",
+    "builtin-baseline": "b40de44891dc1cab11d4722094ae44807a837b98",
     "overrides": [
         { "name": "gdal", "version": "3.6.4" },
         { "name": "proj", "version": "9.0.1" }
+2 −2		.github/workflows/lint_python.yml
+1 −1		.github/workflows/test.yml
+1 −1		LICENSE
+2 −1		README.rst
+14 −10		common_utils.sh
+1 −1		library_builders.sh
+12 −9		osx_utils.sh
+2 −2		tests/test_common_utils.sh
+3 −1		tests/test_fill_pyver.sh
+6 −2		travis_linux_steps.sh