Merge branch 'main' into zlib-ng

.github/renovate.json

@@ -1,17 +1,21 @@
 {
     "$schema": "https://docs.renovatebot.com/renovate-schema.json",
     "extends": [
-        "config:base"
+        "config:recommended"
     ],
     "labels": [
         "Dependency"
     ],
     "packageRules": [
         {
             "groupName": "github-actions",
-            "matchManagers": ["github-actions"],
-            "separateMajorMinor": "false"
+            "matchManagers": [
+                "github-actions"
+            ],
+            "separateMajorMinor": false
         }
     ],
-    "schedule": ["on the 3rd day of the month"]
+    "schedule": [
+        "on the 3rd day of the month"
+    ]
 }

.github/workflows/docs.yml

@@ -33,6 +33,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Set up Python
       uses: actions/setup-python@v5

.github/workflows/lint.yml

@@ -21,6 +21,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: pre-commit cache
       uses: actions/cache@v4

.github/workflows/stale.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 permissions:
-  issues: write
+  contents: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -15,6 +15,8 @@ concurrency:
 jobs:
   stale:
     if: github.repository_owner == 'python-pillow'
+    permissions:
+      issues: write
 
     runs-on: ubuntu-latest
 

.github/workflows/test-cygwin.yml

@@ -48,6 +48,8 @@ jobs:
 
       - name: Checkout Pillow
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Install Cygwin
         uses: cygwin/cygwin-install-action@v4

.github/workflows/test-docker.yml

@@ -47,6 +47,7 @@ jobs:
           debian-12-bookworm-x86,
           debian-12-bookworm-amd64,
           fedora-40-amd64,
+          fedora-41-amd64,
           gentoo,
           ubuntu-22.04-jammy-amd64,
           ubuntu-24.04-noble-amd64,
@@ -64,6 +65,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Build system information
       run: python3 .github/workflows/system-info.py
@@ -101,7 +104,6 @@ jobs:
       with:
         flags: GHA_Docker
         name: ${{ matrix.docker }}
-        gcov: true
         token: ${{ secrets.CODECOV_ORG_TOKEN }}
 
   success:

.github/workflows/test-mingw.yml

@@ -46,6 +46,8 @@ jobs:
     steps:
       - name: Checkout Pillow
         uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up shell
         run: echo "C:\msys64\usr\bin\" >> $env:GITHUB_PATH

.github/workflows/test-valgrind.yml

@@ -40,6 +40,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Build system information
       run: python3 .github/workflows/system-info.py

.github/workflows/test-windows.yml

@@ -44,16 +44,20 @@ jobs:
     steps:
     - name: Checkout Pillow
       uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Checkout cached dependencies
       uses: actions/checkout@v4
       with:
+        persist-credentials: false
         repository: python-pillow/pillow-depends
         path: winbuild\depends
 
     - name: Checkout extra test images
       uses: actions/checkout@v4
       with:
+        persist-credentials: false
         repository: python-pillow/test-images
         path: Tests\test-images
 
@@ -69,16 +73,14 @@ jobs:
     - name: Print build system information
       run: python3 .github/workflows/system-info.py
 
-    - name: Install Python dependencies
-      run: >
-        python3 -m pip install
-        coverage>=7.4.2
-        defusedxml
-        olefile
-        pyroma
-        pytest
-        pytest-cov
-        pytest-timeout
+    - name: Upgrade pip
+      run: |
+        python3 -m pip install --upgrade pip
+
+    - name: Install CPython dependencies
+      if: "!contains(matrix.python-version, 'pypy')"
+      run: |
+        python3 -m pip install PyQt6
 
     - name: Install dependencies
       id: install
@@ -178,7 +180,7 @@ jobs:
     - name: Build Pillow
       run: |
         $FLAGS="-C raqm=vendor -C fribidi=vendor"
-        cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip install -v $FLAGS ."
+        cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip install -v $FLAGS .[tests]"
         & $env:pythonLocation\python.exe selftest.py --installed
       shell: pwsh
 

.github/workflows/test.yml

@@ -63,6 +63,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
@@ -114,7 +116,7 @@ jobs:
         GHA_PYTHON_VERSION: ${{ matrix.python-version }}
 
     - name: Register gcc problem matcher
-      if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'"
+      if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'"
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
 
     - name: Build
@@ -158,7 +160,6 @@ jobs:
       with:
         flags: ${{ matrix.os == 'ubuntu-latest' && 'GHA_Ubuntu' || 'GHA_macOS' }}
         name: ${{ matrix.os }} Python ${{ matrix.python-version }}
-        gcov: true
         token: ${{ secrets.CODECOV_ORG_TOKEN }}
 
   success:

.github/workflows/wheels-dependencies.sh

@@ -1,11 +1,33 @@
 #!/bin/bash
-# Define custom utilities
-# Test for macOS with [ -n "$IS_MACOS" ]
-if [ -z "$IS_MACOS" ]; then
-    export MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
-    export MB_ML_VER=${AUDITWHEEL_POLICY:9}
+
+# Setup that needs to be done before multibuild utils are invoked
+PROJECTDIR=$(pwd)
+if [[ "$(uname -s)" == "Darwin" ]]; then
+    # Safety check - macOS builds require that CIBW_ARCHS is set, and that it
+    # only contains a single value (even though cibuildwheel allows multiple
+    # values in CIBW_ARCHS).
+    if [[ -z "$CIBW_ARCHS" ]]; then
+        echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined."
+        exit 1
+    fi
+    if [[ "$CIBW_ARCHS" == *" "* ]]; then
+        echo "ERROR: Pillow macOS builds only support a single architecture in CIBW_ARCHS."
+        exit 1
+    fi
+
+    # Build macOS dependencies in `build/darwin`
+    # Install them into `build/deps/darwin`
+    WORKDIR=$(pwd)/build/darwin
+    BUILD_PREFIX=$(pwd)/build/deps/darwin
+else
+    # Build prefix will default to /usr/local
+    WORKDIR=$(pwd)/build
+    MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
+    MB_ML_VER=${AUDITWHEEL_POLICY:9}
 fi
-export PLAT=$CIBW_ARCHS
+PLAT=$CIBW_ARCHS
+
+# Define custom utilities
 source wheels/multibuild/common_utils.sh
 source wheels/multibuild/library_builders.sh
 if [ -z "$IS_MACOS" ]; then
@@ -38,35 +60,43 @@ BZIP2_VERSION=1.0.8
 LIBXCB_VERSION=1.17.0
 BROTLI_VERSION=1.1.0
 
+function build_pkg_config {
+    if [ -e pkg-config-stamp ]; then return; fi
+    # This essentially duplicates the Homebrew recipe
+    ORIGINAL_CFLAGS=$CFLAGS
+    CFLAGS="$CFLAGS -Wno-int-conversion"
+    build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \
+        --disable-debug --disable-host-tool --with-internal-glib \
+        --with-pc-path=$BUILD_PREFIX/share/pkgconfig:$BUILD_PREFIX/lib/pkgconfig \
+        --with-system-include-path=$(xcrun --show-sdk-path --sdk macosx)/usr/include
+    CFLAGS=$ORIGINAL_CFLAGS
+    export PKG_CONFIG=$BUILD_PREFIX/bin/pkg-config
+    touch pkg-config-stamp
+}
+
 function build_brotli {
+    if [ -e brotli-stamp ]; then return; fi
     local cmake=$(get_modern_cmake)
     local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz)
     (cd $out_dir \
-        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
+        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
         && make install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libbrotli* /usr/local/lib
-        cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig
-    fi
+    touch brotli-stamp
 }
 
 function build_harfbuzz {
+    if [ -e harfbuzz-stamp ]; then return; fi
     python3 -m pip install meson ninja
 
     local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz)
     (cd $out_dir \
-        && meson setup build --buildtype=release -Dfreetype=enabled -Dglib=disabled)
+        && meson setup build --prefix=$BUILD_PREFIX --libdir=$BUILD_PREFIX/lib --buildtype=release -Dfreetype=enabled -Dglib=disabled)
     (cd $out_dir/build \
         && meson install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libharfbuzz* /usr/local/lib
-    fi
+    touch harfbuzz-stamp
 }
 
 function build {
-    if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then
-        sudo chown -R runner /usr/local
-    fi
     build_xz
     if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then
         yum remove -y zlib-devel
@@ -78,22 +108,27 @@ function build {
         build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto
         build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib
         build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist
-        if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-            cp /usr/local/share/pkgconfig/xcb-proto.pc /usr/local/lib/pkgconfig
-        fi
     else
-        sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc
+        sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc
     fi
     build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib
 
     build_libjpeg_turbo
-    build_tiff
+    if [ -n "$IS_MACOS" ]; then
+        # Custom tiff build to include jpeg; by default, configure won't include
+        # headers/libs in the custom macOS prefix. Explicitly disable webp,
+        # libdeflate and zstd, because on x86_64 macs, it will pick up the
+        # Homebrew versions of those libraries from /usr/local.
+        build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \
+            --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \
+            --disable-webp --disable-libdeflate --disable-zstd
+    else
+        build_tiff
+    fi
+
     build_libpng
     build_lcms2
     build_openjpeg
-    if [ -f /usr/local/lib64/libopenjp2.so ]; then
-        cp /usr/local/lib64/libopenjp2.so /usr/local/lib
-    fi
 
     ORIGINAL_CFLAGS=$CFLAGS
     CFLAGS="$CFLAGS -O3 -DNDEBUG"
@@ -115,32 +150,47 @@ function build {
     build_harfbuzz
 }
 
+# Perform all dependency builds in the build subfolder.
+mkdir -p $WORKDIR
+pushd $WORKDIR > /dev/null
+
 # Any stuff that you need to do before you start building the wheels
 # Runs in the root directory of this repository.
-curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
-untar pillow-depends-main.zip
-
-if [[ -n "$IS_MACOS" ]]; then
-  # libdeflate may cause a minimum target error when repairing the wheel
-  # libtiff and libxcb cause a conflict with building libtiff and libxcb
-  # libxau and libxdmcp cause an issue on macOS < 11
-  # remove cairo to fix building harfbuzz on arm64
-  # remove lcms2 and libpng to fix building openjpeg on arm64
-  # remove jpeg-turbo to avoid inclusion on arm64
-  # remove webp and zstd to avoid inclusion on x86_64
-  # curl from brew requires zstd, use system curl
-  brew remove --ignore-dependencies libpng libtiff libxcb libxau libxdmcp curl cairo lcms2 zstd
-  if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-    brew remove --ignore-dependencies jpeg-turbo
-  else
-    brew remove --ignore-dependencies libdeflate webp
+if [[ ! -d $WORKDIR/pillow-depends-main ]]; then
+  if [[ ! -f $PROJECTDIR/pillow-depends-main.zip ]]; then
+    echo "Download pillow dependency sources..."
+    curl -fSL -o $PROJECTDIR/pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
   fi
+  echo "Unpacking pillow dependency sources..."
+  untar $PROJECTDIR/pillow-depends-main.zip
+fi
 
-  brew install pkg-config
+if [[ -n "$IS_MACOS" ]]; then
+    # Homebrew (or similar packaging environments) install can contain some of
+    # the libraries that we're going to build. However, they may be compiled
+    # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use,
+    # and they may bring in other dependencies that we don't want. The same will
+    # be true of any other locations on the path. To avoid conflicts, strip the
+    # path down to the bare minimum (which, on macOS, won't include any
+    # development dependencies).
+    export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+    export CMAKE_PREFIX_PATH=$BUILD_PREFIX
+
+    # Ensure the basic structure of the build prefix directory exists.
+    mkdir -p "$BUILD_PREFIX/bin"
+    mkdir -p "$BUILD_PREFIX/lib"
+
+    # Ensure pkg-config is available
+    build_pkg_config
+    # Ensure cmake is available
+    python3 -m pip install cmake
 fi
 
 wrap_wheel_builder build
 
+# Return to the project root to finish the build
+popd > /dev/null
+
 # Append licenses
 for filename in wheels/dependency_licenses/*; do
   echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE