diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh index 3a80a7e741d..f86099af16e 100755 --- a/.github/workflows/wheels-dependencies.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -1,11 +1,33 @@ #!/bin/bash -# Define custom utilities -# Test for macOS with [ -n "$IS_MACOS" ] -if [ -z "$IS_MACOS" ]; then - export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} - export MB_ML_VER=${AUDITWHEEL_POLICY:9} + +# Setup that needs to be done before multibuild utils are invoked +PROJECTDIR=$(pwd) +if [[ "$(uname -s)" == "Darwin" ]]; then + # Safety check - macOS builds require that CIBW_ARCHS is set, and that it + # only contains a single value (even though cibuildwheel allows multiple + # values in CIBW_ARCHS). + if [[ -z "$CIBW_ARCHS" ]]; then + echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined." + exit 1 + fi + if [[ "$CIBW_ARCHS" == *" "* ]]; then + echo "ERROR: Pillow macOS builds only support a single architecture in CIBW_ARCHS." + exit 1 + fi + + # Build macOS dependencies in `build/darwin` + # Install them into `build/deps/darwin` + WORKDIR=$(pwd)/build/darwin + BUILD_PREFIX=$(pwd)/build/deps/darwin +else + # Build prefix will default to /usr/local + WORKDIR=$(pwd)/build + MB_ML_LIBC=${AUDITWHEEL_POLICY::9} + MB_ML_VER=${AUDITWHEEL_POLICY:9} fi -export PLAT=$CIBW_ARCHS +PLAT=$CIBW_ARCHS + +# Define custom utilities source wheels/multibuild/common_utils.sh source wheels/multibuild/library_builders.sh if [ -z "$IS_MACOS" ]; then @@ -38,35 +60,43 @@ BZIP2_VERSION=1.0.8 LIBXCB_VERSION=1.17.0 BROTLI_VERSION=1.1.0 +function build_pkg_config { + if [ -e pkg-config-stamp ]; then return; fi + # This essentially duplicates the Homebrew recipe + ORIGINAL_CFLAGS=$CFLAGS + CFLAGS="$CFLAGS -Wno-int-conversion" + build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \ + --disable-debug --disable-host-tool --with-internal-glib \ + --with-pc-path=$BUILD_PREFIX/share/pkgconfig:$BUILD_PREFIX/lib/pkgconfig \ + --with-system-include-path=$(xcrun --show-sdk-path --sdk macosx)/usr/include + CFLAGS=$ORIGINAL_CFLAGS + export PKG_CONFIG=$BUILD_PREFIX/bin/pkg-config + touch pkg-config-stamp +} + function build_brotli { + if [ -e brotli-stamp ]; then return; fi local cmake=$(get_modern_cmake) local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz) (cd $out_dir \ - && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ + && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ && make install) - if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp /usr/local/lib64/libbrotli* /usr/local/lib - cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig - fi + touch brotli-stamp } function build_harfbuzz { + if [ -e harfbuzz-stamp ]; then return; fi python3 -m pip install meson ninja local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz) (cd $out_dir \ - && meson setup build --buildtype=release -Dfreetype=enabled -Dglib=disabled) + && meson setup build --prefix=$BUILD_PREFIX --libdir=$BUILD_PREFIX/lib --buildtype=release -Dfreetype=enabled -Dglib=disabled) (cd $out_dir/build \ && meson install) - if [[ "$MB_ML_LIBC" == "manylinux" ]]; then - cp /usr/local/lib64/libharfbuzz* /usr/local/lib - fi + touch harfbuzz-stamp } function build { - if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then - sudo chown -R runner /usr/local - fi build_xz if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then yum remove -y zlib-devel @@ -78,16 +108,24 @@ function build { build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist - if [[ "$CIBW_ARCHS" == "arm64" ]]; then - cp /usr/local/share/pkgconfig/xcb-proto.pc /usr/local/lib/pkgconfig - fi else - sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc + sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc fi build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib build_libjpeg_turbo - build_tiff + if [ -n "$IS_MACOS" ]; then + # Custom tiff build to include jpeg; by default, configure won't include + # headers/libs in the custom macOS prefix. Explicitly disable webp, + # libdeflate and zstd, because on x86_64 macs, it will pick up the + # Homebrew versions of those libraries from /usr/local. + build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \ + --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \ + --disable-webp --disable-libdeflate --disable-zstd + else + build_tiff + fi + build_libpng build_lcms2 build_openjpeg @@ -112,32 +150,47 @@ function build { build_harfbuzz } +# Perform all dependency builds in the build subfolder. +mkdir -p $WORKDIR +pushd $WORKDIR > /dev/null + # Any stuff that you need to do before you start building the wheels # Runs in the root directory of this repository. -curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip -untar pillow-depends-main.zip - -if [[ -n "$IS_MACOS" ]]; then - # libdeflate may cause a minimum target error when repairing the wheel - # libtiff and libxcb cause a conflict with building libtiff and libxcb - # libxau and libxdmcp cause an issue on macOS < 11 - # remove cairo to fix building harfbuzz on arm64 - # remove lcms2 and libpng to fix building openjpeg on arm64 - # remove jpeg-turbo to avoid inclusion on arm64 - # remove webp and zstd to avoid inclusion on x86_64 - # curl from brew requires zstd, use system curl - brew remove --ignore-dependencies libpng libtiff libxcb libxau libxdmcp curl cairo lcms2 zstd - if [[ "$CIBW_ARCHS" == "arm64" ]]; then - brew remove --ignore-dependencies jpeg-turbo - else - brew remove --ignore-dependencies libdeflate webp +if [[ ! -d $WORKDIR/pillow-depends-main ]]; then + if [[ ! -f $PROJECTDIR/pillow-depends-main.zip ]]; then + echo "Download pillow dependency sources..." + curl -fSL -o $PROJECTDIR/pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip fi + echo "Unpacking pillow dependency sources..." + untar $PROJECTDIR/pillow-depends-main.zip +fi - brew install pkg-config +if [[ -n "$IS_MACOS" ]]; then + # Homebrew (or similar packaging environments) install can contain some of + # the libraries that we're going to build. However, they may be compiled + # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use, + # and they may bring in other dependencies that we don't want. The same will + # be true of any other locations on the path. To avoid conflicts, strip the + # path down to the bare minimum (which, on macOS, won't include any + # development dependencies). + export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + export CMAKE_PREFIX_PATH=$BUILD_PREFIX + + # Ensure the basic structure of the build prefix directory exists. + mkdir -p "$BUILD_PREFIX/bin" + mkdir -p "$BUILD_PREFIX/lib" + + # Ensure pkg-config is available + build_pkg_config + # Ensure cmake is available + python3 -m pip install cmake fi wrap_wheel_builder build +# Return to the project root to finish the build +popd > /dev/null + # Append licenses for filename in wheels/dependency_licenses/*; do echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh index b30b1725f94..ce83a4278cd 100755 --- a/.github/workflows/wheels-test.sh +++ b/.github/workflows/wheels-test.sh @@ -1,12 +1,24 @@ #!/bin/bash set -e +# Ensure fribidi is installed by the system. if [[ "$OSTYPE" == "darwin"* ]]; then - brew install fribidi - export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig" - if [ -f /opt/homebrew/lib/libfribidi.dylib ]; then - sudo cp /opt/homebrew/lib/libfribidi.dylib /usr/local/lib + # If Homebrew is on the path during the build, it may leak into the wheels. + # However, we *do* need Homebrew to provide a copy of fribidi for + # testing purposes so that we can verify the fribidi shim works as expected. + if [[ "$(uname -m)" == "x86_64" ]]; then + HOMEBREW_PREFIX=/usr/local + else + HOMEBREW_PREFIX=/opt/homebrew fi + $HOMEBREW_PREFIX/bin/brew install fribidi + + # Add the lib folder for fribidi so that the vendored library can be found. + # Don't use $HOMEWBREW_PREFIX/lib directly - use the lib folder where the + # installed copy of fribidi is cellared. This ensures we don't pick up the + # Homebrew version of any other library that we're dependent on (most notably, + # freetype). + export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_PREFIX/lib/libfribidi.dylib)) elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then apk add curl fribidi else diff --git a/.gitignore b/.gitignore index 1dd6c917524..3033c2ea7ae 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ lib64/ parts/ sdist/ var/ +wheelhouse/ *.egg-info/ .installed.cfg *.egg @@ -90,5 +91,9 @@ Tests/images/msp Tests/images/picins Tests/images/sunraster +# Test and dependency downloads +pillow-depends-main.zip +pillow-test-images.zip + # pyinstaller *.spec diff --git a/pyproject.toml b/pyproject.toml index bff295bc60f..c6d95f2abe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,10 +94,17 @@ version = { attr = "PIL.__version__" } [tool.cibuildwheel] before-all = ".github/workflows/wheels-dependencies.sh" build-verbosity = 1 + config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable" +# Disable platform guessing on macOS +macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable platform-guessing=disable" + test-command = "cd {project} && .github/workflows/wheels-test.sh" test-extras = "tests" +[tool.cibuildwheel.macos.environment] +PATH = "$(pwd)/build/deps/darwin/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" + [tool.black] exclude = "wheels/multibuild" diff --git a/setup.py b/setup.py index e78f87fe9e0..fbd23a5685a 100644 --- a/setup.py +++ b/setup.py @@ -448,7 +448,7 @@ def _remove_extension(self, name: str) -> None: def get_macos_sdk_path(self) -> str | None: try: sdk_path = ( - subprocess.check_output(["xcrun", "--show-sdk-path"]) + subprocess.check_output(["xcrun", "--show-sdk-path", "--sdk", "macosx"]) .strip() .decode("latin1") ) @@ -606,6 +606,7 @@ def build_extensions(self) -> None: _add_directory(library_dirs, "/usr/X11/lib") _add_directory(include_dirs, "/usr/X11/include") + # Add the macOS SDK path. sdk_path = self.get_macos_sdk_path() if sdk_path: _add_directory(library_dirs, os.path.join(sdk_path, "usr", "lib"))