feat: manylinux_2_34 based on AlmaLinux 9

.github/workflows/build.yml

@@ -36,14 +36,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        policy: ["manylinux2014", "manylinux_2_28", "musllinux_1_1", "musllinux_1_2"]
-        platform: ["x86_64", "s390x", "ppc64le"]
-        include:
-          - policy: "manylinux2014"
+        policy: ["manylinux2014", "manylinux_2_28", "manylinux_2_34", "musllinux_1_1", "musllinux_1_2"]
+        platform: ["x86_64", "i686", "s390x", "ppc64le"]
+        exclude:
+          - policy: "manylinux_2_28"
             platform: "i686"
-          - policy: "musllinux_1_1"
-            platform: "i686"
-          - policy: "musllinux_1_2"
+          - policy: "manylinux_2_34"
             platform: "i686"
 
     env:

.travis.yml

@@ -38,6 +38,14 @@ jobs:
     #  env: POLICY="manylinux_2_28" PLATFORM="s390x"
     #- arch: ppc64le
     #  env: POLICY="manylinux_2_28" PLATFORM="ppc64le"
+    - arch: arm64-graviton2
+      virt: vm
+      group: edge
+      env: POLICY="manylinux_2_34" PLATFORM="aarch64"
+    - arch: s390x
+      env: POLICY="manylinux_2_34" PLATFORM="s390x"
+    - arch: ppc64le
+      env: POLICY="manylinux_2_34" PLATFORM="ppc64le"
     - arch: arm64-graviton2
       virt: vm
       group: edge

README.rst

@@ -37,6 +37,8 @@ The manylinux project supports:
 
 - ``manylinux_2_28`` images for ``x86_64``, ``aarch64``, ``ppc64le`` and ``s390x``.
 
+- ``manylinux_2_34`` images for ``x86_64``, ``aarch64``, ``ppc64le`` and ``s390x``.
+
 - ``musllinux_1_1`` images for ``x86_64``, ``i686``, ``aarch64``, ``ppc64le`` and ``s390x``.
 
 - ``musllinux_1_2`` images for ``x86_64``, ``i686``, ``aarch64``, ``ppc64le`` and ``s390x``.
@@ -95,6 +97,24 @@ done the work for you. The images are uploaded to `quay.io`_ and are tagged
 for repeatable builds.
 
 
+manylinux_2_34 (AlmaLinux 9 based)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Toolchain: GCC 13
+
+- x86_64 image: ``quay.io/pypa/manylinux_2_34_x86_64``
+- aarch64 image: ``quay.io/pypa/manylinux_2_34_aarch64``
+- ppc64le image: ``quay.io/pypa/manylinux_2_34_ppc64le``
+- s390x image: ``quay.io/pypa/manylinux_2_34_s390x``
+
+Built wheels are also expected to be compatible with other
+distros using glibc 2.34 or later, including:
+
+- Debian 12+
+- Ubuntu 21.10+
+- Fedora 35+
+- CentOS/RHEL 9+
+
 manylinux_2_28 (AlmaLinux 8 based)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

build.sh

@@ -46,6 +46,11 @@ if [ "${POLICY}" == "manylinux2014" ]; then
 	else
 		LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64"
 	fi
+elif [ "${POLICY}" == "manylinux_2_34" ]; then
+	BASEIMAGE="${MULTIARCH_PREFIX}almalinux:9"
+	DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-13/root"
+	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
+	LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst"
 elif [ "${POLICY}" == "manylinux_2_28" ]; then
 	BASEIMAGE="${MULTIARCH_PREFIX}almalinux:8"
 	DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-12/root"

docker/Dockerfile

@@ -1,8 +1,8 @@
 # default to latest supported policy, x86_64
-ARG BASEIMAGE=amd64/almalinux:8
-ARG POLICY=manylinux_2_28
+ARG BASEIMAGE=amd64/almalinux:9
+ARG POLICY=manylinux_2_34
 ARG PLATFORM=x86_64
-ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root
+ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-13/root
 ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst
 ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin:
 

docker/build_scripts/build-cpython.sh

@@ -50,6 +50,14 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then
 	export TCLTK_LIBS="-ltk8.6 -ltcl8.6"
 fi
 
+SQLITE_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'sqlite*')
+if [ "${SQLITE_PREFIX}" != "" ]; then
+	case "${CPYTHON_VERSION}" in
+		3.6.*|3.7.*|3.8.*|3.9.*|3.10.*) sed -i "s|/usr/local/include/sqlite3|/opt/_internal/sqlite3/include|g ; s|sqlite_extra_link_args = ()|sqlite_extra_link_args = ('-Wl,--enable-new-dtags,-rpath=/opt/_internal/sqlite3/lib',)|g" setup.py;;
+		*) ;;
+	esac
+fi
+
 OPENSSL_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'openssl*')
 if [ "${OPENSSL_PREFIX}" != "" ]; then
 	CONFIGURE_ARGS="${CONFIGURE_ARGS} --with-openssl=${OPENSSL_PREFIX}"

docker/build_scripts/build-sqlite3.sh

@@ -10,6 +10,11 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
+if [ "${AUDITWHEEL_POLICY}" != "manylinux_2_34" ]; then
+	PREFIX=/usr/local
+else
+	PREFIX=/opt/_internal/sqlite3
+fi
 # Install a more recent SQLite3
 check_var ${SQLITE_AUTOCONF_ROOT}
 check_var ${SQLITE_AUTOCONF_HASH}
@@ -18,23 +23,29 @@ fetch_source ${SQLITE_AUTOCONF_ROOT}.tar.gz ${SQLITE_AUTOCONF_DOWNLOAD_URL}
 check_sha256sum ${SQLITE_AUTOCONF_ROOT}.tar.gz ${SQLITE_AUTOCONF_HASH}
 tar xfz ${SQLITE_AUTOCONF_ROOT}.tar.gz
 pushd ${SQLITE_AUTOCONF_ROOT}
-DESTDIR=/manylinux-rootfs do_standard_install
+# add rpath
+sed -i "s|^Libs:|Libs: -Wl,--enable-new-dtags,-rpath=\${libdir} |g" sqlite3.pc.in
+DESTDIR=/manylinux-rootfs do_standard_install --prefix=${PREFIX}
 popd
 rm -rf ${SQLITE_AUTOCONF_ROOT} ${SQLITE_AUTOCONF_ROOT}.tar.gz
 
-# static library is unused, remove it
-rm /manylinux-rootfs/usr/local/lib/libsqlite3.a
-
 # Strip what we can
 strip_ /manylinux-rootfs
 
 # Install
 cp -rlf /manylinux-rootfs/* /
-if [ "${BASE_POLICY}" == "musllinux" ]; then
-	ldconfig /
-elif [ "${BASE_POLICY}" == "manylinux" ]; then
-	ldconfig
+
+if [ "${PREFIX}" == "/usr/local" ]; then
+	if [ "${BASE_POLICY}" == "musllinux" ]; then
+		ldconfig /
+	elif [ "${BASE_POLICY}" == "manylinux" ]; then
+		ldconfig
+	fi
+else
+	# python >= 3.11
+	mkdir -p /usr/local/lib/pkgconfig/
+	ln -s ${PREFIX}/lib/pkgconfig/sqlite3.pc /usr/local/lib/pkgconfig/sqlite3.pc
 fi
 
 # Clean-up for runtime
-rm -rf /manylinux-rootfs/usr/local/share
+rm -rf /manylinux-rootfs${PREFIX}/share

docker/build_scripts/finalize.sh

@@ -92,7 +92,7 @@ clean_pyc /opt/_internal
 # remove cache
 rm -rf /tmp/* || true
 
-hardlink -cv /opt/_internal
+hardlink -c /opt/_internal
 
 # update system packages
 LC_ALL=C ${MY_DIR}/update-system-packages.sh

docker/build_scripts/install-build-packages.sh

@@ -14,13 +14,16 @@ source $MY_DIR/build_utils.sh
 # make sure the corresponding library is added to RUNTIME_DEPS if applicable
 
 if [ "${BASE_POLICY}" == "manylinux" ]; then
-	COMPILE_DEPS="bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel libidn-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel perl-IPC-Cmd"
+	COMPILE_DEPS="bzip2-devel ncurses-devel readline-devel gdbm-devel libpcap-devel xz-devel openssl openssl-devel keyutils-libs-devel krb5-devel libcom_err-devel curl-devel uuid-devel libffi-devel kernel-headers libdb-devel perl-IPC-Cmd"
 	if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 		PACKAGE_MANAGER=yum
-		COMPILE_DEPS="${COMPILE_DEPS} libXft-devel"
+		COMPILE_DEPS="${COMPILE_DEPS} libidn-devel libXft-devel"
+	elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+		PACKAGE_MANAGER=dnf
+		COMPILE_DEPS="${COMPILE_DEPS} libidn-devel tk-devel"
 	else
 		PACKAGE_MANAGER=dnf
-		COMPILE_DEPS="${COMPILE_DEPS} tk-devel"
+		COMPILE_DEPS="${COMPILE_DEPS} libidn2-devel tk-devel"
 	fi
 elif [ "${BASE_POLICY}" == "musllinux" ]; then
 	PACKAGE_MANAGER=apk
@@ -40,7 +43,7 @@ elif [ "${PACKAGE_MANAGER}" == "apk" ]; then
 elif [ "${PACKAGE_MANAGER}" == "dnf" ]; then
  	dnf -y install --allowerasing ${COMPILE_DEPS}
  	dnf clean all
- 	rm -rf /var/cache/yum
+ 	rm -rf /var/cache/dnf
 else
 	echo "Not implemented"
 	exit 1

docker/build_scripts/install-libxcrypt.sh

@@ -10,11 +10,8 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Get build utilities
 source $MY_DIR/build_utils.sh
 
-if [ "$BASE_POLICY" == "musllinux" ]; then
-	echo "Skip libxcrypt installation on musllinux"
-	exit 0
-elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
-	echo "Skip libxcrypt installation on manylinux_2_28"
+if [ "${AUDITWHEEL_POLICY}" != "manylinux2014" ]; then
+	echo "Skip libxcrypt installation on ${AUDITWHEEL_POLICY}"
 	exit 0
 fi
 

docker/build_scripts/install-runtime-packages.sh

@@ -32,7 +32,7 @@ source $MY_DIR/build_utils.sh
 
 
 # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then
 	MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel"
 elif [ "${BASE_POLICY}" == "musllinux" ]; then
 	MANYLINUX_DEPS="musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev zlib-dev expat-dev"
@@ -42,12 +42,16 @@ else
 fi
 
 # RUNTIME_DEPS: Runtime dependencies. c.f. install-build-packages.sh
-if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
-	RUNTIME_DEPS="zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libidn libcurl uuid libffi libdb"
+if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then
+	RUNTIME_DEPS="zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs libkadm5 libcom_err libcurl uuid libffi libdb"
 	if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
-		RUNTIME_DEPS="${RUNTIME_DEPS} libXft"
+		RUNTIME_DEPS="${RUNTIME_DEPS} libXft libidn"
 	elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
-		RUNTIME_DEPS="${RUNTIME_DEPS} tk"
+		RUNTIME_DEPS="${RUNTIME_DEPS} libidn tk"
+	else
+		RUNTIME_DEPS="${RUNTIME_DEPS} libidn2 tk"
+		# for graalpy
+		RUNTIME_DEPS="${RUNTIME_DEPS} libxcrypt-compat"
 	fi
 elif [ "${BASE_POLICY}" == "musllinux" ]; then
 	RUNTIME_DEPS="zlib bzip2 expat ncurses-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi"
@@ -87,7 +91,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 		curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-10.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-10/repo/custom-1/mayeut-devtoolset-10-custom-1.repo
 	fi
 	fixup-mirrors
-elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then
 	PACKAGE_MANAGER=dnf
 	BASETOOLS="${BASETOOLS} curl glibc-locale-source glibc-langpack-en hardlink hostname libcurl libnsl libxcrypt which"
 	# See https://unix.stackexchange.com/questions/41784/can-yum-express-a-preference-for-x86-64-over-i386-packages
@@ -97,9 +101,14 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
 	# Make sure that locale will not be removed
 	sed -i '/^override_install_langs=/d' /etc/yum.conf
 	dnf -y upgrade
-	dnf -y install dnf-plugins-core
-	dnf config-manager --set-enabled powertools # for yasm
-	TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran"
+	dnf -y install dnf-plugins-core epel-release  # for yasm
+	if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+		dnf config-manager --set-enabled powertools
+		TOOLCHAIN_DEPS="gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ gcc-toolset-12-gcc-gfortran"
+	else
+		dnf config-manager --set-enabled crb
+		TOOLCHAIN_DEPS="gcc-toolset-13-binutils gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran"
+	fi
 	if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
 		TOOLCHAIN_DEPS="${TOOLCHAIN_DEPS} yasm"
 	fi

docker/build_scripts/update-system-packages.sh

@@ -20,10 +20,10 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	fi
 	yum clean all
 	rm -rf /var/cache/yum
-elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then
 	dnf -y upgrade
 	dnf clean all
-	rm -rf /var/cache/yum
+	rm -rf /var/cache/dnf
 elif [ "${BASE_POLICY}" == "musllinux" ]; then
 	apk upgrade --no-cache
 else

tests/run_tests.sh

@@ -10,7 +10,7 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
 	PACKAGE_MANAGER=yum
 elif [ "${AUDITWHEEL_POLICY:0:10}" == "musllinux_" ]; then
 	PACKAGE_MANAGER=apk
-elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ]; then
+elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_28" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux_2_34" ]; then
 	PACKAGE_MANAGER=dnf
 else
 	echo "Unsupported policy: '${AUDITWHEEL_POLICY}'"
@@ -119,6 +119,9 @@ if [ ${EXPECTED_PYTHON_COUNT_ALL} -ne ${PYTHON_COUNT} ]; then
 	exit 1
 fi
 
+# we stopped installing sqlite3 in manylinux_2_34
+SQLITE_PREFIX=$(find /opt/_internal -maxdepth 1 -name 'sqlite*')
+
 # minimal tests for tools that should be present
 auditwheel --version
 autoconf --version
@@ -128,7 +131,9 @@ patchelf --version
 git --version
 cmake --version
 swig -version
-sqlite3 --version
+if [ "${SQLITE_PREFIX}" == "" ]; then
+	sqlite3 --version
+fi
 pipx run nox --version
 pipx install --pip-args='--no-python-version-warning --no-input' nox
 nox --version
@@ -150,13 +155,15 @@ fi
 eval "$(ssh-agent)"
 eval "$(ssh-agent -k)"
 
-# compilation tests, intended to ensure appropriate headers, pkg_config, etc.
-# are available for downstream compile against installed tools
-source_dir="${MY_DIR}/ctest"
-build_dir="$(mktemp -d)"
-cmake -S "${source_dir}" -B "${build_dir}"
-cmake --build "${build_dir}"
-(cd "${build_dir}"; ctest --output-on-failure)
+if [ "${SQLITE_PREFIX}" == "" ]; then
+	# compilation tests, intended to ensure appropriate headers, pkg_config, etc.
+	# are available for downstream compile against installed tools
+	source_dir="${MY_DIR}/ctest"
+	build_dir="$(mktemp -d)"
+	cmake -S "${source_dir}" -B "${build_dir}"
+	cmake --build "${build_dir}"
+	(cd "${build_dir}"; ctest --output-on-failure)
+fi
 
 # https://github.com/pypa/manylinux/issues/1060
 # wrong /usr/local/man symlink