From a94593e3c9ebf4563f855191dd08d4c340a14616 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 13 Feb 2021 18:49:22 +0100 Subject: [PATCH] Add manylinux_2_24 images --- .github/workflows/build.yml | 6 + .travis.yml | 10 +- README.rst | 103 +++++++++--------- build.sh | 5 + docker/Dockerfile | 10 +- .../build_scripts/install-build-packages.sh | 11 +- .../build_scripts/install-runtime-packages.sh | 28 ++++- 7 files changed, 110 insertions(+), 63 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c988aacf..f61c49b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,12 @@ jobs: platform: "x86_64" - policy: "manylinux2014" platform: "ppc64le" + - policy: "manylinux_2_24" + platform: "i686" + - policy: "manylinux_2_24" + platform: "x86_64" + - policy: "manylinux_2_24" + platform: "ppc64le" env: POLICY: ${{ matrix.policy }} diff --git a/.travis.yml b/.travis.yml index de866a7c..1496f662 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,10 +26,18 @@ jobs: virt: vm group: edge env: POLICY="manylinux2014" PLATFORM="aarch64" + - arch: s390x + env: POLICY="manylinux2014" PLATFORM="s390x" #- arch: ppc64le # env: POLICY="manylinux2014" PLATFORM="ppc64le" + - arch: arm64-graviton2 + virt: vm + group: edge + env: POLICY="manylinux_2_24" PLATFORM="aarch64" - arch: s390x - env: POLICY="manylinux2014" PLATFORM="s390x" + env: POLICY="manylinux_2_24" PLATFORM="s390x" + #- arch: ppc64le + # env: POLICY="manylinux2014" PLATFORM="ppc64le" before_install: - if [ -d "${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM}" ]; then cp -rlf ${HOME}/buildx-cache/.buildx-cache-${POLICY}_${PLATFORM} ./; fi diff --git a/README.rst b/README.rst index 9b229f1e..aa838f64 100644 --- a/README.rst +++ b/README.rst @@ -8,18 +8,18 @@ Archives: https://mail.python.org/mailman/listinfo/wheel-builders Older archives: https://groups.google.com/forum/#!forum/manylinux-discuss The goal of the manylinux project is to provide a convenient way to -distribute binary Python extensions as wheels on Linux. This effort -has produced `PEP 513 `_ which -is further enhanced by `PEP 571 `_ -and now `PEP 599 `_ defining -``manylinux2014_*`` platform tags. +distribute binary Python extensions as wheels on Linux. +This effort has produced `PEP 513 `_ (manylinux1), +`PEP 571 `_ (manylinux2010), +`PEP 599 `_ (manylinux2014) and +`PEP 600 `_ (manylinux_x_y). PEP 513 defined ``manylinux1_x86_64`` and ``manylinux1_i686`` platform tags and the wheels were built on Centos5. Centos5 reached End of Life (EOL) on -March 31st, 2017 and thus PEP 571 was proposed. +March 31st, 2017. PEP 571 defined ``manylinux2010_x86_64`` and ``manylinux2010_i686`` platform -tags and the wheels were built on Centos6. Centos6 will reach End of Life (EOL) +tags and the wheels were built on Centos6. Centos6 reached End of Life (EOL) on November 30th, 2020. PEP 599 defines the following platform tags: @@ -41,8 +41,10 @@ PEP 599 defines the following platform tags: Wheels are built on CentOS 7 which will reach End of Life (EOL) on June 30th, 2024. -Code and details regarding ``manylinux1`` can be found here: -`manylinux1 `_. +PEP 600 has been designed to be "future-proof" and does not enforce specific symbols and a specific distro to build. +It only states that a wheel tagged ``manylinux_x_y`` shall work on any distro based on ``glibc>=x.y``. +The manylinux project supports ``manylinux_2_24`` images for ``x86_64``, ``i686``, ``aarch64``, ``ppc64le`` and ``s390x``. + Wheel packages compliant with those tags can be uploaded to `PyPI `_ (for instance with `twine @@ -52,6 +54,8 @@ pip: +-------------------+----------------------------------+ | ``manylinux`` tag | Client-side pip version required | +===================+==================================+ +| ``manylinux_x_y`` | pip >= 20.3 | ++-------------------+----------------------------------+ | ``manylinux2014`` | pip >= 19.3 | +-------------------+----------------------------------+ | ``manylinux2010`` | pip >= 19.0 | @@ -59,7 +63,7 @@ pip: | ``manylinux1`` | pip >= 8.1.0 | +-------------------+----------------------------------+ -The manylinux2014 tags allow projects to distribute wheels that are +The various manylinux tags allow projects to distribute wheels that are automatically installed (and work!) on the vast majority of desktop and server Linux distributions. @@ -73,74 +77,63 @@ Building manylinux-compatible wheels is not trivial; as a general rule, binaries built on one Linux distro will only work on other Linux distros that are the same age or newer. Therefore, if we want to make binaries that run on most Linux distros, we have to use an old enough -distro -- CentOS 7. +distro. -Rather than forcing you to install CentOS 7 yourself, install Python, +Rather than forcing you to install an old distro yourself, install Python, etc., we provide `Docker `_ images where we've done the work for you. The images are uploaded to `quay.io`_ and are tagged for repeatable builds. -manylinux1 (CentOS 5 based) -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -x86-64 image: ``quay.io/pypa/manylinux1_x86_64`` - -.. image:: https://quay.io/repository/pypa/manylinux1_x86_64/status - :target: https://quay.io/repository/pypa/manylinux1_x86_64 -i686 image: ``quay.io/pypa/manylinux1_i686`` - -.. image:: https://quay.io/repository/pypa/manylinux1_i686/status - :target: https://quay.io/repository/pypa/manylinux1_i686 - -manylinux2010 (CentOS 6 based) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +manylinux_2_24 (Debian 9 based) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -x86-64 image: ``quay.io/pypa/manylinux2010_x86_64`` +- x86_64 image: ``quay.io/pypa/manylinux_2_24_x86_64`` +- i686 image: ``quay.io/pypa/manylinux_2_24_i686`` +- aarch64 image: ``quay.io/pypa/manylinux_2_24_aarch64`` +- ppc64le image: ``quay.io/pypa/manylinux_2_24_ppc64le`` +- s390x image: ``quay.io/pypa/manylinux_2_24_s390x`` -.. image:: https://quay.io/repository/pypa/manylinux2010_x86_64/status - :target: https://quay.io/repository/pypa/manylinux2010_x86_64 - -i686 image: ``quay.io/pypa/manylinux2010_i686`` - -.. image:: https://quay.io/repository/pypa/manylinux2010_i686/status - :target: https://quay.io/repository/pypa/manylinux2010_i686 manylinux2014 (CentOS 7 based) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -x86_64 image: ``quay.io/pypa/manylinux2014_x86_64`` +- x86_64 image: ``quay.io/pypa/manylinux2014_x86_64`` +- i686 image: ``quay.io/pypa/manylinux2014_i686`` +- aarch64 image: ``quay.io/pypa/manylinux2014_aarch64`` +- ppc64le image: ``quay.io/pypa/manylinux2014_ppc64le`` +- s390x image: ``quay.io/pypa/manylinux2014_s390x`` -.. image:: https://quay.io/repository/pypa/manylinux2014_x86_64/status - :target: https://quay.io/repository/pypa/manylinux2014_x86_64 -i686 image: ``quay.io/pypa/manylinux2014_i686`` +manylinux2010 (CentOS 6 based - EOL) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. image:: https://quay.io/repository/pypa/manylinux2014_i686/status - :target: https://quay.io/repository/pypa/manylinux2014_i686 +- x86-64 image: ``quay.io/pypa/manylinux2010_x86_64`` +- i686 image: ``quay.io/pypa/manylinux2010_i686`` -aarch64 image: ``quay.io/pypa/manylinux2014_aarch64`` -.. image:: https://quay.io/repository/pypa/manylinux2014_aarch64/status - :target: https://quay.io/repository/pypa/manylinux2014_aarch64 +manylinux1 (CentOS 5 based - EOL) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -ppc64le image: ``quay.io/pypa/manylinux2014_ppc64le`` +Code and details regarding ``manylinux1`` can be found in the `manylinux1 branch `_. -.. image:: https://quay.io/repository/pypa/manylinux2014_ppc64le/status - :target: https://quay.io/repository/pypa/manylinux2014_ppc64le +Support for ``manylinux1`` will `end on January 1st, 2022 `_. -s390x image: ``quay.io/pypa/manylinux2014_s390x`` +- x86-64 image: ``quay.io/pypa/manylinux1_x86_64`` +- i686 image: ``quay.io/pypa/manylinux1_i686`` -.. image:: https://quay.io/repository/pypa/manylinux2014_s390x/status - :target: https://quay.io/repository/pypa/manylinux2014_s390x -These images are rebuilt using GitHub Actions on every commit to this +All images are rebuilt using GitHub Actions / Travis-CI on every commit to this repository; see the `docker/ `_ directory for source code. -The images currently contain: + +Image content +~~~~~~~~~~~~~ + +All images currently contain: - CPython 3.5, 3.6, 3.7, 3.8 and 3.9, installed in ``/opt/python/-``. The directories are named @@ -149,8 +142,7 @@ The images currently contain: can be used to produce wheels named like ``--cp37-cp37m-.whl``. -- Devel packages for all the libraries that PEP 571/599 allows you to - assume are present on the host system +- Development packages for all the libraries that PEP 571/599 list. One should not assume the presence of any other development package. - The `auditwheel `_ tool @@ -170,6 +162,8 @@ current (root) directory: $ PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest ./build.sh +Please not that the Docker build is using `buildx `_. + Example ------- @@ -205,6 +199,9 @@ reached EOL was discussed in `PEP 571 `_. +The proposal for a "future-proof" ``manylinux_x_y`` definition was +discussed in `PEP 600 `_. + This repo also has some analysis code that was used when putting together the original proposal in the ``policy-info/`` directory. diff --git a/build.sh b/build.sh index 78009990..6fefdeb4 100755 --- a/build.sh +++ b/build.sh @@ -53,6 +53,11 @@ elif [ "${POLICY}" == "manylinux2014" ]; then else LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" fi +elif [ "${POLICY}" == "manylinux_2_24" ]; then + BASEIMAGE="${MULTIARCH_PREFIX}debian:9" + DEVTOOLSET_ROOTPATH= + PREPEND_PATH= + LD_LIBRARY_PATH_ARG= else echo "Unsupported policy: '${POLICY}'" exit 1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 51be6809..aba417db 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,10 @@ # default to latest supported policy, x86_64 -ARG BASEIMAGE=amd64/centos:7 -ARG POLICY=manylinux2014 +ARG BASEIMAGE=amd64/debian:9 +ARG POLICY=manylinux_2_24 ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root -ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib -ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: +ARG DEVTOOLSET_ROOTPATH= +ARG LD_LIBRARY_PATH_ARG= +ARG PREPEND_PATH= FROM $BASEIMAGE AS runtime_base ARG POLICY diff --git a/docker/build_scripts/install-build-packages.sh b/docker/build_scripts/install-build-packages.sh index 1c3fa0f2..65707a67 100755 --- a/docker/build_scripts/install-build-packages.sh +++ b/docker/build_scripts/install-build-packages.sh @@ -16,16 +16,25 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == else COMPILE_DEPS="${COMPILE_DEPS} libdb-devel" fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + PACKAGE_MANAGER=apt + COMPILE_DEPS="libz-dev libbz2-dev libexpat1-dev libncurses5-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-kernel-headers" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 fi -if [ ${PACKAGE_MANAGER} == yum ]; then +if [ "${PACKAGE_MANAGER}" == "yum" ]; then yum -y install ${COMPILE_DEPS} yum clean all rm -rf /var/cache/yum +elif [ "${PACKAGE_MANAGER}" == "apt" ]; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -qq -y --no-install-recommends ${COMPILE_DEPS} + apt-get clean -qq + rm -rf /var/lib/apt/lists/* else echo "Not implemented" exit 1 diff --git a/docker/build_scripts/install-runtime-packages.sh b/docker/build_scripts/install-runtime-packages.sh index 10c40774..d7857fe1 100755 --- a/docker/build_scripts/install-runtime-packages.sh +++ b/docker/build_scripts/install-runtime-packages.sh @@ -26,6 +26,8 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}") # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install) if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then MANYLINUX_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel" +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + MANYLINUX_DEPS="libc6-dev libstdc++-6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -39,6 +41,8 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ] || [ "${AUDITWHEEL_POLICY}" == else RUNTIME_DEPS="${RUNTIME_DEPS} libdb" fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + RUNTIME_DEPS="zlib1g libbz2-1.0 libexpat1 libncurses5 libreadline7 tk libgdbm3 libdb5.3 libpcap0.8 liblzma5 libssl1.1 libkeyutils1 libkrb5-3 libcomerr2 libidn2-0 libcurl3 uuid libffi6" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -91,6 +95,14 @@ elif [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then # Install mayeut/devtoolset-9 repo to get devtoolset-9 curl -fsSLo /etc/yum.repos.d/mayeut-devtoolset-9.repo https://copr.fedorainfracloud.org/coprs/mayeut/devtoolset-9/repo/custom-1/mayeut-devtoolset-9-custom-1.repo fi +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + PACKAGE_MANAGER=apt + export DEBIAN_FRONTEND=noninteractive + sed -i 's/none/en_US/g' /etc/apt/apt.conf.d/docker-no-languages + apt-get update -qq + apt-get upgrade -qq -y + apt-get install -qq -y --no-install-recommends ca-certificates gpg curl locales + TOOLCHAIN_DEPS="binutils gcc g++ gfortran" else echo "Unsupported policy: '${AUDITWHEEL_POLICY}'" exit 1 @@ -98,7 +110,7 @@ fi if ! which localedef &> /dev/null; then # somebody messed up glibc-common package to squeeze image size, reinstall the package - if [ ${PACKAGE_MANAGER} == yum ]; then + if [ "${PACKAGE_MANAGER}" == "yum" ]; then yum -y reinstall glibc-common else echo "Not implemented" @@ -109,10 +121,14 @@ fi # upgrading glibc-common can end with removal on en_US.UTF-8 locale localedef -i en_US -f UTF-8 en_US.UTF-8 -if [ ${PACKAGE_MANAGER} == yum ]; then +if [ "${PACKAGE_MANAGER}" == "yum" ]; then yum -y install ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} yum clean all rm -rf /var/cache/yum +elif [ "${PACKAGE_MANAGER}" == "apt" ]; then + apt-get install -qq -y --no-install-recommends ${BASETOOLS} ${TOOLCHAIN_DEPS} ${MANYLINUX_DEPS} ${RUNTIME_DEPS} + apt-get clean -qq + rm -rf /var/lib/apt/lists/* else echo "Not implemented" exit 1 @@ -128,7 +144,9 @@ if [ "${DEVTOOLSET_ROOTPATH:-}" != "" ]; then find $DEVTOOLSET_ROOTPATH/usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf fi -rm -rf /usr/share/backgrounds +if [ -d /usr/share/backgrounds ]; then + rm -rf /usr/share/backgrounds +fi # if we updated glibc, we need to strip locales again... if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then @@ -137,6 +155,10 @@ fi if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then mv -f /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl build-locale-archive +elif [ "${AUDITWHEEL_POLICY}" == "manylinux_2_24" ]; then + rm /usr/lib/locale/locale-archive + localedef -i en_US -f UTF-8 en_US.UTF-8 + update-locale LANG=en_US.UTF-8 fi find /usr/share/locale -mindepth 1 -maxdepth 1 -not \( -name 'en*' -or -name 'locale.alias' \) | xargs rm -rf