[WIP] manylinux2010 with devtoolset 7

.travis.yml

@@ -19,7 +19,7 @@ env:
 
 matrix:
   include:
-    - env: PLATFORM="i686"
+    # - env: PLATFORM="i686"
     - env: PLATFORM="x86_64"
 
 script:

build.sh

@@ -4,5 +4,9 @@
 set -ex
 
 docker/build_scripts/prefetch.sh openssl curl
-docker build --rm -t quay.io/pypa/manylinux1_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/
+if [ $PLATFORM == x86_64 ]; then
+    echo "Building quay.io/pypa/manylinux2010_centos-6-no-vsyscall"
+    docker build --rm -t quay.io/pypa/manylinux2010_centos-6-no-vsyscall -f docker/glibc/Dockerfile docker/glibc/ > /dev/null
+fi
+docker build --rm -t quay.io/pypa/manylinux2010_$PLATFORM:$TRAVIS_COMMIT -f docker/Dockerfile-$PLATFORM docker/
 docker system prune -f

docker/Dockerfile-i686

@@ -1,12 +1,17 @@
-FROM phusion/centos-5-32
-MAINTAINER The ManyLinux project
 
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
+FROM i386/centos:6
+LABEL maintainer="The ManyLinux project"
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
-ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib
-ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+ENV PATH /opt/rh/devtoolset-7/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
+
+COPY ./build_scripts /build_scripts
+COPY /build_scripts/linux32 /usr/bin/linux32
+RUN chmod +x /usr/bin/linux32
 
 COPY build_scripts /build_scripts
 COPY sources /

docker/Dockerfile-x86_64

@@ -1,17 +1,25 @@
-FROM centos:5
-MAINTAINER The ManyLinux project
+# See docker/glibc/
+FROM quay.io/pypa/manylinux2010_centos-6-no-vsyscall
+LABEL maintainer="The ManyLinux project"
 
 ENV LC_ALL en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US.UTF-8
-ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
-ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib
-ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+ENV PATH /opt/rh/devtoolset-7/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/usr/local/lib64:/usr/local/lib
+ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig
+
+
+
+
 
 COPY build_scripts /build_scripts
 COPY sources /
 RUN bash build_scripts/build.sh && rm -r build_scripts
 
 ENV SSL_CERT_FILE=/opt/_internal/certs.pem
 
+
+
+
 CMD ["/bin/bash"]

docker/build_scripts/build.sh

@@ -11,19 +11,10 @@ MY_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Dependencies for compiling Python that we want to remove from
 # the final image after compiling Python
 # GPG installed to verify signatures on Python source tarballs.
-PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel gpg libffi-devel"
+PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel libffi-devel"
 
-# Libraries that are allowed as part of the manylinux1 profile
-MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel  mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
-
-# Centos 5 is EOL and is no longer available from the usual mirrors, so switch
-# to http://vault.centos.org
-# From: https://github.com/rust-lang/rust/pull/41045
-# The location for version 5 was also removed, so now only the specific release
-# (5.11) can be referenced.
-sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
-sed -i 's/mirrorlist/#mirrorlist/' /etc/yum.repos.d/*.repo
-sed -i 's/#\(baseurl.*\)mirror.centos.org\/centos\/$releasever/\1vault.centos.org\/5.11/' /etc/yum.repos.d/*.repo
+# Libraries that are allowed as part of the manylinux2010 profile
+MANYLINUX2010_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel  mesa-libGL-devel libICE-devel libSM-devel ncurses-devel"
 
 # Get build utilities
 source $MY_DIR/build_utils.sh
@@ -43,16 +34,14 @@ yum -y update
 
 # EPEL support
 yum -y install wget
-# https://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
-cp $MY_DIR/epel-release-5-4.noarch.rpm .
-check_sha256sum epel-release-5-4.noarch.rpm $EPEL_RPM_HASH
+# https://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+cp $MY_DIR/epel-release-6-8.noarch.rpm .
+check_sha256sum epel-release-6-8.noarch.rpm $EPEL_RPM_HASH
 
 # Dev toolset (for LLVM and other projects requiring C++11 support)
-wget -q http://people.centos.org/tru/devtools-2/devtools-2.repo
-check_sha256sum devtools-2.repo $DEVTOOLS_HASH
-mv devtools-2.repo /etc/yum.repos.d/devtools-2.repo
-rpm -Uvh --replacepkgs epel-release-5*.rpm
-rm -f epel-release-5*.rpm
+yum -y install centos-release-scl
+rpm -Uvh --replacepkgs epel-release-6*.rpm
+rm -f epel-release-6*.rpm
 
 # from now on, we shall only use curl to retrieve files
 yum -y erase wget
@@ -63,10 +52,10 @@ yum -y install \
     bison \
     bzip2 \
     cmake28 \
-    devtoolset-2-binutils \
-    devtoolset-2-gcc \
-    devtoolset-2-gcc-c++ \
-    devtoolset-2-gcc-gfortran \
+    devtoolset-7-binutils \
+    devtoolset-7-gcc \
+    devtoolset-7-gcc-c++ \
+    devtoolset-7-gcc-gfortran \
     diffutils \
     expat-devel \
     gettext \
@@ -77,13 +66,15 @@ yum -y install \
     unzip \
     which \
     yasm \
+    gpg \
     ${PYTHON_COMPILE_DEPS}
 
 # Build an OpenSSL for both curl and the Pythons. We'll delete this at the end.
 build_openssl $OPENSSL_ROOT $OPENSSL_HASH
 
 # Install curl so we can have TLS 1.2 in this ancient container.
 build_curl $CURL_ROOT $CURL_HASH
+export PATH="/opt/curl/bin:$PATH"
 hash -r
 curl --version
 curl-config --features
@@ -157,8 +148,10 @@ yum -y erase \
     hicolor-icon-theme \
     libX11 \
     wireless-tools \
-    ${PYTHON_COMPILE_DEPS}  > /dev/null 2>&1
-yum -y install ${MANYLINUX1_DEPS}
+    ${PYTHON_COMPILE_DEPS/libffi-devel/}
+# Not uninstalling libffi-devel since it's failing to uninstall https://bugs.centos.org/view.php?id=10828
+
+yum -y install ${MANYLINUX2010_DEPS}
 yum -y clean all > /dev/null 2>&1
 yum list installed
 
@@ -180,7 +173,7 @@ find /opt/_internal -depth \
 for PYTHON in /opt/python/*/bin/python; do
     # Smoke test to make sure that our Pythons work, and do indeed detect as
     # being manylinux compatible:
-    $PYTHON $MY_DIR/manylinux1-check.py
+    $PYTHON $MY_DIR/manylinux-check.py
     # Make sure that SSL cert checking works
     $PYTHON $MY_DIR/ssl-check.py
 done

docker/build_scripts/build_env.sh

@@ -37,5 +37,5 @@ GIT_HASH=ba2fed9d02e424b735e035c4f2b0bdb168ef0df7e35156b5051d900dc7247787
 GIT_DOWNLOAD_URL=https://github.com/git/git/archive
 
 GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
-EPEL_RPM_HASH=0dcc89f9bf67a2a515bad64569b7a9615edc5e018f676a578d5fd0f17d3c81d4
+EPEL_RPM_HASH=e5ed9ecf22d0c4279e92075a64c757ad2b38049bcf5c16c4f2b75d5f6860dc0d
 DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc

docker/build_scripts/build_utils.sh

@@ -54,10 +54,7 @@ function do_cpython_build {
     if [ -e ${prefix}/bin/python3 ]; then
         ln -s python3 ${prefix}/bin/python
     fi
-    # --force-reinstall is to work around:
-    #   https://github.com/pypa/pip/issues/5220
-    #   https://github.com/pypa/get-pip/issues/19
-    ${prefix}/bin/python get-pip.py --force-reinstall
+    ${prefix}/bin/python get-pip.py
     if [ -e ${prefix}/bin/pip3 ] && [ ! -e ${prefix}/bin/pip ]; then
         ln -s pip3 ${prefix}/bin/pip
     fi
@@ -170,15 +167,15 @@ function build_git {
     fetch_source v${git_fname}.tar.gz ${GIT_DOWNLOAD_URL}
     check_sha256sum v${git_fname}.tar.gz ${git_sha256}
     tar -xzf v${git_fname}.tar.gz
-    (cd git-${git_fname} && make install prefix=/usr/local LDFLAGS="-L/usr/local/ssl/lib -ldl" CFLAGS="-I/usr/local/ssl/include" > /dev/null)
+    (cd git-${git_fname} && make install prefix=/usr/local LDFLAGS="-L/opt/curl/lib -L/usr/local/ssl/lib -ldl" CFLAGS="-I/opt/curl/include -I/usr/local/ssl/include" > /dev/null)
     rm -rf git-${git_fname} v${git_fname}.tar.gz
 }
 
 
 function do_curl_build {
     # We do this shared to avoid obnoxious linker issues where git couldn't
     # link properly. If anyone wants to make this build statically go for it.
-    LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static > /dev/null
+    LIBS=-ldl CFLAGS=-Wl,--exclude-libs,ALL ./configure --with-ssl --disable-static --prefix=/opt/curl > /dev/null
     make > /dev/null
     make install > /dev/null
 }

docker/build_scripts/linux32

@@ -0,0 +1,50 @@
+#!/usr/bin/python
+"""
+Emulate linux32, i.e. setarch(8).
+"""
+import os
+import ctypes
+import sys
+
+# Retrieved from a 32-bit CentOS 6.9 installation's
+# /usr/include/sys/personality.h header.
+LINUX_32 = 0x0008
+
+# Grab libc from our process
+process_namespace = ctypes.CDLL(None, use_errno=True)
+# int personality(unsigned long persona);
+__syscall_personality = process_namespace.personality
+__syscall_personality.argtypes = [ctypes.c_ulong]
+__syscall_personality.restype = ctypes.c_int
+
+
+def personality(persona):
+    """
+    Wrap behavior of personality(2).
+    """
+    set_persona = __syscall_personality(persona)
+    if set_persona == -1:
+        errno = ctypes.get_errno()
+        OSError(errno, os.strerror(errno))
+
+    return set_persona
+
+
+personality(LINUX_32)
+
+argv = sys.argv[1:]
+if not argv:
+    os.execlp("/bin/sh", "-sh")
+elif argv[0] in ('-h', '--h', '-help', '--help'):
+    print("""Usage:
+linux32 [<program> [<argument>...]]
+
+Change the reported architecture to 32 bits.
+THIS IS JUST A STUB FOR BOOTSTRAPPING!
+Please install utils-linux-ng for the real executable.
+""")
+elif argv[0].startswith('-'):
+    print("linux32: Unknown option")
+    print("linux32: Try `linux32 --help' for more information.")
+else:
+    os.execvp(argv[0], argv)

docker/build_scripts/manylinux1-check.py → docker/build_scripts/manylinux-check.py

@@ -1,6 +1,6 @@
 # Logic copied from PEP 513
 
-def is_manylinux1_compatible():
+def is_manylinux2010_compatible():
     # Only Linux, and only x86-64 / i686
     from distutils.util import get_platform
     if get_platform() not in ["linux-x86_64", "linux-i686"]:
@@ -14,8 +14,8 @@ def is_manylinux1_compatible():
         # Fall through to heuristic check below
         pass
 
-    # Check glibc version. CentOS 5 uses glibc 2.5.
-    return have_compatible_glibc(2, 5)
+    # Check glibc version. CentOS 6 uses glibc 2.12.
+    return have_compatible_glibc(2, 12)
 
 def have_compatible_glibc(major, minimum_minor):
     import ctypes
@@ -45,9 +45,9 @@ def have_compatible_glibc(major, minimum_minor):
     return True
 
 import sys
-if is_manylinux1_compatible():
-    print("%s is manylinux1 compatible" % (sys.executable,))
+if is_manylinux2010_compatible():
+    print("%s is manylinux2010 compatible" % (sys.executable,))
     sys.exit(0)
 else:
-    print("%s is NOT manylinux1 compatible" % (sys.executable,))
+    print("%s is NOT manylinux2010 compatible" % (sys.executable,))
     sys.exit(1)

docker/build_scripts/py36-requirements.txt

@@ -3,14 +3,14 @@
 certifi==2018.10.15 \
     --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
     --hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a
-auditwheel==1.10.0 \
-    --hash=sha256:185365e972342f9d6ecb412a21eb05c42fd0fff24031c4fb917ad0ac588c389d \
-    --hash=sha256:2f9c89782757099b85593513f0cfc46da038cb4a238829caca29d5372f83c339
+auditwheel==2.0rc1 \
+    --hash=sha256:372f691463c7bec3803a43e2f1fccf048ba5752e12b5a0bbf93e2d0fa0f4a9f5 \
+    --hash=sha256:d07d0907d282168a0468e6cf7b17cefb79115369052b6ced0982c95538592ee8
 # this package required for auditwheel
 pyelftools==0.25 \
     --hash=sha256:89c6da6f56280c37a5ff33468591ba9a124e17d71fe42de971818cbff46c1b24
 # this package required for auditwheel
 typing==3.6.6 \
     --hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
     --hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
-    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d
+    --hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d

docker/deploy.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 docker login -u $QUAY_USERNAME -p $QUAY_PASSWORD quay.io
-tag="quay.io/pypa/manylinux1_$PLATFORM"
+tag="quay.io/pypa/manylinux2010_$PLATFORM"
 docker tag ${tag}:${TRAVIS_COMMIT} ${tag}:latest
 docker push ${tag}:latest

docker/glibc/Dockerfile

@@ -0,0 +1,11 @@
+FROM centos:6 as centos-with-vsyscall
+
+COPY ./build_scripts /build_scripts
+RUN bash /build_scripts/rebuild-glibc-without-vsyscall.sh
+
+FROM centos:6
+LABEL maintainer="The Manylinux project"
+
+COPY --from=centos-with-vsyscall /rpms /rpms
+
+RUN yum -y install /rpms/* && rm -rf /rpms