From 4ffd49ba78fe050bd55a7898e5458156171eac3e Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Thu, 11 Aug 2022 14:47:50 -0700 Subject: [PATCH 1/9] Add support for doing manylinux(_2_28) builds. --- manylinux/README.md | 46 +++++++++++++++++++ manylinux/do-build.sh | 100 ++++++++++++++++++++++++++++++++++++++++++ manylinux/tasks.py | 59 +++++++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 manylinux/README.md create mode 100755 manylinux/do-build.sh create mode 100644 manylinux/tasks.py diff --git a/manylinux/README.md b/manylinux/README.md new file mode 100644 index 000000000..8fe2bff4d --- /dev/null +++ b/manylinux/README.md @@ -0,0 +1,46 @@ +Manylinux build scripts for wxPython +==================================== + +The code in this folder can be used to build a manylinux version of wxPython. +This means that a single wxPython wheel can be used on more than a single Linux +distro like what we've been limited to so far. This is accomplished by including +in the wheel file the shared libraries for all of the dependencies of wxWidgets. +This is primarily the GTK3 libs and all of its dependencies, although there are +some others too. + +In general this works, but it's not a perfect solution. Here are some potential +issues: + +* Since the wheel has its own instance of GTK and other dependent libraries, the + wheel file is quite large compared to one created specifically for the distro + where it will be deployed. (Around 2+ times larger.) + +* The private copy of the GTK libs will not play well with GTK themes and other + resources that are installed on the system. In fact, there may be warnings + printed indicating that "your installation may be broken." + +* In order to get new enough versions of wxWidgets' dependencies, we need to use + the newest (at the time of this writing) version of the manylinux images, + resulting in the platform tag being set to "manylinux_2_28". In other words, + the version of glibc on the target system needs to be >= 2.28, and the wheels + will not work on older systems. A newish version of Pip is also required, one + that recognizes this platform tag. + +How to build +------------ + +Building is simple. You need a system with docker available, and you need a +wxPython sdist to be present in the ../dist folder. This makes it easy to build +using the exact same source code that was used to build already released +versions of wxPython, and also saves the time needed for the code generation +steps. The Python invoke package is also needed, but if you used +../requirements.txt to install packages into your Python then you've already got +it. + +To start a build run a command like the following: + +``` +$ invoke build --pythons "3.8, 3.9, 3.10" +``` + +The resulting manylinux wheels will be moved to ../dist when they are done. diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh new file mode 100755 index 000000000..52628cadc --- /dev/null +++ b/manylinux/do-build.sh @@ -0,0 +1,100 @@ +#!/bin/bash +#-------------------------------------------------------------------------- +# This script is run inside the manylinux Docker container to perform the +# actual build of the Phoenix wheels. See tasks.py and README.md for details. +#-------------------------------------------------------------------------- +set -o errexit +set -o xtrace + +export PYTHONUNBUFFERED=1 + +VERSION=$1 +shift +PYTHONS="$@" +if [ -z $PYTHONS ]; then + PYTHONS=3.8 +fi + +echo VERSION: $VERSION +echo PYTHONS: $PYTHONS + + +function do_setup() { + # Install extra pacakges needed for the build + echo Installing Linux packages... + yum -y -q install \ + freeglut-devel \ + mesa-libGL-devel \ + mesa-libGLU-devel \ + gstreamer1-devel \ + gstreamer1-plugins-base-devel \ + gtk3-devel \ + libjpeg-devel \ + libnotify \ + libnotify-devel \ + libpng-devel \ + libSM-devel \ + libtiff-devel \ + libXtst-devel \ + SDL2-devel \ + webkit2gtk3-devel + + # Unpack the source archive + mkdir /build + cd /build + tar xzf /dist/wxPython-$VERSION.tar.gz +} + + +function do_build() { + # Remove the '.' from the version. IOW, 3.9 --> 39 + py=$(echo $1 | sed 's/\.//') + cd /build/wxPython-$VERSION + + # There's something odd about how the Pythons are set up on this image, + # and/or how virtual envs are created with these Pythons. When using a + # virtual env the python-config tools give the wrong paths for includes, + # libs, etc. Maybe it's too many layers of symlinks? Anyway, we need to use + # the real python binary instead of a virual env for our build because waf + # needs a working python-config. + PYTHON=/opt/python/cp$py-*/bin/python + OLD_PATH=$PATH + export PATH=$(dirname $PYTHON):$PATH + + # build + $PYTHON -m pip install -U -r requirements.txt + $PYTHON -m pip install -U pip setuptools wheel + $PYTHON build.py build_wx + $PYTHON build.py build_py + $PYTHON build.py bdist_wheel + + # do the manylinux magic + auditwheel show dist/wxPython-$VERSION-*linux*.whl + # auditwheel repair -w dist --strip dist/*.whl + auditwheel repair -w dist dist/*.whl + + # do a quickie smoke-test with a virtual env + mkdir tmp + $PYTHON -m venv tmp/test + tmp/test/bin/pip install -U pip + tmp/test/bin/pip install dist/wxPython-*manylinux* + tmp/test/bin/python -c "import wx; print(wx.version())" + rm -rf tmp + + # Save the manylinux wheel to the host folder + mv dist/wxPython-*-manylinux*.whl /dist + rm dist/wxPython-*.whl + export PATH=$OLD_PATH +} + + + +do_setup +for pyver in $PYTHONS; do + do_build $pyver +done + +if [ -n $INTERACTIVE ]; then + exec /bin/bash -i +fi + diff --git a/manylinux/tasks.py b/manylinux/tasks.py new file mode 100644 index 000000000..4c5e53187 --- /dev/null +++ b/manylinux/tasks.py @@ -0,0 +1,59 @@ +import os +import glob +import sys +from invoke import task, run + + +HERE = os.path.abspath(os.path.dirname(__file__)) +IMAGE = 'quay.io/pypa/manylinux_2_28_x86_64' +# TODO: Try the aarch64 image. +# Also, it may be worth trying again to get manylinux_2014 working. + +@task( + help={ + 'cmd': "If given will run this command instead of the image's default command.", + 'keep': "Keep the container when it exits. Otherwise it will be auto-removed." + }, +) +def run(ctx, cmd=None, keep=False, extra=''): + """ + Run the manylinux docker image. + """ + os.chdir(HERE) + dist = os.path.abspath('../dist') + cwd = os.getcwd() + cmd = '' if cmd is None else cmd + rm = '' if keep else '--rm' + ctx.run( + f'docker run -it {rm} -v {dist}:/dist -v {cwd}:/scripts {extra} {IMAGE} {cmd}', + pty=True, echo=True) + + +@task( + help={ + 'pythons': "Comma separated list of Python verions to build for.", + 'keep': "Keep the container when it exits. Otherwise it will be auto-removed.", + 'interactive': "Run a shell when the build script is done so the container can be examined." + } +) +def build(ctx, pythons='', keep=False, interactive=False): + """ + Run the build(s) + """ + # Ensure we've got a source archive available + source = glob.glob('../dist/wxPython-*.tar.gz') + if not source: + print('ERROR: no source archive found in ../dist') + sys.exit(1) + if len(source) > 1: + print('ERROR: Too many source archives found in ../dist') + sys.exit(1) + source = source[0] + version = source[17:-7] + + pythons = pythons.split(',') + pythons = ' '.join(pythons) + cmd = f'/scripts/do-build.sh {version} {pythons}' + extra = '-e INTERACTIVE=1' if interactive else '' + run(ctx, cmd, keep, extra) + From e57162d4a7601c47b6b3e34d10c4456df28a3fb5 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Thu, 11 Aug 2022 15:08:11 -0700 Subject: [PATCH 2/9] The --strip option breaks something, so don't bother. --- manylinux/do-build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh index 52628cadc..f6f39f223 100755 --- a/manylinux/do-build.sh +++ b/manylinux/do-build.sh @@ -4,7 +4,7 @@ # actual build of the Phoenix wheels. See tasks.py and README.md for details. #-------------------------------------------------------------------------- set -o errexit -set -o xtrace +#set -o xtrace export PYTHONUNBUFFERED=1 @@ -70,7 +70,6 @@ function do_build() { # do the manylinux magic auditwheel show dist/wxPython-$VERSION-*linux*.whl - # auditwheel repair -w dist --strip dist/*.whl auditwheel repair -w dist dist/*.whl # do a quickie smoke-test with a virtual env From 81dcdb7dc5d6b7ca88a8d9a0e8f3409dcc49d47c Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Fri, 12 Aug 2022 08:32:57 -0700 Subject: [PATCH 3/9] Be consistent about wildcards used for the wheel files --- manylinux/do-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh index f6f39f223..844b7ef57 100755 --- a/manylinux/do-build.sh +++ b/manylinux/do-build.sh @@ -11,7 +11,7 @@ export PYTHONUNBUFFERED=1 VERSION=$1 shift PYTHONS="$@" -if [ -z $PYTHONS ]; then +if [ "" == "$PYTHONS" ]; then PYTHONS=3.8 fi @@ -70,13 +70,13 @@ function do_build() { # do the manylinux magic auditwheel show dist/wxPython-$VERSION-*linux*.whl - auditwheel repair -w dist dist/*.whl + auditwheel repair -w dist dist/wxPython-$VERSION-*linux*.whl # do a quickie smoke-test with a virtual env mkdir tmp $PYTHON -m venv tmp/test tmp/test/bin/pip install -U pip - tmp/test/bin/pip install dist/wxPython-*manylinux* + tmp/test/bin/pip install dist/wxPython-*manylinux*.whl tmp/test/bin/python -c "import wx; print(wx.version())" rm -rf tmp From 23ed4b8096fd5eab49b92ee87180f207f1172c90 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Fri, 12 Aug 2022 08:33:50 -0700 Subject: [PATCH 4/9] Add support for specifying the architecture, and also using pythons="all" --- manylinux/tasks.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/manylinux/tasks.py b/manylinux/tasks.py index 4c5e53187..eb950825f 100644 --- a/manylinux/tasks.py +++ b/manylinux/tasks.py @@ -3,19 +3,19 @@ import sys from invoke import task, run - HERE = os.path.abspath(os.path.dirname(__file__)) -IMAGE = 'quay.io/pypa/manylinux_2_28_x86_64' -# TODO: Try the aarch64 image. -# Also, it may be worth trying again to get manylinux_2014 working. +IMAGE = 'quay.io/pypa/manylinux_2_28_{}' + @task( help={ 'cmd': "If given will run this command instead of the image's default command.", - 'keep': "Keep the container when it exits. Otherwise it will be auto-removed." + 'keep': "Keep the container when it exits. Otherwise it will be auto-removed.", + 'extra': "Extra flags to pass to the docker command line.", + 'arch': 'The architecture to build for. "x86_64" (default) or "aarch64".' }, ) -def run(ctx, cmd=None, keep=False, extra=''): +def run(ctx, cmd=None, keep=False, extra='', arch='x86_64'): """ Run the manylinux docker image. """ @@ -24,8 +24,9 @@ def run(ctx, cmd=None, keep=False, extra=''): cwd = os.getcwd() cmd = '' if cmd is None else cmd rm = '' if keep else '--rm' + image = IMAGE.format(arch) ctx.run( - f'docker run -it {rm} -v {dist}:/dist -v {cwd}:/scripts {extra} {IMAGE} {cmd}', + f'docker run -it {rm} -v {dist}:/dist -v {cwd}:/scripts {extra} {image} {cmd}', pty=True, echo=True) @@ -33,10 +34,11 @@ def run(ctx, cmd=None, keep=False, extra=''): help={ 'pythons': "Comma separated list of Python verions to build for.", 'keep': "Keep the container when it exits. Otherwise it will be auto-removed.", - 'interactive': "Run a shell when the build script is done so the container can be examined." + 'interactive': "Run a shell when the build script is done so the container can be examined.", + 'arch': 'The architecture to build for. "x86_64" (default) or "aarch64".' } ) -def build(ctx, pythons='', keep=False, interactive=False): +def build(ctx, pythons='', keep=False, interactive=False, arch='x86_64'): """ Run the build(s) """ @@ -50,10 +52,14 @@ def build(ctx, pythons='', keep=False, interactive=False): sys.exit(1) source = source[0] version = source[17:-7] + if pythons == 'all': + pythons = '3.8, 3.9, 3.10' + if pythons == '': + pythons = '3.8' pythons = pythons.split(',') pythons = ' '.join(pythons) cmd = f'/scripts/do-build.sh {version} {pythons}' extra = '-e INTERACTIVE=1' if interactive else '' - run(ctx, cmd, keep, extra) + run(ctx, cmd, keep, extra, arch) From b4e908331a023b59e0d4bb38100a022a6b3a8cab Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Fri, 12 Aug 2022 10:02:04 -0700 Subject: [PATCH 5/9] Better test for interactive mode --- manylinux/do-build.sh | 2 +- manylinux/tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh index 844b7ef57..0dfdcf0b3 100755 --- a/manylinux/do-build.sh +++ b/manylinux/do-build.sh @@ -93,7 +93,7 @@ for pyver in $PYTHONS; do do_build $pyver done -if [ -n $INTERACTIVE ]; then +if [ "$INTERACTIVE" == "yes" ]; then exec /bin/bash -i fi diff --git a/manylinux/tasks.py b/manylinux/tasks.py index eb950825f..2870c5125 100644 --- a/manylinux/tasks.py +++ b/manylinux/tasks.py @@ -60,6 +60,6 @@ def build(ctx, pythons='', keep=False, interactive=False, arch='x86_64'): pythons = pythons.split(',') pythons = ' '.join(pythons) cmd = f'/scripts/do-build.sh {version} {pythons}' - extra = '-e INTERACTIVE=1' if interactive else '' + extra = '-e INTERACTIVE=yes' if interactive else '' run(ctx, cmd, keep, extra, arch) From ceed71747a83013584960206093ddf0182a06d29 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Fri, 12 Aug 2022 14:52:22 -0700 Subject: [PATCH 6/9] Correction in the README --- manylinux/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/manylinux/README.md b/manylinux/README.md index 8fe2bff4d..688107345 100644 --- a/manylinux/README.md +++ b/manylinux/README.md @@ -33,9 +33,7 @@ Building is simple. You need a system with docker available, and you need a wxPython sdist to be present in the ../dist folder. This makes it easy to build using the exact same source code that was used to build already released versions of wxPython, and also saves the time needed for the code generation -steps. The Python invoke package is also needed, but if you used -../requirements.txt to install packages into your Python then you've already got -it. +steps. The Python invoke package is also needed. To start a build run a command like the following: From ad3e54e4751d1939c12908c956b71149c44bf513 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Sat, 13 Aug 2022 12:50:48 -0700 Subject: [PATCH 7/9] Clean up between builds --- manylinux/do-build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh index 0dfdcf0b3..5e8707937 100755 --- a/manylinux/do-build.sh +++ b/manylinux/do-build.sh @@ -84,6 +84,11 @@ function do_build() { mv dist/wxPython-*-manylinux*.whl /dist rm dist/wxPython-*.whl export PATH=$OLD_PATH + + # Clean up the Python parts of this build, since we can do more than one + # build per invocation of the docker image. + rm -rf build/waf + rm -f wx/* || true } From 3d722a809c00facfc1f3f8d04d7137827632ce96 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Sat, 13 Aug 2022 13:45:30 -0700 Subject: [PATCH 8/9] Remove just the *.so files and __pycache__ --- manylinux/do-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manylinux/do-build.sh b/manylinux/do-build.sh index 5e8707937..9d0883172 100755 --- a/manylinux/do-build.sh +++ b/manylinux/do-build.sh @@ -87,8 +87,8 @@ function do_build() { # Clean up the Python parts of this build, since we can do more than one # build per invocation of the docker image. - rm -rf build/waf - rm -f wx/* || true + rm -rf build/waf wx/__pycache__ + rm -f wx/*.so || true } From fd83906a7a4d4b30b64bf1c5689907bacdb57943 Mon Sep 17 00:00:00 2001 From: Robin Dunn Date: Tue, 30 Aug 2022 18:36:56 -0700 Subject: [PATCH 9/9] fix item numbers --- packaging/HOWTO-Release.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packaging/HOWTO-Release.rst b/packaging/HOWTO-Release.rst index 6361f280b..58acb3f5d 100644 --- a/packaging/HOWTO-Release.rst +++ b/packaging/HOWTO-Release.rst @@ -80,15 +80,18 @@ HOWTO Release wxPython Phoenix 17. Tag the released revision in git, using a name like wxPython-4.0.0 (using the actual version number of course.) Push the tag to all remotes. -18. Bump the version numbers in buildtools/version.py appropriately for the +18. Create a release at GitHub. Copy the text about the "Source Code" downloads + from the previous release, and upload the official source tarball. + +19. Bump the version numbers in buildtools/version.py appropriately for the next anticipated release, so future snapshot builds will be recognized as pre-release development versions for the next official release, not the one just completed. -19. If making an announcement about this release, (I think it's okay not to +20. If making an announcement about this release, (I think it's okay not to for minor releases or smallish bug fixes,) send the text in packaging/ANNOUNCE.txt to the email addresses listed at the top of the file. -20. Add a news post to the wxPython site about the release. +21. Add a news post to the wxPython site about the release.