From 7dc500f6f9b00d336a0afb6aac12c75e0f042098 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 14:01:39 +0300 Subject: [PATCH 01/38] Use latest pythia version. --- brink.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brink.conf b/brink.conf index c1ab35c7..cd00a607 100644 --- a/brink.conf +++ b/brink.conf @@ -1,5 +1,5 @@ BASE_REQUIREMENTS='chevah-brink==1.0.2 paver==1.2.4' -PYTHON_CONFIGURATION='default@3.8.6.31955c7' +PYTHON_CONFIGURATION='default@3.8.6.3b1a8ba' # For production packages there are 2 options: BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' #BINARY_DIST_URI='https://bin.chevah.com:20443/production' From 95f797d9551bc7404c040996587f6f1ffee9e7a9 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 14:23:42 +0300 Subject: [PATCH 02/38] Forward-port python-package brink.{conf,sh} updates. --- .github/workflows/bare.yaml | 2 +- brink.conf | 8 ++++---- brink.sh | 19 ++++++++++++------- pythia.conf | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 8eb9b2a8..fca3f6aa 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -54,7 +54,7 @@ jobs: # Workflow won't be cancelled at the first failed job. fail-fast: false matrix: - runs-on: [ windows-2019, windows-2016 ] + runs-on: [ windows-2022, windows-2019 ] timeout-minutes: 20 # Steps represent a sequence of tasks that will be executed as part of the job diff --git a/brink.conf b/brink.conf index cd00a607..abff3784 100644 --- a/brink.conf +++ b/brink.conf @@ -1,13 +1,13 @@ -BASE_REQUIREMENTS='chevah-brink==1.0.2 paver==1.2.4' +BASE_REQUIREMENTS='chevah-brink==1.0.6 paver==1.2.4' PYTHON_CONFIGURATION='default@3.8.6.3b1a8ba' # For production packages there are 2 options: BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' #BINARY_DIST_URI='https://bin.chevah.com:20443/production' # For testing packages, make sure this one is the last uncommented instance: #BINARY_DIST_URI='https://bin.chevah.com:20443/testing' -PIP_INDEX='https://pypi.chevah.com/simple' +PIP_INDEX_URL='https://pypi.chevah.com/simple' # There are 2 build directories used in this repo: -# * $BUILD_DIR (set as 'build') is used for building this Python package. -# * $CHEVAH_BUILD_DIR is used for Buildbot's Python when building through it. +# * $BUILD_DIR is used for building libffi / OpenSSL / Python / etc. +# * $CHEVAH_BUILD_DIR is used by the Python that builds the above. CHEVAH_BUILD_DIR='build-pythia' diff --git a/brink.sh b/brink.sh index b67946a3..c5a0f3f3 100755 --- a/brink.sh +++ b/brink.sh @@ -87,7 +87,7 @@ PYTHON_VERSION='not.defined.yet' PYTHON_PLATFORM='unknown-os-and-arch' PYTHON_NAME='python3.8' BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' -PIP_INDEX='https://pypi.chevah.com' +PIP_INDEX_URL='https://pypi.org/simple' BASE_REQUIREMENTS='' # @@ -256,6 +256,7 @@ update_path_variables() { export CHEVAH_OS=${OS} export CHEVAH_ARCH=${ARCH} export CHEVAH_CACHE=${CACHE_FOLDER} + export PIP_INDEX_URL=${PIP_INDEX_URL} } @@ -307,11 +308,15 @@ pip_install() { echo "::group::pip install $1" set +e + # There is a bug in pip/setuptools when using custom build folders. + # See https://github.com/pypa/pip/issues/3564 + rm -rf ${BUILD_FOLDER}/pip-build ${PYTHON_BIN} -m \ pip install \ - --trusted-host pypi.chevah.com \ - --trusted-host deag.chevah.com \ - --index-url=$PIP_INDEX \ + --trusted-host bin.chevah.com \ + --trusted-host pypi-internal.chevah.com \ + --index-url=$PIP_INDEX_URL \ + --build=${BUILD_FOLDER}/pip-build \ $1 exit_code=$? @@ -531,7 +536,7 @@ install_dependencies(){ # If it's too old, exit with a nice informative message. # If it's supported, return through eval the version numbers to be used for # naming the package, for example: '8' for RHEL 8.2, '2004' for Ubuntu 20.04, -# '312' for Alpine Linux 3.12, '114' for Solaris 11.4. +# '314' for Alpine Linux 3.14, '114' for Solaris 11.4. # check_os_version() { # First parameter should be the human-readable name for the current OS. @@ -796,11 +801,11 @@ detect_os() { ARCH="x64" case "$OS" in win) - # 32bit build on Windows 2016, 64bit otherwise. + # 32bit build on Windows 2019, 64bit otherwise. # Should work with a l10n pack too (tested with French). win_ver=$(systeminfo.exe | head -n 3 | tail -n 1 \ | cut -d ":" -f 2) - if [[ "$win_ver" =~ "Microsoft Windows Server 2016" ]]; then + if [[ "$win_ver" =~ "Microsoft Windows Server 2019" ]]; then ARCH="x86" fi ;; diff --git a/pythia.conf b/pythia.conf index ae2d5db5..a1fc4726 100644 --- a/pythia.conf +++ b/pythia.conf @@ -51,7 +51,7 @@ PIP_LIBRARIES="\ " # Arguments that are sent when using pip. PIP_ARGS="\ - --index-url=https://pypi.chevah.com/simple \ + --index-url=https://bin.chevah.com:20443/pypi/simple \ --trusted-host=pypi.chevah.com \ --no-warn-script-location \ " From b2792937f823235d33859300bc961bf6a91ebb8d Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 14:23:54 +0300 Subject: [PATCH 03/38] Use latest zlib. --- pythia.conf | 2 +- src/zlib/sha512.sum | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythia.conf b/pythia.conf index a1fc4726..e329c722 100644 --- a/pythia.conf +++ b/pythia.conf @@ -7,7 +7,7 @@ DIST_DIR="dist" # Python and lib versions. PYTHON_BUILD_VERSION="3.8.6" LIBFFI_VERSION="3.3" -ZLIB_VERSION="1.2.11" +ZLIB_VERSION="1.2.12" BZIP2_VERSION="1.0.8" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. diff --git a/src/zlib/sha512.sum b/src/zlib/sha512.sum index c8ece673..8bfac5a4 100644 --- a/src/zlib/sha512.sum +++ b/src/zlib/sha512.sum @@ -1 +1 @@ -73fd3fff4adeccd4894084c15ddac89890cd10ef105dd5e1835e1e9bbb6a49ff229713bd197d203edfa17c2727700fce65a2a235f07568212d820dca88b528ae zlib-1.2.11.tar.gz +cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14 zlib-1.2.12.tar.gz From 34bea98918758459b2284136318e0cec67cbaacf Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 14:25:57 +0300 Subject: [PATCH 04/38] Use latest libffi. --- pythia.conf | 2 +- src/libffi/sha512.sum | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythia.conf b/pythia.conf index e329c722..0159a594 100644 --- a/pythia.conf +++ b/pythia.conf @@ -6,7 +6,7 @@ DIST_DIR="dist" # Python and lib versions. PYTHON_BUILD_VERSION="3.8.6" -LIBFFI_VERSION="3.3" +LIBFFI_VERSION="3.4.2" ZLIB_VERSION="1.2.12" BZIP2_VERSION="1.0.8" # Statically build the BSD libedit on selected platforms to get the diff --git a/src/libffi/sha512.sum b/src/libffi/sha512.sum index 497e3960..344aeb00 100644 --- a/src/libffi/sha512.sum +++ b/src/libffi/sha512.sum @@ -1 +1 @@ -61513801a156f11420f541d325de697131846487122d6bdcf5491b18b4da788589f5c0bb07e88e396495d3be5830d74e9135595e2b8ddbfe95c448d8597fbd6f libffi-3.3.tar.gz +31bad35251bf5c0adb998c88ff065085ca6105cf22071b9bd4b5d5d69db4fadf16cadeec9baca944c4bb97b619b035bb8279de8794b922531fddeb0779eb7fb1 libffi-3.4.2.tar.gz From ceb021b11845074714f6d0c13d47b03d6ac007f8 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 14:42:40 +0300 Subject: [PATCH 05/38] Try using latest available Python 3.8.x packages. --- pythia | 2 ++ pythia.conf | 2 +- src/Python-Windows/sha512.sum | 4 ++-- src/Python/sha512.sum | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pythia b/pythia index cc90f807..eacfec64 100755 --- a/pythia +++ b/pythia @@ -144,6 +144,8 @@ build_dep() { # This builds Python itself. build_python() { if [ $OS = "win" ]; then + # Latest 3.8.x version with Windows packages. + export PYTHON_BUILD_VERSION="3.8.10" # Python "build" is a very special case under Windows. execute pushd src/Python-Windows execute ./chevahbs Python $PYTHON_BUILD_VERSION $INSTALL_DIR diff --git a/pythia.conf b/pythia.conf index 0159a594..89e0c44b 100644 --- a/pythia.conf +++ b/pythia.conf @@ -5,7 +5,7 @@ BUILD_DIR="build" DIST_DIR="dist" # Python and lib versions. -PYTHON_BUILD_VERSION="3.8.6" +PYTHON_BUILD_VERSION="3.8.13" LIBFFI_VERSION="3.4.2" ZLIB_VERSION="1.2.12" BZIP2_VERSION="1.0.8" diff --git a/src/Python-Windows/sha512.sum b/src/Python-Windows/sha512.sum index 7a9cb9c9..270e2aca 100644 --- a/src/Python-Windows/sha512.sum +++ b/src/Python-Windows/sha512.sum @@ -1,2 +1,2 @@ -3b463ef64397326ffd5f20009be6c1304a3bd8646e5bf05c0a8be5ed4aabe7b0a5a6133e962f538292e6290a04fbe07b1caa6c09c54d012b0d07b7f025772921 *Python-3.8.6.amd64.zip -fb270ce1f1d0bcb95e6c192a35dda276005f74e3c9d89a3a8f799296e9a1db925155437be977a3f3181c25eb522025895563d2333bc6606353bc024b51379107 *Python-3.8.6.win32.zip +86e55911be78205a61f886feff2195c78a6f158a760cc1697ce4340dcb5ca118360251de2f707b6d2a78b7469d92c87b045b7326d6f194bfa92e665af1cd55a5 *Python-3.8.10.amd64.zip +a6f0c35ba37c07c6e8815fb43c20095541117f3b6cc034b8ef0acfc908de8951bdffa38706bac802f820290b39ae84f934f27a8e32f548735f470277f7a70550 *Python-3.8.10.win32.zip diff --git a/src/Python/sha512.sum b/src/Python/sha512.sum index 9a35875e..82b5f4d9 100644 --- a/src/Python/sha512.sum +++ b/src/Python/sha512.sum @@ -1 +1 @@ -9479a1bed110e4caab0496832c2c40574306c961059de96a54dc93911791b52d949fbdb0b3d0d0a13ab7025e1d823b40ff208a259c310e7013fad1ebfab00f6e Python-3.8.6.tgz +e57f5f5b441e46a742b0147dd7fbfa6b52d550a86e60c9765ecc3c4690e2cdedf197e151c07cd2ea1f75ed9022a2b8ce4850c3d65916eaede1db6feed40b52f6 Python-3.8.13.tgz From d8c718fdd8226b999d76c324078f7cf770e80ffb Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 15:11:25 +0300 Subject: [PATCH 06/38] Fixed downloading latest libffi. --- os_quirks.sh | 2 +- pythia | 2 +- src/libffi/chevahbs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/os_quirks.sh b/os_quirks.sh index 7a023b30..9b320f3f 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -35,7 +35,7 @@ case $OS in # To avoid linking to local libs, build own OpenSSL libs. export BUILD_OPENSSL="yes" # Generic builds on CentOS 5 have issues with Let's Encrypt certs. - export GET_CMD="curl --silent --insecure --output" + export GET_CMD="curl --silent --insecure --location --output" ;; macos) export CC="clang" diff --git a/pythia b/pythia index eacfec64..c5bd8b02 100755 --- a/pythia +++ b/pythia @@ -45,7 +45,7 @@ export CC="gcc" export MAKE="make" # $GET_CMD must save to custom filename, which must be appended before the link. # E.g., to use wget, GET_CMD should be "wget --quiet -O". -export GET_CMD="curl --silent --output" +export GET_CMD="curl --silent --location --output" export SHA_CMD="sha512sum --check --status --warn" export TAR_CMD="tar xfz" export ZIP_CMD="unzip -q" diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index 00b8c3ba..a7acc11d 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -12,7 +12,7 @@ chevahbs_getsources() { local name=$1 local ver=$2 local ext="tar.gz" - local link=ftp://sourceware.org/pub/libffi/"$name"-"$ver"."$ext" + local link=https://github.com/libffi/libffi/releases/download/v"$ver"/"$name"-"$ver"."$ext" download_sources $name $ver $link $ext } From a234311fb8046fa7083361be587a2c0453863c26 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 15:20:12 +0300 Subject: [PATCH 07/38] Use latest OpenSSL when building. --- pythia.conf | 2 +- src/chevah-python-test/test_python_binary_dist.py | 5 ++++- src/openssl/sha512.sum | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pythia.conf b/pythia.conf index 89e0c44b..5d6776eb 100644 --- a/pythia.conf +++ b/pythia.conf @@ -12,7 +12,7 @@ BZIP2_VERSION="1.0.8" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. LIBEDIT_VERSION="20170329-3.1" -OPENSSL_VERSION="1.1.1k" +OPENSSL_VERSION="1.1.1n" # To match the unusual naming scheme for SQLite downloads, eliminate dots from # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". # When updating this, also update the year in src/sqlite/chevahbs, if needed. diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 974f0bc3..da9bbe2f 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -390,9 +390,12 @@ def main(): # On RHEL 8.3, OpenSSL got updated to 1.1.1g. To keep backward # compatibility, link to version 1.1.1c from CentOS 8.2.2004. expecting = u'OpenSSL 1.1.1c FIPS 28 May 2019' + elif CHEVAH_OS == "win": + # Latest Python 3.8 Win packages are left behind. + expecting = u'OpenSSL 1.1.1k 25 Mar 2021' else: # Use latest OpenSSL version when building it from source. - expecting = u'OpenSSL 1.1.1k 25 Mar 2021' + expecting = u'OpenSSL 1.1.1n 15 Mar 2022' if openssl_version != expecting: sys.stderr.write('Expecting %s, got %s.\n' % ( expecting, openssl_version)) diff --git a/src/openssl/sha512.sum b/src/openssl/sha512.sum index 5aba0b90..c23b7db0 100644 --- a/src/openssl/sha512.sum +++ b/src/openssl/sha512.sum @@ -1 +1 @@ -73cd042d4056585e5a9dd7ab68e7c7310a3a4c783eafa07ab0b560e7462b924e4376436a6d38a155c687f6942a881cfc0c1b9394afcde1d8c46bf396e7d51121 openssl-1.1.1k.tar.gz +1937796736613dcf4105a54e42ecb61f95a1cea74677156f9459aea0f2c95159359e766089632bf364ee6b0d28d661eb9957bce8fecc9d2436378d8d79e8d0a4 openssl-1.1.1n.tar.gz From 8dafba782513143787286bcff80be46beb9e627e Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 16:41:26 +0300 Subject: [PATCH 08/38] Back to tried and true Python 3.8.6. This reverts commit ceb021b11845074714f6d0c13d47b03d6ac007f8. --- pythia | 2 -- pythia.conf | 2 +- src/Python-Windows/sha512.sum | 4 ++-- src/Python/sha512.sum | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pythia b/pythia index c5bd8b02..48dee09b 100755 --- a/pythia +++ b/pythia @@ -144,8 +144,6 @@ build_dep() { # This builds Python itself. build_python() { if [ $OS = "win" ]; then - # Latest 3.8.x version with Windows packages. - export PYTHON_BUILD_VERSION="3.8.10" # Python "build" is a very special case under Windows. execute pushd src/Python-Windows execute ./chevahbs Python $PYTHON_BUILD_VERSION $INSTALL_DIR diff --git a/pythia.conf b/pythia.conf index 5d6776eb..1df2a037 100644 --- a/pythia.conf +++ b/pythia.conf @@ -5,7 +5,7 @@ BUILD_DIR="build" DIST_DIR="dist" # Python and lib versions. -PYTHON_BUILD_VERSION="3.8.13" +PYTHON_BUILD_VERSION="3.8.6" LIBFFI_VERSION="3.4.2" ZLIB_VERSION="1.2.12" BZIP2_VERSION="1.0.8" diff --git a/src/Python-Windows/sha512.sum b/src/Python-Windows/sha512.sum index 270e2aca..7a9cb9c9 100644 --- a/src/Python-Windows/sha512.sum +++ b/src/Python-Windows/sha512.sum @@ -1,2 +1,2 @@ -86e55911be78205a61f886feff2195c78a6f158a760cc1697ce4340dcb5ca118360251de2f707b6d2a78b7469d92c87b045b7326d6f194bfa92e665af1cd55a5 *Python-3.8.10.amd64.zip -a6f0c35ba37c07c6e8815fb43c20095541117f3b6cc034b8ef0acfc908de8951bdffa38706bac802f820290b39ae84f934f27a8e32f548735f470277f7a70550 *Python-3.8.10.win32.zip +3b463ef64397326ffd5f20009be6c1304a3bd8646e5bf05c0a8be5ed4aabe7b0a5a6133e962f538292e6290a04fbe07b1caa6c09c54d012b0d07b7f025772921 *Python-3.8.6.amd64.zip +fb270ce1f1d0bcb95e6c192a35dda276005f74e3c9d89a3a8f799296e9a1db925155437be977a3f3181c25eb522025895563d2333bc6606353bc024b51379107 *Python-3.8.6.win32.zip diff --git a/src/Python/sha512.sum b/src/Python/sha512.sum index 82b5f4d9..9a35875e 100644 --- a/src/Python/sha512.sum +++ b/src/Python/sha512.sum @@ -1 +1 @@ -e57f5f5b441e46a742b0147dd7fbfa6b52d550a86e60c9765ecc3c4690e2cdedf197e151c07cd2ea1f75ed9022a2b8ce4850c3d65916eaede1db6feed40b52f6 Python-3.8.13.tgz +9479a1bed110e4caab0496832c2c40574306c961059de96a54dc93911791b52d949fbdb0b3d0d0a13ab7025e1d823b40ff208a259c310e7013fad1ebfab00f6e Python-3.8.6.tgz From a23f1d4f2fcfc9d381d5af207a1d163dde4b33c5 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 17:36:25 +0300 Subject: [PATCH 09/38] Use our own CentOS 5.11 Docker image. --- .github/workflows/docker.yaml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 8514a766..22332338 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -30,7 +30,9 @@ jobs: strategy: fail-fast: false matrix: - container: [ 'alpine:3.14', 'centos:8.2.2004', 'centos:5.11' ] + # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the + # Overview section at https://hub.docker.com/r/proatria/centos for details. + container: [ 'alpine:3.14', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ] timeout-minutes: 30 steps: @@ -50,23 +52,6 @@ jobs: yum -y upgrade yum -y install git curl openssh-clients - # Final CentOS 5 version is used to build the generic Linux package. - - name: CentOS 5.11 setup - if: matrix.container == 'centos:5.11' - run: | - sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo - sed -i s@^#baseurl=http://mirror.centos.org/centos/\$releasever/@baseurl=http://vault.centos.org/5.11/@ /etc/yum.repos.d/*.repo - yum -y upgrade - # Use http://bin.chevah.com:20443/third-party-stuff/centos5/tuxad/ - # when tuxad.de dissapears, it has the minimum required stuff. - rpm -i http://www.tuxad.de/rpms/tuxad-release-5-1.noarch.rpm - yum -y install curl openssh-clients gcc44 make m4 patch unzip wget - ln -s /usr/bin/gcc44 /usr/local/bin/gcc - wget --mirror --no-parent https://bin.chevah.com:20443/third-party-stuff/centos5/endpoint/ - cd bin.chevah.com:20443/third-party-stuff/centos5/endpoint/ - rpm -i local-perl-*.rpm - rpm -i --nodeps git{-core,}-2.5.0-1.ep.x86_64.rpm - - name: Clone repo independently run: | git clone https://github.com/chevah/pythia.git From dfd7214c24c5cc46b00b3fd19947e0c7a24765c6 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 17:49:23 +0300 Subject: [PATCH 10/38] Also install unzip on CentOS 5. --- .github/workflows/docker.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 22332338..1893eaea 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -52,6 +52,14 @@ jobs: yum -y upgrade yum -y install git curl openssh-clients + # Our Chevah Centos 5 image, built for python-package. + # FIXME: Update it to include unzip. + - name: CentOS 5.11 setup + if: matrix.container == 'centos:5.11' + run: | + wget https://vault.centos.org/5.11/os/x86_64/CentOS/unzip-5.52-3.el5.x86_64.rpm + rpm -i unzip-5.52-3.el5.x86_64.rpm + - name: Clone repo independently run: | git clone https://github.com/chevah/pythia.git From 281092d5109f8431567caf50ee4f533ce7b83fa9 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 25 Apr 2022 17:58:49 +0300 Subject: [PATCH 11/38] Updated condition to match CentOS 5 container name. --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 1893eaea..27be8382 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -55,7 +55,7 @@ jobs: # Our Chevah Centos 5 image, built for python-package. # FIXME: Update it to include unzip. - name: CentOS 5.11 setup - if: matrix.container == 'centos:5.11' + if: matrix.container == 'proatria/centos:5.11-chevah1' run: | wget https://vault.centos.org/5.11/os/x86_64/CentOS/unzip-5.52-3.el5.x86_64.rpm rpm -i unzip-5.52-3.el5.x86_64.rpm From d0a6e970e831074a2170e1feff3f7db2ac095e35 Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 26 Apr 2022 12:40:05 +0300 Subject: [PATCH 12/38] Grand rename of the brink and build scripts. --- README.rst | 10 ++-- brink.conf | 13 ----- build.conf | 57 ++++++++++++++++++ pythia => build.sh | 20 +++---- functions_pythia.sh => functions_build.sh | 4 +- publish_dist.sh | 2 +- pythia.conf | 70 +++++------------------ brink.sh => pythia.sh | 20 +++---- src/Python-Windows/chevahbs | 2 +- src/Python/chevahbs | 2 +- src/bzip2/chevahbs | 2 +- src/libedit/chevahbs | 2 +- src/libffi/chevahbs | 2 +- src/openssl/chevahbs | 2 +- src/sqlite-autoconf/chevahbs | 2 +- src/zlib/chevahbs | 2 +- 16 files changed, 106 insertions(+), 106 deletions(-) delete mode 100644 brink.conf create mode 100644 build.conf rename pythia => build.sh (96%) rename functions_pythia.sh => functions_build.sh (98%) rename brink.sh => pythia.sh (97%) diff --git a/README.rst b/README.rst index 1d601044..16d2652e 100644 --- a/README.rst +++ b/README.rst @@ -6,16 +6,16 @@ A derivative of https://github.com/chevah/python-package/. Building: -* ``./pythia build`` +* ``./build.sh build`` Testing: -* ``./pythia test`` -* ``./pythia compat`` +* ``./build.sh test`` +* ``./build.sh compat`` -Use ``./pythia help`` to discover all available commands. +Use ``./build.sh help`` to discover all available commands. -Note that compat tests are currently only working with the ``python2.7`` branch. +Note that compat tests are currently only working on the ``python2.7`` branch. Supported platforms diff --git a/brink.conf b/brink.conf deleted file mode 100644 index abff3784..00000000 --- a/brink.conf +++ /dev/null @@ -1,13 +0,0 @@ -BASE_REQUIREMENTS='chevah-brink==1.0.6 paver==1.2.4' -PYTHON_CONFIGURATION='default@3.8.6.3b1a8ba' -# For production packages there are 2 options: -BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' -#BINARY_DIST_URI='https://bin.chevah.com:20443/production' -# For testing packages, make sure this one is the last uncommented instance: -#BINARY_DIST_URI='https://bin.chevah.com:20443/testing' -PIP_INDEX_URL='https://pypi.chevah.com/simple' - -# There are 2 build directories used in this repo: -# * $BUILD_DIR is used for building libffi / OpenSSL / Python / etc. -# * $CHEVAH_BUILD_DIR is used by the Python that builds the above. -CHEVAH_BUILD_DIR='build-pythia' diff --git a/build.conf b/build.conf new file mode 100644 index 00000000..1df2a037 --- /dev/null +++ b/build.conf @@ -0,0 +1,57 @@ +# Configuration file for Pythia + +# Sub-directories for building from sources and for the final distributable. +BUILD_DIR="build" +DIST_DIR="dist" + +# Python and lib versions. +PYTHON_BUILD_VERSION="3.8.6" +LIBFFI_VERSION="3.4.2" +ZLIB_VERSION="1.2.12" +BZIP2_VERSION="1.0.8" +# Statically build the BSD libedit on selected platforms to get the +# readline module available without linking to the GPL-only readline libs. +LIBEDIT_VERSION="20170329-3.1" +OPENSSL_VERSION="1.1.1n" +# To match the unusual naming scheme for SQLite downloads, eliminate dots from +# the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". +# When updating this, also update the year in src/sqlite/chevahbs, if needed. +SQLITE_VERSION="3330000" +# These are used by get-pip.py. +PIP_VERSION="21.1.1" +SETUPTOOLS_VERSION="53.0.0" + +# Python modules versions to be used everywhere possible. +CFFI_VERSION="1.14.3" +CRYPTOGRAPHY_VERSION="3.4.7" +SUBPROCESS32_VERSION="3.5.4" +BCRYPT_VERSION="3.1.7" +PSUTIL_VERSION="5.7.3" +SETPROCTITLE_VERSION="1.1.10" + +# pycparser is explicitly installed to work around setuptools auto dependencies. +PYCPARSER_VERSION="2.20" + +# Global flags for building required libs. +BUILD_LIBFFI="no" +BUILD_ZLIB="no" +BUILD_BZIP2="yes" +BUILD_LIBEDIT="yes" +BUILD_OPENSSL="no" +BUILD_SQLITE="yes" + +# List of default Python modules installed using pip. +PIP_LIBRARIES="\ + cffi==${CFFI_VERSION} \ + cryptography==${CRYPTOGRAPHY_VERSION} \ + subprocess32==${SUBPROCESS32_VERSION} \ + bcrypt==${BCRYPT_VERSION} \ + psutil==${PSUTIL_VERSION} \ + setproctitle==${SETPROCTITLE_VERSION} + " +# Arguments that are sent when using pip. +PIP_ARGS="\ + --index-url=https://bin.chevah.com:20443/pypi/simple \ + --trusted-host=pypi.chevah.com \ + --no-warn-script-location \ + " diff --git a/pythia b/build.sh similarity index 96% rename from pythia rename to build.sh index 48dee09b..9f2bc710 100755 --- a/pythia +++ b/build.sh @@ -3,11 +3,11 @@ # Pythia's script for building Python. # Set versions for the software to be built and other defaults. -source pythia.conf +source build.conf # Import shared and specific code. source ./functions.sh -source ./functions_pythia.sh +source ./functions_build.sh # Git revision to inject into Python's sys.version string through chevahbs # on non-Windows platforms. Also used for compat tests and archived in the dist. @@ -21,13 +21,13 @@ export BUILD_ZLIB BUILD_BZIP2 BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL # OS detection is slow on Windows, only execute it when the file is missing. if [ ! -r ./BUILD_ENV_VARS ]; then - execute ./brink.sh detect_os + execute ./pythia.sh detect_os fi -# Import build env vars as set by brink.sh. +# Import build env vars as set by pythia.sh. source ./BUILD_ENV_VARS # On Unix, use $ARCH to choose between 32bit or 64bit packages. It's possible -# to force a 32bit build on a 64bit machine, e.g. by setting ARCH in brink.sh +# to force a 32bit build on a 64bit machine, e.g. by setting ARCH in pythia.sh # as "x86" instead of "x64" for a certain platform. # $ARCH is also used when "building" Python on Windows and for testing. # $OS is used when patching/configuring/building/testing. @@ -243,17 +243,17 @@ command_compat() { execute rm -rf compat/ execute git clone https://github.com/chevah/compat.git --depth=1 -b master execute pushd compat - # Copy over current brink stuff, as some changes might require it. - execute cp ../../brink.{conf,sh} ./ + # Copy over current pythia stuff, as some changes might require it. + execute cp ../../pythia.{conf,sh} ./ # Patch compat to use the newly-built Python, then copy it to cache/. - echo -e "\nPYTHON_CONFIGURATION=default@${new_python_conf}" >> brink.conf + echo -e "\nPYTHON_CONFIGURATION=default@${new_python_conf}" >> pythia.conf execute mkdir cache execute cp -r ../"$PYTHON_BUILD_DIR" cache/ # Make sure everything is done from scratch in the current dir. unset CHEVAH_CACHE CHEVAH_BUILD # Some tests might still fail due to causes not related to the new Python. - execute ./brink.sh deps - execute ./brink.sh test_ci + execute ./pythia.sh deps + execute ./pythia.sh test_ci execute popd echo "::endgroup::" diff --git a/functions_pythia.sh b/functions_build.sh similarity index 98% rename from functions_pythia.sh rename to functions_build.sh index 35ba6244..de0bd2eb 100644 --- a/functions_pythia.sh +++ b/functions_build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Pythia-specific functions. +# Pythia-specific build functions. # Global variables. COMMAND="" @@ -102,7 +102,7 @@ chevahbs_build() { build() { # This has the form: "libffi", "zlib", "bzip", "libedit", etc. # It's present in 'src/` and contains `chevahbs`, checksums, patches. - # Also used when downloading the gzipp'ed tarball and unpacking it. + # Also used when downloading the gzip'ed tarball and unpacking it. project_name="$1" # This has the form: "3.2.1", "1.2.11". etc. project_ver="$2" diff --git a/publish_dist.sh b/publish_dist.sh index 2a40a685..56340514 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -31,7 +31,7 @@ esac $sftp_cmd $sftp_opts ${dest_user}@${dest_server} # Get $DIST_DIR. -source pythia.conf +source build.conf # As dist/ is rebuilt on every build, it should only have 1 sub-dir with 1 pkg. upload_dir=$(cd $DIST_DIR && ls -1) diff --git a/pythia.conf b/pythia.conf index 1df2a037..abff3784 100644 --- a/pythia.conf +++ b/pythia.conf @@ -1,57 +1,13 @@ -# Configuration file for Pythia - -# Sub-directories for building from sources and for the final distributable. -BUILD_DIR="build" -DIST_DIR="dist" - -# Python and lib versions. -PYTHON_BUILD_VERSION="3.8.6" -LIBFFI_VERSION="3.4.2" -ZLIB_VERSION="1.2.12" -BZIP2_VERSION="1.0.8" -# Statically build the BSD libedit on selected platforms to get the -# readline module available without linking to the GPL-only readline libs. -LIBEDIT_VERSION="20170329-3.1" -OPENSSL_VERSION="1.1.1n" -# To match the unusual naming scheme for SQLite downloads, eliminate dots from -# the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". -# When updating this, also update the year in src/sqlite/chevahbs, if needed. -SQLITE_VERSION="3330000" -# These are used by get-pip.py. -PIP_VERSION="21.1.1" -SETUPTOOLS_VERSION="53.0.0" - -# Python modules versions to be used everywhere possible. -CFFI_VERSION="1.14.3" -CRYPTOGRAPHY_VERSION="3.4.7" -SUBPROCESS32_VERSION="3.5.4" -BCRYPT_VERSION="3.1.7" -PSUTIL_VERSION="5.7.3" -SETPROCTITLE_VERSION="1.1.10" - -# pycparser is explicitly installed to work around setuptools auto dependencies. -PYCPARSER_VERSION="2.20" - -# Global flags for building required libs. -BUILD_LIBFFI="no" -BUILD_ZLIB="no" -BUILD_BZIP2="yes" -BUILD_LIBEDIT="yes" -BUILD_OPENSSL="no" -BUILD_SQLITE="yes" - -# List of default Python modules installed using pip. -PIP_LIBRARIES="\ - cffi==${CFFI_VERSION} \ - cryptography==${CRYPTOGRAPHY_VERSION} \ - subprocess32==${SUBPROCESS32_VERSION} \ - bcrypt==${BCRYPT_VERSION} \ - psutil==${PSUTIL_VERSION} \ - setproctitle==${SETPROCTITLE_VERSION} - " -# Arguments that are sent when using pip. -PIP_ARGS="\ - --index-url=https://bin.chevah.com:20443/pypi/simple \ - --trusted-host=pypi.chevah.com \ - --no-warn-script-location \ - " +BASE_REQUIREMENTS='chevah-brink==1.0.6 paver==1.2.4' +PYTHON_CONFIGURATION='default@3.8.6.3b1a8ba' +# For production packages there are 2 options: +BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' +#BINARY_DIST_URI='https://bin.chevah.com:20443/production' +# For testing packages, make sure this one is the last uncommented instance: +#BINARY_DIST_URI='https://bin.chevah.com:20443/testing' +PIP_INDEX_URL='https://pypi.chevah.com/simple' + +# There are 2 build directories used in this repo: +# * $BUILD_DIR is used for building libffi / OpenSSL / Python / etc. +# * $CHEVAH_BUILD_DIR is used by the Python that builds the above. +CHEVAH_BUILD_DIR='build-pythia' diff --git a/brink.sh b/pythia.sh similarity index 97% rename from brink.sh rename to pythia.sh index c5a0f3f3..06ff679b 100755 --- a/brink.sh +++ b/pythia.sh @@ -22,14 +22,14 @@ # * CHEVAH_ARCH - CPU type of the current OS # # The build directory is used from CHEVAH_BUILD env, -# then read from brink.conf as CHEVAH_BUILD_DIR, +# then read from pythia.conf as CHEVAH_BUILD_DIR, # and will use a default value if not defined there. # # The cache directory is read the CHEVAH_CACHE env, -# and then read from brink.conf as CHEVAH_CACHE_DIR, +# and then read from pythia.conf as CHEVAH_CACHE_DIR, # and will use a default value if not defined. # -# You can define your own `execute_venv` function in brink.conf with the +# You can define your own `execute_venv` function in pythia.conf with the # command used to execute Python inside the newly virtual environment. # @@ -81,7 +81,7 @@ LOCAL_PYTHON_BINARY_DIST="" OS='not-detected-yet' ARCH='not-detected-yet' -# Initialize default values from brink.conf +# Initialize default values from pythia.conf PYTHON_CONFIGURATION='NOT-YET-DEFINED' PYTHON_VERSION='not.defined.yet' PYTHON_PLATFORM='unknown-os-and-arch' @@ -92,18 +92,18 @@ BASE_REQUIREMENTS='' # # Check that we have a pavement.py file in the current dir. -# If not, we are out of the source's root dir and brink.sh won't work. +# If not, we are out of the source's root dir and pythia.sh won't work. # check_source_folder() { if [ ! -e pavement.py ]; then (>&2 echo 'No "pavement.py" file found in current folder.') - (>&2 echo 'Make sure you are running "brink.sh" from a source folder.') + (>&2 echo 'Make sure you are running "pythia.sh" from a source folder.') exit 8 fi } # Called to trigger the entry point in the virtual environment. -# Can be overwritten in brink.conf +# Can be overwritten in pythia.conf execute_venv() { ${PYTHON_BIN} $PYTHON3_CHECK -c 'from paver.tasks import main; main()' "$@" } @@ -121,13 +121,13 @@ update_venv() { exit_code=$? set -e if [ $exit_code -ne 0 ]; then - (>&2 echo 'Failed to run the initial "./brink.sh deps" command.') + (>&2 echo 'Failed to run the initial "./pythia.sh deps" command.') exit 7 fi } # Load repo specific configuration. -source brink.conf +source pythia.conf clean_build() { @@ -851,7 +851,7 @@ check_source_folder copy_python install_dependencies -# Update brink.conf dependencies when running deps. +# Update pythia.conf dependencies when running deps. if [ "$COMMAND" == "deps" ] ; then install_base_deps fi diff --git a/src/Python-Windows/chevahbs b/src/Python-Windows/chevahbs index b251f5dc..06b35acc 100755 --- a/src/Python-Windows/chevahbs +++ b/src/Python-Windows/chevahbs @@ -7,7 +7,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh # The usual two arguments, e.g. "Python" and "3.9.0", # but the installation path is also needed, because this script is special. diff --git a/src/Python/chevahbs b/src/Python/chevahbs index 73a391ad..374eeffb 100755 --- a/src/Python/chevahbs +++ b/src/Python/chevahbs @@ -7,7 +7,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/bzip2/chevahbs b/src/bzip2/chevahbs index acfc06d2..0f3ea54b 100755 --- a/src/bzip2/chevahbs +++ b/src/bzip2/chevahbs @@ -5,7 +5,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/libedit/chevahbs b/src/libedit/chevahbs index 1f7556b0..c6e98973 100755 --- a/src/libedit/chevahbs +++ b/src/libedit/chevahbs @@ -4,7 +4,7 @@ # Import shared code. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index a7acc11d..ec900226 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -5,7 +5,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/openssl/chevahbs b/src/openssl/chevahbs index 3bfab7ca..1471fbfe 100755 --- a/src/openssl/chevahbs +++ b/src/openssl/chevahbs @@ -5,7 +5,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/sqlite-autoconf/chevahbs b/src/sqlite-autoconf/chevahbs index 71e9142d..0e1dc8b6 100755 --- a/src/sqlite-autoconf/chevahbs +++ b/src/sqlite-autoconf/chevahbs @@ -5,7 +5,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { diff --git a/src/zlib/chevahbs b/src/zlib/chevahbs index 600c2007..4b37fb20 100755 --- a/src/zlib/chevahbs +++ b/src/zlib/chevahbs @@ -5,7 +5,7 @@ # Import shared code. # The relative paths work in both src/blabla and build/blabla. source ../../functions.sh -source ../../functions_pythia.sh +source ../../functions_build.sh chevahbs_getsources() { From 0a2bf13e6b2333ff7ade2d8d8b0bfb8548a5205a Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 26 Apr 2022 12:42:46 +0300 Subject: [PATCH 13/38] Updated GitHub workflows for the grand rename. --- .github/workflows/bare.yaml | 12 ++++++------ .github/workflows/docker.yaml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index fca3f6aa..83c333c4 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -66,10 +66,10 @@ jobs: # Explicitly run our scripts with Bash, not PowerShell (GitHub's default). - name: Build Pythia - run: bash ./pythia build + run: bash ./build.sh build - name: Test Pythia - run: bash ./pythia test + run: bash ./build.sh test # To use an RSA key with SFTPPlus, install upstream OpenSSH package, # which is more finicky in regards to file permissions. @@ -131,10 +131,10 @@ jobs: run: sudo apt-get install libncurses5-dev - name: Build Pythia - run: ./pythia build + run: ./build.sh build - name: Test Pythia - run: ./pythia test + run: ./build.sh test - name: Upload testing package run: | @@ -173,10 +173,10 @@ jobs: sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* - name: Build Pythia - run: ./pythia build + run: ./build.sh build - name: Test Pythia - run: ./pythia test + run: ./build.sh test - name: Upload testing package run: | diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 27be8382..0dfc6d68 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -69,12 +69,12 @@ jobs: - name: Build Pythia run: | cd pythia - ./pythia build + ./build.sh build - name: Test Pythia run: | cd pythia - ./pythia test + ./build.sh test # Using `~/` is problematic under Docker, use `/root/`. # Remove key in same step to avoid leaving it on disk if publishing fails. From 4ab0874c45d551db2eb8b37a0f77b9baeaaa7c41 Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 26 Apr 2022 14:21:25 +0300 Subject: [PATCH 14/38] Move stray pkgconfig files to their place. --- build.conf | 2 +- functions_build.sh | 5 +++++ src/libffi/chevahbs | 14 +------------- src/openssl/chevahbs | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/build.conf b/build.conf index 1df2a037..0b5f3613 100644 --- a/build.conf +++ b/build.conf @@ -1,4 +1,4 @@ -# Configuration file for Pythia +# Configuration file for building Pythia. # Sub-directories for building from sources and for the final distributable. BUILD_DIR="build" diff --git a/functions_build.sh b/functions_build.sh index de0bd2eb..33bb0bfa 100644 --- a/functions_build.sh +++ b/functions_build.sh @@ -212,6 +212,11 @@ cleanup_install_dir() { execute rm -rf "lib/$PYTHON_VERSION/test/" # Remove (mostly OpenSSL) docs and manuals. execute rm -rf share/ + # Move stray pkgconfig/* to lib/pkgconfig/. + if [ -d pkgconfig ]; then + execute mv pkgconfig/* lib/pkgconfig/ + execute rmdir pkgconfig + fi ;; esac execute popd diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index ec900226..4f31fbab 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -29,19 +29,7 @@ chevahbs_compile() { chevahbs_install() { - case $OS in - lnx) - # Installing manually to avoid messing with a lib64/ sub-dir. - execute cp -v *-linux-gnu/.libs/libffi.a "$INSTALL_DIR"/lib/ - execute cp -v *-linux-gnu/libffi.la "$INSTALL_DIR"/lib/ - execute cp -v *-linux-gnu/include/*.h "$INSTALL_DIR"/include/ - execute mkdir "$INSTALL_DIR"/pkgconfig/ - execute cp -v *-linux-gnu/libffi.pc "$INSTALL_DIR"/pkgconfig/ - ;; - *) - execute $MAKE install DESTDIR=$INSTALL_DIR - ;; - esac + execute $MAKE install DESTDIR=$INSTALL_DIR } select_chevahbs_command $@ diff --git a/src/openssl/chevahbs b/src/openssl/chevahbs index 1471fbfe..8a9a2fd8 100755 --- a/src/openssl/chevahbs +++ b/src/openssl/chevahbs @@ -38,7 +38,7 @@ chevahbs_install() { echo "Installing manually to avoid messing with a lib64/ sub-dir:" execute cp -v libcrypto.a libssl.a "$INSTALL_DIR"/lib/ execute cp -rv include/openssl/ "$INSTALL_DIR"/include/ - execute cp -v *.pc "$INSTALL_DIR"/pkgconfig/ + execute cp -v *.pc "$INSTALL_DIR"/lib/pkgconfig/ ;; *) execute $MAKE install From 85a7d21805ee71c9bcd46b21ef86fd9188bfb516 Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 26 Apr 2022 15:07:21 +0300 Subject: [PATCH 15/38] Back to manually installing libffi for CentOS 5. --- src/libffi/chevahbs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/libffi/chevahbs b/src/libffi/chevahbs index 4f31fbab..b76a7f72 100755 --- a/src/libffi/chevahbs +++ b/src/libffi/chevahbs @@ -29,7 +29,19 @@ chevahbs_compile() { chevahbs_install() { - execute $MAKE install DESTDIR=$INSTALL_DIR + case $OS in + lnx) + # Install manually to avoid creating a lib64/ sub-dir on CentOS 5. + execute cp -v *-linux-gnu/.libs/libffi.a "$INSTALL_DIR"/lib/ + execute cp -v *-linux-gnu/libffi.la "$INSTALL_DIR"/lib/ + execute cp -v *-linux-gnu/include/*.h "$INSTALL_DIR"/include/ + execute mkdir "$INSTALL_DIR"/lib/pkgconfig/ + execute cp -v *-linux-gnu/libffi.pc "$INSTALL_DIR"/lib/pkgconfig/ + ;; + *) + execute $MAKE install DESTDIR=$INSTALL_DIR + ;; + esac } select_chevahbs_command $@ From b8b7f6c3e0df451d2e130ecd1b52416099b2cb4f Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 26 Apr 2022 15:33:01 +0300 Subject: [PATCH 16/38] Added strong Bash checks for building. --- build.sh | 11 ++++++++--- functions.sh | 4 +--- os_quirks.sh | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/build.sh b/build.sh index 9f2bc710..a1362036 100755 --- a/build.sh +++ b/build.sh @@ -2,6 +2,11 @@ # # Pythia's script for building Python. +# Script initialization. +set -o nounset +set -o errexit +set -o pipefail + # Set versions for the software to be built and other defaults. source build.conf @@ -87,8 +92,8 @@ command_build() { # following locations, making sure they are picked up when building Python. # $CFLAGS/$CPPFLAGS is another way to ensure this, but it's not as portable. execute mkdir -p "$INSTALL_DIR"/{include,lib} - export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS}" - export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig/:${PKG_CONFIG_PATH}" + export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS:-}" + export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig/:${PKG_CONFIG_PATH:-}" build_dep $BUILD_LIBFFI libffi $LIBFFI_VERSION build_dep $BUILD_ZLIB zlib $ZLIB_VERSION @@ -130,7 +135,7 @@ build_dep() { export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}" fi # Still needed for building cryptography. - export CPPFLAGS="$CPPFLAGS -I${INSTALL_DIR}/include" + export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include" fi elif [ $dep_boolean = "no" ]; then (>&2 echo " Skip building $dep_name") diff --git a/functions.sh b/functions.sh index 19b9fd81..196e1173 100644 --- a/functions.sh +++ b/functions.sh @@ -4,9 +4,7 @@ # # Check if debugging environment variable is set and initialize with 0 if not. -if [ -z "$DEBUG" ] ; then - DEBUG=0 -fi +DEBUG=${DEBUG-0} help_text_help=\ "Show help for a command." diff --git a/os_quirks.sh b/os_quirks.sh index 9b320f3f..8b03dca1 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -40,7 +40,7 @@ case $OS in macos) export CC="clang" export CXX="clang++" - export CFLAGS="$CFLAGS -mmacosx-version-min=10.13" + export CFLAGS="${CFLAGS:-} -mmacosx-version-min=10.13" # setup.py skips building readline by default, as it sets this to # "10.4", and then tries to avoid the broken readline in OS X 10.4. export MACOSX_DEPLOYMENT_TARGET=10.13 @@ -76,14 +76,14 @@ case $OS in export MAKE="gmake" # Needed for the subprocess32 module. # More at https://github.com/google/python-subprocess32/issues/40. - export CFLAGS="$CFLAGS -DHAVE_DIRFD" + export CFLAGS="${CFLAGS:-} -DHAVE_DIRFD" # Arch-specific bits and paths. if [ "${ARCH%64}" = "$ARCH" ]; then # Some libs (e.g. GMP) need to be informed of a 32bit build. export ABI="32" else export CFLAGS="$CFLAGS -m64" - export LDFLAGS="$LDFLAGS -m64 -L/usr/lib/64 -R/usr/lib/64" + export LDFLAGS="${LDFLAGS:-} -m64 -L/usr/lib/64 -R/usr/lib/64" fi # System includes bzip2 libs by default. export BUILD_BZIP2="no" @@ -101,10 +101,10 @@ if [ "${OS%sol*}" = "" ]; then export CFLAGS="$CFLAGS -Kpic" elif [ "${OS%fbsd*}" = "" -o "${OS%obsd*}" = "" ]; then # Use PIC (Position Independent Code) on FreeBSD and OpenBSD with Clang. - export CFLAGS="${CFLAGS} -fPIC" + export CFLAGS="${CFLAGS:-} -fPIC" elif [ "$CC" = "gcc" -a ${ARCH%%64} != "$ARCH" ]; then # Use PIC (Position Independent Code) with GCC on 64-bit arches. - export CFLAGS="${CFLAGS} -fPIC" + export CFLAGS="${CFLAGS:-} -fPIC" fi # Get number of useful CPUs, to enable parallel builds where applicable. @@ -127,4 +127,4 @@ case "$OS" in CPUS=$(getconf _NPROCESSORS_ONLN) ;; esac -export MAKE="$MAKE -j${CPUS}" +export MAKE="${MAKE:-} -j${CPUS}" From 61421f252fda918cb7611c5b08d2a42d54544588 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 14:02:58 +0300 Subject: [PATCH 17/38] Removed scripted OS package management. --- .github/workflows/bare.yaml | 6 +++-- .github/workflows/docker.yaml | 6 +++-- pkg_checks.sh | 42 ++++++++++------------------------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 83c333c4..04d953d3 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -127,8 +127,10 @@ jobs: with: fetch-depth: 2 - - name: Install required packages - run: sudo apt-get install libncurses5-dev + - name: Prepare OS + run: | + sudo apt install libncurses5-dev + sudo apt purge uuid-dev - name: Build Pythia run: ./build.sh build diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 0dfc6d68..86e9fd52 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,7 +41,8 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash openssh-client paxctl + apk add git curl bash gcc make m4 automake libtool patch libffi-dev zlib-dev openssl-dev musl-dev lddtree shadow sudo openssh-client paxctl file unzip + apk del util-linux-dev # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. - name: CentOS 8.2 setup @@ -50,7 +51,8 @@ jobs: sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo sed -i s@^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever/@baseurl=https://vault.centos.org/8.2.2004/@ /etc/yum.repos.d/*.repo yum -y upgrade - yum -y install git curl openssh-clients + yum -y install git curl gcc make m4 automake libtool patch libffi-devel zlib-devel ncurses-devel openssl-devel openssh-clients unzip + yum remove -y e2fsprogs-devel libuuid-devel # Our Chevah Centos 5 image, built for python-package. # FIXME: Update it to include unzip. diff --git a/pkg_checks.sh b/pkg_checks.sh index 8b07bbc1..c0ac0818 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash # # Check for the presence of required packages/commands. -# If possible, install missing ones, typically through sudo. # # This build requires: # * a C compiler, e.g. gcc @@ -16,7 +15,7 @@ # List of OS packages required for building Python/pyOpenSSL/cryptography etc. BASE_PKGS="gcc make m4 automake libtool patch unzip" -DPKG_PKGS="$BASE_PKGS tar diffutils \ +DEB_PKGS="$BASE_PKGS tar diffutils \ git libssl-dev zlib1g-dev libffi-dev libncurses5-dev" RPM_PKGS="$BASE_PKGS tar diffutils \ git-core openssl-devel zlib-devel libffi-devel ncurses-devel" @@ -24,7 +23,7 @@ RPM_PKGS="$BASE_PKGS tar diffutils \ APK_PKGS="$BASE_PKGS file lddtree \ git openssl-dev zlib-dev libffi-dev musl-dev paxctl" # Windows is special, but package management is possible through Chocolatey. -# Curl, sha512sum, and unzip are bundled with MINGW. +# Some tools are bundled with MINGW: curl, sha512sum, unzip. CHOCO_PKGS="" CHOCO_PRESENT="unknown" @@ -45,7 +44,7 @@ case "$OS" in CHECK_CMD="rpm --query" ;; ubuntu*) - PACKAGES="$DPKG_PKGS" + PACKAGES="$DEB_PKGS" CHECK_CMD="dpkg --status" ;; alpine*) @@ -83,6 +82,7 @@ case "$OS" in esac # If $CHECK_CMD is still "command -v", it's only a check for needed commands. +set +o errexit if [ -n "$PACKAGES" ]; then for package in $PACKAGES ; do echo "Checking if $package is available..." @@ -93,32 +93,11 @@ if [ -n "$PACKAGES" ]; then fi done fi +set -o errexit if [ -n "$MISSING_PACKAGES" ]; then (>&2 echo "Missing required dependencies: $MISSING_PACKAGES.") - if [ $CHOCO_PRESENT = "yes" ]; then - echo "## Installing missing Chocolatey packages... ##" - execute choco install --yes --no-progress $MISSING_PACKAGES - else - case "$OS" in - ubuntu*) - echo "## Installing missing dpkg packages... ##" - execute $SUDO_CMD apt install -y $MISSING_PACKAGES - ;; - rhel*|amzn*) - echo "## Installing missing rpm packages... ##" - execute $SUDO_CMD yum install -y $MISSING_PACKAGES - ;; - alpine*) - echo "## Installing missing apk packages... ##" - execute $SUDO_CMD apk add $MISSING_PACKAGES - ;; - *) - (>&2 echo "Don't know how to install those on the current OS.") - exit 149 - ;; - esac - fi + exit 149 fi if [ -n "$PACKAGES" ]; then @@ -143,14 +122,17 @@ fi echo "# Checking if it's possible to avoid linking to system uuid libs... #" case "$OS" in ubuntu*) - execute $SUDO_CMD apt remove -y uuid-dev + $CHECK_CMD uuid \ + && echo "To not link to uuid libs, run: apt remove -y uuid-dev" ;; rhel*|amzn*) - execute $SUDO_CMD yum remove -y e2fsprogs-devel libuuid-devel + $CHECK_CMD libuuid-devel \ + && echo -n "To not link to uuid libs, run: " \ + && echo "yum remove -y e2fsprogs-devel libuuid-devel" ;; alpine*) $CHECK_CMD util-linux-dev \ - && execute $SUDO_CMD apk del util-linux-dev + && echo "To not link to uuid libs, run: apk del util-linux-dev" ;; *) (>&2 echo "Not guarding against linking to uuid libs on this system!") From e0d8d8b50cdbf46edcc5030cefccbd95f70a48f3 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 14:16:57 +0300 Subject: [PATCH 18/38] Try to fix fallout from removing scripted OS pkg mgmt. --- pkg_checks.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg_checks.sh b/pkg_checks.sh index c0ac0818..13d84516 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -13,6 +13,9 @@ # # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh. +# This script has external checks with various exit codes that are checked here. +set +o errexit + # List of OS packages required for building Python/pyOpenSSL/cryptography etc. BASE_PKGS="gcc make m4 automake libtool patch unzip" DEB_PKGS="$BASE_PKGS tar diffutils \ @@ -82,7 +85,6 @@ case "$OS" in esac # If $CHECK_CMD is still "command -v", it's only a check for needed commands. -set +o errexit if [ -n "$PACKAGES" ]; then for package in $PACKAGES ; do echo "Checking if $package is available..." @@ -93,7 +95,6 @@ if [ -n "$PACKAGES" ]; then fi done fi -set -o errexit if [ -n "$MISSING_PACKAGES" ]; then (>&2 echo "Missing required dependencies: $MISSING_PACKAGES.") @@ -122,7 +123,7 @@ fi echo "# Checking if it's possible to avoid linking to system uuid libs... #" case "$OS" in ubuntu*) - $CHECK_CMD uuid \ + $CHECK_CMD uuid-dev \ && echo "To not link to uuid libs, run: apt remove -y uuid-dev" ;; rhel*|amzn*) From f5fe3fec19ace72a31d2706f3111fc4d790de630 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 14:58:36 +0300 Subject: [PATCH 19/38] Build xz for Python's lzma module. --- build.conf | 2 + build.sh | 3 +- os_quirks.sh | 1 + src/Python/disabled_modules.patch | 3 +- .../test_python_binary_dist.py | 13 +++++++ src/xz/chevahbs | 37 +++++++++++++++++++ src/xz/sha512.sum | 1 + 7 files changed, 57 insertions(+), 3 deletions(-) create mode 100755 src/xz/chevahbs create mode 100644 src/xz/sha512.sum diff --git a/build.conf b/build.conf index 0b5f3613..56624c25 100644 --- a/build.conf +++ b/build.conf @@ -9,6 +9,7 @@ PYTHON_BUILD_VERSION="3.8.6" LIBFFI_VERSION="3.4.2" ZLIB_VERSION="1.2.12" BZIP2_VERSION="1.0.8" +XZ_VERSION="5.2.5" # Statically build the BSD libedit on selected platforms to get the # readline module available without linking to the GPL-only readline libs. LIBEDIT_VERSION="20170329-3.1" @@ -36,6 +37,7 @@ PYCPARSER_VERSION="2.20" BUILD_LIBFFI="no" BUILD_ZLIB="no" BUILD_BZIP2="yes" +BUILD_XZ="yes" BUILD_LIBEDIT="yes" BUILD_OPENSSL="no" BUILD_SQLITE="yes" diff --git a/build.sh b/build.sh index a1362036..3661e44d 100755 --- a/build.sh +++ b/build.sh @@ -21,7 +21,7 @@ exit_on_error $? 250 # Export the variables needed by the chevahbs scripts and the test phase. export PYTHON_BUILD_VERSION PYTHIA_VERSION -export BUILD_ZLIB BUILD_BZIP2 BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL +export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL # OS detection is slow on Windows, only execute it when the file is missing. @@ -98,6 +98,7 @@ command_build() { build_dep $BUILD_LIBFFI libffi $LIBFFI_VERSION build_dep $BUILD_ZLIB zlib $ZLIB_VERSION build_dep $BUILD_BZIP2 bzip2 $BZIP2_VERSION + build_dep $BUILD_XZ xz $XZ_VERSION build_dep $BUILD_LIBEDIT libedit $LIBEDIT_VERSION build_dep $BUILD_SQLITE sqlite-autoconf $SQLITE_VERSION build_dep $BUILD_OPENSSL openssl $OPENSSL_VERSION diff --git a/os_quirks.sh b/os_quirks.sh index 8b03dca1..e29de62d 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -13,6 +13,7 @@ case $OS in # But not all are from pypi.org. Wheels copied from other places: # * "setproctitle" from https://www.lfd.uci.edu/~gohlke/pythonlibs/ export BUILD_BZIP2="no" + export BUILD_XZ="no" export BUILD_LIBEDIT="no" export BUILD_SQLITE="no" # On Windows, only one of the installers is downloaded. diff --git a/src/Python/disabled_modules.patch b/src/Python/disabled_modules.patch index 0a96bc9a..33afa959 100644 --- a/src/Python/disabled_modules.patch +++ b/src/Python/disabled_modules.patch @@ -1,7 +1,7 @@ diff -ur Python-3.8.6.orig/setup.py Python-3.8.6.disabled_modules/setup.py --- Python-3.8.6.orig/setup.py 2020-09-23 12:36:32.000000000 +0000 +++ Python-3.8.6.disabled_modules/setup.py 2020-10-30 11:48:50.512622149 +0000 -@@ -24,8 +24,17 @@ +@@ -24,8 +24,16 @@ TEST_EXTENSIONS = True # This global variable is used to hold the list of modules to be disabled. @@ -13,7 +13,6 @@ diff -ur Python-3.8.6.orig/setup.py Python-3.8.6.disabled_modules/setup.py + '_dbm', + '_gdbm', + '_locale', -+ '_lzma', + '_tkinter', + 'nis', + 'readline', diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index da9bbe2f..8093dfe2 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -468,6 +468,19 @@ def main(): sys.stderr.write('"bz2" is missing.\n') exit_code = 149 + try: + import lzma + test_string = b"just a random string to quickly test lzma" + test_string_xzed = lzma.compress(test_string) + if lzma.decompress(test_string_xzed) == test_string: + print('"lzma" module is present.') + else: + sys.stderr.write('"lzma" is present, but broken.\n') + exit_code = 152 + except: + sys.stderr.write('"lzma" is missing.\n') + exit_code = 151 + try: import setproctitle current_process_title = setproctitle.getproctitle() diff --git a/src/xz/chevahbs b/src/xz/chevahbs new file mode 100755 index 00000000..525a7cce --- /dev/null +++ b/src/xz/chevahbs @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Chevah Build Script for xz. + +# Import shared code. +# The relative paths work in both src/blabla and build/blabla. +source ../../functions.sh +source ../../functions_build.sh + + +chevahbs_getsources() { + local name=$1 + local ver=$2 + local ext="tar.gz" + local link=https://tukaani.org/xz/"$name"-"$ver"."$ext" + + download_sources $name $ver $link $ext +} + + +chevahbs_configure() { + CONF_OPTS="--enable-static" + execute ./configure --prefix="" $CONF_OPTS +} + + +chevahbs_compile() { + execute $MAKE +} + + +chevahbs_install() { + execute $MAKE install DESTDIR=$INSTALL_DIR +} + + +select_chevahbs_command $@ diff --git a/src/xz/sha512.sum b/src/xz/sha512.sum new file mode 100644 index 00000000..e72a0025 --- /dev/null +++ b/src/xz/sha512.sum @@ -0,0 +1 @@ +7443674247deda2935220fbc4dfc7665e5bb5a260be8ad858c8bd7d7b9f0f868f04ea45e62eb17c0a5e6a2de7c7500ad2d201e2d668c48ca29bd9eea5a73a3ce xz-5.2.5.tar.gz From 4d1bc2b48bd81a0b309084ed773f295398806d56 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 15:07:54 +0300 Subject: [PATCH 20/38] Download xz from SourceForge to have it work on CentOS 5. --- src/xz/chevahbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xz/chevahbs b/src/xz/chevahbs index 525a7cce..16ce36fb 100755 --- a/src/xz/chevahbs +++ b/src/xz/chevahbs @@ -12,7 +12,7 @@ chevahbs_getsources() { local name=$1 local ver=$2 local ext="tar.gz" - local link=https://tukaani.org/xz/"$name"-"$ver"."$ext" + local link=https://downloads.sourceforge.net/project/lzmautils/"$name"-"$ver"."$ext" download_sources $name $ver $link $ext } From b179a3823f47b8ad417d5110d1f2378a20975f4e Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 15:09:34 +0300 Subject: [PATCH 21/38] Try building xz only on generic Linux. --- .github/workflows/bare.yaml | 2 +- build.conf | 2 +- os_quirks.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 04d953d3..8a486dfc 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -41,7 +41,7 @@ concurrency: # Also increase timeout-minutes for the relevant OS when debugging remotely. # Version 3 of mxschmitt/action-tmate should also work on Windows. env: - TMATE_DEBUG: 'no' + TMATE_DEBUG: 'yes' NGROK_DEBUG: 'no' diff --git a/build.conf b/build.conf index 56624c25..3f7504e6 100644 --- a/build.conf +++ b/build.conf @@ -37,7 +37,7 @@ PYCPARSER_VERSION="2.20" BUILD_LIBFFI="no" BUILD_ZLIB="no" BUILD_BZIP2="yes" -BUILD_XZ="yes" +BUILD_XZ="no" BUILD_LIBEDIT="yes" BUILD_OPENSSL="no" BUILD_SQLITE="yes" diff --git a/os_quirks.sh b/os_quirks.sh index e29de62d..7b9eb995 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -13,7 +13,6 @@ case $OS in # But not all are from pypi.org. Wheels copied from other places: # * "setproctitle" from https://www.lfd.uci.edu/~gohlke/pythonlibs/ export BUILD_BZIP2="no" - export BUILD_XZ="no" export BUILD_LIBEDIT="no" export BUILD_SQLITE="no" # On Windows, only one of the installers is downloaded. @@ -31,6 +30,7 @@ case $OS in # Build as portable as possible, only glibc 2.x should be needed. export BUILD_LIBFFI="yes" export BUILD_ZLIB="yes" + export BUILD_XZ="yes" export BUILD_LIBEDIT="no" # Generic Linux might be an old distro with OpenSSL 0.9.8 libraries. # To avoid linking to local libs, build own OpenSSL libs. From ce4d11866059cb0c500a49cffbeb0f98bd1e1cd9 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 16:04:35 +0300 Subject: [PATCH 22/38] Updated deps checks. --- .../test_python_binary_dist.py | 37 +------------------ 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 8093dfe2..6a3d11c4 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -56,6 +56,7 @@ def get_allowed_deps(): '/lib64/libkeyutils.so.1', '/lib64/libkrb5.so.3', '/lib64/libkrb5support.so.0', + '/lib64/liblzma.so.5', '/lib64/libm.so.6', '/lib64/libnsl.so.1', '/lib64/libpthread.so.0', @@ -66,15 +67,6 @@ def get_allowed_deps(): '/lib64/libz.so.1', ] rhel_version = CHEVAH_OS[4:] - if rhel_version.startswith("7"): - allowed_deps.extend([ - '/lib64/libcrypto.so.10', - '/lib64/libffi.so.6', - '/lib64/libncursesw.so.5', - '/lib64/libpcre.so.1', - '/lib64/libssl.so.10', - '/lib64/libtinfo.so.5', - ]) if rhel_version.startswith("8"): allowed_deps.extend([ '/lib64/libcrypto.so.1.1', @@ -83,32 +75,6 @@ def get_allowed_deps(): '/lib64/libssl.so.1.1', '/lib64/libtinfo.so.6', ]) - elif 'amzn' in CHEVAH_OS: - # Deps for Amazon Linux 2 (x86_64 only). - allowed_deps=[ - '/lib64/libcom_err.so.2', - '/lib64/libcrypto.so.10', - '/lib64/libcrypt.so.1', - '/lib64/libc.so.6', - '/lib64/libdl.so.2', - '/lib64/libffi.so.6', - '/lib64/libgssapi_krb5.so.2', - '/lib64/libk5crypto.so.3', - '/lib64/libkeyutils.so.1', - '/lib64/libkrb5.so.3', - '/lib64/libkrb5support.so.0', - '/lib64/libm.so.6', - '/lib64/libncursesw.so.6', - '/lib64/libpcre.so.1', - '/lib64/libpthread.so.0', - '/lib64/libresolv.so.2', - '/lib64/librt.so.1', - '/lib64/libselinux.so.1', - '/lib64/libssl.so.10', - '/lib64/libtinfo.so.6', - '/lib64/libutil.so.1', - '/lib64/libz.so.1', - ] elif 'ubuntu' in CHEVAH_OS: ubuntu_version = CHEVAH_OS[6:] # Common deps for supported Ubuntu LTS with full paths (x86_64). @@ -116,6 +82,7 @@ def get_allowed_deps(): '/lib/x86_64-linux-gnu/libc.so.6', '/lib/x86_64-linux-gnu/libcrypt.so.1', '/lib/x86_64-linux-gnu/libdl.so.2', + '/lib/x86_64-linux-gnu/liblzma.so.5', '/lib/x86_64-linux-gnu/libm.so.6', '/lib/x86_64-linux-gnu/libpthread.so.0', '/lib/x86_64-linux-gnu/librt.so.1', From 870c252c64b53918da253dcd913931357adc0ccc Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 16:05:12 +0300 Subject: [PATCH 23/38] Try building xz on Alpine. --- .github/workflows/docker.yaml | 2 +- os_quirks.sh | 1 + pkg_checks.sh | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 86e9fd52..0c1309be 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,7 +41,7 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash gcc make m4 automake libtool patch libffi-dev zlib-dev openssl-dev musl-dev lddtree shadow sudo openssh-client paxctl file unzip + apk add git curl bash gcc make m4 automake libtool patch openssl-dev lddtree shadow sudo openssh-client paxctl file unzip apk del util-linux-dev # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. diff --git a/os_quirks.sh b/os_quirks.sh index 7b9eb995..2771915b 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -25,6 +25,7 @@ case $OS in # It's better to run on minimal Alpine containers. export BUILD_LIBFFI="yes" export BUILD_LIBEDIT="no" + export BUILD_XZ="yes" ;; lnx) # Build as portable as possible, only glibc 2.x should be needed. diff --git a/pkg_checks.sh b/pkg_checks.sh index 13d84516..b7e5bb73 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -19,12 +19,12 @@ set +o errexit # List of OS packages required for building Python/pyOpenSSL/cryptography etc. BASE_PKGS="gcc make m4 automake libtool patch unzip" DEB_PKGS="$BASE_PKGS tar diffutils \ - git libssl-dev zlib1g-dev libffi-dev libncurses5-dev" + git zlib1g-dev liblzma-dev libffi-dev libncurses5-dev libssl-dev" RPM_PKGS="$BASE_PKGS tar diffutils \ - git-core openssl-devel zlib-devel libffi-devel ncurses-devel" + git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel" # Alpine's ersatz tar/sha51sum binaries from Busybox are good enough. APK_PKGS="$BASE_PKGS file lddtree \ - git openssl-dev zlib-dev libffi-dev musl-dev paxctl" + git openssl-dev paxctl" # Windows is special, but package management is possible through Chocolatey. # Some tools are bundled with MINGW: curl, sha512sum, unzip. CHOCO_PKGS="" From d83e59cf4961d306e579c9f21892b222e0b541c6 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 16:24:23 +0300 Subject: [PATCH 24/38] Package management fixes. --- .github/workflows/docker.yaml | 4 ++-- os_quirks.sh | 3 +++ pkg_checks.sh | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 0c1309be..414300b6 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,7 +41,7 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash gcc make m4 automake libtool patch openssl-dev lddtree shadow sudo openssh-client paxctl file unzip + apk add git curl bash gcc make m4 automake libtool patch openssl-dev musl-dev lddtree shadow sudo openssh-client paxctl file unzip apk del util-linux-dev # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. @@ -51,7 +51,7 @@ jobs: sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo sed -i s@^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever/@baseurl=https://vault.centos.org/8.2.2004/@ /etc/yum.repos.d/*.repo yum -y upgrade - yum -y install git curl gcc make m4 automake libtool patch libffi-devel zlib-devel ncurses-devel openssl-devel openssh-clients unzip + yum -y install git curl gcc make m4 automake libtool patch libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel openssh-clients unzip yum remove -y e2fsprogs-devel libuuid-devel # Our Chevah Centos 5 image, built for python-package. diff --git a/os_quirks.sh b/os_quirks.sh index 2771915b..07fb9cb7 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -48,6 +48,7 @@ case $OS in export MACOSX_DEPLOYMENT_TARGET=10.13 # System includes bzip2 libs by default. export BUILD_BZIP2="no" + export BUILD_XZ="yes" # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs. # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7. # Therefore, build OpenSSL for both stdlib and cryptography. @@ -61,6 +62,7 @@ case $OS in export BUILD_LIBFFI="yes" # System includes bzip2 libs by default. export BUILD_BZIP2="no" + export BUILD_XZ="yes" # Install package "p5-Digest-SHA" to get shasum binary. export SHA_CMD="shasum --algorithm 512 --check --status --warn" ;; @@ -89,6 +91,7 @@ case $OS in fi # System includes bzip2 libs by default. export BUILD_BZIP2="no" + export BUILD_XZ="yes" # Solaris 11 is much more modern, but still has some quirks. # Multiple system libffi libs present, this is a problem in 11.4. export BUILD_LIBFFI="yes" diff --git a/pkg_checks.sh b/pkg_checks.sh index b7e5bb73..6e7ee130 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -24,7 +24,7 @@ RPM_PKGS="$BASE_PKGS tar diffutils \ git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel" # Alpine's ersatz tar/sha51sum binaries from Busybox are good enough. APK_PKGS="$BASE_PKGS file lddtree \ - git openssl-dev paxctl" + git openssl-dev musl-dev paxctl" # Windows is special, but package management is possible through Chocolatey. # Some tools are bundled with MINGW: curl, sha512sum, unzip. CHOCO_PKGS="" From f32a4548b8c968370db22c26c6e697720bb241c4 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 17:01:49 +0300 Subject: [PATCH 25/38] More xz build tweaks. --- .github/workflows/bare.yaml | 1 + os_quirks.sh | 3 --- pkg_checks.sh | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 8a486dfc..52265402 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -173,6 +173,7 @@ jobs: sudo mv -v /usr/local/bin/git{,.saved} sudo chmod -v a-r /usr/local/include/libintl.h sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* + sudo chmod -v a-r /usr/local/opt/xz/lib/liblzma* - name: Build Pythia run: ./build.sh build diff --git a/os_quirks.sh b/os_quirks.sh index 07fb9cb7..2771915b 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -48,7 +48,6 @@ case $OS in export MACOSX_DEPLOYMENT_TARGET=10.13 # System includes bzip2 libs by default. export BUILD_BZIP2="no" - export BUILD_XZ="yes" # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs. # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7. # Therefore, build OpenSSL for both stdlib and cryptography. @@ -62,7 +61,6 @@ case $OS in export BUILD_LIBFFI="yes" # System includes bzip2 libs by default. export BUILD_BZIP2="no" - export BUILD_XZ="yes" # Install package "p5-Digest-SHA" to get shasum binary. export SHA_CMD="shasum --algorithm 512 --check --status --warn" ;; @@ -91,7 +89,6 @@ case $OS in fi # System includes bzip2 libs by default. export BUILD_BZIP2="no" - export BUILD_XZ="yes" # Solaris 11 is much more modern, but still has some quirks. # Multiple system libffi libs present, this is a problem in 11.4. export BUILD_LIBFFI="yes" diff --git a/pkg_checks.sh b/pkg_checks.sh index 6e7ee130..2130a8aa 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -24,7 +24,7 @@ RPM_PKGS="$BASE_PKGS tar diffutils \ git-core libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel" # Alpine's ersatz tar/sha51sum binaries from Busybox are good enough. APK_PKGS="$BASE_PKGS file lddtree \ - git openssl-dev musl-dev paxctl" + git zlib-dev openssl-dev musl-dev linux-headers paxctl" # Windows is special, but package management is possible through Chocolatey. # Some tools are bundled with MINGW: curl, sha512sum, unzip. CHOCO_PKGS="" From 90b122aba7af4ee1efe09cba3c9a0979559f25c1 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 17:03:47 +0300 Subject: [PATCH 26/38] Install required headers on Alpine. --- .github/workflows/docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 414300b6..7f921a4b 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -41,7 +41,7 @@ jobs: if: startsWith(matrix.container, 'alpine') run: | apk upgrade -U - apk add git curl bash gcc make m4 automake libtool patch openssl-dev musl-dev lddtree shadow sudo openssh-client paxctl file unzip + apk add git curl bash gcc make m4 automake libtool patch zlib-dev openssl-dev musl-dev linux-headers lddtree shadow sudo openssh-client paxctl file unzip apk del util-linux-dev # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g. From 02721b076168b88b0b3665bc4ca8130a038a7ad5 Mon Sep 17 00:00:00 2001 From: dumol Date: Fri, 29 Apr 2022 17:30:22 +0300 Subject: [PATCH 27/38] Try building xz on macOS differently. --- .github/workflows/bare.yaml | 1 - os_quirks.sh | 1 + src/chevah-python-test/test_python_binary_dist.py | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 52265402..8a486dfc 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -173,7 +173,6 @@ jobs: sudo mv -v /usr/local/bin/git{,.saved} sudo chmod -v a-r /usr/local/include/libintl.h sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.* - sudo chmod -v a-r /usr/local/opt/xz/lib/liblzma* - name: Build Pythia run: ./build.sh build diff --git a/os_quirks.sh b/os_quirks.sh index 2771915b..1b81bc63 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -48,6 +48,7 @@ case $OS in export MACOSX_DEPLOYMENT_TARGET=10.13 # System includes bzip2 libs by default. export BUILD_BZIP2="no" + export BUILD_XZ="yes" # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs. # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7. # Therefore, build OpenSSL for both stdlib and cryptography. diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 6a3d11c4..f823a76f 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -144,6 +144,7 @@ def get_allowed_deps(): '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', '/System/Library/Frameworks/Security.framework/Versions/A/Security', '/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration', + '/lib/liblzma.5.dylib', '/usr/lib/libbz2.1.0.dylib', '/usr/lib/libffi.dylib', '/usr/lib/libncurses.5.4.dylib', From 26538c9aeec70b9061130be13cdc9db9ef5e52e1 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 2 May 2022 11:06:47 +0300 Subject: [PATCH 28/38] Accomodate stricter bash checks in functions.sh. --- functions.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/functions.sh b/functions.sh index 196e1173..78de7554 100644 --- a/functions.sh +++ b/functions.sh @@ -9,9 +9,10 @@ DEBUG=${DEBUG-0} help_text_help=\ "Show help for a command." command_help() { - local command=$1 + local command=${1:-} local help_command="help_$command" # Test for a valid help method, otherwise call general help. + set +o errexit type $help_command &> /dev/null if [ $? -eq 0 ]; then $help_command @@ -23,6 +24,7 @@ command_help() { echo -e " $command_name\t${!help_text}" done fi + set -o errexit } # @@ -31,16 +33,17 @@ command_help() { # Select fuctions which are made public. # select_command() { - local command=$1 - shift + local command=${1:-} case $command in "") command_help exit 99 ;; *) + shift # Test for a valid command, otherwise call general help. call_command="command_$command" + set +o errexit type $call_command &> /dev/null if [ $? -eq 0 ]; then $call_command $@ @@ -50,6 +53,7 @@ select_command() { (>&2 echo "Unknown command: ${command}.") exit 98 fi + set -o errexit ;; esac } From 83cc01a22a7beb0e6de8bc9f5b2bc50d67835966 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 2 May 2022 12:40:23 +0300 Subject: [PATCH 29/38] Disabled building shared xz libs. --- pkg_checks.sh | 1 + src/chevah-python-test/test_python_binary_dist.py | 1 - src/xz/chevahbs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg_checks.sh b/pkg_checks.sh index 2130a8aa..9c8c5844 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -115,6 +115,7 @@ command -v makeinfo >/dev/null if [ $? -ne 0 ]; then (>&2 echo "# Missing makeinfo, linking it to /bin/true in ~/bin... #") execute mkdir -p ~/bin + execute rm -f ~/bin/makeinfo execute ln -s /bin/true ~/bin/makeinfo export PATH="$PATH:~/bin/" fi diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index f823a76f..6a3d11c4 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -144,7 +144,6 @@ def get_allowed_deps(): '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', '/System/Library/Frameworks/Security.framework/Versions/A/Security', '/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration', - '/lib/liblzma.5.dylib', '/usr/lib/libbz2.1.0.dylib', '/usr/lib/libffi.dylib', '/usr/lib/libncurses.5.4.dylib', diff --git a/src/xz/chevahbs b/src/xz/chevahbs index 16ce36fb..0bf2e78e 100755 --- a/src/xz/chevahbs +++ b/src/xz/chevahbs @@ -19,7 +19,7 @@ chevahbs_getsources() { chevahbs_configure() { - CONF_OPTS="--enable-static" + CONF_OPTS="--enable-static --disable-shared" execute ./configure --prefix="" $CONF_OPTS } From e9e3d129ed91eee1e0f865bfebec292c5f8e5824 Mon Sep 17 00:00:00 2001 From: dumol Date: Mon, 2 May 2022 13:19:12 +0300 Subject: [PATCH 30/38] Changes after own review. --- .github/workflows/bare.yaml | 2 +- functions_build.sh | 2 +- pkg_checks.sh | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bare.yaml b/.github/workflows/bare.yaml index 8a486dfc..04d953d3 100644 --- a/.github/workflows/bare.yaml +++ b/.github/workflows/bare.yaml @@ -41,7 +41,7 @@ concurrency: # Also increase timeout-minutes for the relevant OS when debugging remotely. # Version 3 of mxschmitt/action-tmate should also work on Windows. env: - TMATE_DEBUG: 'yes' + TMATE_DEBUG: 'no' NGROK_DEBUG: 'no' diff --git a/functions_build.sh b/functions_build.sh index 33bb0bfa..a2d9edd9 100644 --- a/functions_build.sh +++ b/functions_build.sh @@ -102,7 +102,7 @@ chevahbs_build() { build() { # This has the form: "libffi", "zlib", "bzip", "libedit", etc. # It's present in 'src/` and contains `chevahbs`, checksums, patches. - # Also used when downloading the gzip'ed tarball and unpacking it. + # Also used when downloading the gzipped tarball and unpacking it. project_name="$1" # This has the form: "3.2.1", "1.2.11". etc. project_ver="$2" diff --git a/pkg_checks.sh b/pkg_checks.sh index 9c8c5844..99922fe3 100644 --- a/pkg_checks.sh +++ b/pkg_checks.sh @@ -13,9 +13,6 @@ # # On platforms with multiple C compilers, choose by setting CC in os_quirks.sh. -# This script has external checks with various exit codes that are checked here. -set +o errexit - # List of OS packages required for building Python/pyOpenSSL/cryptography etc. BASE_PKGS="gcc make m4 automake libtool patch unzip" DEB_PKGS="$BASE_PKGS tar diffutils \ @@ -84,6 +81,9 @@ case "$OS" in ;; esac +# External checks with various exit codes are checked below. +set +o errexit + # If $CHECK_CMD is still "command -v", it's only a check for needed commands. if [ -n "$PACKAGES" ]; then for package in $PACKAGES ; do @@ -107,6 +107,7 @@ fi # Windows "build" is special, following checks are for other platforms. if [ "$OS" = "win" ]; then + set -o errexit return fi @@ -140,3 +141,6 @@ case "$OS" in (>&2 echo "Not guarding against linking to uuid libs on this system!") ;; esac + +# This script is sourced, execution does not end here. +set -o errexit From 125aa2c4409ca054ac560fa16dea753095b02aab Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 3 May 2022 11:08:07 +0300 Subject: [PATCH 31/38] Updated sqlite. --- build.conf | 2 +- src/sqlite-autoconf/chevahbs | 2 +- src/sqlite-autoconf/sha512.sum | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.conf b/build.conf index 3f7504e6..a142a8d0 100644 --- a/build.conf +++ b/build.conf @@ -17,7 +17,7 @@ OPENSSL_VERSION="1.1.1n" # To match the unusual naming scheme for SQLite downloads, eliminate dots from # the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000". # When updating this, also update the year in src/sqlite/chevahbs, if needed. -SQLITE_VERSION="3330000" +SQLITE_VERSION="3380300" # These are used by get-pip.py. PIP_VERSION="21.1.1" SETUPTOOLS_VERSION="53.0.0" diff --git a/src/sqlite-autoconf/chevahbs b/src/sqlite-autoconf/chevahbs index 0e1dc8b6..5412fdbe 100755 --- a/src/sqlite-autoconf/chevahbs +++ b/src/sqlite-autoconf/chevahbs @@ -14,7 +14,7 @@ chevahbs_getsources() { local name=$1 local ver=$2 local ext="tar.gz" - local link=https://sqlite.org/2020/"$name"-"$ver"."$ext" + local link=https://sqlite.org/2022/"$name"-"$ver"."$ext" download_sources $name $ver $link $ext } diff --git a/src/sqlite-autoconf/sha512.sum b/src/sqlite-autoconf/sha512.sum index ac50db10..34e9f13e 100644 --- a/src/sqlite-autoconf/sha512.sum +++ b/src/sqlite-autoconf/sha512.sum @@ -1 +1 @@ -c0d79d4012a01f12128ab5044b887576a130663245b85befcc0ab82ad3a315dd1e7f54b6301f842410c9c21b73237432c44a1d7c2fe0e0709435fec1f1a20a11 sqlite-autoconf-3330000.tar.gz +02bec3bd0d89f53c81c34a16d9105e157e3cfc9bb9c62900f65fde019a211b4d1b28017affd1785b344d85f742a0c6893b9c5b2702312c3e7ddb78ae8a4c152f sqlite-autoconf-3380300.tar.gz From b761bff70b24dd1a9c808961739a58fc1a6d3a8c Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 3 May 2022 11:26:03 +0300 Subject: [PATCH 32/38] Updated pip and most Python modules. --- build.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build.conf b/build.conf index a142a8d0..c2ec5f7b 100644 --- a/build.conf +++ b/build.conf @@ -19,19 +19,19 @@ OPENSSL_VERSION="1.1.1n" # When updating this, also update the year in src/sqlite/chevahbs, if needed. SQLITE_VERSION="3380300" # These are used by get-pip.py. -PIP_VERSION="21.1.1" -SETUPTOOLS_VERSION="53.0.0" +PIP_VERSION="22.0.4" +SETUPTOOLS_VERSION="62.1.0" # Python modules versions to be used everywhere possible. -CFFI_VERSION="1.14.3" +CFFI_VERSION="1.15.0" CRYPTOGRAPHY_VERSION="3.4.7" SUBPROCESS32_VERSION="3.5.4" -BCRYPT_VERSION="3.1.7" -PSUTIL_VERSION="5.7.3" -SETPROCTITLE_VERSION="1.1.10" +BCRYPT_VERSION="3.2.2" +PSUTIL_VERSION="5.9.0" +SETPROCTITLE_VERSION="1.2.3" # pycparser is explicitly installed to work around setuptools auto dependencies. -PYCPARSER_VERSION="2.20" +PYCPARSER_VERSION="2.21" # Global flags for building required libs. BUILD_LIBFFI="no" From 70c3139a46fca39d9c52ed1f506989a97dfa814f Mon Sep 17 00:00:00 2001 From: dumol Date: Tue, 3 May 2022 12:34:01 +0300 Subject: [PATCH 33/38] Try latest cryptography not requiring Rust. --- build.conf | 2 +- src/chevah-python-test/test_python_binary_dist.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.conf b/build.conf index c2ec5f7b..d177b6d6 100644 --- a/build.conf +++ b/build.conf @@ -24,7 +24,7 @@ SETUPTOOLS_VERSION="62.1.0" # Python modules versions to be used everywhere possible. CFFI_VERSION="1.15.0" -CRYPTOGRAPHY_VERSION="3.4.7" +CRYPTOGRAPHY_VERSION="3.4.8" SUBPROCESS32_VERSION="3.5.4" BCRYPT_VERSION="3.2.2" PSUTIL_VERSION="5.9.0" diff --git a/src/chevah-python-test/test_python_binary_dist.py b/src/chevah-python-test/test_python_binary_dist.py index 6a3d11c4..60daf7e1 100644 --- a/src/chevah-python-test/test_python_binary_dist.py +++ b/src/chevah-python-test/test_python_binary_dist.py @@ -358,8 +358,8 @@ def main(): # compatibility, link to version 1.1.1c from CentOS 8.2.2004. expecting = u'OpenSSL 1.1.1c FIPS 28 May 2019' elif CHEVAH_OS == "win": - # Latest Python 3.8 Win packages are left behind. - expecting = u'OpenSSL 1.1.1k 25 Mar 2021' + # Latest cryptography not requiring Rust has older wheels. + expecting = u'OpenSSL 1.1.1l 24 Aug 2021' else: # Use latest OpenSSL version when building it from source. expecting = u'OpenSSL 1.1.1n 15 Mar 2022' From 1ccb8022610fc70190cc20d7c1bcf74359eb9d80 Mon Sep 17 00:00:00 2001 From: dumol Date: Wed, 4 May 2022 10:51:50 +0300 Subject: [PATCH 34/38] Change after review. Co-authored-by: Adi Roiban --- pythia.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythia.conf b/pythia.conf index abff3784..f963d524 100644 --- a/pythia.conf +++ b/pythia.conf @@ -5,7 +5,7 @@ BINARY_DIST_URI='https://github.com/chevah/pythia/releases/download' #BINARY_DIST_URI='https://bin.chevah.com:20443/production' # For testing packages, make sure this one is the last uncommented instance: #BINARY_DIST_URI='https://bin.chevah.com:20443/testing' -PIP_INDEX_URL='https://pypi.chevah.com/simple' +PIP_INDEX_URL='https://bin.chevah.com:20443/pypi/simple' # There are 2 build directories used in this repo: # * $BUILD_DIR is used for building libffi / OpenSSL / Python / etc. From d3c1b6f30b231d1a026287ac0570348cf14be8e3 Mon Sep 17 00:00:00 2001 From: dumol Date: Wed, 4 May 2022 11:26:05 +0300 Subject: [PATCH 35/38] More changes related to PyPI's index_url. --- build.conf | 6 ------ build.sh | 6 ++++++ pavement.py | 5 +---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/build.conf b/build.conf index d177b6d6..a174a40c 100644 --- a/build.conf +++ b/build.conf @@ -51,9 +51,3 @@ PIP_LIBRARIES="\ psutil==${PSUTIL_VERSION} \ setproctitle==${SETPROCTITLE_VERSION} " -# Arguments that are sent when using pip. -PIP_ARGS="\ - --index-url=https://bin.chevah.com:20443/pypi/simple \ - --trusted-host=pypi.chevah.com \ - --no-warn-script-location \ - " diff --git a/build.sh b/build.sh index 3661e44d..216c0673 100755 --- a/build.sh +++ b/build.sh @@ -177,6 +177,12 @@ command_install_python_modules() { # Install latest PIP, then instruct it to get exact versions of setuptools. # Otherwise, get-pip.py will always try to get latest versions. download_get_pip + # Get $PIP_INDEX_URL. + source pythia.conf + PIP_ARGS="\ + --index-url=${PIP_INDEX_URL} \ + --no-warn-script-location \ + " echo "# Installing latest pip with preferred setuptools version... #" execute $PYTHON_BIN "$BUILD_DIR"/get-pip.py $PIP_ARGS \ pip==$PIP_VERSION --no-setuptools setuptools==$SETUPTOOLS_VERSION diff --git a/pavement.py b/pavement.py index 57d1d5d4..ce476158 100644 --- a/pavement.py +++ b/pavement.py @@ -26,13 +26,10 @@ SETUP['repository']['name'] = u'pythia' SETUP['test']['package'] = None -SETUP['pypi']['index_url'] = 'http://pypi.chevah.com/simple' +SETUP['pypi']['index_url'] = 'https://bin.chevah.com:20443/pypi/simple' SETUP['repository']['name'] = u'pythia' SETUP['repository']['github'] = 'https://github.com/chevah/pythia' -SETUP['buildbot']['builders_filter'] = u'pythia' -SETUP['buildbot']['server'] = 'buildbot.chevah.com' -SETUP['buildbot']['web_url'] = 'https://buildbot.chevah.com:10433' RUN_PACKAGES = [ 'requests~=2.24', From f16f48291717743ace93e9ad9276d3605025c321 Mon Sep 17 00:00:00 2001 From: dumol Date: Wed, 4 May 2022 11:40:31 +0300 Subject: [PATCH 36/38] Even more changes related to PyPI's index_url. --- build.conf | 4 ++++ build.sh | 9 +++------ pythia.sh | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/build.conf b/build.conf index a174a40c..fb16f546 100644 --- a/build.conf +++ b/build.conf @@ -51,3 +51,7 @@ PIP_LIBRARIES="\ psutil==${PSUTIL_VERSION} \ setproctitle==${SETPROCTITLE_VERSION} " +PIP_ARGS="\ + --index-url=${PIP_INDEX_URL} \ + --no-warn-script-location \ + " diff --git a/build.sh b/build.sh index 216c0673..38e97502 100755 --- a/build.sh +++ b/build.sh @@ -7,6 +7,9 @@ set -o nounset set -o errexit set -o pipefail +# Get PIP_INDEX_URL for PIP_ARGS in build.conf. +source pythia.conf + # Set versions for the software to be built and other defaults. source build.conf @@ -177,12 +180,6 @@ command_install_python_modules() { # Install latest PIP, then instruct it to get exact versions of setuptools. # Otherwise, get-pip.py will always try to get latest versions. download_get_pip - # Get $PIP_INDEX_URL. - source pythia.conf - PIP_ARGS="\ - --index-url=${PIP_INDEX_URL} \ - --no-warn-script-location \ - " echo "# Installing latest pip with preferred setuptools version... #" execute $PYTHON_BIN "$BUILD_DIR"/get-pip.py $PIP_ARGS \ pip==$PIP_VERSION --no-setuptools setuptools==$SETUPTOOLS_VERSION diff --git a/pythia.sh b/pythia.sh index 06ff679b..6e1ac6f4 100755 --- a/pythia.sh +++ b/pythia.sh @@ -313,8 +313,6 @@ pip_install() { rm -rf ${BUILD_FOLDER}/pip-build ${PYTHON_BIN} -m \ pip install \ - --trusted-host bin.chevah.com \ - --trusted-host pypi-internal.chevah.com \ --index-url=$PIP_INDEX_URL \ --build=${BUILD_FOLDER}/pip-build \ $1 From fd6ea726989f289cfef49a6a5897d554d8d0041b Mon Sep 17 00:00:00 2001 From: dumol Date: Wed, 4 May 2022 11:41:06 +0300 Subject: [PATCH 37/38] Build xz probably needs to be built on all BSD-derived OS'es. --- os_quirks.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/os_quirks.sh b/os_quirks.sh index 1b81bc63..f2f455b9 100644 --- a/os_quirks.sh +++ b/os_quirks.sh @@ -62,6 +62,7 @@ case $OS in export BUILD_LIBFFI="yes" # System includes bzip2 libs by default. export BUILD_BZIP2="no" + export BUILD_XZ="yes" # Install package "p5-Digest-SHA" to get shasum binary. export SHA_CMD="shasum --algorithm 512 --check --status --warn" ;; @@ -70,6 +71,7 @@ case $OS in export CXX="clang++" # libffi not available in the base system, only as port/package. export BUILD_LIBFFI="yes" + export BUILD_XZ="yes" export SHA_CMD="sha512 -q -c" ;; sol*) @@ -93,6 +95,7 @@ case $OS in # Solaris 11 is much more modern, but still has some quirks. # Multiple system libffi libs present, this is a problem in 11.4. export BUILD_LIBFFI="yes" + export BUILD_XZ="yes" # Native tar is not that compatible, but the GNU tar should be present. export TAR_CMD="gtar xfz" ;; From 533115f86d362b9fa7f4d38603104869fb7fa85c Mon Sep 17 00:00:00 2001 From: dumol Date: Wed, 4 May 2022 11:53:00 +0300 Subject: [PATCH 38/38] One last change related to PyPI's index_url. --- publish_dist.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/publish_dist.sh b/publish_dist.sh index 56340514..31501633 100755 --- a/publish_dist.sh +++ b/publish_dist.sh @@ -31,6 +31,7 @@ esac $sftp_cmd $sftp_opts ${dest_user}@${dest_server} # Get $DIST_DIR. +source pythia.conf source build.conf # As dist/ is rebuilt on every build, it should only have 1 sub-dir with 1 pkg.