Forward-ported fixes from python-package + other updates. (#32)

* Use latest pythia version.

* Forward-port python-package brink.{conf,sh} updates.

* Use latest zlib.

* Use latest libffi.

* Try using latest available Python 3.8.x packages.

* Fixed downloading latest libffi.

* Use latest OpenSSL when building.

* Back to tried and true Python 3.8.6.

This reverts commit ceb021b.

* Use our own CentOS 5.11 Docker image.

* Also install unzip on CentOS 5.

* Updated condition to match CentOS 5 container name.

* Grand rename of the brink and build scripts.

* Updated GitHub workflows for the grand rename.

* Move stray pkgconfig files to their place.

* Back to manually installing libffi for CentOS 5.

* Added strong Bash checks for building.

* Removed scripted OS package management.

* Try to fix fallout from removing scripted OS pkg mgmt.

* Build xz for Python's lzma module.

* Download xz from SourceForge to have it work on CentOS 5.

* Try building xz only on generic Linux.

* Updated deps checks.

* Try building xz on Alpine.

* Package management fixes.

* More xz build tweaks.

* Install required headers on Alpine.

* Try building xz on macOS differently.

* Accomodate stricter bash checks in functions.sh.

* Disabled building shared xz libs.

* Changes after own review.

* Updated sqlite.

* Updated pip and most Python modules.

* Try latest cryptography not requiring Rust.

* Change after review.

Co-authored-by: Adi Roiban <adiroiban@gmail.com>

* More changes related to  PyPI's index_url.

* Even more changes related to PyPI's index_url.

* Build xz probably needs to be built on all BSD-derived OS'es.

* One last change related to PyPI's index_url.

Co-authored-by: Adi Roiban <adiroiban@gmail.com>

Author: dumol

.github/workflows/bare.yaml

@@ -54,7 +54,7 @@ jobs:
       # Workflow won't be cancelled at the first failed job.
       fail-fast: false
       matrix:
-        runs-on: [ windows-2019, windows-2016 ]
+        runs-on: [ windows-2022, windows-2019 ]
     timeout-minutes: 20
 
     # Steps represent a sequence of tasks that will be executed as part of the job
@@ -66,10 +66,10 @@ jobs:
 
     # Explicitly run our scripts with Bash, not PowerShell (GitHub's default).
     - name: Build Pythia
-      run: bash ./pythia build
+      run: bash ./build.sh build
 
     - name: Test Pythia
-      run: bash ./pythia test
+      run: bash ./build.sh test
 
     # To use an RSA key with SFTPPlus, install upstream OpenSSH package,
     # which is more finicky in regards to file permissions.
@@ -127,14 +127,16 @@ jobs:
       with:
         fetch-depth: 2
 
-    - name: Install required packages
-      run: sudo apt-get install libncurses5-dev
+    - name: Prepare OS
+      run: |
+        sudo apt install libncurses5-dev
+        sudo apt purge uuid-dev
 
     - name: Build Pythia
-      run: ./pythia build
+      run: ./build.sh build
 
     - name: Test Pythia
-      run: ./pythia test
+      run: ./build.sh test
 
     - name: Upload testing package
       run: |
@@ -173,10 +175,10 @@ jobs:
         sudo chmod -v a-r /usr/local/opt/gettext/lib/libintl.*
 
     - name: Build Pythia
-      run: ./pythia build
+      run: ./build.sh build
 
     - name: Test Pythia
-      run: ./pythia test
+      run: ./build.sh test
 
     - name: Upload testing package
       run: |

.github/workflows/docker.yaml

@@ -30,7 +30,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container: [ 'alpine:3.14', 'centos:8.2.2004', 'centos:5.11' ]
+        # CentOS 5.11 setup was saved as an image pushed to Docker Hub. See the
+        # Overview section at https://hub.docker.com/r/proatria/centos for details.
+        container: [ 'alpine:3.14', 'centos:8.2.2004', 'proatria/centos:5.11-chevah1' ]
     timeout-minutes: 30
     steps:
 
@@ -39,7 +41,8 @@ jobs:
       if: startsWith(matrix.container, 'alpine')
       run: |
         apk upgrade -U
-        apk add git curl bash openssh-client paxctl
+        apk add git curl bash gcc make m4 automake libtool patch zlib-dev openssl-dev musl-dev linux-headers lddtree shadow sudo openssh-client paxctl file unzip
+        apk del util-linux-dev
 
     # Stick to CentOS 8.2 as OpenSSL got updated in 8.3 from 1.1.1c to 1.1.1g.
     - name: CentOS 8.2 setup
@@ -48,24 +51,16 @@ jobs:
         sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo
         sed -i s@^#baseurl=http://mirror.centos.org/\$contentdir/\$releasever/@baseurl=https://vault.centos.org/8.2.2004/@ /etc/yum.repos.d/*.repo
         yum -y upgrade
-        yum -y install git curl openssh-clients
+        yum -y install git curl gcc make m4 automake libtool patch libffi-devel zlib-devel xz-devel ncurses-devel openssl-devel openssh-clients unzip
+        yum remove -y e2fsprogs-devel libuuid-devel
 
-    # Final CentOS 5 version is used to build the generic Linux package.
+    # Our Chevah Centos 5 image, built for python-package.
+    # FIXME: Update it to include unzip.
     - name: CentOS 5.11 setup
-      if: matrix.container == 'centos:5.11'
+      if: matrix.container == 'proatria/centos:5.11-chevah1'
       run: |
-        sed -i s/^mirrorlist=/#mirrorlist=/ /etc/yum.repos.d/*.repo
-        sed -i s@^#baseurl=http://mirror.centos.org/centos/\$releasever/@baseurl=http://vault.centos.org/5.11/@ /etc/yum.repos.d/*.repo
-        yum -y upgrade
-        # Use http://bin.chevah.com:20443/third-party-stuff/centos5/tuxad/
-        # when tuxad.de dissapears, it has the minimum required stuff.
-        rpm -i http://www.tuxad.de/rpms/tuxad-release-5-1.noarch.rpm
-        yum -y install curl openssh-clients gcc44 make m4 patch unzip wget
-        ln -s /usr/bin/gcc44 /usr/local/bin/gcc
-        wget --mirror --no-parent https://bin.chevah.com:20443/third-party-stuff/centos5/endpoint/
-        cd bin.chevah.com:20443/third-party-stuff/centos5/endpoint/
-        rpm -i local-perl-*.rpm
-        rpm -i --nodeps git{-core,}-2.5.0-1.ep.x86_64.rpm
+        wget https://vault.centos.org/5.11/os/x86_64/CentOS/unzip-5.52-3.el5.x86_64.rpm
+        rpm -i unzip-5.52-3.el5.x86_64.rpm
 
     - name: Clone repo independently
       run: |
@@ -76,12 +71,12 @@ jobs:
     - name: Build Pythia
       run: |
         cd pythia
-        ./pythia build
+        ./build.sh build
 
     - name: Test Pythia
       run: |
         cd pythia
-        ./pythia test
+        ./build.sh test
 
     # Using `~/` is problematic under Docker, use `/root/`.
     # Remove key in same step to avoid leaving it on disk if publishing fails.

README.rst

@@ -6,16 +6,16 @@ A derivative of https://github.com/chevah/python-package/.
 
 Building:
 
-* ``./pythia build``
+* ``./build.sh build``
 
 Testing:
 
-* ``./pythia test``
-* ``./pythia compat``
+* ``./build.sh test``
+* ``./build.sh compat``
 
-Use ``./pythia help`` to discover all available commands.
+Use ``./build.sh help`` to discover all available commands.
 
-Note that compat tests are currently only working with the ``python2.7`` branch.
+Note that compat tests are currently only working on the ``python2.7`` branch.
 
 
 Supported platforms

brink.conf

@@ -0,0 +1,57 @@
+# Configuration file for building Pythia.
+
+# Sub-directories for building from sources and for the final distributable.
+BUILD_DIR="build"
+DIST_DIR="dist"
+
+# Python and lib versions.
+PYTHON_BUILD_VERSION="3.8.6"
+LIBFFI_VERSION="3.4.2"
+ZLIB_VERSION="1.2.12"
+BZIP2_VERSION="1.0.8"
+XZ_VERSION="5.2.5"
+# Statically build the BSD libedit on selected platforms to get the
+# readline module available without linking to the GPL-only readline libs.
+LIBEDIT_VERSION="20170329-3.1"
+OPENSSL_VERSION="1.1.1n"
+# To match the unusual naming scheme for SQLite downloads, eliminate dots from
+# the regular SQLite version, then add 3 zeros. E.g. "3.33.0" -> "3330000".
+# When updating this, also update the year in src/sqlite/chevahbs, if needed.
+SQLITE_VERSION="3380300"
+# These are used by get-pip.py.
+PIP_VERSION="22.0.4"
+SETUPTOOLS_VERSION="62.1.0"
+
+# Python modules versions to be used everywhere possible.
+CFFI_VERSION="1.15.0"
+CRYPTOGRAPHY_VERSION="3.4.8"
+SUBPROCESS32_VERSION="3.5.4"
+BCRYPT_VERSION="3.2.2"
+PSUTIL_VERSION="5.9.0"
+SETPROCTITLE_VERSION="1.2.3"
+
+# pycparser is explicitly installed to work around setuptools auto dependencies.
+PYCPARSER_VERSION="2.21"
+
+# Global flags for building required libs.
+BUILD_LIBFFI="no"
+BUILD_ZLIB="no"
+BUILD_BZIP2="yes"
+BUILD_XZ="no"
+BUILD_LIBEDIT="yes"
+BUILD_OPENSSL="no"
+BUILD_SQLITE="yes"
+
+# List of default Python modules installed using pip.
+PIP_LIBRARIES="\
+    cffi==${CFFI_VERSION} \
+    cryptography==${CRYPTOGRAPHY_VERSION} \
+    subprocess32==${SUBPROCESS32_VERSION} \
+    bcrypt==${BCRYPT_VERSION} \
+    psutil==${PSUTIL_VERSION} \
+    setproctitle==${SETPROCTITLE_VERSION}
+    "
+PIP_ARGS="\
+    --index-url=${PIP_INDEX_URL} \
+    --no-warn-script-location \
+    "

build.conf

@@ -2,12 +2,20 @@
 #
 # Pythia's script for building Python.
 
-# Set versions for the software to be built and other defaults.
+# Script initialization.
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# Get PIP_INDEX_URL for PIP_ARGS in build.conf.
 source pythia.conf
 
+# Set versions for the software to be built and other defaults.
+source build.conf
+
 # Import shared and specific code.
 source ./functions.sh
-source ./functions_pythia.sh
+source ./functions_build.sh
 
 # Git revision to inject into Python's sys.version string through chevahbs
 # on non-Windows platforms. Also used for compat tests and archived in the dist.
@@ -16,18 +24,18 @@ exit_on_error $? 250
 
 # Export the variables needed by the chevahbs scripts and the test phase.
 export PYTHON_BUILD_VERSION PYTHIA_VERSION
-export BUILD_ZLIB BUILD_BZIP2 BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
+export BUILD_ZLIB BUILD_BZIP2 BUILD_XZ BUILD_LIBEDIT BUILD_LIBFFI BUILD_OPENSSL
 
 
 # OS detection is slow on Windows, only execute it when the file is missing.
 if [ ! -r ./BUILD_ENV_VARS ]; then
-    execute ./brink.sh detect_os
+    execute ./pythia.sh detect_os
 fi
-# Import build env vars as set by brink.sh.
+# Import build env vars as set by pythia.sh.
 source ./BUILD_ENV_VARS
 
 # On Unix, use $ARCH to choose between 32bit or 64bit packages. It's possible
-# to force a 32bit build on a 64bit machine, e.g. by setting ARCH in brink.sh
+# to force a 32bit build on a 64bit machine, e.g. by setting ARCH in pythia.sh
 # as "x86" instead of "x64" for a certain platform.
 # $ARCH is also used when "building" Python on Windows and for testing.
 # $OS is used when patching/configuring/building/testing.
@@ -45,7 +53,7 @@ export CC="gcc"
 export MAKE="make"
 # $GET_CMD must save to custom filename, which must be appended before the link.
 # E.g., to use wget, GET_CMD should be "wget --quiet -O".
-export GET_CMD="curl --silent --output"
+export GET_CMD="curl --silent --location --output"
 export SHA_CMD="sha512sum --check --status --warn"
 export TAR_CMD="tar xfz"
 export ZIP_CMD="unzip -q"
@@ -87,12 +95,13 @@ command_build() {
     # following locations, making sure they are picked up when building Python.
     # $CFLAGS/$CPPFLAGS is another way to ensure this, but it's not as portable.
     execute mkdir -p "$INSTALL_DIR"/{include,lib}
-    export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS}"
-    export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig/:${PKG_CONFIG_PATH}"
+    export LDFLAGS="-L${INSTALL_DIR}/lib/ ${LDFLAGS:-}"
+    export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig/:${PKG_CONFIG_PATH:-}"
 
     build_dep $BUILD_LIBFFI   libffi           $LIBFFI_VERSION
     build_dep $BUILD_ZLIB     zlib             $ZLIB_VERSION
     build_dep $BUILD_BZIP2    bzip2            $BZIP2_VERSION
+    build_dep $BUILD_XZ       xz               $XZ_VERSION
     build_dep $BUILD_LIBEDIT  libedit          $LIBEDIT_VERSION
     build_dep $BUILD_SQLITE   sqlite-autoconf  $SQLITE_VERSION
     build_dep $BUILD_OPENSSL  openssl          $OPENSSL_VERSION
@@ -130,7 +139,7 @@ build_dep() {
                 export LDFLAGS="-Wl,-rpath,${INSTALL_DIR}/lib/ ${LDFLAGS}"
             fi
             # Still needed for building cryptography.
-            export CPPFLAGS="$CPPFLAGS -I${INSTALL_DIR}/include"
+            export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_DIR}/include"
         fi
     elif [ $dep_boolean = "no" ]; then
         (>&2 echo "    Skip building $dep_name")
@@ -243,17 +252,17 @@ command_compat() {
     execute rm -rf compat/
     execute git clone https://github.com/chevah/compat.git --depth=1 -b master
     execute pushd compat
-    # Copy over current brink stuff, as some changes might require it.
-    execute cp ../../brink.{conf,sh} ./
+    # Copy over current pythia stuff, as some changes might require it.
+    execute cp ../../pythia.{conf,sh} ./
     # Patch compat to use the newly-built Python, then copy it to cache/.
-    echo -e "\nPYTHON_CONFIGURATION=default@${new_python_conf}" >> brink.conf
+    echo -e "\nPYTHON_CONFIGURATION=default@${new_python_conf}" >> pythia.conf
     execute mkdir cache
     execute cp -r ../"$PYTHON_BUILD_DIR" cache/
     # Make sure everything is done from scratch in the current dir.
     unset CHEVAH_CACHE CHEVAH_BUILD
     # Some tests might still fail due to causes not related to the new Python.
-    execute ./brink.sh deps
-    execute ./brink.sh test_ci
+    execute ./pythia.sh deps
+    execute ./pythia.sh test_ci
 
     execute popd
     echo "::endgroup::"

pythia build.sh

@@ -4,16 +4,15 @@
 #
 
 # Check if debugging environment variable is set and initialize with 0 if not.
-if [ -z "$DEBUG" ] ; then
-    DEBUG=0
-fi
+DEBUG=${DEBUG-0}
 
 help_text_help=\
 "Show help for a command."
 command_help() {
-    local command=$1
+    local command=${1:-}
     local help_command="help_$command"
     # Test for a valid help method, otherwise call general help.
+    set +o errexit
     type $help_command &> /dev/null
     if [ $? -eq 0 ]; then
         $help_command
@@ -25,6 +24,7 @@ command_help() {
             echo -e "    $command_name\t${!help_text}"
         done
     fi
+    set -o errexit
 }
 
 #
@@ -33,16 +33,17 @@ command_help() {
 # Select fuctions which are made public.
 #
 select_command() {
-    local command=$1
-    shift
+    local command=${1:-}
     case $command in
         "")
             command_help
             exit 99
             ;;
         *)
+            shift
             # Test for a valid command, otherwise call general help.
             call_command="command_$command"
+            set +o errexit
             type $call_command &> /dev/null
             if [ $? -eq 0 ]; then
                 $call_command $@
@@ -52,6 +53,7 @@ select_command() {
                 (>&2 echo "Unknown command: ${command}.")
                 exit 98
             fi
+            set -o errexit
         ;;
     esac
 }