Set up the 2.4 branch of the yugabyte-db-thirdparty repo (#64)

- Branch off of the commit 45c97f4, which is the thirdparty commit used in the 2.4 branch of yugabyte/yugabyte-db ( https://github.com/yugabyte/yugabyte-db/tree/2.4 ), which is at the commit yugabyte/yugabyte-db@bcf406c as of this writing.
- GitHub Actions setup similar to the master branch of the yugabyte-db-thirdparty repo (as of the 6a4d8d4 commit).
- Downgrade gperftools to 2.7.
- Fix the Boost source tarball download URL.
- Remove the unused multi_build.py / MultiBuilder functionality.
- Add various possible checksums that we might later need to thirdparty_src_checksums.txt.
- Use the codecheck tool for running mypy on Python code and shellcheck on Bash code.

.github/workflows/linux_build.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+df -H .
+checkout_dir=$PWD
+echo "Building in directory: $checkout_dir"
+docker run -t \
+  --cap-add=SYS_PTRACE \
+  -e GITHUB_TOKEN \
+  -e YB_BUILD_THIRDPARTY_ARGS \
+  -e YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX \
+  "--mount=type=bind,src=$checkout_dir,dst=/opt/yb-build/thirdparty/checkout" \
+  "$YB_DOCKER_IMAGE" \
+  bash -c "
+    set -euo pipefail
+    cd /opt/yb-build/thirdparty/checkout
+    ./build_and_release.sh
+  "

.github/workflows/linux_build.yml

@@ -0,0 +1,97 @@
+name: Linux
+
+# TODO: deduplicate this across multiple workflow files.
+on:
+  push:
+    branches:
+      - master
+      - 2.4
+
+    paths-ignore:
+      - README.md
+
+  # Run tests for any PRs.
+  pull_request:
+    branches:
+      - master
+      - 2.4
+
+    paths-ignore:
+      - README.md
+
+jobs:
+  build:
+    name: ${{ matrix.archive_name_suffix }}
+    runs-on: ubuntu-20.04
+    if: >
+      (github.event_name == 'push' &&
+       !contains(github.event.head_commit.message, 'skip ci') &&
+       !contains(github.event.head_commit.message, 'ci skip')) ||
+      github.event_name == 'pull_request'
+    strategy:
+      # To avoid terminating all jobs when one job fails.
+      fail-fast: false
+      matrix:
+        include:
+          - name: centos7-linuxbrew-gcc5
+            archive_name_suffix: centos7-linuxbrew
+            docker_image: yugabyteci/yb_build_infra_centos7:v2021-03-26T05_02_29
+            build_thirdparty_args: "--toolchain=linuxbrew"
+
+          - name: centos7-gcc-devtoolset-8
+            archive_name_suffix: centos7-devtoolset-8
+            docker_image: yugabyteci/yb_build_infra_centos7:v2021-03-26T05_02_29
+            build_thirdparty_args: "--devtoolset=8"
+
+          - name: centos7-gcc-devtoolset-9
+            archive_name_suffix: centos7-devtoolset-9
+            docker_image: yugabyteci/yb_build_infra_centos7:v2021-03-26T05_02_29
+            build_thirdparty_args: "--devtoolset=9"
+
+          - name: centos7-clang11
+            archive_name_suffix: centos7-clang11
+            docker_image: yugabyteci/yb_build_infra_centos7:v2021-03-26T05_02_29
+            build_thirdparty_args: --toolchain=llvm11
+
+          - name: ubuntu1804-gcc7
+            archive_name_suffix: ubuntu1804-gcc7
+            docker_image: yugabyteci/yb_build_infra_ubuntu1804:v2021-03-26T05_02_28
+            build_thirdparty_args: >-
+              --compiler-prefix=/usr
+              --single-compiler-type=gcc
+              --compiler-suffix=-7
+
+          - name: ubuntu1804-gcc8
+            archive_name_suffix: ubuntu1804-gcc8
+            docker_image: yugabyteci/yb_build_infra_ubuntu1804:v2021-03-26T05_02_28
+            build_thirdparty_args: >-
+              --compiler-prefix=/usr
+              --single-compiler-type=gcc
+              --compiler-suffix=-8
+
+          - name: ubuntu1804-clang10
+            archive_name_suffix: ubuntu1804-clang10
+            docker_image: yugabyteci/yb_build_infra_ubuntu1804:v2021-03-26T05_02_28
+            build_thirdparty_args: >-
+              --single-compiler-type=clang
+              --compiler-prefix=/usr
+              --compiler-suffix=-10
+
+          - name: ubuntu1804-clang11
+            archive_name_suffix: ubuntu1804-clang11
+            docker_image: yugabyteci/yb_build_infra_ubuntu1804:v2021-03-26T05_02_28
+            build_thirdparty_args: >-
+              --single-compiler-type=clang
+              --compiler-prefix=/usr
+              --compiler-suffix=-11
+
+    steps:
+      - name: Clone repository
+        uses: actions/checkout@v1
+
+      - name: Build
+        run: ./.github/workflows/linux_build.sh
+        env:
+          YB_DOCKER_IMAGE: ${{ matrix.docker_image }}
+          YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX: ${{ matrix.archive_name_suffix }}
+          YB_BUILD_THIRDPARTY_ARGS: ${{ matrix.build_thirdparty_args }}

.github/workflows/macos_build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+brew install autoconf automake pkg-config shellcheck
+dirs=( /opt/yb-build/{thirdparty,brew,tmp} )
+sudo mkdir -p "${dirs[@]}"
+sudo chmod 777 "${dirs[@]}"
+./build_and_release.sh

.github/workflows/macos_build.yml

@@ -1,9 +1,11 @@
 name: macOS
 
+# TODO: deduplicate this across multiple workflow files.
 on:
   push:
     branches:
-      - '*'
+      - master
+      - 2.4
 
     paths-ignore:
       - README.md
@@ -12,10 +14,12 @@ on:
   pull_request:
     branches:
       - master
+      - 2.4
 
     paths-ignore:
       - README.md
 
+
 env:
   GITHUB_ORG_AND_REPO: "${{ github.repository }}"
 
@@ -33,8 +37,6 @@ jobs:
       - name: Build YugabyteDB third-party dependencies
         env:
           GITHUB_TOKEN: ${{ secrets.CUSTOM_GITHUB_TOKEN }}
+          YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX: x86_64-macos
         run: |
-          brew install autoconf automake pkg-config
-          sudo mkdir -p /opt/yb-build/{thirdparty,brew,tmp}
-          sudo chmod 777 /opt/yb-build/{thirdparty,brew,tmp}
-          ./build_and_release.sh
+          ./.github/workflows/macos_build.sh

.gitignore

@@ -28,3 +28,6 @@ prebuilt_downloads/
 src/
 venv/
 .pyre
+archive.tar.gz
+archive.tar.gz.sha256
+fossa_modules.yml

branch.txt

@@ -0,0 +1 @@
+2.4

build_and_release.sh

@@ -80,8 +80,6 @@ log "YB_BUILD_THIRDPARTY_ARGS: ${YB_BUILD_THIRDPARTY_ARGS:-undefined}"
 YB_BUILD_THIRDPARTY_EXTRA_ARGS=${YB_BUILD_THIRDPARTY_EXTRA_ARGS:-}
 log "YB_BUILD_THIRDPARTY_EXTRA_ARGS: ${YB_BUILD_THIRDPARTY_EXTRA_ARGS:-undefined}"
 
-log "CIRCLE_PULL_REQUEST: ${CIRCLE_PULL_REQUEST:-undefined}"
-
 # -------------------------------------------------------------------------------------------------
 # Installed tools
 # -------------------------------------------------------------------------------------------------
@@ -128,14 +126,11 @@ fi
 # Check for errors in Python code of this repository
 # -------------------------------------------------------------------------------------------------
 
-( set -x; "$YB_THIRDPARTY_DIR/check_python_code.sh" )
+( set -x; "$YB_THIRDPARTY_DIR/check_code.sh" )
 
 # -------------------------------------------------------------------------------------------------
 
-if [[ -n ${CIRCLE_PULL_REQUEST:-} ]]; then
-  echo "CIRCLE_PULL_REQUEST is set: $CIRCLE_PULL_REQUEST. Will not upload artifacts."
-  unset GITHUB_TOKEN
-elif [[ -z ${GITHUB_TOKEN:-} || $GITHUB_TOKEN == *githubToken* ]]; then
+if [[ -z ${GITHUB_TOKEN:-} || $GITHUB_TOKEN == *githubToken* ]]; then
   echo "This must be a pull request build. Will not upload artifacts."
   GITHUB_TOKEN=""
 else
@@ -146,16 +141,16 @@ fi
 
 original_repo_dir=$PWD
 git_sha1=$( git rev-parse HEAD )
-tag=v$( date +%Y%m%d%H%M%S )-${git_sha1:0:10}
+branch_name=$(<"$YB_THIRDPARTY_DIR/branch.txt")
+tag=v${branch_name}-$( date +%Y%m%d%H%M%S )-${git_sha1:0:10}
 
 archive_dir_name=yugabyte-db-thirdparty-$tag
-if [[ -n $YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX ]]; then
-  effective_suffix="-$YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX"
-else
-  effective_suffix="-$os_name"
+if [[ -z ${YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX:-} ]]; then
+  fatal "YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX is not specified."
 fi
-archive_dir_name+=$effective_suffix
-tag+=$effective_suffix
+to_append="-$YB_THIRDPARTY_ARCHIVE_NAME_SUFFIX"
+archive_dir_name+=$to_append
+tag+=$to_append
 
 build_dir_parent=/opt/yb-build/thirdparty
 repo_dir=$build_dir_parent/$archive_dir_name

build_thirdparty.sh

@@ -18,8 +18,6 @@ set -euo pipefail
 # shellcheck source=./yb-thirdparty-common.sh
 . "${BASH_SOURCE[0]%/*}/yb-thirdparty-common.sh"
 
-check_bash_scripts
-
 echo "YB_THIRDPARTY_DIR=$YB_THIRDPARTY_DIR"
 
 activate_virtualenv

check_bash_scripts.sh → check_code.sh

@@ -5,4 +5,7 @@ set -euo pipefail
 # shellcheck source=./yb-thirdparty-common.sh
 . "${BASH_SOURCE%/*}/yb-thirdparty-common.sh"
 
-check_bash_scripts
+activate_virtualenv
+
+set -x
+codecheck --verbose "$@"

python/build_definitions/boost.py

@@ -33,7 +33,9 @@ def __init__(self) -> None:
         super(BoostDependency, self).__init__(
             name='boost',
             version='1.69.0',
-            url_pattern='https://dl.bintray.com/boostorg/release/{0}/source/boost_{1}.tar.bz2',
+            # URL grabbed from https://www.boost.org/users/history/version_1_69_0.html
+            url_pattern='https://boostorg.jfrog.io/artifactory/main/release/1.69.0/source/'
+                        'boost_1_69_0.tar.bz2',
             build_group=BUILD_GROUP_INSTRUMENTED,
             license='Boost Software License 1.0')
         self.dir = '{}_{}'.format(self.name, self.underscored_version)

python/build_definitions/gperftools.py

@@ -21,7 +21,7 @@ class GPerfToolsDependency(Dependency):
     def __init__(self) -> None:
         super(GPerfToolsDependency, self).__init__(
             name='gperftools',
-            version='2.8',
+            version='2.7',
             url_pattern='https://github.com/gperftools/gperftools/releases/download/gperftools-{0}/'
                         'gperftools-{0}.tar.gz',
             build_group=BUILD_GROUP_INSTRUMENTED)